From e07a865d9c737f8a5f510c0d65eec8ad50d601de Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Tue, 13 Jun 2017 14:47:17 +0200 Subject: [PATCH] Improved background jobs --- .../airing-anime/{airing-anime.go => main.go} | 0 jobs/avatars/Avatar.go | 7 +- jobs/avatars/AvatarOriginalFileOutput.go | 65 +++++++++++++++ jobs/avatars/AvatarWebPFileOutput.go | 27 +++++++ jobs/avatars/AvatarWriter.go | 4 +- jobs/avatars/FSWriter.go | 1 - jobs/avatars/Gravatar.go | 1 + jobs/avatars/download.go | 79 ------------------- jobs/avatars/{avatars.go => main.go} | 60 ++++++++++---- jobs/avatars/webp.go | 51 ------------ jobs/sync-anime/{sync-anime.go => main.go} | 0 .../{clear-sessions.go => main.go} | 0 .../{user-references.go => main.go} | 0 13 files changed, 144 insertions(+), 151 deletions(-) rename jobs/airing-anime/{airing-anime.go => main.go} (100%) create mode 100644 jobs/avatars/AvatarOriginalFileOutput.go create mode 100644 jobs/avatars/AvatarWebPFileOutput.go delete mode 100644 jobs/avatars/FSWriter.go delete mode 100644 jobs/avatars/download.go rename jobs/avatars/{avatars.go => main.go} (53%) delete mode 100644 jobs/avatars/webp.go rename jobs/sync-anime/{sync-anime.go => main.go} (100%) rename patches/clear-sessions/{clear-sessions.go => main.go} (100%) rename patches/user-references/{user-references.go => main.go} (100%) diff --git a/jobs/airing-anime/airing-anime.go b/jobs/airing-anime/main.go similarity index 100% rename from jobs/airing-anime/airing-anime.go rename to jobs/airing-anime/main.go diff --git a/jobs/avatars/Avatar.go b/jobs/avatars/Avatar.go index f6fe3d64..b222c07b 100644 --- a/jobs/avatars/Avatar.go +++ b/jobs/avatars/Avatar.go @@ -1,9 +1,14 @@ package main -import "image" +import ( + "image" + + "github.com/animenotifier/arn" +) // Avatar represents a single image and the name of the format. type Avatar struct { + User *arn.User Image image.Image Data []byte Format string diff --git a/jobs/avatars/AvatarOriginalFileOutput.go b/jobs/avatars/AvatarOriginalFileOutput.go new file mode 100644 index 00000000..00cb50fd --- /dev/null +++ b/jobs/avatars/AvatarOriginalFileOutput.go @@ -0,0 +1,65 @@ +package main + +import ( + "bytes" + "errors" + "image/gif" + "image/jpeg" + "image/png" + "io/ioutil" + + "github.com/animenotifier/arn" + "github.com/nfnt/resize" +) + +// AvatarOriginalFileOutput ... +type AvatarOriginalFileOutput struct { + Directory string + Size int +} + +// SaveAvatar writes the original avatar to the file system. +func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error { + // Determine file extension + extension := "" + + switch avatar.Format { + case "jpg", "jpeg": + extension = ".jpg" + case "png": + extension = ".png" + case "gif": + extension = ".gif" + default: + return errors.New("Unknown format: " + avatar.Format) + } + + // Resize if needed + data := avatar.Data + img := avatar.Image + + if img.Bounds().Dx() != output.Size { + img = resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3) + buffer := new(bytes.Buffer) + + var err error + switch extension { + case ".jpg": + err = jpeg.Encode(buffer, img, nil) + case ".png": + err = png.Encode(buffer, img) + case ".gif": + err = gif.Encode(buffer, img, nil) + } + + if err != nil { + return err + } + + data = buffer.Bytes() + } + + // Write to file + fileName := output.Directory + avatar.User.ID + extension + return ioutil.WriteFile(fileName, data, 0644) +} diff --git a/jobs/avatars/AvatarWebPFileOutput.go b/jobs/avatars/AvatarWebPFileOutput.go new file mode 100644 index 00000000..bc48c05f --- /dev/null +++ b/jobs/avatars/AvatarWebPFileOutput.go @@ -0,0 +1,27 @@ +package main + +import ( + "github.com/animenotifier/arn" + "github.com/nfnt/resize" +) + +// AvatarWebPFileOutput ... +type AvatarWebPFileOutput struct { + Directory string + Size int + Quality float32 +} + +// SaveAvatar writes the avatar in WebP format to the file system. +func (output *AvatarWebPFileOutput) SaveAvatar(avatar *Avatar) error { + img := avatar.Image + + // Resize if needed + if img.Bounds().Dx() != output.Size { + img = resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3) + } + + // Write to file + fileName := output.Directory + avatar.User.ID + ".webp" + return arn.SaveWebP(img, fileName, output.Quality) +} diff --git a/jobs/avatars/AvatarWriter.go b/jobs/avatars/AvatarWriter.go index 54627502..eef1f5fc 100644 --- a/jobs/avatars/AvatarWriter.go +++ b/jobs/avatars/AvatarWriter.go @@ -1,6 +1,6 @@ package main -// AvatarWriter represents a system that saves an avatar locally (in database or as a file, e.g.) -type AvatarWriter interface { +// AvatarOutput represents a system that saves an avatar locally (in database or as a file, e.g.) +type AvatarOutput interface { SaveAvatar(*Avatar) error } diff --git a/jobs/avatars/FSWriter.go b/jobs/avatars/FSWriter.go deleted file mode 100644 index 06ab7d0f..00000000 --- a/jobs/avatars/FSWriter.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/jobs/avatars/Gravatar.go b/jobs/avatars/Gravatar.go index 5eb2b9dc..dedc5f7f 100644 --- a/jobs/avatars/Gravatar.go +++ b/jobs/avatars/Gravatar.go @@ -38,6 +38,7 @@ func (source *Gravatar) GetAvatar(user *arn.User) *Avatar { } return &Avatar{ + User: user, Image: img, Data: data, Format: format, diff --git a/jobs/avatars/download.go b/jobs/avatars/download.go deleted file mode 100644 index b8023964..00000000 --- a/jobs/avatars/download.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - - "github.com/animenotifier/arn" - "github.com/fatih/color" - "github.com/parnurzeal/gorequest" - gravatar "github.com/ungerik/go-gravatar" -) - -func findOriginalAvatar(user *arn.User) string { - return arn.FindFileWithExtension( - user.ID, - avatarsDirectoryOriginal, - []string{ - ".jpg", - ".png", - ".gif", - }, - ) -} - -func downloadAvatar(user *arn.User) bool { - if user.Email == "" { - return false - } - - directory := avatarsDirectoryOriginal - fileName := directory + user.ID - - // Build URL - url := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=pg" - - // Skip existing avatars - if findOriginalAvatar(user) != "" { - color.Yellow(user.Nick) - return true - } - - // Download - response, data, err := gorequest.New().Get(url).EndBytes() - - if err != nil { - color.Red(user.Nick) - return false - } - - contentType := response.Header.Get("content-type") - - if response.StatusCode != 200 { - color.Red(user.Nick) - return false - } - - // Determine file extension - extension := "" - - switch contentType { - case "image/jpeg": - extension = ".jpg" - case "image/png": - extension = ".png" - case "image/gif": - extension = ".gif" - case "image/webp": - extension = ".webp" - } - - fileName += extension - - // Write to disk - ioutil.WriteFile(fileName, data, 0644) - - color.Green(user.Nick) - - return true -} diff --git a/jobs/avatars/avatars.go b/jobs/avatars/main.go similarity index 53% rename from jobs/avatars/avatars.go rename to jobs/avatars/main.go index 96d75722..af0ec884 100644 --- a/jobs/avatars/avatars.go +++ b/jobs/avatars/main.go @@ -3,21 +3,25 @@ package main import ( "fmt" "os" + "reflect" "runtime" "time" + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "github.com/animenotifier/arn" "github.com/fatih/color" ) const ( - networkRateLimit = 100 * time.Millisecond - avatarsDirectoryOriginal = "images/avatars/large/original/" - avatarsDirectoryWebP = "images/avatars/large/webp/" + networkRateLimit = 100 * time.Millisecond + webPQuality = 80 ) var avatarSources []AvatarSource -var avatarWriters []AvatarWriter +var avatarOutputs []AvatarOutput // Main func main() { @@ -29,6 +33,28 @@ func main() { &Gravatar{}, } + // Define the avatar outputs + avatarOutputs = []AvatarOutput{ + &AvatarOriginalFileOutput{ + Directory: "images/avatars/large/original/", + Size: arn.AvatarMaxSize, + }, + &AvatarOriginalFileOutput{ + Directory: "images/avatars/small/original/", + Size: arn.AvatarSmallSize, + }, + &AvatarWebPFileOutput{ + Directory: "images/avatars/large/webp/", + Size: arn.AvatarMaxSize, + Quality: webPQuality, + }, + &AvatarWebPFileOutput{ + Directory: "images/avatars/small/webp/", + Size: arn.AvatarSmallSize, + Quality: webPQuality, + }, + } + // Stream of all users users, _ := arn.AllUsers() @@ -58,29 +84,29 @@ func StartWorkers(queue chan *arn.User, rateLimit time.Duration, work func(*arn. // Work handles a single user. func Work(user *arn.User) { + user.Avatar = "" + for _, source := range avatarSources { avatar := source.GetAvatar(user) if avatar == nil { - fmt.Println(color.RedString("✘"), user.Nick) + fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick) continue } - fmt.Println(color.GreenString("✔"), user.Nick, "|", avatar.Format, avatar.Image.Bounds().Dx(), avatar.Image.Bounds().Dy()) + for _, writer := range avatarOutputs { + err := writer.SaveAvatar(avatar) - for _, writer := range avatarWriters { - writer.SaveAvatar(avatar) + if err != nil { + color.Red(err.Error()) + } } - return + fmt.Println(color.GreenString("✔"), user.Nick, "|", avatar.Format, avatar.Image.Bounds().Dx(), avatar.Image.Bounds().Dy()) + user.Avatar = "/+" + user.Nick + "/avatar" + break } - // if downloadAvatar(user) { - // makeWebPAvatar(user) - // user.Avatar = "/+" + user.Nick + "/avatar" - // } else { - // user.Avatar = "" - // } - - // user.Save() + // Save avatar data + user.Save() } diff --git a/jobs/avatars/webp.go b/jobs/avatars/webp.go deleted file mode 100644 index 32246c87..00000000 --- a/jobs/avatars/webp.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - _ "image/gif" - _ "image/jpeg" - _ "image/png" - "strings" - - "github.com/animenotifier/arn" - "github.com/fatih/color" - "github.com/nfnt/resize" -) - -func makeWebPAvatar(user *arn.User) { - baseName := findOriginalAvatar(user) - - if baseName == "" { - return - } - - original := avatarsDirectoryOriginal + baseName - outFile := avatarsDirectoryWebP + user.ID + ".webp" - - err := avatarToWebP(original, outFile, 80) - - if err != nil { - color.Red(user.Nick + " [WebP]") - } else { - color.Green(user.Nick + " [WebP]") - } -} - -func avatarToWebP(in string, out string, quality float32) error { - img, _, loadErr := arn.LoadImage(in) - - if loadErr != nil { - return loadErr - } - - // Big avatar - saveErr := arn.SaveWebP(img, out, quality) - - if saveErr != nil { - return saveErr - } - - // Small avatar - smallImg := resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3) - saveErr = arn.SaveWebP(smallImg, strings.Replace(out, "large/", "small/", 1), quality) - return saveErr -} diff --git a/jobs/sync-anime/sync-anime.go b/jobs/sync-anime/main.go similarity index 100% rename from jobs/sync-anime/sync-anime.go rename to jobs/sync-anime/main.go diff --git a/patches/clear-sessions/clear-sessions.go b/patches/clear-sessions/main.go similarity index 100% rename from patches/clear-sessions/clear-sessions.go rename to patches/clear-sessions/main.go diff --git a/patches/user-references/user-references.go b/patches/user-references/main.go similarity index 100% rename from patches/user-references/user-references.go rename to patches/user-references/main.go