From bdb8d983d246bfbf066a8251a0c6ee99dc8e0205 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Tue, 13 Jun 2017 13:23:54 +0200 Subject: [PATCH] Improved avatar background job --- jobs/avatars/Avatar.go | 10 +++++ jobs/avatars/AvatarSource.go | 10 +++++ jobs/avatars/AvatarWriter.go | 6 +++ jobs/avatars/FSWriter.go | 1 + jobs/avatars/Gravatar.go | 45 +++++++++++++++++++ jobs/avatars/avatars.go | 87 ++++++++++++++++++++++++++---------- jobs/avatars/webp.go | 2 +- main.go | 4 +- 8 files changed, 139 insertions(+), 26 deletions(-) create mode 100644 jobs/avatars/Avatar.go create mode 100644 jobs/avatars/AvatarSource.go create mode 100644 jobs/avatars/AvatarWriter.go create mode 100644 jobs/avatars/FSWriter.go create mode 100644 jobs/avatars/Gravatar.go diff --git a/jobs/avatars/Avatar.go b/jobs/avatars/Avatar.go new file mode 100644 index 00000000..f6fe3d64 --- /dev/null +++ b/jobs/avatars/Avatar.go @@ -0,0 +1,10 @@ +package main + +import "image" + +// Avatar represents a single image and the name of the format. +type Avatar struct { + Image image.Image + Data []byte + Format string +} diff --git a/jobs/avatars/AvatarSource.go b/jobs/avatars/AvatarSource.go new file mode 100644 index 00000000..5d7c2253 --- /dev/null +++ b/jobs/avatars/AvatarSource.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/animenotifier/arn" +) + +// AvatarSource describes a source where we can find avatar images for a user. +type AvatarSource interface { + GetAvatar(*arn.User) *Avatar +} diff --git a/jobs/avatars/AvatarWriter.go b/jobs/avatars/AvatarWriter.go new file mode 100644 index 00000000..54627502 --- /dev/null +++ b/jobs/avatars/AvatarWriter.go @@ -0,0 +1,6 @@ +package main + +// AvatarWriter represents a system that saves an avatar locally (in database or as a file, e.g.) +type AvatarWriter interface { + SaveAvatar(*Avatar) error +} diff --git a/jobs/avatars/FSWriter.go b/jobs/avatars/FSWriter.go new file mode 100644 index 00000000..06ab7d0f --- /dev/null +++ b/jobs/avatars/FSWriter.go @@ -0,0 +1 @@ +package main diff --git a/jobs/avatars/Gravatar.go b/jobs/avatars/Gravatar.go new file mode 100644 index 00000000..5eb2b9dc --- /dev/null +++ b/jobs/avatars/Gravatar.go @@ -0,0 +1,45 @@ +package main + +import ( + "bytes" + "fmt" + "image" + + "github.com/animenotifier/arn" + "github.com/parnurzeal/gorequest" + gravatar "github.com/ungerik/go-gravatar" +) + +// Gravatar - https://gravatar.com/ +type Gravatar struct{} + +// GetAvatar returns the Gravatar image for a user (if available). +func (source *Gravatar) GetAvatar(user *arn.User) *Avatar { + // If the user has no Email registered we can't get a Gravatar. + if user.Email == "" { + return nil + } + + // Build URL + gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=pg" + + // Download + response, data, networkErr := gorequest.New().Get(gravatarURL).EndBytes() + + if networkErr != nil || response.StatusCode != 200 { + return nil + } + + // Decode + img, format, decodeErr := image.Decode(bytes.NewReader(data)) + + if decodeErr != nil { + return nil + } + + return &Avatar{ + Image: img, + Data: data, + Format: format, + } +} diff --git a/jobs/avatars/avatars.go b/jobs/avatars/avatars.go index 54e5e49e..96d75722 100644 --- a/jobs/avatars/avatars.go +++ b/jobs/avatars/avatars.go @@ -1,45 +1,86 @@ package main import ( + "fmt" "os" "runtime" "time" "github.com/animenotifier/arn" + "github.com/fatih/color" ) const ( - avatarsDirectoryOriginal = "images/avatars/original/" - avatarsDirectoryWebP = "images/avatars/webp/" + networkRateLimit = 100 * time.Millisecond + avatarsDirectoryOriginal = "images/avatars/large/original/" + avatarsDirectoryWebP = "images/avatars/large/webp/" ) +var avatarSources []AvatarSource +var avatarWriters []AvatarWriter + +// Main func main() { + // Switch to main directory os.Chdir("../../") - users, _ := arn.AllUsers() - - usersQueue := make(chan *arn.User) - rateLimiter := time.NewTicker(100 * time.Millisecond) - defer rateLimiter.Stop() - - for w := 0; w < runtime.NumCPU(); w++ { - go func(workerID int) { - for user := range usersQueue { - <-rateLimiter.C - - if downloadAvatar(user) { - makeWebPAvatar(user) - user.Avatar = "/+" + user.Nick + "/avatar" - } else { - user.Avatar = "" - } - - user.Save() - } - }(w) + // Define the avatar sources + avatarSources = []AvatarSource{ + &Gravatar{}, } + // Stream of all users + users, _ := arn.AllUsers() + + // Worker queue + usersQueue := make(chan *arn.User) + StartWorkers(usersQueue, networkRateLimit, Work) + + // We'll send each user to one of the worker threads for user := range users { usersQueue <- user } } + +// StartWorkers creates multiple workers to handle a user each. +func StartWorkers(queue chan *arn.User, rateLimit time.Duration, work func(*arn.User)) { + rateLimiter := time.NewTicker(rateLimit) + + for w := 0; w < runtime.NumCPU(); w++ { + go func() { + for user := range queue { + <-rateLimiter.C + work(user) + } + }() + } +} + +// Work handles a single user. +func Work(user *arn.User) { + for _, source := range avatarSources { + avatar := source.GetAvatar(user) + + if avatar == nil { + fmt.Println(color.RedString("✘"), user.Nick) + continue + } + + fmt.Println(color.GreenString("✔"), user.Nick, "|", avatar.Format, avatar.Image.Bounds().Dx(), avatar.Image.Bounds().Dy()) + + for _, writer := range avatarWriters { + writer.SaveAvatar(avatar) + } + + return + } + + // if downloadAvatar(user) { + // makeWebPAvatar(user) + // user.Avatar = "/+" + user.Nick + "/avatar" + // } else { + // user.Avatar = "" + // } + + // user.Save() +} diff --git a/jobs/avatars/webp.go b/jobs/avatars/webp.go index 3a08edd9..32246c87 100644 --- a/jobs/avatars/webp.go +++ b/jobs/avatars/webp.go @@ -46,6 +46,6 @@ func avatarToWebP(in string, out string, quality float32) error { // Small avatar smallImg := resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3) - saveErr = arn.SaveWebP(smallImg, strings.Replace(out, "webp/", "webp-small/", 1), quality) + saveErr = arn.SaveWebP(smallImg, strings.Replace(out, "large/", "small/", 1), quality) return saveErr } diff --git a/main.go b/main.go index 2092dd58..ceefeaf1 100644 --- a/main.go +++ b/main.go @@ -89,7 +89,7 @@ func main() { } if ctx.CanUseWebP() { - return ctx.File("images/avatars/webp/" + user.ID + ".webp") + return ctx.File("images/avatars/large/webp/" + user.ID + ".webp") } err = errors.New("Your browser doesn't support the WebP image format") @@ -106,7 +106,7 @@ func main() { } if ctx.CanUseWebP() { - return ctx.File("images/avatars/webp-small/" + user.ID + ".webp") + return ctx.File("images/avatars/small/webp/" + user.ID + ".webp") } err = errors.New("Your browser doesn't support the WebP image format")