From 1706ef9bc48bfc7c07896d5eae00b93bc574de84 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Tue, 18 Jul 2017 03:55:47 +0200 Subject: [PATCH] Improved avatars background job --- bots/avatars/avatars.go | 23 +++++++ jobs/avatars/Avatar.go | 87 ------------------------ jobs/avatars/AvatarOriginalFileOutput.go | 60 ---------------- jobs/avatars/AvatarSource.go | 10 --- jobs/avatars/AvatarWebPFileOutput.go | 27 -------- jobs/avatars/AvatarWriter.go | 6 -- jobs/avatars/FileSystem.go | 48 ------------- jobs/avatars/Gravatar.go | 37 ---------- jobs/avatars/MyAnimeList.go | 60 ---------------- jobs/avatars/avatars.go | 36 ++++++---- jobs/jobs.go | 2 +- pages/settings/settings.pixy | 11 ++- 12 files changed, 56 insertions(+), 351 deletions(-) create mode 100644 bots/avatars/avatars.go delete mode 100644 jobs/avatars/Avatar.go delete mode 100644 jobs/avatars/AvatarOriginalFileOutput.go delete mode 100644 jobs/avatars/AvatarSource.go delete mode 100644 jobs/avatars/AvatarWebPFileOutput.go delete mode 100644 jobs/avatars/AvatarWriter.go delete mode 100644 jobs/avatars/FileSystem.go delete mode 100644 jobs/avatars/Gravatar.go delete mode 100644 jobs/avatars/MyAnimeList.go diff --git a/bots/avatars/avatars.go b/bots/avatars/avatars.go new file mode 100644 index 00000000..340308d0 --- /dev/null +++ b/bots/avatars/avatars.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "log" + "net/http" +) + +func refreshAvatar(w http.ResponseWriter, req *http.Request) { + w.Write([]byte("ok")) +} + +var port = "8001" + +func init() { + flag.StringVar(&port, "port", "", "Port the HTTP server should listen on") + flag.Parse() +} + +func main() { + http.HandleFunc("/", refreshAvatar) + log.Fatal(http.ListenAndServe(":"+port, nil)) +} diff --git a/jobs/avatars/Avatar.go b/jobs/avatars/Avatar.go deleted file mode 100644 index f1748c37..00000000 --- a/jobs/avatars/Avatar.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "image" - "net/http" - "strings" - "time" - - "github.com/animenotifier/arn" - "github.com/parnurzeal/gorequest" -) - -var netLog = avatarLog.NewChannel("NET") - -// Avatar represents a single image and the name of the format. -type Avatar struct { - User *arn.User - Image image.Image - Data []byte - Format string -} - -// Extension ... -func (avatar *Avatar) Extension() string { - switch avatar.Format { - case "jpg", "jpeg": - return ".jpg" - case "png": - return ".png" - case "gif": - return ".gif" - default: - return "" - } -} - -// String returns a text representation of the format, width and height. -func (avatar *Avatar) String() string { - return fmt.Sprint(avatar.Format, " | ", avatar.Image.Bounds().Dx(), "x", avatar.Image.Bounds().Dy()) -} - -// AvatarFromURL downloads and decodes the image from an URL and creates an Avatar. -func AvatarFromURL(url string, user *arn.User) *Avatar { - // Download - response, data, networkErrs := gorequest.New().Get(url).EndBytes() - - // Network errors - if len(networkErrs) > 0 { - netLog.Error(user.Nick, url, networkErrs[0]) - return nil - } - - // Retry HTTP only version after 5 seconds if service unavailable - if response == nil || response.StatusCode == http.StatusServiceUnavailable { - time.Sleep(5 * time.Second) - response, data, networkErrs = gorequest.New().Get(strings.Replace(url, "https://", "http://", 1)).EndBytes() - } - - // Network errors on 2nd try - if len(networkErrs) > 0 { - netLog.Error(user.Nick, url, networkErrs[0]) - return nil - } - - // Bad status codes - if response.StatusCode != http.StatusOK { - netLog.Error(user.Nick, url, response.StatusCode) - return nil - } - - // Decode - img, format, decodeErr := image.Decode(bytes.NewReader(data)) - - if decodeErr != nil { - netLog.Error(user.Nick, url, decodeErr) - return nil - } - - return &Avatar{ - User: user, - Image: img, - Data: data, - Format: format, - } -} diff --git a/jobs/avatars/AvatarOriginalFileOutput.go b/jobs/avatars/AvatarOriginalFileOutput.go deleted file mode 100644 index 9d173d5b..00000000 --- a/jobs/avatars/AvatarOriginalFileOutput.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "bytes" - "errors" - "image/gif" - "image/jpeg" - "image/png" - "io/ioutil" - - "github.com/nfnt/resize" -) - -// AvatarOriginalFileOutput ... -type AvatarOriginalFileOutput struct { - Directory string - Size int -} - -// SaveAvatar writes the original avatar to the file system. -func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error { - // Determine file extension - extension := avatar.Extension() - - if extension == "" { - return errors.New("Unknown format: " + avatar.Format) - } - - // Resize if needed - data := avatar.Data - img := avatar.Image - - if img.Bounds().Dx() > output.Size { - img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3) - buffer := new(bytes.Buffer) - - var err error - switch extension { - case ".jpg": - err = jpeg.Encode(buffer, img, nil) - case ".png": - err = png.Encode(buffer, img) - case ".gif": - err = gif.Encode(buffer, img, nil) - } - - if err != nil { - return err - } - - data = buffer.Bytes() - } - - // Set user avatar - avatar.User.Avatar.Extension = extension - - // Write to file - fileName := output.Directory + avatar.User.ID + extension - return ioutil.WriteFile(fileName, data, 0644) -} diff --git a/jobs/avatars/AvatarSource.go b/jobs/avatars/AvatarSource.go deleted file mode 100644 index 5d7c2253..00000000 --- a/jobs/avatars/AvatarSource.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import ( - "github.com/animenotifier/arn" -) - -// AvatarSource describes a source where we can find avatar images for a user. -type AvatarSource interface { - GetAvatar(*arn.User) *Avatar -} diff --git a/jobs/avatars/AvatarWebPFileOutput.go b/jobs/avatars/AvatarWebPFileOutput.go deleted file mode 100644 index a3aaf150..00000000 --- a/jobs/avatars/AvatarWebPFileOutput.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "github.com/animenotifier/arn" - "github.com/nfnt/resize" -) - -// AvatarWebPFileOutput ... -type AvatarWebPFileOutput struct { - Directory string - Size int - Quality float32 -} - -// SaveAvatar writes the avatar in WebP format to the file system. -func (output *AvatarWebPFileOutput) SaveAvatar(avatar *Avatar) error { - img := avatar.Image - - // Resize if needed - if img.Bounds().Dx() > output.Size { - img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3) - } - - // Write to file - fileName := output.Directory + avatar.User.ID + ".webp" - return arn.SaveWebP(img, fileName, output.Quality) -} diff --git a/jobs/avatars/AvatarWriter.go b/jobs/avatars/AvatarWriter.go deleted file mode 100644 index eef1f5fc..00000000 --- a/jobs/avatars/AvatarWriter.go +++ /dev/null @@ -1,6 +0,0 @@ -package main - -// AvatarOutput represents a system that saves an avatar locally (in database or as a file, e.g.) -type AvatarOutput interface { - SaveAvatar(*Avatar) error -} diff --git a/jobs/avatars/FileSystem.go b/jobs/avatars/FileSystem.go deleted file mode 100644 index b97b1363..00000000 --- a/jobs/avatars/FileSystem.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "bytes" - "image" - "io/ioutil" - - "github.com/animenotifier/arn" -) - -var fileSystemLog = avatarLog.NewChannel("SSD") - -// FileSystem loads avatar from the local filesystem. -type FileSystem struct { - Directory string -} - -// GetAvatar returns the local image for the user. -func (source *FileSystem) GetAvatar(user *arn.User) *Avatar { - fullPath := arn.FindFileWithExtension(user.ID, source.Directory, arn.OriginalImageExtensions) - - if fullPath == "" { - fileSystemLog.Error(user.Nick, "Not found on file system") - return nil - } - - data, err := ioutil.ReadFile(fullPath) - - if err != nil { - fileSystemLog.Error(user.Nick, err) - return nil - } - - // Decode - img, format, decodeErr := image.Decode(bytes.NewReader(data)) - - if decodeErr != nil { - fileSystemLog.Error(user.Nick, decodeErr) - return nil - } - - return &Avatar{ - User: user, - Image: img, - Data: data, - Format: format, - } -} diff --git a/jobs/avatars/Gravatar.go b/jobs/avatars/Gravatar.go deleted file mode 100644 index 731d863f..00000000 --- a/jobs/avatars/Gravatar.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "fmt" - "strings" - "time" - - "github.com/animenotifier/arn" - gravatar "github.com/ungerik/go-gravatar" -) - -var gravatarLog = avatarLog.NewChannel("GRA") - -// Gravatar - https://gravatar.com/ -type Gravatar struct { - Rating string - RequestLimiter *time.Ticker -} - -// GetAvatar returns the Gravatar image for a user (if available). -func (source *Gravatar) GetAvatar(user *arn.User) *Avatar { - // If the user has no Email registered we can't get a Gravatar. - if user.Email == "" { - gravatarLog.Error(user.Nick, "No Email") - return nil - } - - // Build URL - gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=" + source.Rating - gravatarURL = strings.Replace(gravatarURL, "http://", "https://", 1) - - // Wait for request limiter to allow us to send a request - <-source.RequestLimiter.C - - // Download - return AvatarFromURL(gravatarURL, user) -} diff --git a/jobs/avatars/MyAnimeList.go b/jobs/avatars/MyAnimeList.go deleted file mode 100644 index e1a30b52..00000000 --- a/jobs/avatars/MyAnimeList.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "net/http" - "regexp" - "time" - - "github.com/animenotifier/arn" - "github.com/parnurzeal/gorequest" -) - -var userIDRegex = regexp.MustCompile(`(\d+)<\/user_id>`) -var malLog = avatarLog.NewChannel("MAL") - -// MyAnimeList - https://myanimelist.net/ -type MyAnimeList struct { - RequestLimiter *time.Ticker -} - -// GetAvatar returns the Gravatar image for a user (if available). -func (source *MyAnimeList) GetAvatar(user *arn.User) *Avatar { - malNick := user.Accounts.MyAnimeList.Nick - - // If the user has no username we can't get an avatar. - if malNick == "" { - malLog.Error(user.Nick, "No MAL nick") - return nil - } - - // Download user info - userInfoURL := "https://myanimelist.net/malappinfo.php?u=" + malNick - response, xml, networkErr := gorequest.New().Get(userInfoURL).End() - - if networkErr != nil { - malLog.Error(user.Nick, userInfoURL, networkErr) - return nil - } - - if response.StatusCode != http.StatusOK { - malLog.Error(user.Nick, userInfoURL, response.StatusCode) - return nil - } - - // Build URL - matches := userIDRegex.FindStringSubmatch(xml) - - if matches == nil || len(matches) < 2 { - malLog.Error(user.Nick, "Could not find user ID") - return nil - } - - malID := matches[1] - malAvatarURL := "https://myanimelist.cdn-dena.com/images/userimages/" + malID + ".jpg" - - // Wait for request limiter to allow us to send a request - <-source.RequestLimiter.C - - // Download - return AvatarFromURL(malAvatarURL, user) -} diff --git a/jobs/avatars/avatars.go b/jobs/avatars/avatars.go index 70eeb352..ac1757f1 100644 --- a/jobs/avatars/avatars.go +++ b/jobs/avatars/avatars.go @@ -15,6 +15,9 @@ import ( "github.com/aerogo/log" "github.com/animenotifier/arn" + "github.com/animenotifier/avatar" + "github.com/animenotifier/avatar/outputs" + "github.com/animenotifier/avatar/sources" "github.com/fatih/color" ) @@ -22,8 +25,8 @@ const ( webPQuality = 80 ) -var avatarSources []AvatarSource -var avatarOutputs []AvatarOutput +var avatarSources []avatar.Source +var avatarOutputs []avatar.Output var avatarLog = log.New() var wg sync.WaitGroup @@ -46,42 +49,42 @@ func main() { defer avatarLog.Flush() // Define the avatar sources - avatarSources = []AvatarSource{ - &Gravatar{ + avatarSources = []avatar.Source{ + &sources.Gravatar{ Rating: "pg", + RequestLimiter: time.NewTicker(100 * time.Millisecond), + }, + &sources.MyAnimeList{ RequestLimiter: time.NewTicker(250 * time.Millisecond), }, - &MyAnimeList{ - RequestLimiter: time.NewTicker(250 * time.Millisecond), - }, - &FileSystem{ + &sources.FileSystem{ Directory: "images/avatars/large/", }, } // Define the avatar outputs - avatarOutputs = []AvatarOutput{ + avatarOutputs = []avatar.Output{ // Original - Large - &AvatarOriginalFileOutput{ + &outputs.OriginalFile{ Directory: "images/avatars/large/", Size: arn.AvatarMaxSize, }, // Original - Small - &AvatarOriginalFileOutput{ + &outputs.OriginalFile{ Directory: "images/avatars/small/", Size: arn.AvatarSmallSize, }, // WebP - Large - &AvatarWebPFileOutput{ + &outputs.WebPFile{ Directory: "images/avatars/large/", Size: arn.AvatarMaxSize, Quality: webPQuality, }, // WebP - Small - &AvatarWebPFileOutput{ + &outputs.WebPFile{ Directory: "images/avatars/small/", Size: arn.AvatarSmallSize, Quality: webPQuality, @@ -126,7 +129,12 @@ func Work(user *arn.User) { user.Avatar.Extension = "" for _, source := range avatarSources { - avatar := source.GetAvatar(user) + avatar, err := source.GetAvatar(user) + + if err != nil { + avatarLog.Error(err) + continue + } if avatar == nil { // fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick) diff --git a/jobs/jobs.go b/jobs/jobs.go index 2c635708..276f5f7c 100644 --- a/jobs/jobs.go +++ b/jobs/jobs.go @@ -23,8 +23,8 @@ var colorPool = []*color.Color{ } var jobs = map[string]time.Duration{ - "active-users": 1 * time.Minute, "forum-activity": 1 * time.Minute, + "active-users": 5 * time.Minute, "anime-ratings": 10 * time.Minute, "airing-anime": 10 * time.Minute, "statistics": 15 * time.Minute, diff --git a/pages/settings/settings.pixy b/pages/settings/settings.pixy index 59a8c0a3..6c89d07f 100644 --- a/pages/settings/settings.pixy +++ b/pages/settings/settings.pixy @@ -117,10 +117,19 @@ component Settings(user *arn.User) select.widget-element.action(id="Avatar.Source", data-field="Avatar.Source", value=user.Settings().Avatar.Source, data-action="save", data-trigger="change") option(value="") Automatic option(value="Gravatar") Gravatar + option(value="URL") Link + option(value="FileSystem") Upload - if "Gravatar" == "Gravatar" + if user.Settings().Avatar.Source == "Gravatar" || (user.Settings().Avatar.Source == "" && user.Avatar.Source == "Gravatar") .profile-image-container.avatar-preview img.profile-image(src=user.Gravatar(), alt="Gravatar") + + if user.Settings().Avatar.Source == "URL" + InputText("Avatar.SourceURL", user.Settings().Avatar.SourceURL, "Link", "Post the link to the image here") + + if user.Settings().Avatar.SourceURL != "" + .profile-image-container.avatar-preview + img.profile-image(src=strings.Replace(user.Settings().Avatar.SourceURL, "http://", "https://", 1), alt="Avatar preview") //- .widget.mountable(data-api="/api/settings/" + user.ID) //- h3.widget-title