Improved avatars background job

This commit is contained in:
Eduard Urbach 2017-07-18 03:55:47 +02:00
parent 1923a1a889
commit 1706ef9bc4
12 changed files with 56 additions and 351 deletions

23
bots/avatars/avatars.go Normal file
View File

@ -0,0 +1,23 @@
package main
import (
"flag"
"log"
"net/http"
)
func refreshAvatar(w http.ResponseWriter, req *http.Request) {
w.Write([]byte("ok"))
}
var port = "8001"
func init() {
flag.StringVar(&port, "port", "", "Port the HTTP server should listen on")
flag.Parse()
}
func main() {
http.HandleFunc("/", refreshAvatar)
log.Fatal(http.ListenAndServe(":"+port, nil))
}

View File

@ -1,87 +0,0 @@
package main
import (
"bytes"
"fmt"
"image"
"net/http"
"strings"
"time"
"github.com/animenotifier/arn"
"github.com/parnurzeal/gorequest"
)
var netLog = avatarLog.NewChannel("NET")
// Avatar represents a single image and the name of the format.
type Avatar struct {
User *arn.User
Image image.Image
Data []byte
Format string
}
// Extension ...
func (avatar *Avatar) Extension() string {
switch avatar.Format {
case "jpg", "jpeg":
return ".jpg"
case "png":
return ".png"
case "gif":
return ".gif"
default:
return ""
}
}
// String returns a text representation of the format, width and height.
func (avatar *Avatar) String() string {
return fmt.Sprint(avatar.Format, " | ", avatar.Image.Bounds().Dx(), "x", avatar.Image.Bounds().Dy())
}
// AvatarFromURL downloads and decodes the image from an URL and creates an Avatar.
func AvatarFromURL(url string, user *arn.User) *Avatar {
// Download
response, data, networkErrs := gorequest.New().Get(url).EndBytes()
// Network errors
if len(networkErrs) > 0 {
netLog.Error(user.Nick, url, networkErrs[0])
return nil
}
// Retry HTTP only version after 5 seconds if service unavailable
if response == nil || response.StatusCode == http.StatusServiceUnavailable {
time.Sleep(5 * time.Second)
response, data, networkErrs = gorequest.New().Get(strings.Replace(url, "https://", "http://", 1)).EndBytes()
}
// Network errors on 2nd try
if len(networkErrs) > 0 {
netLog.Error(user.Nick, url, networkErrs[0])
return nil
}
// Bad status codes
if response.StatusCode != http.StatusOK {
netLog.Error(user.Nick, url, response.StatusCode)
return nil
}
// Decode
img, format, decodeErr := image.Decode(bytes.NewReader(data))
if decodeErr != nil {
netLog.Error(user.Nick, url, decodeErr)
return nil
}
return &Avatar{
User: user,
Image: img,
Data: data,
Format: format,
}
}

View File

@ -1,60 +0,0 @@
package main
import (
"bytes"
"errors"
"image/gif"
"image/jpeg"
"image/png"
"io/ioutil"
"github.com/nfnt/resize"
)
// AvatarOriginalFileOutput ...
type AvatarOriginalFileOutput struct {
Directory string
Size int
}
// SaveAvatar writes the original avatar to the file system.
func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error {
// Determine file extension
extension := avatar.Extension()
if extension == "" {
return errors.New("Unknown format: " + avatar.Format)
}
// Resize if needed
data := avatar.Data
img := avatar.Image
if img.Bounds().Dx() > output.Size {
img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3)
buffer := new(bytes.Buffer)
var err error
switch extension {
case ".jpg":
err = jpeg.Encode(buffer, img, nil)
case ".png":
err = png.Encode(buffer, img)
case ".gif":
err = gif.Encode(buffer, img, nil)
}
if err != nil {
return err
}
data = buffer.Bytes()
}
// Set user avatar
avatar.User.Avatar.Extension = extension
// Write to file
fileName := output.Directory + avatar.User.ID + extension
return ioutil.WriteFile(fileName, data, 0644)
}

View File

@ -1,10 +0,0 @@
package main
import (
"github.com/animenotifier/arn"
)
// AvatarSource describes a source where we can find avatar images for a user.
type AvatarSource interface {
GetAvatar(*arn.User) *Avatar
}

View File

@ -1,27 +0,0 @@
package main
import (
"github.com/animenotifier/arn"
"github.com/nfnt/resize"
)
// AvatarWebPFileOutput ...
type AvatarWebPFileOutput struct {
Directory string
Size int
Quality float32
}
// SaveAvatar writes the avatar in WebP format to the file system.
func (output *AvatarWebPFileOutput) SaveAvatar(avatar *Avatar) error {
img := avatar.Image
// Resize if needed
if img.Bounds().Dx() > output.Size {
img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3)
}
// Write to file
fileName := output.Directory + avatar.User.ID + ".webp"
return arn.SaveWebP(img, fileName, output.Quality)
}

View File

@ -1,6 +0,0 @@
package main
// AvatarOutput represents a system that saves an avatar locally (in database or as a file, e.g.)
type AvatarOutput interface {
SaveAvatar(*Avatar) error
}

View File

@ -1,48 +0,0 @@
package main
import (
"bytes"
"image"
"io/ioutil"
"github.com/animenotifier/arn"
)
var fileSystemLog = avatarLog.NewChannel("SSD")
// FileSystem loads avatar from the local filesystem.
type FileSystem struct {
Directory string
}
// GetAvatar returns the local image for the user.
func (source *FileSystem) GetAvatar(user *arn.User) *Avatar {
fullPath := arn.FindFileWithExtension(user.ID, source.Directory, arn.OriginalImageExtensions)
if fullPath == "" {
fileSystemLog.Error(user.Nick, "Not found on file system")
return nil
}
data, err := ioutil.ReadFile(fullPath)
if err != nil {
fileSystemLog.Error(user.Nick, err)
return nil
}
// Decode
img, format, decodeErr := image.Decode(bytes.NewReader(data))
if decodeErr != nil {
fileSystemLog.Error(user.Nick, decodeErr)
return nil
}
return &Avatar{
User: user,
Image: img,
Data: data,
Format: format,
}
}

View File

@ -1,37 +0,0 @@
package main
import (
"fmt"
"strings"
"time"
"github.com/animenotifier/arn"
gravatar "github.com/ungerik/go-gravatar"
)
var gravatarLog = avatarLog.NewChannel("GRA")
// Gravatar - https://gravatar.com/
type Gravatar struct {
Rating string
RequestLimiter *time.Ticker
}
// GetAvatar returns the Gravatar image for a user (if available).
func (source *Gravatar) GetAvatar(user *arn.User) *Avatar {
// If the user has no Email registered we can't get a Gravatar.
if user.Email == "" {
gravatarLog.Error(user.Nick, "No Email")
return nil
}
// Build URL
gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=" + source.Rating
gravatarURL = strings.Replace(gravatarURL, "http://", "https://", 1)
// Wait for request limiter to allow us to send a request
<-source.RequestLimiter.C
// Download
return AvatarFromURL(gravatarURL, user)
}

View File

@ -1,60 +0,0 @@
package main
import (
"net/http"
"regexp"
"time"
"github.com/animenotifier/arn"
"github.com/parnurzeal/gorequest"
)
var userIDRegex = regexp.MustCompile(`<user_id>(\d+)<\/user_id>`)
var malLog = avatarLog.NewChannel("MAL")
// MyAnimeList - https://myanimelist.net/
type MyAnimeList struct {
RequestLimiter *time.Ticker
}
// GetAvatar returns the Gravatar image for a user (if available).
func (source *MyAnimeList) GetAvatar(user *arn.User) *Avatar {
malNick := user.Accounts.MyAnimeList.Nick
// If the user has no username we can't get an avatar.
if malNick == "" {
malLog.Error(user.Nick, "No MAL nick")
return nil
}
// Download user info
userInfoURL := "https://myanimelist.net/malappinfo.php?u=" + malNick
response, xml, networkErr := gorequest.New().Get(userInfoURL).End()
if networkErr != nil {
malLog.Error(user.Nick, userInfoURL, networkErr)
return nil
}
if response.StatusCode != http.StatusOK {
malLog.Error(user.Nick, userInfoURL, response.StatusCode)
return nil
}
// Build URL
matches := userIDRegex.FindStringSubmatch(xml)
if matches == nil || len(matches) < 2 {
malLog.Error(user.Nick, "Could not find user ID")
return nil
}
malID := matches[1]
malAvatarURL := "https://myanimelist.cdn-dena.com/images/userimages/" + malID + ".jpg"
// Wait for request limiter to allow us to send a request
<-source.RequestLimiter.C
// Download
return AvatarFromURL(malAvatarURL, user)
}

View File

@ -15,6 +15,9 @@ import (
"github.com/aerogo/log" "github.com/aerogo/log"
"github.com/animenotifier/arn" "github.com/animenotifier/arn"
"github.com/animenotifier/avatar"
"github.com/animenotifier/avatar/outputs"
"github.com/animenotifier/avatar/sources"
"github.com/fatih/color" "github.com/fatih/color"
) )
@ -22,8 +25,8 @@ const (
webPQuality = 80 webPQuality = 80
) )
var avatarSources []AvatarSource var avatarSources []avatar.Source
var avatarOutputs []AvatarOutput var avatarOutputs []avatar.Output
var avatarLog = log.New() var avatarLog = log.New()
var wg sync.WaitGroup var wg sync.WaitGroup
@ -46,42 +49,42 @@ func main() {
defer avatarLog.Flush() defer avatarLog.Flush()
// Define the avatar sources // Define the avatar sources
avatarSources = []AvatarSource{ avatarSources = []avatar.Source{
&Gravatar{ &sources.Gravatar{
Rating: "pg", Rating: "pg",
RequestLimiter: time.NewTicker(100 * time.Millisecond),
},
&sources.MyAnimeList{
RequestLimiter: time.NewTicker(250 * time.Millisecond), RequestLimiter: time.NewTicker(250 * time.Millisecond),
}, },
&MyAnimeList{ &sources.FileSystem{
RequestLimiter: time.NewTicker(250 * time.Millisecond),
},
&FileSystem{
Directory: "images/avatars/large/", Directory: "images/avatars/large/",
}, },
} }
// Define the avatar outputs // Define the avatar outputs
avatarOutputs = []AvatarOutput{ avatarOutputs = []avatar.Output{
// Original - Large // Original - Large
&AvatarOriginalFileOutput{ &outputs.OriginalFile{
Directory: "images/avatars/large/", Directory: "images/avatars/large/",
Size: arn.AvatarMaxSize, Size: arn.AvatarMaxSize,
}, },
// Original - Small // Original - Small
&AvatarOriginalFileOutput{ &outputs.OriginalFile{
Directory: "images/avatars/small/", Directory: "images/avatars/small/",
Size: arn.AvatarSmallSize, Size: arn.AvatarSmallSize,
}, },
// WebP - Large // WebP - Large
&AvatarWebPFileOutput{ &outputs.WebPFile{
Directory: "images/avatars/large/", Directory: "images/avatars/large/",
Size: arn.AvatarMaxSize, Size: arn.AvatarMaxSize,
Quality: webPQuality, Quality: webPQuality,
}, },
// WebP - Small // WebP - Small
&AvatarWebPFileOutput{ &outputs.WebPFile{
Directory: "images/avatars/small/", Directory: "images/avatars/small/",
Size: arn.AvatarSmallSize, Size: arn.AvatarSmallSize,
Quality: webPQuality, Quality: webPQuality,
@ -126,7 +129,12 @@ func Work(user *arn.User) {
user.Avatar.Extension = "" user.Avatar.Extension = ""
for _, source := range avatarSources { for _, source := range avatarSources {
avatar := source.GetAvatar(user) avatar, err := source.GetAvatar(user)
if err != nil {
avatarLog.Error(err)
continue
}
if avatar == nil { if avatar == nil {
// fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick) // fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick)

View File

@ -23,8 +23,8 @@ var colorPool = []*color.Color{
} }
var jobs = map[string]time.Duration{ var jobs = map[string]time.Duration{
"active-users": 1 * time.Minute,
"forum-activity": 1 * time.Minute, "forum-activity": 1 * time.Minute,
"active-users": 5 * time.Minute,
"anime-ratings": 10 * time.Minute, "anime-ratings": 10 * time.Minute,
"airing-anime": 10 * time.Minute, "airing-anime": 10 * time.Minute,
"statistics": 15 * time.Minute, "statistics": 15 * time.Minute,

View File

@ -117,11 +117,20 @@ component Settings(user *arn.User)
select.widget-element.action(id="Avatar.Source", data-field="Avatar.Source", value=user.Settings().Avatar.Source, data-action="save", data-trigger="change") select.widget-element.action(id="Avatar.Source", data-field="Avatar.Source", value=user.Settings().Avatar.Source, data-action="save", data-trigger="change")
option(value="") Automatic option(value="") Automatic
option(value="Gravatar") Gravatar option(value="Gravatar") Gravatar
option(value="URL") Link
option(value="FileSystem") Upload
if "Gravatar" == "Gravatar" if user.Settings().Avatar.Source == "Gravatar" || (user.Settings().Avatar.Source == "" && user.Avatar.Source == "Gravatar")
.profile-image-container.avatar-preview .profile-image-container.avatar-preview
img.profile-image(src=user.Gravatar(), alt="Gravatar") img.profile-image(src=user.Gravatar(), alt="Gravatar")
if user.Settings().Avatar.Source == "URL"
InputText("Avatar.SourceURL", user.Settings().Avatar.SourceURL, "Link", "Post the link to the image here")
if user.Settings().Avatar.SourceURL != ""
.profile-image-container.avatar-preview
img.profile-image(src=strings.Replace(user.Settings().Avatar.SourceURL, "http://", "https://", 1), alt="Avatar preview")
//- .widget.mountable(data-api="/api/settings/" + user.ID) //- .widget.mountable(data-api="/api/settings/" + user.ID)
//- h3.widget-title //- h3.widget-title
//- Icon("cogs") //- Icon("cogs")