Improved avatars background job
This commit is contained in:
parent
1923a1a889
commit
1706ef9bc4
23
bots/avatars/avatars.go
Normal file
23
bots/avatars/avatars.go
Normal file
@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func refreshAvatar(w http.ResponseWriter, req *http.Request) {
|
||||
w.Write([]byte("ok"))
|
||||
}
|
||||
|
||||
var port = "8001"
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&port, "port", "", "Port the HTTP server should listen on")
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func main() {
|
||||
http.HandleFunc("/", refreshAvatar)
|
||||
log.Fatal(http.ListenAndServe(":"+port, nil))
|
||||
}
|
||||
@ -1,87 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"image"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/animenotifier/arn"
|
||||
"github.com/parnurzeal/gorequest"
|
||||
)
|
||||
|
||||
var netLog = avatarLog.NewChannel("NET")
|
||||
|
||||
// Avatar represents a single image and the name of the format.
|
||||
type Avatar struct {
|
||||
User *arn.User
|
||||
Image image.Image
|
||||
Data []byte
|
||||
Format string
|
||||
}
|
||||
|
||||
// Extension ...
|
||||
func (avatar *Avatar) Extension() string {
|
||||
switch avatar.Format {
|
||||
case "jpg", "jpeg":
|
||||
return ".jpg"
|
||||
case "png":
|
||||
return ".png"
|
||||
case "gif":
|
||||
return ".gif"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// String returns a text representation of the format, width and height.
|
||||
func (avatar *Avatar) String() string {
|
||||
return fmt.Sprint(avatar.Format, " | ", avatar.Image.Bounds().Dx(), "x", avatar.Image.Bounds().Dy())
|
||||
}
|
||||
|
||||
// AvatarFromURL downloads and decodes the image from an URL and creates an Avatar.
|
||||
func AvatarFromURL(url string, user *arn.User) *Avatar {
|
||||
// Download
|
||||
response, data, networkErrs := gorequest.New().Get(url).EndBytes()
|
||||
|
||||
// Network errors
|
||||
if len(networkErrs) > 0 {
|
||||
netLog.Error(user.Nick, url, networkErrs[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retry HTTP only version after 5 seconds if service unavailable
|
||||
if response == nil || response.StatusCode == http.StatusServiceUnavailable {
|
||||
time.Sleep(5 * time.Second)
|
||||
response, data, networkErrs = gorequest.New().Get(strings.Replace(url, "https://", "http://", 1)).EndBytes()
|
||||
}
|
||||
|
||||
// Network errors on 2nd try
|
||||
if len(networkErrs) > 0 {
|
||||
netLog.Error(user.Nick, url, networkErrs[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
// Bad status codes
|
||||
if response.StatusCode != http.StatusOK {
|
||||
netLog.Error(user.Nick, url, response.StatusCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode
|
||||
img, format, decodeErr := image.Decode(bytes.NewReader(data))
|
||||
|
||||
if decodeErr != nil {
|
||||
netLog.Error(user.Nick, url, decodeErr)
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Avatar{
|
||||
User: user,
|
||||
Image: img,
|
||||
Data: data,
|
||||
Format: format,
|
||||
}
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/nfnt/resize"
|
||||
)
|
||||
|
||||
// AvatarOriginalFileOutput ...
|
||||
type AvatarOriginalFileOutput struct {
|
||||
Directory string
|
||||
Size int
|
||||
}
|
||||
|
||||
// SaveAvatar writes the original avatar to the file system.
|
||||
func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error {
|
||||
// Determine file extension
|
||||
extension := avatar.Extension()
|
||||
|
||||
if extension == "" {
|
||||
return errors.New("Unknown format: " + avatar.Format)
|
||||
}
|
||||
|
||||
// Resize if needed
|
||||
data := avatar.Data
|
||||
img := avatar.Image
|
||||
|
||||
if img.Bounds().Dx() > output.Size {
|
||||
img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3)
|
||||
buffer := new(bytes.Buffer)
|
||||
|
||||
var err error
|
||||
switch extension {
|
||||
case ".jpg":
|
||||
err = jpeg.Encode(buffer, img, nil)
|
||||
case ".png":
|
||||
err = png.Encode(buffer, img)
|
||||
case ".gif":
|
||||
err = gif.Encode(buffer, img, nil)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data = buffer.Bytes()
|
||||
}
|
||||
|
||||
// Set user avatar
|
||||
avatar.User.Avatar.Extension = extension
|
||||
|
||||
// Write to file
|
||||
fileName := output.Directory + avatar.User.ID + extension
|
||||
return ioutil.WriteFile(fileName, data, 0644)
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/animenotifier/arn"
|
||||
)
|
||||
|
||||
// AvatarSource describes a source where we can find avatar images for a user.
|
||||
type AvatarSource interface {
|
||||
GetAvatar(*arn.User) *Avatar
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/animenotifier/arn"
|
||||
"github.com/nfnt/resize"
|
||||
)
|
||||
|
||||
// AvatarWebPFileOutput ...
|
||||
type AvatarWebPFileOutput struct {
|
||||
Directory string
|
||||
Size int
|
||||
Quality float32
|
||||
}
|
||||
|
||||
// SaveAvatar writes the avatar in WebP format to the file system.
|
||||
func (output *AvatarWebPFileOutput) SaveAvatar(avatar *Avatar) error {
|
||||
img := avatar.Image
|
||||
|
||||
// Resize if needed
|
||||
if img.Bounds().Dx() > output.Size {
|
||||
img = resize.Resize(uint(output.Size), 0, img, resize.Lanczos3)
|
||||
}
|
||||
|
||||
// Write to file
|
||||
fileName := output.Directory + avatar.User.ID + ".webp"
|
||||
return arn.SaveWebP(img, fileName, output.Quality)
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
package main
|
||||
|
||||
// AvatarOutput represents a system that saves an avatar locally (in database or as a file, e.g.)
|
||||
type AvatarOutput interface {
|
||||
SaveAvatar(*Avatar) error
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/animenotifier/arn"
|
||||
)
|
||||
|
||||
var fileSystemLog = avatarLog.NewChannel("SSD")
|
||||
|
||||
// FileSystem loads avatar from the local filesystem.
|
||||
type FileSystem struct {
|
||||
Directory string
|
||||
}
|
||||
|
||||
// GetAvatar returns the local image for the user.
|
||||
func (source *FileSystem) GetAvatar(user *arn.User) *Avatar {
|
||||
fullPath := arn.FindFileWithExtension(user.ID, source.Directory, arn.OriginalImageExtensions)
|
||||
|
||||
if fullPath == "" {
|
||||
fileSystemLog.Error(user.Nick, "Not found on file system")
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadFile(fullPath)
|
||||
|
||||
if err != nil {
|
||||
fileSystemLog.Error(user.Nick, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decode
|
||||
img, format, decodeErr := image.Decode(bytes.NewReader(data))
|
||||
|
||||
if decodeErr != nil {
|
||||
fileSystemLog.Error(user.Nick, decodeErr)
|
||||
return nil
|
||||
}
|
||||
|
||||
return &Avatar{
|
||||
User: user,
|
||||
Image: img,
|
||||
Data: data,
|
||||
Format: format,
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/animenotifier/arn"
|
||||
gravatar "github.com/ungerik/go-gravatar"
|
||||
)
|
||||
|
||||
var gravatarLog = avatarLog.NewChannel("GRA")
|
||||
|
||||
// Gravatar - https://gravatar.com/
|
||||
type Gravatar struct {
|
||||
Rating string
|
||||
RequestLimiter *time.Ticker
|
||||
}
|
||||
|
||||
// GetAvatar returns the Gravatar image for a user (if available).
|
||||
func (source *Gravatar) GetAvatar(user *arn.User) *Avatar {
|
||||
// If the user has no Email registered we can't get a Gravatar.
|
||||
if user.Email == "" {
|
||||
gravatarLog.Error(user.Nick, "No Email")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build URL
|
||||
gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=" + source.Rating
|
||||
gravatarURL = strings.Replace(gravatarURL, "http://", "https://", 1)
|
||||
|
||||
// Wait for request limiter to allow us to send a request
|
||||
<-source.RequestLimiter.C
|
||||
|
||||
// Download
|
||||
return AvatarFromURL(gravatarURL, user)
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/animenotifier/arn"
|
||||
"github.com/parnurzeal/gorequest"
|
||||
)
|
||||
|
||||
var userIDRegex = regexp.MustCompile(`<user_id>(\d+)<\/user_id>`)
|
||||
var malLog = avatarLog.NewChannel("MAL")
|
||||
|
||||
// MyAnimeList - https://myanimelist.net/
|
||||
type MyAnimeList struct {
|
||||
RequestLimiter *time.Ticker
|
||||
}
|
||||
|
||||
// GetAvatar returns the Gravatar image for a user (if available).
|
||||
func (source *MyAnimeList) GetAvatar(user *arn.User) *Avatar {
|
||||
malNick := user.Accounts.MyAnimeList.Nick
|
||||
|
||||
// If the user has no username we can't get an avatar.
|
||||
if malNick == "" {
|
||||
malLog.Error(user.Nick, "No MAL nick")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download user info
|
||||
userInfoURL := "https://myanimelist.net/malappinfo.php?u=" + malNick
|
||||
response, xml, networkErr := gorequest.New().Get(userInfoURL).End()
|
||||
|
||||
if networkErr != nil {
|
||||
malLog.Error(user.Nick, userInfoURL, networkErr)
|
||||
return nil
|
||||
}
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
malLog.Error(user.Nick, userInfoURL, response.StatusCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build URL
|
||||
matches := userIDRegex.FindStringSubmatch(xml)
|
||||
|
||||
if matches == nil || len(matches) < 2 {
|
||||
malLog.Error(user.Nick, "Could not find user ID")
|
||||
return nil
|
||||
}
|
||||
|
||||
malID := matches[1]
|
||||
malAvatarURL := "https://myanimelist.cdn-dena.com/images/userimages/" + malID + ".jpg"
|
||||
|
||||
// Wait for request limiter to allow us to send a request
|
||||
<-source.RequestLimiter.C
|
||||
|
||||
// Download
|
||||
return AvatarFromURL(malAvatarURL, user)
|
||||
}
|
||||
@ -15,6 +15,9 @@ import (
|
||||
|
||||
"github.com/aerogo/log"
|
||||
"github.com/animenotifier/arn"
|
||||
"github.com/animenotifier/avatar"
|
||||
"github.com/animenotifier/avatar/outputs"
|
||||
"github.com/animenotifier/avatar/sources"
|
||||
"github.com/fatih/color"
|
||||
)
|
||||
|
||||
@ -22,8 +25,8 @@ const (
|
||||
webPQuality = 80
|
||||
)
|
||||
|
||||
var avatarSources []AvatarSource
|
||||
var avatarOutputs []AvatarOutput
|
||||
var avatarSources []avatar.Source
|
||||
var avatarOutputs []avatar.Output
|
||||
var avatarLog = log.New()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
@ -46,42 +49,42 @@ func main() {
|
||||
defer avatarLog.Flush()
|
||||
|
||||
// Define the avatar sources
|
||||
avatarSources = []AvatarSource{
|
||||
&Gravatar{
|
||||
avatarSources = []avatar.Source{
|
||||
&sources.Gravatar{
|
||||
Rating: "pg",
|
||||
RequestLimiter: time.NewTicker(100 * time.Millisecond),
|
||||
},
|
||||
&sources.MyAnimeList{
|
||||
RequestLimiter: time.NewTicker(250 * time.Millisecond),
|
||||
},
|
||||
&MyAnimeList{
|
||||
RequestLimiter: time.NewTicker(250 * time.Millisecond),
|
||||
},
|
||||
&FileSystem{
|
||||
&sources.FileSystem{
|
||||
Directory: "images/avatars/large/",
|
||||
},
|
||||
}
|
||||
|
||||
// Define the avatar outputs
|
||||
avatarOutputs = []AvatarOutput{
|
||||
avatarOutputs = []avatar.Output{
|
||||
// Original - Large
|
||||
&AvatarOriginalFileOutput{
|
||||
&outputs.OriginalFile{
|
||||
Directory: "images/avatars/large/",
|
||||
Size: arn.AvatarMaxSize,
|
||||
},
|
||||
|
||||
// Original - Small
|
||||
&AvatarOriginalFileOutput{
|
||||
&outputs.OriginalFile{
|
||||
Directory: "images/avatars/small/",
|
||||
Size: arn.AvatarSmallSize,
|
||||
},
|
||||
|
||||
// WebP - Large
|
||||
&AvatarWebPFileOutput{
|
||||
&outputs.WebPFile{
|
||||
Directory: "images/avatars/large/",
|
||||
Size: arn.AvatarMaxSize,
|
||||
Quality: webPQuality,
|
||||
},
|
||||
|
||||
// WebP - Small
|
||||
&AvatarWebPFileOutput{
|
||||
&outputs.WebPFile{
|
||||
Directory: "images/avatars/small/",
|
||||
Size: arn.AvatarSmallSize,
|
||||
Quality: webPQuality,
|
||||
@ -126,7 +129,12 @@ func Work(user *arn.User) {
|
||||
user.Avatar.Extension = ""
|
||||
|
||||
for _, source := range avatarSources {
|
||||
avatar := source.GetAvatar(user)
|
||||
avatar, err := source.GetAvatar(user)
|
||||
|
||||
if err != nil {
|
||||
avatarLog.Error(err)
|
||||
continue
|
||||
}
|
||||
|
||||
if avatar == nil {
|
||||
// fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick)
|
||||
|
||||
@ -23,8 +23,8 @@ var colorPool = []*color.Color{
|
||||
}
|
||||
|
||||
var jobs = map[string]time.Duration{
|
||||
"active-users": 1 * time.Minute,
|
||||
"forum-activity": 1 * time.Minute,
|
||||
"active-users": 5 * time.Minute,
|
||||
"anime-ratings": 10 * time.Minute,
|
||||
"airing-anime": 10 * time.Minute,
|
||||
"statistics": 15 * time.Minute,
|
||||
|
||||
@ -117,10 +117,19 @@ component Settings(user *arn.User)
|
||||
select.widget-element.action(id="Avatar.Source", data-field="Avatar.Source", value=user.Settings().Avatar.Source, data-action="save", data-trigger="change")
|
||||
option(value="") Automatic
|
||||
option(value="Gravatar") Gravatar
|
||||
option(value="URL") Link
|
||||
option(value="FileSystem") Upload
|
||||
|
||||
if "Gravatar" == "Gravatar"
|
||||
if user.Settings().Avatar.Source == "Gravatar" || (user.Settings().Avatar.Source == "" && user.Avatar.Source == "Gravatar")
|
||||
.profile-image-container.avatar-preview
|
||||
img.profile-image(src=user.Gravatar(), alt="Gravatar")
|
||||
|
||||
if user.Settings().Avatar.Source == "URL"
|
||||
InputText("Avatar.SourceURL", user.Settings().Avatar.SourceURL, "Link", "Post the link to the image here")
|
||||
|
||||
if user.Settings().Avatar.SourceURL != ""
|
||||
.profile-image-container.avatar-preview
|
||||
img.profile-image(src=strings.Replace(user.Settings().Avatar.SourceURL, "http://", "https://", 1), alt="Avatar preview")
|
||||
|
||||
//- .widget.mountable(data-api="/api/settings/" + user.ID)
|
||||
//- h3.widget-title
|
||||
|
||||
Loading…
Reference in New Issue
Block a user