Improved background jobs

This commit is contained in:
Eduard Urbach 2017-06-13 14:47:17 +02:00
parent bdb8d983d2
commit e07a865d9c
13 changed files with 144 additions and 151 deletions

View File

@ -1,9 +1,14 @@
package main
import "image"
import (
"image"
"github.com/animenotifier/arn"
)
// Avatar represents a single image and the name of the format.
type Avatar struct {
User *arn.User
Image image.Image
Data []byte
Format string

View File

@ -0,0 +1,65 @@
package main
import (
"bytes"
"errors"
"image/gif"
"image/jpeg"
"image/png"
"io/ioutil"
"github.com/animenotifier/arn"
"github.com/nfnt/resize"
)
// AvatarOriginalFileOutput ...
type AvatarOriginalFileOutput struct {
Directory string
Size int
}
// SaveAvatar writes the original avatar to the file system.
func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error {
// Determine file extension
extension := ""
switch avatar.Format {
case "jpg", "jpeg":
extension = ".jpg"
case "png":
extension = ".png"
case "gif":
extension = ".gif"
default:
return errors.New("Unknown format: " + avatar.Format)
}
// Resize if needed
data := avatar.Data
img := avatar.Image
if img.Bounds().Dx() != output.Size {
img = resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3)
buffer := new(bytes.Buffer)
var err error
switch extension {
case ".jpg":
err = jpeg.Encode(buffer, img, nil)
case ".png":
err = png.Encode(buffer, img)
case ".gif":
err = gif.Encode(buffer, img, nil)
}
if err != nil {
return err
}
data = buffer.Bytes()
}
// Write to file
fileName := output.Directory + avatar.User.ID + extension
return ioutil.WriteFile(fileName, data, 0644)
}

View File

@ -0,0 +1,27 @@
package main
import (
"github.com/animenotifier/arn"
"github.com/nfnt/resize"
)
// AvatarWebPFileOutput ...
type AvatarWebPFileOutput struct {
Directory string
Size int
Quality float32
}
// SaveAvatar writes the avatar in WebP format to the file system.
func (output *AvatarWebPFileOutput) SaveAvatar(avatar *Avatar) error {
img := avatar.Image
// Resize if needed
if img.Bounds().Dx() != output.Size {
img = resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3)
}
// Write to file
fileName := output.Directory + avatar.User.ID + ".webp"
return arn.SaveWebP(img, fileName, output.Quality)
}

View File

@ -1,6 +1,6 @@
package main
// AvatarWriter represents a system that saves an avatar locally (in database or as a file, e.g.)
type AvatarWriter interface {
// AvatarOutput represents a system that saves an avatar locally (in database or as a file, e.g.)
type AvatarOutput interface {
SaveAvatar(*Avatar) error
}

View File

@ -1 +0,0 @@
package main

View File

@ -38,6 +38,7 @@ func (source *Gravatar) GetAvatar(user *arn.User) *Avatar {
}
return &Avatar{
User: user,
Image: img,
Data: data,
Format: format,

View File

@ -1,79 +0,0 @@
package main
import (
"fmt"
"io/ioutil"
"github.com/animenotifier/arn"
"github.com/fatih/color"
"github.com/parnurzeal/gorequest"
gravatar "github.com/ungerik/go-gravatar"
)
func findOriginalAvatar(user *arn.User) string {
return arn.FindFileWithExtension(
user.ID,
avatarsDirectoryOriginal,
[]string{
".jpg",
".png",
".gif",
},
)
}
func downloadAvatar(user *arn.User) bool {
if user.Email == "" {
return false
}
directory := avatarsDirectoryOriginal
fileName := directory + user.ID
// Build URL
url := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=pg"
// Skip existing avatars
if findOriginalAvatar(user) != "" {
color.Yellow(user.Nick)
return true
}
// Download
response, data, err := gorequest.New().Get(url).EndBytes()
if err != nil {
color.Red(user.Nick)
return false
}
contentType := response.Header.Get("content-type")
if response.StatusCode != 200 {
color.Red(user.Nick)
return false
}
// Determine file extension
extension := ""
switch contentType {
case "image/jpeg":
extension = ".jpg"
case "image/png":
extension = ".png"
case "image/gif":
extension = ".gif"
case "image/webp":
extension = ".webp"
}
fileName += extension
// Write to disk
ioutil.WriteFile(fileName, data, 0644)
color.Green(user.Nick)
return true
}

View File

@ -3,21 +3,25 @@ package main
import (
"fmt"
"os"
"reflect"
"runtime"
"time"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"github.com/animenotifier/arn"
"github.com/fatih/color"
)
const (
networkRateLimit = 100 * time.Millisecond
avatarsDirectoryOriginal = "images/avatars/large/original/"
avatarsDirectoryWebP = "images/avatars/large/webp/"
webPQuality = 80
)
var avatarSources []AvatarSource
var avatarWriters []AvatarWriter
var avatarOutputs []AvatarOutput
// Main
func main() {
@ -29,6 +33,28 @@ func main() {
&Gravatar{},
}
// Define the avatar outputs
avatarOutputs = []AvatarOutput{
&AvatarOriginalFileOutput{
Directory: "images/avatars/large/original/",
Size: arn.AvatarMaxSize,
},
&AvatarOriginalFileOutput{
Directory: "images/avatars/small/original/",
Size: arn.AvatarSmallSize,
},
&AvatarWebPFileOutput{
Directory: "images/avatars/large/webp/",
Size: arn.AvatarMaxSize,
Quality: webPQuality,
},
&AvatarWebPFileOutput{
Directory: "images/avatars/small/webp/",
Size: arn.AvatarSmallSize,
Quality: webPQuality,
},
}
// Stream of all users
users, _ := arn.AllUsers()
@ -58,29 +84,29 @@ func StartWorkers(queue chan *arn.User, rateLimit time.Duration, work func(*arn.
// Work handles a single user.
func Work(user *arn.User) {
user.Avatar = ""
for _, source := range avatarSources {
avatar := source.GetAvatar(user)
if avatar == nil {
fmt.Println(color.RedString("✘"), user.Nick)
fmt.Println(color.RedString("✘"), reflect.TypeOf(source).Elem().Name(), user.Nick)
continue
}
for _, writer := range avatarOutputs {
err := writer.SaveAvatar(avatar)
if err != nil {
color.Red(err.Error())
}
}
fmt.Println(color.GreenString("✔"), user.Nick, "|", avatar.Format, avatar.Image.Bounds().Dx(), avatar.Image.Bounds().Dy())
for _, writer := range avatarWriters {
writer.SaveAvatar(avatar)
user.Avatar = "/+" + user.Nick + "/avatar"
break
}
return
}
// if downloadAvatar(user) {
// makeWebPAvatar(user)
// user.Avatar = "/+" + user.Nick + "/avatar"
// } else {
// user.Avatar = ""
// }
// user.Save()
// Save avatar data
user.Save()
}

View File

@ -1,51 +0,0 @@
package main
import (
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"strings"
"github.com/animenotifier/arn"
"github.com/fatih/color"
"github.com/nfnt/resize"
)
func makeWebPAvatar(user *arn.User) {
baseName := findOriginalAvatar(user)
if baseName == "" {
return
}
original := avatarsDirectoryOriginal + baseName
outFile := avatarsDirectoryWebP + user.ID + ".webp"
err := avatarToWebP(original, outFile, 80)
if err != nil {
color.Red(user.Nick + " [WebP]")
} else {
color.Green(user.Nick + " [WebP]")
}
}
func avatarToWebP(in string, out string, quality float32) error {
img, _, loadErr := arn.LoadImage(in)
if loadErr != nil {
return loadErr
}
// Big avatar
saveErr := arn.SaveWebP(img, out, quality)
if saveErr != nil {
return saveErr
}
// Small avatar
smallImg := resize.Resize(arn.AvatarSmallSize, 0, img, resize.Lanczos3)
saveErr = arn.SaveWebP(smallImg, strings.Replace(out, "large/", "small/", 1), quality)
return saveErr
}