WebP bridge is working again

jobs/avatars/Avatar.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"image"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/animenotifier/arn"
@@ -21,6 +22,20 @@ type Avatar struct {
 	Format string
 }
 
+// Extension ...
+func (avatar *Avatar) Extension() string {
+	switch avatar.Format {
+	case "jpg", "jpeg":
+		return ".jpg"
+	case "png":
+		return ".png"
+	case "gif":
+		return ".gif"
+	default:
+		return ""
+	}
+}
+
 // String returns a text representation of the format, width and height.
 func (avatar *Avatar) String() string {
 	return fmt.Sprint(avatar.Format, " | ", avatar.Image.Bounds().Dx(), "x", avatar.Image.Bounds().Dy())
@@ -29,17 +44,23 @@ func (avatar *Avatar) String() string {
 // AvatarFromURL downloads and decodes the image from an URL and creates an Avatar.
 func AvatarFromURL(url string, user *arn.User) *Avatar {
 	// Download
-	response, data, networkErr := gorequest.New().Get(url).EndBytes()
-
-	// Retry after 5 seconds if service unavailable
-	if response.StatusCode == http.StatusServiceUnavailable {
-		time.Sleep(5 * time.Second)
-		response, data, networkErr = gorequest.New().Get(url).EndBytes()
-	}
+	response, data, networkErrs := gorequest.New().Get(url).EndBytes()
 
 	// Network errors
-	if networkErr != nil {
-		netLog.Error(user.Nick, url, networkErr)
+	if len(networkErrs) > 0 {
+		netLog.Error(user.Nick, url, networkErrs[0])
+		return nil
+	}
+
+	// Retry HTTP only version after 5 seconds if service unavailable
+	if response == nil || response.StatusCode == http.StatusServiceUnavailable {
+		time.Sleep(5 * time.Second)
+		response, data, networkErrs = gorequest.New().Get(strings.Replace(url, "https://", "http://", 1)).EndBytes()
+	}
+
+	// Network errors on 2nd try
+	if len(networkErrs) > 0 {
+		netLog.Error(user.Nick, url, networkErrs[0])
 		return nil
 	}
 

jobs/avatars/AvatarOriginalFileOutput.go

@@ -20,16 +20,9 @@ type AvatarOriginalFileOutput struct {
 // SaveAvatar writes the original avatar to the file system.
 func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error {
 	// Determine file extension
-	extension := ""
+	extension := avatar.Extension()
 
-	switch avatar.Format {
-	case "jpg", "jpeg":
-		extension = ".jpg"
-	case "png":
-		extension = ".png"
-	case "gif":
-		extension = ".gif"
-	default:
+	if extension == "" {
 		return errors.New("Unknown format: " + avatar.Format)
 	}
 
@@ -58,6 +51,9 @@ func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error {
 		data = buffer.Bytes()
 	}
 
+	// Set user avatar
+	avatar.User.AvatarExtension = extension
+
 	// Write to file
 	fileName := output.Directory + avatar.User.ID + extension
 	return ioutil.WriteFile(fileName, data, 0644)

jobs/avatars/Gravatar.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/animenotifier/arn"
@@ -26,6 +27,7 @@ func (source *Gravatar) GetAvatar(user *arn.User) *Avatar {
 
 	// Build URL
 	gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=" + source.Rating
+	gravatarURL = strings.Replace(gravatarURL, "http://", "https://", 1)
 
 	// Wait for request limiter to allow us to send a request
 	<-source.RequestLimiter.C

jobs/avatars/avatars.go

@@ -58,26 +58,26 @@ func main() {
 	avatarOutputs = []AvatarOutput{
 		// Original - Large
 		&AvatarOriginalFileOutput{
-			Directory: "images/avatars/large/original/",
+			Directory: "images/avatars/large/",
 			Size:      arn.AvatarMaxSize,
 		},
 
 		// Original - Small
 		&AvatarOriginalFileOutput{
-			Directory: "images/avatars/small/original/",
+			Directory: "images/avatars/small/",
 			Size:      arn.AvatarSmallSize,
 		},
 
 		// WebP - Large
 		&AvatarWebPFileOutput{
-			Directory: "images/avatars/large/webp/",
+			Directory: "images/avatars/large/",
 			Size:      arn.AvatarMaxSize,
 			Quality:   webPQuality,
 		},
 
 		// WebP - Small
 		&AvatarWebPFileOutput{
-			Directory: "images/avatars/small/webp/",
+			Directory: "images/avatars/small/",
 			Size:      arn.AvatarSmallSize,
 			Quality:   webPQuality,
 		},
@@ -87,20 +87,12 @@ func main() {
 		return
 	}
 
-	// Stream of all users
-	users, _ := arn.FilterUsers(func(user *arn.User) bool {
-		return true
-	})
-
-	// Log user count
-	println(len(users), "users")
-
 	// Worker queue
-	usersQueue := make(chan *arn.User)
+	usersQueue := make(chan *arn.User, 512)
 	StartWorkers(usersQueue, Work)
 
 	// We'll send each user to one of the worker threads
-	for _, user := range users {
+	for user := range arn.MustStreamUsers() {
 		usersQueue <- user
 	}
 
@@ -120,7 +112,7 @@ func StartWorkers(queue chan *arn.User, work func(*arn.User)) {
 
 // Work handles a single user.
 func Work(user *arn.User) {
-	user.Avatar = ""
+	user.AvatarExtension = ""
 
 	for _, source := range avatarSources {
 		avatar := source.GetAvatar(user)
@@ -139,10 +131,19 @@ func Work(user *arn.User) {
 		}
 
 		fmt.Println(color.GreenString("✔"), reflect.TypeOf(source).Elem().Name(), "|", user.Nick, "|", avatar)
-		user.Avatar = "/+" + user.Nick + "/avatar"
 		break
 	}
 
+	// Since this a very long running job, refresh user data before saving it.
+	avatarExt := user.AvatarExtension
+	user, err := arn.GetUser(user.ID)
+
+	if err != nil {
+		avatarLog.Error("Can't refresh user info:", user.ID, user.Nick)
+		return
+	}
+
 	// Save avatar data
+	user.AvatarExtension = avatarExt
 	user.Save()
 }