From c729d9d3bab51bfdea62ca7214d9cc49544fcf74 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sat, 8 Jul 2017 15:40:13 +0200 Subject: [PATCH] WebP bridge is working again --- assets.go | 39 ++---------------------- jobs/active-users/active-users.go | 2 +- jobs/avatars/Avatar.go | 39 ++++++++++++++++++------ jobs/avatars/AvatarOriginalFileOutput.go | 14 +++------ jobs/avatars/Gravatar.go | 2 ++ jobs/avatars/avatars.go | 33 ++++++++++---------- mixins/Avatar.pixy | 2 +- mixins/ProfileImage.pixy | 2 +- scripts/AnimeNotifier.ts | 14 +++++++-- scripts/Utils.ts | 14 ++++++++- 10 files changed, 84 insertions(+), 77 deletions(-) diff --git a/assets.go b/assets.go index 1c312697..8f3fc1e1 100644 --- a/assets.go +++ b/assets.go @@ -1,12 +1,9 @@ package main import ( - "errors" - "net/http" "strings" "github.com/aerogo/aero" - "github.com/animenotifier/arn" "github.com/animenotifier/notify.moe/components/js" ) @@ -53,44 +50,12 @@ func init() { // Avatars app.Get("/images/avatars/large/:file", func(ctx *aero.Context) string { - file := strings.TrimSuffix(ctx.Get("file"), ".webp") - - if ctx.CanUseWebP() { - return ctx.File("images/avatars/large/webp/" + file + ".webp") - } - - original := arn.FindFileWithExtension( - file, - "images/avatars/large/original/", - arn.OriginalImageExtensions, - ) - - if original == "" { - return ctx.Error(http.StatusNotFound, "Avatar not found", errors.New("Image not found: "+file)) - } - - return ctx.File(original) + return ctx.File("images/avatars/large/" + ctx.Get("file")) }) // Avatars app.Get("/images/avatars/small/:file", func(ctx *aero.Context) string { - file := strings.TrimSuffix(ctx.Get("file"), ".webp") - - if ctx.CanUseWebP() { - return ctx.File("images/avatars/small/webp/" + file + ".webp") - } - - original := arn.FindFileWithExtension( - file, - "images/avatars/small/original/", - arn.OriginalImageExtensions, - ) - - if original == "" { - return ctx.Error(http.StatusNotFound, "Avatar not found", errors.New("Image not found: "+file)) - } - - return ctx.File(original) + return ctx.File("images/avatars/large/" + ctx.Get("file")) }) // Elements diff --git a/jobs/active-users/active-users.go b/jobs/active-users/active-users.go index 8c520285..ae99e4b4 100644 --- a/jobs/active-users/active-users.go +++ b/jobs/active-users/active-users.go @@ -15,7 +15,7 @@ func main() { // Filter out active users with an avatar users, err := arn.FilterUsers(func(user *arn.User) bool { - return user.IsActive() && user.Avatar != "" + return user.IsActive() && user.AvatarExtension != "" }) if err != nil { diff --git a/jobs/avatars/Avatar.go b/jobs/avatars/Avatar.go index 25c5b21a..f1748c37 100644 --- a/jobs/avatars/Avatar.go +++ b/jobs/avatars/Avatar.go @@ -5,6 +5,7 @@ import ( "fmt" "image" "net/http" + "strings" "time" "github.com/animenotifier/arn" @@ -21,6 +22,20 @@ type Avatar struct { Format string } +// Extension ... +func (avatar *Avatar) Extension() string { + switch avatar.Format { + case "jpg", "jpeg": + return ".jpg" + case "png": + return ".png" + case "gif": + return ".gif" + default: + return "" + } +} + // String returns a text representation of the format, width and height. func (avatar *Avatar) String() string { return fmt.Sprint(avatar.Format, " | ", avatar.Image.Bounds().Dx(), "x", avatar.Image.Bounds().Dy()) @@ -29,17 +44,23 @@ func (avatar *Avatar) String() string { // AvatarFromURL downloads and decodes the image from an URL and creates an Avatar. func AvatarFromURL(url string, user *arn.User) *Avatar { // Download - response, data, networkErr := gorequest.New().Get(url).EndBytes() - - // Retry after 5 seconds if service unavailable - if response.StatusCode == http.StatusServiceUnavailable { - time.Sleep(5 * time.Second) - response, data, networkErr = gorequest.New().Get(url).EndBytes() - } + response, data, networkErrs := gorequest.New().Get(url).EndBytes() // Network errors - if networkErr != nil { - netLog.Error(user.Nick, url, networkErr) + if len(networkErrs) > 0 { + netLog.Error(user.Nick, url, networkErrs[0]) + return nil + } + + // Retry HTTP only version after 5 seconds if service unavailable + if response == nil || response.StatusCode == http.StatusServiceUnavailable { + time.Sleep(5 * time.Second) + response, data, networkErrs = gorequest.New().Get(strings.Replace(url, "https://", "http://", 1)).EndBytes() + } + + // Network errors on 2nd try + if len(networkErrs) > 0 { + netLog.Error(user.Nick, url, networkErrs[0]) return nil } diff --git a/jobs/avatars/AvatarOriginalFileOutput.go b/jobs/avatars/AvatarOriginalFileOutput.go index 8bb6544c..5232ef6b 100644 --- a/jobs/avatars/AvatarOriginalFileOutput.go +++ b/jobs/avatars/AvatarOriginalFileOutput.go @@ -20,16 +20,9 @@ type AvatarOriginalFileOutput struct { // SaveAvatar writes the original avatar to the file system. func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error { // Determine file extension - extension := "" + extension := avatar.Extension() - switch avatar.Format { - case "jpg", "jpeg": - extension = ".jpg" - case "png": - extension = ".png" - case "gif": - extension = ".gif" - default: + if extension == "" { return errors.New("Unknown format: " + avatar.Format) } @@ -58,6 +51,9 @@ func (output *AvatarOriginalFileOutput) SaveAvatar(avatar *Avatar) error { data = buffer.Bytes() } + // Set user avatar + avatar.User.AvatarExtension = extension + // Write to file fileName := output.Directory + avatar.User.ID + extension return ioutil.WriteFile(fileName, data, 0644) diff --git a/jobs/avatars/Gravatar.go b/jobs/avatars/Gravatar.go index 2ec685ce..731d863f 100644 --- a/jobs/avatars/Gravatar.go +++ b/jobs/avatars/Gravatar.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "time" "github.com/animenotifier/arn" @@ -26,6 +27,7 @@ func (source *Gravatar) GetAvatar(user *arn.User) *Avatar { // Build URL gravatarURL := gravatar.Url(user.Email) + "?s=" + fmt.Sprint(arn.AvatarMaxSize) + "&d=404&r=" + source.Rating + gravatarURL = strings.Replace(gravatarURL, "http://", "https://", 1) // Wait for request limiter to allow us to send a request <-source.RequestLimiter.C diff --git a/jobs/avatars/avatars.go b/jobs/avatars/avatars.go index c6a76bbc..6f545413 100644 --- a/jobs/avatars/avatars.go +++ b/jobs/avatars/avatars.go @@ -58,26 +58,26 @@ func main() { avatarOutputs = []AvatarOutput{ // Original - Large &AvatarOriginalFileOutput{ - Directory: "images/avatars/large/original/", + Directory: "images/avatars/large/", Size: arn.AvatarMaxSize, }, // Original - Small &AvatarOriginalFileOutput{ - Directory: "images/avatars/small/original/", + Directory: "images/avatars/small/", Size: arn.AvatarSmallSize, }, // WebP - Large &AvatarWebPFileOutput{ - Directory: "images/avatars/large/webp/", + Directory: "images/avatars/large/", Size: arn.AvatarMaxSize, Quality: webPQuality, }, // WebP - Small &AvatarWebPFileOutput{ - Directory: "images/avatars/small/webp/", + Directory: "images/avatars/small/", Size: arn.AvatarSmallSize, Quality: webPQuality, }, @@ -87,20 +87,12 @@ func main() { return } - // Stream of all users - users, _ := arn.FilterUsers(func(user *arn.User) bool { - return true - }) - - // Log user count - println(len(users), "users") - // Worker queue - usersQueue := make(chan *arn.User) + usersQueue := make(chan *arn.User, 512) StartWorkers(usersQueue, Work) // We'll send each user to one of the worker threads - for _, user := range users { + for user := range arn.MustStreamUsers() { usersQueue <- user } @@ -120,7 +112,7 @@ func StartWorkers(queue chan *arn.User, work func(*arn.User)) { // Work handles a single user. func Work(user *arn.User) { - user.Avatar = "" + user.AvatarExtension = "" for _, source := range avatarSources { avatar := source.GetAvatar(user) @@ -139,10 +131,19 @@ func Work(user *arn.User) { } fmt.Println(color.GreenString("✔"), reflect.TypeOf(source).Elem().Name(), "|", user.Nick, "|", avatar) - user.Avatar = "/+" + user.Nick + "/avatar" break } + // Since this a very long running job, refresh user data before saving it. + avatarExt := user.AvatarExtension + user, err := arn.GetUser(user.ID) + + if err != nil { + avatarLog.Error("Can't refresh user info:", user.ID, user.Nick) + return + } + // Save avatar data + user.AvatarExtension = avatarExt user.Save() } diff --git a/mixins/Avatar.pixy b/mixins/Avatar.pixy index b3f9a103..74bfd4d4 100644 --- a/mixins/Avatar.pixy +++ b/mixins/Avatar.pixy @@ -4,7 +4,7 @@ component Avatar(user *arn.User) component AvatarNoLink(user *arn.User) if user.HasAvatar() - img.user-image.lazy(data-src=user.SmallAvatar(), alt=user.Nick) + img.user-image.lazy(data-src=user.SmallAvatar(), data-webp="true", alt=user.Nick) else SVGAvatar diff --git a/mixins/ProfileImage.pixy b/mixins/ProfileImage.pixy index dc06f415..9b7a7735 100644 --- a/mixins/ProfileImage.pixy +++ b/mixins/ProfileImage.pixy @@ -1,6 +1,6 @@ component ProfileImage(user *arn.User) if user.HasAvatar() - img.profile-image(src=user.LargeAvatar(), alt="Profile image") + img.profile-image.lazy(data-src=user.LargeAvatar(), data-webp="true", alt="Profile image") else svg.profile-image(viewBox="0 0 50 50", alt="Profile image") circle.head(cx="25", cy="19", r="10") diff --git a/scripts/AnimeNotifier.ts b/scripts/AnimeNotifier.ts index 3693693f..c8e32f73 100644 --- a/scripts/AnimeNotifier.ts +++ b/scripts/AnimeNotifier.ts @@ -1,7 +1,7 @@ import { Application } from "./Application" import { Diff } from "./Diff" import { displayAiringDate, displayDate } from "./DateView" -import { findAll, delay } from "./Utils" +import { findAll, delay, canUseWebP } from "./Utils" import { MutationQueue } from "./MutationQueue" import * as actions from "./Actions" @@ -9,6 +9,7 @@ export class AnimeNotifier { app: Application user: HTMLElement title: string + webpEnabled: boolean visibilityObserver: IntersectionObserver imageFound: MutationQueue @@ -80,6 +81,9 @@ export class AnimeNotifier { document.documentElement.classList.add("osx") } + // Check for WebP support + this.webpEnabled = canUseWebP() + // Initiate the elements we need this.user = this.app.find("user") this.app.content = this.app.find("content") @@ -207,7 +211,13 @@ export class AnimeNotifier { lazyLoadImage(img: HTMLImageElement) { // Once the image becomes visible, load it img["became visible"] = () => { - img.src = img.dataset.src + // Replace URL with WebP if supported + if(this.webpEnabled && img.dataset.webp) { + let dot = img.dataset.src.lastIndexOf(".") + img.src = img.dataset.src.substring(0, dot) + ".webp" + } else { + img.src = img.dataset.src + } if(img.naturalWidth === 0) { img.onload = () => { diff --git a/scripts/Utils.ts b/scripts/Utils.ts index b349dc84..17ad8d01 100644 --- a/scripts/Utils.ts +++ b/scripts/Utils.ts @@ -11,5 +11,17 @@ export function delay(millis: number, value?: T): Promise { } export function plural(count: number, singular: string): string { - return (count === 1 || count === -1) ? (count + ' ' + singular) : (count + ' ' + singular + 's') + return (count === 1 || count === -1) ? (count + " " + singular) : (count + " " + singular + "s") +} + +export function canUseWebP(): boolean { + let canvas = document.createElement("canvas") + + if(!!(canvas.getContext && canvas.getContext("2d"))) { + // WebP representation possible + return canvas.toDataURL("image/webp").indexOf("data:image/webp") === 0 + } else { + // In very old browsers (IE 8) canvas is not supported + return false + } } \ No newline at end of file