From bc4e67f71846e2493ba091d462da7654e1590b78 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Wed, 28 Aug 2019 10:07:50 +0900 Subject: [PATCH] Use a CDN for faster AMV delivery --- arn/AMV.go | 72 ++++++++++++++-------- arn/AMVAPI.go | 6 +- arn/APIKeys.go | 3 + arn/Spaces.go | 30 +++++++++ mixins/AMV.pixy | 2 +- pages/episode/episode.go | 26 +------- pages/episode/subtitles.go | 2 +- pages/upload/amv-file.go | 13 ++-- patches/update-amv-info/update-amv-info.go | 27 -------- 9 files changed, 91 insertions(+), 90 deletions(-) create mode 100644 arn/Spaces.go delete mode 100644 patches/update-amv-info/update-amv-info.go diff --git a/arn/AMV.go b/arn/AMV.go index 7829e46b..fbcef12c 100644 --- a/arn/AMV.go +++ b/arn/AMV.go @@ -3,6 +3,7 @@ package arn import ( "errors" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -10,6 +11,7 @@ import ( "github.com/aerogo/nano" "github.com/animenotifier/notify.moe/arn/video" + "github.com/minio/minio-go/v6" ) // AMV is an anime music video. @@ -36,16 +38,37 @@ func (amv *AMV) Link() string { return "/amv/" + amv.ID } +// VideoLink returns the permalink for the video file. +func (amv *AMV) VideoLink() string { + domain := "arn.sfo2.cdn" + + if amv.IsDraft { + domain = "arn.sfo2" + } + + return fmt.Sprintf("https://%s.digitaloceanspaces.com/videos/amvs/%s", domain, amv.File) +} + // TitleByUser returns the preferred title for the given user. func (amv *AMV) TitleByUser(user *User) string { return amv.Title.ByUser(user) } -// SetVideoBytes sets the bytes for the video file. -func (amv *AMV) SetVideoBytes(data []byte) error { +// SetVideoReader sets the bytes for the video file by reading them from the reader. +func (amv *AMV) SetVideoReader(reader io.Reader) error { fileName := amv.ID + ".webm" - filePath := path.Join(Root, "videos", "amvs", fileName) - err := ioutil.WriteFile(filePath, data, 0644) + pattern := amv.ID + ".*.webm" + file, err := ioutil.TempFile("", pattern) + + if err != nil { + return err + } + + filePath := file.Name() + defer os.Remove(filePath) + + // Write file contents + _, err = io.Copy(file, reader) if err != nil { return err @@ -53,6 +76,7 @@ func (amv *AMV) SetVideoBytes(data []byte) error { // Run mkclean optimizedFile := filePath + ".optimized" + defer os.Remove(optimizedFile) cmd := exec.Command( "mkclean", @@ -79,37 +103,35 @@ func (amv *AMV) SetVideoBytes(data []byte) error { return err } - // Now delete the original file and replace it with the optimized file - err = os.Remove(filePath) - - if err != nil { - return err - } - - err = os.Rename(optimizedFile, filePath) - - if err != nil { - return err - } - // Refresh video file info - amv.File = fileName - return amv.RefreshInfo() -} + info, err := video.GetInfo(optimizedFile) -// RefreshInfo refreshes the information about the video file. -func (amv *AMV) RefreshInfo() error { - if amv.File == "" { - return fmt.Errorf("Video file has not been uploaded yet for AMV %s", amv.ID) + if err != nil { + return err } - info, err := video.GetInfo(path.Join(Root, "videos", "amvs", amv.File)) + // Is our storage server available? + if Spaces == nil { + return errors.New("File storage client has not been initialized") + } + + // Make sure the file is public + userMetaData := map[string]string{ + "x-amz-acl": "public-read", + } + + // Upload the file to our storage server + _, err = Spaces.FPutObject("arn", fmt.Sprintf("videos/amvs/%s.webm", amv.ID), optimizedFile, minio.PutObjectOptions{ + ContentType: "video/webm", + UserMetadata: userMetaData, + }) if err != nil { return err } amv.Info = *info + amv.File = fileName return nil } diff --git a/arn/AMVAPI.go b/arn/AMVAPI.go index 63e6f0c2..a6294d98 100644 --- a/arn/AMVAPI.go +++ b/arn/AMVAPI.go @@ -3,8 +3,6 @@ package arn import ( "errors" "fmt" - "os" - "path" "reflect" "github.com/aerogo/aero" @@ -104,8 +102,8 @@ func (amv *AMV) Delete() error { } // Remove file - if amv.File != "" { - err := os.Remove(path.Join(Root, "videos", "amvs", amv.File)) + if amv.File != "" && Spaces != nil { + err := Spaces.RemoveObject("arn", fmt.Sprintf("videos/amvs/%s", amv.File)) if err != nil { return err diff --git a/arn/APIKeys.go b/arn/APIKeys.go index 57334167..af75f5b4 100644 --- a/arn/APIKeys.go +++ b/arn/APIKeys.go @@ -117,4 +117,7 @@ func init() { // Set Anilist API keys anilist.APIKeyID = APIKeys.AniList.ID anilist.APIKeySecret = APIKeys.AniList.Secret + + // Initialize file storage + initSpaces() } diff --git a/arn/Spaces.go b/arn/Spaces.go new file mode 100644 index 00000000..9f1e2dc7 --- /dev/null +++ b/arn/Spaces.go @@ -0,0 +1,30 @@ +package arn + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +// Spaces represents our file storage server. +var Spaces *minio.Client + +// initSpaces starts our file storage client. +func initSpaces() { + if APIKeys.S3.ID == "" || APIKeys.S3.Secret == "" { + return + } + + go func() { + var err error + endpoint := "sfo2.digitaloceanspaces.com" + ssl := true + + // Initiate a client using DigitalOcean Spaces. + Spaces, err = minio.New(endpoint, APIKeys.S3.ID, APIKeys.S3.Secret, ssl) + + if err != nil { + log.Fatal(err) + } + }() +} diff --git a/mixins/AMV.pixy b/mixins/AMV.pixy index fb176a74..27052f8e 100644 --- a/mixins/AMV.pixy +++ b/mixins/AMV.pixy @@ -11,7 +11,7 @@ component AMVMini(amv *arn.AMV, user *arn.User) component AMVVideo(amv *arn.AMV) .video-container(id=amv.ID, data-api="/api/amv/" + amv.ID) video.video.lazy.action(data-action="toggleFullscreen", data-trigger="dblclick", data-id=amv.ID) - source(data-src="https://notify.moe/videos/amvs/" + amv.File, data-type="video/webm") + source(data-src=amv.VideoLink(), data-type="video/webm") //- button.media-play-button //- RawIcon("play") diff --git a/pages/episode/episode.go b/pages/episode/episode.go index ee6b7d2d..bf605f60 100644 --- a/pages/episode/episode.go +++ b/pages/episode/episode.go @@ -2,7 +2,6 @@ package episode import ( "fmt" - "log" "net/http" "github.com/aerogo/aero" @@ -12,27 +11,6 @@ import ( minio "github.com/minio/minio-go/v6" ) -var spaces *minio.Client - -func init() { - if arn.APIKeys.S3.ID == "" || arn.APIKeys.S3.Secret == "" { - return - } - - go func() { - var err error - endpoint := "sfo2.digitaloceanspaces.com" - ssl := true - - // Initiate a client using DigitalOcean Spaces. - spaces, err = minio.New(endpoint, arn.APIKeys.S3.ID, arn.APIKeys.S3.Secret, ssl) - - if err != nil { - log.Fatal(err) - } - }() -} - // Get renders the anime episode. func Get(ctx aero.Context) error { user := utils.GetUser(ctx) @@ -60,8 +38,8 @@ func Get(ctx aero.Context) error { // Does the episode exist? uploaded := false - if spaces != nil { - stat, err := spaces.StatObject("arn", fmt.Sprintf("videos/anime/%s/%d.webm", anime.ID, episodeNumber), minio.StatObjectOptions{}) + if arn.Spaces != nil { + stat, err := arn.Spaces.StatObject("arn", fmt.Sprintf("videos/anime/%s/%d.webm", anime.ID, episodeNumber), minio.StatObjectOptions{}) uploaded = (err == nil) && (stat.Size > 0) } diff --git a/pages/episode/subtitles.go b/pages/episode/subtitles.go index 80a25983..a81f75e5 100644 --- a/pages/episode/subtitles.go +++ b/pages/episode/subtitles.go @@ -29,7 +29,7 @@ func Subtitles(ctx aero.Context) error { ctx.Response().SetHeader("Access-Control-Allow-Origin", "*") ctx.Response().SetHeader("Content-Type", "text/vtt; charset=utf-8") - obj, err := spaces.GetObject("arn", fmt.Sprintf("videos/anime/%s/%d.%s.vtt", anime.ID, episodeNumber, language), minio.GetObjectOptions{}) + obj, err := arn.Spaces.GetObject("arn", fmt.Sprintf("videos/anime/%s/%d.%s.vtt", anime.ID, episodeNumber, language), minio.GetObjectOptions{}) if err != nil { return ctx.Error(http.StatusInternalServerError, err) diff --git a/pages/upload/amv-file.go b/pages/upload/amv-file.go index 63f5972f..8b305c74 100644 --- a/pages/upload/amv-file.go +++ b/pages/upload/amv-file.go @@ -25,20 +25,17 @@ func AMVFile(ctx aero.Context) error { } // Retrieve file from post body - data, err := ctx.Request().Body().Bytes() + reader := ctx.Request().Body().Reader() + defer reader.Close() - if err != nil { - return ctx.Error(http.StatusInternalServerError, "Reading request body failed", err) - } - - // Set amv image file - err = amv.SetVideoBytes(data) + // Set amv video file + err = amv.SetVideoReader(reader) if err != nil { return ctx.Error(http.StatusInternalServerError, "Invalid video format", err) } - // Save image information + // Save video information amv.Save() // Write log entry diff --git a/patches/update-amv-info/update-amv-info.go b/patches/update-amv-info/update-amv-info.go deleted file mode 100644 index 445f920d..00000000 --- a/patches/update-amv-info/update-amv-info.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "github.com/akyoto/color" - "github.com/animenotifier/notify.moe/arn" - "github.com/animenotifier/notify.moe/arn/stringutils" -) - -func main() { - color.Yellow("Updating AMV info") - - defer color.Green("Finished.") - defer arn.Node.Close() - - for amv := range arn.StreamAMVs() { - err := amv.RefreshInfo() - - if err != nil { - color.Red(err.Error()) - continue - } - - color.Green(amv.Title.Canonical) - stringutils.PrettyPrint(amv.Info) - amv.Save() - } -}