web/notify.moe
Code Issues Pull Requests Activity

Simpler image download

Browse Source
This commit is contained in:
Eduard Urbach 2017-11-09 13:28:06 +01:00
parent 2d4b71971e
commit a9505ea07a
6 changed files with 23 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
images/anime/.gitignore vendored
View File

@ -1,2 +1,3 @@
* *
!*/
!.gitignore !.gitignore

2
images/anime/large/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!.gitignore

2
images/anime/medium/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!.gitignore

2
images/anime/original/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!.gitignore

2
images/anime/small/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
*
!.gitignore

25
jobs/anime-images/anime-images.go
View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"os" "os"
"path" "path"
"runtime"
"strings" "strings"
"time" "time"
@ -11,7 +12,6 @@ import (
_ "image/jpeg" _ "image/jpeg"
_ "image/png" _ "image/png"
"github.com/aerogo/flow/jobqueue"
"github.com/aerogo/ipo" "github.com/aerogo/ipo"
"github.com/aerogo/ipo/inputs" "github.com/aerogo/ipo/inputs"
"github.com/aerogo/ipo/outputs" "github.com/aerogo/ipo/outputs"
@ -19,28 +19,26 @@ import (
"github.com/fatih/color" "github.com/fatih/color"
) )
var ticker = time.NewTicker(50 * time.Millisecond) var ticker = time.NewTicker(100 * time.Millisecond)
func main() { func main() {
color.Yellow("Downloading anime images") color.Yellow("Downloading anime images")
defer arn.Node.Close() defer arn.Node.Close()
jobs := jobqueue.New(work) allAnime := arn.AllAnime()
for anime := range arn.StreamAnime() { for index, anime := range allAnime {
jobs.Queue(anime) fmt.Printf("%d / %d\n", index, len(allAnime))
work(anime)
} }
results := jobs.Wait() color.Green("Finished downloading anime images.")
color.Green("Finished downloading %d anime images.", len(results))
// Give file buffers some time, just to be safe // Give file buffers some time, just to be safe
time.Sleep(time.Second) time.Sleep(time.Second)
} }
func work(job interface{}) interface{} { func work(anime *arn.Anime) error {
anime := job.(*arn.Anime)
if !strings.HasPrefix(anime.Image.Original, "//media.kitsu.io/anime/") { if !strings.HasPrefix(anime.Image.Original, "//media.kitsu.io/anime/") {
return nil return nil
} }
@ -59,7 +57,7 @@ func work(job interface{}) interface{} {
webpQuality := 80 webpQuality := 80
jpegQuality := 80 jpegQuality := 80
system := &ipo.System{ system := ipo.System{
Inputs: []ipo.Input{ Inputs: []ipo.Input{
&inputs.NetworkImage{ &inputs.NetworkImage{
URL: anime.Image.Original, URL: anime.Image.Original,
@ -166,5 +164,10 @@ func work(job interface{}) interface{} {
fmt.Println(err) fmt.Println(err)
} }
// Try to free up some memory
system.Inputs = nil
system.Outputs = nil
runtime.GC()
return nil return nil
} }