From 5cc1ded99d022e6c194935b3f8d9e4f0f94d0dfb Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sat, 17 Mar 2018 00:28:36 +0100 Subject: [PATCH] Cleanup --- benchmarks/Components_test.go | 2 +- .../kitsu-import.go} | 34 +++++++++++++------ jobs/{sync-anime => kitsu-import}/shell.go | 2 +- .../files/.gitignore | 0 .../mal-download.go} | 0 jobs/mal-import/mal-import.go | 8 +++-- jobs/sync-shoboi/sync-shoboi.go | 2 +- 7 files changed, 32 insertions(+), 16 deletions(-) rename jobs/{sync-anime/sync-anime.go => kitsu-import/kitsu-import.go} (81%) rename jobs/{sync-anime => kitsu-import}/shell.go (95%) rename jobs/{mal-crawler => mal-download}/files/.gitignore (100%) rename jobs/{mal-crawler/mal-crawler.go => mal-download/mal-download.go} (100%) diff --git a/benchmarks/Components_test.go b/benchmarks/Components_test.go index eb01f210..f60ca1f8 100644 --- a/benchmarks/Components_test.go +++ b/benchmarks/Components_test.go @@ -35,7 +35,7 @@ func BenchmarkRenderAnimeList(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { - components.AnimeList(animeList, user, user) + components.AnimeList(animeList.Items, -1, user, user) } }) } diff --git a/jobs/sync-anime/sync-anime.go b/jobs/kitsu-import/kitsu-import.go similarity index 81% rename from jobs/sync-anime/sync-anime.go rename to jobs/kitsu-import/kitsu-import.go index cef49d87..56fcb97b 100644 --- a/jobs/sync-anime/sync-anime.go +++ b/jobs/kitsu-import/kitsu-import.go @@ -2,16 +2,17 @@ package main import ( "fmt" - "path/filepath" + "net/http" "strings" + "github.com/aerogo/http/client" "github.com/animenotifier/arn" "github.com/animenotifier/kitsu" "github.com/fatih/color" ) func main() { - color.Yellow("Syncing Anime") + color.Yellow("Importing Kitsu anime") defer arn.Node.Close() defer color.Green("Finished.") @@ -26,17 +27,17 @@ func main() { // Iterate over the stream for anime := range allAnime { - sync(anime) + importKitsuAnime(anime) } } -func sync(data *kitsu.Anime) *arn.Anime { +func importKitsuAnime(data *kitsu.Anime) *arn.Anime { anime, err := arn.GetAnime(data.ID) // This stops overwriting existing data - if anime != nil { - return anime - } + // if anime != nil { + // return anime + // } if err != nil { if strings.Contains(err.Error(), "not found") { @@ -62,7 +63,6 @@ func sync(data *kitsu.Anime) *arn.Anime { anime.EpisodeCount = attr.EpisodeCount anime.EpisodeLength = attr.EpisodeLength anime.Status = attr.Status - anime.Image.Extension = filepath.Ext(kitsu.FixImageURL(attr.PosterImage.Original)) // Status "unreleased" means the same as "upcoming" so we should normalize it if anime.Status == "unreleased" { @@ -95,11 +95,12 @@ func sync(data *kitsu.Anime) *arn.Anime { for _, mapping := range data.Mappings { switch mapping.Attributes.ExternalSite { case "myanimelist/anime": - anime.AddMapping("myanimelist/anime", mapping.Attributes.ExternalID, "") + anime.SetMapping("myanimelist/anime", mapping.Attributes.ExternalID, "") case "anidb": - anime.AddMapping("anidb/anime", mapping.Attributes.ExternalID, "") + anime.SetMapping("anidb/anime", mapping.Attributes.ExternalID, "") case "thetvdb", "thetvdb/series": - anime.AddMapping("thetvdb/anime", mapping.Attributes.ExternalID, "") + fmt.Println(mapping.Attributes.ExternalSite, mapping.Attributes.ExternalID) + anime.SetMapping("thetvdb/anime", mapping.Attributes.ExternalID, "") case "thetvdb/season": // Ignore default: @@ -107,6 +108,17 @@ func sync(data *kitsu.Anime) *arn.Anime { } } + return anime + + // Download image + response, err := client.Get(attr.PosterImage.Original).End() + + if err == nil && response.StatusCode() == http.StatusOK { + anime.SetImageBytes(response.Bytes()) + } else { + color.Red("No image for [%s] %s (%d)", anime.ID, anime, response.StatusCode()) + } + // Rating if anime.Rating == nil { anime.Rating = &arn.AnimeRating{} diff --git a/jobs/sync-anime/shell.go b/jobs/kitsu-import/shell.go similarity index 95% rename from jobs/sync-anime/shell.go rename to jobs/kitsu-import/shell.go index 96a3802d..5aa4f6c6 100644 --- a/jobs/sync-anime/shell.go +++ b/jobs/kitsu-import/shell.go @@ -33,7 +33,7 @@ func InvokeShellArgs() bool { panic(errors.New("Anime ID is not the same")) } - anime := sync(kitsuAnime) + anime := importKitsuAnime(kitsuAnime) if verbose { color.Cyan("Kitsu:") diff --git a/jobs/mal-crawler/files/.gitignore b/jobs/mal-download/files/.gitignore similarity index 100% rename from jobs/mal-crawler/files/.gitignore rename to jobs/mal-download/files/.gitignore diff --git a/jobs/mal-crawler/mal-crawler.go b/jobs/mal-download/mal-download.go similarity index 100% rename from jobs/mal-crawler/mal-crawler.go rename to jobs/mal-download/mal-download.go diff --git a/jobs/mal-import/mal-import.go b/jobs/mal-import/mal-import.go index 508da394..a3ef1714 100644 --- a/jobs/mal-import/mal-import.go +++ b/jobs/mal-import/mal-import.go @@ -10,14 +10,18 @@ import ( "github.com/animenotifier/arn" "github.com/animenotifier/mal/parser" + "github.com/fatih/color" ) func main() { + color.Yellow("Importing MAL anime") + defer arn.Node.Close() + defer color.Green("Finished.") - // readFile("../mal-crawler/files/anime-31240.html") + // readFile("../mal-download/files/anime-31240.html") - filepath.Walk("../mal-crawler/files", func(name string, info os.FileInfo, err error) error { + filepath.Walk("../mal-download/files", func(name string, info os.FileInfo, err error) error { if err != nil { fmt.Println(err) return err diff --git a/jobs/sync-shoboi/sync-shoboi.go b/jobs/sync-shoboi/sync-shoboi.go index ab3eb549..5a3adae0 100644 --- a/jobs/sync-shoboi/sync-shoboi.go +++ b/jobs/sync-shoboi/sync-shoboi.go @@ -116,5 +116,5 @@ func search(anime *arn.Anime, title string) { } // This will start a goroutine that saves the anime - anime.AddMapping("shoboi/anime", shoboi.TID, "") + anime.SetMapping("shoboi/anime", shoboi.TID, "") }