From e271893938216281f1333acc2effc65eb3667c1c Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sat, 17 Mar 2018 01:24:06 +0100 Subject: [PATCH] Improved kitsu import --- jobs/kitsu-import/kitsu-import.go | 298 +++++++++++++++--------------- jobs/kitsu-import/shell.go | 7 +- 2 files changed, 151 insertions(+), 154 deletions(-) diff --git a/jobs/kitsu-import/kitsu-import.go b/jobs/kitsu-import/kitsu-import.go index 56fcb97b..d59006ad 100644 --- a/jobs/kitsu-import/kitsu-import.go +++ b/jobs/kitsu-import/kitsu-import.go @@ -2,10 +2,7 @@ package main import ( "fmt" - "net/http" - "strings" - "github.com/aerogo/http/client" "github.com/animenotifier/arn" "github.com/animenotifier/kitsu" "github.com/fatih/color" @@ -23,155 +20,160 @@ func main() { } // Get a stream of all anime - allAnime := kitsu.StreamAnimeWithMappings() + allAnime := kitsu.StreamAnime() // Iterate over the stream for anime := range allAnime { - importKitsuAnime(anime) + sync(anime) } } -func importKitsuAnime(data *kitsu.Anime) *arn.Anime { - anime, err := arn.GetAnime(data.ID) - - // This stops overwriting existing data - // if anime != nil { - // return anime - // } - - if err != nil { - if strings.Contains(err.Error(), "not found") { - anime = &arn.Anime{ - Title: &arn.AnimeTitle{}, - } - } else { - panic(err) - } - } - - attr := data.Attributes - - // General data - anime.ID = data.ID - anime.Type = strings.ToLower(attr.ShowType) - anime.Title.Canonical = attr.CanonicalTitle - anime.Title.English = attr.Titles.En - anime.Title.Romaji = attr.Titles.EnJp - anime.Title.Synonyms = attr.AbbreviatedTitles - anime.StartDate = attr.StartDate - anime.EndDate = attr.EndDate - anime.EpisodeCount = attr.EpisodeCount - anime.EpisodeLength = attr.EpisodeLength - anime.Status = attr.Status - - // Status "unreleased" means the same as "upcoming" so we should normalize it - if anime.Status == "unreleased" { - anime.Status = "upcoming" - } - - // Normalize image extension to .jpg if .jpeg is used - if anime.Image.Extension == ".jpeg" { - anime.Image.Extension = ".jpg" - } - - anime.Summary = arn.FixAnimeDescription(attr.Synopsis) - - if anime.Mappings == nil { - anime.Mappings = []*arn.Mapping{} - } - - // Prefer Shoboi Japanese titles over Kitsu JP titles - if anime.GetMapping("shoboi/anime") != "" { - // Only take Kitsu title when our JP title is empty - if anime.Title.Japanese == "" { - anime.Title.Japanese = attr.Titles.JaJp - } - } else { - // Update JP title with Kitsu JP title - anime.Title.Japanese = attr.Titles.JaJp - } - - // Import mappings - for _, mapping := range data.Mappings { - switch mapping.Attributes.ExternalSite { - case "myanimelist/anime": - anime.SetMapping("myanimelist/anime", mapping.Attributes.ExternalID, "") - case "anidb": - anime.SetMapping("anidb/anime", mapping.Attributes.ExternalID, "") - case "thetvdb", "thetvdb/series": - fmt.Println(mapping.Attributes.ExternalSite, mapping.Attributes.ExternalID) - anime.SetMapping("thetvdb/anime", mapping.Attributes.ExternalID, "") - case "thetvdb/season": - // Ignore - default: - color.Yellow("Unknown mapping: %s %s", mapping.Attributes.ExternalSite, mapping.Attributes.ExternalID) - } - } - - return anime - - // Download image - response, err := client.Get(attr.PosterImage.Original).End() - - if err == nil && response.StatusCode() == http.StatusOK { - anime.SetImageBytes(response.Bytes()) - } else { - color.Red("No image for [%s] %s (%d)", anime.ID, anime, response.StatusCode()) - } - - // Rating - if anime.Rating == nil { - anime.Rating = &arn.AnimeRating{} - } - - if anime.Rating.IsNotRated() { - anime.Rating.Reset() - } - - // Popularity - if anime.Popularity == nil { - anime.Popularity = &arn.AnimePopularity{} - } - - // Trailers - anime.Trailers = []*arn.ExternalMedia{} - - if attr.YoutubeVideoID != "" { - anime.Trailers = append(anime.Trailers, &arn.ExternalMedia{ - Service: "Youtube", - ServiceID: attr.YoutubeVideoID, - }) - } - - // Save in database - anime.Save() - - // Episodes - episodes, err := arn.GetAnimeEpisodes(anime.ID) - - if err != nil || episodes == nil { - episodes := &arn.AnimeEpisodes{ - AnimeID: anime.ID, - Items: []*arn.AnimeEpisode{}, - } - - arn.DB.Set("AnimeEpisodes", anime.ID, episodes) - } - - // Relations - relations, _ := arn.GetAnimeRelations(anime.ID) - - if relations == nil { - relations := &arn.AnimeRelations{ - AnimeID: anime.ID, - Items: []*arn.AnimeRelation{}, - } - - arn.DB.Set("AnimeRelations", anime.ID, relations) - } - - // Log - fmt.Println(color.GreenString("✔"), anime.ID, anime.Title.Canonical) - - return anime +func sync(anime *kitsu.Anime) { + fmt.Println(anime.ID, anime.Attributes.Titles.En) + arn.Kitsu.Set("Anime", anime.ID, anime) } + +// func importKitsuAnime(data *kitsu.Anime) *arn.Anime { +// anime, err := arn.GetAnime(data.ID) + +// // This stops overwriting existing data +// // if anime != nil { +// // return anime +// // } + +// if err != nil { +// if strings.Contains(err.Error(), "not found") { +// anime = &arn.Anime{ +// Title: &arn.AnimeTitle{}, +// } +// } else { +// panic(err) +// } +// } + +// attr := data.Attributes + +// // General data +// anime.ID = data.ID +// anime.Type = strings.ToLower(attr.ShowType) +// anime.Title.Canonical = attr.CanonicalTitle +// anime.Title.English = attr.Titles.En +// anime.Title.Romaji = attr.Titles.EnJp +// anime.Title.Synonyms = attr.AbbreviatedTitles +// anime.StartDate = attr.StartDate +// anime.EndDate = attr.EndDate +// anime.EpisodeCount = attr.EpisodeCount +// anime.EpisodeLength = attr.EpisodeLength +// anime.Status = attr.Status + +// // Status "unreleased" means the same as "upcoming" so we should normalize it +// if anime.Status == "unreleased" { +// anime.Status = "upcoming" +// } + +// // Normalize image extension to .jpg if .jpeg is used +// if anime.Image.Extension == ".jpeg" { +// anime.Image.Extension = ".jpg" +// } + +// anime.Summary = arn.FixAnimeDescription(attr.Synopsis) + +// if anime.Mappings == nil { +// anime.Mappings = []*arn.Mapping{} +// } + +// // Prefer Shoboi Japanese titles over Kitsu JP titles +// if anime.GetMapping("shoboi/anime") != "" { +// // Only take Kitsu title when our JP title is empty +// if anime.Title.Japanese == "" { +// anime.Title.Japanese = attr.Titles.JaJp +// } +// } else { +// // Update JP title with Kitsu JP title +// anime.Title.Japanese = attr.Titles.JaJp +// } + +// // Import mappings +// for _, mapping := range data.Mappings { +// switch mapping.Attributes.ExternalSite { +// case "myanimelist/anime": +// anime.SetMapping("myanimelist/anime", mapping.Attributes.ExternalID, "") +// case "anidb": +// anime.SetMapping("anidb/anime", mapping.Attributes.ExternalID, "") +// case "thetvdb", "thetvdb/series": +// fmt.Println(mapping.Attributes.ExternalSite, mapping.Attributes.ExternalID) +// anime.SetMapping("thetvdb/anime", mapping.Attributes.ExternalID, "") +// case "thetvdb/season": +// // Ignore +// default: +// color.Yellow("Unknown mapping: %s %s", mapping.Attributes.ExternalSite, mapping.Attributes.ExternalID) +// } +// } + +// return anime + +// // Download image +// response, err := client.Get(attr.PosterImage.Original).End() + +// if err == nil && response.StatusCode() == http.StatusOK { +// anime.SetImageBytes(response.Bytes()) +// } else { +// color.Red("No image for [%s] %s (%d)", anime.ID, anime, response.StatusCode()) +// } + +// // Rating +// if anime.Rating == nil { +// anime.Rating = &arn.AnimeRating{} +// } + +// if anime.Rating.IsNotRated() { +// anime.Rating.Reset() +// } + +// // Popularity +// if anime.Popularity == nil { +// anime.Popularity = &arn.AnimePopularity{} +// } + +// // Trailers +// anime.Trailers = []*arn.ExternalMedia{} + +// if attr.YoutubeVideoID != "" { +// anime.Trailers = append(anime.Trailers, &arn.ExternalMedia{ +// Service: "Youtube", +// ServiceID: attr.YoutubeVideoID, +// }) +// } + +// // Save in database +// anime.Save() + +// // Episodes +// episodes, err := arn.GetAnimeEpisodes(anime.ID) + +// if err != nil || episodes == nil { +// episodes := &arn.AnimeEpisodes{ +// AnimeID: anime.ID, +// Items: []*arn.AnimeEpisode{}, +// } + +// arn.DB.Set("AnimeEpisodes", anime.ID, episodes) +// } + +// // Relations +// relations, _ := arn.GetAnimeRelations(anime.ID) + +// if relations == nil { +// relations := &arn.AnimeRelations{ +// AnimeID: anime.ID, +// Items: []*arn.AnimeRelation{}, +// } + +// arn.DB.Set("AnimeRelations", anime.ID, relations) +// } + +// // Log +// fmt.Println(color.GreenString("✔"), anime.ID, anime.Title.Canonical) + +// return anime +// } diff --git a/jobs/kitsu-import/shell.go b/jobs/kitsu-import/shell.go index 5aa4f6c6..5292b57c 100644 --- a/jobs/kitsu-import/shell.go +++ b/jobs/kitsu-import/shell.go @@ -6,7 +6,6 @@ import ( "github.com/animenotifier/arn" "github.com/animenotifier/kitsu" - "github.com/fatih/color" ) // Shell parameters @@ -33,14 +32,10 @@ func InvokeShellArgs() bool { panic(errors.New("Anime ID is not the same")) } - anime := importKitsuAnime(kitsuAnime) + sync(kitsuAnime) if verbose { - color.Cyan("Kitsu:") arn.PrettyPrint(kitsuAnime) - - color.Cyan("ARN:") - arn.PrettyPrint(anime) } return true