diff --git a/jobs/mal-parse/mal-parse.go b/jobs/mal-parse/mal-parse.go index 2a8e8540..aa6df8bd 100644 --- a/jobs/mal-parse/mal-parse.go +++ b/jobs/mal-parse/mal-parse.go @@ -3,7 +3,6 @@ package main import ( "compress/gzip" "errors" - "fmt" "os" "path" "path/filepath" @@ -37,9 +36,11 @@ func main() { } func readFiles(root string, onFile func(string) error) { + count := 0 + filepath.Walk(root, func(name string, info os.FileInfo, err error) error { if err != nil { - fmt.Println(err) + color.Red(err.Error()) return err } @@ -51,8 +52,18 @@ func readFiles(root string, onFile func(string) error) { return nil } - return onFile(name) + count++ + err = onFile(name) + + if err != nil { + color.Red(err.Error()) + } + + // Always continue traversing the directory + return nil }) + + color.Cyan("%d files found", count) } func readAnimeFile(name string) error { diff --git a/jobs/mal-sync/character.go b/jobs/mal-sync/character.go index 916153e0..a14fbe79 100644 --- a/jobs/mal-sync/character.go +++ b/jobs/mal-sync/character.go @@ -21,6 +21,47 @@ func parseCharacterDescription(input string) (output string, attributes []*arn.C for _, paragraph := range paragraphs { // Is paragraph full of attributes? if strings.Contains(paragraph, "\n") { + lines := strings.Split(paragraph, "\n") + var lastAttribute *arn.CharacterAttribute + + for _, line := range lines { + line = strings.Replace(line, " (\n)", "", -1) + + // Remove all kinds of starting and ending parantheses. + if strings.HasPrefix(line, "(") { + line = strings.TrimPrefix(line, "(") + line = strings.TrimSuffix(line, ")") + } + + line = strings.TrimSuffix(line, " (") + line = strings.TrimPrefix(line, ")") + + parts := strings.Split(line, ":") + + if len(parts) < 2 { + // Add to previous attribute + if lastAttribute != nil { + lastAttribute.Value += ", " + line + } + + continue + } + + name := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + if value == "" || value == `"` { + continue + } + + lastAttribute = &arn.CharacterAttribute{ + Name: name, + Value: value, + } + + attributes = append(attributes, lastAttribute) + } + continue } diff --git a/jobs/mal-sync/mal-sync.go b/jobs/mal-sync/mal-sync.go index 4049d3b6..509e0bb1 100644 --- a/jobs/mal-sync/mal-sync.go +++ b/jobs/mal-sync/mal-sync.go @@ -27,31 +27,27 @@ func main() { } // Sync the most important ones first - allAnime := arn.AllAnime() + allAnime := arn.FilterAnime(func(anime *arn.Anime) bool { + return anime.GetMapping("myanimelist/anime") != "" + }) + arn.SortAnimeByQuality(allAnime) + color.Yellow("%d anime found", len(allAnime)) for _, anime := range allAnime { - malID := anime.GetMapping("myanimelist/anime") - - if malID == "" { - continue - } - - syncAnime(anime, malID) + syncAnime(anime, anime.GetMapping("myanimelist/anime")) } // Sync the most important ones first - allCharacters := arn.AllCharacters() + allCharacters := arn.FilterCharacters(func(character *arn.Character) bool { + return character.GetMapping("myanimelist/character") != "" + }) + arn.SortCharactersByLikes(allCharacters) + color.Yellow("%d characters found", len(allCharacters)) for _, character := range allCharacters { - malID := character.GetMapping("myanimelist/character") - - if malID == "" { - continue - } - - syncCharacter(character, malID) + syncCharacter(character, character.GetMapping("myanimelist/character")) } } @@ -59,7 +55,6 @@ func syncAnime(anime *arn.Anime, malID string) { obj, err := malDB.Get("Anime", malID) if err != nil { - fmt.Println(err) return } @@ -91,7 +86,11 @@ func syncCharacter(character *arn.Character, malID string) { obj, err := malDB.Get("Character", malID) if err != nil { - fmt.Println(err) + return + } + + // Skip manually created characters + if character.CreatedBy != "" { return }