Started working on improved character parsing

This commit is contained in:
Eduard Urbach 2018-11-01 02:21:32 +09:00
parent af6d7a6fdb
commit 3ccf470cf0
3 changed files with 72 additions and 21 deletions

View File

@ -3,7 +3,6 @@ package main
import (
"compress/gzip"
"errors"
"fmt"
"os"
"path"
"path/filepath"
@ -37,9 +36,11 @@ func main() {
}
func readFiles(root string, onFile func(string) error) {
count := 0
filepath.Walk(root, func(name string, info os.FileInfo, err error) error {
if err != nil {
fmt.Println(err)
color.Red(err.Error())
return err
}
@ -51,8 +52,18 @@ func readFiles(root string, onFile func(string) error) {
return nil
}
return onFile(name)
count++
err = onFile(name)
if err != nil {
color.Red(err.Error())
}
// Always continue traversing the directory
return nil
})
color.Cyan("%d files found", count)
}
func readAnimeFile(name string) error {

View File

@ -21,6 +21,47 @@ func parseCharacterDescription(input string) (output string, attributes []*arn.C
for _, paragraph := range paragraphs {
// Is paragraph full of attributes?
if strings.Contains(paragraph, "\n") {
lines := strings.Split(paragraph, "\n")
var lastAttribute *arn.CharacterAttribute
for _, line := range lines {
line = strings.Replace(line, " (\n)", "", -1)
// Remove all kinds of starting and ending parantheses.
if strings.HasPrefix(line, "(") {
line = strings.TrimPrefix(line, "(")
line = strings.TrimSuffix(line, ")")
}
line = strings.TrimSuffix(line, " (")
line = strings.TrimPrefix(line, ")")
parts := strings.Split(line, ":")
if len(parts) < 2 {
// Add to previous attribute
if lastAttribute != nil {
lastAttribute.Value += ", " + line
}
continue
}
name := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
if value == "" || value == `"` {
continue
}
lastAttribute = &arn.CharacterAttribute{
Name: name,
Value: value,
}
attributes = append(attributes, lastAttribute)
}
continue
}

View File

@ -27,31 +27,27 @@ func main() {
}
// Sync the most important ones first
allAnime := arn.AllAnime()
allAnime := arn.FilterAnime(func(anime *arn.Anime) bool {
return anime.GetMapping("myanimelist/anime") != ""
})
arn.SortAnimeByQuality(allAnime)
color.Yellow("%d anime found", len(allAnime))
for _, anime := range allAnime {
malID := anime.GetMapping("myanimelist/anime")
if malID == "" {
continue
}
syncAnime(anime, malID)
syncAnime(anime, anime.GetMapping("myanimelist/anime"))
}
// Sync the most important ones first
allCharacters := arn.AllCharacters()
allCharacters := arn.FilterCharacters(func(character *arn.Character) bool {
return character.GetMapping("myanimelist/character") != ""
})
arn.SortCharactersByLikes(allCharacters)
color.Yellow("%d characters found", len(allCharacters))
for _, character := range allCharacters {
malID := character.GetMapping("myanimelist/character")
if malID == "" {
continue
}
syncCharacter(character, malID)
syncCharacter(character, character.GetMapping("myanimelist/character"))
}
}
@ -59,7 +55,6 @@ func syncAnime(anime *arn.Anime, malID string) {
obj, err := malDB.Get("Anime", malID)
if err != nil {
fmt.Println(err)
return
}
@ -91,7 +86,11 @@ func syncCharacter(character *arn.Character, malID string) {
obj, err := malDB.Get("Character", malID)
if err != nil {
fmt.Println(err)
return
}
// Skip manually created characters
if character.CreatedBy != "" {
return
}