160 lines
2.7 KiB
Go
Raw Normal View History

2018-03-08 23:07:48 +00:00
package main
import (
2018-10-29 23:59:35 +00:00
"compress/gzip"
2018-03-08 23:07:48 +00:00
"errors"
"os"
2018-04-12 17:05:05 +00:00
"path"
2018-03-08 23:07:48 +00:00
"path/filepath"
"strings"
2018-04-12 09:05:55 +00:00
"github.com/animenotifier/mal"
2018-03-09 03:32:31 +00:00
"github.com/animenotifier/arn"
2018-03-08 23:07:48 +00:00
"github.com/animenotifier/mal/parser"
2018-03-16 23:28:36 +00:00
"github.com/fatih/color"
2018-03-08 23:07:48 +00:00
)
func main() {
2018-04-12 11:39:58 +00:00
color.Yellow("Parsing MAL files")
2018-03-16 23:28:36 +00:00
defer color.Green("Finished.")
2018-03-21 04:15:03 +00:00
defer arn.Node.Close()
2018-03-08 23:07:48 +00:00
2018-04-12 17:05:05 +00:00
// Invoke via parameters
if InvokeShellArgs() {
return
}
2018-03-08 23:07:48 +00:00
2018-10-31 09:13:18 +00:00
if objectType == "all" || objectType == "anime" {
2018-10-31 09:57:28 +00:00
readFiles(path.Join(arn.Root, "jobs", "mal-download", "anime"), readAnimeFile)
2018-10-31 09:13:18 +00:00
}
2018-10-31 09:57:28 +00:00
if objectType == "all" || objectType == "character" {
readFiles(path.Join(arn.Root, "jobs", "mal-download", "character"), readCharacterFile)
}
}
func readFiles(root string, onFile func(string) error) {
count := 0
2018-10-31 09:57:28 +00:00
filepath.Walk(root, func(name string, info os.FileInfo, err error) error {
if err != nil {
color.Red(err.Error())
2018-10-31 09:57:28 +00:00
return err
}
if info.IsDir() {
return nil
}
if !strings.HasSuffix(name, ".html.gz") {
return nil
}
count++
err = onFile(name)
if err != nil {
color.Red(err.Error())
}
// Always continue traversing the directory
return nil
2018-10-31 09:57:28 +00:00
})
color.Cyan("%d files found", count)
2018-03-08 23:07:48 +00:00
}
2018-10-30 03:01:42 +00:00
func readAnimeFile(name string) error {
2018-03-08 23:07:48 +00:00
file, err := os.Open(name)
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-03-08 23:07:48 +00:00
return err
}
2018-04-25 23:14:23 +00:00
defer file.Close()
2018-10-29 23:59:35 +00:00
reader, err := gzip.NewReader(file)
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-10-29 23:59:35 +00:00
return err
}
anime, characters, err := malparser.ParseAnime(reader)
2018-03-08 23:07:48 +00:00
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-03-08 23:07:48 +00:00
return err
}
if anime.ID == "" {
return errors.New("Empty ID")
}
2018-04-12 09:05:55 +00:00
for _, character := range characters {
obj, err := arn.MAL.Get("Character", character.ID)
if err != nil {
arn.MAL.Set("Character", character.ID, character)
continue
}
existing := obj.(*mal.Character)
modified := false
if existing.Name != character.Name {
existing.Name = character.Name
modified = true
}
2018-10-30 03:01:42 +00:00
if existing.Image != character.Image {
existing.Image = character.Image
2018-04-12 09:05:55 +00:00
modified = true
}
if modified {
arn.MAL.Set("Character", existing.ID, existing)
}
}
2018-10-31 15:04:47 +00:00
// fmt.Println(anime.ID, anime.Title)
2018-03-09 03:32:31 +00:00
arn.MAL.Set("Anime", anime.ID, anime)
2018-03-08 23:07:48 +00:00
return nil
}
2018-10-30 03:01:42 +00:00
func readCharacterFile(name string) error {
file, err := os.Open(name)
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-10-30 03:01:42 +00:00
return err
}
defer file.Close()
reader, err := gzip.NewReader(file)
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-10-30 03:01:42 +00:00
return err
}
character, err := malparser.ParseCharacter(reader)
if err != nil {
2018-10-31 15:04:47 +00:00
color.Red(err.Error())
2018-10-30 03:01:42 +00:00
return err
}
if character.ID == "" {
return errors.New("Empty ID")
}
2018-10-31 15:04:47 +00:00
// fmt.Println(character.ID, character.Name)
2018-10-30 03:01:42 +00:00
arn.MAL.Set("Character", character.ID, character)
return nil
}