Updated MAL tools

This commit is contained in:
Eduard Urbach 2018-10-30 09:35:30 +09:00
parent a559227b65
commit 85cd5538b6
2 changed files with 70 additions and 28 deletions

View File

@ -17,6 +17,7 @@ const (
delayBetweenRequests = 1100 * time.Millisecond
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.20 Safari/537.36"
animeDirectory = "anime"
characterDirectory = "character"
)
var headers = map[string]string{
@ -33,41 +34,59 @@ func main() {
}
// Filter anime with MAL ID
animes := []*arn.Anime{}
var animes []*arn.Anime
for anime := range arn.StreamAnime() {
malID := anime.GetMapping("myanimelist/anime")
if objectType == "all" || objectType == "anime" {
animes = arn.FilterAnime(func(anime *arn.Anime) bool {
return anime.GetMapping("myanimelist/anime") != ""
})
if malID == "" {
continue
}
color.Yellow("Found %d anime", len(animes))
animes = append(animes, anime)
// Sort so that we download the most important ones first
arn.SortAnimeByQuality(animes)
// Create anime directory if it's missing
os.Mkdir(animeDirectory, 0777)
}
color.Yellow("Found %d anime", len(animes))
// Filter characters with MAL ID
var characters []*arn.Character
if objectType == "all" || objectType == "character" {
characters = arn.FilterCharacters(func(character *arn.Character) bool {
return character.GetMapping("myanimelist/character") != ""
})
color.Yellow("Found %d characters", len(characters))
// Sort so that we download the most important ones first
arn.SortCharactersByLikes(characters)
// Create character directory if it's missing
os.Mkdir(characterDirectory, 0777)
}
// We don't need the database anymore
arn.Node.Close()
// Create anime directory if it's missing
os.Mkdir(animeDirectory, 0777)
// Create crawler
malCrawler := crawler.New(
headers,
delayBetweenRequests,
len(animes),
len(animes)+len(characters),
)
// Sort so that we download the most important ones first
arn.SortAnimeByQuality(animes)
// Queue up URLs
count := 0
for _, anime := range animes {
queue(anime, malCrawler)
queueAnime(anime, malCrawler)
count++
}
for _, character := range characters {
queueCharacter(character, malCrawler)
count++
}
@ -78,7 +97,7 @@ func main() {
malCrawler.Wait()
}
func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
func queueAnime(anime *arn.Anime, malCrawler *crawler.Crawler) {
malID := anime.GetMapping("myanimelist/anime")
url := "https://myanimelist.net/anime/" + malID
filePath := fmt.Sprintf("%s/%s.html.gz", animeDirectory, malID)
@ -95,3 +114,21 @@ func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
Raw: true,
})
}
func queueCharacter(character *arn.Character, malCrawler *crawler.Crawler) {
malID := character.GetMapping("myanimelist/character")
url := "https://myanimelist.net/character/" + malID
filePath := fmt.Sprintf("%s/%s.html.gz", characterDirectory, malID)
fileInfo, err := os.Stat(filePath)
if err == nil && time.Since(fileInfo.ModTime()) <= maxAge {
// fmt.Println(color.YellowString(url), "skip")
return
}
malCrawler.Queue(&crawler.Task{
URL: url,
Destination: filePath,
Raw: true,
})
}

View File

@ -8,23 +8,19 @@ import (
)
// Shell parameters
var animeID string
var objectType string
var objectID string
// Shell flags
func init() {
flag.StringVar(&animeID, "id", "", "ID of the notify.moe anime you want to refresh")
flag.StringVar(&objectType, "type", "all", "all | anime | character")
flag.StringVar(&objectID, "id", "", "ID of the notify.moe anime/character you want to refresh")
flag.Parse()
}
// InvokeShellArgs ...
func InvokeShellArgs() bool {
if animeID != "" {
anime, err := arn.GetAnime(animeID)
if err != nil {
panic(err)
}
if objectID != "" {
// Create crawler
malCrawler := crawler.New(
headers,
@ -32,8 +28,17 @@ func InvokeShellArgs() bool {
1,
)
// Queue
queue(anime, malCrawler)
switch objectType {
case "anime":
anime, err := arn.GetAnime(objectID)
arn.PanicOnError(err)
queueAnime(anime, malCrawler)
case "character":
character, err := arn.GetCharacter(objectID)
arn.PanicOnError(err)
queueCharacter(character, malCrawler)
}
// Wait
malCrawler.Wait()