Updated MAL tools

jobs/mal-download/mal-download.go

@@ -17,6 +17,7 @@ const (
 	delayBetweenRequests = 1100 * time.Millisecond
 	userAgent            = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.20 Safari/537.36"
 	animeDirectory       = "anime"
+	characterDirectory   = "character"
 )
 
 var headers = map[string]string{
@@ -33,41 +34,59 @@ func main() {
 	}
 
 	// Filter anime with MAL ID
-	animes := []*arn.Anime{}
+	var animes []*arn.Anime
 
-	for anime := range arn.StreamAnime() {
+	if objectType == "all" || objectType == "anime" {
-		malID := anime.GetMapping("myanimelist/anime")
+		animes = arn.FilterAnime(func(anime *arn.Anime) bool {
+			return anime.GetMapping("myanimelist/anime") != ""
+		})
 
-		if malID == "" {
+		color.Yellow("Found %d anime", len(animes))
-			continue
-		}
 
-		animes = append(animes, anime)
+		// Sort so that we download the most important ones first
+		arn.SortAnimeByQuality(animes)
+
+		// Create anime directory if it's missing
+		os.Mkdir(animeDirectory, 0777)
 	}
 
-	color.Yellow("Found %d anime", len(animes))
+	// Filter characters with MAL ID
+	var characters []*arn.Character
+
+	if objectType == "all" || objectType == "character" {
+		characters = arn.FilterCharacters(func(character *arn.Character) bool {
+			return character.GetMapping("myanimelist/character") != ""
+		})
+
+		color.Yellow("Found %d characters", len(characters))
+
+		// Sort so that we download the most important ones first
+		arn.SortCharactersByLikes(characters)
+
+		// Create character directory if it's missing
+		os.Mkdir(characterDirectory, 0777)
+	}
 
 	// We don't need the database anymore
 	arn.Node.Close()
 
-	// Create anime directory if it's missing
-	os.Mkdir(animeDirectory, 0777)
-
 	// Create crawler
 	malCrawler := crawler.New(
 		headers,
 		delayBetweenRequests,
-		len(animes),
+		len(animes)+len(characters),
 	)
 
-	// Sort so that we download the most important ones first
-	arn.SortAnimeByQuality(animes)
-
 	// Queue up URLs
 	count := 0
 
 	for _, anime := range animes {
-		queue(anime, malCrawler)
+		queueAnime(anime, malCrawler)
+		count++
+	}
+
+	for _, character := range characters {
+		queueCharacter(character, malCrawler)
 		count++
 	}
 
@@ -78,7 +97,7 @@ func main() {
 	malCrawler.Wait()
 }
 
-func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
+func queueAnime(anime *arn.Anime, malCrawler *crawler.Crawler) {
 	malID := anime.GetMapping("myanimelist/anime")
 	url := "https://myanimelist.net/anime/" + malID
 	filePath := fmt.Sprintf("%s/%s.html.gz", animeDirectory, malID)
@@ -95,3 +114,21 @@ func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
 		Raw:         true,
 	})
 }
+
+func queueCharacter(character *arn.Character, malCrawler *crawler.Crawler) {
+	malID := character.GetMapping("myanimelist/character")
+	url := "https://myanimelist.net/character/" + malID
+	filePath := fmt.Sprintf("%s/%s.html.gz", characterDirectory, malID)
+	fileInfo, err := os.Stat(filePath)
+
+	if err == nil && time.Since(fileInfo.ModTime()) <= maxAge {
+		// fmt.Println(color.YellowString(url), "skip")
+		return
+	}
+
+	malCrawler.Queue(&crawler.Task{
+		URL:         url,
+		Destination: filePath,
+		Raw:         true,
+	})
+}

jobs/mal-download/shell.go

@@ -8,23 +8,19 @@ import (
 )
 
 // Shell parameters
-var animeID string
+var objectType string
+var objectID string
 
 // Shell flags
 func init() {
-	flag.StringVar(&animeID, "id", "", "ID of the notify.moe anime you want to refresh")
+	flag.StringVar(&objectType, "type", "all", "all | anime | character")
+	flag.StringVar(&objectID, "id", "", "ID of the notify.moe anime/character you want to refresh")
 	flag.Parse()
 }
 
 // InvokeShellArgs ...
 func InvokeShellArgs() bool {
-	if animeID != "" {
+	if objectID != "" {
-		anime, err := arn.GetAnime(animeID)
-
-		if err != nil {
-			panic(err)
-		}
-
 		// Create crawler
 		malCrawler := crawler.New(
 			headers,
@@ -32,8 +28,17 @@ func InvokeShellArgs() bool {
 			1,
 		)
 
-		// Queue
+		switch objectType {
-		queue(anime, malCrawler)
+		case "anime":
+			anime, err := arn.GetAnime(objectID)
+			arn.PanicOnError(err)
+			queueAnime(anime, malCrawler)
+
+		case "character":
+			character, err := arn.GetCharacter(objectID)
+			arn.PanicOnError(err)
+			queueCharacter(character, malCrawler)
+		}
 
 		// Wait
 		malCrawler.Wait()