From 24fa31171aec338dc0893bd268aa84e524520860 Mon Sep 17 00:00:00 2001
From: Eduard Urbach <e.urbach@gmail.com>
Date: Tue, 13 Nov 2018 16:22:24 +0900
Subject: [PATCH] MAL download can now filter new ones only

---
 jobs/mal-download/mal-download.go | 30 ++++++++++++++++++++++++++----
 jobs/mal-download/shell.go        |  2 ++
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/jobs/mal-download/mal-download.go b/jobs/mal-download/mal-download.go
index 23cd456b..e869445c 100644
--- a/jobs/mal-download/mal-download.go
+++ b/jobs/mal-download/mal-download.go
@@ -5,6 +5,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/animenotifier/arn/osutils"
+
 	"github.com/animenotifier/arn"
 	"github.com/fatih/color"
 
@@ -42,7 +44,13 @@ func main() {
 
 	if objectType == "all" || objectType == "anime" {
 		animes = arn.FilterAnime(func(anime *arn.Anime) bool {
-			return anime.GetMapping("myanimelist/anime") != ""
+			malID := anime.GetMapping("myanimelist/anime")
+
+			if malID == "" {
+				return false
+			}
+
+			return !newOnly || !osutils.Exists(animeFilePath(malID))
 		})
 
 		color.Yellow("Found %d anime", len(animes))
@@ -56,7 +64,13 @@ func main() {
 
 	if objectType == "all" || objectType == "character" {
 		characters = arn.FilterCharacters(func(character *arn.Character) bool {
-			return character.GetMapping("myanimelist/character") != ""
+			malID := character.GetMapping("myanimelist/character")
+
+			if malID == "" {
+				return false
+			}
+
+			return !newOnly || !osutils.Exists(characterFilePath(malID))
 		})
 
 		color.Yellow("Found %d characters", len(characters))
@@ -95,10 +109,18 @@ func main() {
 	malCrawler.Wait()
 }
 
+func animeFilePath(malID string) string {
+	return fmt.Sprintf("%s/%s.html.gz", animeDirectory, malID)
+}
+
+func characterFilePath(malID string) string {
+	return fmt.Sprintf("%s/%s.html.gz", characterDirectory, malID)
+}
+
 func queueAnime(anime *arn.Anime, malCrawler *crawler.Crawler) {
 	malID := anime.GetMapping("myanimelist/anime")
 	url := "https://myanimelist.net/anime/" + malID
-	filePath := fmt.Sprintf("%s/%s.html.gz", animeDirectory, malID)
+	filePath := animeFilePath(malID)
 	fileInfo, err := os.Stat(filePath)
 
 	if err == nil && time.Since(fileInfo.ModTime()) <= maxAge {
@@ -116,7 +138,7 @@ func queueAnime(anime *arn.Anime, malCrawler *crawler.Crawler) {
 func queueCharacter(character *arn.Character, malCrawler *crawler.Crawler) {
 	malID := character.GetMapping("myanimelist/character")
 	url := "https://myanimelist.net/character/" + malID
-	filePath := fmt.Sprintf("%s/%s.html.gz", characterDirectory, malID)
+	filePath := characterFilePath(malID)
 	fileInfo, err := os.Stat(filePath)
 
 	if err == nil && time.Since(fileInfo.ModTime()) <= maxAge {
diff --git a/jobs/mal-download/shell.go b/jobs/mal-download/shell.go
index bd6f72ff..1d5a5068 100644
--- a/jobs/mal-download/shell.go
+++ b/jobs/mal-download/shell.go
@@ -10,11 +10,13 @@ import (
 // Shell parameters
 var objectType string
 var objectID string
+var newOnly bool
 
 // Shell flags
 func init() {
 	flag.StringVar(&objectType, "type", "all", "all | anime | character")
 	flag.StringVar(&objectID, "id", "", "ID of the notify.moe anime/character you want to refresh")
+	flag.BoolVar(&newOnly, "new", false, "Skip existing entries and only download new ones")
 	flag.Parse()
 }