diff --git a/jobs/mal-download/mal-download.go b/jobs/mal-download/mal-download.go index d80a9755..eff0274a 100644 --- a/jobs/mal-download/mal-download.go +++ b/jobs/mal-download/mal-download.go @@ -13,12 +13,24 @@ import ( const ( // The maximum age of files we accept until we force a refresh. - maxAge = 7 * 24 * time.Hour + maxAge = 24 * time.Hour delayBetweenRequests = 1100 * time.Millisecond userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36" ) +var headers = map[string]string{ + "User-Agent": userAgent, + "Accept-Encoding": "gzip", +} + func main() { + defer color.Green("Finished.") + + // Called with arguments? + if InvokeShellArgs() { + return + } + // Filter anime with MAL ID animes := []*arn.Anime{} @@ -42,10 +54,7 @@ func main() { // Create crawler malCrawler := crawler.New( - map[string]string{ - "User-Agent": userAgent, - "Accept-Encoding": "gzip", - }, + headers, delayBetweenRequests, len(animes), ) @@ -57,21 +66,7 @@ func main() { count := 0 for _, anime := range animes { - malID := anime.GetMapping("myanimelist/anime") - url := "https://myanimelist.net/anime/" + malID - filePath := fmt.Sprintf("files/anime-%s.html", malID) - fileInfo, err := os.Stat(filePath) - - if err == nil && time.Since(fileInfo.ModTime()) <= maxAge { - // fmt.Println(color.YellowString(url), "skip") - continue - } - - malCrawler.Queue(&crawler.Task{ - URL: url, - Destination: filePath, - }) - + queue(anime, malCrawler) count++ } @@ -80,7 +75,21 @@ func main() { // Wait for completion malCrawler.Wait() - - // Finished - color.Green("Finished.") +} + +func queue(anime *arn.Anime, malCrawler *crawler.Crawler) { + malID := anime.GetMapping("myanimelist/anime") + url := "https://myanimelist.net/anime/" + malID + filePath := fmt.Sprintf("files/anime-%s.html", malID) + fileInfo, err := os.Stat(filePath) + + if err == nil && time.Since(fileInfo.ModTime()) <= maxAge { + // fmt.Println(color.YellowString(url), "skip") + return + } + + malCrawler.Queue(&crawler.Task{ + URL: url, + Destination: filePath, + }) } diff --git a/jobs/mal-download/shell.go b/jobs/mal-download/shell.go new file mode 100644 index 00000000..748ed976 --- /dev/null +++ b/jobs/mal-download/shell.go @@ -0,0 +1,44 @@ +package main + +import ( + "flag" + + "github.com/aerogo/crawler" + "github.com/animenotifier/arn" +) + +// Shell parameters +var animeID string + +// Shell flags +func init() { + flag.StringVar(&animeID, "id", "", "ID of the notify.moe anime you want to refresh") + flag.Parse() +} + +// InvokeShellArgs ... +func InvokeShellArgs() bool { + if animeID != "" { + anime, err := arn.GetAnime(animeID) + + if err != nil { + panic(err) + } + + // Create crawler + malCrawler := crawler.New( + headers, + delayBetweenRequests, + 1, + ) + + // Queue + queue(anime, malCrawler) + + // Wait + malCrawler.Wait() + return true + } + + return false +}