diff --git a/jobs/crawler/crawler.go b/jobs/crawler/crawler.go index 8e8afd93..1696283e 100644 --- a/jobs/crawler/crawler.go +++ b/jobs/crawler/crawler.go @@ -1,20 +1,38 @@ package main import ( + "fmt" "time" + "github.com/animenotifier/arn" + "github.com/fatih/color" + "github.com/aerogo/crawler" ) const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36" func main() { + defer arn.Node.Close() + malCrawler := crawler.New(userAgent, 1*time.Second, 20000) + count := 0 - malCrawler.Queue(&crawler.Task{ - URL: "https://github.com/animenotifier/notify.moe", - Destination: "file.html", - }) + for anime := range arn.StreamAnime() { + malID := anime.GetMapping("myanimelist/anime") + if malID == "" { + continue + } + + malCrawler.Queue(&crawler.Task{ + URL: "https://myanimelist.net/anime/" + malID, + Destination: fmt.Sprintf("mal/anime-%s.html", malID), + }) + + count++ + } + + color.Yellow("Queued up %d links", count) malCrawler.Wait() }