39 lines
728 B
Go
39 lines
728 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/animenotifier/arn"
|
|
"github.com/fatih/color"
|
|
|
|
"github.com/aerogo/crawler"
|
|
)
|
|
|
|
const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36"
|
|
|
|
func main() {
|
|
defer arn.Node.Close()
|
|
|
|
malCrawler := crawler.New(userAgent, 1*time.Second, 20000)
|
|
count := 0
|
|
|
|
for anime := range arn.StreamAnime() {
|
|
malID := anime.GetMapping("myanimelist/anime")
|
|
|
|
if malID == "" {
|
|
continue
|
|
}
|
|
|
|
malCrawler.Queue(&crawler.Task{
|
|
URL: "https://myanimelist.net/anime/" + malID,
|
|
Destination: fmt.Sprintf("mal/anime-%s.html", malID),
|
|
})
|
|
|
|
count++
|
|
}
|
|
|
|
color.Yellow("Queued up %d links", count)
|
|
malCrawler.Wait()
|
|
}
|