web/notify.moe
Code Issues Pull Requests Activity

Updated crawler settings

Browse Source
This commit is contained in:
Eduard Urbach 2018-03-09 00:22:36 +01:00
parent 6038648e3d
commit a4578998ec
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
jobs/mal-crawler/mal-crawler.go
View File

@ -13,8 +13,9 @@ import (
const ( const (
// The maximum age of files we accept until we force a refresh. // The maximum age of files we accept until we force a refresh.
maxAge = 30 * 24 * time.Hour maxAge = 30 * 24 * time.Hour
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36" delayBetweenRequests = 1000 * time.Millisecond
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36"
) )
func main() { func main() {
@ -41,7 +42,7 @@ func main() {
"User-Agent": userAgent, "User-Agent": userAgent,
"Accept-Encoding": "gzip", "Accept-Encoding": "gzip",
}, },
1500*time.Millisecond, delayBetweenRequests,
len(animes), len(animes),
) )