From a4578998ecd38b32445c123f15872fa52b107d17 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Fri, 9 Mar 2018 00:22:36 +0100 Subject: [PATCH] Updated crawler settings --- jobs/mal-crawler/mal-crawler.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jobs/mal-crawler/mal-crawler.go b/jobs/mal-crawler/mal-crawler.go index 2c68eddb..34ed121c 100644 --- a/jobs/mal-crawler/mal-crawler.go +++ b/jobs/mal-crawler/mal-crawler.go @@ -13,8 +13,9 @@ import ( const ( // The maximum age of files we accept until we force a refresh. - maxAge = 30 * 24 * time.Hour - userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36" + maxAge = 30 * 24 * time.Hour + delayBetweenRequests = 1000 * time.Millisecond + userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.166 Safari/537.36" ) func main() { @@ -41,7 +42,7 @@ func main() { "User-Agent": userAgent, "Accept-Encoding": "gzip", }, - 1500*time.Millisecond, + delayBetweenRequests, len(animes), )