web/notify.moe
Code Issues Pull Requests Activity

Updated MAL tools

Browse Source
This commit is contained in:
Eduard Urbach 2018-10-30 08:59:35 +09:00
parent a9abcf0efb
commit a559227b65
2 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
jobs/mal-download/mal-download.go
View File

@ -15,7 +15,8 @@ const (
// The maximum age of files we accept until we force a refresh. // The maximum age of files we accept until we force a refresh.
maxAge = 24 * time.Hour maxAge = 24 * time.Hour
delayBetweenRequests = 1100 * time.Millisecond delayBetweenRequests = 1100 * time.Millisecond
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.20 Safari/537.36"
animeDirectory = "anime"
) )
var headers = map[string]string{ var headers = map[string]string{
@ -50,7 +51,7 @@ func main() {
arn.Node.Close() arn.Node.Close()
// Create anime directory if it's missing // Create anime directory if it's missing
os.Mkdir("anime", 0777) os.Mkdir(animeDirectory, 0777)
// Create crawler // Create crawler
malCrawler := crawler.New( malCrawler := crawler.New(
@ -80,7 +81,7 @@ func main() {
func queue(anime *arn.Anime, malCrawler *crawler.Crawler) { func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
malID := anime.GetMapping("myanimelist/anime") malID := anime.GetMapping("myanimelist/anime")
url := "https://myanimelist.net/anime/" + malID url := "https://myanimelist.net/anime/" + malID
filePath := fmt.Sprintf("anime/anime-%s.html", malID) filePath := fmt.Sprintf("%s/%s.html.gz", animeDirectory, malID)
fileInfo, err := os.Stat(filePath) fileInfo, err := os.Stat(filePath)
if err == nil && time.Since(fileInfo.ModTime()) <= maxAge { if err == nil && time.Since(fileInfo.ModTime()) <= maxAge {
@ -91,5 +92,6 @@ func queue(anime *arn.Anime, malCrawler *crawler.Crawler) {
malCrawler.Queue(&crawler.Task{ malCrawler.Queue(&crawler.Task{
URL: url, URL: url,
Destination: filePath, Destination: filePath,
Raw: true,
}) })
} }

13
jobs/mal-parse/mal-parse.go
View File

@ -1,6 +1,7 @@
package main package main
import ( import (
"compress/gzip"
"errors" "errors"
"fmt" "fmt"
"os" "os"
@ -36,7 +37,7 @@ func main() {
return nil return nil
} }
if !strings.HasSuffix(name, ".html") { if !strings.HasSuffix(name, ".html.gz") {
return nil return nil
} }
@ -53,7 +54,15 @@ func readFile(name string) error {
} }
defer file.Close() defer file.Close()
anime, characters, err := malparser.ParseAnime(file)
reader, err := gzip.NewReader(file)
if err != nil {
fmt.Println(err)
return err
}
anime, characters, err := malparser.ParseAnime(reader)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)