Migration to nano

This commit is contained in:
Eduard Urbach 2017-11-01 10:28:40 +01:00
parent c60f53ed76
commit b0d03fe8ef
6 changed files with 19 additions and 51 deletions

View File

@ -10,6 +10,7 @@ import (
func main() { func main() {
color.Yellow("Refreshing episode information for each anime.") color.Yellow("Refreshing episode information for each anime.")
defer arn.Node.Close()
if InvokeShellArgs() { if InvokeShellArgs() {
return return
@ -19,7 +20,7 @@ func main() {
mediumPriority := []*arn.Anime{} mediumPriority := []*arn.Anime{}
lowPriority := []*arn.Anime{} lowPriority := []*arn.Anime{}
for anime := range arn.MustStreamAnime() { for anime := range arn.StreamAnime() {
if anime.GetMapping("shoboi/anime") == "" { if anime.GetMapping("shoboi/anime") == "" {
continue continue
} }

View File

@ -9,6 +9,7 @@ import (
func main() { func main() {
color.Yellow("Refreshing osu information") color.Yellow("Refreshing osu information")
defer arn.Node.Close()
ticker := time.NewTicker(500 * time.Millisecond) ticker := time.NewTicker(500 * time.Millisecond)

View File

@ -11,6 +11,7 @@ import (
func main() { func main() {
color.Yellow("Updating search index") color.Yellow("Updating search index")
defer arn.Node.Close()
flow.Parallel( flow.Parallel(
updateAnimeIndex, updateAnimeIndex,
@ -26,13 +27,7 @@ func updateAnimeIndex() {
animeSearchIndex := arn.NewSearchIndex() animeSearchIndex := arn.NewSearchIndex()
// Anime // Anime
animeStream, err := arn.StreamAnime() for anime := range arn.StreamAnime() {
if err != nil {
panic(err)
}
for anime := range animeStream {
if anime.Title.Canonical != "" { if anime.Title.Canonical != "" {
animeSearchIndex.TextToID[strings.ToLower(anime.Title.Canonical)] = anime.ID animeSearchIndex.TextToID[strings.ToLower(anime.Title.Canonical)] = anime.ID
} }
@ -64,21 +59,14 @@ func updateAnimeIndex() {
fmt.Println(len(animeSearchIndex.TextToID), "anime titles") fmt.Println(len(animeSearchIndex.TextToID), "anime titles")
// Save in database // Save in database
err = arn.DB.Set("SearchIndex", "Anime", animeSearchIndex) arn.DB.Set("SearchIndex", "Anime", animeSearchIndex)
if err != nil {
panic(err)
}
} }
func updateUserIndex() { func updateUserIndex() {
userSearchIndex := arn.NewSearchIndex() userSearchIndex := arn.NewSearchIndex()
// Users // Users
userStream, err := arn.StreamUsers() for user := range arn.StreamUsers() {
arn.PanicOnError(err)
for user := range userStream {
if user.HasNick() { if user.HasNick() {
userSearchIndex.TextToID[strings.ToLower(user.Nick)] = user.ID userSearchIndex.TextToID[strings.ToLower(user.Nick)] = user.ID
} }
@ -87,36 +75,28 @@ func updateUserIndex() {
fmt.Println(len(userSearchIndex.TextToID), "user names") fmt.Println(len(userSearchIndex.TextToID), "user names")
// Save in database // Save in database
err = arn.DB.Set("SearchIndex", "User", userSearchIndex) arn.DB.Set("SearchIndex", "User", userSearchIndex)
arn.PanicOnError(err)
} }
func updatePostIndex() { func updatePostIndex() {
postSearchIndex := arn.NewSearchIndex() postSearchIndex := arn.NewSearchIndex()
// Users // Users
postStream, err := arn.StreamPosts() for post := range arn.StreamPosts() {
arn.PanicOnError(err)
for post := range postStream {
postSearchIndex.TextToID[strings.ToLower(post.Text)] = post.ID postSearchIndex.TextToID[strings.ToLower(post.Text)] = post.ID
} }
fmt.Println(len(postSearchIndex.TextToID), "posts") fmt.Println(len(postSearchIndex.TextToID), "posts")
// Save in database // Save in database
err = arn.DB.Set("SearchIndex", "Post", postSearchIndex) arn.DB.Set("SearchIndex", "Post", postSearchIndex)
arn.PanicOnError(err)
} }
func updateThreadIndex() { func updateThreadIndex() {
threadSearchIndex := arn.NewSearchIndex() threadSearchIndex := arn.NewSearchIndex()
// Users // Users
threadStream, err := arn.StreamThreads() for thread := range arn.StreamThreads() {
arn.PanicOnError(err)
for thread := range threadStream {
threadSearchIndex.TextToID[strings.ToLower(thread.Title)] = thread.ID threadSearchIndex.TextToID[strings.ToLower(thread.Title)] = thread.ID
threadSearchIndex.TextToID[strings.ToLower(thread.Text)] = thread.ID threadSearchIndex.TextToID[strings.ToLower(thread.Text)] = thread.ID
} }
@ -124,6 +104,5 @@ func updateThreadIndex() {
fmt.Println(len(threadSearchIndex.TextToID)/2, "threads") fmt.Println(len(threadSearchIndex.TextToID)/2, "threads")
// Save in database // Save in database
err = arn.DB.Set("SearchIndex", "Thread", threadSearchIndex) arn.DB.Set("SearchIndex", "Thread", threadSearchIndex)
arn.PanicOnError(err)
} }

View File

@ -1,7 +1,6 @@
package main package main
import ( import (
"encoding/json"
"fmt" "fmt"
"strings" "strings"
@ -12,6 +11,7 @@ import (
func main() { func main() {
color.Yellow("Syncing Anime") color.Yellow("Syncing Anime")
defer arn.Node.Close()
// In case we refresh only one anime // In case we refresh only one anime
if InvokeShellArgs() { if InvokeShellArgs() {
@ -127,19 +127,7 @@ func sync(data *kitsu.Anime) *arn.Anime {
} }
// Save in database // Save in database
err = anime.Save() anime.Save()
status := ""
if err == nil {
status = color.GreenString("✔")
} else {
color.Red(err.Error())
data, _ := json.MarshalIndent(anime, "", "\t")
fmt.Println(string(data))
status = color.RedString("✘")
}
// Episodes // Episodes
episodes, err := arn.GetAnimeEpisodes(anime.ID) episodes, err := arn.GetAnimeEpisodes(anime.ID)
@ -149,7 +137,7 @@ func sync(data *kitsu.Anime) *arn.Anime {
} }
// Log // Log
fmt.Println(status, anime.ID, anime.Title.Canonical) fmt.Println(color.GreenString("✔"), anime.ID, anime.Title.Canonical)
return anime return anime
} }

View File

@ -10,13 +10,14 @@ import (
func main() { func main() {
color.Yellow("Syncing Shoboi Anime") color.Yellow("Syncing Shoboi Anime")
defer arn.Node.Close()
// Priority queues // Priority queues
highPriority := []*arn.Anime{} highPriority := []*arn.Anime{}
mediumPriority := []*arn.Anime{} mediumPriority := []*arn.Anime{}
lowPriority := []*arn.Anime{} lowPriority := []*arn.Anime{}
for anime := range arn.MustStreamAnime() { for anime := range arn.StreamAnime() {
if anime.GetMapping("shoboi/anime") != "" { if anime.GetMapping("shoboi/anime") != "" {
continue continue
} }
@ -51,7 +52,7 @@ func refreshQueue(queue []*arn.Anime) {
for _, anime := range queue { for _, anime := range queue {
if sync(anime) { if sync(anime) {
arn.PanicOnError(anime.Save()) anime.Save()
count++ count++
} }
} }

View File

@ -21,9 +21,7 @@ func main() {
idList := twistAnime.KitsuIDs() idList := twistAnime.KitsuIDs()
// Save index in cache // Save index in cache
arn.DB.Set("Cache", "animetwist index", &arn.ListOfIDs{ arn.DB.Set("IDList", "animetwist index", idList)
IDList: idList,
})
color.Yellow("Refreshing twist.moe links for %d anime", len(idList)) color.Yellow("Refreshing twist.moe links for %d anime", len(idList))