2017-06-20 21:27:14 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2017-06-20 22:34:18 +00:00
|
|
|
"fmt"
|
2017-06-20 21:27:14 +00:00
|
|
|
"strings"
|
|
|
|
|
2017-06-23 16:22:48 +00:00
|
|
|
"github.com/aerogo/flow"
|
2017-06-20 21:27:14 +00:00
|
|
|
"github.com/animenotifier/arn"
|
2017-06-20 22:34:18 +00:00
|
|
|
"github.com/fatih/color"
|
2017-06-20 21:27:14 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func main() {
|
2017-06-20 22:34:18 +00:00
|
|
|
color.Yellow("Updating search index")
|
2017-11-01 09:28:40 +00:00
|
|
|
defer arn.Node.Close()
|
2017-06-20 22:34:18 +00:00
|
|
|
|
2017-07-20 12:26:43 +00:00
|
|
|
flow.Parallel(
|
|
|
|
updateAnimeIndex,
|
|
|
|
updateUserIndex,
|
|
|
|
updatePostIndex,
|
|
|
|
updateThreadIndex,
|
|
|
|
)
|
2017-06-20 22:34:18 +00:00
|
|
|
|
|
|
|
color.Green("Finished.")
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func updateAnimeIndex() {
|
|
|
|
animeSearchIndex := arn.NewSearchIndex()
|
|
|
|
|
|
|
|
// Anime
|
2017-11-01 09:28:40 +00:00
|
|
|
for anime := range arn.StreamAnime() {
|
2017-07-04 12:34:58 +00:00
|
|
|
if anime.Title.Canonical != "" {
|
|
|
|
animeSearchIndex.TextToID[strings.ToLower(anime.Title.Canonical)] = anime.ID
|
2017-06-28 00:51:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-04 12:34:58 +00:00
|
|
|
if anime.Title.Romaji != "" {
|
|
|
|
animeSearchIndex.TextToID[strings.ToLower(anime.Title.Romaji)] = anime.ID
|
2017-06-20 22:34:18 +00:00
|
|
|
}
|
|
|
|
|
2017-07-04 12:34:58 +00:00
|
|
|
// Make sure we only include Japanese titles that
|
|
|
|
// don't overlap with the English titles.
|
|
|
|
if anime.Title.Japanese != "" && animeSearchIndex.TextToID[strings.ToLower(anime.Title.Japanese)] == "" {
|
2017-06-29 23:14:24 +00:00
|
|
|
animeSearchIndex.TextToID[strings.ToLower(anime.Title.Japanese)] = anime.ID
|
|
|
|
}
|
|
|
|
|
2017-07-04 12:34:58 +00:00
|
|
|
// Same with English titles, don't overwrite other stuff.
|
|
|
|
if anime.Title.English != "" && animeSearchIndex.TextToID[strings.ToLower(anime.Title.English)] == "" {
|
|
|
|
animeSearchIndex.TextToID[strings.ToLower(anime.Title.English)] = anime.ID
|
2017-07-01 22:49:05 +00:00
|
|
|
}
|
|
|
|
|
2017-06-29 23:14:24 +00:00
|
|
|
for _, synonym := range anime.Title.Synonyms {
|
|
|
|
synonym = strings.ToLower(synonym)
|
|
|
|
|
|
|
|
if synonym != "" && len(synonym) <= 10 {
|
|
|
|
animeSearchIndex.TextToID[synonym] = anime.ID
|
|
|
|
}
|
2017-06-20 22:34:18 +00:00
|
|
|
}
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|
|
|
|
|
2017-06-20 22:34:18 +00:00
|
|
|
fmt.Println(len(animeSearchIndex.TextToID), "anime titles")
|
|
|
|
|
2017-06-20 21:27:14 +00:00
|
|
|
// Save in database
|
2017-11-01 09:28:40 +00:00
|
|
|
arn.DB.Set("SearchIndex", "Anime", animeSearchIndex)
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func updateUserIndex() {
|
|
|
|
userSearchIndex := arn.NewSearchIndex()
|
|
|
|
|
|
|
|
// Users
|
2017-11-01 09:28:40 +00:00
|
|
|
for user := range arn.StreamUsers() {
|
2017-07-19 01:45:30 +00:00
|
|
|
if user.HasNick() {
|
2017-06-20 22:34:18 +00:00
|
|
|
userSearchIndex.TextToID[strings.ToLower(user.Nick)] = user.ID
|
|
|
|
}
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|
|
|
|
|
2017-06-20 22:34:18 +00:00
|
|
|
fmt.Println(len(userSearchIndex.TextToID), "user names")
|
|
|
|
|
2017-06-20 21:27:14 +00:00
|
|
|
// Save in database
|
2017-11-01 09:28:40 +00:00
|
|
|
arn.DB.Set("SearchIndex", "User", userSearchIndex)
|
2017-07-20 12:26:43 +00:00
|
|
|
}
|
2017-06-20 21:27:14 +00:00
|
|
|
|
2017-07-20 12:26:43 +00:00
|
|
|
func updatePostIndex() {
|
|
|
|
postSearchIndex := arn.NewSearchIndex()
|
|
|
|
|
|
|
|
// Users
|
2017-11-01 09:28:40 +00:00
|
|
|
for post := range arn.StreamPosts() {
|
2017-07-20 12:26:43 +00:00
|
|
|
postSearchIndex.TextToID[strings.ToLower(post.Text)] = post.ID
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Println(len(postSearchIndex.TextToID), "posts")
|
|
|
|
|
|
|
|
// Save in database
|
2017-11-01 09:28:40 +00:00
|
|
|
arn.DB.Set("SearchIndex", "Post", postSearchIndex)
|
2017-07-20 12:26:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func updateThreadIndex() {
|
|
|
|
threadSearchIndex := arn.NewSearchIndex()
|
|
|
|
|
|
|
|
// Users
|
2017-11-01 09:28:40 +00:00
|
|
|
for thread := range arn.StreamThreads() {
|
2017-07-20 12:26:43 +00:00
|
|
|
threadSearchIndex.TextToID[strings.ToLower(thread.Title)] = thread.ID
|
|
|
|
threadSearchIndex.TextToID[strings.ToLower(thread.Text)] = thread.ID
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|
2017-07-20 12:26:43 +00:00
|
|
|
|
|
|
|
fmt.Println(len(threadSearchIndex.TextToID)/2, "threads")
|
|
|
|
|
|
|
|
// Save in database
|
2017-11-01 09:28:40 +00:00
|
|
|
arn.DB.Set("SearchIndex", "Thread", threadSearchIndex)
|
2017-06-20 21:27:14 +00:00
|
|
|
}
|