Improved search

This commit is contained in:
Eduard Urbach 2017-07-20 14:26:43 +02:00
parent ecb9e4ad86
commit 5ea14e8fc1
10 changed files with 115 additions and 46 deletions

View File

@ -31,12 +31,12 @@ var jobs = map[string]time.Duration{
"popular-anime": 20 * time.Minute, "popular-anime": 20 * time.Minute,
"avatars": 30 * time.Minute, "avatars": 30 * time.Minute,
"twist": 1 * time.Hour, "twist": 1 * time.Hour,
"search-index": 2 * time.Hour,
"sync-shoboi": 8 * time.Hour, "sync-shoboi": 8 * time.Hour,
"refresh-episodes": 10 * time.Hour, "refresh-episodes": 10 * time.Hour,
"refresh-track-titles": 10 * time.Hour, "refresh-track-titles": 10 * time.Hour,
"refresh-osu": 12 * time.Hour, "refresh-osu": 12 * time.Hour,
"sync-anime": 12 * time.Hour, "sync-anime": 12 * time.Hour,
"search-index": 12 * time.Hour,
} }
func main() { func main() {

View File

@ -12,7 +12,12 @@ import (
func main() { func main() {
color.Yellow("Updating search index") color.Yellow("Updating search index")
flow.Parallel(updateAnimeIndex, updateUserIndex) flow.Parallel(
updateAnimeIndex,
updateUserIndex,
updatePostIndex,
updateThreadIndex,
)
color.Green("Finished.") color.Green("Finished.")
} }
@ -71,10 +76,7 @@ func updateUserIndex() {
// Users // Users
userStream, err := arn.StreamUsers() userStream, err := arn.StreamUsers()
arn.PanicOnError(err)
if err != nil {
panic(err)
}
for user := range userStream { for user := range userStream {
if user.HasNick() { if user.HasNick() {
@ -86,8 +88,42 @@ func updateUserIndex() {
// Save in database // Save in database
err = arn.DB.Set("SearchIndex", "User", userSearchIndex) err = arn.DB.Set("SearchIndex", "User", userSearchIndex)
arn.PanicOnError(err)
if err != nil { }
panic(err)
} func updatePostIndex() {
postSearchIndex := arn.NewSearchIndex()
// Users
postStream, err := arn.StreamPosts()
arn.PanicOnError(err)
for post := range postStream {
postSearchIndex.TextToID[strings.ToLower(post.Text)] = post.ID
}
fmt.Println(len(postSearchIndex.TextToID), "posts")
// Save in database
err = arn.DB.Set("SearchIndex", "Post", postSearchIndex)
arn.PanicOnError(err)
}
func updateThreadIndex() {
threadSearchIndex := arn.NewSearchIndex()
// Users
threadStream, err := arn.StreamThreads()
arn.PanicOnError(err)
for thread := range threadStream {
threadSearchIndex.TextToID[strings.ToLower(thread.Title)] = thread.ID
threadSearchIndex.TextToID[strings.ToLower(thread.Text)] = thread.ID
}
fmt.Println(len(threadSearchIndex.TextToID)/2, "threads")
// Save in database
err = arn.DB.Set("SearchIndex", "Thread", threadSearchIndex)
arn.PanicOnError(err)
} }

View File

@ -11,7 +11,7 @@ import (
const requestThreshold = 10 const requestThreshold = 10
var ipToStats = cache.New(30*time.Minute, 15*time.Minute) var ipToStats = cache.New(15*time.Minute, 15*time.Minute)
// IPStats captures the statistics for a single IP. // IPStats captures the statistics for a single IP.
type IPStats struct { type IPStats struct {

View File

@ -13,9 +13,7 @@
.anime-list-item-name .anime-list-item-name
flex 1 flex 1
white-space nowrap clip-long-text
text-overflow ellipsis
overflow hidden
.anime-list-item-episodes .anime-list-item-episodes
horizontal horizontal

View File

@ -1,8 +1,6 @@
.schedule-item-link, .schedule-item-link,
.schedule-item-title .schedule-item-title
white-space nowrap clip-long-text
text-overflow ellipsis
overflow hidden
.schedule-item-link .schedule-item-link
horizontal horizontal

View File

@ -8,11 +8,13 @@ import (
const maxUsers = 6 * 6 const maxUsers = 6 * 6
const maxAnime = 5 * 6 const maxAnime = 5 * 6
const maxPosts = 3
const maxThreads = 3
// Get search page. // Get search page.
func Get(ctx *aero.Context) string { func Get(ctx *aero.Context) string {
term := ctx.Query("q") term := ctx.Query("q")
userResults, animeResults := arn.Search(term, maxUsers, maxAnime) userResults, animeResults, postResults, threadResults := arn.Search(term, maxUsers, maxAnime, maxPosts, maxThreads)
return ctx.HTML(components.SearchResults(term, userResults, animeResults)) return ctx.HTML(components.SearchResults(term, userResults, animeResults, postResults, threadResults))
} }

View File

@ -1,4 +1,4 @@
component SearchResults(term string, users []*arn.User, animeResults []*arn.Anime) component SearchResults(term string, users []*arn.User, animeResults []*arn.Anime, postResults []*arn.Post, threadResults []*arn.Thread)
h1.page-title= "Search: " + term h1.page-title= "Search: " + term
.widgets .widgets
@ -32,9 +32,26 @@ component SearchResults(term string, users []*arn.User, animeResults []*arn.Anim
.widget .widget
h3.widget-title h3.widget-title
Icon("comment") Icon("comment")
span Forums span Forum
p.no-search-results.mountable Forums search coming soon. if len(postResults) == 0 && len(threadResults) == 0
p.no-search-results.mountable No posts found.
else
each thread in threadResults
.mountable(data-mountable-type="forum")
.forum-search-result
a.forum-search-result-title.ajax(href=thread.Link())= thread.Title
if thread.Author().HasNick()
.forum-search-result-author= thread.Author().Nick
.forum-search-result-sample= thread.Text
each post in postResults
.mountable(data-mountable-type="forum")
.forum-search-result
a.forum-search-result-title.ajax(href=post.Link(), data-mountable-type="forum")= post.Thread().Title
if post.Author().HasNick()
.forum-search-result-author= post.Author().Nick
.forum-search-result-sample= post.Text
.widget .widget
h3.widget-title h3.widget-title

View File

@ -2,5 +2,21 @@
width 55px !important width 55px !important
height 78px !important height 78px !important
.forum-search-result
horizontal
.forum-search-result-title
flex 1
clip-long-text
.forum-search-result-author
text-align right
opacity 0.5
.forum-search-result-sample
clip-long-text
margin-bottom 1rem
opacity 0.8
.no-search-results .no-search-results
text-align left text-align left

View File

@ -488,11 +488,9 @@ export class AnimeNotifier {
let time = 0 let time = 0
let start = Date.now() let start = Date.now()
let maxTime = start + maxDelay let maxTime = start + maxDelay
let mutations = []
let mountableTypes = { let mountableTypes = new Map<string, number>()
general: start let mountableTypeMutations = new Map<string, Array<any>>()
}
let collection = document.getElementsByClassName(className) let collection = document.getElementsByClassName(className)
@ -506,43 +504,49 @@ export class AnimeNotifier {
let element = collection.item(i) as HTMLElement let element = collection.item(i) as HTMLElement
let type = element.dataset.mountableType || "general" let type = element.dataset.mountableType || "general"
if(type in mountableTypes) { if(mountableTypes.has(type)) {
time = mountableTypes[type] += delay time = mountableTypes.get(type) + delay
mountableTypes.set(type, time)
} else { } else {
time = mountableTypes[type] = start time = start
mountableTypes.set(type, time)
mountableTypeMutations.set(type, [])
} }
if(time > maxTime) { if(time > maxTime) {
time = maxTime time = maxTime
} }
mutations.push({ mountableTypeMutations.get(type).push({
element, element,
time time
}) })
} }
let mutationIndex = 0 for(let mountableType of mountableTypeMutations.keys()) {
let mutations = mountableTypeMutations.get(mountableType)
let mutationIndex = 0
let updateBatch = () => { let updateBatch = () => {
let now = Date.now() let now = Date.now()
for(; mutationIndex < mutations.length; mutationIndex++) { for(; mutationIndex < mutations.length; mutationIndex++) {
let mutation = mutations[mutationIndex] let mutation = mutations[mutationIndex]
if(mutation.time > now) { if(mutation.time > now) {
break break
}
func(mutation.element)
} }
func(mutation.element) if(mutationIndex < mutations.length) {
window.requestAnimationFrame(updateBatch)
}
} }
if(mutationIndex < mutations.length) { window.requestAnimationFrame(updateBatch)
window.requestAnimationFrame(updateBatch)
}
} }
window.requestAnimationFrame(updateBatch)
} }
diff(url: string) { diff(url: string) {

View File

@ -23,9 +23,7 @@ export class Diff {
} }
Diff.rootContainer.innerHTML = html.replace("<!DOCTYPE html>", "") Diff.rootContainer.innerHTML = html.replace("<!DOCTYPE html>", "")
console.log(aRoot.getElementsByTagName("body")[0]) Diff.childNodes(aRoot, Diff.rootContainer)
console.log(Diff.rootContainer.getElementsByTagName("body")[0])
Diff.childNodes(aRoot.getElementsByTagName("body")[0], Diff.rootContainer.getElementsByTagName("body")[0])
} }
// childNodes diffs the child nodes of 2 given elements and applies DOM mutations. // childNodes diffs the child nodes of 2 given elements and applies DOM mutations.