New tokenizer

This commit is contained in:
Eduard Urbach 2017-07-12 22:45:33 +02:00
parent 9d309b2e8c
commit da57b00ff2

View File

@ -1,7 +1,7 @@
component Japanese(text string)
if arn.ContainsUnicodeLetters(text)
for _, token := range japanese.Tokenize(text)
if token.NeedsFurigana()
for _, token := range arn.JapaneseTokenizer.Tokenize(text)
if token.Furigana
a.japanese(href="http://jisho.org/search/" + token.Original, target="_blank", rel="noopener")
ruby(title=token.Romaji)= token.Original
rt.furigana= token.Hiragana