<commit_before>package controllers

import (
	"encoding/json"
	"fmt"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/jgraham909/revmgo"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"github.com/robfig/revel"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
	revmgo.MongoController
}

type Gloss struct {
	English string
	Tags    []string
	Related []string
	Common  bool
}

type Highlight struct {
	Furigana string
	Japanese string
	Romaji   string
	English  []string
}

type Word struct {
	Romaji    string
	Common    bool
	Dialects  []string
	Fields    []string
	Glosses   []Gloss
	English   []string
	Furigana  string
	Japanese  string
	MainEntry string
	Tags      []string
	Pos       []string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "<strong>" + query + "</strong>"
}

// Wrap the query in <strong> tags so that we can highlight it in the results
func (w *Word) highlightQuery(query string) {
	re := regexp.MustCompile(`\b` + query + `\b`)
	// convert the query to hiragana and katakana. if it's already in
	// hiragana or katakana, it will just be the same.
	kana := kana.NewKana()
	h := kana.Romaji_to_hiragana(query)
	k := kana.Romaji_to_katakana(query)
	// make regular expressions that match the hiragana and katakana
	hiraganaRe := regexp.MustCompile(h)
	katakanaRe := regexp.MustCompile(k)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	katakanaHighlighted := makeStrong(k)<commit_msg>add function to convert query to hiragana and katakana<commit_after>package controllers

import (
	"encoding/json"
	"fmt"
	"github.com/gojp/kana"
	"github.com/gojp/nihongo/app/models"
	"github.com/gojp/nihongo/app/routes"
	"github.com/jgraham909/revmgo"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"github.com/robfig/revel"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"log"
	"regexp"
	"strings"
)

type App struct {
	*revel.Controller
	revmgo.MongoController
}

type Gloss struct {
	English string
	Tags    []string
	Related []string
	Common  bool
}

type Highlight struct {
	Furigana string
	Japanese string
	Romaji   string
	English  []string
}

type Word struct {
	Romaji    string
	Common    bool
	Dialects  []string
	Fields    []string
	Glosses   []Gloss
	English   []string
	Furigana  string
	Japanese  string
	MainEntry string
	Tags      []string
	Pos       []string
}

// wrap a string in strong tags
func makeStrong(query string) string {
	return "<strong>" + query + "</strong>"
}

// convert the query to hiragana and katakana. if it's already in
// hiragana or katakana, it will just be the same.
func convertQueryToKana(query string) (hiragana, katakana string) {
	kana := kana.NewKana()
	h := kana.Romaji_to_hiragana(query)
	k := kana.Romaji_to_katakana(query)
	return h, k
}

// Wrap the query in <strong> tags so that we can highlight it in the results
func (w *Word) highlightQuery(query string) {
	// make regular expression that matches the original query
	re := regexp.MustCompile(`\b` + query + `\b`)
	// convert original query to kana
	h, k := convertQueryToKana(query)
	// make regular expressions that match the hiragana and katakana
	hiraganaRe := regexp.MustCompile(h)
	katakanaRe := regexp.MustCompile(k)
	// wrap the query in strong tags
	queryHighlighted := makeStrong(query)
	katakanaHighlighted := makeStrong(k)
{'fraction_non_alphanumeric': 0.10888825378679623, 'fraction_numerical': 0.002857959416976279, 'mean_word_length': 3.888268156424581, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2604649', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1291, 'n_words': 344}