<commit_before># -*- coding: utf-8 -*-

"""
Japanese Multilingual Named Entity Dictionary (JMnedict) in SQLite format

References:
    ENAMDICT/JMnedict - Japanese Proper Names Dictionary Files
        https://www.edrdg.org/enamdict/enamdict_doc.html
"""

# This code is a part of jamdict library: https://github.com/neocl/jamdict
# :copyright: (c) 2020 <NAME> <<EMAIL>>
# :license: MIT, see LICENSE for more details.

import os
import logging
from typing import Sequence

from puchikarui import Schema
from . import __version__ as JAMDICT_VERSION, __url__ as JAMDICT_URL
from .jmdict import Meta, JMDEntry, KanjiForm, KanaForm, Translation, SenseGloss

# -------------------------------------------------------------------------------
# Configuration
# -------------------------------------------------------------------------------

MY_FOLDER = os.path.dirname(os.path.abspath(__file__))
SCRIPT_FOLDER = os.path.join(MY_FOLDER, 'data')
JMNEDICT_SETUP_FILE = os.path.join(SCRIPT_FOLDER, 'setup_jmnedict.sql')
JMNEDICT_VERSION = '1.08'
JMNEDICT_URL = 'https://www.edrdg.org/enamdict/enamdict_doc.html'
JMNEDICT_DATE = '2020-05-29'
JMNEDICT_SETUP_SCRIPT = '''INSERT INTO meta VALUES ('jmnedict.version', '{jv}');
INSERT INTO meta VALUES ('jmnedict.url', '{ju}');
INSERT INTO meta VALUES ('jmnedict.date', '{jud}');
INSERT INTO meta SELECT 'generator', 'jamdict' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator');
INSERT INTO meta SELECT 'generator_version', '{gv}' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator_version');
INSERT INTO meta SELECT 'generator_url', '{gu}' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator_url');'''.format(
    jv=JMNEDICT_VERSION,
    ju=JMNEDICT_URL,
    jud=JMNEDICT_DATE,
    gv=JAMDICT_VERSION,
    gu=JAMDICT_URL
)


def getLogger():
    return logging.getLogger(__name__)


# -------------------------------------------------------------------------------
# Models
# -------------------------------------------------------------------------------

class JMNEDictSchema(Schema):

    def __init__(self, db_path, *args, **kwargs):
        super().__init__(db_path, *args, **kwargs)
        self.add_script(JMNEDICT_SETUP_SCRIPT)
        self.add_file(JMNEDICT_SETUP_FILE)
        # Meta
        self.add_table('meta', ['key', 'value'], proto=Meta).set_id('key')
        self.add_table('NEEntry', ['idseq'])
        # Kanji
        self.add_table('NEKanji', ['ID', 'idseq', 'text'])
        # Kana
        self.add_table('NEKana', ['ID', 'idseq', 'text', 'nokanji'])
        # Translation (~Sense of JMdict)
        self.add_table('NETranslation', ['ID', 'idseq'])
        self.add_table('NETransType', ['tid', 'text'])
        self.add_table('NETransXRef', ['tid', 'text'])
        self.add_table('NETransGloss', ['tid', 'lang', 'gend', 'text'])


class JMNEDictSQLite(JMNEDictSchema):

    def __init__(self, db_path, *args, **kwargs):
        super().__init__(db_path, *args, **kwargs)

    def all_ne_type(self, ctx=None):
        if ctx is None:
            return self.all_ne_type(ctx=self.ctx())
        else:
            return [x['text'] for x in ctx.execute("SELECT DISTINCT text FROM NETransType")]
        
    def search_ne(self, query, ctx=None, **kwargs) -> Sequence[JMDEntry]:
        # ensure context
        if ctx is None:
            with self.ctx() as ctx:
                return self.search_ne(query, ctx=ctx)
        _is_wildcard_search = '_' in query or '@' in query or '%' in query
        if _is_wildcard_search:
            where = "idseq IN (SELECT idseq FROM NEKanji WHERE text like ?) OR idseq IN (SELECT idseq FROM NEKana WHERE text like ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransGloss ON NETranslation.ID == NETransGloss.tid WHERE NETransGloss.text like ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransType ON NETranslation.ID == NETransType.tid WHERE NETransType.text like ?)"
        else:
            where = "idseq IN (SELECT idseq FROM NEKanji WHERE text == ?) OR idseq IN (SELECT idseq FROM NEKana WHERE text == ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransGloss ON NETranslation.ID == NETransGloss.tid WHERE NETransGloss.text == ?) or idseq in (SELECT idseq FROM NETranslation JOIN NETransType ON NETranslation.ID == NETransType.tid WHERE NETransType.text == ?)"
        getLogger().debug(where)
        params = [query, query, query, query]
        try:
            if query.startswith('id#'):
                query_int = int(query[3:])
                if query_int >= 0:
                    getLogger().debug("Searching NE by ID: {}".format(query_int))
                    where = "idseq = ?"
                    params = [query_int]
        except Exception:
            pass
        # else (a context is provided)
        eids = self.NEEntry.select(where, params, ctx=ctx)
        entries = []
        for e in eids:
            entries.append(self.get_ne(e.idseq, ctx=ctx))
        return entries

    def get_ne(self, idseq, ctx=None) -> JMDEntry:
        # ensure context
        if ctx is None:
            with self.ctx() as new_context:
                return self.get_entry(idseq, new_context)
        # else (a context is provided)
        # select entry & info
        entry = JMDEntry(idseq)<commit_msg>add search_ne_iter()<commit_after># -*- coding: utf-8 -*-

"""
Japanese Multilingual Named Entity Dictionary (JMnedict) in SQLite format

References:
    ENAMDICT/JMnedict - Japanese Proper Names Dictionary Files
        https://www.edrdg.org/enamdict/enamdict_doc.html
"""

# This code is a part of jamdict library: https://github.com/neocl/jamdict
# :copyright: (c) 2020 <NAME> <<EMAIL>>
# :license: MIT, see LICENSE for more details.

import os
import logging
from typing import Sequence

from puchikarui import Schema
from . import __version__ as JAMDICT_VERSION, __url__ as JAMDICT_URL
from .jmdict import Meta, JMDEntry, KanjiForm, KanaForm, Translation, SenseGloss

# -------------------------------------------------------------------------------
# Configuration
# -------------------------------------------------------------------------------

MY_FOLDER = os.path.dirname(os.path.abspath(__file__))
SCRIPT_FOLDER = os.path.join(MY_FOLDER, 'data')
JMNEDICT_SETUP_FILE = os.path.join(SCRIPT_FOLDER, 'setup_jmnedict.sql')
JMNEDICT_VERSION = '1.08'
JMNEDICT_URL = 'https://www.edrdg.org/enamdict/enamdict_doc.html'
JMNEDICT_DATE = '2020-05-29'
JMNEDICT_SETUP_SCRIPT = '''INSERT INTO meta VALUES ('jmnedict.version', '{jv}');
INSERT INTO meta VALUES ('jmnedict.url', '{ju}');
INSERT INTO meta VALUES ('jmnedict.date', '{jud}');
INSERT INTO meta SELECT 'generator', 'jamdict' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator');
INSERT INTO meta SELECT 'generator_version', '{gv}' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator_version');
INSERT INTO meta SELECT 'generator_url', '{gu}' WHERE NOT EXISTS (SELECT 1 FROM meta WHERE key = 'generator_url');'''.format(
    jv=JMNEDICT_VERSION,
    ju=JMNEDICT_URL,
    jud=JMNEDICT_DATE,
    gv=JAMDICT_VERSION,
    gu=JAMDICT_URL
)


def getLogger():
    return logging.getLogger(__name__)


# -------------------------------------------------------------------------------
# Models
# -------------------------------------------------------------------------------

class JMNEDictSchema(Schema):

    def __init__(self, db_path, *args, **kwargs):
        super().__init__(db_path, *args, **kwargs)
        self.add_script(JMNEDICT_SETUP_SCRIPT)
        self.add_file(JMNEDICT_SETUP_FILE)
        # Meta
        self.add_table('meta', ['key', 'value'], proto=Meta).set_id('key')
        self.add_table('NEEntry', ['idseq'])
        # Kanji
        self.add_table('NEKanji', ['ID', 'idseq', 'text'])
        # Kana
        self.add_table('NEKana', ['ID', 'idseq', 'text', 'nokanji'])
        # Translation (~Sense of JMdict)
        self.add_table('NETranslation', ['ID', 'idseq'])
        self.add_table('NETransType', ['tid', 'text'])
        self.add_table('NETransXRef', ['tid', 'text'])
        self.add_table('NETransGloss', ['tid', 'lang', 'gend', 'text'])


class JMNEDictSQLite(JMNEDictSchema):

    def __init__(self, db_path, *args, **kwargs):
        super().__init__(db_path, *args, **kwargs)

    def all_ne_type(self, ctx=None):
        if ctx is None:
            return self.all_ne_type(ctx=self.ctx())
        else:
            return [x['text'] for x in ctx.execute("SELECT DISTINCT text FROM NETransType")]

    def _build_ne_search_query(self, query):
        _is_wildcard_search = '_' in query or '@' in query or '%' in query
        if _is_wildcard_search:
            where = "idseq IN (SELECT idseq FROM NEKanji WHERE text like ?) OR idseq IN (SELECT idseq FROM NEKana WHERE text like ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransGloss ON NETranslation.ID == NETransGloss.tid WHERE NETransGloss.text like ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransType ON NETranslation.ID == NETransType.tid WHERE NETransType.text like ?)"
        else:
            where = "idseq IN (SELECT idseq FROM NEKanji WHERE text == ?) OR idseq IN (SELECT idseq FROM NEKana WHERE text == ?) OR idseq IN (SELECT idseq FROM NETranslation JOIN NETransGloss ON NETranslation.ID == NETransGloss.tid WHERE NETransGloss.text == ?) or idseq in (SELECT idseq FROM NETranslation JOIN NETransType ON NETranslation.ID == NETransType.tid WHERE NETransType.text == ?)"
        params = [query, query, query, query]
        try:
            if query.startswith('id#'):
                query_int = int(query[3:])
                if query_int >= 0:
                    where = "idseq = ?"
                    params = [query_int]
        except Exception:
            pass
        getLogger().debug(f"where={where} | params={params}")
        return where, params

    def search_ne(self, query, ctx=None, **kwargs) -> Sequence[JMDEntry]:
        if ctx is None:
            with self.ctx() as ctx:
                return self.search_ne(query, ctx=ctx)
        where, params = self._build_ne_search_query(query)
        where = 'SELECT idseq FROM NEEntry WHERE ' + where
        entries = []
        for (idseq,) in ctx.conn.cursor().execute(where, params):
            entries.append(self.get_ne(idseq, ctx=ctx))
        return entries

    def search_ne_iter(self, query, ctx=None, **kwargs):
        if ctx is None:
            with self.ctx() as ctx:
                return self.search_ne(query, ctx=ctx)
        where, params = self._build_ne_search_query(query)
        where = 'SELECT idseq FROM NEEntry WHERE ' + where
        for (idseq,) in ctx.conn.cursor().execute(where, params):
            yield self.get_ne(idseq, ctx=ctx)

    def get_ne(self, idseq, ctx=None) -> JMDEntry:
        # ensure context
        if ctx is None:
            with self.ctx() as new_context:
                return self.get_entry(idseq, new_context)
        # else (a context is provided)
        # select entry & info
        entry = JMDEntry(idseq)
{'fraction_non_alphanumeric': 0.17298427415689482, 'fraction_numerical': 0.003817834742296155, 'mean_word_length': 2.4794433902593296, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4708394', 'n_tokens_mistral': 3515, 'n_tokens_neox': 3353, 'n_words': 914}