<commit_before># -*- coding: utf-8 -*-
#
# ibus-replace-with-kanji - Replace with Kanji Japanese input method for IBus
#
# Copyright (c) 2017 Esrille Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import logging
import re

logger = logging.getLogger(__name__)

_re_not_yomi = re.compile(r'[^ぁ-ゖァ-ー―]')
_re_yomi = re.compile(r'^[ぁ-ゖァ-ー―]+[、。，．]?$')

_dic_ver = "v0.4.0"

class Dictionary:

    def __init__(self, path):
        logger.info("Dictionary(%s)", path)

        self.__dict_base = {}
        self.__dict = {}

        self.__yomi = ''
        self.__no = 0
        self.__cand = []
        self.__dirty = False

        self.__orders_path = ''

        # Load Katakana dictionary first so that Katakana words come after Kanji words.
        katakana_path = os.path.join(os.getenv('IBUS_REPLACE_WITH_KANJI_LOCATION'), 'katakana.dic')
        self.__load_dict(self.__dict_base, katakana_path)

        # Load system dictionary
        self.__load_dict(self.__dict_base, path)

        # Load private dictionary
        self.__dict = self.__dict_base.copy()

        my_path = os.path.expanduser('~/.local/share/ibus-replace-with-kanji/my.dic')
        self.__load_dict(self.__dict, my_path, 'a+')

        base = os.path.basename(path)
        if base:
            self.__orders_path = os.path.expanduser('~/.local/share/ibus-replace-with-kanji')
            self.__orders_path = os.path.join(self.__orders_path, base)
            self.__load_dict(self.__dict, self.__orders_path, 'a+', version_checked=False)

    def __load_dict(self, dict, path, mode='r', version_checked = True):
        with open(path, mode) as file:
            file.seek(0, 0)   # in case opened in the 'a+' mode
            for line in file:
                line = line.strip(' \n/')
                if line[0] == ';':
                    if line == "; " + _dic_ver + "\n":
                        version_checked = True
                    continue
                if not line:
                    continue
                if not version_checked:
                    continue
                p = line.split(' ', 1)
                yomi = p[0]
                cand = p[1].strip(' \n/').split('/')
                if not yomi in dict:
                    dict[yomi] = cand
                else:
                    update = list(dict[yomi])
                    for i in reversed(cand):
                        if i in update:
                            update.remove(i)
                        update.insert(0, i)
                    dict[yomi] = update
            logger.info("Loaded %s", path)

    def __str(self, s):
        if s[-1] == '―':
            return s[:-1]<commit_msg>engine/dictionary.py (__load_dict) : Fix version checking<commit_after># -*- coding: utf-8 -*-
#
# ibus-replace-with-kanji - Replace with Kanji Japanese input method for IBus
#
# Copyright (c) 2017 Esrille Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import logging
import re

logger = logging.getLogger(__name__)

_re_not_yomi = re.compile(r'[^ぁ-ゖァ-ー―]')
_re_yomi = re.compile(r'^[ぁ-ゖァ-ー―]+[、。，．]?$')

_dic_ver = "v0.4.0"

class Dictionary:

    def __init__(self, path):
        logger.info("Dictionary(%s)", path)

        self.__dict_base = {}
        self.__dict = {}

        self.__yomi = ''
        self.__no = 0
        self.__cand = []
        self.__dirty = False

        self.__orders_path = ''

        # Load Katakana dictionary first so that Katakana words come after Kanji words.
        katakana_path = os.path.join(os.getenv('IBUS_REPLACE_WITH_KANJI_LOCATION'), 'katakana.dic')
        self.__load_dict(self.__dict_base, katakana_path)

        # Load system dictionary
        self.__load_dict(self.__dict_base, path)

        # Load private dictionary
        self.__dict = self.__dict_base.copy()

        my_path = os.path.expanduser('~/.local/share/ibus-replace-with-kanji/my.dic')
        self.__load_dict(self.__dict, my_path, 'a+')

        base = os.path.basename(path)
        if base:
            self.__orders_path = os.path.expanduser('~/.local/share/ibus-replace-with-kanji')
            self.__orders_path = os.path.join(self.__orders_path, base)
            self.__load_dict(self.__dict, self.__orders_path, 'a+', version_checked=False)

    def __load_dict(self, dict, path, mode='r', version_checked = True):
        with open(path, mode) as file:
            file.seek(0, 0)   # in case opened in the 'a+' mode
            for line in file:
                line = line.strip(' \n/')
                if line[0] == ';':
                    if line == "; " + _dic_ver:
                        version_checked = True
                    continue
                if not line:
                    continue
                if not version_checked:
                    continue
                p = line.split(' ', 1)
                yomi = p[0]
                cand = p[1].strip(' \n/').split('/')
                if not yomi in dict:
                    dict[yomi] = cand
                else:
                    update = list(dict[yomi])
                    for i in reversed(cand):
                        if i in update:
                            update.remove(i)
                        update.insert(0, i)
                    dict[yomi] = update
            logger.info("Loaded %s", path)

    def __str(self, s):
        if s[-1] == '―':
            return s[:-1]
{'fraction_non_alphanumeric': 0.11572700296735905, 'fraction_numerical': 0.006871778853662346, 'mean_word_length': 1.8833858622242232, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471042', 'n_tokens_mistral': 1977, 'n_tokens_neox': 1887, 'n_words': 516}