# Path: janome/dic.py
# MODULE_FST_DATA = 'fst_data%d.py'
# MODULE_ENTRIES_EXTRA = 'entries_extra%d.py'
# MODULE_ENTRIES_COMPACT = 'entries_compact%d.py'
# MODULE_ENTRIES_BUCKETS = 'entries_buckets.py'
# MODULE_CONNECTIONS = 'connections%d.py'
# MODULE_CHARDEFS = 'chardef.py'
# MODULE_UNKNOWNS = 'unknowns.py'
# FILE_USER_FST_DATA = 'user_fst.data'
# FILE_USER_ENTRIES_DATA = 'user_entries.data'
# def save_fstdata(data, dir, part=0):
# def start_save_entries(dir, bucket_idx, morph_offset):
# def end_save_entries(dir, bucket_idx):
# def save_entry(dir, bucket_idx, morph_id, entry):
# def save_entry_buckets(dir, buckets):
# def save_connections(connections, dir='.'):
# def save_chardefs(chardefs, dir='.'):
# def save_unknowns(unknowns, dir='.'):
# def _save(file, data, compresslevel):
# def _load(file):
# def _load_package_data(package, resource):
# def _save_as_module(file, data, binary=False):
# def _start_entries_as_module(file, morph_id_offset):
# def _end_entries_as_module(file):
# def _save_entry_as_module_compact(file, morph_id, entry):
# def _save_entry_as_module_extra(file, morph_id, entry):
#     def lookup(self, s, matcher):
#     def lookup_extra(self, num):
#     def get_trans_cost(self, id1, id2):
#     def __init__(self, entries, connections):
#     def lookup(self, s, matcher):
#     def lookup_extra(self, num):
#     def get_trans_cost(self, id1, id2):
#     def __init__(self, entries_compact, entries_extra, open_files, connections):
#     def lookup(self, s, matcher):
#     def _find_entry(self, idx):
#     def lookup_extra(self, idx):
#     def get_trans_cost(self, id1, id2):
#     def __del__(self):
#     def __init__(self, chardefs, unknowns):
#     def get_char_categories(self, c):
#     def unknown_invoked_always(self, cate):
#     def unknown_grouping(self, cate):
#     def unknown_length(self, cate):
#     def __init__(self, user_dict, enc, type, connections, progress_handler=None):
#     def lookup(self, s):
#     def line_to_entry_ipadic(cls, line):
#     def line_to_entry_simpledic(cls, line):
#     def build_dic(cls, user_dict, enc, dict_type, progress_handler):
#     def save(self, to_dir, compressionlevel=9):
#     def __init__(self, dic_dir, connections):
#     def lookup(self, s):
#     def load_dict(cls, dic_dir):
#     def __init__(self):
# class Dictionary(ABC):
# class RAMDictionary(Dictionary):
# class MMapDictionary(Dictionary):
# class UnknownsDictionary(object):
# class UserDictionary(RAMDictionary):
# class CompiledUserDictionary(RAMDictionary):
# class LoadingDictionaryError(Exception):


import glob
import io
import logging
import struct
import os
import sys
from sysdic import entries_buckets
from sysdic import (
    entries_extra0,
    entries_extra1,
    entries_extra2,
    entries_extra3,
    entries_extra4,
    entries_extra5,
    entries_extra6,
    entries_extra7,
    entries_extra8,
    entries_extra9
)
from sysdic import (
    entries_compact0,
    entries_compact1,
    entries_compact2,
    entries_compact3,
    entries_compact4,
    entries_compact5,
    entries_compact6,
    entries_compact7,
    entries_compact8,
    entries_compact9
)
from sysdic import all_fstdata, entries, connections, chardef, unknowns
from janome.dic import SystemDictionary
                logger.debug(u"check word id & entry for %s" % (surface))
                (matched, outputs) = SYS_DIC.matcher.run(surface.encode('utf8'), common_prefix_match=True)
                if not matched:
                    logger.debug('No match for %s' % (surface))
                    nomatch_count += 1
                for o in outputs:
                    try:
                        word_id = struct.unpack('I', o)[0]
                        logger.debug(u"\tword id : %d" % (word_id))
                        try:
                            entry = SYS_DIC.entries[word_id]
                            if not surface.startswith(entry[0]):
                                logger.warn('Must not match to %s' % entry[0])
                                invalid_match_count += 1
                                break
                        except KeyError:
                            logger.warn('\tCannot find entry')
                            invalid_output_count += 1
                            break
                    except Exception:
                        logger.warn('\tInvalid output')
                        invalid_output_count += 1
                        break
    print('matches = %d' % (input_count - nomatch_count))
    print('no matches = %d' % nomatch_count)
    print('invalid matches = %d' % invalid_match_count)
    print('invalid outputs = %d' % invalid_output_count)

    # validate connection costs
    print('Validate connection costs...')

# Path: janome/dic.py
# MODULE_FST_DATA = 'fst_data%d.py'
# MODULE_ENTRIES_EXTRA = 'entries_extra%d.py'
# MODULE_ENTRIES_COMPACT = 'entries_compact%d.py'
# MODULE_ENTRIES_BUCKETS = 'entries_buckets.py'
# MODULE_CONNECTIONS = 'connections%d.py'
# MODULE_CHARDEFS = 'chardef.py'
# MODULE_UNKNOWNS = 'unknowns.py'
# FILE_USER_FST_DATA = 'user_fst.data'
# FILE_USER_ENTRIES_DATA = 'user_entries.data'
# def save_fstdata(data, dir, part=0):
# def start_save_entries(dir, bucket_idx, morph_offset):
# def end_save_entries(dir, bucket_idx):
# def save_entry(dir, bucket_idx, morph_id, entry):
# def save_entry_buckets(dir, buckets):
# def save_connections(connections, dir='.'):
# def save_chardefs(chardefs, dir='.'):
# def save_unknowns(unknowns, dir='.'):
# def _save(file, data, compresslevel):
# def _load(file):
# def _load_package_data(package, resource):
# def _save_as_module(file, data, binary=False):
# def _start_entries_as_module(file, morph_id_offset):
# def _end_entries_as_module(file):
# def _save_entry_as_module_compact(file, morph_id, entry):
# def _save_entry_as_module_extra(file, morph_id, entry):
#     def lookup(self, s, matcher):
#     def lookup_extra(self, num):
#     def get_trans_cost(self, id1, id2):
#     def __init__(self, entries, connections):
#     def lookup(self, s, matcher):
#     def lookup_extra(self, num):
#     def get_trans_cost(self, id1, id2):
#     def __init__(self, entries_compact, entries_extra, open_files, connections):
#     def lookup(self, s, matcher):
#     def _find_entry(self, idx):
#     def lookup_extra(self, idx):
#     def get_trans_cost(self, id1, id2):
#     def __del__(self):
#     def __init__(self, chardefs, unknowns):
#     def get_char_categories(self, c):
#     def unknown_invoked_always(self, cate):
#     def unknown_grouping(self, cate):
#     def unknown_length(self, cate):
#     def __init__(self, user_dict, enc, type, connections, progress_handler=None):
#     def lookup(self, s):
#     def line_to_entry_ipadic(cls, line):
#     def line_to_entry_simpledic(cls, line):
#     def build_dic(cls, user_dict, enc, dict_type, progress_handler):
#     def save(self, to_dir, compressionlevel=9):
#     def __init__(self, dic_dir, connections):
#     def lookup(self, s):
#     def load_dict(cls, dic_dir):
#     def __init__(self):
# class Dictionary(ABC):
# class RAMDictionary(Dictionary):
# class MMapDictionary(Dictionary):
# class UnknownsDictionary(object):
# class UserDictionary(RAMDictionary):
# class CompiledUserDictionary(RAMDictionary):
# class LoadingDictionaryError(Exception):


# Path: ipadic/validate.py
import glob
import io
import logging
import struct
import os
import sys
from sysdic import entries_buckets
from sysdic import (
    entries_extra0,
    entries_extra1,
    entries_extra2,
    entries_extra3,
    entries_extra4,
    entries_extra5,
    entries_extra6,
    entries_extra7,
    entries_extra8,
    entries_extra9
)
from sysdic import (
    entries_compact0,
    entries_compact1,
    entries_compact2,
    entries_compact3,
    entries_compact4,
    entries_compact5,
    entries_compact6,
    entries_compact7,
    entries_compact8,
    entries_compact9
)
from sysdic import all_fstdata, entries, connections, chardef, unknowns
from janome.dic import SystemDictionary

                logger.debug(u"check word id & entry for %s" % (surface))
                (matched, outputs) = SYS_DIC.matcher.run(surface.encode('utf8'), common_prefix_match=True)
                if not matched:
                    logger.debug('No match for %s' % (surface))
                    nomatch_count += 1
                for o in outputs:
                    try:
                        word_id = struct.unpack('I', o)[0]
                        logger.debug(u"\tword id : %d" % (word_id))
                        try:
                            entry = SYS_DIC.entries[word_id]
                            if not surface.startswith(entry[0]):
                                logger.warn('Must not match to %s' % entry[0])
                                invalid_match_count += 1
                                break
                        except KeyError:
                            logger.warn('\tCannot find entry')
                            invalid_output_count += 1
                            break
                    except Exception:
                        logger.warn('\tInvalid output')
                        invalid_output_count += 1
                        break
    print('matches = %d' % (input_count - nomatch_count))
    print('no matches = %d' % nomatch_count)
    print('invalid matches = %d' % invalid_match_count)
    print('invalid outputs = %d' % invalid_output_count)

    # validate connection costs
    print('Validate connection costs...')

    matrix_file = os.path.join(dicdir, 'matrix.def')