# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):
# 
# Path: common/common_html.py
# def get_head(title, css = None, html5 = False):
# 
# Path: spell/spell.py
# class Speller:
#     def __init__(self, lang):
#     def check(self, word):
#     def suggest(self, word):


import re
import sys
import pywikibot
import pywikibot.pagegenerators as pagegen
import os
import collections
import xml.etree.ElementTree as etree
import hashlib
from pywikibot.data import api
from common import utils
from common import common_html
from spell import spell
    def check_useless_dict_entry(self, variant):
        cache = self.load_dicts(variant)
        transform = [ x[0] for x in self.config[variant]['transform'] ]
        regex = u'(' + u'|'.join(transform) + u')'
        for key in cache:
            local_dict = cache[key][1]
            for word in local_dict:
                if re.search(regex, word) or re.search(regex, local_dict[word]):
                    if type(cache[key][0]) == type(3):
                        print 'global_dict'
                    else:
                        print cache[key][0].encode('utf-8')
                    self.dump_dict_entry(word, local_dict)

    def get_useless_char(self):
        for variant in self.variants:
            if len(self.config[variant]['transform']):
                self.check_useless_dict_entry(variant)

    def find_words(self, words, local_dict, global_dict):
        if local_dict.has_key(words):
            return local_dict[words], False
        if global_dict.has_key(words):
            return global_dict[words], True
        return None, None

    # This mimic code at:
    # https://wikisource.org/wiki/User:He7d3r/Tools/LanguageConverter.js
    def find_repl(self, words_list, i, local_dict, global_dict):
        repl = None

# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):
# 
# Path: common/common_html.py
# def get_head(title, css = None, html5 = False):
# 
# Path: spell/spell.py
# class Speller:
#     def __init__(self, lang):
#     def check(self, word):
#     def suggest(self, word):


# Path: modernization/modernization.py
import re
import sys
import pywikibot
import pywikibot.pagegenerators as pagegen
import os
import collections
import xml.etree.ElementTree as etree
import hashlib
from pywikibot.data import api
from common import utils
from common import common_html
from spell import spell

    def check_useless_dict_entry(self, variant):
        cache = self.load_dicts(variant)
        transform = [ x[0] for x in self.config[variant]['transform'] ]
        regex = u'(' + u'|'.join(transform) + u')'
        for key in cache:
            local_dict = cache[key][1]
            for word in local_dict:
                if re.search(regex, word) or re.search(regex, local_dict[word]):
                    if type(cache[key][0]) == type(3):
                        print 'global_dict'
                    else:
                        print cache[key][0].encode('utf-8')
                    self.dump_dict_entry(word, local_dict)

    def get_useless_char(self):
        for variant in self.variants:
            if len(self.config[variant]['transform']):
                self.check_useless_dict_entry(variant)

    def find_words(self, words, local_dict, global_dict):
        if local_dict.has_key(words):
            return local_dict[words], False
        if global_dict.has_key(words):
            return global_dict[words], True
        return None, None

    # This mimic code at:
    # https://wikisource.org/wiki/User:He7d3r/Tools/LanguageConverter.js
    def find_repl(self, words_list, i, local_dict, global_dict):
        repl = None

        glb = None