# Path: common/ws_utils.py
# def split_page_text(text):
# 
# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):


import sys
import os
import re
import tempfile
import os
import pywikibot
from ws_namespaces import page as page_prefixes, index as index_prefixes
from common import ws_utils
from common import utils
from pywikibot import pagegenerators as pagegen
    from match_and_split import align
        refs.append(it.group(2))
    text = re.sub(u'(?msi)<ref( name=[^>]*)*>.*?</ref>', u'', text)
    for ref in refs:
        text += u'\n' + ref

    text = re.sub(u'(?msi)<ref follow=[^>]*>.*?</ref>', u'', text)

    text = handle_table(text)

    if opt.upper_case:
        text = text.upper()
    return text

def transform_ocr_text(ocr_text, opt):
    ocr_text = re.sub(u'\n«[ ]*', u'\n', ocr_text)
    ocr_text = re.sub(u'-[ ]*\n', u'', ocr_text)
    ocr_text = remove_ocr_template(ocr_text)
    ocr_text = ocr_text.replace(u'\\037', u'')
    ocr_text = ocr_text.replace(u'\\035', u'')
    ocr_text = ocr_text.replace(u'\\013', u'')
    if opt.upper_case:
        ocr_text = ocr_text.upper()
    return ocr_text

def run_diff(ocr_text, text, opt):
    ocr_text = u"\n".join(ocr_text) + u"\n"
    temp1 = tempfile.NamedTemporaryFile(suffix = '.txt')
    utils.write_file(temp1.name, ocr_text)
    text = u"\n".join(text) + u"\n"
    temp2 = tempfile.NamedTemporaryFile(suffix = '.txt')

# Path: common/ws_utils.py
# def split_page_text(text):
# 
# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):


# Path: verify_match/verify_match.py
import sys
import os
import re
import tempfile
import os
import pywikibot
from ws_namespaces import page as page_prefixes, index as index_prefixes
from common import ws_utils
from common import utils
from pywikibot import pagegenerators as pagegen
    from match_and_split import align

        refs.append(it.group(2))
    text = re.sub(u'(?msi)<ref( name=[^>]*)*>.*?</ref>', u'', text)
    for ref in refs:
        text += u'\n' + ref

    text = re.sub(u'(?msi)<ref follow=[^>]*>.*?</ref>', u'', text)

    text = handle_table(text)

    if opt.upper_case:
        text = text.upper()
    return text

def transform_ocr_text(ocr_text, opt):
    ocr_text = re.sub(u'\n«[ ]*', u'\n', ocr_text)
    ocr_text = re.sub(u'-[ ]*\n', u'', ocr_text)
    ocr_text = remove_ocr_template(ocr_text)
    ocr_text = ocr_text.replace(u'\\037', u'')
    ocr_text = ocr_text.replace(u'\\035', u'')
    ocr_text = ocr_text.replace(u'\\013', u'')
    if opt.upper_case:
        ocr_text = ocr_text.upper()
    return ocr_text

def run_diff(ocr_text, text, opt):
    ocr_text = u"\n".join(ocr_text) + u"\n"
    temp1 = tempfile.NamedTemporaryFile(suffix = '.txt')
    utils.write_file(temp1.name, ocr_text)
    text = u"\n".join(text) + u"\n"
    temp2 = tempfile.NamedTemporaryFile(suffix = '.txt')

    utils.write_file(temp2.name, text)