# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json
                    break
            elif matc:
                sep = matc.group(2).strip()
                log("  --> START FOUND " + sep)
                record = True
                if l:
                    re_end = last
                else:
                    re_end = make_end_reg(sep, rich)
            elif copyall is True:
                record = True
                re_end = null_reg
                if n == 0:
                    i = "%s%s %s" % (s, ". -" if re_mat_simple.match(s) else sep, i)
            if record:
                log("     copy alinea: " + i[:10])
                res.append(i)
        # retry and get everything as I before II added if not found
        if not res:
            if not l and enable_copyall and not copyall:
                log("   nothing found, grabbing all article now...")
                return get_mark_from_last(text, s, l, sep=sep, copyall=True)
            print('ERROR: could not retrieve', s, file=sys.stderr)
            return False
        return res

    get_alineas_text = lambda a: clean_text_for_diff([a[k] for k in sorted(a.keys())])

    re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I)
    re_suppr = re.compile(r'\W*suppr(ess|im)', re.I)

# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


# Path: tlfp/tools/complete_articles.py
import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json

                    break
            elif matc:
                sep = matc.group(2).strip()
                log("  --> START FOUND " + sep)
                record = True
                if l:
                    re_end = last
                else:
                    re_end = make_end_reg(sep, rich)
            elif copyall is True:
                record = True
                re_end = null_reg
                if n == 0:
                    i = "%s%s %s" % (s, ". -" if re_mat_simple.match(s) else sep, i)
            if record:
                log("     copy alinea: " + i[:10])
                res.append(i)
        # retry and get everything as I before II added if not found
        if not res:
            if not l and enable_copyall and not copyall:
                log("   nothing found, grabbing all article now...")
                return get_mark_from_last(text, s, l, sep=sep, copyall=True)
            print('ERROR: could not retrieve', s, file=sys.stderr)
            return False
        return res

    get_alineas_text = lambda a: clean_text_for_diff([a[k] for k in sorted(a.keys())])

    re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I)
    re_suppr = re.compile(r'\W*suppr(ess|im)', re.I)

    re_confo = re.compile(r'\W*(conforme|non[\s\-]*modifi)', re.I)