# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json

    if previous_step_metas and not previous_step_metas.get("skip"):
        try:
            assert(previous_step_metas["type"] == "texte")
            oldnum = int(find_num.search(previous_step_metas['id']).group(1))
        except Exception as e:
            print(type(e), e, file=sys.stderr)
            print("Incorrect previous step: %s" % previous_step_metas)
            exit()

    gdoldstep = None
    if anteprevious:
        gdoldstep = {}
        for line in anteprevious:
            if line["type"] == "article":
                keys = list(line['alineas'].keys())
                keys.sort()
                gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys]

    ALL_ARTICLES = []
    def write_json(data):
        nonlocal ALL_ARTICLES
        ALL_ARTICLES.append(data)

    null_reg = re.compile(r'^$')
    re_mat_simple = re.compile(r'[IVXDCLM\d]')
    re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]')
    re_mat_complex2 = re.compile(r'\d+-\d+')
    re_clean_art = re.compile(r'^"?Art\.?\s*', re.I)
    make_sta_reg = lambda x: re.compile(r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x))

# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


# Path: tlfp/tools/complete_articles.py
import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json


    if previous_step_metas and not previous_step_metas.get("skip"):
        try:
            assert(previous_step_metas["type"] == "texte")
            oldnum = int(find_num.search(previous_step_metas['id']).group(1))
        except Exception as e:
            print(type(e), e, file=sys.stderr)
            print("Incorrect previous step: %s" % previous_step_metas)
            exit()

    gdoldstep = None
    if anteprevious:
        gdoldstep = {}
        for line in anteprevious:
            if line["type"] == "article":
                keys = list(line['alineas'].keys())
                keys.sort()
                gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys]

    ALL_ARTICLES = []
    def write_json(data):
        nonlocal ALL_ARTICLES
        ALL_ARTICLES.append(data)

    null_reg = re.compile(r'^$')
    re_mat_simple = re.compile(r'[IVXDCLM\d]')
    re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]')
    re_mat_complex2 = re.compile(r'\d+-\d+')
    re_clean_art = re.compile(r'^"?Art\.?\s*', re.I)
    make_sta_reg = lambda x: re.compile(r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x))

    make_end_reg = lambda x, rich: re.compile(r'^%s[IVXDCLM\d\-]+([\-\.\s]+\d*)*((%s|[A-Z])\s*)*(\(|et\s|%s)' % ('("?[LA][LArRtTO\.\s]+)?' if rich else "", bister, x))