# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json
            if (re_suppr.match(line["statut"]) or (len(alineas) == 1 and re_suppr.match(alineas[0]))) and (line['titre'] not in oldstatus or re_suppr.match(oldstatus[line['titre']])):
               continue
            # Clean empty articles with only "Non modifié" and include text from previous step
            if alineas and re_confo.match(alineas[0]) and alineas[0].endswith(')'):
                if not line['titre'] in oldstep:
                    sys.stderr.write("WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n" % (line['titre'], line['alineas']))
                    # ignore empty non-modified
                    continue
                else:
                    log("DEBUG: get back Art %s" % line['titre'])
                    alineas = oldstep[line['titre']]
            gd_text = []
            enable_copyall = True
            for j, text in enumerate(alineas):
                if "(Non modifi" in text and not line['titre'] in oldstep:
                    sys.stderr.write("WARNING: found repeated article missing %s from previous step: %s\n" % (line['titre'], text))
                elif re_confo_with_txt.search(text):
                    text = re_confo_with_txt.sub(r' \2', text)
                    text = re_clean_subsec_space.sub(r'\1\4 \5', text)
                    gd_text.append(text)
                elif "(Non modifi" in text:
                    part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text)
                    if not part:
                        log("ERROR trying to get non-modifiés")
                        exit()
                    pieces = re_clean_et.sub(',', part[0])
                    log("EXTRACT non-modifiés for " + line['titre'] + ": " + pieces)
                    piece = []
                    for todo in pieces.split(','):
                        # Extract series of non-modified subsections of articles from previous version.

# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


# Path: tlfp/tools/complete_articles.py
import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json

            if (re_suppr.match(line["statut"]) or (len(alineas) == 1 and re_suppr.match(alineas[0]))) and (line['titre'] not in oldstatus or re_suppr.match(oldstatus[line['titre']])):
               continue
            # Clean empty articles with only "Non modifié" and include text from previous step
            if alineas and re_confo.match(alineas[0]) and alineas[0].endswith(')'):
                if not line['titre'] in oldstep:
                    sys.stderr.write("WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n" % (line['titre'], line['alineas']))
                    # ignore empty non-modified
                    continue
                else:
                    log("DEBUG: get back Art %s" % line['titre'])
                    alineas = oldstep[line['titre']]
            gd_text = []
            enable_copyall = True
            for j, text in enumerate(alineas):
                if "(Non modifi" in text and not line['titre'] in oldstep:
                    sys.stderr.write("WARNING: found repeated article missing %s from previous step: %s\n" % (line['titre'], text))
                elif re_confo_with_txt.search(text):
                    text = re_confo_with_txt.sub(r' \2', text)
                    text = re_clean_subsec_space.sub(r'\1\4 \5', text)
                    gd_text.append(text)
                elif "(Non modifi" in text:
                    part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text)
                    if not part:
                        log("ERROR trying to get non-modifiés")
                        exit()
                    pieces = re_clean_et.sub(',', part[0])
                    log("EXTRACT non-modifiés for " + line['titre'] + ": " + pieces)
                    piece = []
                    for todo in pieces.split(','):
                        # Extract series of non-modified subsections of articles from previous version.

                        if " à " in todo: