# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json
                except:
                    print("ERROR: Problem while renumbering sections: ", line['titre'], " is not ", cursec, '\n', file=sys.stderr)
                    # exit()
                if line["id"] != cursec["id"]:
                    log("DEBUG: Matched section %s (%s) with old section %s (%s)" % (line["id"], line.get('titre'), cursec["id"], cursec.get('titre')))
                    line["newid"] = line["id"]
                    line["id"] = cursec["id"]
            write_json(line)
          else:
            keys = list(line['alineas'].keys())
            keys.sort()
            alineas = [line['alineas'][k] for k in keys]
            mult = line['titre'].split(' à ')
            is_mult = (len(mult) > 1)
            if is_mult:
                st = mult[0].strip()
                ed = mult[1].strip()
                if re_suppr.match(line['statut']) or (len(alineas) == 1 and re_suppr.match(alineas[0])):
                    if (st not in oldartids and ed not in oldartids) or (st in oldstatus and re_suppr.match(oldstatus[st]) and ed in oldstatus and re_suppr.match(oldstatus[ed])):
                        log("DEBUG: SKIP already deleted articles %s to %s" % (st, ed))
                        continue
                    log("DEBUG: Marking as deleted articles %s à %s" % (st, ed))
                    mult_type = "sup"
                elif re_confo.match(line['statut']) or (len(alineas) == 1 and re_confo.match(alineas[0])):
                    log("DEBUG: Recovering art conformes %s à %s" % (st, ed))
                    mult_type = "conf"
                else:
                    print("ERROR: Found multiple article which I don't know what to do with", line['titre'], line, file=sys.stderr)
                    exit()
                line['titre'] = st

# Path: tlfp/tools/sort_articles.py
# def split_article(a):
# def quantify_bis(b):
# def type_detail(d):
# def compare_details(a, b):
# def compare_articles(a, b):
# def article_is_lower(a, b):
# 
# Path: tlfp/tools/common.py
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)


# Path: tlfp/tools/complete_articles.py
import sys, re, copy
from requests.structures import CaseInsensitiveDict
from tlfp.tools.sort_articles import bister, article_is_lower
from tlfp.tools.common import clean_text_for_diff, compute_similarity, open_json, print_json

                except:
                    print("ERROR: Problem while renumbering sections: ", line['titre'], " is not ", cursec, '\n', file=sys.stderr)
                    # exit()
                if line["id"] != cursec["id"]:
                    log("DEBUG: Matched section %s (%s) with old section %s (%s)" % (line["id"], line.get('titre'), cursec["id"], cursec.get('titre')))
                    line["newid"] = line["id"]
                    line["id"] = cursec["id"]
            write_json(line)
          else:
            keys = list(line['alineas'].keys())
            keys.sort()
            alineas = [line['alineas'][k] for k in keys]
            mult = line['titre'].split(' à ')
            is_mult = (len(mult) > 1)
            if is_mult:
                st = mult[0].strip()
                ed = mult[1].strip()
                if re_suppr.match(line['statut']) or (len(alineas) == 1 and re_suppr.match(alineas[0])):
                    if (st not in oldartids and ed not in oldartids) or (st in oldstatus and re_suppr.match(oldstatus[st]) and ed in oldstatus and re_suppr.match(oldstatus[ed])):
                        log("DEBUG: SKIP already deleted articles %s to %s" % (st, ed))
                        continue
                    log("DEBUG: Marking as deleted articles %s à %s" % (st, ed))
                    mult_type = "sup"
                elif re_confo.match(line['statut']) or (len(alineas) == 1 and re_confo.match(alineas[0])):
                    log("DEBUG: Recovering art conformes %s à %s" % (st, ed))
                    mult_type = "conf"
                else:
                    print("ERROR: Found multiple article which I don't know what to do with", line['titre'], line, file=sys.stderr)
                    exit()
                line['titre'] = st

            cur = ""