# Path: tlfp/tools/_step_logic.py
# def use_old_procedure(step, dos=None):
# def should_ignore_commission_text(step, dos):
# def is_one_of_the_initial_depots(steps, step_index):
# def get_previous_step(steps, curr_step_index, is_old_procedure=False, get_depot_step=False):
# 
# Path: tlfp/tools/common.py
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)
# 
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def format_display_date(d):
#     return datetime.strftime(datize(d), '%A %d %B %Y').replace(' 0', ' ')
# 
# Path: tlfp/tools/sort_articles.py
# def compare_articles(a, b):
#     if a == b:
#         return 0
#     na = split_article(a)
#     nb = split_article(b)
#     # compare numbers
#     if na[0] != nb[0]:
#         return na[0] - nb[0]
#     # equalize array lengths
#     if len(na) > len(nb):
#         nb += [None] * (len(na) - len(nb))
#     elif len(na) < len(nb):
#         na += [None] * (len(nb) - len(na))
#     # compare details
#     res = compare_details(na[1:], nb[1:])
#     if res != 0:
#         return res
#     ia = na[0]
#     if 'avant' in a.lower():
#         ia -= 1
#     elif 'après' in a.lower():
#         ia += 1
#     ib = nb[0]
#     if 'avant' in b.lower():
#         ib -= 1
#     elif 'après' in b.lower():
#         ib += 1
#     return ia - ib


import re, os, sys
from difflib import ndiff
from functools import cmp_to_key
from tlfp.tools import _step_logic
from tlfp.tools.common import open_json, print_json, clean_text_for_diff, compute_similarity, format_display_date
from tlfp.tools.sort_articles import compare_articles
            else:
                s['length'] = len("\n".join(s['text']))
            s['_original_index'] = article_index
            out['articles'][id]['steps'].append(s)
        old_step_index = nstep

        # except Error as e:
        #     sys.stderr.write("ERROR parsing step %s:\n%s: %s\n" % (str(step)[:50], type(e), e))
        #    exit(1)

    for a in sorted(out['articles']):
        new_steps = []
        for s in out['articles'][a]['steps']:
            del s['text']
            s.pop('_original_index', None)
            if len(new_steps) > 0 and new_steps[-1]['id_step'] == s['id_step']:
                print('same id_step', s['id_step'], file=sys.stderr)
                continue
            new_steps.append(s)
        out['articles'][a]['steps'] = new_steps

    # Set articles' order values after having reinserted missing ones
    orders = {k: n for n, k in enumerate(
        sorted(
            [a['titre'] for a in out['articles'].values() if a['id'] != 'echec'],
            key=cmp_to_key(compare_articles)
        ))
    }
    for a in out['articles'].values():
        if a['id'] == 'echec':

# Path: tlfp/tools/_step_logic.py
# def use_old_procedure(step, dos=None):
# def should_ignore_commission_text(step, dos):
# def is_one_of_the_initial_depots(steps, step_index):
# def get_previous_step(steps, curr_step_index, is_old_procedure=False, get_depot_step=False):
# 
# Path: tlfp/tools/common.py
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)
# 
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def format_display_date(d):
#     return datetime.strftime(datize(d), '%A %d %B %Y').replace(' 0', ' ')
# 
# Path: tlfp/tools/sort_articles.py
# def compare_articles(a, b):
#     if a == b:
#         return 0
#     na = split_article(a)
#     nb = split_article(b)
#     # compare numbers
#     if na[0] != nb[0]:
#         return na[0] - nb[0]
#     # equalize array lengths
#     if len(na) > len(nb):
#         nb += [None] * (len(na) - len(nb))
#     elif len(na) < len(nb):
#         na += [None] * (len(nb) - len(na))
#     # compare details
#     res = compare_details(na[1:], nb[1:])
#     if res != 0:
#         return res
#     ia = na[0]
#     if 'avant' in a.lower():
#         ia -= 1
#     elif 'après' in a.lower():
#         ia += 1
#     ib = nb[0]
#     if 'avant' in b.lower():
#         ib -= 1
#     elif 'après' in b.lower():
#         ib += 1
#     return ia - ib


# Path: tlfp/tools/prepare_articles.py
import re, os, sys
from difflib import ndiff
from functools import cmp_to_key
from tlfp.tools import _step_logic
from tlfp.tools.common import open_json, print_json, clean_text_for_diff, compute_similarity, format_display_date
from tlfp.tools.sort_articles import compare_articles

            else:
                s['length'] = len("\n".join(s['text']))
            s['_original_index'] = article_index
            out['articles'][id]['steps'].append(s)
        old_step_index = nstep

        # except Error as e:
        #     sys.stderr.write("ERROR parsing step %s:\n%s: %s\n" % (str(step)[:50], type(e), e))
        #    exit(1)

    for a in sorted(out['articles']):
        new_steps = []
        for s in out['articles'][a]['steps']:
            del s['text']
            s.pop('_original_index', None)
            if len(new_steps) > 0 and new_steps[-1]['id_step'] == s['id_step']:
                print('same id_step', s['id_step'], file=sys.stderr)
                continue
            new_steps.append(s)
        out['articles'][a]['steps'] = new_steps

    # Set articles' order values after having reinserted missing ones
    orders = {k: n for n, k in enumerate(
        sorted(
            [a['titre'] for a in out['articles'].values() if a['id'] != 'echec'],
            key=cmp_to_key(compare_articles)
        ))
    }
    for a in out['articles'].values():
        if a['id'] == 'echec':

            a['order'] = -1