# Path: tlfp/tools/_step_logic.py
# def use_old_procedure(step, dos=None):
# def should_ignore_commission_text(step, dos):
# def is_one_of_the_initial_depots(steps, step_index):
# def get_previous_step(steps, curr_step_index, is_old_procedure=False, get_depot_step=False):
# 
# Path: tlfp/tools/common.py
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)
# 
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def format_display_date(d):
#     return datetime.strftime(datize(d), '%A %d %B %Y').replace(' 0', ' ')
# 
# Path: tlfp/tools/sort_articles.py
# def compare_articles(a, b):
#     if a == b:
#         return 0
#     na = split_article(a)
#     nb = split_article(b)
#     # compare numbers
#     if na[0] != nb[0]:
#         return na[0] - nb[0]
#     # equalize array lengths
#     if len(na) > len(nb):
#         nb += [None] * (len(na) - len(nb))
#     elif len(na) < len(nb):
#         na += [None] * (len(nb) - len(na))
#     # compare details
#     res = compare_details(na[1:], nb[1:])
#     if res != 0:
#         return res
#     ia = na[0]
#     if 'avant' in a.lower():
#         ia -= 1
#     elif 'après' in a.lower():
#         ia += 1
#     ib = nb[0]
#     if 'avant' in b.lower():
#         ib -= 1
#     elif 'après' in b.lower():
#         ib += 1
#     return ia - ib


import re, os, sys
from difflib import ndiff
from functools import cmp_to_key
from tlfp.tools import _step_logic
from tlfp.tools.common import open_json, print_json, clean_text_for_diff, compute_similarity, format_display_date
from tlfp.tools.sort_articles import compare_articles
                out['sections'][section['id']][step_id]['newnum'] = section['newid']
        last_match_with_previous_step = -1
        for article_index, article in enumerate(data['articles']):
            id = article['titre'].replace(' ', '_')
            if out['articles'].get(id):
                s = create_step(step_id, article=article)
                if 'newtitre' in article:
                    s['newnum'] = article['newtitre']
                txt = clean_text_for_diff(s['text'])

                oldtext = []
                if old_step_index is not None:
                    old_step_id = steps[old_step_index]['directory']
                    for st in out['articles'][id]['steps']:
                        if st['id_step'] == old_step_id:
                            if st['_original_index'] != last_match_with_previous_step + 1:
                                mark_missing_articles_as_deleted(out['articles'], old_step_id, step_id, last_match_with_previous_step, st['_original_index'])
                            last_match_with_previous_step = st['_original_index']
                            if st['status'] != 'sup':
                                oldtext = st['text']
                            break

                if txt and (not oldtext or nstep < depots):
                    s['status'] = 'new' if nstep >= depots else 'none'
                    s['diff'] = 'add'
                    s['n_diff'] = 1
                elif s['status'] == "sup":
                    s['diff'] = 'rem'
                    s['n_diff'] = 0
                else:

# Path: tlfp/tools/_step_logic.py
# def use_old_procedure(step, dos=None):
# def should_ignore_commission_text(step, dos):
# def is_one_of_the_initial_depots(steps, step_index):
# def get_previous_step(steps, curr_step_index, is_old_procedure=False, get_depot_step=False):
# 
# Path: tlfp/tools/common.py
# def open_json(dirpath, filename=None):
#     if filename:
#         path = os.path.join(dirpath, filename)
#     else:
#         path = dirpath
#     try:
#         with open(path, 'r') as f:
#             return json.load(f)
#     except Exception as e:
#         print(type(e), e, file=sys.stderr)
#         sys.stderr.write("ERROR: Could not open file %s" % (path,))
#         raise e
# 
# def print_json(dico, filename=None):
#     jdump = json.dumps(dico, ensure_ascii=False, sort_keys=True, indent=2)
#     if filename:
#         # create parent directories if necessary
#         dirname = os.path.dirname(filename)
#         if dirname:
#             os.makedirs(dirname, exist_ok=True)
# 
#         try:
#             with open("%s.tmp" % filename, 'w') as f:
#                 f.write(jdump)
#             if os.path.exists(filename):
#                 os.remove(filename)
#             os.rename("%s.tmp" % filename, filename)
#         except Exception as e:
#             print(type(e), e, file=sys.stderr)
#             sys.stderr.write("ERROR: Could not write in file %s" % filename)
#             raise e
#     else:
#         print(jdump)
# 
# def clean_text_for_diff(text):
#     if type(text) == list:
#         text = [clean_statuses(re_clean_alin.sub('', t)) for t in text]
#         text = "\n".join([t for t in text if t])
#     else:
#         text = clean_statuses(re_clean_alin.sub('', text))
#     text = clean_balises(text)
#     text = clean_accents(text)
#     text = re_non_alphanum.sub('', text)
#     return text
# 
# def compute_similarity(text1, text2):
#     dmp = diff_match_patch()
#     dmp.Diff_Timeout = 0
#     diff = dmp.diff_main(text1, text2)
# 
#     # similarity
#     common_text = sum([len(txt) for op, txt in diff if op == 0])
#     text_length = max(len(text1), len(text2))
#     sim = common_text / text_length
#     return sim
# 
# def format_display_date(d):
#     return datetime.strftime(datize(d), '%A %d %B %Y').replace(' 0', ' ')
# 
# Path: tlfp/tools/sort_articles.py
# def compare_articles(a, b):
#     if a == b:
#         return 0
#     na = split_article(a)
#     nb = split_article(b)
#     # compare numbers
#     if na[0] != nb[0]:
#         return na[0] - nb[0]
#     # equalize array lengths
#     if len(na) > len(nb):
#         nb += [None] * (len(na) - len(nb))
#     elif len(na) < len(nb):
#         na += [None] * (len(nb) - len(na))
#     # compare details
#     res = compare_details(na[1:], nb[1:])
#     if res != 0:
#         return res
#     ia = na[0]
#     if 'avant' in a.lower():
#         ia -= 1
#     elif 'après' in a.lower():
#         ia += 1
#     ib = nb[0]
#     if 'avant' in b.lower():
#         ib -= 1
#     elif 'après' in b.lower():
#         ib += 1
#     return ia - ib


# Path: tlfp/tools/prepare_articles.py
import re, os, sys
from difflib import ndiff
from functools import cmp_to_key
from tlfp.tools import _step_logic
from tlfp.tools.common import open_json, print_json, clean_text_for_diff, compute_similarity, format_display_date
from tlfp.tools.sort_articles import compare_articles

                out['sections'][section['id']][step_id]['newnum'] = section['newid']
        last_match_with_previous_step = -1
        for article_index, article in enumerate(data['articles']):
            id = article['titre'].replace(' ', '_')
            if out['articles'].get(id):
                s = create_step(step_id, article=article)
                if 'newtitre' in article:
                    s['newnum'] = article['newtitre']
                txt = clean_text_for_diff(s['text'])

                oldtext = []
                if old_step_index is not None:
                    old_step_id = steps[old_step_index]['directory']
                    for st in out['articles'][id]['steps']:
                        if st['id_step'] == old_step_id:
                            if st['_original_index'] != last_match_with_previous_step + 1:
                                mark_missing_articles_as_deleted(out['articles'], old_step_id, step_id, last_match_with_previous_step, st['_original_index'])
                            last_match_with_previous_step = st['_original_index']
                            if st['status'] != 'sup':
                                oldtext = st['text']
                            break

                if txt and (not oldtext or nstep < depots):
                    s['status'] = 'new' if nstep >= depots else 'none'
                    s['diff'] = 'add'
                    s['n_diff'] = 1
                elif s['status'] == "sup":
                    s['diff'] = 'rem'
                    s['n_diff'] = 0
                else:

                    oldtxt = clean_text_for_diff(oldtext)