<commit_before>"""
Experiment to use the JSON formatted data provided by the AN for the doslegs

  python dossier_from_opendata.py <dosleg_url>
"""

import sys
import json
import zipfile
import io

from lawfactory_utils.urls import download, enable_requests_cache, clean_url
import lawfactory_utils.urls

from anpy.dossier import get_legislature


def yield_leafs(etape, path=None):
    if path is None:
        path = []
    if etape.get("actesLegislatifs"):
        for children in to_arr(etape["actesLegislatifs"]["acteLegislatif"]):
            yield from yield_leafs(children, path=path + [etape])
    else:
        yield path, etape


def to_arr(obj):
    if type(obj) is list:
        return obj
    return [obj]


def test_status(url):
    try:
        resp = download(url)
        if resp.status_code != 200:
            return False
    except Exception:
        return False
    return resp


def download_open_data_doslegs(legislature):
    files = {
        15: (
            "Dossiers_Legislatifs_XV.json",
            "http://data.assemblee-nationale.fr/static/openData/repository/15/loi/dossiers_legislatifs/Dossiers_Legislatifs_XV.json.zip",
        ),
        14: (
            "Dossiers_Legislatifs_XIV.json",
            "http://data.assemblee-nationale.fr/static/openData/repository/14/loi/dossiers_legislatifs/Dossiers_Legislatifs_XIV.json.zip",
        ),
    }
    #  TODO: remove this hack when we are able to cache the zip files
    file, file_url = files[legislature]

    CACHE_ENABLED = lawfactory_utils.urls.CACHE_ENABLED
    lawfactory_utils.urls.CACHE_ENABLED = False
    doslegs_resp = download(file_url)
    lawfactory_utils.urls.CACHE_ENABLED = CACHE_ENABLED

    doslegs_zip = zipfile.ZipFile(io.BytesIO(doslegs_resp.content))
    DATA = json.loads(doslegs_zip.open(file).read().decode("utf-8"))

    return DATA


def parse(url, verbose=True, logfile=sys.stderr, cached_opendata_an={}):
    if not verbose:

        def _log(*x):
            return None

    else:

        def _log(*args):
            nonlocal logfile
            print(*args, file=logfile)

    legislature = get_legislature(url)
    if legislature and legislature in cached_opendata_an:
        dossiers_json = cached_opendata_an[legislature]
    else:
        dossiers_json = download_open_data_doslegs(get_legislature(url))

    docs = {doc["uid"]: doc for doc in dossiers_json["export"]["textesLegislatifs"]["document"]}

    for dossier in dossiers_json["export"]["dossiersLegislatifs"]["dossier"]:
        dossier = dossier["dossierParlementaire"]

        titreChemin = dossier["titreDossier"]["titreChemin"]

        # find the right dosleg even if it's an old url
        url_common_part = "{}/dossiers/{}".format(dossier["legislature"], titreChemin)
        if url_common_part not in url:
            continue
        url = "http://www.assemblee-nationale.fr/dyn/{}".format(url_common_part)

        data = {}
        data["urgence"] = False
        url_senat = dossier["titreDossier"]["senatChemin"]
        if url_senat:
            data["url_dossier_senat"] = url_senat
        data["long_title"] = dossier["titreDossier"]["titre"]
        data["url_dossier_assemblee"] = url
        data["assemblee_legislature"] = int(dossier["legislature"])
        data["assemblee_slug"] = dossier["titreDossier"]["titreChemin"]
        data["assemblee_id"] = "%s-%s" % (dossier["legislature"], data["assemblee_slug"])

        data["steps"] = []
        for etape in to_arr(dossier["actesLegislatifs"]["acteLegislatif"]):
            for path, sous_etape in yield_leafs(etape):
                if sous_etape["@xsi:type"] in ("EtudeImpact_Type", "DepotAvisConseilEtat_Type"):
                    continue

                step = {"date": sous_etape.get("dateActe").split("T")[0]}

                if sous_etape["@xsi:type"] == "ProcedureAccelere_Type":
                    data["urgence"] = True<commit_msg>opendata: date can be null<commit_after>"""
Experiment to use the JSON formatted data provided by the AN for the doslegs

  python dossier_from_opendata.py <dosleg_url>
"""

import sys
import json
import zipfile
import io

from lawfactory_utils.urls import download, enable_requests_cache, clean_url
import lawfactory_utils.urls

from anpy.dossier import get_legislature


def yield_leafs(etape, path=None):
    if path is None:
        path = []
    if etape.get("actesLegislatifs"):
        for children in to_arr(etape["actesLegislatifs"]["acteLegislatif"]):
            yield from yield_leafs(children, path=path + [etape])
    else:
        yield path, etape


def to_arr(obj):
    if type(obj) is list:
        return obj
    return [obj]


def test_status(url):
    try:
        resp = download(url)
        if resp.status_code != 200:
            return False
    except Exception:
        return False
    return resp


def download_open_data_doslegs(legislature):
    files = {
        15: (
            "Dossiers_Legislatifs_XV.json",
            "http://data.assemblee-nationale.fr/static/openData/repository/15/loi/dossiers_legislatifs/Dossiers_Legislatifs_XV.json.zip",
        ),
        14: (
            "Dossiers_Legislatifs_XIV.json",
            "http://data.assemblee-nationale.fr/static/openData/repository/14/loi/dossiers_legislatifs/Dossiers_Legislatifs_XIV.json.zip",
        ),
    }
    #  TODO: remove this hack when we are able to cache the zip files
    file, file_url = files[legislature]

    CACHE_ENABLED = lawfactory_utils.urls.CACHE_ENABLED
    lawfactory_utils.urls.CACHE_ENABLED = False
    doslegs_resp = download(file_url)
    lawfactory_utils.urls.CACHE_ENABLED = CACHE_ENABLED

    doslegs_zip = zipfile.ZipFile(io.BytesIO(doslegs_resp.content))
    DATA = json.loads(doslegs_zip.open(file).read().decode("utf-8"))

    return DATA


def parse(url, verbose=True, logfile=sys.stderr, cached_opendata_an={}):
    if not verbose:

        def _log(*x):
            return None

    else:

        def _log(*args):
            nonlocal logfile
            print(*args, file=logfile)

    legislature = get_legislature(url)
    if legislature and legislature in cached_opendata_an:
        dossiers_json = cached_opendata_an[legislature]
    else:
        dossiers_json = download_open_data_doslegs(get_legislature(url))

    docs = {doc["uid"]: doc for doc in dossiers_json["export"]["textesLegislatifs"]["document"]}

    for dossier in dossiers_json["export"]["dossiersLegislatifs"]["dossier"]:
        dossier = dossier["dossierParlementaire"]

        titreChemin = dossier["titreDossier"]["titreChemin"]

        # find the right dosleg even if it's an old url
        url_common_part = "{}/dossiers/{}".format(dossier["legislature"], titreChemin)
        if url_common_part not in url:
            continue
        url = "http://www.assemblee-nationale.fr/dyn/{}".format(url_common_part)

        data = {}
        data["urgence"] = False
        url_senat = dossier["titreDossier"]["senatChemin"]
        if url_senat:
            data["url_dossier_senat"] = url_senat
        data["long_title"] = dossier["titreDossier"]["titre"]
        data["url_dossier_assemblee"] = url
        data["assemblee_legislature"] = int(dossier["legislature"])
        data["assemblee_slug"] = dossier["titreDossier"]["titreChemin"]
        data["assemblee_id"] = "%s-%s" % (dossier["legislature"], data["assemblee_slug"])

        data["steps"] = []
        for etape in to_arr(dossier["actesLegislatifs"]["acteLegislatif"]):
            for path, sous_etape in yield_leafs(etape):
                if sous_etape["@xsi:type"] in ("EtudeImpact_Type", "DepotAvisConseilEtat_Type"):
                    continue

                step = {}

                date = sous_etape.get("dateActe")
                if date:
                    step["date"] = date.split("T")[0]

                if sous_etape["@xsi:type"] == "ProcedureAccelere_Type":
                    data["urgence"] = True
{'fraction_non_alphanumeric': 0.1147582375962631, 'fraction_numerical': 0.003282413836636788, 'mean_word_length': 2.9393336648433617, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1657696', 'n_tokens_mistral': 2710, 'n_tokens_neox': 2649, 'n_words': 485}