<jupyter_start><jupyter_text>Analyse collected data<jupyter_code>%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Import packages<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>data = pd.read_csv("../data/aides_v2.csv")<jupyter_output><empty_output><jupyter_text>Overview<jupyter_code>data.info()
data.head()
data["error_orig"] = 1 - data["error_orig"]
data["error_app"] = 1 - data["error_app"]
n_aides = data["url"].nunique()
print(f"There are {n_aides} unique pages that were analysed")
n_errors_orig = data["error_orig"].sum()
n_errors_app = data["error_app"].sum()
n_errors = (data["error_orig"] + data["error_app"] == 2).sum()

print(f"{n_errors_orig} errors found on orig url")
print(f"{n_errors_app} errors found on app url")
print(f"{n_errors} errors found on both url")
data["type_error_orig"] = data["type_error_orig"].replace("url not starting with http", "url empty or not valid")
data["type_error_app"] = data["type_error_app"].replace("url not starting with http", "url empty or not valid")
data["type_error_orig"].str.split(" to http", expand=True).iloc[:, 0].value_counts()
data["type_error_app"].str.split(" to http", expand=True).iloc[:, 0].value_counts()
n_aide_pdf_criterias = data.pdf_avec_criteres.sum()
criterias = ["conditions", "critères", "éligible", "éligibilité"]
print(f"There are {n_aide_pdf_criterias} pages with a pdf containing one of the following words : {criterias}")
from ast import literal_eval
t = "[1, 2]"
literal_eval(t)
data.loc[data["pdf_avec_criteres"]]["pdfs"].apply(literal_eval).apply(len).value_counts()
(data.loc[data["pdf_avec_criteres"]]["pdfs"].value_counts() == 1).sum()
for aide, pdfs in data.loc[data["pdf_avec_criteres"]].set_index("name")["pdfs"].apply(literal_eval).iteritems():
    print(aide)
    print("\n".join(pdfs))
    print()<jupyter_output>Aide à la création et au développement d'entreprises
http://www.cc-madetmoselle.fr/images/FR-MODIF.pdf
http://www.cc-madetmoselle.fr/images/Documents/REGLEMENT-PLAN.pdf
http://www.cc-madetmoselle.fr/images/Documents/REG-2020-2023.pdf

Aide au développement numérique
http://www.cc-madetmoselle.fr/images/FR-MODIF.pdf
http://www.cc-madetmoselle.fr/images/Documents/REGLEMENT-PLAN.pdf
http://www.cc-madetmoselle.fr/images/Documents/REG-2020-2023.pdf

Financer l'investissement productif
http://ccv2m.fr/wp-content/uploads/2015/05/AET_reglement.pdf
http://ccv2m.fr/wp-content/uploads/2015/05/diagnostic_agricole_foncier.pdf
http://ccv2m.fr/wp-content/uploads/2019/12/tableau-V2M.pdf
http://ccv2m.fr/wp-content/uploads/2021/05/Reglement-interieur-CCV2M-1.pdf

Soutenir l'investissement en immobilier des entreprises dans le cadre d'une création ou extension d'activité
http://www.cc-madetmoselle.fr/images/FR-MODIF.pdf
http://www.cc-madetmoselle.fr/images/Documents/REGLEMENT-PLAN.pdf
http://www.cc-madet[...]
{'fraction_non_alphanumeric': 0.1610054347826087, 'fraction_numerical': 0.01936141304347826, 'mean_word_length': 6.75, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '327753', 'license': 'MIT', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1121, 'n_words': 176, 'path': 'notebooks/analyse_collected_data.ipynb', 'repo_name': 'dataforgoodfr/offseason_missiontransition', 'size': '2944'}