<jupyter_start><jupyter_text>Scrap<jupyter_code>url = "http://www.francetvinfo.fr/elections/resultats/"
resultats = requests.get(url)
soup = BeautifulSoup(resultats.text, 'html.parser')
df = pd.DataFrame(columns=["depart", "code", "circo", "nom", "parti", "score"])
circos_url = []

index=0


for depart in soup.find("section", {"id": "departmentResults"}).findAll("a", href=True):
    url_depart = depart["href"]
    print(url_depart)
    
    soup_depart = BeautifulSoup(requests.get(url_depart).text, 'html.parser')
    depart_name = url_depart.split("/")[-2][:-3]
    depart_number = url_depart.split("_")[-1][:-1]
    i=0
    for circo in (soup_depart.findAll("section")[3].findAll("a", href=True)):
        circo_number = i
        url_circo = circo["href"]
        
        if url_circo in circos_url:
            continue
        else : 
            circos_url.append(url_circo)
            i+=1
        
        try:
            soup_circo = BeautifulSoup(requests.get(url_circo).text, 'html.parser')
        except:
            print("error at : ", url_circo)
            continue
            
        candidats = soup_circo.findAll("section")[3].findAll("div", {"class" : re.compile("^candidate")})[:-2]
        
        for candidat in candidats: 
            try:
                name = candidat.find("span", {"class": "name"}).text
                parti = candidat.find("span", {"class": "party"}).text
                score = candidat.find("span", {"class": "percent"}).text
                df.loc[index] = [depart_name, depart_number, circo_number, name, parti, score]
                index+=1
            except :
                print("error at ", depart_name, circo_number)
                continue


df.shape
df2 = df.copy()<jupyter_output><empty_output><jupyter_text>On récupère à la main certaines valeurs :<jupyter_code>url_alamano = ["http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_1ere-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_2eme-circonscription", 
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_3eme-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_4eme-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_5eme-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_6eme-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/la-reunion_974/la-reunion_7eme-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/guyane_973/guyane_1ere-circonscription",
              "http://www.francetvinfo.fr/elections/resultats/guyane_973/guyane_2eme-circonscription"]
for i, url in enumerate(url_alamano[-2:]):
    soup_circo = BeautifulSoup(requests.get(url).text, 'html.parser')
    for item in soup_circo.findAll("section")[3].findAll("div", {"class" : re.compile("^candidate")})[:-2]:
        name = item.find("span", {"class": "name"}).text
        parti = item.find("span", {"class": "party"}).text
        score = item.find("span", {"class": "percent"}).text
        df2.loc[df2.shape[0]+1] = ["guyane", 973, i, name, parti, score]

df2.shape
df2.head()<jupyter_output><empty_output><jupyter_text>On gère les nom prénom<jupyter_code>df2.score = df2.score.map(lambda x : float(x.replace(",", ".").replace("%", "").strip()))
df2.nom = df2.nom.map(lambda s : unicodedata.normalize('NFD', s).encode('ascii', 'ignore').decode("utf-8") )
df2["prenom"] = df2.nom.map(lambda x : x.split(" ")[0].upper())
df2.nom = df2.nom.map(lambda x : x.split(" ")[1].upper())
df2.circo +=1
df2.head()<jupyter_output><empty_output><jupyter_text>On crée un code pour chaque circo<jupyter_code>def f(x):
    try:
        a = float(x)
    except:
        a = x
    return a
df2.code = df2.code.map(lambda x : str(f(x))[:-2])
df2.circo = df2.circo.map(lambda x : str(x)[:-2])
df2["code_dep"] = df2.code
df2["code"] = df2[['code', 'circo']].apply(lambda x: '|'.join(x), axis=1)<jupyter_output><empty_output><jupyter_text>Export to csv<jupyter_code>df2.to_csv("data/resultats_1ertour.csv", index=False)
df2.head()<jupyter_output><empty_output><jupyter_text>On récupère les pred<jupyter_code>pred = pd.read_excel("data/predictions_du_7_6_15h35.xlsx")
pred.shape
pred["nom"] = pred["nom"].map(lambda x : x.split(" ")[0]) 
df2.merge(pred[["nom", "prenom", "code", "prediction"]], how="left", on=["nom", "prenom", "code"])<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>temp = df2.merge(pred[["nom", "prenom", "code", "prediction"]], how="left", on=["nom", "prenom", "code"])
temp.shape<jupyter_output><empty_output><jupyter_text>On vire les na<jupyter_code>temp = temp.dropna(axis=0)
temp.shape
temp.prediction = temp.prediction*100
np.round(temp["score"].max())<jupyter_output><empty_output><jupyter_text>Quelques indic de perfo  MAE<jupyter_code>mae = []
gap = []
for i in range(int(temp["score"].min()), int(np.round(temp["score"].max())), 1):
    ttemp = temp[temp["score"]>i]
    gap.append(i)
    mae.append(mean_absolute_error(ttemp.prediction, ttemp.score))
plt.plot(gap, mae)
plt.xlabel("seuil de score >")
plt.ylabel("MAE")
plt.show()
mean_absolute_error(temp.prediction, temp.score)<jupyter_output><empty_output><jupyter_text>Répartition des erreurs<jupyter_code>temp["ecart"] = abs(temp.prediction - temp.score)
temp["ecart"].hist(bins=9)
plt.show()<jupyter_output><empty_output><jupyter_text>R2<jupyter_code>r2_score(temp.prediction, temp.score)<jupyter_output><empty_output><jupyter_text>Etude des ballotages<jupyter_code>def qualif(x):
    if x > 50:
        return "E"
    if x > 12.5:
        return "0"
    else: 
        return "N"
    
temp["qualif_true"] = temp["score"].map(lambda x : qualif(x))
temp["qualif_pred"] = temp["prediction"].map(lambda x : qualif(x))
temp[(temp["code"]=="56|4") | (temp["code"]=="75|1") | (temp["code"]=="80|5") | (temp["code"]=="972|3") | (temp["code"]=="986|1") ]
temp.loc[3418,"qualif_true"] = "N"
temp.loc[5116,"qualif_true"] = "N"
temp.loc[7433,"qualif_true"] = "N"
temp.loc[5961,"qualif_true"] = "N"
temp.loc[7510,"qualif_true"] = "N"

print(classification_report(temp["qualif_true"], temp["qualif_pred"]))<jupyter_output>precision    recall  f1-score   support

          0       0.84      0.86      0.85      1572
          E       0.00      0.00      0.00         5
          N       0.96      0.96      0.96      5823

avg / total       0.94      0.94      0.94      7400
{'fraction_non_alphanumeric': 0.18094804010938925, 'fraction_numerical': 0.030233971437253114, 'mean_word_length': 3.7291666666666665, 'pattern_counts': {'":': 7, '<': 48, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '216579', 'license': 'MIT', 'n_tokens_mistral': 2590, 'n_tokens_neox': 2500, 'n_words': 344, 'path': 'legislatives/.ipynb_checkpoints/notebook_scrap_resultats1ertour-checkpoint.ipynb', 'repo_name': 'verycourt/Elections', 'size': '6582'}