<jupyter_start><jupyter_text>Projet de Machine Learning : Test de classification bout en bout   1.Chargement des données<jupyter_code>#importation des modules necessaires
import pyspark
from pyspark.sql import SparkSession
import joblib
#creation d'une session spark 

mon_spark=SparkSession.builder.master("local").appName("MLproject").getOrCreate()

ccdefault = mon_spark.read.format("csv").options(header=True,inferSchema=True).load("Doc_Evaluation/session pratique/data/ccdefault.csv")
# Affichage des 3 premières lignes
ccdefault.show(3)<jupyter_output>+---+---------+---+---------+--------+---+-----+-----+-----+-----+-----+-----+---------+---------+---------+---------+---------+---------+--------+--------+--------+--------+--------+--------+-------+
| ID|LIMIT_BAL|SEX|EDUCATION|MARRIAGE|AGE|PAY_0|PAY_2|PAY_3|PAY_4|PAY_5|PAY_6|BILL_AMT1|BILL_AMT2|BILL_AMT3|BILL_AMT4|BILL_AMT5|BILL_AMT6|PAY_AMT1|PAY_AMT2|PAY_AMT3|PAY_AMT4|PAY_AMT5|PAY_AMT6|DEFAULT|
+---+---------+---+---------+--------+---+-----+-----+-----+-----+-----+-----+---------+---------+---------+---------+---------+---------+--------+--------+--------+--------+--------+--------+-------+
|  1|    20000|  2|        2|       1| 24|    2|    2|   -1|   -1|   -2|   -2|     3913|     3102|      689|        0|        0|        0|       0|     689|       0|       0|       0|       0|      1|
|  2|   120000|  2|        2|       2| 26|   -1|    2|    0|    0|    0|    2|     2682|     1725|     2682|     3272|     3455|     3261|       0|    1000|    1000|    1000|       0|    2000|    [...]<jupyter_text>2. Analyse exploratoire<jupyter_code># Affichage des attributs
ccdefault.printSchema()
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt

# Conversion en dataframe pandas pour une meilleure manipulation
ccdefault.pd=ccdefault.toPandas()


# Changement de nom de colonnnes, la colonne "PAY_0" devrait plutot etre "PAY_1"

ccdefault.pd.rename(columns={'PAY_0':'PAY_1'},inplace = True)
ccdefault.pd.columns
## Quelques statistiques des données 
ccdefault.pd.describe()<jupyter_output><empty_output><jupyter_text>2.1 Distribution des variables continues<jupyter_code>## variables quantitatives 
var_quant=ccdefault.pd[['LIMIT_BAL','AGE','BILL_AMT4','BILL_AMT5','BILL_AMT6','PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']]

var_quant
for col in var_quant:
    plt.figure()
    sns.distplot(var_quant[col])<jupyter_output>/home/tabsoba/ML/lib/python3.8/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)
/home/tabsoba/ML/lib/python3.8/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)
/home/tabsoba/ML/lib/python3.8/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an ax[...]<jupyter_text>2.2 Distribution des variables qualitatives<jupyter_code>## variables qualitatives
var_qual=ccdefault.pd[['SEX','MARRIAGE','EDUCATION','PAY_1','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6']]

var_qual
## Presentation des differentes catégories pour chaque variable qualitative
for col in var_qual:
    print(f'{col:-<50} {var_qual[col].unique()}')
## Presentation des differentes catégories dans chaque variable
for col in var_qual:
    plt.figure()
    var_qual[col].value_counts().plot.pie()<jupyter_output><empty_output><jupyter_text>2.3 Relation entre variables explicatives et variable cible<jupyter_code># Changement du code de 'SEX' (1,2) en 'F' and 'M' 

ccdefault.pd["SEX"]=ccdefault.pd["SEX"].map({1:'M',2:'F'}).astype('category')
ccdefault.pd["SEX"].dtypes

# Creation d'une nouvelle colonne nommée "RET_PAY" indiquant les clients ayant au moins un retard de payement de PAY_1 to Pay_6
# 0 : PAS_RETARD ; 1: RETARD

condition = (ccdefault.pd.PAY_1 >1) | (ccdefault.pd.PAY_2 >1) | (ccdefault.pd.PAY_3 >1) | (ccdefault.pd.PAY_4 >1) | (ccdefault.pd.PAY_5 >1) | (ccdefault.pd.PAY_6 >1)
ccdefault.pd.loc[condition, "RET_PAY"] = 1
ccdefault.pd.loc[ccdefault.pd.RET_PAY.isna(),"RET_PAY"] = 0

ccdefault.pd
# fonction pour representer les relations entre les attributs et la variable cible 

def relation_var(nom_colonne):

    # Get the percentage of default by each group
    status_rembourse_group = pd.crosstab(index=ccdefault.pd['RET_PAY'],columns = ccdefault.pd[nom_colonne], normalize = 'columns')
    
    # Round up to 2 decimal
    status_rembourse_group = status_rembourse_group.apply(lambda x: round(x,2))
    
    labels =  status_rembourse_group.columns
    list1 =  status_rembourse_group.iloc[0].to_list()
    list2 =  status_rembourse_group.iloc[1].to_list()
    
    list1_name = "Remboursé"
    list2_name = "Non Remboursé"
    title = f"Default by {nom_colonne}"
    xlabel = nom_colonne
    ylabel = "Pourcentage de non remboursement"
    
    fig, ax = plt.subplots(figsize=(10, 5))
    bar_width = 0.5
    
    ax1 = ax.bar(labels,list1, bar_width, label = list1_name)
    ax2 = ax.bar(labels,list2, bar_width, bottom = list1, label = list2_name)

    ax.set_title(title, fontweight = "bold")
    ax.set_xlabel(xlabel, fontweight = "bold")
    ax.set_ylabel(ylabel, fontweight = "bold")
    ax.legend(loc="best")
    
    plt.xticks(list(range(len(labels))), labels,rotation=90)
    plt.yticks(fontsize=9)

    for r1, r2 in zip(ax1, ax2):
        h1 = r1.get_height()
        h2 = r2.get_height()
        plt.text(r1.get_x() + r1.get_width() / 2., h1 / 2., f"{h1:.0%}", ha="center", va="center", color="white", fontsize=9, fontweight="bold")
        plt.text(r2.get_x() + r2.get_width() / 2., h1 + h2 / 2., f"{h2:.0%}", ha="center", va="center", color="white", fontsize=9, fontweight="bold")

    plt.show()<jupyter_output><empty_output><jupyter_text>Relation entre la variable cible et le sexe, le status matrimonial et le niveau d'education des clients<jupyter_code>var_qual1=["SEX","MARRIAGE","EDUCATION"]

for col in var_qual1:
    relation_var(col)<jupyter_output><empty_output><jupyter_text>Relation variable cible et age des clients<jupyter_code>bornes= [21,30,40,50,60,70,80]
tranches_ages = ['20-30','30-40','40-50','50-60','60-70','70-80']
ccdefault.pd['AGE'] = pd.cut(ccdefault.pd['AGE'],bins=bornes, labels=tranches_ages ,right=False)

relation_var('AGE')<jupyter_output><empty_output><jupyter_text>2.4 Tendance des paiements entre avril 2005 et septembre 2005<jupyter_code># Extraction des clients ayant au moins un retard de payement entre avril et septembre

retard= ccdefault.pd[ccdefault.pd['RET_PAY']== 1]
tendance= retard[['PAY_6','PAY_5','PAY_4','PAY_3','PAY_2','PAY_1']].sum(axis=0)

fig,ax = plt.subplots()
ax.plot(tendance)
plt.xticks(['PAY_6','PAY_5','PAY_4','PAY_3','PAY_2','PAY_1'],['Apr','May','Jun','Jul','Aug','Sep'])

plt.xlabel('Mois',fontweight='bold')
plt.ylabel('Total mois payés en retard',fontweight='bold')
plt.title('tendance paiements retardés',fontweight='bold')

plt.show()
# Correlation entre la consommation et le temps de retard mis pour le paiement

from matplotlib.pyplot import figure

retard= ccdefault.pd[ccdefault.pd['RET_PAY']== 1]
paiements = [ f"PAY_{i}" for i in range(1, 7) ]
consommation= [ f"BILL_AMT{i}" for i in range(1, 7) ]

fig, ax = plt.subplots(3,2, figsize=(10, 10))

for paie, conso, m in zip(paiements, consommation, ax.flatten()):
    
   
    data = []
    for i in sorted(retard[paie].unique()):
        temp = retard.loc[retard[paie] == i, conso]
        data.append(temp)
    m.boxplot(data, showfliers=False,) 
    m.set_xticklabels(sorted(retard[paie].unique()))
    
plt.show()
# Correlation entre le crédit alloué et la variable cible
# 1: non remboursé ; 0: remboursé

def0 = ccdefault.pd.loc[ccdefault.pd['DEFAULT'] == 0,'LIMIT_BAL']
def1 = ccdefault.pd.loc[ccdefault.pd['DEFAULT'] == 1,'LIMIT_BAL']

fig, ax = plt.subplots()
ax.boxplot([def0, def1],  showfliers=False)

ax.set_xticklabels(['Remboursé',"Non remboursé"],fontweight ='bold')
ax.set_ylabel('Crédit alloué',fontweight ='bold')
ax.set_title('Crédit alloué & Etat du remboursement',fontweight ='bold')

plt.show()<jupyter_output><empty_output><jupyter_text>3. Etude comparative de modèles de machine learning pour la prediction de la variable cible<jupyter_code>## Valeurs manquantes 

for c in ccdefaultNew.columns:
    count=ccdefaultNew.filter(c+" is   NULL" or c+"is ''" or c+"is  NaN" or c+"is  null").count()
    print(str(count) +" valeurs manquantes dans la colonne "+ c)
from pyspark.ml.feature import VectorAssembler, StringIndexer, VectorIndexer, MinMaxScaler
from pyspark.ml import Pipeline
from pyspark.sql.functions import *

from pyspark.ml.classification import LogisticRegression
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.classification import RandomForestClassifier

from pyspark.ml.evaluation import  BinaryClassificationEvaluator

# Drop ID column
ccdefault = ccdefault .select(ccdefault .schema.names[1:])

# Split data into training and test sample
splits = ccdefault.randomSplit([0.75, 0.25])
ccdefault_train = splits[0]
ccdefault_test = splits[1]

# Get and convert categorical features (SEX, EDUCATION, MARRIAGE)
categorical_features = ccdefault.schema.names[1:4]
catVect = VectorAssembler(inputCols = categorical_features, outputCol = "catFeatures")
catIdx = VectorIndexer(inputCol = catVect.getOutputCol(), outputCol = "idxCatFeatures")


# Get and normalize numerical features
numerical_features = ccdefault.schema.names[0:1] + ccdefault.schema.names[4:]
numVect = VectorAssembler(inputCols = numerical_features, outputCol = "numFeatures")
minMax = MinMaxScaler(inputCol = numVect.getOutputCol(), outputCol = "normFeatures")


# Define pipeline 
featVect = VectorAssembler(inputCols=["idxCatFeatures", "normFeatures"], outputCol = "features")
pipeline = Pipeline(stages = [catVect, catIdx, numVect, minMax, featVect])
pipeline_object = pipeline.fit(ccdefault_train)

# Run training and test data through the pipeline
ccdefault_train = pipeline_object.transform(ccdefault_train).select("features", col("DEFAULT").alias("label"))
ccdefault_test = pipeline_object.transform(ccdefault_test).select("features", col("DEFAULT").alias("label"))







accuracy = MulticlassClassificationEvaluator(
    labelCol = "label", predictionCol = "prediction", metricName = "accuracy")
precision = MulticlassClassificationEvaluator(
    labelCol = "label", predictionCol = "prediction", metricName = "weightedPrecision")
recall = MulticlassClassificationEvaluator(
    labelCol = "label", predictionCol = "prediction", metricName = "weightedRecall")<jupyter_output><empty_output><jupyter_text>3.1 Regression logistique<jupyter_code>logit = LogisticRegression(labelCol = "label", featuresCol = "features", maxIter = 20, regParam = 0.2)
model = logit.fit(ccdefault_train)
predictions = model.transform(ccdefault_test)

predictions.select("prediction", "label", "features").show(5)

# select (prediction, true label) and compute test error
evaluator = BinaryClassificationEvaluator() # MulticlassClassificationEvaluator().setLabelCol("label").setPredictionCol("prediction").setMetricName("rmse")
auc = evaluator.evaluate(predictions)
print("AUC = "+ str(auc))<jupyter_output>+----------+-----+--------------------+
|prediction|label|            features|
+----------+-----+--------------------+
|       0.0|    0|[1.0,-3768.0,-393...|
|       0.0|    0|[1.0,-2640.0,0.0,...|
|       1.0|    1|(15,[0,1,2,4,5,6,...|
|       0.0|    0|[1.0,-2000.0,780....|
|       0.0|    0|(15,[0,1,2,4,5,6,...|
+----------+-----+--------------------+
only showing top 5 rows

AUC = 0.9999992147530933<jupyter_text>3.2 Arbre de decision<jupyter_code>dt = DecisionTreeClassifier().setLabelCol("label").setFeaturesCol("features")

# train the model
dtModel = dt.fit(ccdefault_train)

# make predictions on the test data
predictions = dtModel.transform(ccdefault_test)
predictions.select("prediction", "label", "features").show(5)

# select (prediction, true label) and compute test error
evaluator = BinaryClassificationEvaluator() # MulticlassClassificationEvaluator().setLabelCol("label").setPredictionCol("prediction").setMetricName("rmse")
auc = evaluator.evaluate(predictions)
print("AUC = "+ str(auc))<jupyter_output>+----------+-----+--------------------+
|prediction|label|            features|
+----------+-----+--------------------+
|       1.0|    1|[1.0,1.0,-6029.0,...|
|       0.0|    0|[1.0,1.0,-6028.0,...|
|       0.0|    0|[1.0,1.0,-3768.0,...|
|       0.0|    0|[1.0,1.0,-2640.0,...|
|       0.0|    0|(16,[0,1,2,4,5,6,...|
+----------+-----+--------------------+
only showing top 5 rows

AUC = 1.0<jupyter_text>3.3 Forêt aleatoire<jupyter_code># TODO: Replace <FILL IN> with appropriate code

rf =   RandomForestClassifier().setLabelCol("label").setFeaturesCol("features")

# train the model
rfModel = rf.fit(ccdefault_train)

# make predictions on the test data
predictions = rfModel.transform(ccdefault_test)
predictions.select("prediction", "label", "features").show(5)

# select (prediction, true label) and compute test error
evaluator = BinaryClassificationEvaluator() # MulticlassClassificationEvaluator().setLabelCol("label").setPredictionCol("prediction").setMetricName("rmse")
auc = evaluator.evaluate(predictions)
print("AUC = "+ str(auc))<jupyter_output>+----------+-----+--------------------+
|prediction|label|            features|
+----------+-----+--------------------+
|       1.0|    1|[1.0,1.0,-6029.0,...|
|       0.0|    0|[1.0,1.0,-6028.0,...|
|       0.0|    0|[1.0,1.0,-3768.0,...|
|       0.0|    0|[1.0,1.0,-2640.0,...|
|       0.0|    0|(16,[0,1,2,4,5,6,...|
+----------+-----+--------------------+
only showing top 5 rows

Root Mean Squared Error (RMSE) on test data = 1.0
{'fraction_non_alphanumeric': 0.1883062254627033, 'fraction_numerical': 0.03855860908581043, 'mean_word_length': 3.861963190184049, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '4263', 'license': 'MIT', 'n_tokens_mistral': 5330, 'n_tokens_neox': 5079, 'n_words': 1011, 'path': 'Projet_ML.ipynb', 'repo_name': 'maigaabdoul/Projet_ML', 'size': '14264'}