<jupyter_start><jupyter_text>Introduction  Traitement des données<jupyter_code>import os
import pandas as pd
from sklearn.impute import SimpleImputer
import matplotlib.pyplot as plt
import numpy as np
import tarfile
from six.moves import urllib
from sklearn.preprocessing import OrdinalEncoder
import seaborn as sns

DOWNLOAD_ROOT = "https://raw.githubusercontent.com/Snargol/projet-IA-CESI/main/"
DATA_PATH = os.path.join("datasets", "data")
# Méthode de récupération des données
def fetch_data(data_url=DOWNLOAD_ROOT,data_path = DATA_PATH):
  if not os.path.isdir(data_path):
        os.makedirs(data_path)
  employee_url = data_url+"employee_survey_data_full.csv"
  manager_url =  data_url +"manager_survey_data_full.csv"
  general_url =  data_url +"general_data_full.csv"

  employee_path = os.path.join(data_path, "employee_survey_data_full.csv")
  manager_path = os.path.join(data_path, "manager_survey_data_full.csv")
  general_path = os.path.join(data_path, "general_data_full.csv")
  urllib.request.urlretrieve(employee_url, employee_path)
  urllib.request.urlretrieve(manager_url, manager_path)
  urllib.request.urlretrieve(general_url, general_path)

fetch_data()

def load_data(name):
    csv_path = os.path.join("./datasets/data/"+name+".csv")
    return pd.read_csv(csv_path)
def load_final_data():
  general_data = load_data("general_data_full")
  employee_survey_data = load_data("employee_survey_data_full")
  manager_survey_data = load_data("manager_survey_data_full")
  temp_result = pd.merge(general_data,employee_survey_data,on='EmployeeID')
  result = pd.merge(temp_result,manager_survey_data,on='EmployeeID')
  return result

columns_to_fill = ['NumCompaniesWorked','TotalWorkingYears','EnvironmentSatisfaction','JobSatisfaction','WorkLifeBalance',]
binary_columns = ['Attrition','Gender']
nominal_columns = ['Department','EducationField','JobRole','MaritalStatus']
ordinal_columns =  [
    {
        "label":'BusinessTravel',
        "order":['Non-Travel','Travel_Rarely','Travel_Frequently']
    }
]


data = load_final_data()
# Supprime les colonnes dont l'écart-type vaut 0 (une donnée unique pour toutes les lignes de la colonne)
def del_std_of_0(_data):
    _deleted_columns = []
    for each in _data.describe().columns :
      if _data.describe()[str(each)]['std'] == 0:
        _data = _data.drop(columns=[each])
        _deleted_columns.append([each])
    print("Deleted Columns : ")
    print(_deleted_columns)
    return _data
  
data = del_std_of_0(data.copy())

def check_contains_nan(_data):
  count_nan = 0
  for each in columns_to_fill:
    temp_count_nan = count_nan + _data[each].isna().sum()
    count_nan= temp_count_nan
  return count_nan

check_contains_nan(data)

def fill_data(_data_to_process) :

  imputer = SimpleImputer(strategy="median")
  
  _data_to_process_index = data_num.index
  _data_to_process_labels = data_num.columns


  imputer.fit(_data_to_process)
  data_filled_temp = imputer.transform(_data_to_process)
  data_filled = pd.DataFrame(data_filled_temp,columns=_data_to_process_labels)
  return data_filled


def delete_nan_row(_data_to_process):
  for each in columns_to_fill:
    _data_to_process.drop(_data_to_process[_data_to_process[each].isna()].index,inplace=True)
  return _data_to_process

data_num = data.select_dtypes(include=[np.number])
data_num = fill_data(data_num)
# we concat the data
data_cat = data.select_dtypes(object)
dataset = pd.concat([data_num, data_cat], axis=1)
dataset.head()<jupyter_output><empty_output><jupyter_text>Répartition des données<jupyter_code>#Afficher le Dataset pour constater la répartition

fig = plt.figure(figsize=(15,15))
for index,i in enumerate(dataset.columns.tolist()):
    ax=plt.subplot(8,4,index+1)
    sns.countplot(x=i,data=dataset,ax=ax)
fig.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Proportion d'Attrition  TODO : le séparer en plusieurs parties : d'abord sur les ordinal puis les nominales et enfin numérique. Virer avant les données inutiles<jupyter_code>#on trace le nombre d'Attrition et le pourcentage d'attrition
def drawAttritionRepart(listColumns):
    fig = plt.figure(figsize=(15,(len(listColumns) / 2) * 3.75))
    for idx,i in enumerate(listColumns):
        index = idx * 2;
        crosstab = pd.crosstab(index=dataset[i], columns=dataset["Attrition"])
        ax1=plt.subplot(len(listColumns) / 2,4,index+1)
        crosstab.plot(kind="bar",stacked=True,ax=ax1)
        plt.title(i + ' / Attrition | ∑')
        plt.xlabel(i)
        plt.ylabel('nb Employees')

        ax2=plt.subplot(len(listColumns) / 2,4,index+2)
        table= pd.crosstab(dataset[i],dataset.Attrition)
        table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True,ax=ax2)
        plt.title(i + ' / Attrition | %')
        plt.xlabel(i)
        plt.ylabel('Number of Employees')

    fig.tight_layout()
    plt.show()
    
drawAttritionRepart(['Age', 'Education', 'JobLevel', 'NumCompaniesWorked'])
#Afficher le Dataset selon l'attrition afin de déterminer des tendances

fig = plt.figure(figsize=(26,30))
for idx,i in enumerate(dataset.columns.tolist()[:len(dataset) - 2]):
    ax =plt.subplot(7,4,idx+1)
    ax.scatter(dataset[i], dataset.Attrition)
    ax.title.set_text(str(i))
fig.tight_layout()
plt.show()

#Pair-Plot
num_col_eda=['Age','DistanceFromHome','PercentSalaryHike','MonthlyIncome','TotalWorkingYears','YearsAtCompany']
num_attrition=num_col_eda+['Attrition']
sns.pairplot(dataset[num_attrition], hue = 'Attrition')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12368706990220935, 'fraction_numerical': 0.007243752263672582, 'mean_word_length': 5.61437125748503, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '638475', 'license': 'MIT', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1970, 'n_words': 321, 'path': '.ipynb_checkpoints/Projet_IA-checkpoint.ipynb', 'repo_name': 'Snargol/projet-IA-CESI', 'size': '5522'}