<jupyter_start><jupyter_text>**Fancyimpute Missing Data**<jupyter_code>!pip install fancyimpute
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno
%matplotlib inline
from fancyimpute import KNN, NuclearNormMinimization, SoftImpute, IterativeImputer, BiScaler
from fancyimpute import IterativeImputer
from google.colab import files
uploaded = files.upload()
df=pd.read_excel('Imputation_DataSet.xlsx')
df.head(2)
missing_val_count_by_column = (df.isnull().sum())
print(missing_val_count_by_column[missing_val_count_by_column > 0])
df.describe()
#Shows a heatmap of the missing values across the whole data set
plt.figure(figsize=(10,8))
sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='viridis')
missingdata_df=df.columns[df.isnull().any()].tolist()
msno.matrix(df[missingdata_df])
# Shows a bar chart of missing columns
msno.bar(df[missingdata_df], color="blue", log=True, figsize=(30,18))
df.columns
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(df)
df_scaled=scaler.transform(df)
df_scaled=pd.DataFrame(df_scaled, columns=['Surface Treating Pressure', 'Slurry Rate', 'Proppant Concentration'])
df_scaled
df_scaled.describe()
# Use # nearest rows which have a feature to fill in each row's missing features
X_filled_knn = KNN(k=2).fit_transform(df_scaled)
X_filled_knn=pd.DataFrame(X_filled_knn,columns=['Surface Treating Pressure', 'Slurry Rate', 'Proppant Concentration'])
X_filled_knn
X_filled_knn['Surface Treating Pressure']=(X_filled_knn['Surface Treating Pressure']*df['Surface Treating Pressure'].std())+df['Surface Treating Pressure'].mean()
X_filled_knn['Slurry Rate']=(X_filled_knn['Slurry Rate']*df['Slurry Rate'].std())+df['Slurry Rate'].mean()
X_filled_knn['Proppant Concentration']=(X_filled_knn['Proppant Concentration']*df['Proppant Concentration'].std())+df['Proppant Concentration'].mean()
X_filled_knn
X_filled_knn.to_csv('KNN_Imputation.csv', index=False)<jupyter_output><empty_output><jupyter_text>**Applying MICE**<jupyter_code>!pip install impyute
#Appying MICE Imputations
from impyute.imputation.cs import mice
imputed_training=mice(df_scaled.values)
imputed_training=pd.DataFrame(imputed_training,columns=['Surface Treating Pressure', 'Slurry Rate', 'Proppant Concentration'])
imputed_training['Surface Treating Pressure']=(imputed_training['Surface Treating Pressure']*df['Surface Treating Pressure'].std())+df['Surface Treating Pressure'].mean()
imputed_training['Slurry Rate']=(imputed_training['Slurry Rate']*df['Slurry Rate'].std())+df['Slurry Rate'].mean()
imputed_training['Proppant Concentration']=(imputed_training['Proppant Concentration']*df['Proppant Concentration'].std())+df['Proppant Concentration'].mean()
imputed_training
imputed_training.to_csv('MICE_Imputation.csv', index=False)
df.columns<jupyter_output><empty_output><jupyter_text>**Iterative imputer**<jupyter_code>from fancyimpute import IterativeImputer
#Model each feature with missing values as a function of other features, and use that estimate for imputation.
X_filled_ii = IterativeImputer(max_iter=10).fit_transform(df_scaled)
X_filled_ii=pd.DataFrame(X_filled_ii,columns=['Surface Treating Pressure', 'Slurry Rate', 'Proppant Concentration'])
X_filled_ii['Surface Treating Pressure']=(X_filled_ii['Surface Treating Pressure']*df['Surface Treating Pressure'].std())+df['Surface Treating Pressure'].mean()
X_filled_ii['Slurry Rate']=(X_filled_ii['Slurry Rate']*df['Slurry Rate'].std())+df['Slurry Rate'].mean()
X_filled_ii['Proppant Concentration']=(X_filled_ii['Proppant Concentration']*df['Proppant Concentration'].std())+df['Proppant Concentration'].mean()
X_filled_ii.to_csv('Iterative_Imputer_Imputation.csv', index=False)<jupyter_output><empty_output><jupyter_text>**Applying iterative imputer 5 times**<jupyter_code>#Applying iterative imputer 5 times and taking the mean
from fancyimpute import IterativeImputer
XY_incomplete = df_scaled

n_imputations = 5
XY_completed = []
for i in range(n_imputations):
    imputer = IterativeImputer(sample_posterior=True, random_state=i)
    XY_completed.append(imputer.fit_transform(XY_incomplete))

XY_completed_mean = np.mean(XY_completed, 0)
XY_completed_std = np.std(XY_completed, 0)

XY_completed=np.array(XY_completed)
XY_completed_mean=pd.DataFrame(XY_completed_mean,columns=['Surface Treating Pressure', 'Slurry Rate', 'Proppant Concentration'])
XY_completed_mean['Surface Treating Pressure']=(XY_completed_mean['Surface Treating Pressure']*df['Surface Treating Pressure'].std())+df['Surface Treating Pressure'].mean()
XY_completed_mean['Slurry Rate']=(XY_completed_mean['Slurry Rate']*df['Slurry Rate'].std())+df['Slurry Rate'].mean()
XY_completed_mean['Proppant Concentration']=(XY_completed_mean['Proppant Concentration']*df['Proppant Concentration'].std())+df['Proppant Concentration'].mean()
XY_completed_mean.to_csv('Mean_Iterative_Imputer.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1328643216080402, 'fraction_numerical': 0.003417085427135678, 'mean_word_length': 11.44, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '612861', 'license': 'MIT', 'n_tokens_mistral': 1797, 'n_tokens_neox': 1648, 'n_words': 301, 'path': 'Case Study 2/Fancyimpute Missing Data/Fancyimpute Missing Data.ipynb', 'repo_name': 'thirasit/Practical-ML-in-Petrol-with-Python', 'size': '4975'}