<jupyter_start><jupyter_text><jupyter_code>## upload files and import library
from google.colab import files
#uploaded = files.upload()

import pandas as pd
import numpy as np
from scipy.stats.mstats import winsorize
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from scipy.stats.mstats import winsorize
from scipy import stats
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report<jupyter_output><empty_output><jupyter_text>**Import Dataset**<jupyter_code>train = pd.read_csv('train_data.csv',index_col=0)
test = pd.read_csv('test_data.csv',index_col=0)

## clean train
train = train[train.Sector != 'X1']
train = train[train.Sector != 'TRUE']

# ##to test stability-- randomly select 99% of the train data
# train = train.sample(frac=0.99)
train = pd.get_dummies(train, columns=['Sector'])
test = pd.get_dummies(test, columns=['Sector'])<jupyter_output><empty_output><jupyter_text>Version 1: mearge train test<jupyter_code>df = pd.concat([train, test], join="outer") 

df = df.iloc[:-2].drop(['X1','PRICE_VAR','year'],axis=1)
cols = df.select_dtypes(exclude=['float']).columns
df[cols] = df[cols].apply(pd.to_numeric, downcast='float', errors='coerce')


for col in df: 
     df[col] = winsorize(df[col], limits=[0.05,0.05])


X = df.drop(['Class'],axis=1)
y = df['Class']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)
print(y_train.value_counts(normalize=True))
print(y_test.value_counts(normalize=True))<jupyter_output>1.0    0.548414
0.0    0.451586
Name: Class, dtype: float64
1.0    0.560136
0.0    0.439864
Name: Class, dtype: float64<jupyter_text>Version 2: use train and test<jupyter_code># X_train = train.iloc[:-2].drop(['X1','PRICE_VAR','year','Class'],axis=1)
# X_test = test.iloc[:-2].drop(['X1','PRICE_VAR','year','Class'],axis=1)
# cols = X_train.select_dtypes(exclude=['float']).columns
# cols2 = X_test.select_dtypes(exclude=['float']).columns
# X_train[cols] = X_train[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
# X_test[cols2] = X_test[cols2].apply(pd.to_numeric, downcast='float', errors='coerce')

# y_train = train.iloc[:-2][['Class']]
# y_test = test.iloc[:-2][['Class']]

# y_train['Class'] =y_train['Class'].apply(pd.to_numeric, downcast='float', errors='coerce')

# X_train.fillna(0, inplace=True)
# X_test.fillna(0, inplace=True)

# ##check na in y
# y_train[pd.isnull(y_train).any(axis=1)]
# # y_train.dropna(inplace = True)
# # X_train.drop(y_train.index[[7929,12727,12728]],inplace=True)
print('Training Features Shape:', X_train.shape)
print('Training Labels Shape:', y_train.shape)
print('Testing Features Shape:', X_test.shape)
print('Testing Labels Shape:', y_test.shape)<jupyter_output>Training Features Shape: (17660, 232)
Training Labels Shape: (17660,)
Testing Features Shape: (4415, 232)
Testing Labels Shape: (4415,)<jupyter_text>--- **Outliers and Standardization**<jupyter_code># from scipy.stats.mstats import winsorize
# from scipy import stats
# ## remove outliers 5% higher than the max and 5% lower than the min using for loop and winsorize
# for col in X_train: 
#      X_train[col] = winsorize(X_train[col], limits=[0.05,0.05])
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler() # calculate mean and standard deviation of train set
scaler.fit(X_train)


scaler2 = StandardScaler() # calculate mean and standard deviation of test set
scaler2.fit(X_test)


X_train_scaled = scaler.transform(X_train) 
X_test_scaled = scaler2.transform(X_test)<jupyter_output><empty_output><jupyter_text>**Check for balance**<jupyter_code># y_train['Class'].value_counts(normalize=True)
y_train.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>--- **Feature Importance - Random Forest**<jupyter_code>#check
np.any(np.isnan(y_train))
# #Import Random Forest Model
# from sklearn.ensemble import RandomForestClassifier

# #Create a Gaussian Classifier
# clf=RandomForestClassifier(n_estimators=100)

# #Train the model using the training sets y_pred=clf.predict(X_test)
# clf.fit(X_train,y_train)

# y_pred=clf.predict(X_test)
# RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
#             max_depth=None, max_features='auto', max_leaf_nodes=None,
#             min_impurity_decrease=0.0, min_impurity_split=None,
#             min_samples_leaf=1, min_samples_split=2,
#             min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,
#             oob_score=False, random_state=None, verbose=0,
#             warm_start=False)
# feature_imp = pd.Series(clf.feature_importances_,index=X_train.columns).sort_values(ascending=False)
# feature_imp
# feature_imp[:20]

# #Import scikit-learn metrics module for accuracy calculation
# from sklearn import metrics
# # Model Accuracy, how often is the classifier correct?
# print("Accuracy:",metrics.accuracy_score(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>**Test for Stability**<jupyter_code># features = pd.DataFrame()
# i = 1
# while i < 5:
#   train = pd.read_csv('train_data.csv',index_col=0)
#   test = pd.read_csv('test_data.csv',index_col=0)
#   train = train[train.Sector != 'X1']
#   train = train[train.Sector != 'TRUE']
#   train = train.sample(frac=0.99)
#   train = pd.get_dummies(train, columns=['Sector'])
#   test = pd.get_dummies(test, columns=['Sector'])
#   X_train = train.iloc[:-2].drop(['X1','PRICE_VAR','year','Class'],axis=1)
#   X_test = test.iloc[:-2].drop(['X1','PRICE_VAR','year','Class'],axis=1)
#   cols = X_train.select_dtypes(exclude=['float']).columns
#   cols2 = X_test.select_dtypes(exclude=['float']).columns
#   X_train[cols] = X_train[cols].apply(pd.to_numeric, downcast='float', errors='coerce')
#   X_test[cols2] = X_test[cols2].apply(pd.to_numeric, downcast='float', errors='coerce')
#   y_train = train.iloc[:-2][['Class']]
#   y_test = test.iloc[:-2][['Class']]
#   y_train['Class'] =y_train['Class'].apply(pd.to_numeric, downcast='float', errors='coerce')
#   X_train.fillna(0, inplace=True)
#   X_test.fillna(0, inplace=True)
#   for col in X_train: 
#      X_train[col] = winsorize(X_train[col], limits=[0.05,0.05])
#   #Create a Gaussian Classifier
#   clf=RandomForestClassifier(n_estimators=100)
#   #Train the model using the training sets y_pred=clf.predict(X_test)
#   clf.fit(X_train,y_train)
#   y_pred=clf.predict(X_test)
#   feature_imp = pd.DataFrame( clf.feature_importances_,index=X_train.columns).sort_values( by = 0, ascending=False)
#   features =  pd.concat([features, feature_imp], axis=1)
#   i = i + 1
from sklearn.preprocessing import StandardScaler
features = pd.DataFrame()
i = 1
while i < 31:
  X_train_fe, X_valid, y_train_fe, y_valid = train_test_split(X_train, y_train, test_size=0.10, random_state=0)
  # calculate mean and standard deviation of train set
  X_train_fe = StandardScaler().fit(X_train_fe).transform(X_train_fe) 
  X_valid = StandardScaler().fit(X_valid).transform(X_valid)  
  #Create a Gaussian Classifier
  clf=RandomForestClassifier(n_estimators=100)
  #Train the model using the training sets y_pred=clf.predict(X_test)
  clf.fit(X_train_fe,y_train_fe)
  y_pred=clf.predict(X_valid)
  feature_imp = pd.DataFrame( clf.feature_importances_,index=X_train.columns).sort_values( by = 0, ascending=False)
  features =  pd.concat([features, feature_imp], axis=1)
  i = i + 1
# print(features[0].index)
# print(features[1].index)
# print(features[2].index)
features
# Initialize an empty array to hold feature importances
features_imp = features.mean(axis=1)
features_imp.sort_values(ascending=False)[:20]
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
# Creating a bar plot
sns.barplot(x=features_imp.sort_values(ascending=False)[:20], y=features_imp.sort_values(ascending=False)[:20].index,palette = 'Blues')
# Add labels to your graph
plt.xlabel('Feature Importance Score')
plt.ylabel('Features')
plt.title("Important Features")
plt.legend()
# plt.savefig('FE_Plot_RandomForest.png',dpi=300,bbox_inches="tight")
plt.show()

# features['cumulative_importance'] = 

features_cum_plot = np.cumsum(features_imp)
features_cum_plot = pd.DataFrame(features_cum_plot)
features_cum_plot
# 51 features required for 0.90 of cumulative importance
len(features_cum_plot[(features_cum_plot > 0.90).all(1)])

# Cumulative importance plot
    plt.figure(figsize = (8, 6))
    plt.plot(list(range(len(features_cum_plot))), features_cum_plot, 'r-')
    plt.xlabel('Number of Features'); plt.ylabel('Cumulative Importance'); 
    plt.title('Cumulative Feature Importance');
    # plt.savefig('FE__cum_importance_RandomForest.jpg',dpi=300,bbox_inches="tight");
    plt.show();

    
features_imp.sort_values(ascending=False)[:20].index
features_imp.sort_values(ascending=False)[:51].index<jupyter_output><empty_output><jupyter_text>**Save df**<jupyter_code>selected_features = features_imp.sort_values(ascending=False)[:20].index
train_20 = X_train[selected_features]
test_20 = X_test[selected_features]
selected_features = features_imp.sort_values(ascending=False)[:51].index
train_51 = X_train[selected_features]
test_51 = X_test[selected_features]
train_20.to_csv("train_20.csv")
test_20.to_csv("test_20.csv")

train_51.to_csv("train_51.csv")
test_51.to_csv("test_51.csv")
y_train.to_csv("y_train.csv")
y_test.to_csv("y_test.csv")
# sns.heatmap(X_train.corr(),annot=True,cmap='RdYlGn',linewidths=0.2) #data.corr()-->correlation matrix
# fig=plt.gcf()
# fig.set_size_inches(10,8)
# plt.show()
#sns.pairplot(X, hue="Class")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14911193721602645, 'fraction_numerical': 0.026228831061544815, 'mean_word_length': 6.692613185067514, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '437516', 'license': 'MIT', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3577, 'n_words': 609, 'path': 'BA888_FE_Kiki.ipynb', 'repo_name': 'CNIU1997/BA_888', 'size': '9684'}