<jupyter_start><jupyter_text>data cleaning<jupyter_code>train_data = pd.read_csv('train_values.csv')
train_labels = pd.read_csv('train_labels.csv')
train_data = train_data.join(train_labels['damage_grade'])
train_data = pd.get_dummies(train_data)
dropped_columns = ['building_id', 'geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id']
train_data = train_data.drop(dropped_columns, axis=1).dropna()
train_data = train_data.replace([np.inf, -np.inf], np.nan).dropna(how="any")
# removing outliers 
len(train_data["age"][train_data["age"] > 300]) / len(train_data["age"])
len(train_data["area_percentage"][train_data["area_percentage"] > 70]) / len(train_data["area_percentage"])
train_data = train_data[train_data["age"] < 500]
train_data = train_data[train_data["area_percentage"] < 70]
X = train_data.drop('damage_grade', axis=1)
y = train_data['damage_grade']
# Create correlation matrix
corr_matrix = X.corr().abs()

# Select upper triangle of correlation matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))

# Find index of feature columns with correlation greater than 0.95
to_drop = [column for column in upper.columns if any(upper[column] >= 0.5)]
X = X.drop(X[to_drop], axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
#X_test = sc.transform(X_test)
# from sklearn.decomposition import FastICA
pca = PCA(n_components=.95)
pca.fit(X_train)
X_transform = pca.transform(X_train)
pca.components_.shape
plt.figure(figsize=(20,20))
plt.matshow(pca.components_, cmap='viridis', fignum=1, aspect='auto')
labs = []
plt.yticks(range(40), [f'Comp# {i}' for i in range(1, 41)],fontsize=10)
plt.colorbar()
plt.xticks(range(44), np.array(X.columns),rotation=90,ha='left')
plt.rc('xtick', labelsize=10)
plt.rc('ytick', labelsize=10)
plt.rc('axes', labelsize=10)
plt.show();
pca.components_ 
a = np.array(pca.components_[1])
np.where(a > 0.5)
index = np.unique(np.concatenate([i[0] for i in [np.where(i > 0.5) for i in pca.components_]]))
X_reduced = X.iloc[:, index]
X_reduced
X_reduced.columns
X_reduced.shape
X_train_reduced, X_test_reduced, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
logistic = LogisticRegression(solver='lbfgs', multi_class='multinomial', random_state=42, max_iter=500)
cross_val_score(logistic, X_train_reduced, y_train, cv=3)
logistic = LogisticRegression(penalty='l1', solver='saga', multi_class='multinomial', random_state=42, max_iter=500)
cross_val_score(logistic, X_train_reduced, y_train, cv=3)
mlp = MLPClassifier(activation='logistic', random_state=42)
mlp.fit(X_train_reduced, y_train)
mlp.score(X_train_reduced, y_train)
mlp.score(X_test_reduced, y_test)
clf = RandomForestClassifier(n_estimators=100, criterion='entropy').fit(X_train_reduced, y_train)
clf.score(X_train_reduced, y_train)
clf.score(X_test_reduced, y_test)
feature_imp = pd.Series(clf.feature_importances_, 
                       index=X_train_reduced.columns).sort_values(ascending=False)
feature_imp<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1510164569215876, 'fraction_numerical': 0.02516940948693127, 'mean_word_length': 8.064327485380117, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '538298', 'license': 'MIT', 'n_tokens_mistral': 1269, 'n_tokens_neox': 1186, 'n_words': 194, 'path': 'EDA_comp.ipynb', 'repo_name': 'sandeepsainath/Earthquake-Damage', 'size': '3099'}