<jupyter_start><jupyter_text>First SubmissionData Science Nigeria 2021 AI Bootcamp Qualification Hackathon by Data Science Nigeria<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression, LinearRegression, Ridge, SGDClassifier, RANSACRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.preprocessing import StandardScaler, LabelEncoder, MinMaxScaler, RobustScaler, Normalizer,\
Binarizer, FunctionTransformer
import seaborn as sns
import xgboost
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score, precision_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score<jupyter_output><empty_output><jupyter_text>1. Read Data<jupyter_code>train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
sample_submission = pd.read_csv('samplesubmission.csv')
desc = train.describe().T
# miss = ((len(train) - desc['count']) /len(train)) * 100
high_std_cols = list(desc['std'][desc['std'] > 4].index)
high_std_cols
train.head(10)
train.columns
test.head(5)
sample_submission.head(4)<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>train.shape, test.shape, sample_submission.shape
target = [cols for cols in train.columns if  cols not in test.columns]
target
# train_copy = train.copy()
# for colname in train_copy.select_dtypes("object"):
#     train_copy[colname], _ = train_copy[colname].factorize()
# train_copy.fillna(train_copy['Disposable_Income'].mean(), inplace = True)
a  = (
    train.groupby("Marital_Status")  # for each state               # select the income
    .transform("mean")         # and compute its mean
    .round(2)
)
a
y = train[target[0]]
y
from sklearn.impute import SimpleImputer 
imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
train['Disposable_Income'].fillna(train['Disposable_Income'].mean(), inplace = True)
test['Disposable_Income'].fillna(train['Disposable_Income'].mean(), inplace = True)
# train['Date_Customer'] = pd.to_datetime(train['Date_Customer'])
# test['Date_Customer'] = pd.to_datetime(test['Date_Customer'])
# train['Date_Customer']
train.describe()
train.info()
categorical_feat = list(train.select_dtypes(include = 'object').columns)
categorical_feat
numerical_feat = list(train.select_dtypes(exclude = 'object').columns)
numerical_feat.remove('Response')
numerical_feat, len(numerical_feat)
# label encoding 
# le = LabelEncoder()
# for i in categorical_feat:
#     train[i] = le.fit_transform(train[i])
#     test[i] = le.fit_transform(test[i])
# train.head()
# numerical_feat_to_choose = set(np.random.choice(numerical_feat, 20))
# numerical_feat_to_choose<jupyter_output><empty_output><jupyter_text>Feature SelectionUsing Pearson Correlation<jupyter_code>plt.figure(figsize = (20,10))
cor = train.corr()
sns.heatmap(cor, annot = True, cmap = plt.cm.CMRmap_r)
plt.show()<jupyter_output><empty_output><jupyter_text>Using the below function to select highly correlated features<jupyter_code>def correlation(dataset, threshold):
    col_corr = set()
    corr_matrix = dataset.corr()
    for i in range(len(corr_matrix.columns)):
        for j in range(i):
            if abs(corr_matrix.iloc[i,j] >= threshold):
                colname = corr_matrix.columns[i]
                col_corr.add(colname)
    return col_corr
corr_features = correlation(train, 0.7)
len(corr_features)
train.drop(corr_features, axis = 1, inplace = True)
test.drop(corr_features, axis = 1, inplace = True)
numerical_feat.remove('Amount_on_MeatProducts')
high_std_cols.remove('Amount_on_MeatProducts')
absCorWithTarget = []
for var in train[numerical_feat]:
    absCorWithTarget.append(round(abs(y.corr(train[numerical_feat][var])),4))
cor_with_target = pd.DataFrame(absCorWithTarget, index = numerical_feat, columns = ['CorrWithTTarget'])
cor_with_target.head()<jupyter_output><empty_output><jupyter_text>Data Scaling<jupyter_code>scaler = MinMaxScaler()
X = pd.DataFrame(scaler.fit_transform(train[numerical_feat]), columns = numerical_feat)
testing = pd.DataFrame(scaler.fit_transform(test[numerical_feat]), columns = numerical_feat)
X<jupyter_output><empty_output><jupyter_text>Defining ModelSince there are no Missing values we go straight to splitting our data for model fitting<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=0)
# Train the Model
lr = LogisticRegression()
lr.fit(X_train, y_train)
# from sklearn.neighbors import KNeighborsClassifier
# # Predict
# # We use both the 'X_train and X_test just to ensre we are not over fitting'
pred = lr.predict(X_test)
print('Testing set Evalution:')
print(f'Classifier f1_score value is: {f1_score(y_test, pred)}')
precision_score(y_test, pred, average='micro')<jupyter_output><empty_output><jupyter_text>Submitting Section<jupyter_code>prediction = lr.predict(testing)
submission_file = pd.DataFrame({'Id': test.ID, 'Response': prediction})
submission_file.reset_index(drop = True, inplace = True)
submission_file
sns.countplot(x = 'Response', data = submission_file)
submission_file.to_csv('my_second9.csv', index = False)<jupyter_output><empty_output><jupyter_text>LINEAR REGRESSION<jupyter_code>from sklearn.datasets import make_regression, make_blobs, make_classification, make_circles
X,Y = make_regression(n_features = 1, noise = 10, n_samples = 1000)
plt.scatter(X, Y, s=5)
lr = LinearRegression(normalize = False, n_jobs = 1,fit_intercept = True, copy_X = True)
ridge = Ridge(alpha = .1)
ridge.fit([[0,0],[0,0],[1,1]], [0,.1,1])
lr.fit([[0,0],[0,0],[1,1]], [0,.1,1])
ridge.coef_, lr.coef_
outliers = Y[950:] - 600
outliers
Y_out = np.append(Y[:950],outliers)
plt.scatter(X,Y_out, s=5)
A, b,w = make_regression(n_features = 10, noise = 10, n_samples = 10, coef = True, random_state = 1, bias = 3.5)
alphas = np.logspace(-6,6,200)
alphas[:20]
coefs = []
for a in alphas:
    ridge = Ridge(alpha = a, fit_intercept = True)
    ridge.fit(A,b)
    coefs.append(ridge.coef_)
ax = plt.gca()
ax.plot(alphas, coefs)
ax.set_xscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>LOGISTIC REGRESSION<jupyter_code>C,d = make_blobs(n_features=2, n_samples = 1000, cluster_std = 2, centers=2)
plt.scatter(C[:, 0], C[:, 1], s=10, c=d)
h = .02
x_min, x_max  = C[:, 0].min() - .5, C[:, 0].max() + .5
y_min, y_max  = C[:, 1].min() - .5, C[:, 1].max() + .5
                          
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
lr.fit(C,d)
Z = lr.predict(np.c_[xx.ravel(), yy.ravel()])
Z= Z.reshape(xx.shape)
plt.pcolormesh(xx, yy, Z, cmap = plt.cm.Paired, shading='auto')
plt.scatter(C[:, 0], C[:, 1],c=d, s=10)<jupyter_output><empty_output><jupyter_text>OUTLINE LEARNING MODELS<jupyter_code>E,f = make_classification(n_classes=2, n_features=10, n_samples = 10000)
E_train,E_test,f_train,f_test = train_test_split(E,f)
sgd = SGDClassifier(n_iter_no_change = 10)
sgd.partial_fit(E_train[:1500], f_train[:1500], classes = [0,1])
sgd.score(E_test,f_test)
sgd.partial_fit(E_train[1500:5000], f_train[1500:5000], classes = [0,1])
sgd.score(E_test,f_test)<jupyter_output><empty_output><jupyter_text>Robust Regressionfitting a regression in the presence of currupt data: either outliers, or error in the modelThe techniques used are RANSAC, Theil and HuberRegressor<jupyter_code>ransac = RANSACRegressor()
ransac.fit(C,d)<jupyter_output><empty_output><jupyter_text>Polynomial Features<jupyter_code>G,h = make_circles(n_samples=1000, noise=.04)
plt.scatter(G[:,0],G[:, 1], c=h, s=10)
from sklearn.preprocessing import PolynomialFeatures
pol = PolynomialFeatures(degree=2)
G_tf = pol.fit_transform(G)
G_tf[:5]
pol_lr = LogisticRegression()
pol_lr = lr.fit(G_tf[:500], h[:500])
h.shape
G_pred = pol_lr.predict(G_tf[500:])
h_pred = np.where(G_pred > 1, 1,0)
pol_lr.score(G_tf[500:], h[500:])
pol_lr.coef_
corpus = [
    'this is the first line',
    'this is the second line',
    'typing the third sentence',
]<jupyter_output><empty_output><jupyter_text>Text Processing<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
cv  = CountVectorizer()
df = pd.DataFrame({
    'text': corpus
})
df
cv.fit_transform(df.text).toarray()
cv.vocabulary_
from adspy_shared_utilities import load_crime_dataset
crime = load_crime_dataset()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13679301685997847, 'fraction_numerical': 0.021642951094104985, 'mean_word_length': 6.6875, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '159750', 'license': 'Apache-2.0', 'n_tokens_mistral': 3260, 'n_tokens_neox': 3079, 'n_words': 592, 'path': 'zindi_first.ipynb', 'repo_name': 'lawallanre00490038/Zindi-Conpetition_2', 'size': '8363'}