<jupyter_start><jupyter_text>Load/Clean Data<jupyter_code>data = pd.read_csv('ABNB_NY_V2.csv')
summary = data.describe()
print({x: len(data[x].unique()) for x in data.columns})
summary
print(data['neighbourhood_group'].unique())
print(data['room_type'].unique())
len(data[data['last_review'].isna()])<jupyter_output>['Brooklyn' 'Manhattan' 'Queens' 'Staten Island' 'Bronx']
['Private room' 'Entire home/apt' 'Shared room']<jupyter_text>Convert Features to Types<jupyter_code>data['price'] = data['price'].astype('float')
data['minimum_nights'] = data['minimum_nights'].astype('float')
data['number_of_reviews'] = data['number_of_reviews'].astype('float')
data['calculated_host_listings_count'] = data[
    'calculated_host_listings_count'].astype('float')
data['availability_365'] = data['availability_365'].astype('float')
data['dist_subway'] = data['dist_subway'].astype('float')
data['cnt_all'] = data['cnt_all'].astype('float')
data['cnt_all'] = data['cnt_all'].astype('float')
data['cnt_entire_home'] = data['cnt_entire_home'].astype('float')
data['cnt_private_room'] = data['cnt_private_room'].astype('float')
data['cnt_shared_room'] = data['cnt_shared_room'].astype('float')
data['avgprice_all'] = data['avgprice_all'].astype('float')
data['avgprice_entire_home'] = data['avgprice_entire_home'].astype('float')
data['avgprice_private_room'] = data['avgprice_private_room'].astype('float')
data['avgprice_shared_room'] = data['avgprice_shared_room'].astype('float')
data['neighbourhood_group'] = data['neighbourhood_group'].astype('category')
data['neighbourhood'] = data['neighbourhood'].astype('category')
data['ZIP'] = data['ZIP'].astype('category')
data['last_review'] = pd.to_datetime(data['last_review'])
data['days_since_last_review'] = (
    dt.datetime.strptime('2020-01-01', '%Y-%m-%d') -
    data['last_review']).dt.days
data.days_since_last_review.fillna(data.days_since_last_review.max(),
                                   inplace=True)
data.drop(columns=['last_review'], inplace=True)<jupyter_output><empty_output><jupyter_text>Drop useless cols<jupyter_code>data.drop(
    columns=['id', 'name', 'host_id', 'host_name', 'neighbourhood', 'ZIP'],
    inplace=True)
data.describe()<jupyter_output><empty_output><jupyter_text>Define feature set<jupyter_code>data.dtypes
target_col = 'room_type'<jupyter_output><empty_output><jupyter_text>Price Outliers - Drop stuff with zero price- Drop anything with price >= 300<jupyter_code>data = data[data.price != 0].copy()
data = data[data.price < 300].copy()
data = data[data.minimum_nights <= 20].copy()
data.to_csv('airbnb_clean.csv', index=False)<jupyter_output><empty_output><jupyter_text>Add Response Column<jupyter_code>room_types = {'Entire home/apt': 0, 'Private room': 1, 'Shared room': 2}
data['Response'] = data.apply(lambda x: room_types[x.room_type], axis=1)
#data.drop(axis=0, labels=[x for x in data.index if data.iloc[x].Response == 2], inplace=True)
data.dtypes[data.dtypes == np.float64].index
data.dtypes[data.dtypes == 'category']<jupyter_output><empty_output><jupyter_text>Feature / Target Split<jupyter_code>data_final = data
features, target = data_final.drop(
    columns=['Response']), data_final.loc[:, ['Response']]
features.shape, target.shape<jupyter_output><empty_output><jupyter_text>Train/Test Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(features,
                                                    target,
                                                    test_size=0.2,
                                                    random_state=0)
classes_test = ClassBalance(
    labels=['Entire home/apt', 'Private room', 'Shared room'])
classes_test.fit(y_train['Response'], y_test['Response'])<jupyter_output><empty_output><jupyter_text>Data Transforms    - OHE Encode all categoricals    - Standard Scale all numerics (sklearn is not R - it does not do this by default)    - This needs to be done on train/test set separately (create a pipeline)         https://scikit-learn.org/stable/modules/cross_validation.html        https://scikit-learn.org/stable/modules/compose.htmlcombining-estimators<jupyter_code>#OH encode
label_encode = [
    x for x, y in dict(X_train.dtypes).items()
    if type(y) == pd.CategoricalDtype
]
for var in label_encode:
    #impute to Mode
    print(f"{var} was OHE'd")
    X_train[var].fillna(X_train[var].mode()[0], inplace=True)
    X_test[var].fillna(X_train[var].mode()[0], inplace=True)
    cat_list_train = pd.get_dummies(X_train[var], prefix=var)
    cat_list_test = pd.get_dummies(X_test[var], prefix=var)
    X_train_new = X_train.join(cat_list_train)
    X_train = X_train_new
    X_test_new = X_test.join(cat_list_test)
    X_test = X_test_new

normalize = [x for x, y in dict(X_train.dtypes).items() if y == np.float64]
for col in normalize:
    #change impution scheme?
    print(f"{col} was Standard Scaled")
    X_train[col].fillna(X_train[col].median(), inplace=True)
    X_test[col].fillna(X_train[col].median(), inplace=True)
    normalizer = StandardScaler()
    ar_train = X_train[col].to_numpy().reshape(-1, 1)
    normalizer.fit(ar_train)
    X_train.loc[:, col] = normalizer.transform(ar_train)
    ar_test = X_test[col].to_numpy().reshape(-1, 1)
    X_test.loc[:, col] = normalizer.transform(ar_test)

X_train.drop(columns=label_encode, inplace=True)
X_test.drop(columns=label_encode, inplace=True)
X_train.drop(columns=target_col, inplace=True)
X_test.drop(columns=target_col, inplace=True)<jupyter_output>neighbourhood_group was OHE'd
latitude was Standard Scaled
longitude was Standard Scaled
price was Standard Scaled
minimum_nights was Standard Scaled
number_of_reviews was Standard Scaled
reviews_per_month was Standard Scaled
calculated_host_listings_count was Standard Scaled
availability_365 was Standard Scaled
dist_subway was Standard Scaled
cnt_all was Standard Scaled
cnt_entire_home was Standard Scaled
cnt_private_room was Standard Scaled
cnt_shared_room was Standard Scaled
avgprice_all was Standard Scaled
avgprice_entire_home was Standard Scaled
avgprice_private_room was Standard Scaled
avgprice_shared_room was Standard Scaled
days_since_last_review was Standard Scaled<jupyter_text>Baseline accuracy rates<jupyter_code>base_line_accuracy_train = {
    'class_0': (y_train[y_train.Response == 0].shape[0] / y_train.shape[0]),
    'class_1': (y_train[y_train.Response == 1].shape[0] / y_train.shape[0]),
    'class_2': (y_train[y_train.Response == 2].shape[0] / y_train.shape[0])
}
base_line_accuracy_test = {
    'class_0': (y_test[y_test.Response == 0].shape[0] / y_test.shape[0]),
    'class_1': (y_test[y_test.Response == 1].shape[0] / y_test.shape[0]),
    'class_2': (y_test[y_test.Response == 2].shape[0] / y_test.shape[0])
}

print(base_line_accuracy_train)
print(base_line_accuracy_test)<jupyter_output>{'class_0': 0.46599676455948236, 'class_1': 0.5083685913389746, 'class_2': 0.025634644101543057}
{'class_0': 0.4682677949228472, 'class_1': 0.5068442010950722, 'class_2': 0.024888003982080638}<jupyter_text>Create Models (50 Pts)	Create a logistic regression model and a support vector machine model for the classification task involved with your dataset. Assess how well each model performs (use 80/20 training/testing split for your data). Adjust parameters of the models to make them more accurate. If your dataset size requires the use of stochastic gradient descent, then linear kernel only is fine to use. That is, the SGDClassifier is fine to use for optimizing logistic regression and linear support vector machines. For many problems, SGD will be required in order to train the SVM model in a reasonable timeframe.<jupyter_code>?LogisticRegression<jupyter_output><empty_output><jupyter_text>https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.htmlhttps://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html Fit a Logistic on all predictors- Using standard techniques in scikit-learn, we will attempt to isolate the best parameters for:    - Inverse of regularization strength (C):         - lower values imply stronger regularization penalty and slower fit times    - Tolerance (tol):        - tolerance for stopping criteria<jupyter_code>logistic = LogisticRegression(
     max_iter=10000
    ,fit_intercept=True # always fit an intercept term
    ,n_jobs=-1 # use all processors
)
pipe = Pipeline(steps=[('logistic', logistic)])
param_grid = {
    'logistic__C': np.logspace(-4, 4, 4),
    'logistic__class_weight':[None, 'balanced'],
    'logistic__penalty':['l1', 'l2','elasticnet', 'none'],
    'logistic__multi_class':['auto', 'ovr', 'multinomial'],
    'logistic__dual':[True, False],
    'logistic__solver':['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']
}

search = GridSearchCV(pipe, param_grid, n_jobs=-1)
search.fit(X_train, y_train.values.ravel())
print("Best parameter (CV score=%0.3f):" % search.best_score_)
print(search.best_params_)
results = pd.DataFrame(search.cv_results_)
results<jupyter_output><empty_output><jupyter_text>Analyzing the Logistic Classifier With Parameters Specified<jupyter_code>lgrg_all = LogisticRegression(
    C=.001,
    solver='liblinear',
    dual=False,
    fit_intercept=True,
    intercept_scaling=1
    #, l1_ratio=None
    ,
    max_iter=100,
    multi_class='auto',
    n_jobs=None,
    penalty='l2',
    random_state=0,
    tol=0.0001,
    verbose=0,
    class_weight='balanced',
    warm_start=False)

lgrg_all.fit(X_train, y_train.values.ravel())
cutoffs_lgrg_all, scores_lgrg_all = analyze_classifier(lgrg_all, X_train,
                                                       X_test, y_train, y_test)
predict_logit_all = lgrg_all.predict(X_test)
prob_logit_all = lgrg_all.predict_proba(X_test)
acc_logit_all = round(accuracy_score(y_test, predict_logit_all), 5)
print(f'Logit (all features) Accuracy {acc_logit_all}')
lgrg_all.coef_
from sklearn.svm import SVC
support_vec_model = SVC(gamma='auto', kernel='linear', C=0.01)
support_vec_model.fit(X_train, y_train.values.ravel())
analyze_classifier(support_vec_model, X_train, X_test, y_train, y_test)<jupyter_output><empty_output><jupyter_text>Random forest classifier - adding feature imporance analysis  https://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html<jupyter_code>rf_classifier = RandomForestClassifier(
    #max_depth=None,
    random_state=0,
    #bootstrap=True,
    #oob_score=True,
    #max_samples = 50,
    criterion='gini',
    n_estimators=200)
rf_classifier.fit(X_train, y_train.values.ravel())
cutoffs_rf, scores_rf = analyze_classifier(rf_classifier, X_train, X_test,
                                           y_train, y_test)
predict_rf = rf_classifier.predict(X_test)
prob_rf = rf_classifier.predict_proba(X_test)
acc_rf = round(accuracy_score(y_test, predict_rf), 5)
print(f'RF Accuracy {acc_rf}')
importances = rf_classifier.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf_classifier.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X_train.shape[1]):
    print(f'{f+1}:{X_train.columns[f]} {importances[indices[f]]:.5f}')

# Plot the feature importances of the forest
plt.figure()
plt.title("Feature importances")
plt.bar(range(X_train.shape[1]),
        importances[indices],
        color="r",
        yerr=std[indices],
        align="center")
plt.xticks(range(X_train.shape[1]), indices)
plt.xlim([-1, X_train.shape[1]])
plt.show()<jupyter_output>Feature ranking:
1:latitude 0.38412
2:longitude 0.04932
3:price 0.04502
4:minimum_nights 0.04465
5:number_of_reviews 0.04366
6:reviews_per_month 0.04255
7:calculated_host_listings_count 0.04173
8:availability_365 0.03891
9:dist_subway 0.03822
10:cnt_all 0.03672
11:cnt_entire_home 0.03507
12:cnt_private_room 0.03455
13:cnt_shared_room 0.03332
14:avgprice_all 0.03294
15:avgprice_entire_home 0.03109
16:avgprice_private_room 0.02611
17:avgprice_shared_room 0.01854
18:days_since_last_review 0.01418
19:neighbourhood_group_Bronx 0.00402
20:neighbourhood_group_Brooklyn 0.00303
21:neighbourhood_group_Manhattan 0.00151
22:neighbourhood_group_Queens 0.00054
23:neighbourhood_group_Staten Island 0.00021<jupyter_text>PCA Pipelinehttps://scikit-learn.org/stable/auto_examples/compose/plot_digits_pipe.html<jupyter_code># Define a pipeline to search for the best combination of PCA truncation
# and classifier regularization.
pca = PCA()
# set the tolerance to a large value to make the example faster
logistic = LogisticRegression(max_iter=10000, tol=0.1)
pipe = Pipeline(steps=[('pre-process',
                        StandardScaler()), ('pca', pca), ('logistic',
                                                          logistic)])
#Continuous predictors only
continuous = X_train.dtypes[X_train.dtypes == 'float64'].index

# Parameters of pipelines can be set using ‘__’ separated parameter names:
param_grid = {
    'pca__n_components': [1, len(continuous)],
    'logistic__C': np.logspace(-4, 4, 4),
}
search = GridSearchCV(pipe, param_grid, n_jobs=-1)
search.fit(X_train[continuous], y_train.values.ravel())
print("Best parameter (CV score=%0.3f):" % search.best_score_)
print(search.best_params_)

# Plot the PCA spectrum
pca.fit(X_train[continuous])

fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True, figsize=(6, 6))
ax0.plot(np.arange(1, pca.n_components_ + 1),
         pca.explained_variance_ratio_,
         '+',
         linewidth=2)
ax0.set_ylabel('PCA explained variance ratio')

ax0.axvline(search.best_estimator_.named_steps['pca'].n_components,
            linestyle=':',
            label='n_components chosen')
ax0.legend(prop=dict(size=12))

# For each number of components, find the best classifier results
results = pd.DataFrame(search.cv_results_)
components_col = 'param_pca__n_components'
best_clfs = results.groupby(components_col).apply(
    lambda g: g.nlargest(1, 'mean_test_score'))

best_clfs.plot(x=components_col,
               y='mean_test_score',
               yerr='std_test_score',
               legend=False,
               ax=ax1)
ax1.set_ylabel('Classification accuracy (val)')
ax1.set_xlabel('n_components')

plt.xlim(-1, 20)

plt.tight_layout()
plt.show()
pca.explained_variance_ratio_.sum()
pca.explained_variance_
pca.explained_variance_ratio_
pca.n_features_
pca.n_components_
len(pca.components_[0])<jupyter_output><empty_output><jupyter_text>Pipeline Hyperparameter Tuninghttps://towardsdatascience.com/logistic-regression-model-tuning-with-scikit-learn-part-1-425142e01af5<jupyter_code>pipe = Pipeline(steps=[('classifier', RandomForestClassifier())
                       ])  #RF here is just a dummy

param_grid = [{
    'classifier': [LogisticRegression()],
    'classifier__penalty': ['l1', 'l2'],
    'classifier__C': np.logspace(-4, 4, 20),
    'classifier__solver': ['liblinear']
}, {
    'classifier': [RandomForestClassifier()],
    'classifier__n_estimators': list(range(10, 101, 10)),
    'classifier__max_features': list(range(6, 32, 5))
}]

clf = GridSearchCV(pipe, param_grid=param_grid, cv=5, verbose=True, n_jobs=-1)
best_clf = clf.fit(X_train, y_train.values.ravel())
best_clf.best_estimator_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13107210938013542, 'fraction_numerical': 0.03181489515545915, 'mean_word_length': 5.347100542344597, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 62, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '151880', 'license': 'MIT', 'n_tokens_mistral': 5681, 'n_tokens_neox': 5305, 'n_words': 964, 'path': 'mini_lab_1.ipynb', 'repo_name': 'fastest-parrot/ml1-lab1', 'size': '15213'}