<jupyter_start><jupyter_text>Load and prep Yelp reviews data<jupyter_code>## Load Yelp Business data
biz_f = open(folder_yelp + '/yelp_academic_dataset_business.json')
biz_df = pd.DataFrame([json.loads(x) for x in biz_f.readlines()])
biz_f.close()

## Load Yelp Reviews data
review_file = open(folder_yelp + '/yelp_academic_dataset_review.json')
review_df = pd.DataFrame([json.loads(x) for x in review_file.readlines()])
review_file.close()
biz_df.shape
review_df.shape
biz_df[:1]
biz_df.dtypes
biz_df['categories'] = biz_df['categories'].astype(str)
# Pull out only Nightlife and Restaurants businesses
two_biz = biz_df[biz_df.apply(lambda x: 'Nightlife' in x['categories'] or 'Restaurants' in x['categories'], axis=1)]
two_biz.shape
biz_df.shape
## Join with the reviews to get all reviews on the two types of business
twobiz_reviews = two_biz.merge(review_df, on='business_id', how='inner')
twobiz_reviews.shape
## Python Object Serialization - Pickle
twobiz_reviews.to_pickle(folder_python_object + '/fe_04_01_twobiz_reviews.pkl')
twobiz_reviews = pd.read_pickle(folder_python_object + '/fe_04_01_twobiz_reviews.pkl')
# Trim away the features we won't use
twobiz_reviews = twobiz_reviews[['business_id', 
                                 'name', 
                                 'stars_y', 
                                 'text', 
                                 'categories']]
# Create the target column--True for Nightlife businesses, and False otherwise
twobiz_reviews['target'] = twobiz_reviews.apply(lambda x: 'Nightlife' in x['categories'], axis=1)
## Now pull out each class of reviews separately, 
## so we can create class-balanced samples for training
nightlife = twobiz_reviews[twobiz_reviews.apply(lambda x: 'Nightlife' in x['categories'], axis=1)]
restaurants = twobiz_reviews[twobiz_reviews.apply(lambda x: 'Restaurants' in x['categories'], axis=1)]

nightlife.shape
restaurants.shape
nightlife_subset = nightlife.sample(frac=0.1, random_state=123)
restaurant_subset = restaurants.sample(frac=0.033, random_state=123)
nightlife_subset.shape
restaurant_subset.shape
nightlife_subset.to_pickle(folder_python_object + '/fe_04_01_nightlife_subset.pkl')
restaurant_subset.to_pickle(folder_python_object + '/fe_04_01_restaurant_subset.pkl')
nightlife_subset = pd.read_pickle(folder_python_object + '/fe_04_01_nightlife_subset.pkl')
restaurant_subset = pd.read_pickle(folder_python_object + '/fe_04_01_restaurant_subset.pkl')
combined = pd.concat([nightlife_subset, restaurant_subset])
combined['target'] = combined.apply(lambda x: 'Nightlife' in x['categories'], axis=1)
print(combined.shape)
print(combined.groupby(['target'])['text'].count())
combined[:3]
# Split into training and test data sets
training_data, test_data = modsel.train_test_split(combined, 
                                                   train_size=0.7, 
                                                   random_state=123)
training_data.shape
test_data.shape
# Represent the review text as a bag-of-words 
bow_transform = text.CountVectorizer()
X_tr_bow = bow_transform.fit_transform(training_data['text'])
print(len(bow_transform.vocabulary_))
print(type(bow_transform.vocabulary_))
mydict = bow_transform.vocabulary_
# get last 10 words here
demo = {k: mydict[k] for k in sorted(mydict.keys())[-10:]}
demo
print(X_tr_bow.shape)
print(type(X_tr_bow))
X_tr_bow[1, :10].toarray()
X_te_bow = bow_transform.transform(test_data['text'])
y_tr = training_data['target']
y_te = test_data['target']
# Create the tf-idf representation using the bag-of-words matrix
tfidf_trfm = text.TfidfTransformer(norm=None)
X_tr_tfidf = tfidf_trfm.fit_transform(X_tr_bow)
X_te_tfidf = tfidf_trfm.transform(X_te_bow)
# l2-normalize the bag-of-words representation
X_tr_l2 = preproc.normalize(X_tr_bow, axis=0)
X_te_l2 = preproc.normalize(X_te_bow, axis=0)<jupyter_output><empty_output><jupyter_text>Classify with logistic regression<jupyter_code>def simple_logistic_classify(X_tr, y_tr, X_test, y_test, description, _C=1.0):
    ## Helper function to train a logistic classifier and score on test data
    m = LogisticRegression(C=_C).fit(X_tr, y_tr)
    s = m.score(X_test, y_test)
    print ('Test score with', description, 'features:', s)
    return m
m1 = simple_logistic_classify(X_tr_bow, y_tr, X_te_bow, y_te, 'bow')
m3 = simple_logistic_classify(X_tr_tfidf, y_tr, X_te_tfidf, y_te, 'tf-idf')
m2 = simple_logistic_classify(X_tr_l2, y_tr, X_te_l2, y_te, 'l2-normalized')<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:765: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)<jupyter_text>Tune regularization parameters using grid search<jupyter_code>param_grid_ = {'C': [1e-5, 1e-3, 1e-1, 1e0, 1e1, 1e2]}
bow_search   = modsel.GridSearchCV(LogisticRegression(), cv=5, param_grid=param_grid_)
tfidf_search = modsel.GridSearchCV(LogisticRegression(), cv=5, param_grid=param_grid_)
l2_search    = modsel.GridSearchCV(LogisticRegression(), cv=5, param_grid=param_grid_)
bow_search.fit(X_tr_bow, y_tr)
bow_search.best_score_
l2_search.fit(X_tr_l2, y_tr)
l2_search.best_score_
tfidf_search.fit(X_tr_tfidf, y_tr)
tfidf_search.best_score_
bow_search.best_params_
l2_search.best_params_
tfidf_search.best_params_
bow_search.cv_results_
import pickle
results_file = open(folder_python_object + '/fe_04_02_tfidf_gridcv_results.pkl', 'wb')
pickle.dump(bow_search, results_file, -1)
pickle.dump(tfidf_search, results_file, -1)
pickle.dump(l2_search, results_file, -1)
results_file.close()
pkl_file = open(folder_python_object + '/fe_04_02_tfidf_gridcv_results.pkl', 'rb')
bow_search = pickle.load(pkl_file)
tfidf_search = pickle.load(pkl_file)
l2_search = pickle.load(pkl_file)
pkl_file.close()
bow_search.cv_results_['mean_test_score']

search_results = pd.DataFrame.from_dict({'bow': bow_search.cv_results_['mean_test_score'],
                               'tfidf': tfidf_search.cv_results_['mean_test_score'],
                               'l2': l2_search.cv_results_['mean_test_score']})
search_results<jupyter_output><empty_output><jupyter_text>Plot cross validation results<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")
ax = sns.boxplot(data=search_results, width=0.4)
ax.set_ylabel('Accuracy', size=14)
ax.tick_params(labelsize=14)
plt.savefig(folder_graph + '/fe_04_02_tfidf_gridcv_results.png')
m1 = simple_logistic_classify(X_tr_bow, y_tr, X_te_bow, y_te, 'bow', _C=bow_search.best_params_['C'])
m2 = simple_logistic_classify(X_tr_l2, y_tr, X_te_l2, y_te, 'l2-normalized', _C=l2_search.best_params_['C'])
m3 = simple_logistic_classify(X_tr_tfidf, y_tr, X_te_tfidf, y_te, 'tf-idf', _C=tfidf_search.best_params_['C'])
bow_search.cv_results_['mean_test_score']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1143056524773203, 'fraction_numerical': 0.017585484996510817, 'mean_word_length': 5.773156899810964, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '453746', 'license': 'MIT', 'n_tokens_mistral': 2730, 'n_tokens_neox': 2628, 'n_words': 466, 'path': '04.01-06_Tf-idf.ipynb', 'repo_name': 'sondywoo/feature-engineering-book', 'size': '7165'}