<jupyter_start><jupyter_text>Neighborhood Vibe Analysis<jupyter_code>import numpy as np
import pandas as pd
import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.cluster import KMeans, MiniBatchKMeans
import nltk
from nltk import wordpunct_tokenize
from nltk.stem import WordNetLemmatizer
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
import math as math<jupyter_output><empty_output><jupyter_text>Prerequisite<jupyter_code># download wordnet for future analysis
nltk.download('wordnet')<jupyter_output>[nltk_data] Downloading package wordnet to /Users/gaotian/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>Read dataset<jupyter_code>df = pd.read_csv('../data/listings_2019.csv')
df.head(3)<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code># Extract columns
col = ['id', 'name', 'summary', 'space', 
       'neighborhood_overview', 'transit', 'host_id', 
       'neighbourhood_cleansed', 'latitude', 'longitude']
df = df[col]
df.isnull().sum()
df['neighborhood_overview'].fillna('Unknown', inplace=True)
# Examing changes
df.isnull().sum()
new_arr = ['Allston', 'Back Bay', 'Beacon Hill','Brighton',
             'Chinatown', 'Dorchester', 'Downtown', 'East Boston', 'Fenway',
           'Jamaica Plain', 'Mission Hill', 'North End', 'South Boston',
             'South End', 'West End']
indexes_to_drop = []
for i in df.index:
    if df.iloc[i]['neighbourhood_cleansed'] not in new_arr:
        indexes_to_drop.append(i)

# Delete these row indexes from dataFrame
df.drop(df.index[indexes_to_drop] , inplace=True)<jupyter_output><empty_output><jupyter_text>Build Model<jupyter_code>class LemmaTokenizer(object):
    """Custom tokenizer class that stems tokens"""
    def __init__(self):
        self.wnl = WordNetLemmatizer()
    def __call__(self, doc):
        return [self.wnl.lemmatize(t) for t in wordpunct_tokenize(doc) if len(t.strip()) > 1]
def show_topn(classifier, vectorizer, categories, n):
    """Returns the top n features that characterize each category"""
    feature_names = np.asarray(vectorizer.get_feature_names())
    for i, category in enumerate(categories):
        topn = np.argsort(classifier.coef_[i])[-n:] #sorts in ascending order
        print('{}: {}'.format(category, ", ".join(feature_names[topn])))
def save_topn(classifier, vectorizer, categories, n, outdict):
    """Returns the top n features that characterize eachc category, and save result in outdict"""
    feature_names = np.asarray(vectorizer.get_feature_names())
    for i, category in enumerate(categories):
        topn = np.argsort(classifier.coef_[i])[-n:]
        outdict[i] = feature_names[topn]
pipeline = Pipeline([('tfidf', TfidfVectorizer(ngram_range=(1,2), stop_words='english', tokenizer=LemmaTokenizer())),
                     ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                           alpha=1e-3, random_state=5)),])


le = LabelEncoder()
new_target = le.fit_transform(df['neighbourhood_cleansed'])
model = pipeline.fit(df['neighborhood_overview'], new_target)
# Show top 5 words result
show_topn(model.named_steps['clf'],
          model.named_steps['tfidf'],
          le.inverse_transform(model.named_steps['clf'].classes_), 5)<jupyter_output>Allston: greater boston, student, harvard square, harvard, allston
Back Bay: historic bay, fashionable, shopping, newbury, bay
Beacon Hill: state house, state, hill, beacon hill, beacon
Brighton: college, brighton center, college boston, boston college, brighton
Chinatown: rated craft, avery, avery bar, culture, chinatown
Dorchester: diverse neighborhood, savin hill, savin, ashmont, dorchester
Downtown: tavern, crossing highlight, downtown crossing, crossing, heart downtown
East Boston: property le, church quincy, airport, east, east boston
Fenway: sweet cheek, fenway park, kenmore square, kenmore, fenway
Jamaica Plain: jp, pond, plain, jamaica plain, jamaica
Mission Hill: number hidden, phone number, phone, mission hill, mission
North End: little, italian, north, little italy, italy
South Boston: beach, southie, broadway, seaport, south boston
South End: gallery, copley, end, south, south end
West End: (. mi, west, mi ),, west end, university mi<jupyter_text>Word Cloud<jupyter_code># Create wordcloud based on top-30 words
Keywords = {}
save_topn(model.named_steps['clf'], 
          model.named_steps['tfidf'], 
          le.inverse_transform(model.named_steps['clf'].classes_), 
          30,
          outdict=Keywords)
fig,axes=plt.subplots(5, 3, figsize=(30,30))

# Add title
arr = le.inverse_transform(model.named_steps['clf'].classes_)
cnt = 0
for i in range(len(axes)): 
    for j in range(len(axes[i])):
        if cnt == 15:
            break
        axes[i,j].set_title(arr[cnt])
        cnt += 1
        
for i in range(15):
    wordlist = list(Keywords[i])
    wc = WordCloud(background_color='white', max_words=30, relative_scaling=0.2).generate(" ".join(wordlist))
    print(wc)
    axes[math.floor(i/3),i%3].imshow(wc)<jupyter_output><wordcloud.wordcloud.WordCloud object at 0x1a3061a1d0>
<wordcloud.wordcloud.WordCloud object at 0x1a30643be0>
<wordcloud.wordcloud.WordCloud object at 0x1105e4eb8>
<wordcloud.wordcloud.WordCloud object at 0x1105cbd68>
<wordcloud.wordcloud.WordCloud object at 0x1105d0da0>
<wordcloud.wordcloud.WordCloud object at 0x1105cba20>
<wordcloud.wordcloud.WordCloud object at 0x1105d64a8>
<wordcloud.wordcloud.WordCloud object at 0x1105cb400>
<wordcloud.wordcloud.WordCloud object at 0x1105d6cc0>
<wordcloud.wordcloud.WordCloud object at 0x1105cb780>
<wordcloud.wordcloud.WordCloud object at 0x1105d6be0>
<wordcloud.wordcloud.WordCloud object at 0x1105cba58>
<wordcloud.wordcloud.WordCloud object at 0x1105da0f0>
<wordcloud.wordcloud.WordCloud object at 0x1105d02b0>
<wordcloud.wordcloud.WordCloud object at 0x1105daeb8>
{'fraction_non_alphanumeric': 0.1155437841416984, 'fraction_numerical': 0.02416818407548419, 'mean_word_length': 5.16530612244898, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '563860', 'license': 'MIT', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2008, 'n_words': 493, 'path': 'src/neighborhood_vibe.ipynb', 'repo_name': 'feiyue33/cs506-final-project', 'size': '6041'}