<jupyter_start><jupyter_text>For Exploring Changes Over Time<jupyter_code>import pandas as pd
import numpy as np
import nltk
import sklearn
import string, re
import urllib
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import OneHotEncoder,StandardScaler
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestRegressor
from nltk.corpus import stopwords
april = pd.read_csv('../Data/2018/listings42018sum.csv')
may = pd.read_csv('../Data/2018/listings52018sum.csv')
july = pd.read_csv('../Data/2018/listings72018sum.csv')
august = pd.read_csv('../Data/2018/listings82018sum.csv')
september = pd.read_csv('../Data/2018/listings92018sum.csv')
october = pd.read_csv('../Data/2018/listings102018sum.csv')
november = pd.read_csv('../Data/2018/listings112018sum.csv')
december = pd.read_csv('../Data/2018/listings122018sum.csv')
april['month'] = 'april'
may['month'] = 'may'
july['month'] = 'july'
august['month'] = 'august'
september['month'] = 'september'
october['month'] = 'october'
november['month'] = 'november'
december['month'] = 'december'
frames = [april,may,july,august,september,october,november,december]
data_2018 = pd.concat(frames)
data_2018.head()
data_2018['reviews_per_month'] = data_2018['reviews_per_month'].fillna(0)
ohe = OneHotEncoder(sparse=False)
neigh_group = ohe.fit_transform(data_2018[['neighbourhood_group']])
neigh_group_cat = ohe.categories_
neigh = ohe.fit_transform(data_2018[['neighbourhood']])
neigh_cat = ohe.categories_
room = ohe.fit_transform(data_2018[['room_type']])
room_cat = ohe.categories_
month_group = ohe.fit_transform(data_2018[['month']])
month_group_cat = ohe.categories_
def rename(name_of_columns,pre_addition):
    new_list = []
    for x in name_of_columns:
        for x in x:
            new_list.append(pre_addition+ '' + x)
    return new_list
new_neigh_group_cat = rename(neigh_group_cat,'neighbourhood_group: ')
new_neigh_cat = rename(neigh_cat,'neighbourhood: ')
new_room_cat = rename(room_cat,'room_type: ')
new_month_group_cat = rename(month_group_cat,'month: ')
neigh_group_df = pd.DataFrame(data=neigh_group,columns=new_neigh_group_cat)
neigh_df = pd.DataFrame(data=neigh,columns=new_neigh_cat)
room_type_df = pd.DataFrame(data=room,columns = new_room_cat)
month_df = pd.DataFrame(data=month_group,columns=new_month_group_cat)
stopwords_list = stopwords.words('english') + list(string.punctuation)
vectorizer = TfidfVectorizer(strip_accents='unicode',stop_words=stopwords_list,min_df=30, ngram_range=(1,3))
data_2018.fillna({'name':''}, inplace=True)
tf_idf = vectorizer.fit_transform(data_2018['name'])
nlp_name = pd.DataFrame(tf_idf.toarray(), columns=vectorizer.get_feature_names())
clean_data = data_2018.drop(columns = ['id','month','host_name','host_id','last_review','name',
month_df
clean_data['longitude'] = clean_data['longitude'].round(decimals=5)
clean_data['latitude'] = clean_data['latitude'].round(decimals=5)
clean_data = clean_data.reset_index()
clean_data = pd.concat([clean_data,neigh_group_df,neigh_df,room_type_df,nlp_name,month_df],axis=1)
X = clean_data.drop(columns = ['price'])
y = clean_data['price']
ss = StandardScaler()
X_ss = ss.fit_transform(X)
Xtrain,Xtest,ytrain,ytest = train_test_split(X_ss,y)
clean_data = data_2018.drop(columns = ['id','month','host_name','host_id','last_review','name',
                                       'neighbourhood_group','neighbourhood','room_type'])

rfr = RandomForestRegressor(n_estimators=1000,min_samples_split=5,min_samples_leaf=3,random_state=42)
rfr.fit(Xtrain,ytrain)
rfr.score(Xtrain,ytrain)
ypredtrain = rfr.predict(Xtrain)
ypredtest = rfr.predict(Xtest)
from sklearn.metrics import r2_score
print(r2_score(ytrain,ypredtrain))
print(r2_score(ytest,ypredtest))
ids = data_2018.id
ids = ids.reset_index
ids = pd.DataFrame(ids)
ids = ids.reset_index()
ids =ids.drop(columns = 'index')
ids['id'].value_counts()s
data_2018[data_2018['id']==23748596]
import matplotlib.pyplot as plt
from pandasql import sqldf
pysqldf = lambda q: sqldf(q, globals())

query1 = pysqldf("""SELECT month,avg(price) AS price FROM data_2018 group by month
            ORDER BY 
            case 
                when month = 'april' then 1
                when month = 'may' then 2
                when month = 'july' then 3
                when month = 'august' then 4
                when month = 'september' then 5
                when month = 'october' then 6
                when month = 'november' then 7
                else 8
                end""")


plt.bar(x=query1['month'],height=(query1['price']-148))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13704819277108435, 'fraction_numerical': 0.0387263339070568, 'mean_word_length': 5.679597701149425, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '71501', 'license': 'MIT', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1672, 'n_words': 263, 'path': 'Notebooks/Exploratory Notebooks/Combining_multiple_sets.ipynb', 'repo_name': 'trossII/SeattleBnB', 'size': '4648'}