<jupyter_start><jupyter_text>Stats for All<jupyter_code>print("total reviews cnt :", len(dataset))
print("unique alias count:", len(dataset.alias.unique()))
dataset.head()
# all dataset
print("ratingValue sum :", dataset.ratingValue.sum())
print("ratingValue mean:", dataset.ratingValue.mean())
# only TS - Restaurants
print(dataset.loc[dataset.alias.isin(conditions)].ratingValue.sum())
print(dataset.loc[dataset.alias.isin(conditions)].ratingValue.mean())
dataset.ratingValue.value_counts()
dataset.alias.value_counts()[:10]<jupyter_output><empty_output><jupyter_text>Yelp Business Info<jupyter_code>dataset_businesses = pd.read_csv('../../data/raw/yp_competitors.csv')
len(dataset_businesses)
dataset_businesses.head()<jupyter_output><empty_output><jupyter_text>Matching Records for Both Businesses and Reviews<jupyter_code>dataset_businesses = dataset_businesses.loc[dataset_businesses.alias.isin(dataset.alias)]
len(set(dataset_businesses.alias))
dataset_businesses.price.value_counts()
dataset_businesses.head()<jupyter_output><empty_output><jupyter_text>Generated Business Info  Generated for All<jupyter_code>from nltk import word_tokenize
from nltk import Text
from nltk.corpus import stopwords
from nltk import sent_tokenize, word_tokenize

# using nltk.Text object
def create_text(reviews):
    result = []
    for review in reviews:
        result.extend(word_tokenize(review))
    return Text(result)

def lexical_diversity(text):
    return round(len(set(text)) / len(text), 5)

def vocabulary_size(text):
    text.tokens = [x.lower() for x in text.tokens if x.isalpha()]
    vocab = len(set(text))
    return vocab

def content_size(tokens):
    sw_list = stopwords.words('english')
    content = [w for w in tokens if w.lower() not in sw_list]
    return round(len(content) / len(tokens), 5)
reviews = create_text(dataset.description)
_text = "".join(dataset.description)
len(word_tokenize(_text))
len(reviews.tokens)
len(sent_tokenize(_text))
vocabulary_size(reviews)
lexical_diversity(reviews)
sorted(reviews.vocab().most_common(10), reverse=True)
maxlen = max(len(word) for word in reviews.tokens)
[word for word in reviews.tokens if len(word) == (maxlen)]
fd = reviews.vocab()
cumulative = 0
most_common_words = [word for (word, count) in fd.most_common()]
for rank, word in enumerate(most_common_words):
    cumulative += fd.freq(word)
    print("%3d %6.2f%% %s" % (rank + 1, cumulative * 100, word))
    if cumulative > 0.25:
        break
reviews.dispersion_plot(["yum", "parking", "terrible", "hate", "clean", "kitchen", "staff", "manager"])
reviews.vocab().plot(20, cumulative=50)
reviews.vocab()["common"]
content_size(reviews.tokens)
some_words_freq = sorted(w for w in set(reviews.vocab()) if reviews.vocab()[w] > 3000 and reviews.vocab()[w] < 4000)
print(some_words_freq)
from nltk import FreqDist
word_lengths = FreqDist(len(w) for w in reviews)
word_lengths.plot()<jupyter_output><empty_output><jupyter_text>Generated for Each Business<jupyter_code>from nltk.corpus import stopwords
from string import punctuation
modals = ['can', 'will', 'must', 'should', 'might']
from nltk import sent_tokenize, word_tokenize
import random
import itertools

def generate_features(dataset, business):
    data = dataset.loc[dataset.alias == business].copy()
    data['sent_tokens'] = data.description.apply(lambda x: sent_tokenize(x))
    data['word_tokens'] = data.description.apply(lambda x: word_tokenize(x))
    data_sents = list(itertools.chain(*data.sent_tokens))
    data_words = list(itertools.chain(*data.word_tokens))
    data_cntnt = list(w for w in data_words if w.lower() not in stopwords.words('english'))
    data_vocab = set(i.lower() for i in data_words if i.isalpha())
    
    feature_list = {
        'alias': business,
        'review_count': len(data),
        "rating_avg": round(data.ratingValue.mean(), 4),
        "rating_sum": data.ratingValue.sum(),
        'num_of_sents': len(data_sents),
        'num_of_words': len(data_words),
        'num_of_vocab': len(data_vocab),
        'num_of_punctuation': len(list(i for i in data_words if i in punctuation)),
        'num_of_modals': len(list(i for i in data_words if i.lower() in modals)),
        'avg_words_in_sents': round((len(data_words) / len(data_sents)), 4),
        'ratio_content': round((len(data_cntnt) / len(data_words)), 4),
        'ratio_lexical': round((len(set(data_words)) / len(data_words)), 4)
    }
    
    return feature_list
# features for a sample
businesses = set(dataset.alias)
results_sample = list()

for i, business in enumerate(random.sample(businesses, 3)):
    features = generate_features(dataset, business)
    results_sample.append(features)
pd_info_sample = pd.DataFrame(results_sample)
# print sample 
pd_info_sample.sort_values("num_of_words", ascending=False)
# features for all 
businesses = set(dataset.alias)
results = []

for business in businesses:
    features = generate_features(dataset, business)
    results.append(features)
    
pd_info = pd.DataFrame(results)
pd_info.rename(columns={'review_count': 'num_of_reviews'}, inplace=True)
# print all ../../processed/raw/yp_competitors_info.csv
pd_info.to_csv("../../data/processed/yp_competitors_info.csv", index=False)
pd.read_csv("../../data/processed/yp_competitors_info.csv").info()
bins = [0, 100, 500, 1000, 2000, 3000, 5000]
lbls = [1, 2, 3, 4, 5, 6, 7, 8, 9]
pd_bins = pd.cut(pd_info.num_of_reviews, bins, lbls).value_counts()
pd_bins.plot(title='Binned Review Count').tick_params(axis='x', labelrotation=45)
bins = [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]
pd_bins = pd.cut(pd_info.rating_avg, bins).value_counts()
pd_bins.plot(title='Binned Rating').tick_params(axis='x', labelrotation=45)<jupyter_output><empty_output><jupyter_text>Time Analysis Business Info<jupyter_code>from numpy import arange
from matplotlib import pyplot
import nltk
from nltk import ConditionalFreqDist
import datetime
# Debug only
# dataset = dataset.iloc[:5000, :]
dataset.shape
dataset.head()<jupyter_output><empty_output><jupyter_text>For All<jupyter_code>cfdist = ConditionalFreqDist(
    (row.ratingValue, word)
    for i, row in dataset.iterrows()
    for word in word_tokenize(row.description)
    if word in modals
)
cfdist_ts_restaurants = ConditionalFreqDist(
    (row.ratingValue, word)
    for i, row in dataset[dataset.alias.isin(conditions)].iterrows()
    for word in word_tokenize(row.description)
    if word in modals
)
modals = ['can', 'could', 'may', 'might', 'must', 'will']
# func
def bar_chart(categories, words, counts, title, ax):
    "Plot a bar chart showing counts for each word by category"
    colors = 'rgbcmyk' # red, green, blue, cyan, magenta, yellow, black
    ind = arange(len(words))
    width = 1 / (len(categories) + 1)
    bar_groups = []
    for c in range(len(categories)):
        bars = plt.bar(ind+c*width, counts[categories[c]], width, color=colors[c % len(colors)])
        bar_groups.append(bars)
    plt.xticks(ind+width, words)
    plt.legend([b[0] for b in bar_groups], categories, loc='upper left')
    plt.ylabel('Frequency')
    plt.title('Frequency of Six Modal Verbs by Rating - ' + title)
# total num of each modal
counts = {}
counts_ts = {}
for i in range(1, 6):
    counts[i] = [cfdist[i][word] for word in modals]
    counts_ts[i] = [cfdist_ts_restaurants[i][word] for word in modals]

bar_chart(range(1, 6), modals, counts, 'All', ax1)
# total num of each modal
counts = {}
for i in range(1, 6):
    counts[i] = [cfdist_ts_restaurants[i][word] for word in modals]

# plot
bar_chart(range(1, 6), modals, counts, 'TS Restaurants', ax2)
cfdist.plot(samples=modals, conditions=range(1,6))
cfdist.tabulate(samples=modals)
# commented are not in the list :/
conditions=[
    'kimos-maui-lahaina',
    'sunnyside-tahoe-city-2',
    'dukes-huntington-beach-huntington-beach-2',
    # 'dukes-la-jolla-la-jolla',
    # 'dukes-malibu-malibu-2',
    'dukes-beach-house-lahaina',
    # 'dukes-kauai-lihue-3',
    # 'dukes-waikiki-honolulu-2',
    # 'hula-grill-waikiki-honolulu-3',
    'hula-grill-kaanapali-lahaina-2',
    # 'keokis-paradise-koloa',
    'leilanis-lahaina-2'
]<jupyter_output><empty_output><jupyter_text>For Each Business<jupyter_code>dataset.dataPublished = pd.to_datetime(dataset.dataPublished)
dataset["day"] = dataset.dataPublished.apply(lambda x: x.strftime("%a"))
cfd_days = ConditionalFreqDist(
    (business.alias, business.day) 
    for index, business in dataset.iterrows()
)
pd_days = pd.DataFrame.from_dict(cfd_days, orient='index', dtype=int).fillna(0)
pd_days.columns = 'Cnt_Day_' + pd_days.columns
pd_days.reset_index(level=0, inplace=True)
pd_days.rename(columns={'index': 'alias'}, inplace=True)

pd_days.to_csv("../../data/processed/yp_competitors_day.csv", index=False)
pd_days.head()
days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
sorted_columns_d = []
for day in days:
    val = [i for i in pd_days.columns if day in i]
    sorted_columns_d.append(''.join(val))
pd_days = pd_days.reindex(['alias'] + sorted_columns_d, axis=1)
pd_days.head()
# Overall Dataset
import matplotlib.pyplot as plt

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 4))
k = pd_days.iloc[:, 1:].sum().plot(
    alpha=0.90, rot=45, xticks=range(7), grid=True, ax=ax1, title="All"
)
l = pd_days[pd_days.alias.isin(conditions)].iloc[:, 1:].sum().plot(
    alpha=0.90, rot=45, xticks=range(7), grid=True, ax=ax2, title="TS Restaurants"
)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 4))
k = pd_days.iloc[:, 1:].mean().plot(
    alpha=0.90, rot=45, xticks=range(7), grid=True, ax=ax1, title="All - Mean"
)
l = pd_days[pd_days.alias.isin(conditions)].iloc[:, 1:].mean().plot(
    alpha=0.90, rot=45, xticks=range(7), grid=True, ax=ax2, title="TS Restaurants - Mean"
)
cfd_days.plot(conditions=conditions, samples=days)
cfd_days.tabulate(samples=days, conditions=conditions)
dataset.dataPublished = pd.to_datetime(dataset.dataPublished)
dataset["mon"] = dataset.dataPublished.apply(lambda x: x.strftime("%b"))
cfd = ConditionalFreqDist(
    (business.alias, business.mon) 
    for index, business in dataset.iterrows()
)
pd_months = pd.DataFrame.from_dict(cfd, orient='index', dtype=int).fillna(0)
pd_months.columns = 'Cnt_Mon_' + pd_months.columns
pd_months.reset_index(level=0, inplace=True)
pd_months.rename(columns={'index': 'alias'}, inplace=True)

pd_months.to_csv("../../data/processed/yp_competitors_mon.csv", index=False)
pd_months.head()
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
sorted_columns = []
for mon in months:
    val = [i for i in pd_months.columns if mon in i]
    sorted_columns.append(''.join(val))
pd_months = pd_months.reindex(['alias'] + sorted_columns, axis=1)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 4))
k = pd_months.iloc[:, 1:].sum().plot(
    alpha=0.90, rot=45, xticks=range(12), grid=True, ax=ax1, title="All"
)
l = pd_months[pd_months.alias.isin(conditions)].iloc[:, 1:].sum().plot(
    alpha=0.90, rot=45, xticks=range(12), grid=True, ax=ax2, title="TS Restaurants"
)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 4))
k = pd_months.iloc[:, 1:].mean().plot(
    alpha=0.90, rot=45, xticks=range(12), grid=True, ax=ax1, title="All"
)
l = pd_months[pd_months.alias.isin(conditions)].iloc[:, 1:].mean().plot(
    alpha=0.90, rot=45, xticks=range(12), grid=True, ax=ax2, title="TS Restaurants"
)
cfd.plot(conditions=conditions, samples=months)
cfd.tabulate(conditions=conditions, samples=months)<jupyter_output>Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec 
                       kimos-maui-lahaina 198 141 189 189 242 218 250 265 233 266 201 176 
                   sunnyside-tahoe-city-2  29  28  29  26  38  78 150 164  80  32  24  34 
dukes-huntington-beach-huntington-beach-2 109 105 126 143 151 118 187 168 134 133 101  86 
                dukes-beach-house-lahaina 112  99 111 120 173 163 190 172 170 176 146 132 
           hula-grill-kaanapali-lahaina-2 187 136 176 210 259 305 270 278 260 262 221 189 
                       leilanis-lahaina-2 152  97 129 130 186 211 233 224 172 180 152 146<jupyter_text>Time Plots  For All<jupyter_code>pd.to_timedelta(5, unit='D')
dataset['date_5_days_ago'] = pd.to_datetime(dataset['dataPublished']) - pd.to_timedelta(5, unit='d')
dataset.dataPublished = pd.to_datetime(dataset.dataPublished)
dfgb = dataset.groupby([pd.Grouper(key="dataPublished", freq="Q-DEC")])
dfgb_mean = dfgb.ratingValue.mean()
dfgb_mean[dfgb_mean.index > datetime.datetime(2000, 1, 1)].plot('line', figsize=(17, 5))
dfgb_mean = dfgb.ratingValue.count()
dfgb_mean[dfgb_mean.index > datetime.datetime(2000, 1, 1)].plot('line', figsize=(17, 5))
dfgb_mean = dfgb.ratingValue.sum()
dfgb_mean[dfgb_mean.index > datetime.datetime(2000, 1, 1)].plot('line', figsize=(17, 5))
a = pd.DataFrame()
a['raw'] = dataset.dataPublished
a['edited'] = dataset.dataPublished.apply(lambda t: t.to_period(freq='M'))
a.head()
a = pd.DataFrame()
a['raw'] = dataset.dataPublished
a['edited'] = dataset.groupby([pd.Grouper(key="dataPublished", freq="W-MON")])
a.head()
dataset.iloc[:100,0:].groupby([pd.Grouper(key="dataPublished", freq="W-MON")]).head()<jupyter_output><empty_output><jupyter_text>For TS Restaurants<jupyter_code>fig = plt.figure(figsize=(17, 6))
ax = fig.add_subplot()
desired_date = datetime.datetime(2016, 1, 1)

a = dataset.groupby([pd.Grouper(key="dataPublished", freq="W-MON")])['ratingValue'].mean()
a.name = 'All'
b = dataset.loc[dataset.alias.isin(conditions)].groupby([pd.Grouper(key="dataPublished", freq="W-MON")])['ratingValue'].mean()
b.name = 'TS Restaurants'
a[a.index > desired_date].plot('line', title="All", ax=ax)
b[b.index > desired_date].plot('line', title="TS Restaurants", ax=ax)
plt.legend()
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Merging DataFrames<jupyter_code>dataset_businesses.info()
pd_info.info()
pd_days.info()
pd_months.info()
merged = pd.merge(dataset_businesses, pd_info, on='alias').merge(pd_days, on='alias').merge(pd_months, on='alias')
print(merged.columns)
merged.to_csv('../../data/processed/yp_competitors.csv', index=False, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15790220708253494, 'fraction_numerical': 0.032999787098147756, 'mean_word_length': 5.296693476318141, 'pattern_counts': {'":': 2, '<': 45, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '558523', 'license': 'MIT', 'n_tokens_mistral': 5599, 'n_tokens_neox': 5163, 'n_words': 1051, 'path': 'notebooks/eda/.ipynb_checkpoints/businesses_rws-checkpoint.ipynb', 'repo_name': 'metinsenturk/semantic-analysis', 'size': '14091'}