<jupyter_start><jupyter_text>libraries<jupyter_code>import sys
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import en_core_web_sm
from spacy.matcher import Matcher
from spacy.matcher import PhraseMatcher
from spacy import displacy


%matplotlib inline

# to import Database class from data_collection folder
module_path = os.path.abspath(os.path.join('../..')+'/data/data_collection')
if module_path not in sys.path:
    sys.path.append(module_path)

# now that the folder is in the path, ../data_collection/database.py can be imported
from storage_managers.database import Database<jupyter_output><empty_output><jupyter_text>Import reviews data and target feature<jupyter_code>db = Database()

# get halal-reviews (reviews that include the word 'halal')
reviews_sql = '''SELECT * FROM reviews'''
reviews_df = db.select_df(reviews_sql)
print('- {} reviews containing the word halal were scraped'.format(reviews_df.shape[0]))

# get target restaurants-of-interest list 
file_path = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/target_feature/label_target.csv'
target_df = pd.read_csv(file_path, index_col=0)
target_df['halal'] = target_df['halal'].str.replace('FLASE', 'FALSE')
target_df['halal'] = target_df['halal'].apply(lambda x: True if x =='TRUE' else False)
halal_frac = target_df['halal'].sum()/target_df.shape[0]

print('- {:.0f}% of the {} restaurants-of-interest are halal'.format(halal_frac*100, target_df.shape[0]))<jupyter_output>- 5092 reviews containing the word halal were scraped
- 73% of the 838 restaurants-of-interest are halal<jupyter_text>Feature Engineering 1. 'halal' in business name<jupyter_code># patch missing platform_ids and mismatch in target data

# import original businesses data
rest_sql = '''SELECT * FROM businesses WHERE url LIKE '%yelp%' '''
rest_df = db.select_df(rest_sql)

# drop Aya Kitchen
aya_id = 'y6BfLt9Gvrq2JsJvjkjdIQ'
reviews_df.drop(reviews_df[reviews_df['restaurant_id'] == aya_id].index, inplace=True)

# patch platform_id in target_df
target_df = target_df.merge(rest_df[['platform_id', 'url']], how='left', on='url')
target_df.drop('platform_id_x', inplace=True, axis=1)
target_df = target_df.rename(columns={'platform_id_y' : 'platform_id'})
# group reviews per restaurant
grouped_reviews_df = reviews_df.groupby('restaurant_id').agg(lambda x: ' '.join(x)) # combine review text
grouped_reviews_df['review_date'] = grouped_reviews_df['review_date'].apply(lambda x: x.split()) # make dates list
grouped_reviews_df['review_count'] = grouped_reviews_df['review_date'].apply(lambda x: len(x)) # count reviews per restaurnat
grouped_reviews_df.head()

# merge restaurant name 
grouped_reviews_df = grouped_reviews_df.merge(target_df[['platform_id', 'name', 'total_review_count', 'halal']], how='left', left_index=True, right_on='platform_id')

grouped_reviews_df.index = grouped_reviews_df['platform_id']
grouped_reviews_df.drop('platform_id', inplace=True, axis=1)
grouped_reviews_df = grouped_reviews_df.rename(columns={'name': 'restaurant_name', 'review_count' : 'halal_review_count'})

# add boolean column for 'halal' in restaurant name
grouped_reviews_df['halal_in_name'] = grouped_reviews_df.apply(lambda row: True if 'halal' in row['restaurant_name'].lower() else False, axis=1)

plt.figure(figsize=(14,8))
g = sns.countplot(x='halal', hue='halal_in_name', data=grouped_reviews_df)
g.set_title('Distribution of halal_in_name feature', size=14)
g_labels = g.set_xticklabels(g.get_xticklabels(), rotation=45, horizontalalignment='right')
for p in g.patches:
    g.annotate('{:.0f}'.format(p.get_height()), (p.get_x()+0.3, p.get_height()),
                ha='center', va='bottom', color= 'black')<jupyter_output><empty_output><jupyter_text>2. Percentage of reviews including the word halal out of all reviews<jupyter_code># calculate percentage of halal-containing reviews out of total reviews
grouped_reviews_df['halal_review_percent'] = grouped_reviews_df.apply(lambda row: row['halal_review_count']/row['total_review_count'], axis =1)

# plot distribution for halal vs non-halal restaurants
halal_target = grouped_reviews_df[grouped_reviews_df['halal']]['halal_review_percent']
non_halal_target = grouped_reviews_df[~grouped_reviews_df['halal']]['halal_review_percent']
fig = plt.figure(figsize=(14,8))
g1 = sns.distplot(halal_target*100, kde=False, norm_hist=True)
g2 = sns.distplot(non_halal_target*100, kde=False, norm_hist=True)
g1.set_title('Distribution of the percent of halal-reviews out of all reviews for restaurant-of-interest', size=14)
g1.set_xlabel('% of halal review', size=14)
g1.set_ylabel('Count', size=14)
plt.legend(labels=['Halal restaurants','Non-halal restaurants'], prop={'size': 10})
plt.show()
grouped_reviews_df[(grouped_reviews_df['halal_review_percent'] > 0.9) & (~grouped_reviews_df['halal'])]<jupyter_output><empty_output><jupyter_text>- The higher percentage of halal-related reviews in non-halal restaurants is probably related to food carts or restaurants with few reviews to start with and they have comparison to halal-guys or halal street cuisine   3. Spacy nlp 3.1. Includes halal-relevant terms- List of halal-relevant terms: [{LEMMA: be},{LOWER: halal}], [{LOWER: halal}, {LOWER: meat/options/chicken/beef/lamb/spot/restaurant/burger}]<jupyter_code>## run nlp of grouped review text and save Doc to dataframe
nlp = en_core_web_sm.load()
grouped_reviews_df['doc'] = grouped_reviews_df['review_text'].apply(lambda x: nlp(x))

# initialize phrase matcher
matcher = PhraseMatcher(nlp.vocab)

# # specify match pattern for matcher
# pattern1 = [{'LEMMA': 'be'},{'LOWER': 'halal'}]
# pattern2 = [{'LOWER': 'halal'},{'LOWER': 'meat'}]
# pattern3 = [{'LOWER': 'halal'},{'LOWER': 'beef'}]
# pattern4 = [{'LOWER': 'halal'},{'LOWER': 'lamb'}]
# pattern5 = [{'LOWER': 'halal'},{'LOWER': 'chicken'}]
# pattern6 = [{'LOWER': 'halal'},{'LOWER': 'burger'}]
# pattern7 = [{'LOWER': 'halal'},{'LOWER': 'options'}]
# pattern8 = [{'LOWER': 'halal'},{'LOWER': 'place'}]
# pattern9 = [{'LOWER': 'halal'},{'LOWER': 'spot'}]
# pattern10 = [{'LOWER': 'halal'},{'LOWER': 'restaurant'}]
# matcher.add('be_halal', None, pattern1)

matcher.add('halal', None, nlp('is halal'), nlp("'s halal"), nlp('halal meat'), nlp('halal beef'),
            nlp('halal lamb'), nlp('halal goat'), nlp('halal chicken'), nlp('halal burger'), 
            nlp('halal options'), nlp('halal place'), nlp('halal spot'), nlp('halal restaurant'))

# match and print the first few setences
c=10
for i, doc in grouped_reviews_df['doc'].iteritems():
    match = matcher(doc)
    count = len(match)
    grouped_reviews_df.loc[i, 'halal_relevant_count'] = count
    grouped_reviews_df.loc[i, 'halal_relevant'] = True if count>0 else False
    if c>0:
        for match_id, start, end in match:
            print('{}:'.format(i), doc[start:end].sent)
        c -= 1
# visualize presence of halal_relevant terms vs target feature
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['halal_relevant'])<jupyter_output><empty_output><jupyter_text>3.2. Includes negation of halal<jupyter_code># look for negation tokens. Then check if halal is in token.head or token.children
# print first few results
c=5
for i, doc in grouped_reviews_df['doc'].iteritems():
    grouped_reviews_df.loc[i, 'halal_negation_count'] = 0
    grouped_reviews_df.loc[i, 'halal_negation'] = False
    for token in doc:
        # for negation tokens, check if 'halal' is in token.head or children
        if (token.dep_ == 'neg') & ((token.head.text.lower() == 'halal') | ('halal' in [child.text.lower() for child in token.head.children])):
            grouped_reviews_df.loc[i, 'halal_negation'] = True
            grouped_reviews_df.loc[i, 'halal_negation_count'] = grouped_reviews_df.loc[i, 'halal_negation_count'] + 1
            if c>0:
                sent = doc[token.i: token.i+1].sent
                print('\n',sent)
                print('Head is: {}'.format(token.head), token.head.pos_)
                print('Children list:', *[child.text for child in token.head.children])
                print('Children list:', *[child.pos_ for child in token.head.children])
                c-=1
# visualize halal negation vs. target feature
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['halal_negation'])<jupyter_output><empty_output><jupyter_text>3.3. Incidences of 'halal guys'/ 'halal truck'<jupyter_code># initialize phrase matcher
matcher = PhraseMatcher(nlp.vocab)

# add phrases
matcher.add('halal', None, nlp('halal guys'), nlp('halal truck'))

# match and print the first few setences
c=10
for i, doc in grouped_reviews_df['doc'].iteritems():
    match = matcher(doc)
    count = len(match)
    grouped_reviews_df.loc[i, 'halal_truck_count'] = count
    grouped_reviews_df.loc[i, 'halal_truck'] = True if count>0 else False
    if c>0:
        for match_id, start, end in match:
            print('{}:'.format(i), doc[start:end].sent)
        c -= 1
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['halal_truck'])<jupyter_output><empty_output><jupyter_text>3.4. Includes non-halal terms- List of non-halal terms: pork, bacon, ham, alcohol, beer, wine, shortening, lard, geltin<jupyter_code># initialize matcher
matcher = Matcher(nlp.vocab)

# add pattern
pattern1 = [{'LOWER': 'pork'}]; matcher.add('non-halal', None, pattern1)
pattern2 = [{'LOWER': 'bacon'}]; matcher.add('non-halal', None, pattern2)
pattern3 = [{'LOWER': 'ham'}]; matcher.add('non-halal', None, pattern3)
pattern4 = [{'LOWER': 'alcohol'}]; matcher.add('non-halal', None, pattern4)
pattern5 = [{'LOWER': 'beer'}]; matcher.add('non-halal', None, pattern5)
pattern6 = [{'LOWER': 'wine'}]; matcher.add('non-halal', None, pattern6)
pattern7 = [{'LOWER': 'shortening'}]; matcher.add('non-halal', None, pattern7)
pattern8 = [{'LOWER': 'lard'}]; matcher.add('non-halal', None, pattern8)
pattern9 = [{'LOWER': 'gelatin'}]; matcher.add('non-halal', None, pattern9)
pattern10 = [{'LOWER': 'bar'}]; matcher.add('non-halal', None, pattern10)

# match and print the first few setences
c=10
for i, doc in grouped_reviews_df['doc'].iteritems():
    match = matcher(doc)
    count = len(match)
    grouped_reviews_df.loc[i, 'non_halal_relevant_count'] = count
    grouped_reviews_df.loc[i, 'non_halal_relevant'] = True if count>0 else False
    if c>0:
        for match_id, start, end in match:
            print('{}:'.format(i), doc[start:end].sent)
        c -= 1
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['non_halal_relevant'])<jupyter_output><empty_output><jupyter_text>3.5. Includes CreekStone<jupyter_code># initialize matcher
matcher = Matcher(nlp.vocab)

# add pattern
pattern = [{'LOWER': 'creekstone'}]
matcher.add('creekstone', None, pattern)

# match and print the first few setences
c=2
for i, doc in grouped_reviews_df['doc'].iteritems():
    match = matcher(doc)
    count = len(match)
    grouped_reviews_df.loc[i, 'creekstone_count'] = count
    grouped_reviews_df.loc[i, 'creekstone'] = True if count>0 else False
    if count>0 and c>0:
        for match_id, start, end in match:
            print('{}:'.format(i), doc[start:end].sent)
        c -= 1
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['creekstone'])<jupyter_output><empty_output><jupyter_text>3.6. Incidences of beef/ trukey bacon<jupyter_code># initialize matcher
matcher = Matcher(nlp.vocab)

# add pattern
pattern1 = [{'LOWER': 'beef'}, {'LOWER': 'bacon'}]; matcher.add('halal_bacon', None, pattern1)
pattern2 = [{'LOWER': 'turkey'}, {'LOWER': 'bacon'}]; matcher.add('halal_bacon', None, pattern2)

# match and print the first few setences
c=2
for i, doc in grouped_reviews_df['doc'].iteritems():
    match = matcher(doc)
    count = len(match)
    grouped_reviews_df.loc[i, 'hala_bacon_count'] = count
    grouped_reviews_df.loc[i, 'halal_bacon'] = True if count>0 else False
    if count>0 and c>0:
        for match_id, start, end in match:
            print('{}:'.format(i), doc[start:end].sent)
        c -= 1
pd.crosstab(grouped_reviews_df['halal'], grouped_reviews_df['halal_bacon'])<jupyter_output><empty_output><jupyter_text>4. Pickle dataframe and save it<jupyter_code>grouped_reviews_df.columns<jupyter_output><empty_output><jupyter_text>4.1. pickle dataframe with categorial features only<jupyter_code># to_pickle_df = grouped_reviews_df[['halal', 'halal_in_name', 'halal_review_percent', 'halal_relevant',
#                                    'halal_negation', 'halal_truck', 'creekstone', 'non_halal_relevant',
#                                    'halal_bacon']]
# file_name = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/features_draft_1.pkl'
# to_pickle_df.to_pickle(file_name)<jupyter_output><empty_output><jupyter_text>4.2. pickle dataframe with numerical features only<jupyter_code># to_pickle_df = grouped_reviews_df[['halal', 'halal_in_name', 'halal_review_percent', 'halal_relevant_count',
#                                    'halal_negation_count', 'halal_truck_count','non_halal_relevant_count',
#                                    'creekstone_count', 'hala_bacon_count']]
# file_name = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/features_draft_2.pkl'
# to_pickle_df.to_pickle(file_name)<jupyter_output><empty_output><jupyter_text>5. Make data frame with individual reviews for classification- Features to merge from business data:    - 'halal_in_name'    - percentage of halal reviews out of total reviews    - Halal calssification (target)- Need to figure out a way combine review predictions to a restaurant score<jupyter_code># make sure Aya Kitchen reviews are removed
aya_id = 'y6BfLt9Gvrq2JsJvjkjdIQ'
if aya_id in reviews_df.restaurant_id.unique().tolist():
    reviews_df.drop(reviews_df[reviews_df['restaurant_id'] == aya_id].index, inplace=True)

# merge fatures from grouped_reviews
reviews_df2 = reviews_df.merge(grouped_reviews_df[['halal_in_name', 'halal_review_percent', 'halal']],
                 left_on='restaurant_id', right_index=True)
print('{:.0f}% of the {} reviews are for halal restaurants'.format(reviews_df2.halal.sum()/ reviews_df2.shape[0]*100,
                                                               reviews_df2.shape[0]))
# define functions for nlp feature extraction
def is_halal_relevant(doc_col):
    # initialize phrase matcher
    matcher = PhraseMatcher(nlp.vocab)
    # add phrases
    matcher.add('halal', None, nlp('is halal'), nlp("'s halal"), nlp('halal meat'), nlp('halal beef'),
                nlp('halal lamb'), nlp('halal goat'), nlp('halal chicken'), nlp('halal burger'), 
                nlp('halal options'), nlp('halal place'), nlp('halal spot'), nlp('halal restaurant'))
    # phrase match
    res = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
    return res

def is_halal_negation(doc_col):
    res = []
    for i, doc in doc_col.iteritems():
        found = False
        for token in doc:
            # for negation tokens, check if 'halal' is in token.head or children
            if (token.dep_ == 'neg') & (token.head.text.lower() == 'halal'):
                res.append(True)
                found = True
                break
            elif (token.dep_ == 'neg') & ('halal' in [child.text.lower() for child in token.head.children]):
                res.append(True)
                found = True
                break
        if not found:
            res.append(False)
    return res

def is_halal_truck(doc_col):
    # initialize phrase matcher
    matcher = PhraseMatcher(nlp.vocab)
    # add phrases
    matcher.add('halal', None, nlp('halal guys'), nlp('halal truck'))
    # phrase match
    res = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
    return res

def is_non_halal_relevant_and_count(doc_col):
    # initialize matcher
    matcher = Matcher(nlp.vocab)
    # add patterns
    pattern1 = [{'LOWER': 'pork'}]; matcher.add('non-halal', None, pattern1)
    pattern2 = [{'LOWER': 'bacon'}]; matcher.add('non-halal', None, pattern2)
    pattern3 = [{'LOWER': 'ham'}]; matcher.add('non-halal', None, pattern3)
    pattern4 = [{'LOWER': 'alcohol'}]; matcher.add('non-halal', None, pattern4)
    pattern5 = [{'LOWER': 'beer'}]; matcher.add('non-halal', None, pattern5)
    pattern6 = [{'LOWER': 'wine'}]; matcher.add('non-halal', None, pattern6)
    pattern7 = [{'LOWER': 'shortening'}]; matcher.add('non-halal', None, pattern7)
    pattern8 = [{'LOWER': 'lard'}]; matcher.add('non-halal', None, pattern8)
    pattern9 = [{'LOWER': 'gelatin'}]; matcher.add('non-halal', None, pattern9)
    pattern10 = [{'LOWER': 'bar'}]; matcher.add('non-halal', None, pattern10)
    # match
    bool_list = []
    count = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        bool_list.append(True if len(match) else False)
        count.append(len(match))
    return bool_list, count


def is_creekstone(doc_col):
    # initialize matcher
    matcher = Matcher(nlp.vocab)
    # add pattern
    pattern = [{'LOWER': 'creekstone'}]
    matcher.add('creekstone', None, pattern)
    #match
    res = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
    return res

def is_halal_bacon(doc_col):
    # initialize matcher
    matcher = Matcher(nlp.vocab)
    # add patterns
    pattern1 = [{'LOWER': 'beef'}, {'LOWER': 'bacon'}]; matcher.add('halal_bacon', None, pattern1)
    pattern2 = [{'LOWER': 'turkey'}, {'LOWER': 'bacon'}]; matcher.add('halal_bacon', None, pattern2)
    # match
    res = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
    return res
# generate col with Spacy Doc for each review to facilitate matching
# started 10:57am - 11:00am (13 mins to run)
reviews_df2['doc'] = reviews_df2['review_text'].apply(lambda x: nlp(x))

reviews_df2['halal_relevant'] = is_halal_relevant(reviews_df2['doc'])
reviews_df2['halal_negation'] = is_halal_negation(reviews_df2['doc'])
reviews_df2['halal_truck'] = is_halal_truck(reviews_df2['doc'])
reviews_df2['non_halal_relevant'], reviews_df2['non_halal_relevant_count'] = is_non_halal_relevant_and_count(reviews_df2['doc'])
reviews_df2['creekstone'] = is_creekstone(reviews_df2['doc'])
reviews_df2['halal_bacon'] = is_halal_bacon(reviews_df2['doc'])
# visualize presence of halal_relevant terms vs target feature
pd.crosstab(reviews_df2['halal'], reviews_df2['halal_relevant'])
# visualize halal negation vs. target feature
pd.crosstab(reviews_df2['halal'], reviews_df2['halal_negation'])
# visualize halal truck vs. target feature
pd.crosstab(reviews_df2['halal'], reviews_df2['halal_truck'])
# presence of non-halal terms vs. target feature
pd.crosstab(reviews_df2['halal'], reviews_df2['non_halal_relevant'])<jupyter_output><empty_output><jupyter_text>Note: Making this feature account for the position of 'halal' and make sure it's following the negation only shifted 9 reviews from halal/has halal negation from halal/has no halal negation. i.e. not worth the extra calulation.<jupyter_code># mention of halal bacon vs. target feature
pd.crosstab(reviews_df2['halal'], reviews_df2['halal_bacon'])<jupyter_output><empty_output><jupyter_text>How to normalize date column?  6. Pickle granulated categorical review data<jupyter_code># to_pickle_df3 = reviews_df2[['halal', 'halal_in_name', 'halal_review_percent', 'halal_relevant',
#                                    'halal_negation', 'halal_truck','non_halal_relevant',
#                                    'non_halal_relevant_count', 'creekstone', 'halal_bacon', 'restaurant_id']]
# file_name3 = os.getcwd() + '/features_draft_3.pkl'
# to_pickle_df3.to_pickle(file_name3)
reviews_df2.groupby('restaurant_id')['halal'].apply(lambda x: np.sum(x)/len(x)).value_counts()<jupyter_output><empty_output><jupyter_text>7. More Features<jupyter_code># add halal_neg_percent feature 
# number of negated halal mention divided by total number of reviews that include the word halal
grouped_reviews_df['halal_negation_percent'] = grouped_reviews_df['halal_negation_count'] / grouped_reviews_df['halal_review_count']

# plot distribution for halal vs non-halal restaurants
halal_target = grouped_reviews_df[grouped_reviews_df['halal']]['halal_negation_percent']
non_halal_target = grouped_reviews_df[~grouped_reviews_df['halal']]['halal_negation_percent']
fig = plt.figure(figsize=(14,8))
g1 = sns.distplot(halal_target*100, kde=False, norm_hist=True)
g2 = sns.distplot(non_halal_target*100, kde=False, norm_hist=True)
g1.set_title('Negated halal mentions count/ total reviews that include word halal', size=14)
g1.set_xlabel('% of halal review', size=14)
g1.set_ylabel('Frequency', size=14)
plt.legend(labels=['Halal restaurants','Non-halal restaurants'], prop={'size': 10})
plt.show()<jupyter_output><empty_output><jupyter_text>8. Pickle dataframe with all features<jupyter_code>grouped_reviews_df.columns
# file_name4 = os.getcwd() + '/restaurant_cat_and_num.pkl'
# grouped_reviews_df.drop(['review_text', 'review_date', 'restaurant_name', 'doc'] ,axis=1).to_pickle(file_name4)<jupyter_output><empty_output><jupyter_text>9. Pickle whole dataframe if needed later<jupyter_code>file_name5 = os.getcwd() + '/features_draft_v2.2.pkl'
grouped_reviews_df.to_pickle(file_name5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14314300339708688, 'fraction_numerical': 0.012331890734794546, 'mean_word_length': 4.99609375, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '109201', 'license': 'FTL', 'n_tokens_mistral': 7806, 'n_tokens_neox': 7579, 'n_words': 1618, 'path': 'src/features/feature_engineering/Feature_Engineering_v2.ipynb', 'repo_name': 'WAzaizeh/Halal_o_Meter', 'size': '21489'}