<jupyter_start><jupyter_text>libraries<jupyter_code>import sys
import os
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import en_core_web_sm
from collections import Counter
from spacy.matcher import PhraseMatcher, Matcher


%matplotlib inline

# to import Database class from data_collection folder
module_path = os.path.abspath(os.path.join('../..')+'/data/data_collection')
if module_path not in sys.path:
    sys.path.append(module_path)

# now that the folder is in the path, ../data_collection/database.py can be imported
from storage_managers.database import Database<jupyter_output><empty_output><jupyter_text>Import raw features dataframe with Spacy Docs<jupyter_code># read pickle of dataframe with all reviews data, including Spacy Doc, and features
data_df = pd.read_pickle('/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/features_draft_v2.2.pkl')<jupyter_output><empty_output><jupyter_text>Import FP from XGBoost with oversampling model in Baseline_v4<jupyter_code># read pickles
FP_list = pd.read_pickle(os.getcwd() + '/FP_list_v2.pkl')
TN_list = pd.read_pickle(os.getcwd() + '/TN_list_v2.pkl')

# save FP reviews to csv for easier viewing
FP_df = data_df.loc[FP_list]
TN_df = data_df.loc[TN_list]
# FP_df.to_csv(os.getcwd() + '/FP_reviews_v2.csv')

# get word frequency
def most_common_n(text, n):
    nlp = en_core_web_sm.load()
    doc = nlp(text) # since I cannot join Doc objects easily I'll combine the strings
    #remove stopwords, punctuations, and whitespaces
    words = [token.lemma_.lower() for token in doc if token.is_stop != True and token.is_punct != True and token.is_space != True]
    word_freq = Counter(words)
    common_words = word_freq.most_common(n)
    return (common_words)
    
FP_top_50 = most_common_n(". ".join(FP_df['review_text']), 50) # 50 most common words from all FP reviews
TN_top_50 = most_common_n(". ".join(TN_df['review_text']), 50) # 50 most common words from all TN reviews
print(FP_top_50)
print(TN_top_50)<jupyter_output>[('food', 148), ('halal', 147), ('good', 118), ('cart', 95), ('chicken', 95), ('like', 87), ('sauce', 78), ('order', 69), ('come', 60), ('place', 55), ('$', 55), ('rice', 55), ('lamb', 53), ('try', 51), ('time', 50), ('eat', 49), ('get', 48), ('meat', 48), ('taste', 44), ('go', 44), ('great', 42), ('sandwich', 39), ('salad', 37), ('know', 36), ('service', 35), ('definitely', 35), ('white', 35), ('restaurant', 34), ('hot', 34), ('platter', 34), ('think', 32), ('guy', 31), ('lunch', 31), ('line', 30), ('delicious', 30), ('dish', 30), ('gyro', 28), ('grill', 28), ('want', 28), ('noodle', 28), ('ask', 27), ('price', 27), ('meal', 26), ('long', 26), ('burger', 26), ('pita', 26), ('day', 25), ('little', 25), ('find', 24), ('fry', 24)]<jupyter_text>get sentencs around the word halal<jupyter_code>nlp = en_core_web_sm.load()
doc = nlp(". ".join(FP_df['review_text']))
sents = [doc[token.i:token.i-1].sent for token in doc if token.text.lower() == 'halal']
sents[:5]
from spacy import displacy

displacy.render(sents[0], style="dep")<jupyter_output><empty_output><jupyter_text>New Features:    - Seperate different meats in halal_relevant b/c their distribution seems to be different in FP & TN    - Add 'only' matching if sentence also includes halal to see if it improves model. If not, match for 'only' with halal as part of token's head/ children.  1. Seperate halal meats<jupyter_code># initialize nlp if necessary
nlp = en_core_web_sm.load()

def halal_match_and_count(doc_col, pattern):
    # initialize phrase matcher
    matcher = PhraseMatcher(nlp.vocab)
    # add phrases
    matcher.add(pattern, None, nlp(pattern))
    # phrase match
    res = []
    count = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
        count.append(len(match))
    return res, count

def get_match_data(text):
    pattern = ' '.join(['halal', text])
    bool_col_name = pattern.replace(' ', '_')
    count_col_name = bool_col_name + '_count'
    data_df[bool_col_name], data_df[count_col_name] = halal_match_and_count(data_df['doc'], pattern)
    if (bool_col_name in data_df.columns) and (count_col_name in data_df.columns):
        print('Matched and counted "Halal {}"'.format(text))

word_list = ['chicken', 'beef', 'lamb', 'goat', 'burger', 'fried', 'meat']
for word in word_list:
    get_match_data(word)
pd.crosstab(data_df['halal'], data_df['halal_meat'])<jupyter_output><empty_output><jupyter_text>2. Redefine halal_relevant to include 'is halal' only and change name to is_halal<jupyter_code># initialize nlp if necessary
nlp = en_core_web_sm.load()

def halal_matches_and_counts(doc_col, patterns):
    # initialize phrase matcher
    matcher = PhraseMatcher(nlp.vocab)
    # add phrases
    nlp_patterns = [nlp(pattern) for pattern in patterns]
    matcher.add('matcher', None, *nlp_patterns)
    # phrase match
    res = []
    count = []
    for i, doc in doc_col.iteritems():
        match = matcher(doc)
        res.append(True if len(match) else False)
        count.append(len(match))
    return res, count

data_df['is_halal'], data_df['is_halal_count'] = halal_matches_and_counts(data_df['doc'], ('is halal', "'s halal", 'halal place', 'halal spot', 'halal options', 'restaurant', 'halal place'))
pd.crosstab(data_df['halal'], data_df['is_halal'])<jupyter_output><empty_output><jupyter_text>3. Roughly detect partial halal establishments<jupyter_code># initialize nlp if necessary
nlp = en_core_web_sm.load()

def double_match_and_count(doc_col, word_1, word_2):
    # initialize phrase matcher
    matcher_1 = Matcher(nlp.vocab)
    matcher_2 = Matcher(nlp.vocab)
    # add matching patterns
    pattern1 = [{'LOWER': word_1}]; matcher_1.add(word_1, None, pattern1)
    pattern2 = [{'LOWER': word_2}]; matcher_2.add(word_2, None, pattern2)
    # match for word_1 first, which preferably should be less common
    res = []
    count = []
    for i, doc in doc_col.iteritems():
        double_match_counter = 0 
        first_matches = matcher_1(doc)
        for match_id, start, end in first_matches:
            doc2 = nlp(doc[start].sent.text)
            second_match = matcher_2(doc2)
            if len(second_match):
                double_match_counter+=1
        res.append(True if double_match_counter>0 else False)
        count.append(double_match_counter)
    return res, count
    
data_df['partial_halal'], data_df['partial_halal_count'] = double_match_and_count(data_df['doc'], 'only', 'halal')
pd.crosstab(data_df['halal'], data_df['partial_halal'])<jupyter_output><empty_output><jupyter_text>Pickle dataframe with features for modeling -Version 4<jupyter_code>file_name_v3 = os.getcwd() + '/restaurant_cat_and_num_v3.pkl'
data_df.drop(['review_text', 'review_date', 'restaurant_name', 'doc', 'halal_relevant_count', 'halal_relevant'] ,axis=1).to_pickle(file_name_v3)<jupyter_output><empty_output><jupyter_text>4. Detect comparative sentences<jupyter_code>'''
using fine-grained POS tags
token.tag_ = JJR (adjective, comparative), RBR (adverb, comparative)
'''

## find sentences that has tokens with these tags in the first 20 docs and print them
def detetct_comparative(doc_col):
    # initialize phrase matcher
    matcher = Matcher(nlp.vocab)
    # add phrases
    comparative_pattern = [{'TAG': {'IN': ['JJR', 'RBR']}}]
    matcher.add('matcher', None, comparative_pattern)
    # match pattern and look for halal in the comparative term's subtree
    res = []
    count = []
    for i, doc in doc_col.iteritems():
        matches = matcher(doc)
        comp_halal_counter = 0
        for match_id, start, end in matches:
            if 'halal' in [word.text for word in doc[start].head.subtree]:
                comp_halal_counter += 1
        res.append(True if comp_halal_counter>0 else False)
        count.append(comp_halal_counter)
    return res, count
            
            
#         res.append(True if len(match) else False)
#         count.append(len(match))
#     return res, count

data_df['comparative_halal'], data_df['comparative_halal_count'] = detetct_comparative(data_df['doc'])

# normalize count feature by dividing by total halal count
data_df['comparative_halal_percent'] = data_df['comparative_halal_count'] / data_df['halal_review_count']
pd.crosstab(data_df['halal'], data_df['comparative_halal'])<jupyter_output><empty_output><jupyter_text>Pickle dataframe with features for modeling -Version 4<jupyter_code>file_name_v4 = os.getcwd() + '/restaurant_cat_and_num_v4.pkl'
data_df.drop(['review_text', 'review_date', 'restaurant_name', 'doc'] ,axis=1).to_pickle(file_name_v4)
data_df.columns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14059633027522936, 'fraction_numerical': 0.01938073394495413, 'mean_word_length': 4.779324055666004, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '639310', 'license': 'FTL', 'n_tokens_mistral': 3230, 'n_tokens_neox': 3040, 'n_words': 753, 'path': 'src/features/feature_engineering/Feature_Engineering_v4.ipynb', 'repo_name': 'WAzaizeh/Halal_o_Meter', 'size': '8720'}