<jupyter_start><jupyter_text>Yelp (csv)  [optional] only run if **train_data_full.csv** not exists Merging seperated csv data into one Pandas dataframe<jupyter_code>import pandas as pd
import time
import datetime
from pathlib import Path
data_dir = Path.cwd() / 'datasets/yelp'
# data_dir = Path.home() / 'GenderPerformance/datasets/yelp'
# 0: business, 1: checkin, 2: photo, 3: review, 4: tip, 5: user
# data_class = {'business': 'business', 'checkin': 'checkin', 'photo': 'photo', 'review': 'review', 'tip': 'tip', 'user': 'user'}

business_df = pd.read_csv(data_dir / 'business.csv', usecols=['business_id', 'name', 'categories'])
review_df = pd.read_csv(data_dir / 'review.csv', usecols=['review_id', 'user_id', 'business_id', 'stars', 'date', 'useful', 'text'])
user_df = pd.read_csv(data_dir / 'user.csv', usecols=['user_id', 'name'])

def get_business_category(business_id):
    return business_df[business_df['business_id'] == business_id]['categories']

def get_username(user_id):
    return user_df[user_df['user_id'] == user_id]['name']

def date_to_timestamp(date_str):
    return time.mktime(datetime.datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S').timetuple())
    
# refining interested attributes
business_df.rename(columns={'name': 'business_name'}, inplace=True)
business_df = business_df.dropna(axis=0)
user_df.rename(columns={'name': 'user_name'}, inplace=True)<jupyter_output><empty_output><jupyter_text>Extracting  categories correlated to 'restaurants' coproduct for NaturalExptCategory(not_pairwise), not used here<jupyter_code>import pandas as pd
data_dir = Path.home() / 'GenderPerformance/datasets/yelp'

business_df = pd.read_csv(data_dir + 'business.csv', usecols=['business_id', 'name', 'categories'])
business_df['categories_lower'] = business_df['categories'].apply(lambda x: str(x).lower())
# categories = business_df['categories_lower'].unique()
categories = business_df['categories_lower']
# directly correlated to 'restaurants'
split_cates = [None] * len(categories)
correlated_categories = {}
for c_k, c_v in enumerate(categories):
    if 'restaurants' in c_v:
        split_cates[c_k] = c_v.lower().split(',')
        for c in split_cates[c_k]:
            c_striped = c.strip()
            correlated_categories[c_striped] = correlated_categories.get(c_striped, 0) + 1
 
with open(data_dir / 'directly_correlated_categories.json', 'w') as f:
    json.dump(correlated_categories, f)

sorted_correlated_categories = sorted(correlated_categories.items(), key=lambda kv: kv[1], reverse=True)
print(sorted_correlated_categories[:11])
# correlated_categories = {'restaurants': False, 'chinese': False}
import queue
q = queue.Queue()
for key in correlated_categories:
    q.put(key)
print('# of directly correlated: ', len(correlated_categories))

def bfs_category():
    while not q.empty():
        element = q.get()
        if element is None:
            break;

#         print('element:', element)
        for c in categories:
            if element in c:
                split_categories = c.lower().split(',')
                for split_categoy in split_categories:
                    split_categoy = split_categoy.strip()
                    correlated_categories[split_categoy] = correlated_categories.get(split_categoy, 0) + 1
                    if split_categoy not in correlated_categories:
#                         print('new:', split_categoy)
                        q.put(split_categoy)
bfs_category()
print('# of directly and undirectly correlated: ', len(correlated_categories))
# len(correlated_categories)
sorted_cates = sorted(correlated_categories.items(), key=lambda kv: kv[1], reverse=True)
print(sorted_cates[:11])
len(correlated_categories)
import json
with open(data_dir / 'all_correlated_categories.json', 'w') as f:
    json.dump(correlated_categories, f)<jupyter_output><empty_output><jupyter_text>Counting top N unique independent categoriescoproduct for NaturalExptCategory(not_pairwise), not used here<jupyter_code>business_df = business_df.dropna(axis=0)
categories_column =  business_df.categories
category_freq = {}
for cate in categories_column:
    for sub_cate in cate.lower().split(','):
        sub_cate = sub_cate.strip()
        category_freq[sub_cate] = category_freq.get(sub_cate, 0) + 1
sorted_category_freq = sorted(category_freq.items(), key=lambda kv: kv[1], reverse=True)
with open(data_dir / 'top_10_categories.json', 'w') as f:
    json.dump(sorted_category_freq[:10], f)
len(sorted_category_freq)
sorted_category_freq
import json
directly = True
if directly:
    with open(data_dir / 'directly_correlated_categories.json') as json_file:
        restaurants_related_categories = json.load(json_file)
else:
    with open(data_dir / 'all_correlated_categories.json', 'w') as json_file:
        restaurants_related_categories = json.load(json_file)
interest_categories = {}
top_n = 5
for category_tuple in sorted_category_freq:
    if len(interest_categories) >= top_n:
        break
    
#     print(category_tuple[0], correlated_categories[category_tuple[0]])
    if category_tuple[0] not in restaurants_related_categories:
        interest_categories[category_tuple[0]] = category_tuple[1]


interest_categories['restaurants'] = correlated_categories['restaurants']
print('interested categories:', interest_categories)
print(sorted_category_freq[:10])
import json
with open(data_dir / 'top_n_correlated_categories.json', 'w') as f:
    json.dump(interest_categories, f)<jupyter_output><empty_output><jupyter_text>Merging dataframes<jupyter_code>merged_df = pd.merge(review_df, business_df, on='business_id', how='outer')
merged_df = pd.merge(merged_df, user_df, on='user_id', how='outer')
merged_df = merged_df.dropna(axis=0) # drop rows where NAN exists
merged_df['timestamp'] = merged_df['date'].apply(lambda x:date_to_timestamp(x))
merged_df['text'].replace(to_replace=[r"\\t|\\n|\\r", "\t|\n|\r"], value=["",""], regex=True, inplace=True)
merged_df.to_csv(data_dir / 'train_data_full.csv', index=False)
merged_df.head()
merged_df = pd.read_csv(data_dir / 'train_data_full.csv')
merged_df.head()
data_dir = Path.home() / 'GenderPerformance/datasets/yelp'
business_df = pd.read_csv(data_dir + 'business.csv', usecols=['business_id', 'categories'])
merged_df = pd.merge(merged_df, business_df, on='business_id', how='outer')
merged_df.to_csv('train_data_full_cate.csv', index=False)<jupyter_output><empty_output><jupyter_text>Reading complete data from csv file<jupyter_code>import os.path
import pandas as pd
if not os.path.exists(data_dir / 'train_data_full.csv'):
    merged_df.to_csv('train_data_full.csv', index=False)

if merged_df is None:
    merged_df = pd.read_csv(data_dir / 'train_data_full.csv')
merged_df.head()
import gender_guesser.detector as gender

d = gender.Detector(case_sensitive=False)

import string
translator = str.maketrans('', '', string.punctuation)

def get_gender(name):
    try:
        n = name.lower()
        if ('mom' in n) or ('girl' in n) or ('angel' in n) or ('mum' in n) or ('mother' in n) or ('woman' in n):
            return 'female'
        if ('boy' in n) or ('dude' in n):
            return 'male'
        temp = name.translate(translator).split()
    except:
        return 'unknown'
    
    if len(temp) > 0:
        first_name = temp[0]
        first_name = ''.join([i for i in first_name if not i.isdigit()])
        gender = d.get_gender(first_name,'usa')
        return gender
    else:
        return 'unknown'
print(get_gender())
merged_df['gender'] = merged_df['user_name'].apply(lambda x:get_gender(x))
merged_df.head()
disclosed_flags = merged_df['gender'].isin(['female', 'male'])
undisclosed_flags = ~disclosed_flags
disclosed_gender_df = merged_df[disclosed_flags]
disclosed_gender_df.head()
disclosed_gender_df = disclosed_gender_df.drop(columns=['gender'])
disclosed_gender_df.head()
undisclosed_gender_df = merged_df[undisclosed_flags]
undisclosed_gender_df = undisclosed_gender_df.drop(columns=['Gender'])
undisclosed_gender_df.head()<jupyter_output><empty_output><jupyter_text>Save disclosed and undisclosed whole data to csv, if csv file exists, reading them in<jupyter_code>import os.path
import pandas as pd
if not os.path.exists(data_dir / 'disclosed_dataset.csv'):
    print('saving disclosed dataset to csv')
    disclosed_gender_df.to_csv(data_dir / 'disclosed_dataset.csv', index=False)
else:
    disclosed_gender_df = pd.read_csv(data_dir / 'disclosed_dataset.csv')

if not os.path.exists(data_dir / 'undisclosed_dataset.csv'):
    print('saving undisclosed dataset to csv')
    undisclosed_gender_df.to_csv(data_dir / 'undisclosed_dataset.csv', index=False)
else:
    undisclosed_gender_df = pd.read_csv(data_dir / 'undisclosed_dataset.csv')
undisclosed_gender_df = undisclosed_gender_df[['user_id', 'text', 'gender']]
# undisclosed_gender_df.replace(to_replace=[r"\\t|\\n|\\r", "\t|\n|\r"], value=["",""], regex=True, inplace=True)
undisclosed_gender_df.to_csv(data_dir / 'undisclosed_id_text_gender.csv', index=False, header=False)
mapping = {'female' : 1, 'male' : 0}
disclosed_dataset_df = disclosed_gender_df[['user_name', 'text', 'gender']]
disclosed_dataset_df.replace({'gender': mapping}, inplace=True)
disclosed_dataset_df.head()<jupyter_output><empty_output><jupyter_text>plot data distribution<jupyter_code>DF_shape = disclosed_dataset_df.loc[disclosed_dataset_df['gender'] == 1].shape
DM_shape = disclosed_dataset_df.loc[disclosed_dataset_df['gender'] == 0].shape
UNDIS_shape = undisclosed_gender_df.shape
total_size = disclosed_dataset_df.shape[0] + UNDIS_shape[0]
print(total_size, disclosed_dataset_df.shape[0], UNDIS_shape[0], DM_shape[0], DF_shape[0])
import matplotlib.pyplot as plt
labels = 'SM', 'SW', 'Performing'
sizes = [DM_shape[0]/total_size, DF_shape[0]/total_size, UNDIS_shape[0]/total_size]
explode = (0, 0, 0)  # only "explode" the 2nd slice (i.e. 'Hogs')

fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
        shadow=True, startangle=90)
ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

plt.show()<jupyter_output><empty_output><jupyter_text>Split disclosed dataset into train, test and validation train: text, gender test: name, text, gender validation: text, gender<jupyter_code>from sklearn.model_selection import train_test_split

# dataset_df = pd.read_csv('train_dataset.csv')
if 'disclosed_dataset_df' in locals():
    # 80/20 train/test
    train_df, test_df = train_test_split(disclosed_dataset_df, test_size=0.2)
    # 80/20 train/validation
    train_df, validation_df = train_test_split(train_df, test_size=0.2)
# extracting related attributes for training, validation and test
train_gender_text_df = train_df[['gender', 'text']]
validation_gender_text_df = validation_df[['gender', 'text']]
test_name_text_gender_df = test_df[['user_name', 'text', 'gender']]
# v2_test_df = pd.read_csv('v2_test_data.csv', engine='python',  encoding='utf-8', error_bad_lines=False, header=None)
# train_gender_text_df.replace(to_replace=[r"\\t|\\n|\\r", "\t|\n|\r"], value=["",""], regex=True, inplace=True)
train_gender_text_df.to_csv(data_dir / 'training_gender_text.csv', index=False, header=False)
# validation_gender_text_df.replace(to_replace=[r"\\t|\\n|\\r", "\t|\n|\r"], value=["",""], regex=True, inplace=True)
validation_gender_text_df.to_csv(data_dir / 'validation_gender_text.csv', index=False, header=False)
# test_name_text_gender_df.replace(to_replace=[r"\\t|\\n|\\r", "\t|\n|\r"], value=["",""], regex=True, inplace=True)
test_name_text_gender_df.to_csv(data_dir / 'test_name_text_gender.csv', index=False, header=False)<jupyter_output><empty_output><jupyter_text>Helper function removing escape chars and checking if successfully removed<jupyter_code># Legacy
def remove_(x):
    x = str(x).replace('\n', ' ')
    return x

v1_test_df['Review'] = v1_test_df[1].apply(lambda x:remove_(x))

def check(x):
    if '\n' in x:
        return True
    return False

v1_test_df['check'] = v1_test_df['Review'].apply(lambda x:check(x))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1397608370702541, 'fraction_numerical': 0.006726457399103139, 'mean_word_length': 5.7017250973845295, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '392277', 'license': 'MIT', 'n_tokens_mistral': 4240, 'n_tokens_neox': 4113, 'n_words': 802, 'path': 'YelpDataset.ipynb', 'repo_name': 'wiekern/GenderPerformance', 'size': '12042'}