<jupyter_start><jupyter_text>Part III: Feature engineering and data preparation<jupyter_code>import pandas as pd
# read csv from Part II as DataFrame
df = pd.read_csv("../Dataset/thebiglist_clean_extended.csv")
df
# get value_counts of Brand
top_thirty_brands = df["Brand"].value_counts()[:30].index
top_thirty_brands
# create a new column called newBrand
new_brand = []

for brand in df["Brand"]:
    if brand in top_thirty_brands:
        new_brand.append(brand)
    else:
        new_brand.append("Other")

df["newBrand"] = new_brand
df
len(df["newBrand"].value_counts())
len(df["Brand"].value_counts())
# examine Style
top_four_style = df["Style"].value_counts().index[:4]

# loop through style and create newStyle
new_style = []

for style in df["Style"]:
    if style in top_four_style:
        new_style.append(style)
    else:
        new_style.append("Other")
        
df["newStyle"] = new_style
df
df["newStyle"].value_counts()
df["Country"].value_counts()
# the top few should account for about 80% of dataset
len(df) * 0.8
df["Country"].value_counts()[:10].sum() # top 10
10 / 51 * 100
# 80-20 rule
# top 20% accounts for 80%

# this case
# first 19% accounts for about 80% of all noodles
top_ten_countries = df["Country"].value_counts()[:10].index
top_ten_countries
new_country = []

for country in df["Country"]:
    if country in top_ten_countries:
        new_country.append(country)
    else:
        new_country.append("Other")
        
df["newCountry"] = new_country
df
df["newCountry"].value_counts()
brand_dummy = pd.get_dummies(df["newBrand"], drop_first=True,
                            prefix="from")
brand_dummy
style_dummy = pd.get_dummies(df["newStyle"], drop_first=True,
                            prefix="is")
style_dummy
country_dummy = pd.get_dummies(df["newCountry"], drop_first=True,
                            prefix="in")
country_dummy
# df_temp = df.drop(["Brand", "Variety", "Style", "Country",
#                   "newBrand", "newStyle", "newCountry"], axis=1)
# or
df_temp = df[["Stars", "isSpicy", "hasChicken", "hasBeef", "hasSeafood"]]

df_temp
df_final = pd.concat([df_temp, brand_dummy, style_dummy, country_dummy],
                    axis=1)

df_final
df_final.to_csv("../Dataset/thebiglist_clean_extended_final.csv", index=None)
pd.read_csv("../Dataset/thebiglist_clean_extended_final.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1379746835443038, 'fraction_numerical': 0.014345991561181435, 'mean_word_length': 3.991578947368421, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '23261', 'license': 'MIT', 'n_tokens_mistral': 842, 'n_tokens_neox': 800, 'n_words': 174, 'path': 'Notebooks/Project Ramen Rating Analysis Part III.ipynb', 'repo_name': 'Gamers-Blended/Ramen-Rating-Analysis', 'size': '2370'}