<jupyter_start><jupyter_text>Part 2: Full Data Workflow A-Z  Data Preparation and Feature Creation  Arithmetic Operations<jupyter_code>import pandas as pd
import numpy as np
titanic = pd.read_csv("titanic.csv")
titanic.head()
titanic.info()
titanic.age.fillna(titanic.age.mean(), inplace = True)
titanic.head(10)<jupyter_output><empty_output><jupyter_text>Add/Sub/Mul/Div of Columns<jupyter_code>titanic.sibsp + titanic.parch
titanic.sibsp.add(titanic.parch)
titanic["no_relat"] = titanic.sibsp.add(titanic.parch)
titanic.head()
sales = pd.read_csv("sales.csv", index_col = 0)
sales
sales.Mon + sales.Thu
sales.Mon.add(sales.Thu, fill_value=0)
sales["perc_Bonus"] = [0.12, 0.15, 0.10, 0.20]
sales
sales.Thu * sales.perc_Bonus
sales.Thu.mul(sales.perc_Bonus, fill_value=0)
sales.iloc[:, :-1].sum(axis = 1).mul(sales.perc_Bonus)
sales["Bonus"] = sales.iloc[:, :-1].sum(axis = 1).mul(sales.perc_Bonus)
sales<jupyter_output><empty_output><jupyter_text>Add/Sub/Mul/Div with Scaler Value<jupyter_code>titanic.head()
1912 - titanic.age
titanic["YoB"] = titanic.age.sub(1912).mul(-1)
titanic.head()
fx_rate = 1.1
titanic["EUR_fare"] = titanic.fare.div(fx_rate)
titanic.head()
titanic.drop(columns = ["sibsp", "parch", "deck", "YoB", "EUR_fare"], inplace =True)
titanic.head()
sales
fixed_costs = 5
sales.iloc[:, :-2].sub(fixed_costs, fill_value = 0)
perc_Bonus = 0.1
sales.iloc[:, :-2].mul(perc_Bonus, fill_value = 0)
sales.iloc[:,:-2]
lot_size = 10
bonus_per_lot = 1.25
sales.iloc[:, :-2].floordiv(lot_size, fill_value = 0).mul(bonus_per_lot).sum(axis = 1)<jupyter_output><empty_output><jupyter_text>Transformation / Mapping<jupyter_code>summer = pd.read_csv("summer.csv")
summer.head()
sample = summer.sample(n = 7, random_state = 123).sort_values(by = "Year")
sample
city_country = {"Paris":"France", "Mexico":"Mexico", "Montreal":"Canada", "Moscow":"Russia", "Barcelona":"Spain", "Athens": "Greece"}
city_country
sample.City.map(city_country)
sample["Host_Country"] = sample.City.map(city_country)
sample
titanic.head()
mapper = {1:"First", 2:"Second", 3:"Third"}
titanic.pclass.map(mapper)
titanic.pclass = titanic.pclass.map(mapper)
titanic.head()<jupyter_output><empty_output><jupyter_text>Conditional Transformation<jupyter_code>titanic.head(10)
titanic.no_relat == 0
pd.Series(np.where(titanic.no_relat == 0, "Yes", "No"))
titanic["alone"] = pd.Series(np.where(titanic.no_relat == 0, "Yes", "No"))
titanic.head(10)
titanic["child"] = pd.Series(np.where(titanic.age < 18, "Yes", "No"))
titanic.head(10)<jupyter_output><empty_output><jupyter_text>Discretization and Binning with pd.cut() (Part 1)<jupyter_code>titanic.head(10)
age_bins = [0, 10, 18, 30, 55, 100]
cats = pd.cut(titanic.age, age_bins, right = False)
cats
cats.value_counts()
titanic["age_cat"] = cats
titanic.head()
titanic.groupby("age_cat").survived.mean()
group_names = ["child", "teenager", "young_adult", "adult", "elderly"]
pd.cut(titanic.age, age_bins, right = False, labels = group_names)
titanic["age_cat"] = pd.cut(titanic.age, age_bins, right = False, labels = group_names)
titanic.head(10)
titanic.age_cat<jupyter_output><empty_output><jupyter_text>Discretization and Binning with pd.cut() (Part 2)<jupyter_code>titanic.fare
pd.cut(titanic.fare, 5, precision= 3)
titanic["fare_cat"] = pd.cut(titanic.fare, 5, precision= 0)
titanic.head(10)
titanic.fare_cat.value_counts()<jupyter_output><empty_output><jupyter_text>Discretization and Binning with pd.qcut()<jupyter_code>titanic.head()
pd.qcut(titanic.fare, 5) 
titanic["fare_cat"] = pd.qcut(titanic.fare, 5) 
titanic.head()
titanic.fare_cat.value_counts()
pd.qcut(titanic.fare, [0, 0.1, 0.25, 0.5, 0.9, 1], precision = 0) 
fare_labels =["very_cheap", "cheap", "moderate", "exp", "very_exp"]
titanic["fare_cat"] =  pd.qcut(titanic.fare, [0, 0.1, 0.25, 0.5, 0.9, 1], precision = 0, labels = fare_labels) 
titanic.head()
titanic.fare_cat.value_counts()
titanic.groupby(["age_cat", "fare_cat"]).survived.mean().unstack()<jupyter_output><empty_output><jupyter_text>Caps and Floors<jupyter_code>titanic.head()
import matplotlib.pyplot as plt
titanic.fare.plot(figsize = (12,8))
plt.show()
titanic.fare.describe()
titanic.fare.sort_values(ascending = False)
fare_cap = 250
titanic.loc[titanic.fare > fare_cap, "fare"] = fare_cap
fare_floor = 5
titanic.loc[titanic.fare < fare_floor, "fare"] = fare_floor
titanic.head()<jupyter_output><empty_output><jupyter_text>Scaling / Standardization<jupyter_code>titanic.head()
titanic.describe()
import matplotlib.pyplot as plt
titanic.fare.plot(figsize = (12,8))
titanic.age.plot(figsize = (12,8))
plt.show()
mean_age = titanic.age.mean()
mean_fare = titanic.fare.mean()
std_age = titanic.age.std()
std_fare = titanic.fare.std()
titanic["age_z"] = round((titanic.age-mean_age) / std_age,2)
titanic["fare_z"] = round((titanic.fare-mean_fare) / std_fare,2)
titanic.head(10)
round(titanic.describe(),2)
titanic.fare_z.plot(figsize = (12,8))
titanic.age_z.plot(figsize = (12,8))
plt.show()
#titanic.to_csv("titanic_prep.csv", index = False)
titanic.head()
titanic.drop(labels = ["age", "alone", "child", "age_z", "fare_z", "fare_cat"], axis = 1, inplace = False)<jupyter_output><empty_output><jupyter_text>Creating Dummy Variables<jupyter_code>titanic.head()
titanic.drop(labels = ["age", "alone", "child", "age_z", "fare_z", "fare_cat"], axis = 1, inplace = True)
titanic.head()
titanic_d = pd.get_dummies(titanic, columns = ["sex", "pclass", "embarked", "age_cat"], drop_first=True)
titanic_d.head()
titanic_d.info()<jupyter_output><empty_output><jupyter_text>String Operations<jupyter_code>import pandas as pd
summer = pd.read_csv("summer.csv")
summer.head()
summer.Athlete = summer.Athlete.str.title()
summer.Athlete.str.split(", ", n = 1, expand = True)
summer[["Surname", "First_Name"]] = summer.Athlete.str.split(", ", n = 1, expand = True)
summer.head()
summer["Surname"] = summer.Surname.str.strip()
summer["First_Name"] = summer.First_Name.str.strip()
summer.drop(columns = "Athlete")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18013045659809332, 'fraction_numerical': 0.024251547081451746, 'mean_word_length': 7.203017832647462, 'pattern_counts': {'":': 6, '<': 51, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '153385', 'license': 'CC0-1.0', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2497, 'n_words': 386, 'path': '04 - Data Analysis With Pandas/notebooks/12_DataPreparationFeatureCreation.ipynb', 'repo_name': 'hossainlab/PY4R', 'size': '5979'}