<jupyter_start><jupyter_text>Session 3<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Matplotlib (cont.)<jupyter_code>x = np.arange(0, 1000) # 100000
y = np.random.randint(0, 100, size=1000)
z = 10*np.random.randn(1000)+50
plt.style.use("ggplot") # seaborn
a = plt.hist(y, bins=20, alpha=0.5)
plt.hist(z, bins=20, alpha=0.5)
plt.show()
print("frequencies: {}".format(a[0]))
print("bins start point: {}".format(a[1]))

plt.bar(x, y)
plt.show()
y = y.tolist()
y.append(250)
plt.boxplot(y)
plt.show()<jupyter_output><empty_output><jupyter_text>detect anomalies<jupyter_code>max_x, min_x = 10, -10
mu = 0
sigma = 0.7

x = np.linspace(min_x, max_x, 1000)
y = stats.norm.pdf(x, loc=mu, scale=sigma)
y_c = stats.norm.cdf(x, loc=mu, scale=sigma)
plt.figure(figsize=(14, 8))
plt.plot(x, y)
plt.plot(x, y_c)

plt.legend(["pdf", "cdf"], loc=2)
plt.show()<jupyter_output><empty_output><jupyter_text>Pandas<jupyter_code>import pandas as pd
df = pd.read_csv("./data/titanic_train.csv")
df = df.set_index("PassengerId")
df.head()
df.loc[4] # index
df.iloc[3][2] # position
df.Age > 80
df[df.Age>=70]
df = pd.read_csv("./data/titanic_train.csv", index_col="PassengerId")
# df = df.reset_index()
df.head()
df.info()
df.isnull().sum()
df.isnull().sum().sum()
df.describe().transpose()
df.columns
df = df.drop(['Ticket'], axis=1)
age_mean = df['Age'].mean()
print(age_mean)
embark_mode = (df["Embarked"].mode().values[0])
print(embark_mode)
df['Age'] = df['Age'].fillna(age_mean)
df['Embarked'] = df['Embarked'].fillna(embark_mode)
df['Cabin'] = df['Cabin'].fillna("U")
print(type(df[["Age"]]))
print(type(df["Age"]))
pd.get_dummies(df["Sex"], prefix="sex")
TITLE_LIST = ['Mrs', 'Mr', 'Master', 'Miss', 'Major', 'Rev', 'Dr', 'Ms',
              'Mlle', 'Col', 'Capt', 'Mme', 'Countess', 'Don', 'Jonkheer']


def substring_exist(string, substrings):
    for substring in substrings:
        if str.find(string, substring) != -1:
            return substring
    print(string)
    return np.nan

def replace_titles(x):
    title = x['Title']
    if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col']:
        # return 'Mr'
        return 0
    elif title in ['Countess', 'Mme']:
        # return 'Mrs'
        return 0.75
    elif title in ['Mlle', 'Ms']:
        # return 'Miss'
        return 1
    elif title == 'Dr':
        if x['Sex'] == 'Male':
            # return 'Mr'
            return 0
        else:
            # return 'Mrs'
            return 0.75
    else:
        # return title
        return 0.5

df['Title'] = df['Name'].map(
    lambda x: substring_exist(x, TITLE_LIST))
CABIN_LIST = ['A', 'B', 'C', 'D', 'E', 'F', 'T', 'G', 'U']
def replace_deck(x):
    deck = x['Deck']
    if deck == "U":
        return 0
    elif deck == "G":
        return 0.15
    elif deck == "F":
        return 0.3
    elif deck == "E":
        return 0.45
    elif deck == "D":
        return 0.6
    elif deck == "C":
        return 0.75
    elif deck == "B":
        return 0.9
    elif deck == "A":
        return 1.1
    elif deck == "T":
        return 1.2
    else:
        return -1
def replace_embark(x):
    embarked = x['Embarked']
    if embarked == "S":
        return 0
    elif embarked == "Q":
        return 1
    else:
        return -1
df['Title'] = df.apply(replace_titles, axis=1)
df['Embarked'] = df.apply(replace_embark, axis=1)
df['Deck'] = df['Cabin'].map(lambda x: substring_exist(x, CABIN_LIST))
df['Deck'] = df.apply(replace_deck, axis=1)

df['Family_Size'] = df['SibSp']+df['Parch']
df['Fare_Per_Person'] = df['Fare']/(df['Family_Size']+1)
def replace_sex(x):
    sex = x['Sex']
    if sex == "male":
        return 0
    else:
        return 1
df['Sex'] = df.apply(replace_sex, axis=1)
df["Pclass_inv"] = df["Pclass"].map(lambda x: 1/x)
df["Age"].plot(kind="hist")

std_dev = np.std(df["Age"])
df["Age_norm"] = -(df.Age - age_mean)/std_dev

df.Age_norm.plot(kind="hist")

df.head()
df = df.drop("Name", axis=1)
df.tail()
df.sort_values(by="Age", ascending=False, na_position="first")
df.to_csv("./data/neat.csv")
# merge<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19995204986813714, 'fraction_numerical': 0.026372572524574443, 'mean_word_length': 3.462032085561497, 'pattern_counts': {'":': 12, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '204345', 'license': 'Apache-2.0', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1663, 'n_words': 350, 'path': 'TA/Session3.ipynb', 'repo_name': 'mmheydari97/datamining-spring', 'size': '4171'}