<jupyter_start><jupyter_text>Pandas Data Grammar IllustrationThis notebook illustrates the concepts of grammar of data in pandas using the bank dataset.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
# Read the data file into a DataFrame
import os

PROJECT_NAME = "bank"

home_dir = os.path.expanduser("~")
course_id = 'cpsc6300'
project_dir = os.path.join(home_dir, course_id, PROJECT_NAME)
project_data_dir = os.path.join(project_dir, "data", "bank")
data_file_path = os.path.join(project_data_dir, "bank-additional-bank-additional-full.csv")
df = pd.read_csv(data_file_path, sep=";")<jupyter_output><empty_output><jupyter_text>Select a data slice<jupyter_code>### head() and tail()
df.head(5)
df.tail(5)<jupyter_output><empty_output><jupyter_text>bracket selection<jupyter_code>df[['age', 'job']]<jupyter_output><empty_output><jupyter_text>iloc<jupyter_code>### Read the iloc documentation
help(df.iloc)
df.iloc[[10, 12, 15], [0, 1]]<jupyter_output><empty_output><jupyter_text>loc<jupyter_code># df.loc
help(df.loc)
df.loc[[0, 13]]
df.loc[[0, 13], ["age", "job"]]
df.loc[df.education == 'basic.4y']<jupyter_output><empty_output><jupyter_text>Sort<jupyter_code>### sort_values
help(df.sort_values)
df.sort_values(by=["education", "age"])
### sort_values
help(df.sort_index)
df.sort_index(ascending=False)
df.loc[df.education == 'basic.4y']<jupyter_output><empty_output><jupyter_text>Unqiue values<jupyter_code>### unqiue
help(pd.Series.unique)
df["job"].unique()<jupyter_output><empty_output><jupyter_text>drop_duplicates<jupyter_code>help(pd.DataFrame.drop_duplicates)
df[["education", "job"]].drop_duplicates()<jupyter_output><empty_output><jupyter_text>Transform<jupyter_code>education_map = {
     'illiterate': 0,
     'basic.4y': 4,
     'basic.6y': 6,
     'basic.9y': 9,
     'high.school': 12,
     'professional.course': 12,
     'university.degree': 16,
     'unknown': np.NaN,
}
education_map
df['education_numeric'] = df["education"].map(education_map)
help(pd.Series.map)
df.head()<jupyter_output><empty_output><jupyter_text>Assignment<jupyter_code>df.loc[df.poutcome == 'nonexistent', "poutcome"] = np.nan<jupyter_output><empty_output><jupyter_text>Sample<jupyter_code>df.sample(n=5, random_state=1)<jupyter_output><empty_output><jupyter_text>Aggregated statistics<jupyter_code>df.age.max(), df.age.min()
df[["age", "nr.employed"]].describe()<jupyter_output><empty_output><jupyter_text>Delete Columns<jupyter_code>df.drop(columns=["contact"], inplace=True)<jupyter_output><empty_output><jupyter_text>Group By<jupyter_code>df.groupby("education").sum()
help(df.groupby)<jupyter_output>Help on method groupby in module pandas.core.frame:

groupby(by=None, axis=0, level=None, as_index: bool = True, sort: bool = True, group_keys: bool = True, squeeze: bool = False, observed: bool = False) -> 'groupby_generic.DataFrameGroupBy' method of pandas.core.frame.DataFrame instance
    Group DataFrame using a mapper or by a Series of columns.
    
    A groupby operation involves some combination of splitting the
    object, applying a function, and combining the results. This can be
    used to group large amounts of data and compute operations on these
    groups.
    
    Parameters
    ----------
    by : mapping, function, label, or list of labels
        Used to determine the groups for the groupby.
        If ``by`` is a function, it's called on each value of the object's
        index. If a dict or Series is passed, the Series or dict VALUES
        will be used to determine the groups (the Series' values are first
        aligned; see ``.align()`` method). If an ndarray [...]
{'fraction_non_alphanumeric': 0.13915416098226466, 'fraction_numerical': 0.010368349249658937, 'mean_word_length': 6.077220077220077, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '64265', 'license': 'MIT', 'n_tokens_mistral': 1360, 'n_tokens_neox': 1305, 'n_words': 275, 'path': 'numpy_pandas_matplotlib/data_grammar.ipynb', 'repo_name': 'cdhumme/notebooks', 'size': '3665'}