<jupyter_start><jupyter_text>SeriesImagine you have a list my_high_scores = [3, 24, 5]<jupyter_code>my_high_scores = [3, 24, 5]
my_high_scores
import pandas as pd

s = pd.Series(my_high_scores)
s<jupyter_output><empty_output><jupyter_text>We get a type and an index<jupyter_code>s?
# Accessing elements
# s[0], s[1:]
s[1:]
s.name = "High Scores"
s<jupyter_output><empty_output><jupyter_text>The index<jupyter_code>levels = [
    "Underground",
    "Athletic",
    "Castle"
]
levels
s.index = levels
s
s[0]
s.iloc[1]
s
high_scores = {
    "Underground": 3,
    "Athletic": 24,
    "Castle": 5
}
pd.Series(high_scores)<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code>s.sum(), s.mean(), s.std()
s.min(), s.argmin(), s.idxmin()<jupyter_output><empty_output><jupyter_text>Mathematical operations<jupyter_code>s2 = s - 1
s2
s3 = s / s2
s3
# what if index does't match?

s / pd.Series([3], index=["Underground"])<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>s.plot(kind="bar");
# try bar plot<jupyter_output><empty_output><jupyter_text>DataFrame<jupyter_code>high_scores = {
    "Me": my_high_scores,
    "James": [3, 52, 63]
}
df = pd.DataFrame(high_scores, index=levels)
df
pd.DataFrame?
df?
# numpy under the hood<jupyter_output><empty_output><jupyter_text>info, describe<jupyter_code>df.tail()
df.max()
# returns a Series!
df.cumsum()
# Are we plotting super mario coins?
#df.plot(kind="bar");
df.cumsum().plot();
# Try plotting cumsum!<jupyter_output><empty_output><jupyter_text>Copy high scores from [speedrun.com](https://www.speedrun.com/sm64120_Star)<jupyter_code>df = pd.read_excel("~/Desktop/supermario64_120stars.xlsx")
df.head()
df = pd.read_excel("~/Desktop/supermario64_120stars.xlsx", usecols="A:F", index_col=0)
df.head()<jupyter_output><empty_output><jupyter_text>Getting familiar with our df:<jupyter_code># df.tail()
# df.shape
# df.info()
# df.describe()
df.shape<jupyter_output><empty_output><jupyter_text>Boolean indexing<jupyter_code>#df.Verified
df[["Verified"]]
df.Verified != "Yes"
df[df.Verified != "Yes"]<jupyter_output><empty_output><jupyter_text>[pd.DataFrame.apply](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.apply.html)`func = lambda col: col*2``df.apply(func, axis=1)`<jupyter_code># look at the Date column
df["Date"].head()
# https://stackoverflow.com/questions/28268818/how-to-find-the-date-n-days-ago-in-python

import datetime
from dateutil.relativedelta import relativedelta

def get_past_date(str_days_ago):
    TODAY = datetime.date.today()
    splitted = str_days_ago.split()
    if len(splitted) == 1 and splitted[0].lower() == 'today':
        return str(TODAY.isoformat())
    elif len(splitted) == 1 and splitted[0].lower() == 'yesterday':
        date = TODAY - relativedelta(days=1)
        return str(date.isoformat())
    elif splitted[1].lower() in ['hour', 'hours', 'hr', 'hrs', 'h']:
        date = datetime.datetime.now() - relativedelta(hours=int(splitted[0]))
        return str(date.date().isoformat())
    elif splitted[1].lower() in ['day', 'days', 'd']:
        date = TODAY - relativedelta(days=int(splitted[0]))
        return str(date.isoformat())
    elif splitted[1].lower() in ['wk', 'wks', 'week', 'weeks', 'w']:
        date = TODAY - relativedelta(weeks=int(splitted[0]))
        return str(date.isoformat())
    elif splitted[1].lower() in ['mon', 'mons', 'month', 'months', 'm']:
        date = TODAY - relativedelta(months=int(splitted[0]))
        return str(date.isoformat())
    elif splitted[1].lower() in ['yrs', 'yr', 'years', 'year', 'y']:
        date = TODAY - relativedelta(years=int(splitted[0]))
        return str(date.isoformat())
    else:
        return "Wrong Argument format"

# apply get_past_date to date
#dates = df.Date.apply(get_past_date)
df.dtypes<jupyter_output><empty_output><jupyter_text>- pd.DataFrame.isna()- pd.DataFrame.dropna()<jupyter_code>df.Date.isna().any()
#df.Date.isna().any()
df.loc[df.Date.isna()]<jupyter_output><empty_output><jupyter_text>Ignore NaN values<jupyter_code>dates_notna = df.Date.dropna().apply(get_past_date)
dates_notna.head(10)
df.head(10)
df["date"] = dates_notna<jupyter_output><empty_output><jupyter_text>The index is not unique!<jupyter_code>df.head(10)
#df.reset_index(inplace=True)
df = df.reset_index()
#df.head()
# mention drop_duplicates
#dates = df.Date[df.Date.notna()].apply(get_past_date)
dates_notna = df.Date.dropna().apply(get_past_date)
dates_notna.head(8)
df["date"] = dates_notna
df.head(8)<jupyter_output><empty_output><jupyter_text>dtypes<jupyter_code>df.plot(x="date", y="Position");
df.date[0]
df.dtypes<jupyter_output><empty_output><jupyter_text>Dtype Conversion- pd.to_datetime- pd.to_timestamp- pd.to_numeric- pd.to_timedelta- pd.DataFrame.astype<jupyter_code>pd.to_datetime(df["date"])
df["date"] = pd.to_datetime(df["date"])
df.dtypes
df.plot(x="Position", y="date");
df.plot.scatter(x="Position", y="date");
df.plot.scatter(x="Position", y="date", logx=True);
# ignore?
df["Platform"].unique()<jupyter_output><empty_output><jupyter_text>- df.value.str<jupyter_code>df.Time[df.Time.str.contains("13")]
time = df.Time.str.extract(r'(?P<hours>\d+)h (?P<minutes>\d+)m (?P<seconds>\d+)s')

time["hours"] = pd.to_numeric(time["hours"])
time["minutes"] = pd.to_numeric(time["minutes"])
time["seconds"] = pd.to_numeric(time["seconds"])
time.head()

seconds = time.hours*3600 + time.minutes*60 + time.seconds
df["seconds"] = seconds
df.head()
# we could also convert seconds to timedelta

pd.to_timedelta(df["seconds"], unit="s")<jupyter_output><empty_output><jupyter_text>Column slicing<jupyter_code>df.loc[:, ["Position", "Player", "Verified", "date", "seconds"]]
df = df.loc[:, ["Position", "Player", "Verified", "date", "seconds"]]
df<jupyter_output><empty_output><jupyter_text>groupby<jupyter_code>df["playerlen"] = df["Player"].str.len()
df
groups = df.groupby("playerlen")
groups
playerlen, group = groups.__iter__().__next__()
group
for (playerlen, df_group) in groups:
    print(f"{len(df_group)} players have {playerlen} letters")
groups.mean()<jupyter_output><empty_output><jupyter_text>- value_counts- sort_index, sort_values<jupyter_code>df.groupby("playerlen").mean().plot(y="seconds");
# sort_index, sort_value
df.value_counts("playerlen").sort_index().plot(kind="pie");<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1811422243315931, 'fraction_numerical': 0.013763644992880873, 'mean_word_length': 6.437647058823529, 'pattern_counts': {'":': 5, '<': 88, '<?xml version=': 0, '>': 88, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '221803', 'license': 'MIT', 'n_tokens_mistral': 2583, 'n_tokens_neox': 2466, 'n_words': 372, 'path': 'Gentle introduction.ipynb', 'repo_name': 'SebastianRoll/slides_pandas', 'size': '6321'}