<jupyter_start><jupyter_text>Aggregation and Grouping<jupyter_code>planets = sns.load_dataset('planets') #Seaborn Planets dataset
planets.shape
planets.head()
planets.isnull().any()
planets.dropna().describe()
planets.groupby('method')['orbital_period'].median()
planets.groupby('method')['year'].describe()
decade = 10 * (planets['year'] // 10)
decade = decade.astype(str) + 's'
decade.name = 'decade'
planets.groupby(['method', decade])['number'].sum().unstack().fillna(0)<jupyter_output><empty_output><jupyter_text>Pivot Tables<jupyter_code>pd.DataFrame.pivot_table?
titanic = sns.load_dataset('titanic')
titanic.head()
titanic.pivot_table('survived', index='sex', columns='class')
age = pd.cut(titanic['age'], [0, 18, 80]) #Create bins
fare = pd.qcut(titanic['fare'],2)       #quantiles
#titanic.pivot_table('survived', ['sex', age], 'class')
titanic.pivot_table('survived', ['sex', age], [fare, 'class'])<jupyter_output><empty_output><jupyter_text>Example USA Births<jupyter_code>#Download the data
!curl -O https://raw.githubusercontent.com/jakevdp/data-CDCbirths/master/births.csv
births = pd.read_csv('births.csv')
births.head()
births['decade'] = 10 * (births['year'] // 10)
births.pivot_table('births', index='decade', columns='gender', aggfunc='sum')
%matplotlib inline
import matplotlib.pyplot as plt
import matplotlib as mpl
sns.set() #Seaborn styles
births.pivot_table('births', index='year', columns='gender', aggfunc='sum').plot()
plt.ylabel('total births per year')
quartiles = np.percentile(births['births'], [25,50,75])
mu = quartiles[1]
sig = 0.74 * (quartiles[2] - quartiles[0])
#Extract outliers based on Gaussian distribution interquartile range
births = births.query('(births > @mu - 5 * @sig) & (births < @mu + 5 * @sig)')
#Some days are null
births['day'] = births['day'].astype(int)
#Create a datetime index
births.index = pd.to_datetime(10000 * births.year +
                              100 * births.month +
                              births.day, format='%Y%m%d')
births['dayofweek'] = births.index.dayofweek
births.pivot_table('births', index='dayofweek',
                    columns='decade', aggfunc='mean').plot()
plt.gca().set_xticklabels(['Mon', 'Tues', 'Wed', 'Thurs', 'Fri', 'Sat', 'Sun'])
plt.ylabel('mean births by day');
births_by_date = births.pivot_table('births', [births.index.month, births.index.day])
births_by_date.head()
#The year 2012 is just a dummy
births_by_date.index = [pd.datetime(2012, month, day) 
                        for (month, day) in births_by_date.index]
fig, ax = plt.subplots(figsize=(12,4))
births_by_date.plot(ax=ax);<jupyter_output><empty_output><jupyter_text>Vectorized String Operations<jupyter_code>monte = pd.Series(['<NAME>', '<NAME>', '<NAME>',
'<NAME>', '<NAME>', '<NAME>'])
monte.str.lower()
monte.str.len()
monte.str.startswith('T')
monte.str.split().str.get(-1)
monte.str[0:3]
full_monte = pd.DataFrame({'name': monte,
                           'info': ['B|C|D', 'B|D', 'A|C', 'B|D', 'B|C',
                           'B|C|D']})
full_monte
full_monte['info'].str.get_dummies('|')<jupyter_output><empty_output><jupyter_text>Working with Time Series<jupyter_code>import numpy as np
date = np.array('2015-07-04', dtype=np.datetime64)
date
date + np.arange(12)
import pandas as pd
date = pd.to_datetime('4th of July, 2015')
print(date)
date.strftime('%A')
from datetime import datetime
dates = pd.to_datetime([datetime(2015, 7, 3), '4th of July, 2015',
'2015-Jul-6', '07-07-2015', '20150708'])
dates
dates.to_period('D')
dates - dates[0]<jupyter_output><empty_output><jupyter_text>Example: Visualizing Seattle Bicycle Counts<jupyter_code>!curl -o FremontBridge.csv https://data.seattle.gov/api/views/65db-xm6k/rows.csv?accessType=DOWNLOAD
data = pd.read_csv('FremontBridge.csv', index_col = 'Date', parse_dates = True)
data.head()
data.columns = ['West', 'East']
data['Total'] = data.eval('West + East')
data.dropna().describe()
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn; seaborn.set()
data.plot()
plt.ylabel('Hourly Bicycle Count')
#resample the data by week
weekly = data.resample('W').sum()
weekly.plot(style=[':','--','-'])
plt.ylabel('Weekly bicycle count')
#organize data using rolling mean for 30-day rolling 
daily = data.resample('D').sum()
daily.rolling(30, center=True).sum().plot(style=[':','--','-'])
plt.ylabel('mean hourly count')
#Use a Gaussian window to smooth the result
daily.rolling(50, center=True, win_type='gaussian').sum(std=10).plot(style=[':','--','-'])
#Average traffic as a function of the time of day

by_time = data.groupby(data.index.time).mean()
hourly_ticks = 4 * 60 * 60 * np.arange(6)
by_time.plot(xticks=hourly_ticks, style=[':','--','-'])
# How things change based on the day of the week
by_weekday = data.groupby(data.index.dayofweek).mean()
by_weekday.index = ['Mon', 'Tues', 'Wed', 'Thurs', 'Fri', 'Sat', 'Sun']
by_weekday.plot(style=[':','--','-'])
#Analyze week against weekend hourly traffic
weekend = np.where(data.index.weekday < 5, 'Weekday', 'Weekend')
by_time = data.groupby([weekend, data.index.time]).mean()
fig, ax = plt.subplots(1,2,figsize=(14,5))
by_time.loc['Weekday'].plot(ax=ax[0], title='Weekdays', xticks=hourly_ticks,
                            style=[':','--','-'])
by_time.loc['Weekend'].plot(ax=ax[1], title='Weekends', xticks=hourly_ticks,
                            style=[':','--','-'])<jupyter_output><empty_output><jupyter_text>High-Performance Pandas<jupyter_code>#The instruction eval lets you perform memory efficient operations
# pd.eval() supports boolean operations but df.eval() doesn't, 
# use df.query instead
rng = np.random.RandomState(42)
df = pd.DataFrame(rng.rand(1000,3), columns=['A','B','C'])
df.head()
#The same operation
%timeit result1 = (df['A']+df['B']/(df['C'] - 1))
%timeit result2 = pd.eval('(df.A + df.B)/(df.C - 1)')
%timeit result3 = df.eval('(A + B) / (C - 1)')
#And can do assignment at the same time
df.eval('D = (A - B) / C', inplace=True)
df.head()
column_mean = df.mean(1)
result1 = df['A'] + column_mean
result2 = df.eval('A+@column_mean') #references local variables
np.allclose(result1, result2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19286528750610848, 'fraction_numerical': 0.023456589021013195, 'mean_word_length': 5.316872427983539, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '281762', 'license': 'MIT', 'n_tokens_mistral': 2432, 'n_tokens_neox': 2263, 'n_words': 446, 'path': 'Python Data Science Handbook/5. PANDAS ADVANCED 2.ipynb', 'repo_name': 'Hector-Restrepo/AI-ML', 'size': '6174'}