<jupyter_start><jupyter_text>Aggregating statistics<jupyter_code>import pandas as pd
air_quality = pd.read_pickle('air_quality.pkl')
air_quality.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 95685 entries, 0 to 95684
Data columns (total 27 columns):
 #   Column                 Non-Null Count  Dtype          
---  ------                 --------------  -----          
 0   date_time              95685 non-null  datetime64[ns] 
 1   PM2.5                  95685 non-null  float64        
 2   PM10                   95685 non-null  float64        
 3   SO2                    95685 non-null  float64        
 4   NO2                    95685 non-null  float64        
 5   CO                     95685 non-null  float64        
 6   O3                     95685 non-null  float64        
 7   TEMP                   95685 non-null  float64        
 8   PRES                   95685 non-null  float64        
 9   DEWP                   95685 non-null  float64        
 10  RAIN                   95685 non-null  float64        
 11  wd                     95685 non-null  object         
 12  WSPM                   95685 non-null  float64[...]<jupyter_text>Series/one column of a DataFrame<jupyter_code>air_quality['TEMP'].count()
air_quality['TEMP'].mean()
air_quality['TEMP'].std()
air_quality['TEMP'].min()
air_quality['TEMP'].max()
air_quality['TEMP'].quantile(0.25)
air_quality['TEMP'].median()
air_quality['TEMP'].describe()
air_quality['RAIN'].sum()
air_quality['PM2.5_category'].mode()
air_quality['PM2.5_category'].nunique()
air_quality['PM2.5_category'].describe()<jupyter_output><empty_output><jupyter_text>DataFrame by columns<jupyter_code>air_quality.count()
air_quality.mean()
air_quality.mean(numeric_only=True)
air_quality[['PM2.5', 'TEMP']].mean()
air_quality[['PM2.5', 'TEMP']].min()
air_quality[['PM2.5', 'TEMP']].max()
air_quality.describe().T
air_quality.describe(include=['object', 'category', 'bool'])
air_quality[['PM2.5_category', 'TEMP_category', 'hour']].mode()
air_quality['hour'].value_counts()
air_quality[['PM2.5', 'TEMP']].agg('mean')
air_quality[['PM2.5', 'TEMP']].mean()
air_quality[['PM2.5', 'TEMP']].agg(['min', 'max', 'mean'])
air_quality[['PM2.5', 'PM2.5_category']].agg(['min', 'max', 'mean', 'nunique'])
air_quality[['PM2.5', 'PM2.5_category']].agg({'PM2.5': 'mean', 'PM2.5_category': 'nunique'})
air_quality.agg({'PM2.5': ['min', 'max', 'mean'], 'PM2.5_category': 'nunique'})
def max_minus_min(s):
    return s.max() - s.min()
max_minus_min(air_quality['TEMP'])
air_quality[['PM2.5', 'TEMP']].agg(['min', 'max', max_minus_min])
41.6 - (-16.8)<jupyter_output><empty_output><jupyter_text>DataFrame by rows<jupyter_code>air_quality[['PM2.5', 'PM10']]
air_quality[['PM2.5', 'PM10']].min()
air_quality[['PM2.5', 'PM10']].min(axis=1)
air_quality[['PM2.5', 'PM10']].mean(axis=1)
air_quality[['PM2.5', 'PM10']].sum(axis=1)<jupyter_output><empty_output><jupyter_text>Grouping by<jupyter_code>air_quality.groupby(by='PM2.5_category')
air_quality.groupby(by='PM2.5_category').groups
air_quality['PM2.5_category'].head(20)
air_quality.groupby(by='PM2.5_category').groups.keys()
air_quality.groupby(by='PM2.5_category').get_group('Good')
air_quality.sort_values('date_time')
air_quality.sort_values('date_time').groupby(by='year').first()
air_quality.sort_values('date_time').groupby(by='year').last()
air_quality.groupby('TEMP_category').size()
air_quality['TEMP_category'].value_counts(sort=False)
air_quality.groupby('quarter').mean()
#air_quality[['PM2.5', 'TEMP']].groupby('quarter').mean() # KeyError: 'quarter'
air_quality[['PM2.5', 'TEMP', 'quarter']].groupby('quarter').mean()
air_quality.groupby('quarter')[['PM2.5', 'TEMP']].mean()
air_quality.groupby('quarter').mean()[['PM2.5', 'TEMP']]
air_quality.groupby('quarter')[['PM2.5', 'TEMP']].describe()
air_quality.groupby('quarter')[['PM2.5', 'TEMP']].agg(['min', 'max'])
air_quality.groupby('day_of_week_name')[['PM2.5', 'TEMP', 'RAIN']].agg({'PM2.5': ['min', 'max', 'mean'], 'TEMP': 'mean', 'RAIN': 'mean'})
air_quality.groupby(['quarter', 'TEMP_category'])[['PM2.5', 'TEMP']].mean()
air_quality.groupby(['TEMP_category', 'quarter'])[['PM2.5', 'TEMP']].mean()
air_quality.groupby(['year', 'quarter', 'month'])['TEMP'].agg(['min', 'max'])<jupyter_output><empty_output><jupyter_text>Pivoting tables<jupyter_code>import pandas as pd
student = pd.read_csv('student.csv')
student.info()
student
pd.pivot_table(student, 
               index='sex')
pd.pivot_table(student, 
               index=['sex', 'internet']
              )
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score')
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score', 
               aggfunc='mean')
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score', 
               aggfunc='median')
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score', 
               aggfunc=['min', 'mean', 'max'])
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score', 
               aggfunc='mean',
               columns='studytime'
              )
student[(student['sex']=='M') & (student['internet']=='no') & (student['studytime']=='4. >10 hours')]
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values='score', 
               aggfunc='mean',
               columns='studytime',  
               fill_value=-999)
pd.pivot_table(student, 
               index=['sex', 'internet'], 
               values=['score', 'age'],
               aggfunc='mean', 
               columns='studytime', 
               fill_value=-999)
pd.pivot_table(student, 
               index=['sex'], 
               values='score', 
               aggfunc='mean',                
               columns=['internet', 'studytime'],
               fill_value=-999)
pd.pivot_table(student, 
               index='familysize',
               values='score', 
               aggfunc='mean',
               columns='sex'
              )
pd.pivot_table(student, 
               index='familysize',
               values='score', 
               aggfunc='mean',
               columns='sex', 
               margins=True,
               margins_name='Average score total')
student[student['sex']=='F'].mean()
pd.pivot_table(student, 
               index='studytime', 
               values=['age', 'score'], 
               aggfunc={'age': ['min', 'max'],
                        'score': 'median'}, 
               columns='sex')
pd.pivot_table(student,  
               index='studytime', 
               values='score',
               aggfunc=lambda s: s.max() - s.min(),
               columns='sex'
               )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1858562244301578, 'fraction_numerical': 0.034921098772647576, 'mean_word_length': 2.6860527732902533, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '2498', 'license': 'MIT', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2267, 'n_words': 317, 'path': 'Exploring+data+(Exploratory+Data+Analysis)+(1).ipynb', 'repo_name': 'PacktPublishing/Python-for-Data-Analysis-step-by-step-with-projects-', 'size': '6844'}