<jupyter_start><jupyter_text>1. Pandas Foundations<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
austinAirport = pd.read_csv('data/austin_airport_departure_data_2015_july.csv', skiprows=15)
austinAirport.drop('Unnamed: 17', axis=1, inplace=True)
austinAirport.info()
austinAirport.shape
austinAirport.head()
austinAirport = pd.read_csv('data/austin_airport_departure_data_2015_july.csv', skiprows=15, parse_dates=[1])
austinAirport.drop('Unnamed: 17', axis=1, inplace=True)
austinAirport.head()
autoMpg = pd.read_csv('data/auto-mpg.csv')
autoMpg.info()
autoMpg.shape
autoMpg.head()
lifeExpectancy = pd.read_csv('data/life_expectancy_at_birth.csv').iloc[:, 1:]
lifeExpectancy.info()
lifeExpectancy.shape
lifeExpectancy.head()
messy = pd.read_csv('data/messy_stock_data.tsv', sep='\s', skiprows=[0,1,2,4,7,10])
messy.info()
messy.shape
messy
noaa = pd.read_csv('data/NOAA_QCLCD_2011_hourly_13904.txt', sep=',', header=None)
noaa.info()
noaa.shape
noaa.head()
percentBachelors = pd.read_csv('data/percent-bachelors-degrees-women-usa.csv')
percentBachelors.info()
percentBachelors.shape
percentBachelors.head()
tips = pd.read_csv('data/tips.csv')
tips.info()
tips.shape
tips.head()
titanic = pd.read_csv('data/titanic.csv')
titanic.info()
titanic.shape
titanic.head()
weather = pd.read_csv('data/weather_data_austin_2010.csv')
weather.info()
weather.shape
weather.head()
worldInd = pd.read_csv('data/world_ind_pop_data.csv')
worldInd.info()
worldInd.shape
worldInd.head()
worldPop = pd.read_csv('data/world_population.csv', parse_dates=True, index_col='Year')
worldPop.info()
worldPop.shape
worldPop.head()<jupyter_output><empty_output><jupyter_text>1.1. Building dataframes from scratch<jupyter_code>data = {
    'weekdays': ['Sun', 'Sun', 'Mon', 'Mon'],
    'cities': ['Austin', 'Dallas', 'Austin', 'Dallas'],
    'visitors': [139, 237, 326, 456],
    'signups': [7, 12, 3, 5]
}

users = pd.DataFrame(data)
users
weekdays = ['Sun', 'Sun', 'Mon', 'Mon']
cities = ['Austin', 'Dallas', 'Austin', 'Dallas']
visitors = [139, 237, 326, 456]
signups = [7, 12, 3, 5]

list_labels = ['weekdays','cities','visitors','signups']
list_cols = [weekdays,cities,visitors,signups]
zipped = dict(list(zip(list_labels, list_cols)))
zipped
users = pd.DataFrame(zipped)
users
users['fees'] = 0
users
heights = [59.0,65.2,62.9,65.4,63.7,65.7,64.1]
data = {'heights':heights, 'sex':'M'}
results = pd.DataFrame(data)
results
ord('A')

[chr(i) for i in range(65, 65+7)]
results.columns = ['height (in)', 'sex']
results.index = ['A', 'B', 'C', 'D', 'E', 'F', 'G']
results<jupyter_output><empty_output><jupyter_text>1.2. Importing & exporting data  1.3. Plotting with pandas<jupyter_code>austinAirport = pd.read_csv('data/austin_airport_departure_data_2015_july.csv', 
                            skiprows=15, parse_dates=[1], 
                            index_col='Date (MM/DD/YYYY)')

austinAirport.drop('Unnamed: 17', axis=1, inplace=True)
austinAirport.head()
plt.plot(austinAirport['Actual Elapsed Time(Minutes)'])
plt.show()
austinAirport['Actual Elapsed Time(Minutes)'].plot()
plt.show()
austinAirport['Actual Elapsed Time(Minutes)'].plot()
plt.yscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>2. Visual exploratory data analysis<jupyter_code>austinAirport.plot(x='Scheduled Elapsed Time(Minutes)', y='Actual Elapsed Time(Minutes)')
plt.show()
austinAirport.plot(x='Scheduled Elapsed Time(Minutes)', y='Actual Elapsed Time(Minutes)', kind='scatter')
plt.show()
austinAirport.plot(y='Actual Elapsed Time(Minutes)', kind='box')
plt.show()
austinAirport.plot(y='Actual Elapsed Time(Minutes)', kind='hist')
plt.show()
austinAirport.plot(y='Actual Elapsed Time(Minutes)', kind='hist', bins=30, range=(0, 400), density=True)
plt.show()
austinAirport.plot(y='Actual Elapsed Time(Minutes)', kind='hist', bins=30, range=(0, 400), density=True)
plt.show()
austinAirport.plot(y='Actual Elapsed Time(Minutes)', kind='hist', bins=30, range=(0, 400), 
                   cumulative=True, density=True)
plt.show()<jupyter_output><empty_output><jupyter_text>2.1. Statistical exploratory data analysis<jupyter_code>austinAirport.describe()
austinAirport.mean()
austinAirport.std()
austinAirport.median()
q = [0.25, 0.75]
austinAirport.quantile(q)
austinAirport.plot(kind='box')
plt.show()<jupyter_output><empty_output><jupyter_text>2.2. Separating populations<jupyter_code>austinAirport['Destination Airport '].describe()
austinAirport['Destination Airport '].unique()<jupyter_output><empty_output><jupyter_text>3. Indexing time series<jupyter_code>austinAirport.head()
austinAirport.loc['2015-07-01'][:2]
austinAirport.loc['2015'][-2:]
austinAirport.loc['2015-07-01':'2015-07-03'][:2]
evening_2_11 = pd.to_datetime(['2015-2-11 20:00', '2015-2-11 21:00', '2015-2-11 22:00', '2015-2-11 23:00'])
evening_2_11<jupyter_output><empty_output><jupyter_text>3.1. Resampling time series data<jupyter_code>daily_mean = austinAirport.resample('D').mean()
daily_mean.tail()
daily_mean.loc['2015-07-31', 'Flight Number']
austinAirport.loc['2015-07-31', 'Flight Number'].mean()
austinAirport.resample('D').sum().tail()
austinAirport.resample('W').count()
austinAirport.resample('2W').count()
one_day = austinAirport.loc['2015-07-31', 'Flight Number']
one_day.tail()
'''ValueError: cannot reindex a non-unique index with a method or limit'''
# one_day.resample('4H').ffill()<jupyter_output><empty_output><jupyter_text>3.2. Manipulating time series data<jupyter_code>austinAirport['Tail Number'].str.upper().head()
austinAirport['Tail Number'].str.contains('607').sum()
austinAirport[austinAirport['Tail Number'].str.contains('607')==True]
austinAirport.index.to_series().dt.hour.head()
austinAirport.index.to_series().dt.tz_localize('US/Central').head()
austinAirport.index.to_series().dt.tz_localize('US/Central').dt.tz_convert('US/Eastern').head()
worldPop.resample('A').first()
worldPop.resample('A').first().interpolate('linear')<jupyter_output><empty_output><jupyter_text>3.3. Time series visualization<jupyter_code>austinAirport.loc['2015-07-01':'2015-07-02', 'Actual Elapsed Time(Minutes)'].plot(style='k.-')
plt.show()
austinAirport['Actual Elapsed Time(Minutes)'].plot(kind='area')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18066280566280565, 'fraction_numerical': 0.045527670527670525, 'mean_word_length': 7.5515818431911965, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '514738', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 2667, 'n_tokens_neox': 2441, 'n_words': 355, 'path': 'pandas/pandas-foundations/pandas-foundations.ipynb', 'repo_name': 'shawnngtq/data-science', 'size': '6216'}