<jupyter_start><jupyter_text>Python Data Analyis  Import all necessary modules<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Series<jupyter_code>np.random.randn(5)
labels = ['a', 'b', 'c', 'd', 'e']
s = pd.Series(np.random.randn(5), index=labels)
'b' in s
s['b']
s.to_dict()
s = pd.Series(s.to_dict(), index=['b', 'e', 'a', 'd', 'f'])
s
s.dropna()
s * 2
s[3:]
s[:3]
s.index<jupyter_output><empty_output><jupyter_text>DataFrame: 2D collection of Series<jupyter_code>df = pd.DataFrame({'a': np.random.randn(6),
                'b': ['foo', 'bar'] * 3,
                'c': np.random.randn(6)})
df['d'] = range(6)
df
df.info()
df.dtypes
df['b']    # by column
df[:3]
df[:-2]
df[-2:]   # the last 2 rows
df[['a','b']]
df.loc[2]    # does equal thing as above. In this case, 3rd row. 
df.loc[2, 'b']
df.loc[2:4, 'b']
df.loc[2:4, ['b', 'c']]
df.loc[2:4, 'b':'c']   # slices by column - essentially the same as above.
df.loc[[0, 2, 4], ['b', 'c', 'd']]   # pass a list of rows and columns I want to select out
df['c'] > 0
df.loc[df['c'] > 0]   # boolean arrays
df.index
df.columns
df.T
df.T.T
df['a'].dot(df['a'])<jupyter_output><empty_output><jupyter_text>Hierarchical indexing<jupyter_code>index = pd.MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
                                   ['one', 'two', 'three']],
                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]])
hdf = pd.DataFrame(np.random.randn(10, 3), index=index,
                columns=['A', 'B', 'C'])
hdf
hdf.loc['foo']
hdf.loc['bar','one']['A']<jupyter_output><empty_output><jupyter_text>Working with real dataset<jupyter_code>data = pd.read_csv("store_sales.txt")
data
data.info()
data.describe()
data['Sales'].sum()
data[(data['Province'] == 'Yukon')]
data[['Province', 'Product Category']]
data[['Province', 'Product Category']].drop_duplicates()
data.groupby(['Province', 'Product Category'])
data.groupby(['Province', 'Product Category']).groups
data.groupby(['Province', 'Product Category']).groups.keys()
len(data.groupby(['Province', 'Product Category']).groups[('Alberta', 'Furniture')])
data.groupby(['Province', 'Product Category']).agg({'Sales':'sum'})
data.groupby(['Province', 'Product Category']).agg({'Sales':['sum', 'mean']})
data.groupby(['Province', 'Product Category']).agg({'Sales':['sum', 'mean', lambda x: x.max()]})
def get_range(x):
    return x.max() - x.min()
data.groupby(['Province', 'Product Category']).agg({'Sales':['sum', 'mean', get_range]})
pi_data = data.pivot_table(
    index=['Province','Customer Segment'], 
    columns=['Product Category'], 
    values=['Sales'], 
    aggfunc=[np.sum, np.mean])
pi_data
pi_data.to_csv("/Users/ramanathanhari/pi_data.csv")<jupyter_output><empty_output><jupyter_text>Time Series analysis<jupyter_code>sd = pd.read_csv("stock_data.csv")
sd
sd.info()
sd.describe()
sd.cov()
sd.corr()
plt.matshow(sd.corr())
import seaborn as sns

f, ax = plt.subplots(figsize=(10, 8))
corr = sd.corr()
sns.heatmap(corr, mask=np.zeros_like(corr, dtype=np.bool), cmap=sns.diverging_palette(220, 10, as_cmap=True),
            square=True, ax=ax)
sd['AAPL'].rolling(5).mean()
sd['AAPL'].rolling(5).apply(get_range)
sd_apple = sd[['AAPL']].copy()
sd_apple
sd_apple['returns'] = np.log(sd_apple['AAPL'] / sd_apple['AAPL'].shift(1))
sd_apple
sd_apple['SMA_1'] = sd_apple['AAPL'].rolling(60).mean()
sd_apple
sd_apple['SMA_2'] = sd_apple['AAPL'].rolling(10).mean()
sd_apple
sd_apple[['AAPL','SMA_1', 'SMA_2']].plot(figsize=(10,6))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22283205268935236, 'fraction_numerical': 0.020032930845225026, 'mean_word_length': 4.975409836065574, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '141624', 'license': 'MIT', 'n_tokens_mistral': 1531, 'n_tokens_neox': 1456, 'n_words': 279, 'path': '2016/Python 101 Workshop- NMIMS/NMIMS_Workshop.ipynb', 'repo_name': 'GoelJatin/HydPy-meetups', 'size': '3644'}