<jupyter_start><jupyter_text>***Capítulo 8 - Python para análise de dados***<jupyter_code>#!pip install pandas==1.3.2
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Indexação hierárquica<jupyter_code>data = pd.Series(np.random.randn(9), index=[['a','a','a','b','b','c','c','d','d'],[1,2,3,1,3,1,2,2,3]])
data
data.index
data['b']
data['b':'c']
data.loc[['b','d']]
data.loc[:,2]
data.unstack()
data.unstack().stack()
frame = pd.DataFrame(np.arange(12).reshape((4,3)), index=[['a','a','b','b'], [1,2,1,2]],columns=[['Ohio','Ohio','Colorado'],['Green','Red','Green']])
frame
frame.index.names = ['key1', 'key2']
frame.columns.names = ['state','color']
frame
frame['Ohio']<jupyter_output><empty_output><jupyter_text>outra forma de criar as mesmas colunas:*MultiIndex.from_arrays([['Ohio', 'Ohio', 'Colorado'], ['Green','Red','Green']], names=['state','color'])* **Reorganizando e ordenando níveis**<jupyter_code>#swaplevel troca a ordem do conteudo passado por argumento
frame.swaplevel('key1','key2')
#Ordena os index
frame.sort_index(level=1)
frame.swaplevel (0,1).sort_index(level=0)<jupyter_output><empty_output><jupyter_text>**Estatística de resumo por nível**<jupyter_code>frame.sum(level='key2')
frame.sum(level='color', axis=1)<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: FutureWarning: Using the level keyword in DataFrame and Series aggregations is deprecated and will be removed in a future version. Use groupby instead. df.sum(level=1) should use df.groupby(level=1).sum().
  """Entry point for launching an IPython kernel.<jupyter_text>**Indexando com as colunas de um DataFrame**<jupyter_code>frame = pd.DataFrame({'a':range(7), 'b':range(7,0,-1), 'c':['one','one','one','two','two','two','two'], 'd':[0,1,2,0,1,2,3]})
frame
frame2 = frame.set_index(['c','d'])
frame2
frame.set_index(['c','d'], drop=False)
frame2.reset_index()<jupyter_output><empty_output><jupyter_text>Combinando e mesclando conjuntos de dados **Junções no DataFrame no estilo de banco de dados**<jupyter_code>df1 = pd.DataFrame({'key':['b','b','a','c','a','a','b'], 'data1':range(7)})
df2 = pd.DataFrame({'key':['a','b','d'], 'data2': range(3)})
df1
df2
pd.merge(df1,df2)
pd.merge(df1,df2, on='key')
df3 = pd.DataFrame({'lkey':['b','b','a','c','a','a','b'], 'data1':range(7)})
df4 = pd.DataFrame({'rkey':['a','b','d'], 'data2':range(3)})
pd.merge(df3,df4, left_on='lkey',right_on='rkey')
pd.merge(df1,df2, how='outer')
df1 = pd.DataFrame({'key':['b','b','a','c','a','b'], 'data1':range(6)})
df1
df2 = pd.DataFrame({'key':['a','b','a','b','d'], 'data2': range(5)})
df2
pd.merge(df1,df2, on='key', how='left')
left = pd.DataFrame({'key1':['foo','foo','bar'], 'key2':['one','two','one'],'lval':[1,2,3]})
right = pd.DataFrame({'key1':['foo','foo','bar','bar'],'key2':['one','one','one','two'],'rval':[4,5,6,7]})
#indicator aponta de onde vieram os dados
pd.merge(left, right, on=['key1','key2'], how='outer', indicator=True)
pd.merge(left, right, on='key1')
pd.merge(left, right, on='key1', suffixes=('_left','_right'))<jupyter_output><empty_output><jupyter_text>**Fazendo merge com base no índice**<jupyter_code>left1 = pd.DataFrame({'key':['a','b','a','a','b','c'], 'value': range(6)})
left1
right1 = pd.DataFrame({'group_val':[3.5, 7]}, index=['a','b'])
right1
pd.merge(left1, right1, left_on='key', right_index=True)
pd.merge(left1, right1, left_on='key', right_index=True, how='outer')
lefth = pd.DataFrame({'key1':['Ohio','Ohio','Ohio','Nevada','Nevada'], 'key2':[2000,2001,2002,2001,2002],'data':np.arange(5.)})
lefth
righth = pd.DataFrame(np.arange(12).reshape((6,2)), index=[['Nevada', 'Nevada','Ohio','Ohio','Ohio','Ohio'],[2001,2000,2000,2000,2001,2002]], columns=['event1','event2'])
righth
pd.merge(lefth,righth, left_on=['key1','key2'], right_index=True)
pd.merge(lefth,righth, left_on=['key1','key2'], right_index=True, how='outer')
left2 = pd.DataFrame([[1.,2.], [3.,4.], [5.,6.]], index=['a','c','e'], columns=['ohio','Nevada'])
left2
right2 = pd.DataFrame([[7., 8.], [9.,10.], [11.,12.], [13,14]], index=['b','c','d','e'], columns=['Missouri','Alabama'])
right2

pd.merge(left2, right2, how='outer', left_index=True, right_index=True)
#outra maneira de fazer o exemplo anterior
left2.join(right2, how='outer')
left1.join(right1, on='key')
another = pd.DataFrame([[7.,8.], [9.,10.], [11., 12.], [16.,17.]], index=['a','c','e','f'], columns=['New York', 'Oregon'])
another
left2.join([right2, another])
left2.join([right2, another], how='outer')<jupyter_output><empty_output><jupyter_text>**Concatenando ao longo de um eixo**<jupyter_code>arr = np.arange(12).reshape((3,4))
arr
np.concatenate([arr, arr], axis=1)
s1 = pd.Series([0,1], index=['a','b'])
s2 = pd.Series([2,3,4], index=['c','d','e'])
s3 = pd.Series([5,6], index=['f','g'])
pd.concat([s1,s2,s3])
pd.concat([s1,s2,s3], axis=1)
s4 = pd.concat([s1, s3])
s4
pd.concat([s1,s4], axis=1)
pd.concat([s1,s4], axis=1, join='inner')
#especificar os eixos da concatenação
pd.concat([s1,s4], axis=1).reindex(columns=['a','c','b','e'])
result = pd.concat([s1,s1,s3], keys=['one','two','three'])
result
result.unstack()
pd.concat([s1,s2,s3], axis=1, keys=['one','two','three'])
df1 = pd.DataFrame(np.arange(6).reshape(3,2), index=['a','b','c'], columns=['onde','two'])
df1
df2 = pd.DataFrame(5 + np.arange(4).reshape(2,2), index=['a','c'], columns=['three','four'])
df2
pd.concat([df1, df2], axis=1, keys=['level1','level2'])
pd.concat({'level1':df1,'level2':df2}, axis=1)
pd.concat([df1, df2], axis=1, keys=['level1','level2'], names=['upper','lower'])
df1 = pd.DataFrame(np.random.randn(3,4), columns=['a','b','c','d'])
df1
df2 = pd.DataFrame(np.random.randn(2,3), columns=['b','d','a'])
df2
#ignore_index ignora o index que vem do outro dataframe e cria uma sequencia
pd.concat([df1,df2], ignore_index=True)<jupyter_output><empty_output><jupyter_text>**Combinando dados com sobreposição**<jupyter_code>a = pd.Series([np.nan, 2.5, 0.0, 3.5, 4.5, np.nan], index=['f','e','d','c','b','a'])
a
b = pd.Series([0., np.nan, 2., np.nan, np.nan, 5.], index=['a','b','c','d','e','f'])
b
np.where(pd.isnull(a), b, a)
#mesma coisa do exemplo anterior
b.combine_first(a)
df1 = pd.DataFrame({'a':[1., np.nan, 5.,np.nan], 'b':[np.nan, 2., np.nan, 6.], 'c': range(2,18,4)})
df1
df2 = pd.DataFrame({'a':[5., 4., np.nan, 3., 7.], 'b':[np.nan, 3.,4.,6.,8.]})
df2
df1.combine_first(df2)<jupyter_output><empty_output><jupyter_text>Reformatação e pivoteamento<jupyter_code>data = pd.DataFrame(np.arange(6).reshape((2,3)), index=pd.Index(['Ohio','Colorado'], name='state'), columns=pd.Index(['One','Two','three'], name='number'))
data
result = data.stack()
result
result.unstack()
result.unstack(0)
result.unstack('state')
s1 = pd.Series([0,1,2,3], index=['a','b','c','d'])
s2 = pd.Series([4,5,6], index=['c','d','e'])
data2 = pd.concat([s1,s2], keys=['one','two'])
data2
data2.unstack()
data2.unstack().stack()
data2.unstack().stack(dropna=False)
df = pd.DataFrame({'left':result,'right':result + 5}, columns = pd.Index(['left','right'], name='side'))
df
df.unstack('state')
df.unstack('state').stack('side')<jupyter_output><empty_output><jupyter_text>**Fazendo o pivoteamento de um formato "longo" para um formato "largo"** Uma maneira de armazenar séries temporais<jupyter_code>data = pd.read_csv('/content/drive/MyDrive/dados_pandas/pydata-book-2nd-edition/examples/macrodata.csv')
data.head()
periods = pd.PeriodIndex(year=data.year, quarter=data.quarter, name='date')
columns = pd.Index(['realgdp', 'infl','unemp'], name='item')
data = data.reindex(columns=columns)
data.index = periods.to_timestamp('D','end')
ldata = data.stack().reset_index().rename(columns={0:'value'})
ldata[:10]
pivoted = ldata.pivot('date','item','value')
pivoted
ldata['value2'] = np.random.randn(len(ldata))
ldata[:10]
pivoted = ldata.pivot('date','item')
pivoted[:5]
pivoted['value'][:5]
unstacked = ldata.set_index(['date','item']).unstack('item')
unstacked[:7]<jupyter_output><empty_output><jupyter_text>**Pivoteamento de formato "largo" para o formato "longo"**<jupyter_code>df = pd.DataFrame({'key': ['foo','bar','baz'], 'A':[1,2,3], 'B':[4,5,6], 'C':[7,8,9]})
df
melted = pd.melt(df,['key'])
melted
reshaped = melted.pivot('key', 'variable','value')
reshaped
reshaped.reset_index()
pd.melt(df, id_vars=['key'], value_vars=['A','B'])
pd.melt(df, value_vars=['A','B','C'])
pd.melt(df, value_vars=['key','A','B'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.26734790874524716, 'fraction_numerical': 0.04336977186311787, 'mean_word_length': 9.15319662243667, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '598053', 'license': 'MIT', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3524, 'n_words': 536, 'path': 'tratamento_dados.ipynb', 'repo_name': 'anicelysantos/book-python-para-analise-de-dados', 'size': '8416'}