<jupyter_start><jupyter_text>Introduction to Data Science            Dr.	<NAME>            CS-533                        <NAME>            Student ID: 114104708            Homework 4 Series and Frames                                     Unit 31        Geting Used to Pandas Data Structures<jupyter_code>import pandas as pd
# The last value is wrokg, we will fix it later!
inflation = pd.Series((2.2, 3.4,2.8,1.6,2.3,2.7,3.4,3.2,2.8,3.8,-0.4,1.6,3.2,2.1,1.5,1.5))
inflation
len(inflation)
inflation.values
inflation.index
inflation.index.values
import numpy as np
inflation = pd.Series({1999:2.2,2000:3.4,2001:2.8,2002:1.6,2003:2.3,2004:2.6,2005:3.4,2006:3.2,2007:2.8,2008:3.8,2009:-0.4,2010:1.6,2011:3.2,2012:2.1,2013:1.5,2014:1.6,2015:np.nan})

inflation.index = pd.Index(range(1999, 2016))
inflation[2015]=np.nan
inflation
inflation.index.name = "Year"
inflation.name = "%"
inflation.head()
inflation.tail()<jupyter_output><empty_output><jupyter_text>The following section we don't have all the data but I will do my best to replicate the excercises by loading a csv file<jupyter_code>tmpdata=pd.read_csv("./data.csv")
alco2009 = tmpdata.loc[tmpdata['Year'] == 2009]
alco2009.set_index('State',inplace=True)
alco2009.drop(['Year'], axis=1, inplace=True)
alco2009
alco2009["Wine"].head()
alco2009.Beer.tail()
alco2009["Total"]=0
alco2009.head()<jupyter_output>D:\sw\anaconda\envs\fastai\lib\site-packages\ipykernel_launcher.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  """Entry point for launching an IPython kernel.<jupyter_text>Unit 32        Reshaping Data<jupyter_code>alco2009.columns.values
alco2009.index.values
alco2009.reset_index().set_index("Beer").head()
alco2009.ix["Nebraska"]
"Samoa" in alco2009.index
s_states = [state for state in alco2009.index if state[0] == 'S'] + ["Samoa"]
drinks = list(alco2009.columns) + ["Water"]
nan_alco = alco2009.reindex(s_states, columns=drinks)
nan_alco
alco=pd.read_csv("./data.csv")
state = ""
test=[]
for a in alco.itertuples():
    if isinstance(a.State, float) == False:
        state = a.State
    test.append(list([state, a.Year]))
alco
multi = pd.MultiIndex.from_tuples(test, names=["State","Year"])
alco.drop(['Year',"State"], axis=1, inplace=True)
alco.index=multi
import pickle
with open("alco.pickle","wb") as oFile:
    pickle.dump(alco, oFile)
alco.ix['Wyoming'].head()
alco.ix['Wyoming', 1999]
tall_alco = alco.stack()
tall_alco.index.names = ['State','Year','Drink']
tall_alco.head()
wide_alco = alco.unstack()
wide_alco.head()<jupyter_output><empty_output><jupyter_text>Unit 33        Handling Missing Data<jupyter_code>nan_alco.dropna(how="all")
nan_alco.dropna(how="all", axis=1)

nan_alco.dropna()
nan_alco.isnull()

nan_alco.notnull()
sp = nan_alco['Spirits'] # Selected a column with dirty rows
clean = sp.notnull() # The clean rows
sp[-clean] = sp[clean].mean() # Impute the clean mean into the dirty rows
nan_alco

nan_alco.fillna(0)

nan_alco.fillna(method="ffill")<jupyter_output><empty_output><jupyter_text>Unit 34        Combining Data<jupyter_code>population=pd.read_csv("./population.csv",thousands=',')
population=population.set_index("State")
alco2009.drop(columns="Total",inplace=True)
population.head()
df = pd.merge(alco2009.reset_index(),
population.reset_index()).set_index("State")
df.head()
df = pd.merge(alco2009, population, left_index=True, right_index=True)
df.head()

population.join(alco2009).tail(10)
pd.concat([alco2009, population], axis=1).tail()
pop_ca=pd.read_csv("./1710000501-eng.csv",thousands=',')
pop_ca.set_index("State", inplace=True)
pop_ca.head()
pop_na = pd.concat([population, pop_ca], keys=["US", "CA"])
pop_na.index.names = ("Country", "State")
pop_na<jupyter_output><empty_output><jupyter_text>Unit 35        Ordering and Describing Data<jupyter_code>population.sort_index().head()

population.sort_values("Population").head()
pop_by_state = population.sort_index()
pop_by_state.rank().head()

alco2009.max()
alco2009.min(axis=1)
alco2009.sum()
alco.ix['Hawaii'].cumsum().head()
alco.ix['Hawaii'].diff().head()
dna = "AGTCCGCGAATACAGGCTCGGT"
dna_as_series = pd.Series(list(dna), name="genes")
dna_as_series.head()
dna_as_series.unique()

dna_as_series.value_counts().sort_index()

valid_nucs = list("ACGT")
dna_as_series.isin(valid_nucs).all()<jupyter_output><empty_output><jupyter_text>Unit 36        Transforming Data<jupyter_code>alco["Total"] = alco.Wine + alco.Spirits + alco.Beer
alco.head()
np.log10(alco.Total).head()
dna = "AGTCCGCGAATACAGGCTCGGT"
dna1 = dna.replace("C", "")
dna2 = dna.replace("T", "")
dna_as_series1 = pd.Series(list(dna1), name="genes") # No C's
dna_as_series2 = pd.Series(list(dna2), name="genes") # No T's
dna_as_series1.value_counts() + dna_as_series2.value_counts()

# We want to group by the "Year" column
alco_noidx = alco.reset_index()
sum_alco = alco_noidx.groupby("Year").sum()
sum_alco.tail()
regions=pd.read_csv("./regions.csv")
state2reg = dict()
for data in regions.itertuples():
    state2reg.update({data.state:data.Region})
state2reg
alco2009.groupby(state2reg).mean()
cats = pd.cut(alco2009['Wine'], 3).head()
cats
cats = pd.cut(alco2009['Wine'], 3, labels=("Low", "Moderate", "Heavy"))
cats.head()
cats = pd.cut(alco2009['Wine'], 3, labels=False).head()
cats
quants = pd.qcut(alco2009['Wine'], 3, labels=("Low", "Moderate", "Heavy"))
quants.head()
pd.get_dummies(state2reg).sort_index().head()
with_state = alco2009.reset_index()
abbrevs = with_state["State"].map(lambda x: x[:3].upper())
abbrevs.head()
wine_state = alco2009["Wine"] > alco2009["Wine"].mean()
beer_state = alco2009["Beer"] > alco2009["Beer"].mean()
pd.crosstab(wine_state, beer_state)<jupyter_output><empty_output><jupyter_text>Unit 37        Taming Pandas File I/O<jupyter_code>regions = pd.read_csv("original_regions.csv",
header=None,
names=("region", "division", "state"))
state2reg_series = regions.ffill().set_index("state")["region"]
state2reg_series.head()
state2reg = state2reg_series.to_dict()
state2reg<jupyter_output><empty_output><jupyter_text>Lynx Trappings            Section "Your Turn"            Chapter 6            Page: 119                         Step 1        Import Libraries, making input directory and saving file<jupyter_code>import os
import urllib
import pandas as pd
import numpy as np
inFilePath="./cache"
if not os.path.exists(inFilePath):
    print("Directory ",inFilePath," doesn't exist. Creating it")
    os.mkdir(inFilePath)
else:
    print("Directory ",inFilePath," exist. Continuing")

fileToWork = urllib.request.URLopener()
fileToWork.retrieve("http://vincentarelbundock.github.io/Rdatasets/csv/datasets/lynx.csv", inFilePath+"/lynx.csv")<jupyter_output><empty_output><jupyter_text>Step 2        Read data and save it as a DataFrame<jupyter_code>data = pd.read_csv(inFilePath+"/lynx.csv",index_col=0)
data.head()<jupyter_output><empty_output><jupyter_text>Step 3        Generate a new column for decades<jupyter_code>data["decade"]=((np.trunc((data.time/10)))*10).astype(int)
data.head()<jupyter_output><empty_output><jupyter_text>Step 4        Rearrange columns<jupyter_code>data=data[['decade', 'time', 'value']]
data.head()<jupyter_output><empty_output><jupyter_text>Step 5        Set decade as index<jupyter_code>data=data.set_index("decade")
data.head()<jupyter_output><empty_output><jupyter_text>Step 6        Make new dictionary for final data<jupyter_code>res = {}
for dec in list(set(data.index.values)):
    res[dec]=(data.ix[dec].value.sum())
res<jupyter_output><empty_output><jupyter_text>Step 7        Generate new DataFrame from Dictionary and sort it<jupyter_code>finalDF = pd.DataFrame.from_dict(res, orient='index',columns=['trappings'])
finalDF.index.name = "decade"
finalDF.sort_values("trappings",ascending=False, inplace=True)
finalDF<jupyter_output><empty_output><jupyter_text>Step 8        Make out directory<jupyter_code>outFilePath="./doc"
fileName="/result.csv"
if not os.path.exists(outFilePath):
    print("Directory ",outFilePath," doesn't exist. Creating it")
    os.mkdir(outFilePath)
else:
    print("Directory ",outFilePath," exist. Continuing")<jupyter_output><empty_output><jupyter_text>Step 9        Saving results to CSV file<jupyter_code>finalDF.to_csv(outFilePath+fileName)<jupyter_output><empty_output><jupyter_text>Step 10        Displaying results<jupyter_code>%matplotlib inline  

import matplotlib.pyplot as plt
finalDF.plot.bar()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15671468401486988, 'fraction_numerical': 0.04426115241635688, 'mean_word_length': 5.854299363057325, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 76, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '116598', 'license': 'MIT', 'n_tokens_mistral': 3528, 'n_tokens_neox': 3232, 'n_words': 541, 'path': 'introds/hw/L4-SeriesAndFrames/series-frames.ipynb', 'repo_name': 'gcaracas/bsu', 'size': '8633'}