<jupyter_start><jupyter_text><jupyter_code>#basic importing and aliasing
import numpy as np
import pandas as pd
#prevent dataframe truncation by adjusting pandas display options
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)<jupyter_output><empty_output><jupyter_text>Reading in Data<jupyter_code>#read in csv
black_friday = pd.read_csv('https://raw.githubusercontent.com/pierretd/datasets-1/master/BlackFriday.csv')
print(black_friday.head())

#read in json
#json load tutorial: https://chrisalbon.com/python/data_wrangling/load_json_file_into_pandas/
space = pd.read_json('https://raw.githubusercontent.com/pierretd/datasets-1/master/PeopleInSpaceNow.json')
print(space.head())

#read in tsv
fast_fm = pd.read_csv('https://raw.githubusercontent.com/pierretd/datasets-1/master/FastFmProfiles.tsv', delimiter='\t', encoding='utf-8')
print(fast_fm.head())
#uploading a file
from google.colab import files
upload = files.upload()
#renaming headers during upload
url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/autos/imports-85.data'
auto = pd.read_csv(url,names=['symboling','norm_loss','make','fuel','aspiration','doors',
                              'bod_style','drv_wheels','eng_loc','wheel_base','length','width',
                              'height','curb_weight','engine','cylinders','engine_size',
                              'fuel_system','bore','stroke','compression','hp','peak_rpm',
                              'city_mpg','hgwy_mpg','price'])<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>#rename columns after reading in
flag_data = pd.read_csv(flag_data_url, header=None)
feature_map = {0: 'name',
               1: 'landmass',
               2: 'zone',
               3: 'area',
               4: 'population',
               5: 'language',
               6: 'religion',
               7: 'bars',
               8: 'stripes',
               9: 'colours',
               10: 'red',
               11: 'green',
               12: 'blue',
               13: 'gold',
               14: 'white',
               15: 'black',
               16: 'orange',
               17: 'mainhue',
               18: 'circles',
               19: 'crosses',
               20: 'saltires',
               21: 'quarters',
               22: 'sunstars',
               23: 'crescent',
               24: 'triangle',
               25: 'icon',
               26: 'animate',
               27: 'text',
               28: 'topleft',
               29: 'botright'}

flag_data.rename(columns=feature_map, inplace=True)
#replacing values in a data set with a new value
auto = auto.replace('?', np.NaN)
#setting categories to type 'category' for faster operations
auto['make'] = auto['make'].astype('category')
#label encoding
auto['make'] = auto['make'].cat.codes
#one hot encoding
auto = pd.get_dummies(auto, columns=['make'], prefix = ['make'])
auto.head()
!pip install category_encoders as ce
#binary encoding
encoder = ce.BinaryEncoder(cols=['make'])
auto = encoder.fit_transform(auto)
#filling null values (will not run, just example)
df.fillna(0) #fill nulls with a value
df.fillna(method='ffill') #fill based on a method chosen from  {‘backfill’, ‘bfill’, ‘pad’, ‘ffill’, None}
df.fillna(value={'A': 0, 'B': 1, 'C': 2, 'D': 3}) #fill each feature with a different value
df.fillna(value=values, limit=1) #fill a set number of nulls
#dropping null values
df.dropna() #drop rows
df.dropna(axis='columns') #drop columns
df.dropna(axis='rows', thresh=3) # drop rows with fewer non-nulls than the threshhold
#add a column (will not run, just example)
df['new feature'] = 'Value' #new column with same value for all rows
df['new feature'] = df2['df2 feature'] #new column from another dataframe column
#combine two data frames (will not run, just example)
df3 = df1.append(df2)
#use np.where to encode a category based on values in another category (will not run, just example)
df['feature1'] = np.where(df['feature2'].str.contains('value'), 1, 0)
#loop to impute values based on whether they are numeric or categorical (will not run, just example)
from pandas.api.types import is_numeric_dtype

for header in comic_chars_df:
  if is_numeric_dtype(comic_chars_df[header]):
    #do something with the numeric features here
  else:
    #do something with the categorical features here<jupyter_output><empty_output><jupyter_text>Exploring Data<jupyter_code>#show X rows in the dataframe, no value defaults to 5
auto.head()
#summary of the dataframe
auto.info()
#get basic stats on any numeric features
auto.describe()
#dimensions of the dataframe
auto.shape
#what are the data types of the different features
auto.dtypes
#how many items in each feature of a dataframe
auto.count()
#count how many of each item in a feature
auto['make'].value_counts()
#count how many unique values in a category
auto['make'].value_counts().count()
auto['make'].nunique()
#how many null or non-null values are there and what features are they in
auto.isnull().sum()
auto.notnull().sum()
#addittional useful pandas calls
df.tail() 
df.apply()
df.map()
df.between()
df.unique()
df.index
df.values
df.isin()
df.astype()
df.iloc[]
df.loc[]
df.groupby()
df.T
df.interpolate() #https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.interpolate.html
#binning with pd.cut (will not run, just example)
time_bins = pd.cut(df['time'], 5) #create 5 bins of time
#crosstab for quick small tables (will not run, just example)
pd.crosstab(df['purchased'], time_bins, normalize='columns')
#pivot table
#https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.pivot_table.html
table = pd.pivot_table(auto, values=['length'],index=['make'],columns=['bod_style']) #pivot table showing length values against make and bod_style

table = pd.pivot_table(auto, values=['length'],index=['make'],columns=['bod_style'], aggfunc=np.sum) #pivot table showing summing length values against make and bod_style<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13615101904443702, 'fraction_numerical': 0.01252923488138991, 'mean_word_length': 3.7104642014162077, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '181582', 'license': 'MIT', 'n_tokens_mistral': 1991, 'n_tokens_neox': 1887, 'n_words': 523, 'path': 'Data_Cleaning_and_Exploring.ipynb', 'repo_name': 'Granero0011/DS-Unit-1-Sprint-1-Dealing-With-Data', 'size': '5986'}