<jupyter_start><jupyter_text>Import pandas<jupyter_code>import pandas<jupyter_output><empty_output><jupyter_text>Check pandas version<jupyter_code>print(pandas.__version__)<jupyter_output>0.20.2<jupyter_text>Data Import Load a csv file<jupyter_code>csv_file = pandas.read_csv('./data/feature_engineering.csv')<jupyter_output><empty_output><jupyter_text>Check the type of structure of csv_file<jupyter_code>type(csv_file)<jupyter_output><empty_output><jupyter_text>Now load html table using a url (requires internet connectivity)<jupyter_code>url = 'http://www.fdic.gov/bank/individual/failed/banklist.html'

dfs = pandas.read_html(url)
type(dfs)
csv_file<jupyter_output><empty_output><jupyter_text>Load the product data  Data Selection<jupyter_code>product_df = pandas.read_csv('data/product_data.csv')<jupyter_output><empty_output><jupyter_text>Print the product data<jupyter_code>product_df<jupyter_output><empty_output><jupyter_text>Select a column<jupyter_code>product_df['ShippingLocation']<jupyter_output><empty_output><jupyter_text>Select multiple column<jupyter_code>listOfCols = ['ProductName','Cost']
product_df[listOfCols]
product_df[['ProductName','Cost']]
#For selecting rows with index = 2
index = 2
product_df.loc[index]
#For selecting rows with index less than 2
index = 2
product_df.loc[:index]
#Select top 2 rows
row = 2
product_df.iloc[:row]
#Select bottom 2 rows
row = 2
product_df.iloc[-row:]
#Select all rows with 3 columns
product_df.iloc[:,1:4]
product_df[product_df['Cost']>=30000]
product_df[(product_df['Cost']>=30000) & (product_df['ShippingLocation']=='Delhi') ]
product_df[(product_df['Cost']>=30000) | (product_df['ShippingLocation']=='Delhi') ]
product_df[~(product_df['Cost']>=30000)]<jupyter_output><empty_output><jupyter_text>Data Cleaning Renaming the columns<jupyter_code>product_df= product_df.rename(columns={'Cost': 'price',
    'shippingLocation': 'destination'})<jupyter_output><empty_output><jupyter_text>Insert a column with some value<jupyter_code>product_df['NumberOfProduct'] = 1<jupyter_output><empty_output><jupyter_text>Checking for null values<jupyter_code>product_df.isnull()
product_df['price'].isnull()<jupyter_output><empty_output><jupyter_text>Drop null values<jupyter_code>#Drop all rows having any null value
product_df.dropna(axis=0)

#Drop all columns having any null value
product_df.dropna(axis=1)

#Drop all rows having less than k null values
product_df.dropna(axis=0,thresh=5)

#Drop all columns having all null values
product_df.dropna(axis=0,how='all')<jupyter_output><empty_output><jupyter_text>Replace values<jupyter_code>#Replace all occurrence of 0 with 1
product_df.replace(0,1)<jupyter_output><empty_output><jupyter_text>Replace NA values<jupyter_code>#Fill all NA values,
product_df.fillna(0)<jupyter_output><empty_output><jupyter_text>Grouping the data<jupyter_code>#Grouping on data by SellerName
product_df.groupby('SellerName')
#We can also group data based on multiple columns
regiment_preScore = product_df.groupby(['SellerName','ShippingLocation'])
regiment_preScore<jupyter_output><empty_output><jupyter_text>Apply Statistics<jupyter_code>product_df['price'].groupby(product_df['SellerName']).mean()
product_df['price'].groupby(product_df['SellerName']).count()
product_df['price'].groupby(product_df['SellerName']).min()
product_df['price'].groupby(product_df['SellerName']).median()
product_df['price'].groupby(product_df['SellerName']).std()<jupyter_output><empty_output><jupyter_text>correlation<jupyter_code>product_df.corr()<jupyter_output><empty_output><jupyter_text>Get Summary of DataFrame<jupyter_code>product_df.describe()<jupyter_output><empty_output><jupyter_text>Sort values<jupyter_code>#Sort values by cost
product_df.sort_values(by='price')
#If we want to sort values in descending order
product_df.sort_values(by='price',ascending=False)
#We can also sort values by two or more features
product_df.sort_values(by=['SellerName','price'])
product_df.dtypes
product_df['price'].astype(str)<jupyter_output><empty_output><jupyter_text>Apply functions<jupyter_code>#Declare lambda function
smallText =lambda x: x.lower()

#Apply on any column of dataframe
product_df['ShippingLocation'].apply(smallText)<jupyter_output><empty_output><jupyter_text>Normalizing categorical features Min Max Normalization<jupyter_code>from sklearn import preprocessing

data = {'price': [492, 286, 487, 519, 541, 429]}
price_frame = pandas.DataFrame(data)

min_max_normalizer = preprocessing.MinMaxScaler()
scaled_data = min_max_normalizer.fit_transform(price_frame)
price_frame_normalized = pandas.DataFrame(scaled_data)
price_frame_normalized<jupyter_output><empty_output><jupyter_text>Z-Score Normalization<jupyter_code>from sklearn import preprocessing

data = {'price': [492, 286, 487, 519, 541, 429]}
price_frame = pandas.DataFrame(data)

min_max_normalizer = preprocessing.scale(price_frame)
price_frame_normalized = pandas.DataFrame(min_max_normalizer,columns=['price'])
price_frame_normalized<jupyter_output><empty_output><jupyter_text>Bin the data<jupyter_code>from sklearn import preprocessing

data = {'age': [28,27,31,31,15,41,61,26,85,9,88,51,26,52,19]}
age_frame = pandas.DataFrame(data)

bins = [0, 20, 40, 60, 100]
group_names = ['Less Than 20 years', '20 to 40 years', '40 to 60 years', '60+ years']

age_frame['categories'] = pandas.cut(age_frame['age'], bins, labels=group_names)
age_frame<jupyter_output><empty_output><jupyter_text>Convert Categorical variable to numeric<jupyter_code>from sklearn import preprocessing

encoder = preprocessing.LabelEncoder()
encoder.fit_transform(["Delhi", "Pune", "Mumbai", "Delhi"])
encoder.inverse_transform([2,1,1,0])
dummy_frame = pandas.get_dummies(age_frame.categories)
dummy_frame
import pandas
startDate = '2017-07-06'
endDate = '2017-12-18'
Dates = pandas.DataFrame(pandas.date_range(startDate, endDate),columns=['Dates'])
Dates
import numpy
Dates['day'] = Dates['Dates'].map(lambda x:x.day)
Dates['month'] = Dates['Dates'].map(lambda x:x.month)
Dates['da_of_week'] = Dates['Dates'].map(lambda x:x.weekday_name)
Dates['is_weekend'] = Dates['Dates'].map(lambda x:numpy.is_busday(x, weekmask='1111100' ))
Dates
from sklearn.feature_extraction.text import CountVectorizer
comments=['nice product',
         'bad condition',
         'shiiping was bad',
         'great delivery']
count_vect = CountVectorizer()
X_train_counts = count_vect.fit_transform(comments)
word_frame = pandas.DataFrame(X_train_counts.toarray())

#Get all column name and integer index mapping
word_dict = dict((v, k) for k, v in count_vect.vocabulary_.items())

#Now replace integer column names with words
word_frame = word_frame.rename(columns=word_dict)
word_frame
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_transformer = TfidfVectorizer()
#tfidf_transformer.fit(X_train_counts)
X_train_tfidf = tfidf_transformer.fit_transform(comments)

word_frame = pandas.DataFrame(X_train_tfidf.toarray())

#Get all column name and integer index mapping
word_dict = dict((v, k) for k, v in tfidf_transformer.vocabulary_.items())

#Now replace integer column names with words
word_frame = word_frame.rename(columns=word_dict)

word_frame<jupyter_output><empty_output><jupyter_text>Dimentionality Reduction<jupyter_code>from sklearn.decomposition import PCA
pca = PCA()
pca.fit_transform(word_frame)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12501700448918515, 'fraction_numerical': 0.021765746156985444, 'mean_word_length': 10.31076923076923, 'pattern_counts': {'":': 0, '<': 104, '<?xml version=': 0, '>': 108, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '610161', 'license': 'MIT', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2590, 'n_words': 442, 'path': 'Chapter 03/3. Feature Engineering.ipynb', 'repo_name': 'bpbpublications/-', 'size': '7351'}