<jupyter_start><jupyter_text><jupyter_code>### EDA Project 1
# Removed invalid data - which is incorrect Ex Rating > 5 is invalid
# Checked null values since nof of rows were around 0.02% we removed those rows
# We checked Category vs Rating -> Ratings for all the categories are almost similar. Hence there is no speciality
# Checked Installs vs Rating  -> Some +ve correlation found btw Installs & Ratings

#Importing header files
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

path = '/content/drive/My Drive/Colab Notebooks/EDA Project - Play Store.csv'

data = pd.read_csv(path)
pd.set_option('max_columns', 13)
print(data.head(2))

# Observation on ratings - Found that Ratings > 5, which is invalid
data['Rating'].plot(kind='hist')
plt.show()

# Cleaning data whose rating > 5
data = data[data['Rating']<=5.0]
data['Rating'].plot(kind='hist')
plt.show()

# Average rating of application in store is around 4 which is very high
print('Average of the ratings :', data['Rating'].mean())

# Null Value Treatment

# Exploration of Null values
total_null = data.isnull().sum()
percent_null = (total_null/len(data)) * 100
missing_data = pd.concat([total_null, percent_null],  axis=1, keys=['Total','Percent'])
print(missing_data)

# Cleaning Null Value Rows
data = data.dropna()
total_null_1 = data.isnull().sum()
percent_null_1 = (total_null_1/len(data)) * 100
missing_data_1 = pd.concat([total_null_1, percent_null_1],  axis=1, keys=['Total','Percent'])
print(missing_data_1)

# Category vs Rating
cp = sns.catplot(x="Category", y="Rating", data=data, kind="box", height = 10)
cp.set_xticklabels(rotation=90)
cp.set_titles('Rating vs Category [BoxPlot]')

plt.show()

#Importing header files
from sklearn.preprocessing import MinMaxScaler, LabelEncoder
import re

# Installs vs Ratings

#Removing `,` &  '+'  from the column
data['Installs']=data['Installs'].str.replace(',','')
data['Installs']=data['Installs'].str.replace('+','')

#Converting the column to `int` datatype
data['Installs'] = data['Installs'].astype(int)

#Label encoding the column to reduce the effect of a large range of values
le=LabelEncoder()
data['Installs']=le.fit_transform(data['Installs'])
print(le.fit_transform(data['Installs']))

#Plotting Regression plot between Rating and Installs
plt.figure(figsize = (10,10))
sns.regplot(x="Installs", y="Rating", color = 'teal',data=data)
plt.title('Rating vs Installs[RegPlot]',size = 20)


print(data['Installs'].value_counts())

#print(data['Installs'].cat.codes)
print(data['Installs'].unique())
from google.colab import drive
drive.mount('/content/drive')
# import packages
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

path='/content/drive/My Drive/Colab Notebooks/Saleprice- EDA & Data Preprocessing.csv'

# Task 1 - Split into training and test datasets
df = pd.read_csv(path)
X = df.iloc[:,0:7]
y = df.iloc[:,7]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2)

#print(X_train)
#print(X_test)

# Task 2 - Detect Outlier - Scatterplot

fig, ((ax_1, ax_2), (ax_3, ax_4)) = plt.subplots(2,2, figsize=(15,15))
ax_1.scatter(X_train['LotFrontage'], y_train, color='red')
ax_1.set_title('SalePrice vs LotFrontage')
ax_1.set_xlabel('LotFrontage')
ax_1.set_ylabel('SalePrice')

ax_2.scatter(X_train['TotalBsmtSF'], y_train, color='blue')
ax_2.set_xlabel('TotalBsmtSF')
ax_2.set_ylabel('SalePrice')
ax_2.set_title('SalePrice vs TotalBsmtSF')

ax_3.scatter(X_train['GrLivArea'], y_train, color='green')
ax_3.set_xlabel('GrLivArea')
ax_3.set_ylabel('SalePrice')
ax_3.set_title('SalePrice vs GrLivArea')

ax_4.scatter(X_train['LotArea'], y_train, color='black')
ax_4.set_xlabel('LotArea')
ax_4.set_ylabel('SalePrice')
ax_4.set_title('SalePrice vs LotArea')

plt.show()


# Task 3 - Treatment of outliers -  Out LIARS!
train = pd.concat([X_train, y_train], axis=1)

mask1 = train['LotFrontage']<300
mask2 = train['TotalBsmtSF']<5000
mask3 = train['GrLivArea']<4500
mask4 = train['LotArea']<100000

train = train[ mask1 & mask2 & mask3 & mask4 ]


#Task 4 - Identify percentage of missing values
# Detecting missing data

# Split into feature and target
X_train, y_train = train.iloc[:,:7], train[['SalePrice']]

# percentage of missing values for every feature
missing_columns = (X_train.isnull().sum()*100) / len(X_train)
mask = missing_columns > 50
columns = missing_columns[mask].index.tolist()
print(columns)


# percentage of rows with more than 5 missing values
rows_percentage = (1 - (len(X_train.dropna(thresh=5)) / len(X_train)))*100      # thresh - Require that many non-NA values.
print(rows_percentage)



## Handling missing data

# Import packages
from sklearn.preprocessing import Imputer
dict_new = {'Attchd':0,'Detchd':1,'BuiltIn':2,'2Types':3,'CarPort':4,'Basment':5}
X_train['GarageType'] = X_train['GarageType'].map(dict_new)
X_test['GarageType'] = X_test['GarageType'].map(dict_new)

# Custom imputers
mean_imputer = Imputer(strategy='mean')
mode_imputer = Imputer(strategy='most_frequent')

# Drop PoolQC
X_train = X_train.drop(columns=['PoolQC'], axis=1)
X_test = X_test.drop(columns=['PoolQC'], axis=1)

# Train the imputer
mode_imputer = mode_imputer.fit(X_train[['GarageType']])
mean_imputer = mean_imputer.fit(X_train[['LotFrontage']])

# Apply the imputer 
X_train['GarageType'] = mode_imputer.transform(X_train[['GarageType']])
X_test['GarageType'] = mode_imputer.transform(X_test[['GarageType']])
X_train['LotFrontage'] = mean_imputer.transform(X_train[['LotFrontage']])
X_test['LotFrontage'] = mean_imputer.transform(X_test[['LotFrontage']])


print(X_train.info)
  from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code

Enter your authorization code:
··········
Mounted at /content/drive
{'fraction_non_alphanumeric': 0.1475932419509085, 'fraction_numerical': 0.028849218999043674, 'mean_word_length': 6.84375, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '639169', 'license': 'MIT', 'n_tokens_mistral': 2435, 'n_tokens_neox': 2312, 'n_words': 485, 'path': 'EDA_&_Data_Preprocessing.ipynb', 'repo_name': 'daizybhadresha/ga-learner-dsmp-repo', 'size': '6274'}