<jupyter_start><jupyter_text>Importing the Libraries<jupyter_code>#Importing libraries for cleaning
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Importing Training and Test Data<jupyter_code>#Using Pandas Library to import training data
train_data = pd.read_csv("drive/My Drive/Train.csv")

#Using Pandas Library to import test data
test_data = pd.read_csv("drive/My Drive/Test.csv")<jupyter_output><empty_output><jupyter_text>Missing Values Code<jupyter_code>train_data.head()
train_data['join_date']
#Check Missing Values in a Dataset
def NACount(data):
  NaCount = {}
  for feature in data.columns:
    NaCount[feature] = data[feature].isna().sum()

  return NaCount
#Check Unique Values in Training Data
def UniqueCount(data):
  UnCount = {}
  for feature in data.columns:
    UnCount[feature] = data[feature].unique()

  return UnCount
#Run Function on training Data

print(NACount(train_data))
#Run Function on test Data

print(NACount(test_data))
#Run Unique Values Function on test Data

print(UniqueCount(test_data))
#Run  Unique Values Function on training Data

print(UniqueCount(train_data))
#Drop the two records with missing join dates
train_data.dropna(inplace = True)
print(NACount(train_data))
#Drop the one record with a missing join date from the Test Data
test_data.dropna(inplace = True)
print(NACount(train_data))
#Replacing little f with big F
train_data.marital_status.replace(["f"], ["F"], inplace = True)
print(NACount(test_data))<jupyter_output>{'ID': 0, 'join_date': 0, 'sex': 0, 'marital_status': 0, 'birth_year': 0, 'branch_code': 0, 'occupation_code': 0, 'occupation_category_code': 0, 'P5DA': 0, 'RIBP': 0, '8NN1': 0, '7POT': 0, '66FJ': 0, 'GYSR': 0, 'SOP4': 0, 'RVSZ': 0, 'PYUQ': 0, 'LJR9': 0, 'N2MW': 0, 'AHXO': 0, 'BSTQ': 0, 'FM3X': 0, 'K6QO': 0, 'QBOL': 0, 'JWFN': 0, 'JZ9D': 0, 'J9JW': 0, 'GHYX': 0, 'ECY3': 0}<jupyter_text>Data Type Checking<jupyter_code>def catDataType(data):
  for feature in data.columns:
    if data[feature].dtype == "object" and feature != "ID":
      data[feature] = data[feature].astype('category')

  return data


#Check the data types of the training data
train_data.dtypes
#Check the data types of the test data
test_data.dtypes

cut_bins = [1930, 1950, 1970, 1990, 2000, 2020]
train_data['birth_year'] = pd.cut(train_data['birth_year'], bins=cut_bins)
test_data['birth_year'] = pd.cut(test_data['birth_year'], bins=cut_bins)
#Convert the join date to datetime
train_data['join_date'] = pd.to_datetime(train_data['join_date'])
test_data['join_date'] = pd.to_datetime(test_data['join_date'])
train_data.dtypes

train_data['join_date'], bins_jd = pd.cut(train_data['join_date'], bins=5, retbins=True)
test_data['join_date'] = pd.cut(test_data['join_date'], bins = bins_jd)
train_data.join_date.value_counts()
#Convert the categorial features into the category data type
train_data = catDataType(train_data)
test_data = catDataType(test_data)
train_data.dtypes
test_data.join_date.value_counts()
train_data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 29130 entries, 0 to 29131
Data columns (total 29 columns):
 #   Column                    Non-Null Count  Dtype   
---  ------                    --------------  -----   
 0   ID                        29130 non-null  object  
 1   join_date                 29130 non-null  category
 2   sex                       29130 non-null  category
 3   marital_status            29130 non-null  category
 4   birth_year                29130 non-null  category
 5   branch_code               29130 non-null  category
 6   occupation_code           29130 non-null  category
 7   occupation_category_code  29130 non-null  category
 8   P5DA                      29130 non-null  int64   
 9   RIBP                      29130 non-null  int64   
 10  8NN1                      29130 non-null  int64   
 11  7POT                      29130 non-null  int64   
 12  66FJ                      29130 non-null  int64   
 13  GYSR                      29130 non-null  int6[...]<jupyter_text>Exploratory Data Analysis Function<jupyter_code>#Importing the graphing libraries
import matplotlib.pyplot as plt 
import seaborn as sns
#Defining graphing function
def ExploreGraphs(x_feature, y_result, df, choice, custom):
  
  #Setting the visual features of the graph
  sns.set_style("darkgrid")
  plt.figure(figsize=(10,10))
  
  #Bar Chart choice
  if choice == "bar":

    barplot = sns.barplot(x = x_feature, y = y_result, data = df, ci=custom)
    plt.show()

  #Histogram choice
  elif choice == "hist":

    histplot = plt.hist(x = df[x_feature])
    plt.show()

  #Count Plot Choice
  elif choice == "count":

    barplot = sns.countplot(x = x_feature, data = df)
    plt.show()

  #Heat map choice
  elif choice == "heat":

    plt.figure(figsize=(30,30))
    sns.heatmap(df.corr(), vmax=0.8, square=True, annot = True)

  #Pair plot choice
  elif choice == "pair":

    sns.pairplot(df)

  #In case the choice input is not defined in the function
  else:
    print("Not a valid choice")<jupyter_output><empty_output><jupyter_text>Graphing Area<jupyter_code>'''
How to use function
FIRST PARAMETER: X Feature (Pandas Column Name)

SECOND PARAMETER: Y FEATURE (Pandas Column Name)

THIRD PARAMETER: DATA (Pandas Dataframe)

FOURTH PARAMETER: Choice of graph: "bar" for Bar Plot (Good for comparing one insurance product with a feature),
                                    "hist" for Histogram (Good for continuous distrbution of one feature),
                                    "count" for a count plot (Good for the discontinuous distribution of one feature),
                                    "heat" for a heatmap between the variables of the dataset,
                                    "pair" for a pairplot between the variables of a dataset (Be careful of the number of features you're comparing)

FIFTH PARAMETER: Confidence Interval error bar for Bar Plot = number (number% Confidence Interval), None (No error bar) 
'''
ExploreGraphs("occupation_code", "RVSZ", test_data, "bar", 95)<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>#Creating a copy of the features
train_data_features = train_data.iloc[:,1:8]
test_data_features = test_data.iloc[:,1:8]

#Creating a copy of the products
train_data_products = train_data.iloc[:,8:]
test_data_products = test_data.iloc[:,8:]
test_data_products.head()
train_data_features.drop(columns = 'occupation_code', inplace = True)
test_data_features.drop(columns = 'occupation_code', inplace = True)
test_data_features
test_data.birth_year.value_counts()
pd.DataFrame(train_data_features).to_csv("drive/My Drive/train_data_features.csv")
pd.DataFrame(test_data_features).to_csv("drive/My Drive/test_data_features.csv")
pd.DataFrame(train_data_products).to_csv("drive/My Drive/train_data_products.csv")
pd.DataFrame(test_data_products).to_csv("drive/My Drive/test_data_products.csv")<jupyter_output><empty_output><jupyter_text>Encoding the Categorial Data<jupyter_code>#Importing One Hot Encoding libraries
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
train_data_features.join_date.unique

ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0,1,2,3,4,5])], remainder='passthrough')
train_data_features = (ct.fit_transform(train_data_features))
test_data_features = (ct.fit_transform(test_data_features))

test_data<jupyter_output><empty_output><jupyter_text>Split Encoded Data<jupyter_code>from sklearn.model_selection import train_test_split
X_train_70, X_valid_30, Y_train_70, Y_valid_30 = train_test_split(train_data_features, train_data_products, test_size = 0.3, random_state=1)<jupyter_output><empty_output><jupyter_text>Feature Scaling<jupyter_code>X_train_70.dtype
from sklearn.preprocessing import StandardScaler
sc = StandardScaler(with_mean=False)
X_train_70 = sc.fit_transform(X_train_70)
X_valid_30 = sc.transform(X_valid_30)
test_data_features = sc.fit_transform(test_data_features)<jupyter_output><empty_output><jupyter_text>Export Cleaned Data<jupyter_code>pd.DataFrame(X_train_70).to_csv("drive/My Drive/X_train_70.csv")
pd.DataFrame(X_valid_30).to_csv("drive/My Drive/X_valid_30.csv")
pd.DataFrame(Y_train_70).to_csv("drive/My Drive/Y_train_70.csv")
pd.DataFrame(Y_valid_30).to_csv("drive/My Drive/Y_valid_30.csv")
pd.DataFrame(test_data_features).to_csv("drive/My Drive/encoded_test_data_features.csv")
pd.DataFrame(test_data_products).to_csv("drive/My Drive/encoded_test_data_products.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10997084548104956, 'fraction_numerical': 0.029037900874635567, 'mean_word_length': 4.44853875476493, 'pattern_counts': {'":': 6, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '608491', 'license': 'MIT', 'n_tokens_mistral': 3104, 'n_tokens_neox': 2882, 'n_words': 696, 'path': 'Data_Cleaning_Code.ipynb', 'repo_name': 'malcolmRite/House-Price-Predictions', 'size': '8575'}