<jupyter_start><jupyter_text><jupyter_code># Importing all the libraries for the project
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
# Setting figure size
sns.set(rc={'figure.figsize':(15,10)})
# Extraction Process from csv to pandas dataframe
df = pd.read_csv("price_train.csv")
df.head()
# function to print and calculate total amenities


def total_amenities(x):
  total_amen = []
  for i in x:
    z = i.replace('{' , '')
    z = z.replace('}' , '')
    z = z.replace('"' , '').split(',')
    for j in z:
      if j not in total_amen:
        total_amen.append(j)
        print(j)

  print(len(total_amen))


total_amenities(df['amenities'])
# Univariate Analysis of each column to get to know dataset more clearly
# function to present the count plot graph
def CountPlot(x,df,xl='x-axis',yl='Total Count',title='Countplot',z=0):
  length = len(df)
  g=sns.countplot(x=x,data=df)
  for p in g.patches:
    height = p.get_height()
    g.text(p.get_x()+p.get_width()/2.,height + 20,'{:1.2f}%'.format(height/length*100),ha="center") 
  g.set(xlabel=xl, ylabel=yl, title=title)
  plt.xticks(rotation=z)
  plt.show()
# Visualizing countplot of column 'host_is_superhost'
CountPlot('host_is_superhost',df,'Host is Superhost','Total Count of Response','Count Plot of response of Host is SuperHost')
# Visualizing countplot of column 'host_response_time'
CountPlot('host_response_time',df,'Host Response Time','Total Count of Response','Count Plot of Host Response Time',45)
# Now creating ETL pipline class which will Extract the csv data, transform into required format and then save the new dataset as csv.
class ETL:
  def __init__(self, Path,Train=True):
    self.df = pd.read_csv(Path)
    self.Train = Train
    self.Transform()
  
  # Method to Transform the data into required format

  def Transform(self):
    # Deleting X1 and id columns as they both are for identification 
    self.df = self.df.drop(['X1','id'],axis=1)

    # Function to Convert the boolean values into Numerical values
    def bool_to_num(x):
      if x==True:
        return 1
      else:
        return 0

    # Applying the bool_to_num function on 'host_is_superhost','host_identity_verified' and 'instant_bookable'
    self.df['host_is_superhost'] = self.df['host_is_superhost'].apply(bool_to_num)
    self.df['host_identity_verified'] = self.df['host_identity_verified'].apply(bool_to_num)
    self.df['instant_bookable'] = self.df['instant_bookable'].apply(bool_to_num)

    # Function to split the number and % and convert it into numerical values
    def percentage_to_num(x):
      return int(x.split('%')[0])

    # Applying the string_to_num function on 'host_response_rate'
    self.df['host_response_rate'] = self.df['host_response_rate'].apply(percentage_to_num)

    # Function to split the number and $ and convert it into numerical values
    def dollar_to_num(x):
      if x == '0':
        return float(x)
      else:
        try:
          return float(x.split('$')[1])
        except:
          return float(x.replace(',' , '').split('$')[1])

    
    # Applying the string_to_num function on 'cleaning_fee' and 'price'
    self.df['cleaning_fee'] = self.df['cleaning_fee'].apply(dollar_to_num)
    if self.Train:
      self.df['price'] = self.df['price'].apply(dollar_to_num)

    # creating instance of labelencoder
    labelencoder = LabelEncoder()

    # Assigning numerical values and storing in same column (host_response_time, neighbourhood_group_cleansed, property_type, room_type, bed_type, cancellation_policy)
    label_encoding_cols = ['host_response_time', 'neighbourhood_group_cleansed', 'property_type', 'room_type', 'bed_type', 'cancellation_policy']
    for i in label_encoding_cols:
      self.df[i] = labelencoder.fit_transform(self.df[i])


    # As there are total 178 amenities, we are creating a column in which total amenities of that entry is present.
    # Function to convert amenities column into total amenities present for that entry
    def total_amenities(x):
      total = 0
      z = x.replace('{' , '')
      z = z.replace('}' , '')
      z = z.replace('"' , '').split(',')
      for j in z:
        total = total + 1
      return total

    
    # Applying the total_amenities function on 'amenities'
    self.df['amenities'] = self.df['amenities'].apply(total_amenities)
        

      

  # Function to print and save transformed data
  def Load(self):
    self.df.to_csv("Transformed_Data.csv")
    return self.df


etl = ETL("price_train.csv")
data = etl.Load()
data.head(10)
# Installing lazy predict on google collab. As lazy predict gives rough idea of the models and their respective accuracies. 
!pip install lazypredict
# Importing required libararies to train the model

import lazypredict
from sklearn.model_selection import train_test_split
from lazypredict.Supervised import LazyRegressor
# First we are splitting the dataset into X and Y where X are the all columns except 'Price' and Y is 'Price' column
X = data.drop(['price'],axis=1)
X.head(10)
Y = data['price'].copy()
Y.head(10)
# Now Function to Standardized the dataframe
def Standard_convert(X):
  from sklearn.preprocessing import StandardScaler
  scaler = StandardScaler()
  X_standard = scaler.fit_transform(X)
  X_new = pd.DataFrame(X_standard,columns=X.columns)
  return X_new

X_new = Standard_convert(X)
X_new.head(10)
# Now splitting the data into Training and Testing
X_train, X_test, y_train, y_test = train_test_split(X_new, Y, test_size = 0.3, random_state = 420)
# Now applying Lazy Predict
reg = LazyRegressor(verbose=0,ignore_warnings=True,custom_metric=None, predictions=True)
models,pred = reg.fit(X_train, X_test, y_train, y_test)
models = models[0:-1]
models = models.reset_index()
models
# Function to print Bar plot
def BarPlot(x,y,df,xl='x-axis',yl='y-aixs',title='Barplot',z=0):
  length = len(df)
  g=sns.barplot(x = x, y = y, data = df,
            palette = 'hls') 
  g.set(xlabel=xl, ylabel=yl, title=title)
  plt.xticks(rotation=z)
  plt.show()
BarPlot('Model','R-Squared',models,xl="Models", yl="R-Squared Value", title="Comparision of models",z=90)
BarPlot('Model','RMSE',models,xl="Models", yl="RMSE Value", title="Comparision of models",z=90)
BarPlot('Model','Time Taken',models,xl="Models", yl="Time Taken", title="Comparision of models",z=90)
# As per the above plots, it is clear that the best model for this dataset is Random Forest Regression
# Hence, training a Random Forest Regression model, Calculate its prediction matrices and the use the model to predict the Testing dataset provided.
# Checking the X_new
# Descriptive statistics for each column
X_new.describe()
from sklearn.ensemble import RandomForestRegressor

regressor = RandomForestRegressor(n_estimators=200, random_state=0)
regressor.fit(X_train, y_train)
y_pred = regressor.predict(X_test)
from sklearn import metrics

print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
# Now predicting the given testing dataset
test_etl = ETL("price_test.csv",False)
Val = test_etl.Load()
Val.head(10)
training_val = Standard_convert(Val)
training_val.head(10)
Prediction_Result = regressor.predict(training_val)
Prediction_Result = pd.DataFrame(Prediction_Result,columns=['Result'])
Prediction_Result.to_csv("Prediction_Result.csv")
Prediction_Result.head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11496943685784887, 'fraction_numerical': 0.008193523214982443, 'mean_word_length': 4.2779684282772825, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '545111', 'license': 'MIT', 'n_tokens_mistral': 2615, 'n_tokens_neox': 2447, 'n_words': 679, 'path': 'ejam_Jay.ipynb', 'repo_name': 'anuragmishra218/ETL_Visualization_Prediction_Python', 'size': '7689'}