<jupyter_start><jupyter_text>EDA<jupyter_code>import pandas as pd
import numpy as np

# I will now read in the csv
# https://www.kaggle.com/dgomonov/new-york-city-airbnb-open-data
df = pd.read_csv("AB_NYC_2019 (7).csv")

# This function will check to make sure that it was read in correctly
print(df.shape)
df.head()
df.rename(columns={'neighbourhood_group': 'Borough', 'neighbourhood': 'Neighbourhood', 'room_type': 'Room_type', 'minimum_nights': 'Minimum_nights', 'availability_365': 'Availability_365', 'price': 'Price'}, inplace=True)
df.columns
df.drop(['id','name','host_id','host_name', 'number_of_reviews', 'last_review', 'reviews_per_month', 'calculated_host_listings_count', 'longitude', 'latitude'], axis=1, inplace=True)
df.head()
# Columns 'to_list()'
df.columns.to_list()
# This will rearrange the columns
df = df[["Borough",
         "Neighbourhood",
         "Room_type",
         "Availability_365",
         "Minimum_nights",
         "Price"]]

# This will check to make sure that they were done correctly
print(df.shape)
df.head()
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')
# Defining Function -- ** nunique() **
def data_distributions(df):

  # predifined columns
  cols = ["Borough",
          "Neighbourhood",
          "Room_type",
          "Availability_365",
          "Minimum_nights"]

  # For loop for specified columns
  for col in cols:
    print(col + ":", df[col].nunique())
    print("\n")
    print(df[col].value_counts(normalize=True))
    print("-----\n")


# Now we need to call the function on the df
data_distributions(df)
# Defining Function -- ** value_counts() **
def values_function(df):
  # Predefined column 
   cols = ["Borough",
          "Neighbourhood",
          "Room_type",
          "Availability_365",
          "Minimum_nights"]

   for c in cols:
     print("---- %s ---" % c)
     print(df[c].value_counts())
     print("\n")

# Calling the Function
values_function(df)
# Now I need to check for any NaN's
df.isnull().sum()
# Next will be to describe the metrics
df.describe()
import pandas.util.testing as tm
import numpy as np
import matplotlib
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.cm as cmx
import matplotlib.colors as colors 
%matplotlib inline
# Not Removing Anything
dfx = df

# CHECK:
print(df.shape)
df.head()
# Name Column -- CHECK
dfx["Price"][:10]<jupyter_output><empty_output><jupyter_text>Modeling PreProcessing<jupyter_code># I need to install category encoders
!pip install category_encoders
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
import category_encoders as ce 

import warnings
warnings.filterwarnings("ignore")

# This will use a defining function
def preprocessing(df):
  """
  Precprocesses the data.

  Input: DataFrame

  Output: X_train, X_test, y_train, y_test
  """

  # Copy the DF
  dfx = df.copy()

  # Targets and Features
  target = "Price"
  features = ["Borough",
              "Neighbourhood",
              "Room_type",
              "Minimum_nights",
              "Availability_365"]

  # This will create the X Features Matrix
  X = dfx[features]

  # This will create the y target vector
  y = dfx[target]

  # This function will map - 'room_type'
  room_type_dict = {"Shared room":1, "Private room":2, "Entire home/apt":3}
  X.iloc[:, 4].map(room_type_dict)
  # print(X["room_type"])

  # Now we will create the Train_test_split
  X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    test_size=.2,
                                                    random_state=42)

  # Preprocess Pipline -- OrdinalEncoder and StandardScaler
  preprocess = make_pipeline(
      ce.OrdinalEncoder(),
      StandardScaler()
  )

  # Now we will Fit Transform and Transform Training and Testing Data
  X_train = preprocess.fit_transform(X_train)
  X_test = preprocess.fit_transform(X_test)

  # I now need to create a DataFrame for X Matrices
  X_train_df = pd.DataFrame(X_train, columns=features)
  X_test_df = pd.DataFrame(X_test, columns=features)
  print(X_train_df.shape,
        X_test_df.shape,
        X_train.shape,
        X_test.shape,
        y_train.shape,
        y_test.shape)

  # Return
  return X_train_df, X_test_df, X_train, X_test, y_train, y_test

# Calling function
X_train_df, X_test_df, X_train, X_test, y_train, y_test = preprocessing(dfx)

# Check it out
X_train_df.head()<jupyter_output>(39116, 5) (9779, 5) (39116, 5) (9779, 5) (39116,) (9779,)<jupyter_text>Linear Regression<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score

# Instantiate Model
lin_reg = LinearRegression()

# Fitting the Model to the training data
lin_reg.fit(X_train_df, y_train)

# Predicting the Training Price based on Training Data
y_pred_train = lin_reg.predict(X_train_df)
y_pred_test = lin_reg.predict(X_test_df)

# Reporting Metrics Function
def reporting_metrics(y_vector, y_pred_vector):
  mse = mean_squared_error(y_vector, y_pred_vector)
  rmse = np.sqrt(mse)
  mae = mean_absolute_error(y_vector, y_pred_vector)
  r2 = r2_score(y_vector, y_pred_vector)
  print("DATA METRICS \n")

  print("Mean Absolute Error:", mae)
  print("Mean Squared Error:", mse)
  print("Root Mean Squared Error:", rmse)
  print("R^2:", r2)

  print(".....")


# Calling Function -- ** TRAINING DATA **
reporting_metrics(y_train, y_pred_train)

# Intercept
print("Intercept:", lin_reg.intercept_)
print("\n")

# Coefficients
coefs = pd.Series(lin_reg.coef_, X_train_df.columns)
print("Coefficients:")
print(coefs)
# Calling Function -- ** TESTING DATA **
reporting_metrics(y_test, y_pred_test)

# Intercept
print("Intercept:", lin_reg.intercept_)
print("\n")

# Coefficients
coefs = pd.Series(lin_reg.coef_, X_test_df.columns)
print("Coefficients:")
print(coefs)<jupyter_output>DATA METRICS 

Mean Absolute Error: 75.00544773709035
Mean Squared Error: 40247.45484923003
Root Mean Squared Error: 200.61768329145372
R^2: 0.09021116691410547
.....
Intercept: 153.75158502914408


Coefficients:
Borough              1.456468
Neighbourhood       -0.056954
Room_type          -60.019330
Minimum_nights       3.696906
Availability_365    20.545710
dtype: float64<jupyter_text>Random Forest Regressor<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor

# Instantiate Model
rf_reg = RandomForestRegressor()

# Fitting the Model to the training data
rf_reg.fit(X_train_df, y_train)

# Predicting
y_pred_train = rf_reg.predict(X_train_df)
y_pred_test = rf_reg.predict(X_test_df)

# Reporting Metrics -- TRAINING DATA
reporting_metrics(y_train, y_pred_train)

# Get Feature Importances
importances = pd.Series(rf_reg.feature_importances_, X_train_df.columns)

# Plot Top N Features Importances
n = 5
plt.figure(figsize = (5, n/2))
plt.title(f"Top {n} Most Important Training Features")
importances.sort_values()[-n:].plot.barh(color = "green");
from sklearn.externals import joblib
from joblib import dump, load

dump(rf_reg, 'rf_reg.joblib')
joblib.load
# Reporting Metrics -- TESTING DATA
reporting_metrics(y_test, y_pred_test)

# Get Feature Importances
importances = pd.Series(rf_reg.feature_importances_, X_test_df.columns)

# Plot Top N Features Importances
n = 5
plt.figure(figsize = (5, n/2))
plt.title(f"Top {n} Most Important Testing Features")
importances.sort_values()[-n:].plot.barh(color = "red");<jupyter_output>DATA METRICS 

Mean Absolute Error: 91.9342476811693
Mean Squared Error: 68627.98222340636
Root Mean Squared Error: 261.9694299406066
R^2: -0.5513272105768163
.....
{'fraction_non_alphanumeric': 0.11166602637981816, 'fraction_numerical': 0.03419131771033423, 'mean_word_length': 4.203197868087941, 'pattern_counts': {'":': 4, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '63294', 'license': 'MIT', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2675, 'n_words': 655, 'path': 'notebooks/EDA/optimal_airbnb_EDA (2).ipynb', 'repo_name': 'optimal-airbnb/ds', 'size': '7809'}