<jupyter_start><jupyter_text>Lambda School Data Science*Unit 2, Sprint 3, Module 1*--- Define ML problemsYou will use your portfolio project dataset for all assignments this sprint. AssignmentComplete these tasks for your project, and document your decisions.- [ ] Choose your target. Which column in your tabular dataset will you predict?- [ ] Is your problem regression or classification?- [ ] How is your target distributed?    - Classification: How many classes? Are the classes imbalanced?    - Regression: Is the target right-skewed? If so, you may want to log transform the target.- [ ] Choose which observations you will use to train, validate, and test your model.    - Are some observations outliers? Will you exclude them?    - Will you do a random split or a time-based split?- [ ] Choose your evaluation metric(s).    - Classification: Is your majority class frequency > 50% and < 70% ? If so, you can just use accuracy if you want. Outside that range, accuracy could be misleading. What evaluation metric will you choose, in addition to or instead of accuracy?- [ ] Begin to clean and explore your data.- [ ] Begin to choose which features, if any, to exclude. Would some features "leak" future information?<jupyter_code># dataset location - http://insideairbnb.com/get-the-data.html

import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Kaggle-Challenge/master/data/'
    !pip install category_encoders==2.*
    !pip install pandas-profiling==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'
import pandas as pd
from sklearn.model_selection import train_test_split

listings = pd.read_csv('listings.csv')

listings.shape
listings.head()
# Target column is "price" and will need Regression model type

listings['price'].value_counts()


listings['price'].describe()
%matplotlib inline
import seaborn as sns
sns.distplot(listings['price']);

# Target appeared right-skewed so need to apply log transformation

import numpy as np
price_log = np.log1p(listings['price'])
import matplotlib.pyplot as plt

sns.distplot(price_log)
plt.title('Log-transformed target'); # price here is log dollars
#Removing Outliers?

import numpy as np

listings = listings[(listings['price'] >= np.percentile(listings['price'], 0.5)) & 
        (listings['price'] <= np.percentile(listings['price'], 99.5)) & 
        (listings['latitude'] >= np.percentile(listings['latitude'], 0.05)) & 
        (listings['latitude'] < np.percentile(listings['latitude'], 99.95)) &
        (listings['longitude'] >= np.percentile(listings['longitude'], 0.05)) & 
        (listings['longitude'] <= np.percentile(listings['longitude'], 99.95))]
# Feature selection, splitting the data

from sklearn.model_selection import train_test_split

train, test = train_test_split(listings, test_size=0.20, random_state=42 )

train, val = train_test_split(train, test_size=0.20, random_state=42 )

train.shape, test.shape, val.shape
# The status_group column is the target
target = 'price'

# Get a dataframe with all train columns except the target & id
train_features = train.drop(columns=[target, 'id'])

# Get a list of the numeric features
numeric_features = train_features.select_dtypes(include='number').columns.tolist()

# Get a series with the cardinality of the nonnumeric features
cardinality = train_features.select_dtypes(exclude='number').nunique()

# Get a list of all categorical features with cardinality <= 50
categorical_features = cardinality[cardinality <= 100].index.tolist()

# Combine the lists 
features = numeric_features + categorical_features
print(len(features))
print(features)
X_train = train[features]
y_train = train[target]
X_val = val[features]
y_val = val[target]
X_test = test[features]
y_test = test[target]
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_predict, cross_val_score

pipeline = make_pipeline(
    ce.OneHotEncoder(use_cat_names=True), 
    SimpleImputer(strategy='mean'), 
    LinearRegression()
)
# Apply logarithmic transformation to data set before fitting the model

y_train_log = np.log1p(y_train)
y_val_log = np.log1p(y_val)
pipeline.fit(X_train, y_train_log)
y_pred_log = pipeline.predict(X_val)
y_pred = np.expm1(y_pred_log)
print(mean_absolute_error(y_val, y_pred))<jupyter_output>82.0520173464766
{'fraction_non_alphanumeric': 0.10172075800479198, 'fraction_numerical': 0.013722500544543673, 'mean_word_length': 5.188679245283019, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '420540', 'license': 'MIT', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1357, 'n_words': 440, 'path': 'module1/NB_LS_DS_231_assignment.ipynb', 'repo_name': 'nikibhatt/DS-Unit-2-Applied-Modeling', 'size': '4591'}