<jupyter_start><jupyter_text>EDA, Cleaning, Train Test Split<jupyter_code>df = pd.read_csv('train.csv')
df.head()
df = df.drop(['bed_type', 'city', 'first_review', 'host_has_profile_pic', 'host_identity_verified', 
              'host_response_rate', 'last_review', 'neighbourhood', 'thumbnail_url', 'host_since', 'id', 'latitude', 
              'longitude', 'name', 'number_of_reviews', 'review_scores_rating', 'amenities', 'instant_bookable', 
              'cleaning_fee', 'zipcode'], axis=1)
print(df.shape)
df.head()
df.dropna(inplace=True)
df.shape
X = df.drop('log_price', axis=1)
y = df.log_price
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=.2, random_state=23)<jupyter_output><empty_output><jupyter_text>Transforming text and data into a sparse matrix so it can be analyzed with model<jupyter_code>X_train = X_train.copy()
X_train['description'] = X_train['description'].str.lower()

X_val = X_val.copy()
X_val['description'] = X_val['description'].str.lower()
X_train['description'] = X_train['description'].replace('[^a-zA-Z0-9]', ' ', regex = True)
X_val['description'] = X_val['description'].replace('[^a-zA-Z0-9]', ' ', regex = True)
tf = TfidfVectorizer()
## Make sure to only transform validation/testing data as the transformer has already been fit
X_train_tfidf = tf.fit_transform(X_train['description'])
X_val_tfidf = tf.transform(X_val['description'])
enc = DictVectorizer()
X_train_tfidf
X_train_categ.toarray()
## Make sure to only transform validation/testing data as the transformer has already been fit
X_train_categ = enc.fit_transform(X_train[['property_type', 'room_type', 'accommodates', 'bathrooms', 'cancellation_policy', 'bedrooms', 'beds']].to_dict('records'))
X_val_categ = enc.transform(X_val[['property_type', 'room_type', 'accommodates', 'bathrooms', 'cancellation_policy', 'bedrooms', 'beds']].to_dict('records'))
# Using hstack to concat the 2 arrays
X_trained = hstack([X_train_tfidf,X_train_categ])
X_vals = hstack([X_val_tfidf, X_val_categ])
# Using ridge regression model
clf = Ridge(alpha=1.0, random_state=23)
clf.fit(X_trained, y_train)
rslt = clf.predict(X_vals)
rslt
# Adding predictions of validation data to DF and transforming price from log price to a dollar amount
X_val['pred_price'] = rslt
X_val['pred_price'] = np.exp(X_val['pred_price'])
X_val['actual_price'] = y_val
X_val['actual_price'] = np.exp(X_val['actual_price'])
X_val.head()
MSE(rslt,y_val)
clf.score(X_vals, y_val)
X_val.head(15)
X_val.tail(15)<jupyter_output><empty_output><jupyter_text>Testing different ideas until I figured out one that worked which is above<jupyter_code>nlp = spacy.load("en_core_web_lg")
def tokenize(document):
    
    doc = nlp(document)
    
    return [token.lemma_.strip() for token in doc if (token.is_stop != True) and (token.is_punct != True)]
tf = TfidfVectorizer(analyzer='word',
                     ngram_range=(1, 2),
                     min_df=1,
                     max_df=.99,
                     stop_words='english',
                     tokenizer=tokenize)

tfidf_matrix = tf.fit_transform(df['description'])
tfidf_matrix.shape
dtm = pd.DataFrame(tfidf_matrix.todense(), columns=tf.get_feature_names())
dtm.tail()
df.reset_index(inplace=True)
df.drop('index', axis=1)
def reduceCardinality(series):
  top10 = series.value_counts()[:10].index
  return series.apply(lambda x: 'Other' if x not in top10 else x)
df['property_type'] = reduceCardinality(df['property_type'])
df['property_type'].value_counts()
new_df = pd.concat([df, dtm], axis=1)
new_df = new_df.copy()
X = new_df.drop(['log_price', 'description', 'index'], axis=1)
y = new_df.log_price
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=.2, random_state=23)
print(X_train.shape)
print(X_val.shape)
print(y_train.shape)
print(y_val.shape)
clf = Ridge()
ord = OrdinalEncoder()
ohe = OneHotEncoder()
X_train.head()
pipe = Pipeline([
                ('encode',OneHotEncoder()),
                #('impute', SimpleImputer()),
                #('scale',MinMaxScaler()),
                ('model',Ridge(random_state=41))
                ])
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_val)
model = Ridge(solver="sag", fit_intercept=True, random_state=369)
pipe = Pipeline([
                ('encode',OneHotEncoder()),
                #('impute', SimpleImputer()),
                #('scale',MinMaxScaler()),
                ('model',model)
                ])
pipe.fit(X_train, y_train)
y_pred = pipe.predict(X_val)
X_dummies = csr_matrix(pd.get_dummies(df[['property_type', 'room_type', 'cancellation_policy']], sparse=True).values)
df.dtypes
X_dummies
csr_df = hstack((X_dummies, tfidf_matrix)).tocsr()
X1 = csr_df
X1
model = Ridge(solver="saga", fit_intercept=False, random_state=23)
model.fit(X1, y)
X1_train, X1_val, y1_train, y1_val = train_test_split(X1, y, test_size=.2, random_state=23)
X1_train.reshape(-1,1)
model.fit(X1_train, y1_train)
y_pred = model.predict(X_val)
x2 = pd.DataFrame(X1.todense())
x2.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15332401676981433, 'fraction_numerical': 0.01217807945697744, 'mean_word_length': 5.036144578313253, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '662722', 'license': 'MIT', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1831, 'n_words': 341, 'path': 'BW_testing.ipynb', 'repo_name': 'Build-Week-Airbnb-6/data-science', 'size': '5009'}