<jupyter_start><jupyter_text>Problem definition Apply regression models to predict Airbnb prices  Load the data<jupyter_code>df_train = pd.read_csv('data/train.csv')
df_test = pd.read_csv('data/test.csv')
print(df_train.shape, df_test.shape)<jupyter_output>(51000, 29) (23111, 28)<jupyter_text>Feature Engineering<jupyter_code>print(df_train.columns)
df_train.head()

#Check the bedrooms based on Bed Type
df_train['bedrooms'] = np.where(df_train['bed_type'].str.contains('Real Bed') & df_train['bedrooms'].isnull(), 1, 0)
df_train['beds'] = np.where(df_train['bed_type'].str.contains('Real Bed') & df_train['beds'].isnull(), 1, 0)

df_test['bedrooms'] = np.where(df_test['bed_type'].str.contains('Real Bed') & df_test['bedrooms'].isnull(), 1, 0)
df_test['beds'] = np.where(df_test['bed_type'].str.contains('Real Bed') & df_test['beds'].isnull(), 1, 0)

#df_train['zipcode'] = df_train['zipcode'].str.strip()
#df_train['zipcode'] = df_train['zipcode'].str.replace(' ','').astype(str)
#df_train['zipcode'] = df_train['zipcode'].str.replace('.0','').astype(str)
#df_train['zipcode'] = df_train['zipcode'].str.replace('\r','').astype(str)
#df_train['zipcode'] = df_train['zipcode'].str.replace('\n','').astype(str)

#df_test['zipcode'] = df_test['zipcode'].str.strip()
#df_test['zipcode'] = df_test['zipcode'].str.replace(' ','').astype(str)
#df_test['zipcode'] = df_test['zipcode'].str.replace('.0','').astype(str)
#df_test['zipcode'] = df_test['zipcode'].str.replace('\r','').astype(str)
#df_test['zipcode'] = df_test['zipcode'].str.replace('\n','').astype(str)


## Populate blank fields
df_train['bathrooms'].fillna(1, inplace=True)

#########Added for validations
print(df_train.describe())
print(df_train.count())
# Verify group for any null values
print("")
print("(Number of Blank values)")
print("")
print(df_train.isnull().sum())


print("(---DF_TRAIN---)")
print(df_train.dtypes)
print(df_train.shape)
print(df_train.describe())
print('Corr:',df_train.corr())

print("(---DF_TEST---)")
print(df_test.dtypes)
print(df_test.shape)
print(df_test.describe())
print('Corr:',df_test.corr())


# Crashes at Prepare Submission
# ValueError: shapes (23111,22) and (30,1) not aligned: 22 (dim 1) != 30 (dim 0)
#Air Conditioning
#df_train['AC'] = np.where(df_train['amenities'].str.contains('Air cond'), 'Yes', 'No')
#df_ac = pd.get_dummies(df_train['AC'])
#df_train = pd.concat([df_train, df_ac], axis=1)


#Air Conditioning
#df_test['AC'] = np.where(df_test['amenities'].str.contains('Air cond'), 'Yes', 'No')
#df_ac = pd.get_dummies(df_test['AC'])
#df_test = pd.concat([df_test, df_ac], axis=1)


#TV
# Error at Prepare Submission
# ValueError: shapes (23111,22) and (30,1) not aligned: 22 (dim 1) != 30 (dim 0)
#df_train['TV'] = np.where(df_train['amenities'].str.contains('TV'), 'Yes', 'No')
#df_tv = pd.get_dummies(df_train['TV'])
#df_train = pd.concat([df_train, df_tv], axis=1)

#TV
#df_test['TV'] = np.where(df_test['amenities'].str.contains('TV'), 'Yes', 'No')
#df_tv = pd.get_dummies(df_test['TV'])
#df_test = pd.concat([df_test, df_tv], axis=1)


sns.set(rc={'figure.figsize':(10,7)})
correlation_matrix = df_train.corr().round(2)
# annot = True to print the values inside the square
sns.heatmap(data=correlation_matrix, annot=True)
plt.show()


#Keyerror not in index
#df_zipcode = pd.get_dummies(df_train['zipcode'])
#df_train = pd.concat([df_train, df_zipcode], axis=1)

df_room_type = pd.get_dummies(df_train['room_type'])
df_train = pd.concat([df_train, df_room_type], axis=1)

df_city = pd.get_dummies(df_train['city'])
df_train = pd.concat([df_train, df_city], axis=1)

#Keyerror not in index?
#df_neigh = pd.get_dummies(df_train['neighbourhood'])
#df_train = pd.concat([df_train, df_neigh], axis=1)


##############TEST

#df_zipcode = pd.get_dummies(df_test['zipcode'])
#df_test = pd.concat([df_test, df_zipcode], axis=1)

df_room_type = pd.get_dummies(df_test['room_type'])
df_test = pd.concat([df_test, df_room_type], axis=1)

df_city = pd.get_dummies(df_test['city'])
df_test = pd.concat([df_test, df_city], axis=1)

#df_neigh = pd.get_dummies(df_test['neighbourhood'])
#df_test = pd.concat([df_test, df_neigh], axis=1)


#Try to retrieve amenities
#df_kitchen = df_train.loc[df_train['amenities'].str.contains('Kitchen')]
#print(df_kitchen)
#df_train['review_scores_rating'] = np.where(df_train['number_of_reviews'] == 0,0)
#df_train['review_scores_rating'] = np.where(df_train['number_of_reviews'] == 0, 0, 1)

# IF Nmber of reviews is 0 Then 0% rating
df_train['review_scores_rating'] = np.where(df_train['number_of_reviews']==0, 0, df_train['review_scores_rating'])
df_test['review_scores_rating'] = np.where(df_test['number_of_reviews']==0, 0, df_test['review_scores_rating'])

df_train['review_scores_rating'].fillna(1, inplace=True)
df_test['review_scores_rating'].fillna(1, inplace=True)


#print(df_train['review_scores_rating'])
#print(df_train['review_scores_rating'].isnull().sum())
#df_train['review_scores_rating'].fillna(1, inplace=True)
#f['Amenities_2'] = np.where(df_train['amenities'].str.contains('Kitchen'),1)
#If kitchen exists retrieve
# df_train['Amenities_2'] = np.where(df_train['amenities'].str.contains('Kitchen'), 1, 0)
# print(df_train['Amenities_2'])
# print(df_train.describe())

#Kitchen
df_train['Kitchen'] = np.where(df_train['amenities'].str.contains('Kitchen'), 'Yes', 'No')
df_kitchen = pd.get_dummies(df_train['Kitchen'])
df_train = pd.concat([df_train, df_kitchen], axis=1)

#Air Conditioning
df_train['AC'] = np.where(df_train['amenities'].str.contains('Air cond'), 'Yes', 'No')
df_ac = pd.get_dummies(df_train['AC'])
df_train = pd.concat([df_train, df_ac], axis=1)


###### TEST

#Kitchen
df_test['Kitchen'] = np.where(df_test['amenities'].str.contains('Kitchen'), 'Yes', 'No')
df_kitchen = pd.get_dummies(df_test['Kitchen'])
df_test = pd.concat([df_test, df_kitchen], axis=1)

#Air Conditioning
df_test['AC'] = np.where(df_test['amenities'].str.contains('Air cond'), 'Yes', 'No')
df_ac = pd.get_dummies(df_test['AC'])
df_test = pd.concat([df_test, df_ac], axis=1)


#print(df_train.describe())
#print(df_test.describe())

#df_train.head()
#print(df_train.isnull().sum())

#df_test.head()
#print(df_test.isnull().sum())
# select the columns
 
X_columns = ['accommodates', 'bathrooms', 'bedrooms', 'beds']+ list(df_room_type.columns) + list(df_city.columns) + list(df_kitchen.columns) #+ list(df_tv.columns)#+ list(df_ac.columns)#+ list(df_neigh.columns)#+ list(df_zipcode.columns)
y_column = ['log_price']
# handle missing values
df_train = df_train[X_columns + y_column]
print(df_train.shape)
df_train = df_train.fillna(0.0) # probably not a good idea for 'review_scores_rating'
print(df_train.shape)<jupyter_output>(51000, 18)
(51000, 18)<jupyter_text>Model Training<jupyter_code># split the data using sklearn

threshold = 0.8
X = df_train[X_columns]
y = df_train[y_column]


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1.0-threshold, shuffle=True)

print('X_train', X_train.shape)
print('y_train', y_train.shape)
print('X_test', X_test.shape)
print('y_test', y_test.shape)


# train a linear regression
model = LinearRegression()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)<jupyter_output><empty_output><jupyter_text>Model Evaluation<jupyter_code>rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE', round(rmse, 2))
plt.scatter(y_test, y_pred, alpha=0.3)
plt.plot(range(0,10), range(0,10), '--r', alpha=0.3, label='Line1')
plt.title('Linear Regression')
plt.xlabel('True Value')
plt.ylabel('Predict Value')
plt.show()<jupyter_output>RMSE 0.48<jupyter_text>Prepare submission<jupyter_code>df_prediction = df_test[X_columns].fillna(0.0)
df_test['log_price'] = model.predict(df_prediction)
df_test[['id', 'log_price']]
df_test[['id', 'log_price']].to_csv('submission_linear_regression.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19892952720785012, 'fraction_numerical': 0.01707658977953358, 'mean_word_length': 8.26564344746163, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '103349', 'license': 'MIT', 'n_tokens_mistral': 3233, 'n_tokens_neox': 3112, 'n_words': 475, 'path': 'kaggle_submission - 20190516.ipynb', 'repo_name': 'Clercy/cebd1260kaggle1', 'size': '7847'}