<jupyter_start><jupyter_text>Feature Engineering<jupyter_code># moscow additional data
driver = ogr.GetDriverByName("ESRI Shapefile")
shp = driver.Open("data/administrative-divisions-of-moscow/moscow_adm.shp")
layer = shp.GetLayer()
# coordinates have different sizes in list
shape_json = json.loads(layer.GetFeature(0).ExportToJson())
len_list = len(np.array(shape_json["geometry"]["coordinates"]).shape)

shape_json1 = json.loads(layer.GetFeature(1).ExportToJson())
len_list1 = len(np.array(shape_json1["geometry"]["coordinates"]).shape)
len_list, len_list1
# sub_area
okrug = []
sub_area = []
coordinates = []
for i in range(layer.GetFeatureCount()):
    sub_area.append(layer.GetFeature(i).GetField("RAION"))
    okrug.append(layer.GetFeature(i).GetField("OKRUGS"))
    shape_json = json.loads(layer.GetFeature(i).ExportToJson())
    if len(np.array(shape_json["geometry"]["coordinates"]).shape) == 2:
        coordinates.append(np.mean(np.array(shape_json["geometry"]["coordinates"][0][0]), axis=0))
    else:
        coordinates.append(np.mean(np.array(shape_json["geometry"]["coordinates"][0]), axis=0))<jupyter_output><empty_output><jupyter_text>Train<jupyter_code># timestamp
"""
train["year"] = train.timestamp.apply(lambda x: x.year)
train["month"] = train.timestamp.apply(lambda x: x.month)
train["day"] = train.timestamp.apply(lambda x: x.day)
train["week"] = train.timestamp.apply(lambda x: x.week)
train["week_day"] = train.timestamp.apply(lambda x: x.dayofweek)
"""
# to-do add season for housing prices
# house
train["diff_max_floor"] = train.max_floor - train.floor
train["perc_max_floor"] = train.floor / train.max_floor
train["diff_full_life_sq"] = train.full_sq - train.life_sq
train["perc_full_life_sq"] = train.life_sq / train.full_sq
train["diff_full_kitch_sq"] = train.full_sq - train.kitch_sq
train["perc_full_kitch_sq"] = train.kitch_sq / train.full_sq
train["diff_full_life_kitch_sq"] = train.full_sq - train.life_sq - train.kitch_sq
train["perc_full_extra_sq"] = (train.life_sq + train.kitch_sq) / train.full_sq
train["sum_life_kitch"] = train.life_sq + train.kitch_sq
train["roomsize"] = (train.life_sq - train.kitch_sq) / train.num_room
train["age_at_sale"] = (train.timestamp.apply(lambda x: x.year) - train.build_year).apply(lambda x: x if x < 1500 else None)
# school 
train["perc_preschool"] = train.children_preschool / train.preschool_quota
train["perc_school"] = train.children_school / train.school_quota
# sub-area

kremlin = (37.617499, 55.752023) # long / # lat
moscow_uni = (37.528798, 55.704098)

moscow_df = pd.DataFrame({"sub_area": sub_area, "okrug": okrug, "coordinates": coordinates})
moscow_df["distance_from_kremlin"] = moscow_df.coordinates.apply(lambda x: haversine(x[0], x[1], kremlin[0], kremlin[1]))
moscow_df["distance_from_moscow_uni"] = moscow_df.coordinates.apply(lambda x: haversine(x[0], x[1], moscow_uni[0], moscow_uni[1]))
                                                                                        
moscow_df.drop("coordinates", axis=1, inplace=True)
train = pd.merge(train, moscow_df, how="left", on="sub_area")

# to-do maybe add distance to financial district
# reset index
train.set_index("id", inplace=True)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code># timestamp
"""
test["year"] = test.timestamp.apply(lambda x: x.year)
test["month"] = test.timestamp.apply(lambda x: x.month)
test["day"] = test.timestamp.apply(lambda x: x.day)
test["week"] = test.timestamp.apply(lambda x: x.week)
test["week_day"] = test.timestamp.apply(lambda x: x.dayofweek)
"""
# house
test["diff_max_floor"] = test.max_floor - test.floor
test["perc_max_floor"] = test.floor / test.max_floor
test["diff_full_life_sq"] = test.full_sq - test.life_sq
test["perc_full_life_sq"] = test.life_sq / test.full_sq
test["diff_full_kitch_sq"] = test.full_sq - test.kitch_sq
test["perc_full_kitch_sq"] = test.kitch_sq / test.full_sq
test["diff_full_life_kitch_sq"] = test.full_sq - test.life_sq - test.kitch_sq
test["perc_full_extra_sq"] = (test.life_sq + test.kitch_sq) / test.full_sq
test["sum_life_kitch"] = test.life_sq + test.kitch_sq
test["roomsize"] = (test.life_sq - test.kitch_sq) / test.num_room
test["age_at_sale"] = (test.timestamp.apply(lambda x: x.year) - test.build_year).apply(lambda x: x if x < 1500 else None)
# school 
test["perc_preschool"] = test.children_preschool / test.preschool_quota
test["perc_school"] = test.children_school / test.school_quota
# sub-area

kremlin = (37.617499, 55.752023) # long / # lat
moscow_uni = (37.528798, 55.704098)

moscow_df = pd.DataFrame({"sub_area": sub_area, "okrug": okrug, "coordinates": coordinates})
moscow_df["distance_from_kremlin"] = moscow_df.coordinates.apply(lambda x: haversine(x[0], x[1], kremlin[0], kremlin[1]))
moscow_df["distance_from_moscow_uni"] = moscow_df.coordinates.apply(lambda x: haversine(x[0], x[1], moscow_uni[0], moscow_uni[1]))
                                                                                        
moscow_df.drop("coordinates", axis=1, inplace=True)
test = pd.merge(test, moscow_df, how="left", on="sub_area")

# to-do maybe add distance to financial district
# reset index
test.set_index("id", inplace=True)<jupyter_output><empty_output><jupyter_text>Transformation<jupyter_code>y_train = train["price_doc"]
x_train = train.drop(["timestamp", "price_doc"], axis=1)
# transform non-numerical variables
for c in x_train.columns:
    if x_train[c].dtype == "object":
        lbl = preprocessing.LabelEncoder()
        lbl.fit(list(x_train[c].values)) 
        x_train[c] = lbl.transform(list(x_train[c].values))
# replace missing values with mean values
"""
for c in x_train.columns:
    x_train[c].fillna(x_train[c].mean(), inplace=True)
"""
x_test = test.drop(["timestamp"], axis=1)
# transform non-numerical variables
for c in x_test.columns:
    if x_test[c].dtype == "object":
        lbl = preprocessing.LabelEncoder()
        lbl.fit(list(x_test[c].values)) 
        x_test[c] = lbl.transform(list(x_test[c].values))
# replace missing values with mean values
"""
for c in x_test.columns:
    x_test[c].fillna(x_test[c].mean(), inplace=True)
"""
xgb_params = {
    "eta": 0.05,
    "max_depth": 5,
    "subsample": 0.7,
    "colsample_bytree": 0.7,
    "objective": "reg:linear",
    "eval_metric": "rmse",
    "silent": 1,
    "seed":42
}
dtrain = xgb.DMatrix(x_train, y_train)
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=20, 
                   verbose_eval=50, show_stdv=False)
cv_output[["train-rmse-mean", "test-rmse-mean"]].plot()
pass
num_boost_rounds = len(cv_output)
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_rounds)
xgb.plot_importance(model, max_num_features=20)
pass
pred = model.predict(dtrain)
error = np.mean(abs(pred - y_train)) / len(y_train)
n = 1 / len(y_train)
rmsle = np.sqrt(1/n * np.sum(np.power(np.log(pred + 1) - np.log(y_train.values + 1), 2)))
print("RMSLE: {rmsle}, Error: {error}".format(rmsle=rmsle, error=error))
dtest = xgb.DMatrix(x_test)
y_predict = model.predict(dtest)
output = pd.DataFrame({"id": x_test.index, "price_doc": y_predict})
output.to_csv("submissions_add_features.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15926748057713652, 'fraction_numerical': 0.018174250832408437, 'mean_word_length': 5.119694397283531, 'pattern_counts': {'":': 18, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '323332', 'license': 'MIT', 'n_tokens_mistral': 2753, 'n_tokens_neox': 2679, 'n_words': 433, 'path': 'Kaggle/Sberbank Russian Housing Market/additional_feature_engineering.ipynb', 'repo_name': 'datitran/Krimskrams', 'size': '7208'}