<jupyter_start><jupyter_text>Plot correlations<jupyter_code>data = df[:150]
sns.relplot(x="price", y="engV", data=data).set_xticklabels(rotation=30)
sns.relplot(x="mileage", y="price", data=data)
sns.relplot(x="year", y="mileage", hue="car", data=data).set_xticklabels(rotation=30);
sns.relplot(x="year", y="price", kind="line", data=data).set_xticklabels(rotation=30);
data.hist(figsize=(16, 20), bins=50, xlabelsize=8, ylabelsize=8)
data = df.dropna()
corr = data.corr()
cmap = sns.diverging_palette(230, 20, as_cmap=True)
sns.heatmap(corr, cmap=cmap)
for c in df.columns:
    if df[c].dtype == 'object':
        lbl = preprocessing.LabelEncoder()
        lbl.fit(list(df[c].values)) 
        df[c] = lbl.transform(list(df[c].values))
corr = df.corr()
sns.heatmap(corr, cmap=cmap)
data[data.engV > 9]
data[data.engV < 0.8]
sns.boxplot(data.price)
data[data['price'] > 200000]
data.describe()
df.price.plot()
cheap = data[data['price'] < 500]
outliers = cheap['model'].value_counts()
outliers = outliers[outliers <= 1].index
cheap = cheap.loc[~cheap['model'].isin(outliers)]

fig, ax = plt.subplots(figsize=(15,6))
plt.xticks(rotation=30)
chart = sns.countplot(x='car', data=cheap)
# annotation here
for p in chart.patches:
             chart.annotate("%.0f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()),
                 ha='center', va='center', fontsize=10, color='black', xytext=(0, 5),
                 textcoords='offset points')<jupyter_output><empty_output><jupyter_text>Можно заметить, что дешёвый ВАЗ, АЗЛК и ЗАЗ ещё могут стоить ниже 500$ поэтому посмотрим машины, которые стоят 0:<jupyter_code>cheap = data[data['price'] == 0]
outliers = cheap['model'].value_counts()
outliers = outliers[outliers <= 1].index
cheap = cheap.loc[~cheap['model'].isin(outliers)]

fig, ax = plt.subplots(figsize=(15,6))
plt.xticks(rotation=30)
chart = sns.countplot(x='car', data=cheap)
# annotation here
for p in chart.patches:
             chart.annotate("%.0f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()),
                 ha='center', va='center', fontsize=10, color='black', xytext=(0, 5),
                 textcoords='offset points')<jupyter_output><empty_output><jupyter_text>Make the new columns with mean price for cars<jupyter_code>df = pd.read_csv('data/car_ad.csv')
df.shape
df = df.loc[(df.price > 300) & (df.price < df.price.quantile(0.99))]
df.groupby("car").mean().sort_values(by='price', ascending=False).price
df['mean_model_price'] = df.groupby("car").price.transform(np.mean)
df
from sklearn.model_selection import train_test_split

data_train, data_val = train_test_split(df, train_size=0.8, random_state=27)
data_train.to_csv("data/train.csv", index=False)
data_val.to_csv("data/val.csv", index=False)<jupyter_output><empty_output><jupyter_text>Это уже либо выбросы, либо новые машины, цена на которые ещё не установлена. Это можно проверить посмотрев на пробег и год.<jupyter_code>cheap = data[(data['price'] < 300) & (data['mileage'] == 0) & (data['year'] == 2016)].car
cheap.value_counts()<jupyter_output><empty_output><jupyter_text>Что ж, можно сказать, что эти машины являются новыми, так как пробега нет, год последний (датасет до 2016 года), а цена 0. Или же просто некорректно выставлены эти признаки. Но непонятно почему тогда они попали в этот датасет с бу машинами, разве что автосалоны как рекламу выложили объявления. Так как целью является определение оценочной стоимости по рынку бу машин в Украине, то правильным решением станет отбросить машины с ценой ниже 300 и пробегом 0 миль.<jupyter_code>for col in df.columns:
    if df[col].dtype == 'object':
        print(col, ' ==> ', len(df[col].unique()))<jupyter_output><empty_output><jupyter_text>Ок, значит car & model будет LabelEncoding, остальноё OneHotEncoding. upd: отказался, так как особого профита нет. Теперь всё LabelEncoder  Write function and start train models<jupyter_code># Пришлось упростить подготовку данных, чтоб сделать простую спецификацию для автоматической проверки проекта

def preprocess(df, label=True):
    # fill nans
    df['drive'] = df['drive'].fillna(df['drive'].mode()[0])
    df['engV'] = df['engV'].fillna(df['engV'].median())
    
    # remove outliers
    df = df[df['mileage'].between(df['mileage'].quantile(0.05) , df['mileage'].quantile(0.95))]
    df = df[df.engV.between(df['engV'].quantile(0), df['engV'].quantile(0.99))]
    
    # Label Encoder
    if label:
        for c in df.columns:
            if df[c].dtype == 'object':
                lbl = preprocessing.LabelEncoder()
                lbl.fit(list(df[c].values)) 
                df[c] = lbl.transform(list(df[c].values))
    
    return df
def load_data(splitted=True, labeled=True):
    '''
    df - dataset
    splitted:
        if True, then get splitted data train.csv & val.csv
        else take all dataset
    '''
    if splitted:
        df_train = pd.read_csv('data/train.csv')
        df_test = pd.read_csv('data/val.csv')
        df_train = preprocess(df_train, labeled)
        df_test = preprocess(df_test, labeled)

        X_train = df_train.drop(columns=['price'], axis=0)
        y_train = df_train['price']

        X_test = df_test.drop(columns=['price'], axis=0)
        y_test = df_test['price']
        return X_train, X_test, y_train, y_test
    else:
        df = pd.read_csv('data/car_ad.csv')
        df = preprocess(df, labeled)
        X = df.drop(columns=['price'], axis=0)
        y = df['price']
        X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, random_state=27)
        return X_train, X_test, y_train, y_test
# Metric for regression
from sklearn.metrics import make_scorer

def mean_absolute_percentage_error(y_true, y_pred): 
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

mape = make_scorer(mean_absolute_percentage_error, greater_is_better=False)
X_train, X_test, y_train, y_test = load_data()
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score


classifiers = [
    KNeighborsRegressor(4),
    SVR(),
    DecisionTreeRegressor(),
    RandomForestRegressor(),
    AdaBoostRegressor(),
    GradientBoostingRegressor(),
    XGBRegressor(),
    LinearRegression()
]

log_cols = ["Classifier", "Accuracy"]
log = pd.DataFrame(columns=log_cols)

acc_dict = {}

for model in classifiers:
    name = model.__class__.__name__
    
    # Bundle preprocessing and modeling code in a pipeline
    my_pipeline = Pipeline(steps=[('model', model)])

    acc = -1 * cross_val_score(my_pipeline, X_train, y_train,
                                  cv=5,
                                  scoring=mape).mean()
    
    print(model, ', MAPE: ', acc)
    
    if name in acc_dict:
        acc_dict[name] += acc
    else:
        acc_dict[name] = acc

for clf in acc_dict:
    log_entry = pd.DataFrame([[clf, acc_dict[clf]]], columns = log_cols)
    log = log.append(log_entry)

plt.xlabel('Accuracy')
plt.title("Classifier Accuracy (less is better)")

sns.set_color_codes("muted")
sns.barplot(x = "Accuracy", y = 'Classifier', data = log, color = "b")<jupyter_output>KNeighborsRegressor(n_neighbors=4) , MAPE:  44.22267320278519
SVR() , MAPE:  93.8002722036617
DecisionTreeRegressor() , MAPE:  25.06415897738794
RandomForestRegressor() , MAPE:  19.258298573571913
AdaBoostRegressor() , MAPE:  130.56325446452018
GradientBoostingRegressor() , MAPE:  25.312440735635693
XGBRegressor(base_score=None, booster=None, colsample_bylevel=None,
             colsample_bynode=None, colsample_bytree=None, gamma=None,
             gpu_id=None, importance_type='gain', interaction_constraints=None,
             learning_rate=None, max_delta_step=None, max_depth=None,
             min_child_weight=None, missing=nan, monotone_constraints=None,
             n_estimators=100, n_jobs=None, num_parallel_tree=None,
             random_state=None, reg_alpha=None, reg_lambda=None,
             scale_pos_weight=None, subsample=None, tree_method=None,
             validate_parameters=None, verbosity=None) , MAPE:  18.967459773905137
LinearRegression() , MAPE:  78.46732725167013<jupyter_text>Теперь, так как XGBRegressor и RandomForest показали себя с наилучшей стороны в базовой конфигурации, подберём наилучшие параметры методом GridSearchCV для этих моделей.<jupyter_code>from sklearn.model_selection import GridSearchCV
from xgboost import XGBRegressor

xgb_model = XGBRegressor()

parameters = {'nthread':[2],
              'objective':['reg:squarederror'],
              'learning_rate': [0.01, 0.03, 0.05], 
              'max_depth': [5,9,12],
              'min_child_weight': [1,5],
              'subsample': [0.75],
              'colsample_bytree': [1],
              'n_estimators': [1000],
              'eval_metric': ["mape"]}

clf = GridSearchCV(xgb_model, parameters, n_jobs=5, 
                   cv=5, 
                   scoring=mape,
                   verbose=2, refit=True)

clf.fit(X_train, y_train)
clf.best_score_
clf.best_params_<jupyter_output><empty_output><jupyter_text>Best params for XGBoostRegressor---```python{'colsample_bytree': 1, 'eval_metric': 'mape', 'learning_rate': 0.01, 'max_depth': 15, 'min_child_weight': 1, 'n_estimators': 500, 'nthread': 2, 'objective': 'reg:squarederror', 'subsample': 0.75} ``` ---  Dump model params<jupyter_code>import pickle

pickle.dump(clf, open("xgbr.pickle", "wb"))<jupyter_output><empty_output><jupyter_text>Grid search and train Random Forest model<jupyter_code>random_forest_model = RandomForestRegressor()

parameters = {
    'bootstrap': [True, False],
    'max_depth': [30, 60, 90, 120],
    'max_features': [3, 6, 9],
    'min_samples_leaf': [4, 5, 7],
    'min_samples_split': [8, 10, 12],
    'n_estimators': [100, 500, 1000]
}

rf_grid = GridSearchCV(random_forest_model, parameters, n_jobs=-1, 
                   cv=5, 
                   scoring=mape,
                   verbose=1)

rf_grid.fit(X_train, y_train)
rf_grid.best_score_
rf_grid.best_params_
rf_pred = rf_grid.predict(X_test)
print(mean_absolute_percentage_error(y_test, rf_pred))
pickle.dump(rf_grid, open("random_forest.pickle.dat", "wb"))<jupyter_output><empty_output><jupyter_text>Как видно, RandomForest дал неплохой результат на тестовой выборке MEPA:15.94%, но XGBRegressor обеспечил результатом MEPA:14.35%, поэтому будем использовать его. Вероятно можно было улучшить результат, использовав OHEncoder и добавив ещё какие-то признаки, но этого должно быть достаточно для поставленной задачи.  Try to log target function and train XGBRegressor with the best params<jupyter_code>best_params = {
     'colsample_bytree': [1],
     'eval_metric': ['mape'],
     'learning_rate': [0.01],
     'max_depth': [15],
     'min_child_weight': [1],
     'n_estimators': [500],
     'nthread': [2],
     'objective': ['reg:squarederror'],
     'subsample': [0.75]}

y_train_log, y_test_log = np.log(y_train), np.log(y_test)

xgb_log_model = XGBRegressor()

clf_log = GridSearchCV(xgb_log_model, best_params, n_jobs=5, 
                   cv=5, 
                   scoring=mape,
                   verbose=2, refit=True)

clf_log.fit(X_train, y_train_log)
pred_log = clf_log.predict(X_test)
pred_log
print(mean_absolute_percentage_error(np.exp(y_test_log), np.exp(pred_log)))<jupyter_output>14.527449112900086<jupyter_text>В логарифмировании ЦФ не оказалось смысла.  Experiments with the best model<jupyter_code>from sklearn.model_selection import GridSearchCV
from xgboost import XGBRegressor


best_params = {
     'colsample_bytree': [1],
     'eval_metric': ['mape'],
     'learning_rate': [0.01],
     'max_depth': [9],
     'min_child_weight': [1],
     'n_estimators': [1000],
     'nthread': [2],
     'objective': ['reg:squarederror'],
     'subsample': [0.75]}

xgb_model = XGBRegressor()

clf = GridSearchCV(xgb_model, best_params, n_jobs=5, 
                   cv=5, 
                   scoring=mape,
                   verbose=2, refit=True)

clf.fit(X_train, y_train)
clf.best_score_
pred = clf.predict(X_test)
print(mean_absolute_percentage_error(y_test, pred))
import pickle

pickle.dump(clf, open("models/xgbr.pickle", "wb"))
X_train<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1345061975209916, 'fraction_numerical': 0.030547780887644944, 'mean_word_length': 4.067260940032415, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '54194', 'license': 'Apache-2.0', 'n_tokens_mistral': 4923, 'n_tokens_neox': 4711, 'n_words': 901, 'path': '.ipynb_checkpoints/EDA, feature engineering and split data-checkpoint.ipynb', 'repo_name': 'HuginnM/UsedCarsUA', 'size': '12505'}