<jupyter_start><jupyter_text>Загрузим данные<jupyter_code>from google.colab import drive 
drive.mount('/content/gdrive')
df = pd.read_csv('gdrive/My Drive/data/dz3/train.csv')#
df_test = pd.read_csv('gdrive/My Drive/data/dz3/test.csv')#
df.head()
df_macro = pd.read_csv('gdrive/My Drive/data/dz3/macro.csv')
df = df.merge(df_macro, how = 'inner',on='timestamp')
df_test = df_test.merge(df_macro, how = 'inner',on='timestamp')
df.head()<jupyter_output><empty_output><jupyter_text>Посмотрим на количество пропусков данных по признакам<jupyter_code>def print_count_nan(df):
  row_count = df.shape[0]
  for column in df.columns:
    nan_count = df[column].isna().sum()
    if  nan_count > 0:
      print(column,'  ',nan_count,' (', "{0:.3f}".format(nan_count/row_count), '%)')
    else:
      print(column,' нет')

print_count_nan(df)<jupyter_output>id  нет
timestamp  нет
full_sq  нет
life_sq    6383  ( 0.209 %)
floor    167  ( 0.005 %)
max_floor    9572  ( 0.314 %)
material    9572  ( 0.314 %)
build_year    13605  ( 0.446 %)
num_room    9572  ( 0.314 %)
kitch_sq    9572  ( 0.314 %)
state    13559  ( 0.445 %)
product_type  нет
sub_area  нет
area_m  нет
raion_popul  нет
green_zone_part  нет
indust_part  нет
children_preschool  нет
preschool_quota    6688  ( 0.219 %)
preschool_education_centers_raion  нет
children_school  нет
school_quota    6685  ( 0.219 %)
school_education_centers_raion  нет
school_education_centers_top_20_raion  нет
hospital_beds_raion    14441  ( 0.474 %)
healthcare_centers_raion  нет
university_top_20_raion  нет
sport_objects_raion  нет
additional_education_raion  нет
culture_objects_top_25  нет
culture_objects_top_25_raion  нет
shopping_centers_raion  нет
office_raion  нет
thermal_power_plant_raion  нет
incineration_raion  нет
oil_chemistry_raion  нет
radiation_raion  нет
railroad_terminal_raion  нет
big_marke[...]<jupyter_text>Удалим выбросы<jupyter_code>df_loc = df.loc[(df['full_sq']>600)]
df.drop(axis=0,index=df_loc.index,inplace=True)<jupyter_output><empty_output><jupyter_text>Заполним пропуски<jupyter_code>df_X=df.fillna(df.mean())
df_y = pd.DataFrame(df['price_doc'])
df_X.drop(columns = ['id', 'timestamp','price_doc'],inplace = True)
df_test1=df_test.drop(columns = ['id', 'timestamp'])
df_X.head()
!pip install -q catboost shap
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

from catboost import CatBoostRegressor
import catboost
import matplotlib.pyplot as plt
plt.figure(figsize=(8,6))
df2=df_X[df_X['life_sq']<1000]
plt.scatter(df2['full_sq'], df2['life_sq'])
plt.xlabel('full', fontsize=12)
plt.ylabel('life', fontsize=12)
plt.show()
plt.figure(figsize=(8,6))
df2=df_X[df_X['life_sq']<1000]
plt.scatter(df['price_doc'], df['full_sq'])
plt.xlabel('price', fontsize=12)
plt.ylabel('full', fontsize=12)
plt.show()<jupyter_output><empty_output><jupyter_text>Найдем категориальные признаки<jupyter_code>from numpy.core.numerictypes import obj2sctype
categorical_column = []
yes_no_column = []
numeric_column = []
float_column = []
for column in df_X.columns:
  if df_X[column].dtypes == object:
    #check yes-no column
    flag = False
    if df_X[column].isna().sum() == 0:
      flag = True
      for value in df_X[column].value_counts().index:
        if not value in ['yes','no']:
          flag = False
    if flag:
      yes_no_column.append(column) 
    else:
      categorical_column.append(column)
  else:
    if df_X[column].dtypes == float:
      float_column.append(column)
    else:
      numeric_column.append(column)
#print(numeric_column)
print(len(categorical_column))
print(categorical_column)
print(len(float_column))
print(float_column)
print(len(yes_no_column))
print(yes_no_column)

"""print(len(yes_no_column))
print(yes_no_column)
for column in yes_no_column:
  print('-------------------------------')
  print(column)
  print(df_X[column].value_counts())"""
yes_no_column
for column in yes_no_column:
  df_X.loc[(df_X[column] == 'no'),column] = '0'
  df_X.loc[(df_X[column] == 'yes'),column] = '1'
  df_X[column].astype(int)

  df_test.loc[(df_test[column] == 'no'),column] = '0'
  df_test.loc[(df_test[column] == 'yes'),column] = '1'
  df_test[column].astype(int)  

"""for column in float_column:
  df_X[column]=str(df_X[column])
  df_test[column]=str(df_test[column])"""
#X_train_us, X_test_us, y_train_us, y_test_us = train_test_split(df_X, df_y, test_size=0.2, random_state=15, shuffle=True)

"""train_pool = catboost.Pool(
    X_train_us,
    label=y_train_us,
    cat_features=categorical_column+float_column
)

test_pool = catboost.Pool(
    X_test_us,
    label=y_test_us,
    cat_features=categorical_column+float_column
)

test2_pool = catboost.Pool(
    df_test,
    label=None,
    cat_features=categorical_column+float_column
)
val_pool = catboost.Pool(
    val_X,
    label=val_y,
    cat_features=cat_features
)"""
df_X.shape,df_test1.shape
"""df_X['testOrtrain']=1
df_test1['testOrtrain']=0
df_all = df_X.append( df_test1, ignore_index=True, verify_integrity=False, sort=False)
df_all.shape"""<jupyter_output><empty_output><jupyter_text>Создание числовых признаков через OneHotEncoder<jupyter_code>from sklearn.preprocessing import OneHotEncoder
def cat_one_encoding(df):
  cat_onehot = OneHotEncoder(handle_unknown='ignore', sparse=False)
  ans = cat_onehot.fit_transform(df[categorical_column])
  df_results = pd.DataFrame(ans,columns=cat_onehot.get_feature_names_out(
      categorical_column),
      index=df.index)

  df_results=df[numeric_column+float_column+yes_no_column].join(df_results)
  #df_results=df_results.join(df[])
  #df_results=df_results.join(df[])
  """df_results=df_results.join(df['testOrtrain'])
  
  df_test2 = df_results[df_results['testOrtrain']==0].copy()
  df_test2.drop(columns = ['testOrtrain'],inplace = True)
  df_X2 = df_results[df_results['testOrtrain']==1].copy()
  df_X2.drop(columns = ['testOrtrain'],inplace = True)"""
  return df_results

df_X2 = cat_one_encoding(df_X)
df_test2 = cat_one_encoding(df_test)

X_train_us, X_test_us, y_train_us, y_test_us = train_test_split(df_X2, df_y, test_size=0.2, random_state=15, shuffle=True)

df_test2.shape,df_X2.shape
#X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=1)<jupyter_output><empty_output><jupyter_text>Обучение модели<jupyter_code>"""cbr = CatBoostRegressor(iterations=5000,learning_rate=0.011, depth=5, verbose=0, eval_metric='Quantile', l2_leaf_reg=3, rsm=0.09, random_seed=12)
cbr.fit(X_train_us, y_train_us, plot=True, baseline=[1000] * len(X_train_us))
y_cbr_pr_us = cbr.predict(X_test_us)
print('RMSE - ', np.sqrt(mean_squared_error(np.log(y_cbr_pr_us), np.log(y_test_us))))"""
"""cbr = CatBoostRegressor(iterations=5000,learning_rate=0.011, depth=5, verbose=0, eval_metric='Quantile', l2_leaf_reg=3, rsm=0.09, random_seed=12)
cbr.fit(train_pool, plot=True, baseline=[1000] * len(X_train_us))
y_cbr_pr_us = cbr.predict(X_test_us)
print('RMSE - ', np.sqrt(mean_squared_error(np.log(y_cbr_pr_us), np.log(y_test_us))))
price = cbr.predict(df_test2)
"""

"""from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold


# define dataset

model = GradientBoostingRegressor()
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)

n_scores = cross_val_score(model, df_X2, df_y, scoring='roc_auc', cv=cv, n_jobs=-1, error_score='raise')
print('roc_auc: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))
# fit the model on the whole dataset
#model = GradientBoostingRegressor()
#model.fit(df_X2, df_y)
price = model.predict(df_test2)"""
from catboost import Pool, cv
params = {"iterations": 100,
          "depth": 2,
          "loss_function": "RMSE",
          "verbose": False}
cv_dataset = Pool(data=df_X2,
                  label=df_y)
cv_test = Pool(data=df_test2,label = None)
scores = cv(cv_dataset,
            params,
            fold_count=2, 
            plot="True")
from catboost import CatBoostRegressor
cat = CatBoostRegressor()
cat.fit(df_X2,df_y,verbose=False, plot=True)
price= cat.predict(cv_test)<jupyter_output><empty_output><jupyter_text>Сохранение результата<jupyter_code>df_res=pd.DataFrame()
df_res['id']=df_test['id']
df_res['price_doc']=price.round(2)
df_res.to_csv('submission.csv',sep=',', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13920285544318858, 'fraction_numerical': 0.027364663890541343, 'mean_word_length': 6.028428093645485, 'pattern_counts': {'":': 4, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '364147', 'license': 'Apache-2.0', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3380, 'n_words': 504, 'path': 'ROZA_2_DZ3.ipynb', 'repo_name': 'RozaT1011/First', 'size': '8405'}