<jupyter_start><jupyter_text>添加特征文本特征  数据特征<jupyter_code># 文本特征
feature_cnt = 25
tfidf = TfidfVectorizer(max_features=feature_cnt)
items_df['item_name_len'] = items_df['item_name'].map(len)
items_df['item_name_wc'] = items_df['item_name'].map(lambda x: len(str(x).split(' ')))  # word count

txtFeatures = pd.DataFrame(tfidf.fit_transform(items_df['item_name']).toarray())
cols = txtFeatures.columns
for i in range(feature_cnt):
    items_df['item_name_tfidf_' + str(i)] = txtFeatures[cols[i]]
items_df.head()
# 对item_category 进行同样操作
#Text Features
feature_cnt = 25
tfidf = TfidfVectorizer(max_features=feature_cnt)
item_categories_df['item_category_name_len'] = item_categories_df['item_category_name'].map(len)  #Lenth of Item Category Description
item_categories_df['item_category_name_wc'] = item_categories_df['item_category_name'].map(lambda x: len(str(x).split(' '))) #Item Category Description Word Count
txtFeatures = pd.DataFrame(tfidf.fit_transform(item_categories_df['item_category_name']).toarray())
cols = txtFeatures.columns
for i in range(feature_cnt):
    item_categories_df['item_category_name_tfidf_' + str(i)] = txtFeatures[cols[i]]
item_categories_df.head()
# 对 shop进行同样操作
#Text Features
feature_cnt = 25
tfidf = TfidfVectorizer(max_features=feature_cnt)
shops_df['shop_name_len'] = shops_df['shop_name'].map(len)  #Lenth of Shop Name
shops_df['shop_name_wc'] = shops_df['shop_name'].map(lambda x: len(str(x).split(' '))) #Shop Name Word Count
txtFeatures = pd.DataFrame(tfidf.fit_transform(shops_df['shop_name']).toarray())
cols = txtFeatures.columns
for i in range(feature_cnt):
    shops_df['shop_name_tfidf_' + str(i)] = txtFeatures[cols[i]]
shops_df.head()<jupyter_output><empty_output><jupyter_text>分离月份数据<jupyter_code>train_df['date'] = pd.to_datetime(train_df['date'], format='%d.%m.%Y')
train_df['month'] = train_df['date'].dt.month
train_df['year'] = train_df['date'].dt.year
train_df = train_df.drop(['date', 'item_price'], axis = 1)
cols = [c for c in train_df.columns if c not in ['item_cnt_day']]
train = train_df.groupby(cols, as_index=False)[['item_cnt_day']].sum()
train = train.rename(columns={'item_cnt_day':'item_cnt_month'})
# 对月份求平均
shop_item_monthly_mean = train[['shop_id', 'item_id', 'item_cnt_month']].groupby(['shop_id', 'item_id'], 
                                                                                as_index=False)[['item_cnt_month']].mean()
shop_item_monthly_mean = shop_item_monthly_mean.rename(columns={'item_cnt_month':'item_cnt_month_mean'})
# 添加平均值特征
train = pd.merge(train, shop_item_monthly_mean, how='left', on=['shop_id', 'item_id'])
shop_item_prev_month = train[train['date_block_num'] == 33][['shop_id', 'item_id', "item_cnt_month"]]
shop_item_prev_month = shop_item_prev_month.rename(columns={'item_cnt_month':'item_cnt_prev_month'})
shop_item_prev_month.head()
# 添加前一个月的数据，构成时间序列
train = pd.merge(train, shop_item_prev_month, how='left', on=['shop_id', 'item_id']).fillna(0.)
# item 特征
train = pd.merge(train, items_df, how='left', on='item_id')
#Item Category features
train = pd.merge(train, item_categories_df, how='left', on='item_category_id')
#Shops features
train = pd.merge(train, shops_df, how='left', on='shop_id')

train.head()
test_df.head()
test_df['month'] = 11
test_df['year'] = 2015
test_df['date_block_num'] = 34
test = test_df
#Add Mean Feature
test = pd.merge(test, shop_item_monthly_mean, how='left', on=['shop_id','item_id']).fillna(0.)
#Add Previous Month Feature
test = pd.merge(test, shop_item_prev_month, how='left', on=['shop_id','item_id']).fillna(0.)
#Items features
test = pd.merge(test, items_df, how='left', on='item_id')
#Item Category features
test = pd.merge(test, item_categories_df, how='left', on='item_category_id')
#Shops features
test = pd.merge(test, shops_df, how='left', on='shop_id')
test['item_cnt_month'] = 0.
test.head()<jupyter_output><empty_output><jupyter_text>类别特征编码<jupyter_code>for c in ['shop_name','item_name','item_category_name']:
    lbl = sklearn.preprocessing.LabelEncoder()
    lbl.fit(list(train[c].unique())+list(test[c].unique()))
    train[c] = lbl.transform(train[c].astype(str))
    test[c] = lbl.transform(test[c].astype(str))
    print(c)
train.head()
# 训练和预测
col = [c for c in train.columns if c not in ['item_cnt_month']]
x1 = train[train['date_block_num'] < 33]
y1 = np.log1p(x1['item_cnt_month'].clip(0., 20.))
x1 = x1[col]
x2 = train[train['date_block_num'] == 33]
y2 = np.log1p(x2['item_cnt_month'].clip(0., 20.))
x2 = x2[col]
# 模型
reg = sklearn.ensemble.ExtraTreesRegressor(n_estimators=25, n_jobs=-1, max_depth=15, random_state=18)
reg.fit(x1, y1)
y_pred = reg.predict(x2)
print('RMSE:', np.sqrt(sklearn.metrics.mean_squared_error(y2.clip(0.,20.),y_pred.clip(0.,20.))))
reg.fit(train[col], train['item_cnt_month'].clip(0. ,20.))
test['item_cnt_month'] = reg.predict(test[col]).clip(0.,20.)
test[['ID','item_cnt_month']].to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>其他模型 lightbgm xgboost<jupyter_code>from multiprocessing import *
#XGBoost
def xgb_rmse(preds, y):
    y = y.get_label()
    score = np.sqrt(sklearn.metrics.mean_squared_error(y.clip(0., 20.), preds.clip(0., 20.)))
    return 'RMSE', score
params = {'eta':0.2, 'max_depth':4, 'objective':'reg:linear', 'eval_metric':'rmse', 'seed':18, 'silent':True}
watchlist = [(xgb.DMatrix(x1, y1), 'train'), (xgb.DMatrix(x2, y2), 'vaild')]
xgb_model = xgb.train(params, xgb.DMatrix(x1, y1), 100, watchlist, verbose_eval=10, feval=xgb_rmse, 
                     maximize=False, early_stopping_rounds=20)
 test['item_cnt_month'] = xgb_model.predict(xgb.DMatrix(test[col]), ntree_limit=xgb_model.best_ntree_limit)
 test[['ID','item_cnt_month']].to_csv('xgb_submission.csv', index=False)

#LightGBM
def lgb_rmse(preds, y):
    y = np.array(list(y.get_label()))
    score = np.sqrt(sklearn.metrics.mean_squared_error(y.clip(0.,20.), preds.clip(0.,20.)))
    return 'RMSE', score, False
params = {'learning_rate': 0.2, 'max_depth': 7, 'boosting': 'gbdt',
          'objective': 'regression', 'metric': 'mse', 'is_training_metric': False, 'seed': 18}
lgb_model = lgb.train(params, lgb.Dataset(x1, label=y1), 100, 
                      lgb.Dataset(x2, label=y2), feval=lgb_rmse, verbose_eval=10, early_stopping_rounds=20)
test['item_cnt_month'] = lgb_model.predict(test[col], num_iteration=lgb_model.best_iteration)
test[['ID','item_cnt_month']].to_csv('lgb_submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17482951022938623, 'fraction_numerical': 0.01735895846249225, 'mean_word_length': 6.937269372693727, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '634645', 'license': 'Apache-2.0', 'n_tokens_mistral': 2609, 'n_tokens_neox': 2564, 'n_words': 378, 'path': 'competitive-data-science-predict-future-sales/01.ipynb', 'repo_name': 'yunshuipiao/sw_kaggle', 'size': '6452'}