<jupyter_start><jupyter_text>train data process<jupyter_code># train_tmp_click_log = 'data/train_tmp/click_log.csv'
train_click_log_path = 'data/train_preliminary/click_log.csv'
train_click_log = pd.read_csv(train_click_log_path)
train_click_log.head(3)
train_click_log.dtypes
train_click_log.time.unique().shape, train_click_log.user_id.unique().shape, train_click_log.creative_id.unique().shape, train_click_log.click_times.unique().shape
train_click_log.shape
train_ad = pd.read_csv('data/train_preliminary/ad.csv')
train_ad.head(5)
train_ad.shape
train_ad.dtypes
train_ad.product_id[train_ad.product_id=='\\N']='0'
train_ad.industry[train_ad.industry=='\\N']='0'
train_ad.product_id = train_ad.product_id.astype(int)
train_ad.industry = train_ad.industry.astype(int)
train_ad.dtypes
train_user = pd.read_csv('data/train_preliminary/user.csv')
train_user.head(2)
train_user.dtypes<jupyter_output><empty_output><jupyter_text>merge user.csv ad.csv to click_log.csv<jupyter_code>train_clicklog_ad = pd.merge(train_click_log, train_ad, on='creative_id', how='left')
train_clicklog_ad.head()
train_clicklog_ad.shape
train_clicklog_ad_user = pd.merge(train_clicklog_ad, train_user, on='user_id', how='left')
train_clicklog_ad_user.head()
train_clicklog_ad_user.dtypes<jupyter_output><empty_output><jupyter_text>write to csv file<jupyter_code># train_clicklog_ad_user.to_csv('data/train_preliminary/clicklog_ad_user.csv',index=False)<jupyter_output><empty_output><jupyter_text>Test data process<jupyter_code># train_tmp_click_log = 'data/train_tmp/click_log.csv'
test_click_log_path = 'data/test/click_log.csv'
test_click_log = pd.read_csv(test_click_log_path)
test_click_log.time.unique().shape, test_click_log.user_id.unique().shape, test_click_log.creative_id.unique().shape, test_click_log.click_times.unique().shape
test_click_log.info()
test_ad = pd.read_csv('data/train_preliminary/ad.csv')
test_ad.product_id[test_ad.product_id=='\\N']='0'
test_ad.industry[test_ad.industry=='\\N']='0'
test_ad.product_id = test_ad.product_id.astype(int)
test_ad.industry = test_ad.industry.astype(int)
test_ad.dtypes
test_ad.head()<jupyter_output><empty_output><jupyter_text>在click_log.csv中有20多万的creative_id没有在ad.csv出现,使用inner方式保留两个表的公共creative_id的行<jupyter_code># test_clicklog_ad = pd.merge(test_click_log, test_ad, on='creative_id', how='left')
# test_click_ad_dropna = test_clicklog_ad.dropna()
test_clicklog_ad = pd.merge(test_click_log, test_ad, on='creative_id', how='inner')
test_clicklog_ad.head()
# 类型转换为int
# test_clicklog_ad_drop_userid.ad_id = test_clicklog_ad_drop_userid.product_id.astype(int)
# test_clicklog_ad_drop_userid.product_id = test_clicklog_ad_drop_userid.product_id.astype(int)
# test_clicklog_ad_drop_userid.product_category = test_clicklog_ad_drop_userid.product_id.astype(int)
# test_clicklog_ad_drop_userid.advertiser_id = test_clicklog_ad_drop_userid.product_id.astype(int)
# test_clicklog_ad_drop_userid.industry = test_clicklog_ad_drop_userid.product_id.astype(int)
test_clicklog_ad.info()
test_clicklog_ad.info()

test_clicklog_ad.to_csv('data/test/clicklog_ad.csv', index=False)
# 删除重复user_id
# test_clicklog_ad_drop_userid = test_click_ad_dropna.drop_duplicates('user_id')
# test_clicklog_ad_drop_userid.info()
# test_clicklog_ad_drop_userid.to_csv('data/test/clicklog_ad.csv', index=False)
train_clicklog_ad_user.info()
train_clicklog_ad_user.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12158590308370044, 'fraction_numerical': 0.0026431718061674008, 'mean_word_length': 10.909090909090908, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '132290', 'license': 'MIT', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1399, 'n_words': 112, 'path': 'process_data_old.ipynb', 'repo_name': 'sunlanchang/Tencent-Ads-Algo-Comp-2020', 'size': '3405'}