<jupyter_start><jupyter_text>public score : 0.16293<jupyter_code>result_df<jupyter_output><empty_output><jupyter_text>안되겠다 트리50개로 해봐야지csv로 뽑고 => h2o.ai에서 진행<jupyter_code>%%time
use_col = ["srch_co","srch_ci","user_location_region",\
               "hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","srch_children_cnt"]
print("read the test.csv")
test = pd.read_csv("../data/test.csv")
test = test[use_col]

test["srch_ci"] = pd.to_datetime(test["srch_ci"], errors="coerce")
test["srch_co"] = pd.to_datetime(test["srch_co"], errors="coerce")
test["period"] = test["srch_co"] - test["srch_ci"]
test["period"] = (test["period"] / np.timedelta64(1, 'D')).fillna(0.0).astype(int)
test = test.drop(["srch_co","srch_ci"], axis=1)
test["num"] = 1
test["srch_adults_cnt"] = test["srch_adults_cnt"].apply(lambda x: 3 if x>=3 else x)
test = test.drop(["num","srch_children_cnt"], axis=1)

test = test[["hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","period","user_location_region"]]

print("modeling strart")
model = RandomForestClassifier(n_estimators=10, max_depth=7, n_jobs=-1, random_state=777)
print('='*50)
print('# Test shape : {}'.format(test.shape))

model.fit(train_x,train_y)

preds = model.predict_proba(test)
preds = np.fliplr(np.argsort(preds, axis=1))

print("save file")

result_df = pd.DataFrame([ " ".join(row) for row in preds[:,:5].astype(str)], columns=["hotel_cluster"])
result_df.index.names = ["id"]
file_name = datetime.now().strftime("result_%Y%m%d%H%M%S") + '.csv'
result_df.to_csv(os.path.join('../output',file_name), index=True)
train_sum = pd.concat([train_x,train_y], axis=1)
train_sum.to_csv("train_data.csv")
test.to_csv("test_data.csv")
# 하나로 정리
%%time

print('preprocessing train_data')
use_col = ["srch_co","srch_ci","user_location_region",\
               "hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","srch_children_cnt","hotel_cluster"]

train_y = train[["hotel_cluster"]]

train_x = train[use_col]
train_x["srch_ci"] = pd.to_datetime(train_x["srch_ci"], errors="coerce")
train_x["srch_co"] = pd.to_datetime(train_x["srch_co"], errors="coerce")
train_x["period"] = train_x["srch_co"] - train_x["srch_ci"]
train_x["period"] = (train_x["period"] / np.timedelta64(1, 'D')).astype(int)
train_x = train_x.drop(["srch_co","srch_ci"], axis=1)
train_x["srch_adults_cnt"] = train_x["srch_adults_cnt"].apply(lambda x: 3 if x>=3 else x)
train_x = train_x.drop(["srch_children_cnt"], axis=1)
train_x = train_x[["hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","period","user_location_region"]]



use_col = ["srch_co","srch_ci","user_location_region",\
               "hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","srch_children_cnt"]
print("read the test.csv")
test = pd.read_csv("../data/test.csv")
test = test[use_col]

print("preprocessing test_data")

test["srch_ci"] = pd.to_datetime(test["srch_ci"], errors="coerce")
test["srch_co"] = pd.to_datetime(test["srch_co"], errors="coerce")
test["period"] = test["srch_co"] - test["srch_ci"]
test["period"] = (test["period"] / np.timedelta64(1, 'D')).fillna(0.0).astype(int)
test = test.drop(["srch_co","srch_ci"], axis=1)
test["num"] = 1
test["srch_adults_cnt"] = test["srch_adults_cnt"].apply(lambda x: 3 if x>=3 else x)
test = test.drop(["num","srch_children_cnt"], axis=1)

test = test[["hotel_market","srch_destination_id","hotel_country","srch_adults_cnt","period","user_location_region"]]

print("modeling strart")
model = RandomForestClassifier(n_estimators=10, max_depth=7, n_jobs=-1, random_state=777)
print('='*50)
print('# Test shape : {}'.format(test.shape))

model.fit(train_x,train_y)

preds = model.predict_proba(test)
preds = np.fliplr(np.argsort(preds, axis=1))

print("save file")

result_df = pd.DataFrame([ " ".join(row) for row in preds[:,:5].astype(str)], columns=["hotel_cluster"])
result_df.index.names = ["id"]
file_name = datetime.now().strftime("result_%Y%m%d%H%M%S") + '.csv'
result_df.to_csv(os.path.join('../output',file_name), index=True)
train[["srch_children_cnt","srch_adults_cnt"]].groupby(["srch_children_cnt"]).agg
from csv import DictReader
from collections import defaultdict
from datetime import datetime

start = datetime.now()

def get_top5(d):
    return " ".join(sorted(d, key=d.get, reverse=True)[:5])

destination_clusters = defaultdict(lambda: defaultdict(int))

for i, row in enumerate(DictReader(open("../data/train.csv"))):
	destination_clusters[row["srch_destination_id"]][row["hotel_cluster"]] += 1
	if i % 1000000 == 0:
		print("%s\t%s"%(i, datetime.now() - start))

most_frequent = defaultdict(str)

for k in destination_clusters:
	most_frequent[k] = get_top5(destination_clusters[k])

with open("pred_sub.csv", "w") as outfile:
	outfile.write("id,hotel_cluster\n")
	for i, row in enumerate(DictReader(open("../data/test.csv"))):
		outfile.write("%d,%s\n"%(i,most_frequent[row["srch_destination_id"]]))
		if i % 1000000 == 0:
			print("%s\t%s"%(i, datetime.now() - start))<jupyter_output>0	0:00:00.032000
1000000	0:00:08.536000
2000000	0:00:16.857000
3000000	0:00:25.250000
4000000	0:00:33.751000
5000000	0:00:42.339000
6000000	0:00:50.776000
7000000	0:00:59.239000
8000000	0:01:07.823000
9000000	0:01:16.249000
10000000	0:01:24.588000
11000000	0:01:32.984000
12000000	0:01:41.549000
13000000	0:01:50.041000
14000000	0:01:58.427000
15000000	0:02:06.812000
16000000	0:02:15.193000
17000000	0:02:23.633000
18000000	0:02:31.937000
19000000	0:02:40.275000
20000000	0:02:48.656000
21000000	0:02:57.063000
22000000	0:03:05.426000
23000000	0:03:15.156000
24000000	0:03:23.927000
25000000	0:03:32.309000
26000000	0:03:41.202000
27000000	0:03:49.622000
28000000	0:03:58.008000
29000000	0:04:06.977000
30000000	0:04:15.666000
31000000	0:04:24.836000
32000000	0:04:34.716000
33000000	0:04:43.532000
34000000	0:04:52.305000
35000000	0:05:00.588000
36000000	0:05:08.876000
37000000	0:05:17.220000
0	0:05:23.496000
1000000	0:05:33.737000
2000000	0:05:44.842000
{'fraction_non_alphanumeric': 0.18850267379679145, 'fraction_numerical': 0.13920454545454544, 'mean_word_length': 7.75, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '473007', 'license': 'MIT', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2512, 'n_words': 354, 'path': 'notebook/06. Feature Engineering.ipynb', 'repo_name': 'zzsza/Kaggle_Expedia-hotel-recommendations', 'size': '5984'}