<jupyter_start><jupyter_text>IMDB<jupyter_code>imdb0 = pd.read_csv("./training-curves/results.csv")
imdb1 = pd.read_csv("./training-curves/results (1).csv") 
imdb2 = pd.read_csv("./training-curves/results (2).csv")
imdb3 = pd.read_csv("./training-curves/results (3).csv")
imdb4 = pd.read_csv("./training-curves/imdb0.csv")
imdb5 = pd.read_csv("./training-curves/imdb1.csv")
imdb6 = pd.read_csv("./training-curves/imdb2.csv")
imdb7 = pd.read_csv("./training-curves/imdb3.csv")
imdb0.head()
imdb1.head()
imdb1.loc[:, 'Trial-ID'] = imdb1.loc[:, 'Trial-ID'] + imdb0['Trial-ID'].max()
imdb2.loc[:, 'Trial-ID'] = imdb2.loc[:, 'Trial-ID'] + imdb1['Trial-ID'].max()
imdb3.loc[:, 'Trial-ID'] = imdb3.loc[:, 'Trial-ID'] + imdb2['Trial-ID'].max()
imdb4.loc[:, 'Trial-ID'] = imdb4.loc[:, 'Trial-ID'] + imdb3['Trial-ID'].max()
imdb5.loc[:, 'Trial-ID'] = imdb5.loc[:, 'Trial-ID'] + imdb4['Trial-ID'].max()
imdb6.loc[:, 'Trial-ID'] = imdb6.loc[:, 'Trial-ID'] + imdb5['Trial-ID'].max()
imdb7.loc[:, 'Trial-ID'] = imdb7.loc[:, 'Trial-ID'] + imdb6['Trial-ID'].max()
imdbnew = pd.concat([imdb0, imdb1, imdb2, imdb3, imdb4, imdb5, imdb6, imdb7])
imdbnew.to_csv("training-curves/imdb_complete.csv", index=False)<jupyter_output><empty_output><jupyter_text>IMDB CuDNN<jupyter_code>cudnnimdb = [pd.read_csv("./imdblstm/imdb_cudnn_{}.csv".format(i)) for i in range(1,6)]
for i in range(1,5):
    cudnnimdb[i].loc[:, 'Trial-ID'] = cudnnimdb[i].loc[:, 'Trial-ID'] + cudnnimdb[i-1]['Trial-ID'].max()
cudnnimdb_merged = pd.concat(cudnnimdb)
cudnnimdb_merged['Trial-ID'].max()
cudnnimdb_completed = cudnnimdb_merged.query("Status == 'COMPLETED'").sort_values(by="Trial-ID").drop(['Status', 'Iteration', 'test_acc'], axis=1)
(cudnnimdb_completed.groupby('dropout_embedding').agg('count').loc[:,'Trial-ID'] != 25).sum()
len(cudnnimdb_completed.groupby('dropout_embedding').agg('count').loc[:,'Trial-ID'])
cudnnimdb_final = pd.merge(cudnnimdb_completed, cudnnimdb_merged.loc[cudnnimdb_merged.test_acc.notnull(),['Trial-ID', 'test_acc']], on=['Trial-ID'] )
cudnnimdb_final.head()
cudnnimdb_final.to_csv("imdblstm/imdb_results.csv", index=False)<jupyter_output><empty_output><jupyter_text>IMDB CuDNN New Split<jupyter_code>df = pd.read_csv("./imdblstm/{}-{}.csv".format(name, 0))
df = df.loc[(df.Iteration == 15) | (df.Status == "COMPLETED")]
test_performance = df.loc[(df.Iteration == 15) & (df.Status == "INTERMEDIATE")].dropna(axis=1, how='all')
valid_performance = df.loc[df.Status == 'COMPLETED'].dropna(axis=1, how='all')
joined_df = valid_performance.merge(test_performance.loc[:, ['Trial-ID', 'test_acc']], on='Trial-ID')
joined_df.to_csv("imdblstm/imdb-new-split-0.csv", index=False)
n=7
name = 'imdb-new-split'
cudnnimdb = [pd.read_csv("./imdblstm/{}-{}.csv".format(name, i)).query("Status == 'COMPLETED'") for i in range(n)]
for i in range(1, n):
    cudnnimdb[i].loc[:, 'Trial-ID'] = cudnnimdb[i].loc[:, 'Trial-ID'] + cudnnimdb[i-1]['Trial-ID'].max()
cudnnimdb_merged = pd.concat(cudnnimdb)
cudnnimdb_merged['Trial-ID'].max()
cudnnimdb_merged.groupby(['dropout_embedding', 'embedding_regularizer', 'dropout_lstm']).agg('count').Objective.value_counts()
cudnnimdb_merged.head()
cudnnimdb_merged.sort_values(by="Trial-ID").drop(['Status', 'Iteration'], axis=1).to_csv("imdblstm/imdb_new_split.csv", index=False)
(cudnnimdb_merged.groupby('dropout_embedding').agg('count').loc[:,'Trial-ID'] != 25).sum()
len(cudnnimdb_merged.groupby('dropout_embedding').agg('count').loc[:,'Trial-ID'])<jupyter_output><empty_output><jupyter_text>MNIST<jupyter_code>mnist0 = pd.read_csv("./mnistcnn/results0.csv")
mnist1 = pd.read_csv("./mnistcnn/results1.csv")
mnist2 = pd.read_csv("./mnistcnn/results2.csv")
mnist3 = pd.read_csv("./mnistcnn/results3.csv")
mnist4 = pd.read_csv("./mnistcnn/results4.csv")
mnist0.head()
mnist1.head()
mnist2.head()
mnist3.head()
mnist1.loc[:, 'Trial-ID'] = mnist1.loc[:, 'Trial-ID'] + mnist0['Trial-ID'].max()
mnist2.loc[:, 'Trial-ID'] = mnist2.loc[:, 'Trial-ID'] + mnist1['Trial-ID'].max()
mnist3.loc[:, 'Trial-ID'] = mnist3.loc[:, 'Trial-ID'] + mnist2['Trial-ID'].max()
mnist4.loc[:, 'Trial-ID'] = mnist4.loc[:, 'Trial-ID'] + mnist3['Trial-ID'].max()
mnistnew = pd.concat([mnist0, mnist1, mnist2, mnist3, mnist4])
mnistnew.query("Status == 'COMPLETED'").to_csv("./mnistcnn/mnist_with_test_set_merged.csv", index=False)<jupyter_output><empty_output><jupyter_text>MNIST CNN Unshuffled<jupyter_code>mnist_unshuffled = [pd.read_csv("./mnistcnn/mnistcnn-unshuffled-{}.csv".format(i)) for i in range(6)]
for i in range(1,6):
    mnist_unshuffled[i].loc[:, 'Trial-ID'] = mnist_unshuffled[i].loc[:, 'Trial-ID'] + mnist_unshuffled[i-1]['Trial-ID'].max()
mnist_unshuffled_merged = pd.concat(mnist_unshuffled)
mnist_unshuffled_merged['Trial-ID'].max()
mnist_unshuffled_merged.query("Status == 'COMPLETED'").to_csv("./mnistcnn/mnist-unshuffled-merged.csv", index=False)<jupyter_output><empty_output><jupyter_text>Boston<jupyter_code>n=5
name = 'boston-new-split-'
boston = [pd.read_csv("./bostongbrt/{}{}.csv".format(name, i)).query("Status == 'COMPLETED'") for i in range(n)]
for i in range(1,n):
    boston[i].loc[:, 'Trial-ID'] = boston[i].loc[:, 'Trial-ID'] + boston[i-1]['Trial-ID'].max()
boston_merged = pd.concat(boston)
boston_merged['Trial-ID'].max()
boston_merged.groupby(['lr_scaler']).agg('count').Objective.value_counts()
boston_merged.to_csv("./bostongbrt/{}merged.csv".format(name), index=False)
boston_merged.head()<jupyter_output><empty_output><jupyter_text>Cartpole<jupyter_code>n=4
name = 'cartpole-revision-'
cartpole = [pd.read_csv("./cartpole/{}{}.csv".format(name, i)).query("Status == 'COMPLETED'") for i in range(n)]
for i in range(1,n):
    cartpole[i].loc[:, 'Trial-ID'] = cartpole[i].loc[:, 'Trial-ID'] + cartpole[i-1]['Trial-ID'].max()
cartpole_merged = pd.concat(cartpole)
cartpole_merged['Trial-ID'].max()
cartpole_merged.groupby(['cliprange']).agg('count').Objective.value_counts()
cartpole_merged['counts'] = cartpole_merged.groupby(['cliprange']).Objective.transform('count')
cartpole_final = cartpole_merged.query('counts == 25').drop(columns=['counts'])
cartpole_final.to_csv("./cartpole/{}merged.csv".format(name), index=False)
cartpole_final.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22776432606941083, 'fraction_numerical': 0.01791767554479419, 'mean_word_length': 7.294511378848728, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '85037', 'license': 'MIT', 'n_tokens_mistral': 2748, 'n_tokens_neox': 2645, 'n_words': 285, 'path': 'merge-datasets-util.ipynb', 'repo_name': 'LarsHH/reproducible-hyperparameter-optimization-jcgs-2019', 'size': '6195'}