<jupyter_start><jupyter_text>What are the instructions for short trajectories?<jupyter_code># 5-9
data_short = list(filter(filter_5_9, full_tr_data))

print (len(data_short))

instr_set_short = set()
instr_list_short = []
for d in data_short:
    instr_set_short.add(d['instructions'][0])
    instr_list_short.append(d['instructions'][0])

print (len(instr_set_short))

counter = Counter(instr_list_short).most_common(30)
counter
sum([t[1] for t in counter])<jupyter_output><empty_output><jupyter_text>data 100 - small_three_houses_single_goal_short (training same as val)<jupyter_code>def filter_data_100(d):
    if filter_instruction_most_common(d) and filter_5_9(d):
        return True
    else:
        return False 
# 5-9, 'find a sink in the kitchen '
data_100 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_data_100(full_tr_data[idx])]
len(data_100)
#  ('ac26ZMwG7aT', 38),
#  ('8WUmhLawc2A', 37),
#  ('cV4RVeZvu5T', 26)
#  3 houses, 5-9, 'find a sink in the kitchen '
data_100 = [d for d in data_100 if d['scan'] in ['ac26ZMwG7aT', '8WUmhLawc2A', 'cV4RVeZvu5T']]
len(data_100)
data_100 = [d for d in data_100 if 156450 != d['path_id']]
len(data_100)
data_100_scans = [d['scan'] for d in data_100]
len(set(data_100_scans))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_three_houses_single_goal_short.json", "w") as f:
    json.dump(data_100, f)<jupyter_output><empty_output><jupyter_text>data 1078 - small_three_goals_short -- training<jupyter_code>def filter_data_1078(d):
    if filter_instruction_most_common_sets(d) and filter_5_9(d):
        return True
    else:
        return False  
# any scans, length 5-9, 
# 'find a sink in the kitchen', 'find a refrigerator in the kitchen', 'find a picture in the living room'
data_1078 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_data_1078(full_tr_data[idx])]
print ("Data length = ", len(data_1078))

data_1078_scans = [d['scan'] for d in data_1078]
print ("Number of unique houses = ", len(set(data_1078_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_three_goals_short.json", "w") as f:
    json.dump(data_1078, f)
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_three_goals_short.json", "w") as f:
    json.dump(data_1078, f)
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_three_goals_short.json", "w") as f:
    json.dump(data_1078, f)<jupyter_output><empty_output><jupyter_text>data 1078 - small_three_goals_short -- val (less than 1078)<jupyter_code>def filter_data_1078_val(d, scans):
    if filter_instruction_most_common_sets(d) and filter_5_9(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_1078_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_1078_val(full_val_seen_data[idx], data_1078_scans)]
print ("Data length = ", len(data_1078_val_seen))

data_1078_val_seen_scans = [d['scan'] for d in data_1078_val_seen]
print ("Number of unique houses = ", len(set(data_1078_val_seen_scans)))
data_1078_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_data_1078(full_val_unseen_data[idx])]
print ("Data length = ", len(data_1078_val_unseen))

data_1078_val_unseen_scans = [d['scan'] for d in data_1078_val_unseen]
print ("Number of unique houses = ", len(set(data_1078_val_unseen_scans)))<jupyter_output>Data length =  60
Number of unique houses =  5<jupyter_text>data 1862 - small_three_goals_anylength -- training<jupyter_code># any length
# 'find a sink in the kitchen', 'find a refrigerator in the kitchen', 'find a picture in the living room'
data_1862 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_instruction_most_common_sets(full_tr_data[idx])]
print ("Data length = ", len(data_1862))

data_1862_scans = [d['scan'] for d in data_1862]
print ("Number of unique houses = ", len(set(data_1862_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_three_goals_anylength.json", "w") as f:
    json.dump(data_1862, f)<jupyter_output><empty_output><jupyter_text>data 1862 - small_three_goals_anylength -- val (less than 1078)<jupyter_code>def filter_data_1862_val(d, scans):
    if filter_instruction_most_common_sets(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_1862_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_1862_val(full_val_seen_data[idx], data_1862_scans)]
print ("Data length = ", len(data_1862_val_seen))

data_1862_val_seen_scans = [d['scan'] for d in data_1862_val_seen]
print ("Number of unique houses = ", len(set(data_1862_val_seen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_three_goals_anylength.json", "w") as f:
    json.dump(data_1862_val_seen, f)
data_1862_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_instruction_most_common_sets(full_val_unseen_data[idx])]
print ("Data length = ", len(data_1862_val_unseen))

data_1862_val_unseen_scans = [d['scan'] for d in data_1862_val_unseen]
print ("Number of unique houses = ", len(set(data_1862_val_unseen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_three_goals_anylength.json", "w") as f:
    json.dump(data_1862_val_unseen, f)<jupyter_output><empty_output><jupyter_text>data 3160 - small_ten_goals_short -- training<jupyter_code>def filter_data_3160(d):
    if filter_instruction_most_common_sets_top_10(d) and filter_5_9(d):
        return True
    else:
        return False  
# any scans, length 5-9, 
# top 10 language instructions
data_3160 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_data_3160(full_tr_data[idx])]
print ("Data length = ", len(data_3160))

data_3160_scans = [d['scan'] for d in data_3160]
print ("Number of unique houses = ", len(set(data_3160_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_ten_goals_short.json", "w") as f:
    json.dump(data_3160, f)<jupyter_output><empty_output><jupyter_text>data 3160 - small_ten_goals_short -- val (less than 3160)<jupyter_code>def filter_data_3160_val(d, scans):
    if filter_instruction_most_common_sets_top_10(d) and filter_5_9(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_3160_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_3160_val(full_val_seen_data[idx], data_3160_scans)]
print ("Data length = ", len(data_3160_val_seen))

data_3160_val_seen_scans = [d['scan'] for d in data_3160_val_seen]
print ("Number of unique houses = ", len(set(data_3160_val_seen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_ten_goals_short.json", "w") as f:
    json.dump(data_3160_val_seen, f)
data_3160_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_data_3160(full_val_unseen_data[idx])]
print ("Data length = ", len(data_3160_val_unseen))

data_3160_val_unseen_scans = [d['scan'] for d in data_3160_val_unseen]
print ("Number of unique houses = ", len(set(data_3160_val_unseen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_ten_goals_short.json", "w") as f:
    json.dump(data_3160_val_unseen, f)<jupyter_output><empty_output><jupyter_text>data 5775 - small_ten_goals_anylength -- training<jupyter_code># anylength
# top 10 language instructions
data_5775 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_instruction_most_common_sets_top_10(full_tr_data[idx])]
print ("Data length = ", len(data_5775))

data_5775_scans = [d['scan'] for d in data_5775]
print ("Number of unique houses = ", len(set(data_5775_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_ten_goals_anylength.json", "w") as f:
    json.dump(data_5775, f)<jupyter_output><empty_output><jupyter_text>data 5775 - small_ten_goals_anylength -- val (less than 5775)<jupyter_code>def filter_data_5775_val(d, scans):
    if filter_instruction_most_common_sets_top_10(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_5775_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_5775_val(full_val_seen_data[idx], data_5775_scans)]
print ("Data length = ", len(data_5775_val_seen))

data_5775_val_seen_scans = [d['scan'] for d in data_5775_val_seen]
print ("Number of unique houses = ", len(set(data_5775_val_seen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_ten_goals_anylength.json", "w") as f:
    json.dump(data_5775_val_seen, f)
data_5775_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_instruction_most_common_sets_top_10(full_val_unseen_data[idx])]
print ("Data length = ", len(data_5775_val_unseen))

data_5775_val_unseen_scans = [d['scan'] for d in data_5775_val_unseen]
print ("Number of unique houses = ", len(set(data_5775_val_unseen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_ten_goals_anylength.json", "w") as f:
    json.dump(data_5775_val_unseen, f)<jupyter_output><empty_output><jupyter_text>data 7954 - small_thirty_goals_short -- training<jupyter_code>def filter_data_7954(d):
    if filter_instruction_most_common_sets_top_30(d) and filter_5_9(d):
        return True
    else:
        return False   
# any scans, length 5-9, 
# 'find a sink in the kitchen', 'find a refrigerator in the kitchen', 'find a picture in the living room'
data_7954 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_data_dummy(full_tr_data[idx])]
print ("Data length = ", len(data_7954))

data_7954_scans = [d['scan'] for d in data_7954]
print ("Number of unique houses = ", len(set(data_7954_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_thirty_goals_short.json", "w") as f:
    json.dump(data_7954, f)<jupyter_output><empty_output><jupyter_text>data 7954 - small_thirty_goals_short -- val (less than 7954)<jupyter_code>def filter_data_7954_val(d, scans):
    if filter_instruction_most_common_sets_top_30(d) and filter_5_9(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_7954_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_7954_val(full_val_seen_data[idx], data_7954_scans)]
print ("Data length = ", len(data_7954_val_seen))

data_7954_val_seen_scans = [d['scan'] for d in data_7954_val_seen]
print ("Number of unique houses = ", len(set(data_7954_val_seen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_thirty_goals_short.json", "w") as f:
    json.dump(data_7954_val_seen, f)
data_7954_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_data_7954(full_val_unseen_data[idx])]
print ("Data length = ", len(data_7954_val_unseen))

data_7954_val_unseen_scans = [d['scan'] for d in data_7954_val_unseen]
print ("Number of unique houses = ", len(set(data_7954_val_unseen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_thirty_goals_short.json", "w") as f:
    json.dump(data_7954_val_unseen, f)<jupyter_output><empty_output><jupyter_text>data 14413 - small_thirty_goals_anylength -- training<jupyter_code># anylength
# top 30 language instructions
data_14413 = [full_tr_data[idx] for idx in range(len(full_tr_data)) if filter_instruction_most_common_sets_top_30(full_tr_data[idx])]
print ("Data length = ", len(data_14413))

data_14413_scans = [d['scan'] for d in data_14413]
print ("Number of unique houses = ", len(set(data_14413_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_train_small_thirty_goals_anylength.json", "w") as f:
    json.dump(data_14413, f)<jupyter_output><empty_output><jupyter_text>data 14413 - small_thirty_goals_anylength -- val (less than 14413)<jupyter_code>def filter_data_14413_val(d, scans):
    if filter_instruction_most_common_sets_top_30(d) and (d['scan'] in scans):
        return True
    else:
        return False  
data_14413_val_seen = [full_val_seen_data[idx] for idx in range(len(full_val_seen_data)) if filter_data_14413_val(full_val_seen_data[idx], data_14413_scans)]
print ("Data length = ", len(data_14413_val_seen))

data_14413_val_seen_scans = [d['scan'] for d in data_14413_val_seen]
print ("Number of unique houses = ", len(set(data_14413_val_seen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_seen_small_thirty_goals_anylength.json", "w") as f:
    json.dump(data_14413_val_seen, f)
data_14413_val_unseen = [full_val_unseen_data[idx] for idx in range(len(full_val_unseen_data)) if filter_instruction_most_common_sets_top_30(full_val_unseen_data[idx])]
print ("Data length = ", len(data_14413_val_unseen))

data_14413_val_unseen_scans = [d['scan'] for d in data_14413_val_unseen]
print ("Number of unique houses = ", len(set(data_14413_val_unseen_scans)))
with open("/home/hoyeung/blob_matterport3d/asknav/asknav_val_unseen_small_thirty_goals_anylength.json", "w") as f:
    json.dump(data_14413_val_unseen, f)<jupyter_output><empty_output><jupyter_text>Philly run yaml template<jupyter_code>seeds_10 = [42, 677, 848, 163, 620, 511, 102, 158, 2, 204]
seeds_20 = [677, 848, 163, 620, 511, 102, 158, 2, 204, 471, 154, 220, 748, 623,  47, 882, 136, 315,  48]

# exper_name = "20200126_philly_aggrevate_seeds"
# size = 3160
# data_suffix = "small_ten_goals_short"
# seeds = seeds_10

exper_name = "20200126_philly_aggrevate_seeds"
size = 1078
data_suffix = "small_three_goals_short"
seeds = seeds_10

for num in seeds:
    text = """
- name: data_{size}_seed_{num}
  sku: G1
  command:
  - cd tasks/VNLA/scripts/ && bash train_experiments.sh "{exper_name}" "data_{size}_seed_{num}" "configs/experiment.json" -batch_size 100 -start_beta_decay 100 -decay_beta_every 100 -min_history_to_learn 1000 -save_every 1000 -log_every 1000 -plot_to_philly 0 -data_suffix {data_suffix} -dropout_ratio 0.5 -agent_end_criteria 2.5 -n_iters 50000 -seed {num}
  """.format(size=size, num=num, data_suffix=data_suffix, exper_name=exper_name)
    print(text)<jupyter_output>- name: data_1078_seed_42
  sku: G1
  command:
  - cd tasks/VNLA/scripts/ && bash train_experiments.sh "20200126_philly_aggrevate_seeds" "data_1078_seed_42" "configs/experiment.json" -batch_size 100 -start_beta_decay 100 -decay_beta_every 100 -min_history_to_learn 1000 -save_every 1000 -log_every 1000 -plot_to_philly 0 -data_suffix small_three_goals_short -dropout_ratio 0.5 -agent_end_criteria 2.5 -n_iters 50000 -seed 42
  

- name: data_1078_seed_677
  sku: G1
  command:
  - cd tasks/VNLA/scripts/ && bash train_experiments.sh "20200126_philly_aggrevate_seeds" "data_1078_seed_677" "configs/experiment.json" -batch_size 100 -start_beta_decay 100 -decay_beta_every 100 -min_history_to_learn 1000 -save_every 1000 -log_every 1000 -plot_to_philly 0 -data_suffix small_three_goals_short -dropout_ratio 0.5 -agent_end_criteria 2.5 -n_iters 50000 -seed 677
  

- name: data_1078_seed_848
  sku: G1
  command:
  - cd tasks/VNLA/scripts/ && bash train_experiments.sh "20200126_philly_aggrevate_seeds" [...]
{'fraction_non_alphanumeric': 0.11262530302037607, 'fraction_numerical': 0.06918692262333748, 'mean_word_length': 6.697428139183056, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '112824', 'license': 'MIT', 'n_tokens_mistral': 6674, 'n_tokens_neox': 5951, 'n_words': 1117, 'path': 'code/tasks/VNLA/notebooks/make_toy_datasets_20200124.ipynb', 'repo_name': 'Chucooleg/vnla', 'size': '15263'}