Upload 40 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/instructions_processed_LP/BERT/data/alfred_data/train_task_desc_to_task_id_oct24.p filter=lfs diff=lfs merge=lfs -text
+models/instructions_processed_LP/BERT/data/alfred_data/train_text_with_ppdl_low_appended_new_split_oct24.p filter=lfs diff=lfs merge=lfs -text

models/instructions_processed_LP/ALFRED_task_helper.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Mar 27 16:49:38 2021
+
+@author: soyeonmin
+"""
+import pickle
+import alfred_utils.gen.constants as constants
+import string
+
+exclude = set(string.punctuation)
+task_type_dict = {2: 'pick_and_place_simple',
+ 5: 'look_at_obj_in_light',
+ 1: 'pick_and_place_with_movable_recep',
+ 3: 'pick_two_obj_and_place',
+ 6: 'pick_clean_then_place_in_recep',
+ 4: 'pick_heat_then_place_in_recep',
+ 0: 'pick_cool_then_place_in_recep'}
+
+
+def read_test_dict(test, appended, unseen):
+    if test:
+        if appended:
+            if unseen:
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_tests_unseen_new_split_GT_aug28.p", "rb"))
+            else:
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_tests_seen_new_split_GT_aug28.p", "rb"))
+        else:
+            if unseen:
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_test_unseen_916_noappended.p", "rb"))
+                
+                # REALFRED
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_test_unseen_noappended_new_split_bert_trained.p", "rb"))
+
+            else:
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_test_seen_916_noappended.p", "rb"))
+                # REALFRED
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_test_seen_noappended_new_split_bert_trained.p", "rb"))
+                
+    else:
+        if appended:
+            if unseen:
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_val_unseen_appended.p", "rb"))
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_unseen_new_split_GT_aug28.p", "rb"))
+                
+                # tests_unseen
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_unseen_new_split_GT_aug28_t.p", "rb"))
+            else:
+                # # return pickle.load(open("models/instructions_processed_LP/instruction2_params_val_seen_appended.p", "rb"))
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_seen_new_split_GT_aug28.p", "rb"))
+                
+                # tests_seen
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_seen_new_split_GT_aug28_t.p", "rb"))
+        else:
+            if unseen:
+                # REALFRED
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_val_unseen_916_noappended.p", "rb"))
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_unseen_noappended_new_split.p", "rb"))
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_unseen_new_split_GT_aug28_t.p", "rb"))
+            else:
+                # return pickle.load(open("models/instructions_processed_LP/instruction2_params_val_seen_916_noappended.p", "rb"))
+                return pickle.load(open("models/instructions_processed_LP/instruction2_params_valid_seen_new_split_GT_aug28_t.p", "rb"))
+
+def exist_or_no(string):
+    if string == '' or string == False:
+        return 0
+    else:
+        return 1
+
+def none_or_str(string):
+    if string == '':
+        return None
+    else:
+        return string
+
+def get_arguments_test(test_dict, instruction):
+    task_type, mrecep_target, object_target, parent_target, sliced = \
+        test_dict[instruction]['task_type'],  test_dict[instruction]['mrecep_target'], test_dict[instruction]['object_target'], test_dict[instruction]['parent_target'],\
+             test_dict[instruction]['sliced']
+
+    if isinstance(task_type, int):
+        task_type = task_type_dict[task_type]
+    return instruction, task_type, mrecep_target, object_target, parent_target, sliced 
+        
+
+def get_arguments(traj_data):
+    task_type = traj_data['task_type']
+    try:
+        # r_idx = traj_data['repeat_idx']
+        r_idx = traj_data['ann']['repeat_idx']
+    except:
+        r_idx = 0
+    language_goal_instr = traj_data['turk_annotations']['anns'][r_idx]['task_desc']
+    
+    sliced = exist_or_no(traj_data['pddl_params']['object_sliced'])
+    mrecep_target = none_or_str(traj_data['pddl_params']['mrecep_target'])
+    object_target = none_or_str(traj_data['pddl_params']['object_target'])
+    parent_target = none_or_str(traj_data['pddl_params']['parent_target'])
+    #toggle_target = none_or_str(traj_data['pddl_params']['toggle_target'])
+    
+    return language_goal_instr, task_type, mrecep_target, object_target, parent_target, sliced
+
+def add_target(target, target_action, list_of_actions):
+    if target in [a for a in constants.OPENABLE_CLASS_LIST if not(a == 'Box')]:
+        list_of_actions.append((target, "OpenObject"))
+    list_of_actions.append((target, target_action))
+    if target in [a for a in constants.OPENABLE_CLASS_LIST if not(a == 'Box')]:
+        list_of_actions.append((target, "CloseObject"))
+    return list_of_actions
+
+def determine_consecutive_interx(list_of_actions, previous_pointer, sliced=False):
+    returned, target_instance = False, None
+    if previous_pointer <= len(list_of_actions)-1:
+        if list_of_actions[previous_pointer][0] == list_of_actions[previous_pointer+1][0]:
+            returned = True
+            #target_instance = list_of_target_instance[-1] #previous target
+            target_instance = list_of_actions[previous_pointer][0]
+        #Micorwave or Fridge
+        elif list_of_actions[previous_pointer][1] == "OpenObject" and list_of_actions[previous_pointer+1][1] == "PickupObject":
+            returned = True
+            #target_instance = list_of_target_instance[0]
+            target_instance = list_of_actions[0][0]
+            if sliced:
+                #target_instance = list_of_target_instance[3]
+                target_instance = list_of_actions[3][0]
+        #Micorwave or Fridge
+        elif list_of_actions[previous_pointer][1] == "PickupObject" and list_of_actions[previous_pointer+1][1] == "CloseObject":
+            returned = True
+            #target_instance = list_of_target_instance[-2] #e.g. Fridge
+            target_instance = list_of_actions[previous_pointer-1][0]
+        #Faucet
+        elif list_of_actions[previous_pointer+1][0] == "Faucet" and list_of_actions[previous_pointer+1][1] in ["ToggleObjectOn", "ToggleObjectOff"]:
+            returned = True
+            target_instance = "Faucet"
+        #Pick up after faucet 
+        elif list_of_actions[previous_pointer][0] == "Faucet" and list_of_actions[previous_pointer+1][1] == "PickupObject":
+            returned = True
+            #target_instance = list_of_target_instance[0]
+            target_instance = list_of_actions[0][0]
+            if sliced:
+                #target_instance = list_of_target_instance[3]
+                target_instance = list_of_actions[3][0]
+    return returned, target_instance
+            
+def get_list_of_highlevel_actions(traj_data, test=False, test_dict=None, args_nonsliced=False, appended=False):
+    if not(test):
+        language_goal, task_type, mrecep_target, obj_target, parent_target,  sliced = get_arguments(traj_data)
+    if test:
+        r_idx = traj_data['ann']['repeat_idx']
+        instruction = traj_data['turk_annotations']['anns'][r_idx]['task_desc']
+        # aug28
+
+        #if appended:
+        instruction = instruction.lower()
+        instruction = ''.join(ch for ch in instruction if ch not in exclude)
+        language_goal, task_type, mrecep_target, obj_target, parent_target,  sliced  = get_arguments_test(test_dict, instruction)
+            
+    #obj_target = 'Tomato'
+    #mrecep_target = "Plate"
+    if parent_target == "Sink":
+        parent_target = "SinkBasin"
+    if parent_target == "Bathtub":
+        parent_target = "BathtubBasin"
+    
+    #Change to this after the sliced happens
+    if args_nonsliced:
+        if sliced == 1:
+            obj_target = obj_target +'Sliced'
+        #Map sliced as the same place in the map, but like "|SinkBasin" look at the objectid
+
+    
+    categories_in_inst = []
+    list_of_highlevel_actions = []
+    second_object = []
+    caution_pointers = []
+    #obj_target = "Tomato"
+
+    #if sliced:
+    #    obj_target = obj_target +'Sliced'
+
+    if sliced == 1:
+       list_of_highlevel_actions.append(("Knife", "PickupObject"))
+       list_of_highlevel_actions.append((obj_target, "SliceObject"))
+       caution_pointers.append(len(list_of_highlevel_actions))
+       list_of_highlevel_actions.append(("SinkBasin", "PutObject"))
+       categories_in_inst.append(obj_target)
+       
+    if sliced:
+        obj_target = obj_target +'Sliced'
+
+    
+    if task_type == 'pick_cool_then_place_in_recep': #0 in new_labels 
+       list_of_highlevel_actions.append((obj_target, "PickupObject"))
+       caution_pointers.append(len(list_of_highlevel_actions))
+       list_of_highlevel_actions = add_target("Fridge", "PutObject", list_of_highlevel_actions)
+       list_of_highlevel_actions.append(("Fridge", "OpenObject"))
+       list_of_highlevel_actions.append((obj_target, "PickupObject"))
+       list_of_highlevel_actions.append(("Fridge", "CloseObject"))
+       caution_pointers.append(len(list_of_highlevel_actions))
+       list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+       categories_in_inst.append(obj_target)
+       categories_in_inst.append("Fridge")
+       categories_in_inst.append(parent_target)
+       
+    elif task_type == 'pick_and_place_with_movable_recep': #1 in new_labels
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(mrecep_target, "PutObject", list_of_highlevel_actions)
+        list_of_highlevel_actions.append((mrecep_target, "PickupObject"))
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append(mrecep_target)
+        categories_in_inst.append(parent_target)
+    
+    elif task_type == 'pick_and_place_simple':#2 in new_labels 
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        #list_of_highlevel_actions.append((parent_target, "PutObject"))
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append(parent_target)
+        
+    
+    elif task_type == 'pick_heat_then_place_in_recep': #4 in new_labels
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))  
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target("Microwave", "PutObject", list_of_highlevel_actions)
+        list_of_highlevel_actions.append(("Microwave", "ToggleObjectOn" ))
+        list_of_highlevel_actions.append(("Microwave", "ToggleObjectOff" ))
+        list_of_highlevel_actions.append(("Microwave", "OpenObject"))
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))
+        list_of_highlevel_actions.append(("Microwave", "CloseObject"))
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append("Microwave")
+        categories_in_inst.append(parent_target)
+        
+    elif task_type == 'pick_two_obj_and_place': #3 in new_labels
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))  
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        if parent_target in constants.OPENABLE_CLASS_LIST:
+            second_object = [False] * 4
+        else:
+            second_object = [False] * 2
+        if sliced:
+            second_object = second_object + [False] * 3
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))  
+        #caution_pointers.append(len(list_of_highlevel_actions))
+        second_object.append(True)
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        second_object.append(False)
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append(parent_target)
+        
+        
+    elif task_type == 'look_at_obj_in_light': #5 in new_labels
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))
+        #if toggle_target == "DeskLamp":
+        #    print("Original toggle target was DeskLamp")
+        toggle_target = "FloorLamp"
+        list_of_highlevel_actions.append((toggle_target, "ToggleObjectOn" ))
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append(toggle_target)
+        
+    elif task_type == 'pick_clean_then_place_in_recep': #6 in new_labels
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))  
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions.append(("SinkBasin", "PutObject"))  #Sink or SinkBasin? 
+        list_of_highlevel_actions.append(("Faucet", "ToggleObjectOn"))
+        list_of_highlevel_actions.append(("Faucet", "ToggleObjectOff"))
+        list_of_highlevel_actions.append((obj_target, "PickupObject"))  
+        caution_pointers.append(len(list_of_highlevel_actions))
+        list_of_highlevel_actions = add_target(parent_target, "PutObject", list_of_highlevel_actions)
+        categories_in_inst.append(obj_target)
+        categories_in_inst.append("SinkBasin")
+        categories_in_inst.append("Faucet")
+        categories_in_inst.append(parent_target)
+    else:
+        raise Exception("Task type not one of 0, 1, 2, 3, 4, 5, 6!")
+
+    if sliced == 1:
+       if not(parent_target == "SinkBasin"):
+            categories_in_inst.append("SinkBasin")
+    
+    #return [(goal_category, interaction), (goal_category, interaction), ...]
+    print("instruction goal is ", language_goal)
+    #list_of_highlevel_actions = [ ('Microwave', 'OpenObject'), ('Microwave', 'PutObject'), ('Microwave', 'CloseObject')]
+    #list_of_highlevel_actions = [('Microwave', 'OpenObject'), ('Microwave', 'PutObject'), ('Microwave', 'CloseObject'), ('Microwave', 'ToggleObjectOn'), ('Microwave', 'ToggleObjectOff'), ('Microwave', 'OpenObject'), ('Apple', 'PickupObject'), ('Microwave', 'CloseObject'), ('Fridge', 'OpenObject'), ('Fridge', 'PutObject'), ('Fridge', 'CloseObject')]
+    #categories_in_inst = ['Microwave', 'Fridge']
+    return list_of_highlevel_actions, categories_in_inst, second_object, caution_pointers

models/instructions_processed_LP/BERT/best_models/base.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2099e73c1ebdf22a9ae9ac122beaaf29da9e2cccf4eabf659919a20f42e9108b
+size 438040777

models/instructions_processed_LP/BERT/best_models/mrecep.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47ad0207d1672ff9a12af409a7681eb16be8faf7af7465c2e40804deae2fee2
+size 438419145

models/instructions_processed_LP/BERT/best_models/object.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56b153e583afa38ac5a7369d72f67a99092678e29472995cf62c4fa48fefe0c3
+size 438419145

models/instructions_processed_LP/BERT/best_models/parent.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c0b683d812d0fb0adaa6a92107c586e3dfb22bf20a43836df8b34a0ca317671
+size 438148425

models/instructions_processed_LP/BERT/best_models/sliced.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962b51b86dd1dcca9611ca0a1f6bc0b6a44d18e8de94cb07406459b5333ec749
+size 438025417

models/instructions_processed_LP/BERT/data/alfred_data/create_text_with_pddl_low_appended.py

@@ -0,0 +1,112 @@
+import os
+import pandas as pd
+import argparse
+import json
+import pickle
+import string
+parser = argparse.ArgumentParser()
+parser.add_argument('--data_path', type=str, default="/media/user/data/FILM/alfred_data_all/json_new", help="where to look for the generated data")
+parser.add_argument('--split', type=str, default="tests_unseen")
+parser.add_argument('-o','--output_name', type=str)
+args = parser.parse_args()
+data_path = args.data_path
+split = args.split
+exclude = set(string.punctuation)
+result = dict()
+traj_data_path = "/media/user/data/alfred_4.3.0/gen/dataset/Finished"
+traj_data_path = "/media/user/data/FILM/alfred_data_all/json_2.1.0/valid_unseen"
+task_to_path = dict()
+desc_to_gt_params = dict()     # 'wash the brown vegetable and put it on the counter’: 
+JSON_FILENAME = "traj_data.json"
+for dir_name, _, _ in os.walk(traj_data_path):
+    if "trial_" in dir_name and (not "raw_images" in dir_name) and (not "pddl_states" in dir_name) and (not "video" in dir_name):
+        json_file = os.path.join(dir_name, JSON_FILENAME)
+        if not os.path.isfile(json_file):
+            continue
+        task = json_file.split('/')[-2]
+        task_to_path[task] = json_file
+# print(task_to_path)
+
+task_types = {"pick_cool_then_place_in_recep": 0, "pick_and_place_with_movable_recep": 1, "pick_and_place_simple": 2, "pick_two_obj_and_place": 3, "pick_heat_then_place_in_recep": 4, "look_at_obj_in_light": 5, "pick_clean_then_place_in_recep": 6}
+
+# 'task_desc': task
+# {'wash the brown vegetable and put it on the counter’: 'trial_T20230514_192232_882070',
+# 'wash the brown vegetable and put it on the counter’: 'trial_T20230514_192232_882070', ...}
+x_task = dict()
+
+result['x'] = []; result['x_low'] = []
+n = 0
+d = 0
+# with open('../../../../../alfred_data_small/splits/REALFRED_splits.json', 'r') as f:
+# with open('alfred_data_small/splits/REALFRED_splits.json', 'r') as f:
+with open('../../../../../alfred_data_small/splits/oct21.json', 'r') as f:
+
+
+    splits = json.load(f)
+tasks = list()
+for i in splits[split]:
+    task = i["task"]
+    if task not in tasks:
+        tasks.append(task)
+        
+for task in tasks:   
+    with open(os.path.join(data_path,task,'pp','ann_0.json')) as f:
+        ann_0 = json.load(f)
+    
+    anns = ann_0['turk_annotations']['anns']    # anns = [{"assignment_id", "high_descs", "task_desc"},{},{}]
+    for j in anns:
+        task_desc = j['task_desc']
+        if task_desc[-1] == '.':
+            task_desc = task_desc[:-1]
+        task_desc = task_desc.lower()
+        task_desc = ''.join(ch for ch in task_desc if ch not in exclude)
+        if task_desc in result['x']:
+            print(task, task_desc)
+            d += 1        
+        result['x'].append(task_desc)
+        
+        x_low = ''
+        for k in j['high_descs']:
+            if k[-1] == '.':
+                k = k[:-1]
+            k = k.lower()
+            k = ''.join(ch for ch in k if ch not in exclude)
+            x_low = x_low + k + '[SEP]'
+        x_low = x_low[:-6]
+        result['x_low'].append(x_low)
+        n += 1
+        
+        # x_task
+        x_task[task_desc] = task
+        path = task_to_path[task]
+        with open(path) as f:
+            traj_data = json.load(f)
+        params = dict()
+        task_type = task_types[traj_data['task_type']]
+        params['task_type'] = task_type
+        if traj_data['pddl_params']['mrecep_target'] != "":
+            params['mrecep_target'] = traj_data['pddl_params']['mrecep_target']
+        else:
+            params['mrecep_target'] = None
+        params['object_target'] = traj_data['pddl_params']['object_target']
+        if traj_data['pddl_params']['parent_target'] != "":
+            params['parent_target'] = traj_data['pddl_params']['parent_target']
+        else:
+            params['parent_target'] = None
+        if traj_data['pddl_params']['object_sliced']:
+            params['sliced'] = 1
+        else:
+            params['sliced'] = 0
+                    
+        desc_to_gt_params[task_desc] = params
+        
+        
+        
+        
+# pickle.dump(result, open(args.output_name + ".p", "wb"))
+pickle.dump(result, open(args.split + '_text_with_ppdl_low_appended.p', 'wb'))
+# pickle.dump(x_task, open(args.split + '_task_desc_to_task_id.p', 'wb'))
+pickle.dump(desc_to_gt_params, open('../../../instruction2_params_test_unseen_noappended_GT.p', 'wb'))
+print(f"num of annotations: {n}")
+print(f"num of duplication: {d}")
+print(f"num of not duplication: {n-d}")

models/instructions_processed_LP/BERT/data/alfred_data/create_text_with_pddl_low_appended_for_new_split.py

@@ -0,0 +1,191 @@
+import os
+import pandas as pd
+import argparse
+import json
+import pickle
+import string
+parser = argparse.ArgumentParser()
+parser.add_argument('--data_path', type=str, default="/media/user/data2/FILM/alfred_data_all/json_2.1.0", help="where to look for the generated data") # annotation path
+parser.add_argument('--split', type=str, default="valid_unseen")
+# parser.add_argument('-o','--output_name', type=str)
+args = parser.parse_args()
+data_path = args.data_path
+split = args.split
+exclude = set(string.punctuation)
+result = dict()
+
+# Path of 'traj_data.json' files that were created when the trajectory was generated
+# traj_data_path = "/media/user/data/alfred_4.3.0/gen/dataset/Finished"
+
+
+traj_data_path = "/media/user/data2/FILM/alfred_data_all/Re_json_2.1.0"
+task_to_path = dict()
+desc_to_gt_params = dict()     # 'wash the brown vegetable and put it on the counter’: 
+JSON_FILENAME = "traj_data.json"
+for dir_name, _, _ in os.walk(traj_data_path):
+    if "trial_" in dir_name and (not "raw_images" in dir_name) and (not "pddl_states" in dir_name) and (not "video" in dir_name):
+        json_file = os.path.join(dir_name, JSON_FILENAME)
+        if not os.path.isfile(json_file):
+            continue
+        task = json_file.split('/')[-2]
+        task_to_path[task] = json_file
+# print(task_to_path)
+
+task_types = {"pick_cool_then_place_in_recep": 0, "pick_and_place_with_movable_recep": 1, "pick_and_place_simple": 2, "pick_two_obj_and_place": 3, "pick_heat_then_place_in_recep": 4, "look_at_obj_in_light": 5, "pick_clean_then_place_in_recep": 6}
+
+# 'task_desc': task
+# {'wash the brown vegetable and put it on the counter’: 'trial_T20230514_192232_882070',
+# 'wash the brown vegetable and put it on the counter’: 'trial_T20230514_192232_882070', ...}
+x_task = dict()
+
+if args.split in ['tests_seen', 'tests_unseen']:
+    result['x'] = []; result['x_low'] = []
+if args.split in ['train', 'valid_seen', 'valid_unseen']:
+    result['x'] = []; result['y'] = []; result['s'] = []; result['mrecep_targets'] = []; result['object_targets'] = []; result['parent_targets'] = []; result['toggle_targets'] = []; result['x_low'] = []
+
+n = 0
+d = 0
+# with open('../../../../../alfred_data_small/splits/REALFRED_splits.json', 'r') as f:
+# with open('alfred_data_small/splits/REALFRED_splits.json', 'r') as f:
+# with open('../../../../../alfred_data_small/splits/aug28.json', 'r') as f:
+
+# with open('../../../../../alfred_data_small/splits/oct24.json', 'r') as f:
+with open('/media/user/data2/FILM/alfred_data_small/splits/oct24.json', 'r') as f:
+    splits = json.load(f)
+tasks = list()
+for i in splits[split]:
+    task = i["task"]
+    print(task)
+    if task not in tasks:
+        tasks.append(task)
+print("\n\n")
+for task in tasks:
+    with open(os.path.join(traj_data_path,task,'pp','ann_0.json')) as f:
+        ann_0 = json.load(f)
+
+    # anns = ann_0['turk_annotations']['anns']    # anns = [{"assignment_id", "high_descs", "task_desc"},{},{}]
+
+    ##### Without annotation #####
+    # anns = [ann_0['template']]
+
+    #0514
+    anns = ann_0['turk_annotations']['anns']
+
+    for j in anns:
+        task_desc = j['task_desc']
+        if len(task_desc)> 0:
+            if task_desc[-1] == '.':
+                task_desc = task_desc[:-1]
+        task_desc = task_desc.lower()
+        task_desc = ''.join(ch for ch in task_desc if ch not in exclude)
+        if task_desc in result['x']:
+            print(task, task_desc)
+            print("====")
+            d += 1        
+        result['x'].append(task_desc)
+
+        x_low = ''
+
+
+        ##### without template #####
+        # for k in j['high_descs']:
+
+        ##### Without annotation #####
+        # for k in j['high_descs'][:-1]:
+        for k in j['high_descs']:
+            if len(k)>0:
+                if k[-1] == '.':
+                    k = k[:-1]
+            k = k.lower()
+            k = ''.join(ch for ch in k if ch not in exclude)
+            x_low = x_low + k + '[SEP]'
+        x_low = x_low[:-5]
+        result['x_low'].append(x_low)
+        n += 1
+        
+        
+        # Extract from traj_data.json (GT)
+        # x_task
+        x_task[task_desc] = task
+
+        task_trial = task[-29:]
+        path = task_to_path[task_trial]
+        with open(path) as f:
+            traj_data = json.load(f)
+        
+        params = dict()
+        task_type = task_types[traj_data['task_type']]
+        params['task_type'] = task_type
+        if traj_data['pddl_params']['mrecep_target'] != "":
+            params['mrecep_target'] = traj_data['pddl_params']['mrecep_target']
+        else:
+            params['mrecep_target'] = None
+        params['object_target'] = traj_data['pddl_params']['object_target']
+        if traj_data['pddl_params']['parent_target'] != "":
+            params['parent_target'] = traj_data['pddl_params']['parent_target']
+        else:
+            params['parent_target'] = None
+        if traj_data['pddl_params']['object_sliced']:
+            params['sliced'] = 1
+        else:
+            params['sliced'] = 0
+        if traj_data['pddl_params']['toggle_target'] != "":
+            params['toggle_target'] = traj_data['pddl_params']['toggle_target']
+        else:
+            params['toggle_target'] = None
+        
+        desc_to_gt_params[task_desc] = params
+        
+        if args.split in ['train', 'valid_seen', 'valid_unseen']:
+            # obj2idx_new_split, recep2idx_new_split
+            import pickle
+            # obj2idx = pickle.load(open('alfred_dicts/obj2idx_new_split.p', 'rb'))
+            # recep2idx = pickle.load(open('alfred_dicts/recep2idx_new_split.p', 'rb'))
+            # toggle2idx = pickle.load(open('alfred_dicts/toggle2idx.p', 'rb'))
+            obj2idx = pickle.load(open('models/instructions_processed_LP/BERT/data/alfred_data/alfred_dicts/obj2idx_new_split.p', 'rb'))
+            recep2idx = pickle.load(open('models/instructions_processed_LP/BERT/data/alfred_data/alfred_dicts/recep2idx_new_split.p', 'rb'))
+            toggle2idx = pickle.load(open('models/instructions_processed_LP/BERT/data/alfred_data/alfred_dicts/toggle2idx.p', 'rb'))
+            
+            result['y'].append(task_type)
+            
+            if traj_data['pddl_params']['object_sliced']:
+                result['s'].append(1)
+            else:
+                result['s'].append(0)
+            
+            if traj_data['pddl_params']['mrecep_target'] != "":
+                result['mrecep_targets'].append(obj2idx[traj_data['pddl_params']['mrecep_target']])
+            else:
+                result['mrecep_targets'].append(obj2idx[None])                
+
+            result['object_targets'].append(obj2idx[traj_data['pddl_params']['object_target']])
+            
+            if traj_data['pddl_params']['parent_target'] != "":
+                result['parent_targets'].append(recep2idx[traj_data['pddl_params']['parent_target']])
+            else:
+                result['parent_targets'].append(recep2idx[None])
+                
+            if traj_data['pddl_params']['toggle_target'] != "":
+                result['toggle_targets'].append(toggle2idx[traj_data['pddl_params']['toggle_target']])
+            else:
+                result['toggle_targets'].append(toggle2idx[None])
+
+
+# pickle.dump(result, open(args.output_name + ".p", "wb"))
+# pickle.dump(result, open(args.split + '_text_with_ppdl_low_appended_new_split_aug28_t.p', 'wb'))
+# pickle.dump(x_task, open(args.split + '_task_desc_to_task_id_aug28_t.p', 'wb'))
+# pickle.dump(desc_to_gt_params, open('../../../instruction2_params_' + args.split + '_new_split_GT_aug28_t.p', 'wb'))
+# print(f"num of annotations: {n}")
+# print(f"num of duplication: {d}")
+# print(f"num of not duplication: {n-d}")
+
+# 0514
+# pickle.dump(result, open(args.split + '_text_with_ppdl_low_appended_new_split_oct24.p', 'wb'))
+# pickle.dump(x_task, open(args.split + '_task_desc_to_task_id_oct24.p', 'wb'))
+# pickle.dump(desc_to_gt_params, open('../../../instruction2_params_' + args.split + '_new_split_GT_oct24.p', 'wb'))
+pickle.dump(result, open('models/instructions_processed_LP/BERT/data/alfred_data/'+args.split + '_text_with_ppdl_low_appended_new_split_oct24.p', 'wb'))
+pickle.dump(x_task, open('models/instructions_processed_LP/BERT/data/alfred_data/'+args.split + '_task_desc_to_task_id_oct24.p', 'wb'))
+pickle.dump(desc_to_gt_params, open('models/instructions_processed_LP/instruction2_params_' + args.split + '_new_split_GT_oct24.p', 'wb'))
+print(f"num of annotations: {n}")
+print(f"num of duplication: {d}")
+print(f"num of not duplication: {n-d}")

models/instructions_processed_LP/BERT/data/alfred_data/train_task_desc_to_task_id_oct24.p

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d48d00106a760795caef5f1dbbe76162925fd450a43c2bfeb406453974b26f3
+size 2003819

models/instructions_processed_LP/BERT/data/alfred_data/train_text_with_ppdl_low_appended_new_split_oct24.p

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33039ca77d04ab3e9806d7de5e687620144dbcffc42efcead167c77054e69307
+size 9665585

models/instructions_processed_LP/BERT/end_to_end_outputs.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Feb  3 23:52:39 2021
+
+@author: soyeonmin
+"""
+
+#Run base model (into templates) and then extract arguments
+
+import random
+import time
+import torch
+from torch import nn
+import pickle
+import glob
+import argparse
+import os
+parser = argparse.ArgumentParser()
+parser.add_argument('-sp','--split', type=str, choices=['valid_unseen', 'valid_seen', 'tests_seen', 'tests_unseen'], required=True)
+parser.add_argument('-m','--model_saved_folder_name', type=str, required=True, default='best_models')
+parser.add_argument('-o','--output_name', type=str, required=True)
+parser.add_argument('--no_appended', action='store_true')
+
+args = parser.parse_args()
+
+
+def accuracy(y_pred, y_batch):
+    #y_pred has shape [batch, no_classes]
+    maxed = torch.max(y_pred, 1)
+    y_hat = maxed.indices
+    num_accurate = torch.sum((y_hat == y_batch).long())
+    train_accuracy = num_accurate/ y_hat.shape[0]
+    return train_accuracy.item()
+
+def accurate_both(y_pred1, y_batch1, y_pred2, y_batch2):
+    #
+    maxed1 = torch.max(y_pred1, 1)
+    y_hat1 = maxed1.indices
+    #
+    maxed2 = torch.max(y_pred2, 1)
+    y_hat2 = maxed2.indices
+    #
+    num_both_accurate = torch.sum((y_hat1 == y_batch1).long() * (y_hat2 == y_batch2).long())
+    train_accuracy = num_both_accurate/ y_hat1.shape[0]
+    return train_accuracy.item()
+    
+
+    
+
+
+#Load data
+import pickle
+val_set_unseen = pickle.load(open('data/alfred_data/'+ args.split + '_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+desc2id = pickle.load(open('data/alfred_data/'+ args.split + '_task_desc_to_task_id_oct24.p', 'rb'))
+
+# REALFRED new split
+obj2idx = pickle.load(open('data/alfred_data/alfred_dicts/obj2idx_new_split.p', 'rb'))
+recep2idx = pickle.load(open('data/alfred_data/alfred_dicts/recep2idx_new_split.p', 'rb'))
+
+toggle2idx = pickle.load(open('data/alfred_data/alfred_dicts/toggle2idx.p', 'rb'))
+
+idx2obj = {v:k for k, v in obj2idx.items()}
+idx2recep = {v:k for k, v in recep2idx.items()}
+idx2toggle = {v:k for k, v in toggle2idx.items()}
+
+#These are based on new labels
+task_to_label_mapping = {'mrecep_target':[1], 'object_target': [0,1,2,3,4,5,6],\
+                         'parent_target':[0,1,2,3,4,6], 'toggle_target': [5],
+                         'sliced':[0,1,2,3,4,5,6]}
+    
+#Set device
+if torch.cuda.is_available():
+    device = torch.device('cuda')
+else:
+    device = torch.device('cpu')
+
+
+save_folder_name = args.model_saved_folder_name
+#Base model
+from transformers import BertForSequenceClassification
+base_model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=7).to(device)
+
+base_model_name = 'base.pt'
+base_model.load_state_dict(torch.load(os.path.join(save_folder_name, base_model_name)))
+base_model.eval()
+
+#Run data through base model and get label 
+from transformers import BertTokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+
+#Put validation data into the base_model
+x_val_seen = val_set_unseen['x_low'] #; y_val_seen = val_set_seen['y']
+if args.no_appended:
+    x_val_seen = val_set_unseen['x']
+
+
+encoding_v_s = tokenizer(x_val_seen, return_tensors='pt', padding=True, truncation=True)
+input_ids_val_seen = encoding_v_s['input_ids'].to(device)
+attention_mask_val_seen = encoding_v_s['attention_mask'].to(device) 
+
+N= 10
+y_hat_list_vs = []
+if input_ids_val_seen.shape[0]%N!=0:
+    until = int(input_ids_val_seen.shape[0]/N)+1
+else:
+    until = int(input_ids_val_seen.shape[0]/N)
+for b in range(until):
+    input_ids_batch = input_ids_val_seen[N*b:N*(b+1)].to(device)
+    attention_mask_batch = attention_mask_val_seen[N*b:N*(b+1)].to(device)    
+    outputs = base_model(input_ids_batch, attention_mask=attention_mask_batch)
+    predicted_templates = torch.max(outputs.logits, 1).indices
+    y_hat_list_vs += predicted_templates.cpu().numpy().tolist()
+
+del outputs
+del base_model
+vs_idx2predicted_label = {i:y for i, y in enumerate(y_hat_list_vs)}
+
+#Now extract the arguments 
+global c
+c = 0
+def get_prediction(classifier, N, input_ids, attention_mask):
+    y_hat_list = []
+    for b in range(int(input_ids.shape[0]/N)+1):
+        if b!=int(input_ids.shape[0]/N):
+            input_ids_batch = input_ids[N*b:N*(b+1)].to(device)
+        else:
+            input_ids_batch = input_ids[N*b:].to(device)
+        attention_mask_batch = attention_mask[N*b:N*(b+1)].to(device)
+        
+        #outputs = base_model(input_ids_batch, attention_mask=attention_mask_batch, labels=labels_batch.view(1,-1))
+        # print('input_ids_batch: ', input_ids_batch.shape)
+        outputs = classifier(input_ids_batch, attention_mask=attention_mask_batch)
+        global c
+        c += 1
+        try:
+            print(c, torch.max(outputs.logits, 1).indices)
+        except:
+            print(outputs)
+            print(c)
+        # print(outputs.logits)
+        predicted_templates = torch.max(outputs.logits, 1).indices
+        del outputs
+        y_hat_list += predicted_templates.cpu().numpy().tolist()
+    return y_hat_list
+
+x_val_seen_p = [str(int(vs_idx2predicted_label[i])) + ' ' + x for i, x in enumerate(x_val_seen)]
+
+encoding_v_s = tokenizer(x_val_seen_p, return_tensors='pt', padding=True, truncation=True)
+print(encoding_v_s)
+input_ids_val_seen = encoding_v_s['input_ids'].to(device)
+print(input_ids_val_seen.shape)
+attention_mask_val_seen = encoding_v_s['attention_mask'].to(device) 
+
+parent_target_classifier = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(recep2idx)).to(device)   
+parent_target_classifier.load_state_dict(torch.load(os.path.join(save_folder_name, 'parent.pt')))
+parent_outputs_hat = get_prediction(parent_target_classifier, 9, input_ids_val_seen, attention_mask_val_seen)
+del parent_target_classifier
+c = 0
+
+object_target_classifier = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(obj2idx)).to(device)
+object_target_classifier.load_state_dict(torch.load(os.path.join(save_folder_name, 'object.pt')))
+object_outputs_hat = get_prediction(object_target_classifier, 9, input_ids_val_seen, attention_mask_val_seen)
+del object_target_classifier
+c = 0
+
+sliced_target_classifier = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2).to(device)
+sliced_target_classifier.load_state_dict(torch.load(os.path.join(save_folder_name, 'sliced.pt')))
+sliced_outputs_hat = get_prediction(sliced_target_classifier, 9, input_ids_val_seen, attention_mask_val_seen)
+del sliced_target_classifier
+c = 0
+
+mrecep_target_classifier = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(obj2idx)).to(device)
+mrecep_target_classifier.load_state_dict(torch.load(os.path.join(save_folder_name, 'mrecep.pt')))
+mrecep_outputs_hat = get_prediction(mrecep_target_classifier, 9, input_ids_val_seen, attention_mask_val_seen)
+del mrecep_target_classifier
+
+instructions= val_set_unseen['x_low']
+if args.no_appended:
+    instructions= val_set_unseen['x']
+instructions= val_set_unseen['x']    
+instruction2_params_test_unseen = {}
+for i, instruction in enumerate(instructions):
+    task_type = vs_idx2predicted_label[i]
+    object_target = idx2obj[object_outputs_hat[i]]
+    if parent_outputs_hat == None:
+        parent_target = None
+    else:
+        parent_target = idx2recep[parent_outputs_hat[i]]
+    if mrecep_outputs_hat == None:
+        mrecep_target = None
+    else:
+        mrecep_target = idx2obj[mrecep_outputs_hat[i]]
+    sliced_target = sliced_outputs_hat[i]
+    
+    if task_type == 5:
+        parent_target = None
+    # if task_type !=1:
+    #     mrecep_target = None
+
+    instruction2_params_test_unseen[instruction] = {'task_type': task_type, \
+                                                    'mrecep_target': mrecep_target,\
+                                                    'sliced': sliced_target,\
+                                                      'object_target': object_target,\
+                                                      'parent_target': parent_target}
+
+
+pickle.dump(instruction2_params_test_unseen, open("../" + args.output_name + ".p", "wb"))
+
+

models/instructions_processed_LP/BERT/train_bert_args.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Feb  3 21:49:31 2021
+
+@author: soyeonmin
+"""
+from tensorboardX import SummaryWriter
+from tqdm import trange
+import random
+import time
+import torch
+from torch import nn
+import os
+import glob
+from collections import OrderedDict
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('-lr','--learning_rate', type=float, default=1e-5, help="learning rate")
+parser.add_argument('-s','--seed', type=int, default=0, help="seed")
+parser.add_argument('-l','--label', type=int, default=1, help="template")
+parser.add_argument('-d','--decay', type=float, default=0.5, help="template")
+parser.add_argument('-dt','--decay_term', type=int, default=50, help="template")
+parser.add_argument('-t','--task', type=str, help="type in one of mrecep, object, parent, toggle, sliced")
+parser.add_argument('-v','--verbose', type=int, default=0, help="print training output")
+parser.add_argument('-model_type','--model_type', type=str, default='bert-base', help="one of roberta-large, roberta-base, bert-base, bert-large")
+parser.add_argument('-load','--load', type=str, default='', help="one of roberta-large, roberta-base, bert-base, bert-large")
+parser.add_argument('-no_divided_label','--no_divided_label', action='store_true')
+parser.add_argument('--no_appended', action='store_true')
+
+
+args = parser.parse_args()
+
+#Set device
+if torch.cuda.is_available():
+    device = torch.device('cuda')
+else:
+    device = torch.device('cpu')
+    
+import pickle
+template_by_label = pickle.load(open('data/alfred_data/alfred_dicts/template_by_label.p', 'rb'))
+# train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended.p', 'rb'))
+# val_set_seen = pickle.load(open('data/alfred_data/val_seen_text_with_ppdl_low_appended.p', 'rb'))
+# val_set_unseen = pickle.load(open('data/alfred_data/val_unseen_text_with_ppdl_low_appended.p', 'rb'))
+
+# obj2idx = pickle.load(open('data/alfred_data/alfred_dicts/obj2idx.p', 'rb'))
+# recep2idx = pickle.load(open('data/alfred_data/alfred_dicts/recep2idx.p', 'rb'))
+# train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended_new_split.p', 'rb'))
+# val_set_seen = pickle.load(open('data/alfred_data/valid_seen_text_with_ppdl_low_appended_new_split.p', 'rb'))
+# val_set_unseen = pickle.load(open('data/alfred_data/valid_unseen_text_with_ppdl_low_appended_new_split.p', 'rb'))
+
+# 0514
+train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+val_set_seen = pickle.load(open('data/alfred_data/valid_seen_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+val_set_unseen = pickle.load(open('data/alfred_data/valid_unseen_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+
+obj2idx = pickle.load(open('data/alfred_data/alfred_dicts/obj2idx_new_split.p', 'rb'))
+recep2idx = pickle.load(open('data/alfred_data/alfred_dicts/recep2idx_new_split.p', 'rb'))
+toggle2idx = pickle.load(open('data/alfred_data/alfred_dicts/toggle2idx.p', 'rb'))
+
+assert args.task in ['mrecep', 'object', 'parent', 'toggle', 'sliced']
+if args.task == 'mrecep':
+    label_arg = 'mrecep_targets'; num_labels = len(obj2idx)
+elif args.task == 'object':
+    label_arg = 'object_targets'; num_labels = len(obj2idx)
+elif args.task == 'parent':
+    label_arg = 'parent_targets'; num_labels = len(recep2idx)
+elif args.task == 'toggle':
+    label_arg = 'toggle_targets'; num_labels = len(toggle2idx)
+elif args.task == 'sliced':
+    label_arg = 's'; num_labels = 2
+
+
+
+
+
+if args.model_type in ['bert-base', 'bert-large'] : 
+    from transformers import BertTokenizer as Tokenizer
+    tok_type = args.model_type + '-uncased'
+    from transformers import BertForSequenceClassification as BertModel
+elif args.model_type in ['roberta-base', 'roberta-large']:
+    from transformers import RobertaTokenizer as Tokenizer
+    tok_type = args.model_type
+    from transformers import RobertaForSequenceClassification as BertModel
+tokenizer = Tokenizer.from_pretrained(tok_type)
+
+torch.manual_seed(args.seed)
+model = BertModel.from_pretrained(tok_type, num_labels=num_labels)
+
+if len(args.load) >0:
+    sd = OrderedDict()
+    sd_ori = torch.load(args.load, map_location = 'cpu')
+    for k in sd_ori:
+        if 'classifier' in k:
+            if k =='classifier.weight':
+                sd[k] = model.classifier.weight
+            elif k =='classifier.bias':
+                sd[k] = model.classifier.bias
+        else:
+            sd[k] = sd_ori[k]
+    del sd_ori
+    
+    model.load_state_dict(sd)
+
+model = model.to(device)
+
+####################################################
+## 1. Prepare Data
+####################################################
+
+
+from random import shuffle
+import numpy as np
+x_train = train_set['x_low']; y_train = train_set['y']
+if args.no_appended:
+    x_train = train_set['x']
+#if args.no_divided_label:
+x_train = [str(y_train[i])+ ' ' + x for i, x in enumerate(x_train)]
+
+if args.task == 'sliced' or args.no_divided_label:
+    x_ones = [i for i in range(len(x_train))]
+else:
+    x_ones = [i for i, lab in enumerate(y_train) if lab == args.label]
+random.seed(0)
+shuffled = [i for i in x_ones]
+shuffle(shuffled)
+#x_train = np.array(train_set['x_low'])[shuffled]
+x_train = np.array(x_train)[shuffled]
+x_train = x_train.tolist()
+encoding = tokenizer(x_train, return_tensors='pt', padding=True, truncation=True)
+input_ids = encoding['input_ids'].to(device) #has shape [21025, 37]
+attention_mask = encoding['attention_mask'].to(device) #also has shape [21025, 37]
+
+label_train = np.array(train_set[label_arg])[shuffled]
+label_train = torch.tensor(label_train).to(device)
+
+
+
+x_val_seen = val_set_seen['x_low']; y_val_seen = val_set_seen['y']
+if args.no_appended:
+    x_val_seen = val_set_seen['x']
+x_val_seen = [str(y_val_seen[i])+ ' ' + x for i, x in enumerate(x_val_seen)]
+
+if args.task == 'sliced' or args.no_divided_label:
+    random.seed(0)
+    x_ones_vs = random.sample(range(len(x_val_seen)), 100)
+else:
+    x_ones_vs = [i for i, lab in enumerate(y_val_seen) if lab == args.label]
+x_val_seen = np.array(x_val_seen)[x_ones_vs].tolist()
+encoding_v_s = tokenizer(x_val_seen, return_tensors='pt', padding=True, truncation=True)
+input_ids_val_seen = encoding_v_s['input_ids'].to(device)
+attention_mask_val_seen = encoding_v_s['attention_mask'].to(device) 
+
+label_val_seen = [val_set_seen[label_arg][i] for i in x_ones_vs]
+label_val_seen = torch.tensor(label_val_seen).to(device)
+
+
+
+####################################################
+## 2. Do training
+####################################################
+
+model.train()
+from transformers import AdamW
+learning_rate = args.learning_rate
+optimizer = AdamW(model.parameters(), lr=learning_rate)
+if 'base' in args.model_type:
+    N= 64
+else:
+    N = 32
+
+def accuracy(y_pred, y_batch):
+    #y_pred has shape [batch, no_classes]
+    maxed = torch.max(y_pred, 1)
+    y_hat = maxed.indices
+    num_accurate = torch.sum((y_hat == y_batch).float())
+    train_accuracy = num_accurate/ y_hat.shape[0]
+    return train_accuracy.item()
+
+def accurate_total(y_pred, y_batch):
+    #y_pred has shape [batch, no_classes]
+    maxed = torch.max(y_pred, 1)
+    y_hat = maxed.indices
+    num_accurate = torch.sum((y_hat == y_batch).float())
+    return num_accurate
+
+
+if args.no_appended:
+    super_folder = 'saved_models_noappended_new_split_oct24/'
+else:
+    super_folder = 'saved_models_appended_new_split_oct24/'
+
+if args.task == 'sliced':
+    save_folder_name = args.task +'/' + args.model_type + '_lr_' + str(args.learning_rate) + 'seed_' + str(args.seed) + 'decay_' + str(args.decay) +'/'
+else:
+    save_folder_name = args.task +  str(args.label) + '/' + args.model_type+ '_lr_' + str(args.learning_rate) + 'seed_' + str(args.seed) + 'decay_' + str(args.decay) +'/'
+if not os.path.exists(super_folder+'argument_models/' + save_folder_name):
+    os.makedirs(super_folder+'argument_models/' + save_folder_name)
+    
+accuracy_dictionary = {'training_loss': [], 'training':[], 'test':[]}
+start_train_time = time.time()
+
+summary_writer = SummaryWriter(log_dir=super_folder+'argument_models/' + save_folder_name)
+train_iter = 0
+
+for t in trange(50):
+    model.train()
+    if t>0 and (t+1)%args.decay_term ==0:
+        learning_rate *= args.decay
+        optimizer = AdamW(model.parameters(), lr=learning_rate)
+    avg_training_loss = 0.0
+    training_acc = 0.0
+    for b in trange(int(input_ids.shape[0]/N)):
+        input_ids_batch = input_ids[N*b:N*(b+1)].to(device)
+        labels_batch = label_train[N*b:N*(b+1)].to(device)
+        attention_mask_batch = attention_mask[N*b:N*(b+1)].to(device)
+        optimizer.zero_grad()
+        #forward pass
+        outputs = model(input_ids_batch, attention_mask=attention_mask_batch, labels=labels_batch.view(1,-1))
+        loss = outputs.loss
+        summary_writer.add_scalar('train/loss', loss, train_iter)
+        
+        
+        num_acc = accurate_total(y_pred=outputs.logits, y_batch=labels_batch.view(-1))
+        #if t ==0:
+        #    print("loss at step ", t, " : ", loss.item())
+        loss.backward()
+        train_iter+=1
+        optimizer.step()
+        #
+        avg_training_loss += loss
+        training_acc += num_acc 
+    avg_training_loss *= 1/int(input_ids.shape[0]/N)
+    #Print & Evaluate
+    if args.verbose:
+        print("loss at step ", t, " : ", loss.item())
+        print("training accuracy: ", training_acc/input_ids.shape[0])
+    #evaluate
+    with torch.no_grad():
+        model.eval()
+        outputs_val_seen = model(input_ids_val_seen, attention_mask=attention_mask_val_seen)
+        val_seen_acc = accuracy(y_pred=outputs_val_seen.logits, y_batch=label_val_seen.view(-1))
+        if args.verbose:
+            print("validation accuracy (seen): ", val_seen_acc)
+        del outputs_val_seen
+    
+        model_name = 'epoch_' + str(t) + '.pt'
+        torch.save(model.state_dict(), super_folder+'argument_models/' + save_folder_name + model_name)
+        accuracy_dictionary['training_loss'].append(loss.item())
+        accuracy_dictionary['training'].append(training_acc/input_ids.shape[0])
+        accuracy_dictionary['test'].append(val_seen_acc)
+        summary_writer.add_scalar('valid/accuracy', val_seen_acc, train_iter)
+        
+        print("Epoch " + str(t) + "\n")
+        print("training loss: " + str(accuracy_dictionary['training_loss'][t]))
+        print("training accuracy: " + str(accuracy_dictionary['training'][t]))
+        print("validation (seen) accuracy: " + str(accuracy_dictionary['test'][t]))
+    
+#Get the highest accuracy and delete the rest 
+highest_test = np.argwhere(accuracy_dictionary['test'] == np.amax(accuracy_dictionary['test']))
+highest_test = highest_test.flatten().tolist()
+
+training_acc_highest_h = np.argmax([accuracy_dictionary['training'][h].detach().cpu().numpy() for h in highest_test])
+best_t = highest_test[training_acc_highest_h]
+print("The best model is 'epoch_"+ str(best_t)+ '.pt')
+
+#Delete every model except for best_t 
+file_path = super_folder+'argument_models/' + save_folder_name + "epoch_" + str(best_t) + ".pt"
+if os.path.isfile(file_path):
+    for CleanUp in glob.glob(super_folder+'argument_models/' + save_folder_name + '*.pt'):
+        if not CleanUp.endswith(file_path):    
+            os.remove(CleanUp)
+
+#Save training/ test accuracy dictionary in a txt file
+f = open(super_folder+ 'argument_models/' + save_folder_name +  "training_log.txt", "w")
+for t in range(100):
+
+    if t >= len(accuracy_dictionary['training_loss']):
+        break
+    if t == best_t:
+        f.write("===========================================\n")
+        # print("===========================================\n")
+    f.write("Epoch " + str(t) + "\n")
+    f.write("training loss: " + str(accuracy_dictionary['training_loss'][t]) + "\n")
+    f.write("training accuracy: " + str(accuracy_dictionary['training'][t]) + "\n")
+    f.write("validation (seen) accuracy: " + str(accuracy_dictionary['test'][t]) + "\n")
+    
+    # print("Epoch " + str(t) + "\n")
+    # print("training loss: " + str(accuracy_dictionary['training_loss'][t]) + "\n")
+    # print("training accuracy: " + str(accuracy_dictionary['training'][t]) + "\n")
+    # print("validation (seen) accuracy: " + str(accuracy_dictionary['test'][t]) + "\n")
+    if t == best_t:
+        f.write("===========================================\n")
+        # print("===========================================\n")
+f.close()
+
+
+
+#Print training/ test accuracy for t
+print("Saved and finished")
+

models/instructions_processed_LP/BERT/train_bert_base.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jan 20 11:41:44 2021
+
+@author: soyeonmin
+"""
+
+#Train BERT on ALFRED data
+import tensorboard
+from tensorboardX import SummaryWriter
+import random
+import time
+import torch
+from torch import nn
+import os
+from tqdm import trange
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('-lr','--learning_rate', type=float, help="learning rate")
+parser.add_argument('-model_type','--model_type', type=str, default='bert-base', help="one of roberta-large, roberta-base, bert-base, bert-large")
+parser.add_argument('-task_desc_only','--task_desc_only', action='store_true', help="one of roberta-large, roberta-base, bert-base, bert-large")
+parser.add_argument('--no_appended', action='store_true')
+
+
+args = parser.parse_args()
+
+
+##########
+#### 1. Data loading and processing
+#########
+
+#Set device
+if torch.cuda.is_available():
+    device = torch.device('cuda')
+else:
+    device = torch.device('cpu')
+    
+
+import pickle
+directory = 'data/alfred_data/'
+# train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended.p', 'rb'))
+# val_set_seen = pickle.load(open('data/alfred_data/val_seen_text_with_ppdl_low_appended.p', 'rb'))
+# val_set_unseen = pickle.load(open('data/alfred_data/val_unseen_text_with_ppdl_low_appended.p', 'rb'))
+
+# train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended_new_split.p', 'rb'))
+# val_set_seen = pickle.load(open('data/alfred_data/valid_seen_text_with_ppdl_low_appended_new_split.p', 'rb'))
+# val_set_unseen = pickle.load(open('data/alfred_data/valid_unseen_text_with_ppdl_low_appended_new_split.p', 'rb'))
+
+train_set = pickle.load(open('data/alfred_data/train_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+val_set_seen = pickle.load(open('data/alfred_data/valid_seen_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+val_set_unseen = pickle.load(open('data/alfred_data/valid_unseen_text_with_ppdl_low_appended_new_split_oct24.p', 'rb'))
+
+#Now process like huggingface
+if args.no_appended:
+    x_train = train_set['x']; y_train = train_set['y']
+else:
+    x_train = train_set['x_low']; y_train = train_set['y']
+labels = torch.tensor(y_train).to(device)
+
+if args.model_type in ['bert-base', 'bert-large'] : 
+    from transformers import BertTokenizer as Tokenizer
+    tok_type = args.model_type + '-uncased'
+    from transformers import BertForSequenceClassification as BertModel
+elif args.model_type in ['roberta-base', 'roberta-large']:
+    from transformers import RobertaTokenizer as Tokenizer
+    tok_type = args.model_type
+    from transformers import RobertaForSequenceClassification as BertModel
+tokenizer = Tokenizer.from_pretrained(tok_type)
+
+model = BertModel.from_pretrained(tok_type, num_labels=7).to(device)
+encoding = tokenizer(x_train, return_tensors='pt', padding=True, truncation=True)
+
+input_ids = encoding['input_ids'].to(device) #has shape [21025, 37]
+attention_mask = encoding['attention_mask'].to(device) #also has shape [21025, 37]
+
+
+#Get input_ids, attention_mask for val_seen, val_unseen
+x_val_seen = val_set_seen['x_low']; y_val_seen = val_set_seen['y']
+if args.no_appended:
+    x_val_seen = val_set_seen['x']
+#Just sample a few from x_val_seen, y_val_seen
+random.seed(0)
+sampled_val_seen = random.sample(range(len(x_val_seen)), 100)
+x_val_seen = [x_val_seen[i] for i in sampled_val_seen]; y_val_seen = [y_val_seen[i] for i in sampled_val_seen]
+labels_val_seen = torch.tensor(y_val_seen).to(device)
+encoding_v_s = tokenizer(x_val_seen, return_tensors='pt', padding=True, truncation=True)
+input_ids_val_seen = encoding_v_s['input_ids'].to(device)
+attention_mask_val_seen = encoding_v_s['attention_mask'].to(device)
+
+x_val_unseen = val_set_unseen['x_low']; y_val_unseen = val_set_unseen['y']
+if args.no_appended:
+    x_val_unseen = val_set_unseen['x']
+random.seed(1)
+sampled_val_unseen = random.sample(range(len(x_val_unseen)), 100)
+x_val_unseen = [x_val_unseen[i] for i in sampled_val_unseen]; y_val_unseen = [y_val_unseen[i] for i in sampled_val_unseen]
+labels_val_unseen = torch.tensor(y_val_unseen).to(device)
+encoding_v_us = tokenizer(x_val_unseen, return_tensors='pt', padding=True, truncation=True)
+input_ids_val_unseen = encoding_v_us['input_ids'].to(device)
+attention_mask_val_unseen = encoding_v_us['attention_mask'].to(device)
+
+
+
+##########
+#### 2. Do training
+#########
+
+model.train()
+from transformers import AdamW
+optimizer = AdamW(model.parameters(), lr=args.learning_rate)
+N = 16 #batch size
+if args.no_appended:
+    N=64
+save_folder_name = args.model_type + '_lr_' + str(args.learning_rate) +'/'
+if args.no_appended:
+    super_folder = 'saved_models_noappended_new_split_oct24/'
+else:
+    super_folder = 'saved_models_appended_new_split_oct24/'
+if not os.path.exists(super_folder + save_folder_name):
+    os.makedirs(super_folder + save_folder_name)
+
+
+def accuracy(y_pred, y_batch):
+    #y_pred has shape [batch, no_classes]
+    maxed = torch.max(y_pred, 1)
+    y_hat = maxed.indices
+    num_accurate = torch.sum((y_hat == y_batch).float())
+    train_accuracy = num_accurate/ y_hat.shape[0]
+    return train_accuracy.item()
+
+def accurate_total(y_pred, y_batch):
+    #y_pred has shape [batch, no_classes]
+    maxed = torch.max(y_pred, 1)
+    y_hat = maxed.indices
+    num_accurate = torch.sum((y_hat == y_batch).float())
+    return num_accurate
+
+start_train_time = time.time()
+summary_writer = SummaryWriter(log_dir=super_folder + save_folder_name)
+train_iter = 0
+for t in trange(50):
+    model.train()
+    avg_training_loss = 0.0
+    training_acc = 0.0
+    for b in trange(int(input_ids.shape[0]/N)):
+        input_ids_batch = input_ids[N*b:N*(b+1)].to(device)
+        labels_batch = labels[N*b:N*(b+1)].to(device)
+        attention_mask_batch = attention_mask[N*b:N*(b+1)].to(device)
+        optimizer.zero_grad()
+        #forward pass
+        outputs = model(input_ids_batch, attention_mask=attention_mask_batch, labels=labels_batch.view(1,-1))
+        loss = outputs.loss
+        summary_writer.add_scalar('train/loss', loss, train_iter)
+
+        
+        
+        num_acc = accurate_total(y_pred=outputs.logits, y_batch=labels_batch.view(-1))
+        loss.backward()
+        train_iter+=1
+        optimizer.step()
+        #
+        avg_training_loss += loss
+        training_acc += num_acc 
+    avg_training_loss *= 1/int(input_ids.shape[0]/N)
+    #Print & Evaluate
+    if t % 1 ==0:
+        with torch.no_grad():
+            model.eval()
+            print("loss at step ", t, " : ", loss.item())
+            print("training accuracy: ", training_acc/input_ids.shape[0])
+            #evaluate
+            outputs_val_seen = model(input_ids_val_seen, attention_mask=attention_mask_val_seen)
+            print("validation accuracy (seen): ", accuracy(y_pred=outputs_val_seen.logits, y_batch=labels_val_seen.view(-1)))
+            del outputs_val_seen
+            outputs_val_unseen = model(input_ids_val_unseen, attention_mask=attention_mask_val_unseen)
+            print("validation accuracy (unseen): ", accuracy(y_pred=outputs_val_unseen.logits, y_batch=labels_val_unseen.view(-1)))
+            
+            model_name = 'epoch_' + str(t) + '.pt'
+            torch.save(model.state_dict(), super_folder + save_folder_name + model_name)

models/instructions_processed_LP/compare_BERT_pred_with_GT_oct24.py

@@ -0,0 +1,83 @@
+import pickle
+
+splits = ["valid_seen","valid_unseen","tests_seen", "tests_unseen"]
+split = "valid_unseen"
+
+pred = pickle.load(open('instruction2_params_'+split+'_appended_new_split_oct24.p','rb'))
+gt = pickle.load(open('instruction2_params_'+split+'_new_split_GT_oct24.p','rb'))
+
+task_desc_list = list(pred.keys())
+# gt = list(gt.items())
+task_types = {"pick_cool_then_place_in_recep": 0, "pick_and_place_with_movable_recep": 1, "pick_and_place_simple": 2, "pick_two_obj_and_place": 3, "pick_heat_then_place_in_recep": 4, "look_at_obj_in_light": 5, "pick_clean_then_place_in_recep": 6}
+
+c0=0; c1=0; c2=0; c3=0; c4=0; c5=0; c6=0
+nt=0; nm=0; no=0; np=0; ns=0; nt=0
+parent_pred = []; parent_gt=[]; parents = []
+# num_of_tables_in_parent = 0
+for i in range(len(pred)):
+    n = 0
+    task = task_desc_list[i]
+    print("##########################################################################")
+    print(task)
+    if pred[task]['task_type'] == gt[task]['task_type']:
+        nt += 1; n += 1
+    else:
+        print('Task type: ', pred[task]['task_type'], gt[task]['task_type'])
+        
+    if pred[task]['mrecep_target'] == gt[task]['mrecep_target']:
+        nm += 1; n += 1
+    else:
+        print('mrecep: ', pred[task]['mrecep_target'], gt[task]['mrecep_target'])
+        
+    if pred[task]['object_target'] == gt[task]['object_target']:
+        no += 1; n += 1
+    else:
+        print('object_target: ', pred[task]['object_target'], gt[task]['object_target'])
+        
+    if pred[task]['parent_target'] == gt[task]['parent_target']:
+        np += 1; n += 1
+
+    else:
+        print('parent_target: ', pred[task]['parent_target'], gt[task]['parent_target'])
+        parent_pred.append(pred[task]['parent_target'])
+        parent_gt.append(gt[task]['parent_target'])
+        parents.append([pred[task]['parent_target'],gt[task]['parent_target']])
+        
+    if pred[task]['sliced'] == gt[task]['sliced']:
+        ns += 1; n += 1
+    else:
+        print('sliced: ', pred[task]['sliced'], gt[task]['sliced'])
+    # if pred[task]['toggle_target'] == gt[task]['toggle_target']:
+    #     nt += 1; n += 1
+    print("##########################################################################")
+
+    if n == 0:
+        c0 += 1
+    elif n == 1:
+        c1 += 1
+    elif n == 2:
+        c2 += 1
+    elif n == 3:
+        c3 += 1
+    elif n == 4:
+        c4 += 1
+    elif n == 5:
+        c5 += 1
+
+print("################################################")
+print(split)
+print(f"Total number of task_desc: {len(pred), len(gt)}\n")
+print(f"task_type accuracy: {round(nt/len(pred)*100, 2)}")
+print(f"mrecep_target accuracy: {round(nm/len(pred)*100, 2)}")
+print(f"object_target accuracy: {round(no/len(pred)*100, 2)}")
+print(f"parent_target accuracy: {round(np/len(pred)*100, 2)}")
+print(f"sliced accuracy: {round(ns/len(pred)*100, 2)}")
+print()
+print(f"0 Correct: {round(c0/len(pred)*100, 2)}")
+print(f"1 Correct: {round(c1/len(pred)*100, 2)}")
+print(f"2 Correct: {round(c2/len(pred)*100, 2)}")
+print(f"3 Correct: {round(c3/len(pred)*100, 2)}")
+print(f"4 Correct: {round(c4/len(pred)*100, 2)}")
+print(f"5 Correct: {round(c5/len(pred)*100, 2)}")
+print('\nTotal all correct accuracy: ', str(round(c5/len(pred)*100, 2)))
+print("################################################")