non-LFS commit

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.glb filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,152 @@
+import gradio as gr
+import os
+import re
+
+from leo.inference import inference
+
+MESH_DIR = 'assets/mesh'
+MESH_NAMES = sorted([os.path.splitext(fname)[0] for fname in os.listdir(MESH_DIR)])
+STEP_COUNTS = 6
+
+def change_scene(dropdown_scene: str):
+    # reset 3D scene and chatbot history
+    return os.path.join(MESH_DIR, f'{dropdown_scene}.glb')
+
+with gr.Blocks(title='LEO Demo') as demo:
+    gr.HTML(value="<h1 align='center'>Task-oriented Sequential Grounding in 3D Scenes </h1>")
+
+    with gr.Row():
+        with gr.Column(scale=5):
+            dropdown_scene = gr.Dropdown(
+                choices=MESH_NAMES,
+                value='scene0050_00',
+                interactive=True,
+                label='Select a 3D scene',
+            )
+            model_3d = gr.Model3D(
+                value=os.path.join(MESH_DIR, f'scene0050_00.glb'),
+                clear_color=[0.0, 0.0, 0.0, 0.0],
+                label='3D Scene',
+                camera_position=(80, 100, 6),
+                height=659,
+            )
+            gr.HTML(
+                """<center><strong>
+                👆 SCROLL and DRAG on the 3D Scene
+                to zoom in/out and rotate. Press CTRL and DRAG to pan.
+                </strong></center>
+                """
+            )
+    
+    dropdown_scene.change(
+        fn=change_scene,
+        inputs=[dropdown_scene],
+        outputs=[model_3d],
+        queue=False
+    )
+
+    # LEO task-to-plan inference wrapper
+    def leo_task_to_plan(task_description):
+        task_input = {
+            "task_description": task_description,
+            "scan_id": "scene0050_00"
+        }
+        plan = inference("scene0050_00", task_input, predict_mode=True)
+        plan = plan[0]['pred_plan_text']
+        # parts = re.split(r'(\d+\.)', plan)[1:]
+        # steps = [parts[i] + parts[i + 1].rstrip() for i in range(0, len(parts), 2)]
+        return plan
+
+    # LEO ground inference wrapper
+    def leo_plan_to_masks(task_description, *action_steps):
+        formatted_action_steps = [
+            {"action": step, "target_id": "unknown", "label": "unknown"} for step in action_steps if step != ""
+        ]
+        task_input = {
+            "task_description": task_description,
+            "action_steps": formatted_action_steps,
+            "scan_id": "scene0050_00"
+        }
+        masks = inference("scene0050_00", task_input, predict_mode=False)
+        masks = [tensor.item() for tensor in masks]
+        return [f"assets/mask/scene0050_00/scene0050_00_obj_{mask}.glb" for mask in masks] + ["assets/mask/scene0050_00/scene0050_00_obj_empty.glb"] * (STEP_COUNTS - len(masks))
+
+    # LEO task-to-plan and ground inference wrapper
+    def leo_task_to_plan_and_masks(task_description):
+        task_input = {
+            "task_description": task_description,
+            "scan_id": "scene0050_00"
+        }
+        plan = inference("scene0050_00", task_input, predict_mode=True)
+        plan_text = plan[0]['pred_plan_text']
+        parts = re.split(r'(\d+\.)', plan_text)[1:]
+        steps = [parts[i] + parts[i + 1].rstrip() for i in range(0, len(parts), 2)]
+        steps += ["### PLANNING HAS ENDED, SEE ABOVE FOR DETAILS ###"] * (STEP_COUNTS - len(steps))
+        
+        masks = plan[0]['predict_object_id']
+        mask_paths = [f"assets/mask/scene0050_00/scene0050_00_obj_{mask}.glb" for mask in masks]
+        mask_paths += ["assets/mask/scene0050_00/scene0050_00_obj_empty.glb"] * (STEP_COUNTS - len(masks)) # fill with empty mask
+        
+        output = []
+        for i in range(STEP_COUNTS):
+            output.append(steps[i])
+            output.append(mask_paths[i])
+        return output
+    
+    with gr.Tab("LEO Task-to-Plan"):
+        gr.Interface(
+            fn=leo_task_to_plan,
+            inputs=[gr.Textbox(label="Task Description")],
+            outputs=["text"],
+            examples=[
+                ["Freshen up in the bathroom."]
+            ],
+            title="LEO Task-to-Plan: Input task, Output plan text"
+        )
+
+    with gr.Tab("LEO Plan-to-Masks"):
+        gr.Interface(
+            fn=leo_plan_to_masks,
+            inputs=[gr.Textbox(label="Task Description")] + [gr.Textbox(label=f"Action Step {i+1}") for i in range(STEP_COUNTS)],
+            outputs=[gr.Model3D(
+                clear_color=[0.0, 0.0, 0.0, 0.0], camera_position=(80, 100, 6), label=f"3D Model for Step {i+1} (if the step exists)") for i in range(STEP_COUNTS)],
+            examples=[
+                ["Retrieve an item from the backpack.", "1. Walk to the ottoman located near the brown leather armchair.", "2. Choose the black backpack resting on this ottoman.", "3. Open the backpack to find the needed item."] + [""] * (STEP_COUNTS - 3)
+            ],
+            title="LEO Plan-to-Masks: Input plan, Output 3D Masks for each step, Red denotes predicted target object"
+        )
+
+    with gr.Tab("LEO Task-to-Plan and Masks"):
+        gr.Interface(
+            fn=leo_task_to_plan_and_masks,
+            inputs=[gr.Textbox(label="Task Description")],
+            outputs=[
+                item for sublist in zip(
+                    [gr.Textbox(label=f"Action Step {i+1}") for i in range(STEP_COUNTS)],
+                    [gr.Model3D(
+                        clear_color=[0.0, 0.0, 0.0, 0.0],
+                        camera_position=(80, 100, 6),
+                        label=f"3D Model for Step {i+1} (if the step exists)"
+                    ) for i in range(STEP_COUNTS)]
+                ) for item in sublist
+            ],
+            examples=[
+                ["Retrieve an item from the backpack."]
+            ],
+            title="LEO Task-to-Plan and Masks: Input task, Output plan text and 3D Masks for each step, Red denotes predicted target object",
+            # js="""
+            # function() {
+            #     const stepCounts = """ + str(STEP_COUNTS) + """;
+            #     const stepElems = document.querySelectorAll('.output_interface .textbox_output');
+            #     const modelElems = document.querySelectorAll('.output_interface .model3d_output');
+            #     for (let i = 0; i < stepCounts; i++) {
+            #         if (stepElems[i].value === '### PLANNING HAS ENDED, SEE ABOVE FOR DETAILS ###' || modelElems[i].src.includes('scene0050_00_obj_empty.glb')) {
+            #             stepElems[i].style.display = 'none';
+            #             modelElems[i].style.display = 'none';
+            #         }
+            #     }
+            # }
+            # """
+        )
+
+demo.queue().launch(share=True, allowed_paths=['assets'])

assets/meta/scannetv2-labels.combined.tsv

@@ -0,0 +1,608 @@
+id	raw_category	category	count	nyu40id	eigen13id	nyuClass	nyu40class	eigen13class	ModelNet40	ModelNet10	ShapeNetCore55	synsetoffset	wnsynsetid	wnsynsetkey	mpcat40	mpcat40index
+1	wall	wall	8277	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+2	chair	chair	4646	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+22	books	book	1678	23	2	book	books	Books					n02870526	book.n.11	objects	39
+3	floor	floor	1553	2	5	floor	floor	Floor					n03365592	floor.n.01	floor	2
+5	door	door	1483	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+1163	object	object	1313	40	7		otherprop	Objects							objects	39
+16	window	window	1209	9	13	window	window	Window					n04587648	window.n.01	window	9
+4	table	table	1170	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+56	trash can	trash can	1090	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+13	pillow	pillow	937	18	7	pillow	pillow	Objects			pillow	3938244	n03938244	pillow.n.01	cushion	8
+15	picture	picture	862	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+41	ceiling	ceiling	806	22	3	ceiling	ceiling	Ceiling					n02990373	ceiling.n.01	ceiling	17
+26	box	box	775	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+161	doorframe	doorframe	768	8	12	door	door	Wall	door					doorframe.n.01	door	4
+19	monitor	monitor	765	40	7	monitor	otherprop	Objects	monitor	monitor	tv or monitor	3211117	n03782190	monitor.n.04	objects	39
+7	cabinet	cabinet	731	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+9	desk	desk	680	14	10	desk	desk	Table	desk	desk	table	4379243	n03179701	desk.n.01	table	5
+8	shelf	shelf	641	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+10	office chair	office chair	595	5	4	chair	chair	Chair	chair	chair	chair	3001627	n04373704	swivel_chair.n.01	chair	3
+31	towel	towel	570	27	7	towel	towel	Objects					n04459362	towel.n.01	towel	20
+6	couch	couch	502	6	9	sofa	sofa	Sofa	sofa	sofa	sofa	4256520	n04256520	sofa.n.01	sofa	10
+14	sink	sink	488	34	7	sink	sink	Objects	sink				n04223580	sink.n.01	sink	15
+48	backpack	backpack	479	40	7	backpack	otherprop	Objects					n02769748	backpack.n.01	objects	39
+28	lamp	lamp	419	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+11	bed	bed	370	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+18	bookshelf	bookshelf	360	10	6	bookshelf	bookshelf	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+71	mirror	mirror	349	19	7	mirror	mirror	Objects					n03773035	mirror.n.01	mirror	21
+21	curtain	curtain	347	16	13	curtain	curtain	Window	curtain				n03151077	curtain.n.01	curtain	12
+40	plant	plant	331	40	7	plant	otherprop	Objects	plant				n00017222	plant.n.02	plant	14
+52	whiteboard	whiteboard	327	30	7	whiteboard	whiteboard	Objects					n03211616	display_panel.n.01	board_panel	35
+96	radiator	radiator	322	39	6	radiator	otherfurniture	Furniture					n04041069	radiator.n.02	misc	40
+22	book	book	318	23	2	book	books	Books					n02870526	book.n.11	objects	39
+29	kitchen cabinet	kitchen cabinet	310	3	6	cabinet	cabinet	Furniture					n02933112	cabinet.n.01	cabinet	7
+49	toilet paper	toilet paper	291	40	7	toilet paper	otherprop	Objects					n15075141	toilet_tissue.n.01	objects	39
+29	kitchen cabinets	kitchen cabinet	289	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+23	armchair	armchair	281	5	4	chair	chair	Chair	chair	chair	chair	3001627	n02738535	armchair.n.01	chair	3
+63	shoes	shoe	272	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+24	coffee table	coffee table	258	7	10	coffee table	table	Table	table	table	table	4379243	n03063968	coffee_table.n.01	table	5
+17	toilet	toilet	256	33	7	toilet	toilet	Objects	toilet	toilet			n04446276	toilet.n.01	toilet	18
+47	bag	bag	252	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+32	clothes	clothes	248	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+46	keyboard	keyboard	246	40	7	keyboard	otherprop	Objects	keyboard		computer keyboard	3085013	n03085013	computer_keyboard.n.01	objects	39
+65	bottle	bottle	226	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+97	recycling bin	recycling bin	225	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+34	nightstand	nightstand	224	32	6	night stand	night stand	Furniture	night_stand	night_stand			n03015254	chest_of_drawers.n.01	chest_of_drawers	13
+38	stool	stool	221	40	7	stool	otherprop	Objects	stool				n04326896	stool.n.01	stool	19
+33	tv	tv	219	25	11	television	television	TV			tv or monitor	3211117	n03211117	display.n.06	tv_monitor	22
+75	file cabinet	file cabinet	217	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+36	dresser	dresser	213	17	6	dresser	dresser	Furniture	dresser	dresser			n03015254	chest_of_drawers.n.01	chest_of_drawers	13
+64	computer tower	computer tower	203	40	7	computer	otherprop	Objects					n03082979	computer.n.01	objects	39
+32	clothing	clothes	165	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+101	telephone	telephone	164	40	7	telephone	otherprop	Objects			telephone	4401088	n04401088	telephone.n.01	objects	39
+130	cup	cup	157	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+27	refrigerator	refrigerator	154	24	6	refridgerator	refridgerator	Furniture					n04070727	refrigerator.n.01	appliances	37
+44	end table	end table	147	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+131	jacket	jacket	146	40	7	jacket	otherprop	Objects					n03589791	jacket.n.01	clothes	38
+55	shower curtain	shower curtain	144	28	7	shower curtain	shower curtain	Objects	curtain				n04209239	shower_curtain.n.01	curtain	12
+42	bathtub	bathtub	144	36	7	bathtub	bathtub	Objects	bathtub	bathtub	tub	2808440	n02808440	bathtub.n.01	bathtub	25
+59	microwave	microwave	141	40	7	microwave	otherprop	Objects			microwave	3761084	n03761084	microwave.n.02	appliances	37
+159	kitchen counter	kitchen counter	140	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+74	sofa chair	sofa chair	129	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+82	paper towel dispenser	paper towel dispenser	129	40	7	paper towel dispenser	otherprop	Objects							objects	39
+1164	bathroom vanity	bathroom vanity	126	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	table	5
+93	suitcase	suitcase	118	40	7	luggage	otherprop	Objects					n02773838	bag.n.06	objects	39
+77	laptop	laptop	111	40	7	laptop	otherprop	Objects	laptop		laptop	3642806	n03642806	laptop.n.01	objects	39
+67	ottoman	ottoman	111	39	6	ottoman	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+128	shower walls	shower wall	109	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+50	printer	printer	106	40	7	printer	otherprop	Objects			printer	4004475	n04004475	printer.n.03	appliances	37
+35	counter	counter	104	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+69	board	board	100	38	7	board	otherstructure	Objects							board_panel	35
+100	soap dispenser	soap dispenser	99	40	7		otherprop	Objects					n04254120	soap_dispenser.n.01	objects	39
+62	stove	stove	95	38	7	stove	otherstructure	Objects			stove	4330267	n04330267	stove.n.02	appliances	37
+105	light	light	93	38	7	light	otherstructure	Objects					n03665366	light.n.02	lighting	28
+1165	closet wall	closet wall	90	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+165	mini fridge	mini fridge	87	24	6	refridgerator	refridgerator	Furniture					n03273913	electric_refrigerator.n.01	appliances	37
+7	cabinets	cabinet	79	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+5	doors	door	76	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+76	fan	fan	75	40	7	fan	otherprop	Objects					n03320046	fan.n.01	misc	40
+230	tissue box	tissue box	73	40	7	tissue box	otherprop	Objects					n02883344	box.n.01	objects	39
+54	blanket	blanket	72	40	7	blanket	otherprop	Objects					n02849154	blanket.n.01	objects	39
+125	bathroom stall	bathroom stall	71	38	7		otherstructure	Objects					n02873839	booth.n.02	misc	40
+72	copier	copier	70	40	7		otherprop	Objects					n03257586	duplicator.n.01	appliances	37
+68	bench	bench	66	39	6	bench	otherfurniture	Furniture	bench		bench	2828884	n02828884	bench.n.01	seating	34
+145	bar	bar	66	38	7	bar	otherstructure	Objects					n02788689	bar.n.03	misc	40
+157	soap dish	soap dish	65	40	7	soap dish	otherprop	Objects					n04254009	soap_dish.n.01	objects	39
+1166	laundry hamper	laundry hamper	65	40	7	laundry basket	otherprop	Objects							objects	39
+132	storage bin	storage bin	63	40	7	storage bin	otherprop	Objects							objects	39
+1167	bathroom stall door	bathroom stall door	62	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+232	light switch	light switch	61	38	7	light switch	otherstructure	Objects					n04372370	switch.n.01	misc	40
+134	coffee maker	coffee maker	61	40	7		otherprop	Objects					n03063338	coffee_maker.n.01	appliances	37
+51	tv stand	tv stand	61	39	6	tv stand	otherfurniture	Furniture	tv_stand				n03290653	entertainment_center.n.01	furniture	36
+250	decoration	decoration	60	40	7		otherprop	Objects					n03169390	decoration.n.01	misc	40
+1168	ceiling light	ceiling light	59	38	7	light	otherstructure	Objects					n03665366	light.n.02	lighting	28
+342	range hood	range hood	59	38	7	range hood	otherstructure	Objects	range_hood				n04053677	range_hood.n.01	misc	40
+89	blackboard	blackboard	58	38	7	blackboard	otherstructure	Objects					n02846511	blackboard.n.01	board_panel	35
+103	clock	clock	58	40	7	clock	otherprop	Objects			clock	3046257	n03046257	clock.n.01	objects	39
+99	wardrobe closet	wardrobe	54	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+95	rail	rail	53	38	7	railing	otherstructure	Objects					n04047401	railing.n.01	railing	30
+154	bulletin board	bulletin board	53	38	7	board	otherstructure	Objects					n03211616	display_panel.n.01	board_panel	35
+140	mat	mat	52	20	5	floor mat	floor mat	Floor					n03727837	mat.n.01	floor	2
+1169	trash bin	trash bin	52	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+193	ledge	ledge	51	38	7		otherstructure	Objects					n09337253	ledge.n.01	misc	40
+116	seat	seat	49	39	6	furniture	otherfurniture	Furniture					n04161981	seat.n.03	furniture	36
+202	mouse	mouse	49	40	7	mouse	otherprop	Objects					n03793489	mouse.n.04	objects	39
+73	basket	basket	48	40	7	basket	otherprop	Objects			basket	2801938	n02801938	basket.n.01	objects	39
+78	shower	shower	48	38	7		otherstructure	Objects					n04208936	shower.n.01	shower	23
+1170	dumbbell	dumbbell	48	40	7		otherprop	Objects					n03255030	dumbbell.n.01	objects	39
+79	paper	paper	46	26	7	paper	paper	Objects					n14974264	paper.n.01	objects	39
+80	person	person	46	31	7	person	person	Objects	person				n05217688	person.n.02	misc	40
+141	windowsill	windowsill	45	38	7		otherstructure	Objects					n04590263	windowsill.n.01	window	9
+57	closet	closet	45	39	6	wardrobe	otherfurniture	Furniture	wardrobe						misc	40
+102	bucket	bucket	45	40	7	bucket	otherprop	Objects					n02909870	bucket.n.01	misc	40
+261	sign	sign	44	40	7	sign	otherprop	Objects					n04217882	signboard.n.01	objects	39
+118	speaker	speaker	43	40	7	speaker	otherprop	Objects			speaker	3691459	n03691459	loudspeaker.n.01	objects	39
+136	dishwasher	dishwasher	43	38	7	dishwasher	otherstructure	Objects			dishwasher	3207941	n03207941	dishwasher.n.01	appliances	37
+98	container	container	43	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1171	stair rail	stair rail	42	38	7	banister	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+170	shower curtain rod	shower curtain rod	42	40	7		otherprop	Objects							curtain	12
+1172	tube	tube	41	40	7		otherprop	Objects							misc	40
+1173	bathroom cabinet	bathroom cabinet	39	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+79	papers	paper	39	26	7	paper	paper	Objects					n14974264	paper.n.01	objects	39
+221	storage container	storage container	39	40	7	container	otherprop	Objects							objects	39
+570	paper bag	paper bag	39	37	7	bag	bag	Objects					n04122825	sack.n.01	objects	39
+138	paper towel roll	paper towel roll	39	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+168	ball	ball	39	40	7	ball	otherprop	Objects							objects	39
+276	closet doors	closet door	38	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+106	laundry basket	laundry basket	37	40	7	laundry basket	otherprop	Objects			basket	2801938	n03050864	clothes_hamper.n.01	objects	39
+214	cart	cart	37	40	7	cart	otherprop	Objects					n03484083	handcart.n.01	shelving	31
+276	closet door	closet door	35	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+323	dish rack	dish rack	35	40	7	dish rack	otherprop	Objects					n03207630	dish_rack.n.01	objects	39
+58	stairs	stairs	35	38	7	stairs	otherstructure	Objects					n04298308	stairway.n.01	stairs	16
+86	blinds	blinds	35	13	13	blinds	blinds	Window					n02851099	blind.n.03	blinds	32
+2	stack of chairs	chair	35	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+399	purse	purse	34	40	7	purse	otherprop	Objects					n02774152	bag.n.04	objects	39
+121	bicycle	bicycle	33	40	7	bicycle	otherprop	Objects			bicycle	2834778	n02834778	bicycle.n.01	objects	39
+185	tray	tray	32	40	7	tray	otherprop	Objects					n04476259	tray.n.01	objects	39
+300	plunger	plunger	30	40	7		otherprop	Objects					n03970156	plunger.n.03	objects	39
+180	paper cutter	paper cutter	30	40	7	paper cutter	otherprop	Objects					n03886940	paper_cutter.n.01	objects	39
+163	toilet paper dispenser	toilet paper dispenser	29	40	7		otherprop	Objects							objects	39
+26	boxes	box	29	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+66	bin	bin	28	40	7	bin	otherprop	Objects					n02839910	bin.n.01	objects	39
+208	toilet seat cover dispenser	toilet seat cover dispenser	28	40	7		otherprop	Objects							objects	39
+112	guitar	guitar	28	40	7	guitar	otherprop	Objects	guitar		guitar	3467517	n03467517	guitar.n.01	objects	39
+540	mailboxes	mailbox	28	29	7	box	box	Objects			mailbox	3710193	n03710193	mailbox.n.01	misc	40
+395	handicap bar	handicap bar	27	38	7	bar	otherstructure	Objects							misc	40
+166	fire extinguisher	fire extinguisher	27	40	7	fire extinguisher	otherprop	Objects					n03345837	fire_extinguisher.n.01	misc	40
+122	ladder	ladder	27	39	6	ladder	otherfurniture	Furniture	stairs				n03632277	ladder.n.01	stairs	16
+120	column	column	26	38	7	column	otherstructure	Objects					n03074380	column.n.06	column	24
+107	pipe	pipe	25	40	7	pipe	otherprop	Objects					n03944672	pipe.n.02	misc	40
+283	vacuum cleaner	vacuum cleaner	25	40	7		otherprop	Objects					n04517823	vacuum.n.04	objects	39
+88	plate	plate	24	40	7	plate	otherprop	Objects					n03959485	plate.n.04	objects	39
+90	piano	piano	24	39	6	piano	otherfurniture	Furniture	piano		piano	3928116	n03928116	piano.n.01	furniture	36
+177	water cooler	water cooler	24	39	6	water cooler	otherfurniture	Furniture					n04559166	water_cooler.n.01	misc	40
+1174	cd case	cd case	24	40	7		otherprop	Objects							objects	39
+562	bowl	bowl	24	40	7	bowl	otherprop	Objects	bowl		bowl	2880940	n02880940	bowl.n.03	objects	39
+1175	closet rod	closet rod	24	40	7		otherprop	Objects					n04100174	rod.n.01	misc	40
+1156	bathroom counter	bathroom counter	24	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+84	oven	oven	23	38	7	oven	otherstructure	Objects					n03862676	oven.n.01	appliances	37
+104	stand	stand	23	39	6	stand	otherfurniture	Furniture	table	table	table	4379243	n04301000	stand.n.04	table	5
+229	scale	scale	23	40	7	scale	otherprop	Objects					n04141975	scale.n.07	objects	39
+70	washing machine	washing machine	23	39	6	washing machine	otherfurniture	Furniture			washing_machine	4554684	n04554684	washer.n.03	appliances	37
+325	broom	broom	22	40	7	broom	otherprop	Objects					n02906734	broom.n.01	objects	39
+169	hat	hat	22	40	7	hat	otherprop	Objects					n03497657	hat.n.01	clothes	38
+128	shower wall	shower wall	22	1	12	wall	wall	Wall					n04208936	shower.n.01	wall	1
+331	guitar case	guitar case	21	40	7	guitar case	otherprop	Objects							objects	39
+87	rack	rack	21	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+488	water pitcher	water pitcher	21	40	7	pitcher	otherprop	Objects					n03950228	pitcher.n.02	objects	39
+776	laundry detergent	laundry detergent	21	40	7		otherprop	Objects							objects	39
+370	hair dryer	hair dryer	21	40	7	hair dryer	otherprop	Objects					n03483316	hand_blower.n.01	objects	39
+191	pillar	pillar	21	38	7	column	otherstructure	Objects					n03073977	column.n.07	column	24
+748	divider	divider	20	40	7		otherprop	Objects							wall	1
+242	power outlet	power outlet	19	40	7		otherprop	Objects							misc	40
+45	dining table	dining table	19	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+417	shower floor	shower floor	19	2	5	floor	floor	Floor					n04208936	shower.n.01	floor	2
+70	washing machines	washing machine	19	39	6	washing machine	otherfurniture	Furniture			washing_machine	4554684	n04554684	washer.n.03	appliances	37
+188	shower door	shower door	19	8	12	door	door	Wall	door				n04208936	shower.n.01	door	4
+1176	coffee kettle	coffee kettle	18	40	7	pot	otherprop	Objects					n03612814	kettle.n.01	objects	39
+1177	wardrobe cabinet	wardrobe 	18	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+1178	structure	structure	18	38	7		otherstructure	Objects							misc	40
+18	bookshelves	bookshelf	17	10	6	bookshelf	bookshelf	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+110	clothes dryer	clothes dryer	17	39	6		otherfurniture	Furniture					n03251766	dryer.n.01	appliances	37
+148	toaster	toaster	17	40	7	toaster	otherprop	Objects					n04442312	toaster.n.02	appliances	37
+63	shoe	shoe	17	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+155	ironing board	ironing board	16	39	6	ironing board	otherfurniture	Furniture					n03586090	ironing_board.n.01	objects	39
+572	alarm clock	alarm clock	16	40	7	alarm clock	otherprop	Objects			clock	3046257	n02694662	alarm_clock.n.01	objects	39
+1179	shower head	shower head	15	38	7		otherstructure	Objects							shower	23
+28	lamp base	lamp	15	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+392	water bottle	water bottle	15	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n04557648	water_bottle.n.01	objects	39
+1180	keyboard piano	keyboard piano	15	39	6	piano	otherfurniture	Furniture	piano		piano	3928116	n03928116	piano.n.01	furniture	36
+609	projector screen	projector screen	15	38	7	projector screen	otherstructure	Objects							misc	40
+1181	case of water bottles	case of water bottles	15	40	7		otherprop	Objects							objects	39
+195	toaster oven	toaster oven	14	40	7	toaster oven	otherprop	Objects					n04442441	toaster_oven.n.01	appliances	37
+581	music stand	music stand	14	39	6	music stand	otherfurniture	Furniture					n03801760	music_stand.n.01	furniture	36
+58	staircase	stairs	14	38	7	stairs	otherstructure	Objects					n04298308	stairway.n.01	stairs	16
+1182	coat rack	coat rack	14	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	3
+1183	storage organizer	storage organizer	14	40	7		otherprop	Objects							shelving	3
+139	machine	machine	14	40	7	machine	otherprop	Objects					n03699975	machine.n.01	appliances	37
+1184	folded chair	folded chair	14	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1185	fire alarm	fire alarm	14	40	7		otherprop	Objects					n03343737	fire_alarm.n.02	misc	40
+156	fireplace	fireplace	13	38	7	fireplace	otherstructure	Objects					n03346455	fireplace.n.01	fireplace	27
+408	vent	vent	13	40	7		otherprop	Objects					n04526241	vent.n.01	misc	40
+213	furniture	furniture	13	39	6	furniture	otherfurniture	Furniture					n03405725	furniture.n.01	furniture	36
+1186	power strip	power strip	13	40	7		otherprop	Objects							objects	39
+1187	calendar	calendar	13	40	7		otherprop	Objects							objects	39
+1188	poster	poster	13	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+115	toilet paper holder	toilet paper holder	13	40	7	toilet paper holder	otherprop	Objects							objects	39
+1189	potted plant	potted plant	12	40	7	plant	otherprop	Objects	plant				n00017222	plant.n.02	plant	14
+304	stuffed animal	stuffed animal	12	40	7	stuffed animal	otherprop	Objects					n04399382	teddy.n.01	objects	39
+1190	luggage	luggage	12	40	7	luggage	otherprop	Objects					n02774630	baggage.n.01	objects	39
+21	curtains	curtain	12	16	13	curtain	curtain	Window	curtain				n03151077	curtain.n.01	curtain	12
+312	headphones	headphones	12	40	7		otherprop	Objects					n03261776	earphone.n.01	objects	39
+233	crate	crate	12	39	6	crate	otherfurniture	Furniture					n03127925	crate.n.01	objects	39
+286	candle	candle	12	40	7	candle	otherprop	Objects	lamp				n02948072	candle.n.01	objects	39
+264	projector	projector	12	40	7	projector	otherprop	Objects					n04009552	projector.n.02	objects	39
+110	clothes dryers	clothes dryer	12	39	6		otherfurniture	Furniture					n03251766	dryer.n.01	appliances	37
+1191	mattress	mattress	12	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+356	dustpan	dustpan	12	40	7		otherprop	Objects					n03259009	dustpan.n.02	objects	39
+25	drawer	drawer	11	39	6	drawer	otherfurniture	Furniture					n03233905	drawer.n.01	furniture	36
+750	rod	rod	11	40	7		otherprop	Objects			pistol	3948459	n03427202	gat.n.01	misc	40
+269	globe	globe	11	40	7	globe	otherprop	Objects							objects	39
+307	footrest	footrest	11	39	6	foot rest	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+410	piano bench	piano bench	11	39	6	piano bench	otherfurniture	Furniture	bench		bench	2828884	n02828884	bench.n.01	seating	34
+730	breakfast bar	breakfast bar	11	38	7	bar	otherstructure	Objects							counter	26
+216	step stool	step stool	11	40	7	step stool	otherprop	Objects	stool				n04315713	step_stool.n.01	stool	19
+1192	hand rail	hand rail	11	38	7	railing	otherstructure	Objects							railing	30
+119	vending machine	vending machine	11	40	7	machine	otherprop	Objects					n04525305	vending_machine.n.01	appliances	37
+682	ceiling fan	ceiling fan	11	40	7	fan	otherprop	Objects					n03320046	fan.n.01	misc	40
+434	swiffer	swiffer	11	40	7		otherprop	Objects							objects	39
+126	foosball table	foosball table	11	39	6	foosball table	otherfurniture	Furniture	table	table	table	4379243	n04379243	table.n.02	table	5
+919	jar	jar	11	40	7	jar	otherprop	Objects			jar	3593526	n03593526	jar.n.01	objects	39
+85	footstool	footstool	11	39	6	ottoman	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+1193	folded table	folded table	10	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+108	round table	round table	10	7	10	table	table	Table	table	table	table	4379243	n04114554	round_table.n.02	table	5
+135	hamper	hamper	10	40	7	basket	otherprop	Objects			basket	2801938	n03482405	hamper.n.02	objects	39
+1194	poster tube	poster tube	10	40	7		otherprop	Objects							objects	39
+432	case	case	10	40	7	case	otherprop	Objects							objects	39
+53	carpet	carpet	10	40	7	rug	otherprop	Objects					n04118021	rug.n.01	floor	2
+1195	thermostat	thermostat	10	40	7		otherprop	Objects					n04422875	thermostat.n.01	misc	40
+111	coat	coat	10	40	7	jacket	otherprop	Objects					n03057021	coat.n.01	clothes	38
+305	water fountain	water fountain	10	38	7	water fountain	otherstructure	Objects					n03241335	drinking_fountain.n.01	misc	40
+1125	smoke detector	smoke detector	10	40	7		otherprop	Objects							misc	40
+13	pillows	pillow	9	18	7	pillow	pillow	Objects			pillow	3938244	n03938244	pillow.n.01	cushion	8
+1196	flip flops	flip flops	9	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+1197	cloth	cloth	9	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+1198	banner	banner	9	40	7		otherprop	Objects					n02788021	banner.n.01	misc	40
+1199	clothes hanger	clothes hanger	9	40	7		otherprop	Objects					n03057920	coat_hanger.n.01	objects	39
+1200	whiteboard eraser	whiteboard eraser	9	40	7		otherprop	Objects							objects	39
+378	iron	iron	9	40	7		otherprop	Objects					n03584829	iron.n.04	objects	39
+591	instrument case	instrument case	9	40	7	case	otherprop	Objects							objects	39
+49	toilet paper rolls	toilet paper	9	40	7	toilet paper	otherprop	Objects					n15075141	toilet_tissue.n.01	objects	39
+92	soap	soap	9	40	7	soap	otherprop	Objects					n04253437	soap.n.01	objects	39
+1098	block	block	9	40	7		otherprop	Objects							misc	40
+291	wall hanging	wall hanging	8	40	7		otherprop	Objects					n03491178	hanging.n.01	picture	6
+1063	kitchen island	kitchen island	8	38	7	kitchen island	otherstructure	Objects					n03620600	kitchen_island.n.01	counter	26
+107	pipes	pipe	8	38	7		otherstructure	Objects							misc	40
+1135	toothbrush	toothbrush	8	40	7	toothbrush	otherprop	Objects					n04453156	toothbrush.n.01	objects	39
+189	shirt	shirt	8	40	7		otherprop	Objects					n04197391	shirt.n.01	clothes	38
+245	cutting board	cutting board	8	40	7	cutting board	otherprop	Objects					n03025513	chopping_board.n.01	objects	39
+194	vase	vase	8	40	7	vase	otherprop	Objects	vase		jar	3593526	n04522168	vase.n.01	objects	39
+1201	shower control valve	shower control valve	8	38	7		otherstructure	Objects					n04208936	shower.n.01	shower	23
+386	exercise machine	exercise machine	8	40	7	machine	otherprop	Objects							gym_equipment	33
+1202	compost bin	compost bin	8	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+857	shorts	shorts	8	40	7	shorts	otherprop	Objects							clothes	38
+452	tire	tire	8	40	7		otherprop	Objects					n04440749	tire.n.01	objects	39
+1203	teddy bear	teddy bear	7	40	7	stuffed animal	otherprop	Objects					n04399382	teddy.n.01	objects	39
+346	bathrobe	bathrobe	7	40	7		otherprop	Objects					n02807616	bathrobe.n.01	clothes	38
+152	handrail	handrail	7	38	7	railing	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+83	faucet	faucet	7	40	7	faucet	otherprop	Objects			faucet	3325088	n03325088	faucet.n.01	misc	40
+1204	pantry wall	pantry wall	7	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+726	thermos	thermos	7	40	7	flask	otherprop	Objects	bottle		bottle	2876657	n04422727	thermos.n.01	objects	39
+61	rug	rug	7	40	7	rug	otherprop	Objects					n04118021	rug.n.01	floor	2
+39	couch cushions	cushion	7	18	7	pillow	pillow	Objects					n03151500	cushion.n.03	cushion	8
+1117	tripod	tripod	7	39	6	stand	otherfurniture	Furniture					n04485082	tripod.n.01	objects	39
+540	mailbox	mailbox	7	29	7	box	box	Objects			mailbox	3710193	n03710193	mailbox.n.01	misc	40
+1205	tupperware	tupperware	7	40	7		otherprop	Objects							objects	39
+415	shoe rack	shoe rack	7	40	7	shoe rack	otherprop	Objects							shelving	31
+31	towels	towel	6	27	7	towel	towel	Objects					n04459362	towel.n.01	towel	20
+1206	beer bottles	beer bottle	6	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+153	treadmill	treadmill	6	39	6	treadmill	otherfurniture	Furniture					n04477387	treadmill.n.01	gym_equipment	33
+1207	salt	salt	6	40	7		otherprop	Objects							objects	39
+129	chest	chest	6	39	6	chest	otherfurniture	Furniture	dresser	dresser					chest_of_drawers	13
+220	dispenser	dispenser	6	40	7		otherprop	Objects					n03210683	dispenser.n.01	objects	39
+1208	mirror doors	mirror door	6	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+231	remote	remote	6	40	7		otherprop	Objects			remote_control	4074963	n04074963	remote_control.n.01	objects	39
+1209	folded ladder	folded ladder	6	39	6	ladder	otherfurniture	Furniture	stairs				n03632277	ladder.n.01	misc	40
+39	cushion	cushion	6	18	7	pillow	pillow	Objects					n03151500	cushion.n.03	cushion	8
+1210	carton	carton	6	40	7		otherprop	Objects							objects	39
+117	step	step	6	38	7		otherstructure	Objects					n04314914	step.n.04	misc	40
+822	drying rack	drying rack	6	39	6	drying rack	otherfurniture	Furniture							shelving	31
+238	slippers	slipper	6	40	7	shoe	otherprop	Objects					n04241394	slipper.n.01	clothes	38
+143	pool table	pool table	6	39	6	pool table	otherfurniture	Furniture	table	table	table	4379243	n03982430	pool_table.n.01	table	5
+1211	soda stream	soda stream	6	40	7		otherprop	Objects							objects	39
+228	toilet brush	toilet brush	6	40	7	toilet brush	otherprop	Objects							objects	39
+494	loft bed	loft bed	6	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+226	cooking pot	cooking pot	6	40	7	pot	otherprop	Objects							objects	39
+91	heater	heater	6	39	6	heater	otherfurniture	Furniture					n03508101	heater.n.01	misc	40
+1072	messenger bag	messenger bag	6	37	7	bag	bag	Objects							objects	39
+435	stapler	stapler	6	40	7	stapler	otherprop	Objects					n04303497	stapler.n.01	objects	39
+1165	closet walls	closet wall	5	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+345	scanner	scanner	5	40	7		otherprop	Objects							appliances	37
+893	elliptical machine	elliptical machine	5	40	7	machine	otherprop	Objects							gym_equipment	33
+621	kettle	kettle	5	40	7	pot	otherprop	Objects					n03612814	kettle.n.01	objects	39
+1212	metronome	metronome	5	40	7		otherprop	Objects					n03757604	metronome.n.01	objects	39
+297	dumbell	dumbell	5	40	7		otherprop	Objects							objects	39
+1213	music book	music book	5	23	2	book	books	Books					n02870526	book.n.11	objects	39
+1214	rice cooker	rice cooker	5	40	7		otherprop	Objects							objects	39
+1215	dart board	dart board	5	38	7	board	otherstructure	Objects					n03162940	dartboard.n.01	objects	39
+529	sewing machine	sewing machine	5	40	7	sewing machine	otherprop	Objects					n04179913	sewing_machine.n.01	objects	39
+1216	grab bar	grab bar	5	38	7	railing	otherstructure	Objects							railing	30
+1217	flowerpot	flowerpot	5	40	7	vase	otherprop	Objects	vase		jar	3593526	n04522168	vase.n.01	objects	39
+1218	painting	painting	5	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+1219	railing	railing	5	38	7	railing	otherstructure	Objects					n04047401	railing.n.01	railing	30
+1220	stair	stair	5	38	7	stairs	otherstructure	Objects	stairs				n04314914	step.n.04	stairs	16
+525	toolbox	toolbox	5	39	6	chest	otherfurniture	Furniture					n04452615	toolbox.n.01	objects	39
+204	nerf gun	nerf gun	5	40	7		otherprop	Objects							objects	39
+693	binders	binder	5	40	7	binder	otherprop	Objects							objects	39
+179	desk lamp	desk lamp	5	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+1221	quadcopter	quadcopter	5	40	7		otherprop	Objects							objects	39
+1222	pitcher	pitcher	5	40	7	pitcher	otherprop	Objects					n03950228	pitcher.n.02	objects	39
+1223	hanging	hanging	5	40	7		otherprop	Objects							misc	40
+1224	mail	mail	5	40	7		otherprop	Objects							misc	40
+1225	closet ceiling	closet ceiling	5	22	3	ceiling	ceiling	Ceiling					n02990373	ceiling.n.01	ceiling	17
+1226	hoverboard	hoverboard	5	40	7		otherprop	Objects							objects	39
+1227	beanbag chair	beanbag chair	5	39	6	bean bag	otherfurniture	Furniture					n02816656	beanbag.n.01	chair	3
+571	water heater	water heater	5	40	7	water heater	otherprop	Objects					n04560113	water_heater.n.01	misc	40
+1228	spray bottle	spray bottle	5	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+556	rope	rope	5	40	7	rope	otherprop	Objects					n04108268	rope.n.01	objects	39
+280	plastic container	plastic container	5	40	7	container	otherprop	Objects							objects	39
+1229	soap bottle	soap bottle	5	40	7	soap	otherprop	Objects							objects	39
+1230	ikea bag	ikea bag	4	37	7	bag	bag	Objects				2773838	n02773838	bag.n.06	objects	39
+1231	sleeping bag	sleeping bag	4	40	7		otherprop	Objects					n04235860	sleeping_bag.n.01	objects	39
+1232	duffel bag	duffel bag	4	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+746	frying pan	frying pan	4	40	7	frying pan	otherprop	Objects					n03400231	frying_pan.n.01	objects	39
+1233	oven mitt	oven mitt	4	40	7		otherprop	Objects							objects	39
+1234	pot	pot	4	40	7	pot	otherprop	Objects					n04235860	sleeping_bag.n.01	objects	39
+144	hand dryer	hand dryer	4	40	7		otherprop	Objects							objects	39
+282	dollhouse	dollhouse	4	39	6	doll house	otherfurniture	Furniture					n03219483	dollhouse.n.01	objects	39
+167	shampoo bottle	shampoo bottle	4	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1235	hair brush	hair brush	4	40	7		otherprop	Objects					n02908217	brush.n.02	objects	39
+1236	tennis racket	tennis racket	4	40	7		otherprop	Objects					n04409806	tennis_racket.n.01	objects	39
+1237	display case	display case	4	40	7	case	otherprop	Objects							objects	39
+234	ping pong table	ping pong table	4	39	6	ping pong table	otherfurniture	Furniture	table	table	table	4379243	n04379243	table.n.02	table	5
+563	boiler	boiler	4	40	7		otherprop	Objects							misc	40
+1238	bag of coffee beans	bag of coffee beans	4	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+1239	bananas	banana	4	40	7		otherprop	Objects					n00021265	food.n.01	objects	39
+1240	carseat	carseat	4	40	7		otherprop	Objects							misc	40
+366	helmet	helmet	4	40	7		otherprop	Objects			helmet	3513137	n03513137	helmet.n.02	clothes	38
+816	umbrella	umbrella	4	40	7	umbrella	otherprop	Objects					n04507155	umbrella.n.01	objects	39
+1241	coffee box	coffee box	4	40	7		otherprop	Objects							objects	39
+719	envelope	envelope	4	40	7	envelope	otherprop	Objects					n03291819	envelope.n.01	objects	39
+284	wet floor sign	wet floor sign	4	40	7	sign	otherprop	Objects							misc	40
+1242	clothing rack	clothing rack	4	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+247	controller	controller	4	40	7		otherprop	Objects					n03096960	control.n.09	objects	39
+1243	bath walls	bathroom wall	4	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+1244	podium	podium	4	39	6		otherfurniture	Furniture					n03159640	dais.n.01	furniture	36
+1245	storage box	storage box	4	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1246	dolly	dolly	4	40	7		otherprop	Objects							misc	40
+1247	shampoo	shampoo	3	40	7		otherprop	Objects					n04183516	shampoo.n.01	objects	39
+592	paper tray	paper tray	3	40	7	paper tray	otherprop	Objects							objects	39
+385	cabinet door	cabinet door	3	8	12	door	door	Wall	door						door	4
+1248	changing station	changing station	3	40	7		otherprop	Objects							misc	40
+1249	poster printer	poster printer	3	40	7	printer	otherprop	Objects			printer	4004475	n04004475	printer.n.03	appliances	37
+133	screen	screen	3	40	7		otherprop	Objects					n03151077	curtain.n.01	curtain	12
+301	soap bar	soap bar	3	38	7	bar	otherstructure	Objects							objects	39
+1250	crutches	crutches	3	40	7		otherprop	Objects					n03141823	crutch.n.01	objects	39
+379	studio light	studio light	3	38	7	light	otherstructure	Objects							lighting	28
+130	stack of cups	cup	3	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+1251	toilet flush button	toilet flush button	3	40	7		otherprop	Objects							objects	39
+450	trunk	trunk	3	40	7		otherprop	Objects							misc	40
+1252	grocery bag	grocery bag	3	37	7	bag	bag	Objects			suitcase	2773838	n03461288	grocery_bag.n.01	objects	39
+316	plastic bin	plastic bin	3	40	7	bin	otherprop	Objects							objects	39
+1253	pizza box	pizza box	3	29	7	box	box	Objects							objects	39
+385	cabinet doors	cabinet door	3	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	door	4
+1254	legs	legs	3	31	7	person	person	Objects	person				n05217688	person.n.02	misc	40
+461	car	car	3	40	7	car	otherprop	Objects	car		car	2958343	n02958343	car.n.01	misc	40
+1255	shaving cream	shaving cream	3	40	7		otherprop	Objects					n04186051	shaving_cream.n.01	objects	39
+1256	luggage stand	luggage stand	3	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+599	shredder	shredder	3	40	7		otherprop	Objects					n04210120	shredder.n.01	objects	39
+281	statue	statue	3	40	7	sculpture	otherprop	Objects					n04306847	statue.n.01	misc	40
+1257	urinal	urinal	3	33	7	toilet	toilet	Objects	toilet	toilet			n04515991	urinal.n.01	toilet	18
+1258	hose	hose	3	40	7		otherprop	Objects					n03539875	hose.n.03	misc	40
+1259	bike pump	bike pump	3	40	7		otherprop	Objects							objects	39
+319	coatrack	coatrack	3	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+1260	bear	bear	3	40	7		otherprop	Objects							objects	39
+28	wall lamp	lamp	3	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+1261	humidifier	humidifier	3	40	7		otherprop	Objects							objects	39
+546	toothpaste	toothpaste	3	40	7	toothpaste	otherprop	Objects							objects	39
+1262	mouthwash bottle	mouthwash bottle	3	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1263	poster cutter	poster cutter	3	40	7		otherprop	Objects							objects	39
+1264	golf bag	golf bag	3	37	7	bag	bag	Objects			suitcase	2773838	n03445617	golf_bag.n.01	objects	39
+1265	food container	food container	3	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1266	camera	camera	3	40	7		otherprop	Objects							objects	39
+28	table lamp	lamp	3	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n04380533	table_lamp.n.01	lighting	28
+1267	yoga mat	yoga mat	3	20	5	floor mat	floor mat	Floor					n03727837	mat.n.01	floor	2
+1268	card	card	3	40	7		otherprop	Objects							objects	39
+1269	mug	mug	3	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+188	shower doors	shower door	3	38	7		otherstructure	Objects					n04208936	shower.n.01	door	4
+689	cardboard	cardboard	3	40	7		otherprop	Objects							objects	39
+1270	rack stand	rack stand	3	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+1271	boxes of paper	boxes of paper	3	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1272	flag	flag	3	40	7		otherprop	Objects							misc	40
+354	futon	futon	3	39	6	mattress	otherfurniture	Furniture					n03408444	futon.n.01	sofa	10
+339	magazine	magazine	3	40	7	magazine	otherprop	Objects					n06595351	magazine.n.01	objects	39
+1009	exit sign	exit sign	3	40	7	exit sign	otherprop	Objects							misc	40
+1273	rolled poster	rolled poster	3	40	7		otherprop	Objects							objects	39
+1274	wheel	wheel	3	40	7		otherprop	Objects							objects	39
+15	pictures	picture	3	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+1275	blackboard eraser	blackboard eraser	3	40	7	eraser	otherprop	Objects					n03294833	eraser.n.01	objects	39
+361	organizer	organizer	3	40	7		otherprop	Objects					n03918737	personal_digital_assistant.n.01	objects	39
+1276	doll	doll	3	40	7	toy	otherprop	Objects					n03219135	doll.n.01	objects	39
+326	book rack	book rack	3	39	6	bookrack	otherfurniture	Furniture							objects	39
+1277	laundry bag	laundry bag	3	40	7	laundry basket	otherprop	Objects			basket	2801938	n03050864	clothes_hamper.n.01	objects	39
+1278	sponge	sponge	3	40	7		otherprop	Objects					n01906749	sponge.n.04	objects	39
+116	seating	seat	3	39	6	furniture	otherfurniture	Furniture					n04161981	seat.n.03	furniture	36
+1184	folded chairs	folded chair	2	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1279	lotion bottle	lotion bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+212	can	can	2	40	7	can	otherprop	Objects			can	2946921	n02946921	can.n.01	objects	39
+1280	lunch box	lunch box	2	40	7		otherprop	Objects							objects	39
+1281	food display	food display	2	40	7		otherprop	Objects							misc	40
+794	storage shelf	storage shelf	2	40	7		otherprop	Objects							shelving	31
+1282	sliding wood door	sliding wood door	2	40	7		otherprop	Objects							door	4
+955	pants	pants	2	40	7		otherprop	Objects					n04489008	trouser.n.01	clothes	38
+387	wood	wood	2	40	7		otherprop	Objects							misc	40
+69	boards	board	2	38	7	board	otherstructure	Objects							board_panel	35
+65	bottles	bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+523	washcloth	washcloth	2	40	7		otherprop	Objects					n04554523	washcloth.n.01	towel	20
+389	workbench	workbench	2	39	6	bench	otherfurniture	Furniture	bench		table	4379243	n04600486	workbench.n.01	table	5
+29	open kitchen cabinet	kitchen cabinet	2	3	6	cabinet	cabinet	Furniture					n02933112	cabinet.n.01	cabinet	7
+1283	organizer shelf	organizer shelf	2	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+146	frame	frame	2	38	7		otherstructure	Objects							misc	40
+130	cups	cup	2	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+372	exercise ball	exercise ball	2	40	7	ball	otherprop	Objects					n04285146	sports_equipment.n.01	gym_equipment	33
+289	easel	easel	2	39	6	stand	otherfurniture	Furniture					n03262809	easel.n.01	furniture	36
+440	garbage bag	garbage bag	2	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+321	roomba	roomba	2	40	7		otherprop	Objects							objects	39
+976	garage door	garage door	2	38	7	garage door	otherstructure	Objects	door						door	4
+1256	luggage rack	luggage stand	2	39	6	stand	otherfurniture	Furniture					n04038440		shelving	31
+1284	bike lock	bike lock	2	40	7		otherprop	Objects							objects	39
+1285	briefcase	briefcase	2	40	7		otherprop	Objects					n02900705	briefcase.n.01	objects	39
+357	hand towel	hand towel	2	27	7	towel	towel	Objects					n03490006	hand_towel.n.01	towel	20
+1286	bath products	bath product	2	40	7		otherprop	Objects							objects	39
+1287	star	star	2	40	7		otherprop	Objects					n09444783	star.n.03	misc	40
+365	map	map	2	40	7	map	otherprop	Objects					n03720163	map.n.01	misc	40
+1288	coffee bean bag	coffee bean bag	2	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+81	headboard	headboard	2	39	6	headboard	otherfurniture	Furniture					n03502200	headboard.n.01	bed	11
+1289	ipad	ipad	2	40	7		otherprop	Objects							objects	39
+1290	display rack	display rack	2	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+948	traffic cone	traffic cone	2	40	7	cone	otherprop	Objects	cone						objects	39
+174	toiletry	toiletry	2	40	7		otherprop	Objects					n04447443	toiletry.n.01	objects	39
+1028	canopy	canopy	2	40	7		otherprop	Objects							misc	40
+1291	massage chair	massage chair	2	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1292	paper organizer	paper organizer	2	40	7		otherprop	Objects							objects	39
+1005	barricade	barricade	2	40	7		otherprop	Objects							misc	40
+235	platform	platform	2	38	7		otherstructure	Objects							misc	40
+1293	cap	cap	2	40	7	hat	otherprop	Objects					n03497657	hat.n.01	clothes	38
+1294	dumbbell plates	dumbbell plates	2	40	7		otherprop	Objects							objects	39
+1295	elevator	elevator	2	38	7		otherstructure	Objects							misc	40
+1296	cooking pan	cooking pan	2	40	7	pan	otherprop	Objects					n03880531	pan.n.01	objects	39
+1297	trash bag	trash bag	2	37	7	bag	bag	Objects							objects	39
+1298	santa	santa	2	40	7		otherprop	Objects							misc	40
+1299	jewelry box	jewelry box	2	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1300	boat	boat	2	40	7		otherprop	Objects							misc	40
+1301	sock	sock	2	21	7	clothes	clothes	Objects					n04254777	sock.n.01	clothes	38
+1051	kinect	kinect	2	40	7	kinect	otherprop	Objects							objects	39
+566	crib	crib	2	39	6	crib	otherfurniture	Furniture							furniture	36
+1302	plastic storage bin	plastic storage bin	2	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1062	cooler	cooler	2	24	6	refridgerator	refridgerator	Furniture					n03102654	cooler.n.01	appliances	37
+1303	kitchen apron	kitchen apron	2	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+1304	dishwashing soap bottle	dishwashing soap bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1305	xbox controller	xbox controller	2	40	7		otherprop	Objects							objects	39
+1306	banana holder	banana holder	2	40	7		otherprop	Objects							objects	39
+298	ping pong paddle	ping pong paddle	2	40	7		otherprop	Objects							table	5
+1307	airplane	airplane	2	40	7		otherprop	Objects							misc	40
+1308	conditioner bottle	conditioner bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1309	tea kettle	tea kettle	2	40	7	tea kettle	otherprop	Objects					n04397768	teakettle.n.01	objects	39
+43	bedframe	bedframe	2	39	6		otherfurniture	Furniture					n02822579	bedstead.n.01	bed	11
+1310	wood beam	wood beam	2	38	7		otherstructure	Objects							beam	29
+593	toilet paper package	toilet paper package	2	40	7		otherprop	Objects							objects	39
+1311	wall mounted coat rack	wall mounted coat rack	2	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+1312	film light	film light	2	40	7		otherprop	Objects							lighting	28
+749	ceiling lamp	ceiling lamp	1	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+623	chain	chain	1	40	7		otherprop	Objects							chair	3
+1313	sofa	sofa	1	6	9	sofa	sofa	Sofa	sofa	sofa	sofa	4256520	n04256520	sofa.n.01	sofa	10
+99	closet wardrobe	wardrobe	1	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+265	sweater	sweater	1	40	7		otherprop	Objects					n04370048	sweater.n.01	clothes	38
+1314	kitchen mixer	kitchen mixer	1	40	7		otherprop	Objects							appliances	37
+99	wardrobe	wardrobe	1	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+1315	water softener	water softener	1	40	7		otherprop	Objects							misc	40
+448	banister	banister	1	38	7	banister	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+257	trolley	trolley	1	40	7	trolley	otherprop	Objects					n04335435	streetcar.n.01	misc	40
+1316	pantry shelf	pantry shelf	1	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+786	sofa bed	sofa bed	1	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+801	loofa	loofa	1	40	7		otherprop	Objects							objects	39
+972	shower faucet handle	shower faucet handle	1	40	7	handle	otherprop	Objects							shower	23
+1317	toy piano	toy piano	1	40	7	toy	otherprop	Objects					n03964744	plaything.n.01	objects	39
+1318	fish	fish	1	40	7		otherprop	Objects					n02512053	fish.n.01	objects	39
+75	file cabinets	file cabinet	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n03337140	file.n.03	cabinet	7
+657	cat litter box	cat litter box	1	29	7	box	box	Objects							objects	39
+561	electric panel	electric panel	1	40	7		otherprop	Objects							misc	40
+93	suitcases	suitcase	1	40	7	luggage	otherprop	Objects					n02774630	baggage.n.01	objects	39
+513	curtain rod	curtain rod	1	38	7	curtain rod	otherstructure	Objects							curtain	12
+411	bunk bed	bunk bed	1	39	6	bunk bed	otherfurniture	Furniture	bed	bed	bed	2818832	n02920259	bunk_bed.n.01	bed	11
+1122	chandelier	chandelier	1	38	7	chandelier	otherstructure	Objects					n03005285	chandelier.n.01	lighting	28
+922	tape	tape	1	40	7	tape	otherprop	Objects							objects	39
+88	plates	plate	1	40	7		otherprop	Objects					n03959485	plate.n.04	objects	39
+518	alarm	alarm	1	40	7	alarm	otherprop	Objects			clock	3046257	n02694662	alarm_clock.n.01	objects	39
+814	fire hose	fire hose	1	40	7		otherprop	Objects					n03346004	fire_hose.n.01	misc	40
+1319	toy dinosaur	toy dinosaur	1	40	7	toy	otherprop	Objects					n03964744	plaything.n.01	objects	39
+1320	cone	cone	1	40	7		otherprop	Objects							objects	39
+649	glass doors	glass door	1	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+607	hatrack	hatrack	1	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+819	subwoofer	subwoofer	1	40	7	speaker	otherprop	Objects			speaker	3691459	n04349401	subwoofer.n.01	objects	39
+1321	fire sprinkler	fire sprinkler	1	40	7		otherprop	Objects							misc	40
+1322	trash cabinet	trash cabinet	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+1204	pantry walls	pantry wall	1	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+227	photo	photo	1	40	7	photo	otherprop	Objects					n03925226	photograph.n.01	picture	6
+817	barrier	barrier	1	40	7		otherprop	Objects					n02796623	barrier.n.01	misc	40
+130	stacks of cups	cup	1	40	7		otherprop	Objects					n03147509	cup.n.01	objects	39
+712	beachball	beachball	1	40	7	ball	otherprop	Objects					n02814224	beach_ball.n.01	objects	39
+1323	folded boxes	folded boxes	1	40	7		otherprop	Objects							objects	39
+1324	contact lens solution bottle	contact lens solution bottle	1	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+673	covered box	covered box	1	29	7	box	box	Objects							objects	39
+459	folder	folder	1	40	7	folder	otherprop	Objects					n03376279	folder.n.02	objects	39
+643	mail trays	mail tray	1	40	7	mail tray	otherprop	Objects							objects	39
+238	slipper	slipper	1	40	7		otherprop	Objects					n04241394	slipper.n.01	clothes	38
+765	magazine rack	magazine rack	1	39	6	stand	otherfurniture	Furniture					n03704549	magazine_rack.n.01	shelving	31
+1008	sticker	sticker	1	40	7	sticker	otherprop	Objects					n07272545	gummed_label.n.01	objects	39
+225	lotion	lotion	1	40	7		otherprop	Objects					n03690938	lotion.n.01	objects	39
+1083	buddha	buddha	1	40	7		otherprop	Objects							objects	39
+813	file organizer	file organizer	1	40	7		otherprop	Objects							objects	39
+138	paper towel rolls	paper towel roll	1	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+1145	night lamp	night lamp	1	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+796	fuse box	fuse box	1	40	7		otherprop	Objects							misc	40
+1325	knife block	knife block	1	40	7		otherprop	Objects							objects	39
+363	furnace	furnace	1	39	6	furnace	otherfurniture	Furniture					n03404449	furnace.n.01		
+1174	cd cases	cd case	1	40	7		otherprop	Objects							objects	39
+38	stools	stool	1	40	7	stool	otherprop	Objects	stool				n04326896	stool.n.01	stool	19
+1326	hand sanitzer dispenser	hand sanitzer dispenser	1	40	7		otherprop	Objects					n04254120	soap_dispenser.n.01	objects	39
+997	teapot	teapot	1	40	7	tea pot	otherprop	Objects					n04398044	teapot.n.01	objects	39
+1327	pen holder	pen holder	1	40	7		otherprop	Objects							objects	39
+1328	tray rack	tray rack	1	40	7		otherprop	Objects							objects	39
+1329	wig	wig	1	40	7		otherprop	Objects					n04584207	wig.n.01	objects	39
+182	switch	switch	1	40	7		otherprop	Objects					n04372370	switch.n.01	misc	40
+280	plastic containers	plastic container	1	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1330	night light	night light	1	40	7		otherprop	Objects							lighting	28
+1331	notepad	notepad	1	40	7		otherprop	Objects							objects	39
+1332	mail bin	mail bin	1	40	7		otherprop	Objects							misc	40
+1333	elevator button	elevator button	1	40	7		otherprop	Objects							misc	40
+939	gaming wheel	gaming wheel	1	40	7		otherprop	Objects							objects	39
+1334	drum set	drum set	1	40	7		otherprop	Objects							objects	39
+480	cosmetic bag	cosmetic bag	1	37	7	bag	bag	Objects							objects	39
+907	coffee mug	coffee mug	1	40	7	vessel	otherprop	Objects			cup or mug	3797390	n03063599	coffee_mug.n.01	objects	39
+1335	closet shelf	closet shelf	1	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+1336	baby mobile	baby mobile	1	40	7		otherprop	Objects							objects	39
+829	diaper bin	diaper bin	1	40	7	bin	otherprop	Objects							objects	39
+947	door wall	door wall	1	1	12	wall	wall	Wall							wall	1
+1116	stepstool	stepstool	1	40	7	step stool	otherprop	Objects							objects	39
+599	paper shredder	shredder	1	40	7		otherprop	Objects					n04210120	shredder.n.01	objects	39
+733	dress rack	dress rack	1	40	7		otherprop	Objects					n03238762	dress_rack.n.01	misc	40
+123	cover	cover	1	40	7	blanket	otherprop	Objects							objects	39
+506	shopping bag	shopping bag	1	37	7	bag	bag	Objects					n04204081	shopping_bag.n.01	objects	39
+569	sliding door	sliding door	1	8	12	door	door	Wall	door				n04239074	sliding_door.n.01	door	4
+1337	exercise bike	exercise bike	1	40	7	machine	otherprop	Objects					n04210120	shredder.n.01	gym_equipment	33
+1338	recliner chair	recliner chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03238762	dress_rack.n.01	chair	3
+1314	kitchenaid mixer	kitchen mixer	1	40	7		otherprop	Objects							appliances	37
+1339	soda can	soda can	1	40	7	can	otherprop	Objects			can	2946921	n02946921	can.n.01	objects	39
+1340	stovetop	stovetop	1	38	7	stove	otherstructure	Objects			stove	4330267	n04330267	stove.n.02	appliances	37
+851	stepladder	stepladder	1	39	6	ladder	otherfurniture	Furniture	stairs				n04315599	step_ladder.n.01	stairs	16
+142	tap	tap	1	40	7	faucet	otherprop	Objects			faucet	3325088	n04559451	water_faucet.n.01	objects	39
+436	cable	cable	1	40	7	cables	otherprop	Objects							objects	39
+1341	baby changing station	baby changing station	1	39	6		otherfurniture	Furniture							furniture	36
+1342	costume	costume	1	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+885	rocking chair	rocking chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n04099969	rocking_chair.n.01	chair	3
+693	binder	binder	1	40	7	binder	otherprop	Objects							objects	39
+815	media center	media center	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+401	towel rack	towel rack	1	40	7		otherprop	Objects					n04459773	towel_rack.n.01	misc	40
+1343	medal	medal	1	40	7		otherprop	Objects							objects	39
+1184	stack of folded chairs	folded chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1344	telescope	telescope	1	40	7		otherprop	Objects					n04403638	telescope.n.01	objects	39
+1345	closet doorframe	closet doorframe	1	8	12	door	door	Wall	door						door	4
+160	glass	glass	1	38	7	glass	otherstructure	Objects					n03438257	glass.n.02	misc	40
+1126	baseball cap	baseball cap	1	40	7		otherprop	Objects			cap	2954340	n02799323	baseball_cap.n.01	clothes	38
+1346	battery disposal jar	battery disposal jar	1	40	7	jar	otherprop	Objects			jar	3593526	n03593526	jar.n.01	objects	39
+332	mop	mop	1	40	7		otherprop	Objects					n04367480	swab.n.02	objects	39
+397	tank	tank	1	40	7		otherprop	Objects							objects	39
+643	mail tray	mail tray	1	40	7	mail tray	otherprop	Objects							objects	39
+551	centerpiece	centerpiece	1	40	7	centerpiece	otherprop	Objects					n02994419	centerpiece.n.02	objects	39
+1163	stick	stick	1	40	7	stick	otherprop	Objects							objects	39
+1347	closet floor	closet floor	1	2	5	floor	floor	Floor					n03365592	floor.n.01	floor	2
+1348	dryer sheets	dryer sheets	1	40	7		otherprop	Objects							objects	39
+803	bycicle	bycicle	1	40	7		otherprop	Objects							misc	40
+484	flower stand	flower stand	1	39	6	stand	otherfurniture	Furniture							furniture	36
+1349	air mattress	air mattress	1	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02690809	air_mattress.n.01	bed	11
+1350	clip	clip	1	40	7		otherprop	Objects							objects	39
+222	side table	side table	1	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+1253	pizza boxes	pizza box	1	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1351	display	display	1	39	7		otherfurniture	Furniture					n03211117	display.n.06	misc	40
+1352	postcard	postcard	1	40	7		otherprop	Objects							objects	39
+828	display sign	display sign	1	40	7	sign	otherprop	Objects							misc	40
+1353	paper towel	paper towel	1	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+612	boots	boot	1	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+1354	tennis racket bag	tennis racket bag	1	40	7		otherprop	Objects							objects	39
+1355	air hockey table	air hockey table	1	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+1301	socks	sock	1	21	7	clothes	clothes	Objects					n04254777	sock.n.01	clothes	38
+1356	food bag	food bag	1	37	7	bag	bag	Objects							objects	39
+1199	clothes hangers	clothes hanger	1	40	7		otherprop	Objects					n03057920	coat_hanger.n.01	misc	40
+1357	starbucks cup	starbucks cup	1	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39

assets/meta/scannetv2_raw_categories.json

@@ -0,0 +1,609 @@
+[
+  "wall",
+  "chair",
+  "books",
+  "floor",
+  "door",
+  "object",
+  "window",
+  "table",
+  "trash can",
+  "pillow",
+  "picture",
+  "ceiling",
+  "box",
+  "doorframe",
+  "monitor",
+  "cabinet",
+  "desk",
+  "shelf",
+  "office chair",
+  "towel",
+  "couch",
+  "sink",
+  "backpack",
+  "lamp",
+  "bed",
+  "bookshelf",
+  "mirror",
+  "curtain",
+  "plant",
+  "whiteboard",
+  "radiator",
+  "book",
+  "kitchen cabinet",
+  "toilet paper",
+  "kitchen cabinets",
+  "armchair",
+  "shoes",
+  "coffee table",
+  "toilet",
+  "bag",
+  "clothes",
+  "keyboard",
+  "bottle",
+  "recycling bin",
+  "nightstand",
+  "stool",
+  "tv",
+  "file cabinet",
+  "dresser",
+  "computer tower",
+  "clothing",
+  "telephone",
+  "cup",
+  "refrigerator",
+  "end table",
+  "jacket",
+  "shower curtain",
+  "bathtub",
+  "microwave",
+  "kitchen counter",
+  "sofa chair",
+  "paper towel dispenser",
+  "bathroom vanity",
+  "suitcase",
+  "laptop",
+  "ottoman",
+  "shower walls",
+  "printer",
+  "counter",
+  "board",
+  "soap dispenser",
+  "stove",
+  "light",
+  "closet wall",
+  "mini fridge",
+  "cabinets",
+  "doors",
+  "fan",
+  "tissue box",
+  "blanket",
+  "bathroom stall",
+  "copier",
+  "bench",
+  "bar",
+  "soap dish",
+  "laundry hamper",
+  "storage bin",
+  "bathroom stall door",
+  "light switch",
+  "coffee maker",
+  "tv stand",
+  "decoration",
+  "ceiling light",
+  "range hood",
+  "blackboard",
+  "clock",
+  "wardrobe closet",
+  "rail",
+  "bulletin board",
+  "mat",
+  "trash bin",
+  "ledge",
+  "seat",
+  "mouse",
+  "basket",
+  "shower",
+  "dumbbell",
+  "paper",
+  "person",
+  "windowsill",
+  "closet",
+  "bucket",
+  "sign",
+  "speaker",
+  "dishwasher",
+  "container",
+  "stair rail",
+  "shower curtain rod",
+  "tube",
+  "bathroom cabinet",
+  "papers",
+  "storage container",
+  "paper bag",
+  "paper towel roll",
+  "ball",
+  "closet doors",
+  "laundry basket",
+  "cart",
+  "closet door",
+  "dish rack",
+  "stairs",
+  "blinds",
+  "stack of chairs",
+  "purse",
+  "bicycle",
+  "tray",
+  "plunger",
+  "paper cutter",
+  "toilet paper dispenser",
+  "boxes",
+  "bin",
+  "toilet seat cover dispenser",
+  "guitar",
+  "mailboxes",
+  "handicap bar",
+  "fire extinguisher",
+  "ladder",
+  "column",
+  "pipe",
+  "vacuum cleaner",
+  "plate",
+  "piano",
+  "water cooler",
+  "cd case",
+  "bowl",
+  "closet rod",
+  "bathroom counter",
+  "oven",
+  "stand",
+  "scale",
+  "washing machine",
+  "broom",
+  "hat",
+  "shower wall",
+  "guitar case",
+  "rack",
+  "water pitcher",
+  "laundry detergent",
+  "hair dryer",
+  "pillar",
+  "divider",
+  "power outlet",
+  "dining table",
+  "shower floor",
+  "washing machines",
+  "shower door",
+  "coffee kettle",
+  "wardrobe cabinet",
+  "structure",
+  "bookshelves",
+  "clothes dryer",
+  "toaster",
+  "shoe",
+  "ironing board",
+  "alarm clock",
+  "shower head",
+  "lamp base",
+  "water bottle",
+  "keyboard piano",
+  "projector screen",
+  "case of water bottles",
+  "toaster oven",
+  "music stand",
+  "staircase",
+  "coat rack",
+  "storage organizer",
+  "machine",
+  "folded chair",
+  "fire alarm",
+  "fireplace",
+  "vent",
+  "furniture",
+  "power strip",
+  "calendar",
+  "poster",
+  "toilet paper holder",
+  "potted plant",
+  "stuffed animal",
+  "luggage",
+  "curtains",
+  "headphones",
+  "crate",
+  "candle",
+  "projector",
+  "clothes dryers",
+  "mattress",
+  "dustpan",
+  "drawer",
+  "rod",
+  "globe",
+  "footrest",
+  "piano bench",
+  "breakfast bar",
+  "step stool",
+  "hand rail",
+  "vending machine",
+  "ceiling fan",
+  "swiffer",
+  "foosball table",
+  "jar",
+  "footstool",
+  "folded table",
+  "round table",
+  "hamper",
+  "poster tube",
+  "case",
+  "carpet",
+  "thermostat",
+  "coat",
+  "water fountain",
+  "smoke detector",
+  "pillows",
+  "flip flops",
+  "cloth",
+  "banner",
+  "clothes hanger",
+  "whiteboard eraser",
+  "iron",
+  "instrument case",
+  "toilet paper rolls",
+  "soap",
+  "block",
+  "wall hanging",
+  "kitchen island",
+  "pipes",
+  "toothbrush",
+  "shirt",
+  "cutting board",
+  "vase",
+  "shower control valve",
+  "exercise machine",
+  "compost bin",
+  "shorts",
+  "tire",
+  "teddy bear",
+  "bathrobe",
+  "handrail",
+  "faucet",
+  "pantry wall",
+  "thermos",
+  "rug",
+  "couch cushions",
+  "tripod",
+  "mailbox",
+  "tupperware",
+  "shoe rack",
+  "towels",
+  "beer bottles",
+  "treadmill",
+  "salt",
+  "chest",
+  "dispenser",
+  "mirror doors",
+  "remote",
+  "folded ladder",
+  "cushion",
+  "carton",
+  "step",
+  "drying rack",
+  "slippers",
+  "pool table",
+  "soda stream",
+  "toilet brush",
+  "loft bed",
+  "cooking pot",
+  "heater",
+  "messenger bag",
+  "stapler",
+  "closet walls",
+  "scanner",
+  "elliptical machine",
+  "kettle",
+  "metronome",
+  "dumbell",
+  "music book",
+  "rice cooker",
+  "dart board",
+  "sewing machine",
+  "grab bar",
+  "flowerpot",
+  "painting",
+  "railing",
+  "stair",
+  "toolbox",
+  "nerf gun",
+  "binders",
+  "desk lamp",
+  "quadcopter",
+  "pitcher",
+  "hanging",
+  "mail",
+  "closet ceiling",
+  "hoverboard",
+  "beanbag chair",
+  "water heater",
+  "spray bottle",
+  "rope",
+  "plastic container",
+  "soap bottle",
+  "ikea bag",
+  "sleeping bag",
+  "duffel bag",
+  "frying pan",
+  "oven mitt",
+  "pot",
+  "hand dryer",
+  "dollhouse",
+  "shampoo bottle",
+  "hair brush",
+  "tennis racket",
+  "display case",
+  "ping pong table",
+  "boiler",
+  "bag of coffee beans",
+  "bananas",
+  "carseat",
+  "helmet",
+  "umbrella",
+  "coffee box",
+  "envelope",
+  "wet floor sign",
+  "clothing rack",
+  "controller",
+  "bath walls",
+  "podium",
+  "storage box",
+  "dolly",
+  "shampoo",
+  "paper tray",
+  "cabinet door",
+  "changing station",
+  "poster printer",
+  "screen",
+  "soap bar",
+  "crutches",
+  "studio light",
+  "stack of cups",
+  "toilet flush button",
+  "trunk",
+  "grocery bag",
+  "plastic bin",
+  "pizza box",
+  "cabinet doors",
+  "legs",
+  "car",
+  "shaving cream",
+  "luggage stand",
+  "shredder",
+  "statue",
+  "urinal",
+  "hose",
+  "bike pump",
+  "coatrack",
+  "bear",
+  "wall lamp",
+  "humidifier",
+  "toothpaste",
+  "mouthwash bottle",
+  "poster cutter",
+  "golf bag",
+  "food container",
+  "camera",
+  "table lamp",
+  "yoga mat",
+  "card",
+  "mug",
+  "shower doors",
+  "cardboard",
+  "rack stand",
+  "boxes of paper",
+  "flag",
+  "futon",
+  "magazine",
+  "exit sign",
+  "rolled poster",
+  "wheel",
+  "pictures",
+  "blackboard eraser",
+  "organizer",
+  "doll",
+  "book rack",
+  "laundry bag",
+  "sponge",
+  "seating",
+  "folded chairs",
+  "lotion bottle",
+  "can",
+  "lunch box",
+  "food display",
+  "storage shelf",
+  "sliding wood door",
+  "pants",
+  "wood",
+  "boards",
+  "bottles",
+  "washcloth",
+  "workbench",
+  "open kitchen cabinet",
+  "organizer shelf",
+  "frame",
+  "cups",
+  "exercise ball",
+  "easel",
+  "garbage bag",
+  "roomba",
+  "garage door",
+  "luggage rack",
+  "bike lock",
+  "briefcase",
+  "hand towel",
+  "bath products",
+  "star",
+  "map",
+  "coffee bean bag",
+  "headboard",
+  "ipad",
+  "display rack",
+  "traffic cone",
+  "toiletry",
+  "canopy",
+  "massage chair",
+  "paper organizer",
+  "barricade",
+  "platform",
+  "cap",
+  "dumbbell plates",
+  "elevator",
+  "cooking pan",
+  "trash bag",
+  "santa",
+  "jewelry box",
+  "boat",
+  "sock",
+  "kinect",
+  "crib",
+  "plastic storage bin",
+  "cooler",
+  "kitchen apron",
+  "dishwashing soap bottle",
+  "xbox controller",
+  "banana holder",
+  "ping pong paddle",
+  "airplane",
+  "conditioner bottle",
+  "tea kettle",
+  "bedframe",
+  "wood beam",
+  "toilet paper package",
+  "wall mounted coat rack",
+  "film light",
+  "ceiling lamp",
+  "chain",
+  "sofa",
+  "closet wardrobe",
+  "sweater",
+  "kitchen mixer",
+  "wardrobe",
+  "water softener",
+  "banister",
+  "trolley",
+  "pantry shelf",
+  "sofa bed",
+  "loofa",
+  "shower faucet handle",
+  "toy piano",
+  "fish",
+  "file cabinets",
+  "cat litter box",
+  "electric panel",
+  "suitcases",
+  "curtain rod",
+  "bunk bed",
+  "chandelier",
+  "tape",
+  "plates",
+  "alarm",
+  "fire hose",
+  "toy dinosaur",
+  "cone",
+  "glass doors",
+  "hatrack",
+  "subwoofer",
+  "fire sprinkler",
+  "trash cabinet",
+  "pantry walls",
+  "photo",
+  "barrier",
+  "stacks of cups",
+  "beachball",
+  "folded boxes",
+  "contact lens solution bottle",
+  "covered box",
+  "folder",
+  "mail trays",
+  "slipper",
+  "magazine rack",
+  "sticker",
+  "lotion",
+  "buddha",
+  "file organizer",
+  "paper towel rolls",
+  "night lamp",
+  "fuse box",
+  "knife block",
+  "furnace",
+  "cd cases",
+  "stools",
+  "hand sanitzer dispenser",
+  "teapot",
+  "pen holder",
+  "tray rack",
+  "wig",
+  "switch",
+  "plastic containers",
+  "night light",
+  "notepad",
+  "mail bin",
+  "elevator button",
+  "gaming wheel",
+  "drum set",
+  "cosmetic bag",
+  "coffee mug",
+  "closet shelf",
+  "baby mobile",
+  "diaper bin",
+  "door wall",
+  "stepstool",
+  "paper shredder",
+  "dress rack",
+  "cover",
+  "shopping bag",
+  "sliding door",
+  "exercise bike",
+  "recliner chair",
+  "kitchenaid mixer",
+  "soda can",
+  "stovetop",
+  "stepladder",
+  "tap",
+  "cable",
+  "baby changing station",
+  "costume",
+  "rocking chair",
+  "binder",
+  "media center",
+  "towel rack",
+  "medal",
+  "stack of folded chairs",
+  "telescope",
+  "closet doorframe",
+  "glass",
+  "baseball cap",
+  "battery disposal jar",
+  "mop",
+  "tank",
+  "mail tray",
+  "centerpiece",
+  "stick",
+  "closet floor",
+  "dryer sheets",
+  "bycicle",
+  "flower stand",
+  "air mattress",
+  "clip",
+  "side table",
+  "pizza boxes",
+  "display",
+  "postcard",
+  "display sign",
+  "paper towel",
+  "boots",
+  "tennis racket bag",
+  "air hockey table",
+  "socks",
+  "food bag",
+  "clothes hangers",
+  "starbucks cup"
+]

leo/grounding_head.py

@@ -0,0 +1,20 @@
+import torch
+from torch import nn
+from leo.utils import get_mlp_head
+
+
+class SequentialGroundHead(nn.Module):
+    def __init__(self, hidden_size=4096):
+        super().__init__()        
+        # grounding head
+        self.og3d_head = get_mlp_head(
+            hidden_size * 2, hidden_size // 2,
+            1, dropout=0.1
+        )
+
+    def forward(self, obj_embeds, grd_embdes, obj_masks=None):
+        txt_embeds = grd_embdes
+        og3d_logits = self.og3d_head(torch.cat((obj_embeds, txt_embeds.repeat(1, obj_embeds.shape[1], 1)), dim=2)).squeeze(2)
+        if obj_masks is not None:
+            og3d_logits = og3d_logits.masked_fill_(obj_masks.logical_not(), -float('inf')) 
+        return og3d_logits

leo/img_encoder.py

@@ -0,0 +1,161 @@
+import timm
+import torch
+import numpy as np
+import torch.nn as nn
+from einops import rearrange
+
+
+def disabled_train(self, mode=True):
+    """
+    Overwrite model.train with this function to make sure train/eval mode does not change anymore
+    """
+    return self
+
+
+def simple_conv_and_linear_weights_init(m):
+    if type(m) in [
+        nn.Conv1d,
+        nn.Conv2d,
+        nn.Conv3d,
+        nn.ConvTranspose1d,
+        nn.ConvTranspose2d,
+        nn.ConvTranspose3d,
+    ]:
+        weight_shape = list(m.weight.data.size())
+        fan_in = np.prod(weight_shape[1:4])
+        fan_out = np.prod(weight_shape[2:4]) * weight_shape[0]
+        w_bound = np.sqrt(6.0 / (fan_in + fan_out))
+        m.weight.data.uniform_(-w_bound, w_bound)
+        if m.bias is not None:
+            m.bias.data.fill_(0)
+    elif type(m) == nn.Linear:
+        simple_linear_weights_init(m)
+
+def simple_linear_weights_init(m):
+    if type(m) == nn.Linear:
+        weight_shape = list(m.weight.data.size())
+        fan_in = weight_shape[1]
+        fan_out = weight_shape[0]
+        w_bound = np.sqrt(6.0 / (fan_in + fan_out))
+        m.weight.data.uniform_(-w_bound, w_bound)
+        if m.bias is not None:
+            m.bias.data.fill_(0)
+
+
+class Backbone2DWrapper(nn.Module):
+
+    def __init__(self, model, tag, freeze=True):
+        super().__init__()
+        self.model = model
+        self.tag = tag
+        self.freeze = freeze
+        if 'convnext' in tag:
+            self.out_channels = 1024
+        elif 'swin' in tag:
+            self.out_channels = 1024
+        elif 'vit' in tag:
+            self.out_channels = 768
+        elif 'resnet' in tag:
+            self.out_channels = 2048
+        else:
+            raise NotImplementedError
+
+        if freeze:
+            for param in self.parameters():
+                param.requires_grad = False
+            self.eval()
+            self.train = disabled_train
+
+    def forward_normal(self, x, flat_output=False):
+        feat = self.model.forward_features(x)
+        if 'swin' in self.tag:
+            feat = rearrange(feat, 'b h w c -> b c h w')
+        if 'vit_base_32_timm_laion2b' in self.tag or 'vit_base_32_timm_openai' in self.tag:
+            # TODO: [CLS] is prepended to the patches.
+            feat = rearrange(feat[:, 1:], 'b (h w) c -> b c h w', h=7)
+        if flat_output:
+            feat = rearrange(feat, 'b c h w -> b (h w) c')
+        return feat
+
+    @torch.no_grad()
+    def forward_frozen(self, x, flat_output=False):
+        return self.forward_normal(x, flat_output)
+
+    def forward(self, x, flat_output=False):
+        if self.freeze:
+            return self.forward_frozen(x, flat_output)
+        else:
+            return self.forward_normal(x, flat_output)
+        
+def convnext_base_laion2b(pretrained=False, freeze=True, **kwargs):
+    m = timm.create_model(
+        'convnext_base.clip_laion2b',
+        pretrained=pretrained
+    )
+    if kwargs.get('reset_clip_s2b2'):
+        s = m.state_dict()
+        for i in s.keys():
+            if 'stages.3.blocks.2' in i and ('weight' in i or 'bias' in i):
+                s[i].normal_()
+        m.load_state_dict(s, strict=True)
+
+    return Backbone2DWrapper(m, 'convnext_base_laion2b', freeze=freeze)
+
+
+class GridFeatureExtractor2D(nn.Module):
+    def __init__(self, backbone_name='convnext_base', backbone_pretrain_dataset='laion2b', use_pretrain=True, freeze=True, pooling='avg'):
+        super().__init__()
+
+        init_func_name = '_'.join([backbone_name, backbone_pretrain_dataset])
+        init_func = globals().get(init_func_name)
+        if init_func and callable(init_func):
+            self.backbone = init_func(pretrained=use_pretrain, freeze=freeze)
+        else:
+            raise NotImplementedError(f"Backbone2D does not support {init_func_name}")
+
+        self.pooling = pooling
+        if self.pooling:
+            if self.pooling == 'avg':
+                self.pooling_layers = nn.Sequential(
+                    nn.AdaptiveAvgPool2d(output_size=(1,1)),
+                    nn.Flatten()
+                )
+                self.out_channels = self.backbone.out_channels
+            elif self.pooling == 'conv':
+                self.pooling_layers = nn.Sequential(
+                    nn.Conv2d(self.backbone.out_channels, 64, 1),
+                    nn.ReLU(inplace=True),
+                    nn.Conv2d(64, 32, 1),
+                    nn.Flatten()
+                )
+                self.pooling_layers.apply(simple_conv_and_linear_weights_init)
+                self.out_channels = 32 * 7 * 7   # hardcode for 224x224
+            elif self.pooling in ['attn', 'attention']:
+                self.visual_attention = nn.Sequential(
+                    nn.Conv2d(self.backbone.out_channels, self.backbone.out_channels, 1),
+                    nn.ReLU(inplace=True),
+                    nn.Conv2d(self.backbone.out_channels, self.backbone.out_channels, 1),
+                )
+                self.visual_attention.apply(simple_conv_and_linear_weights_init)
+                def _attention_pooling(x):
+                    B, C, H, W = x.size()
+                    attn = self.visual_attention(x)
+                    attn = attn.view(B, C, -1)
+                    x = x.view(B, C, -1)
+                    attn = attn.softmax(dim=-1)
+                    x = torch.einsum('b c n, b c n -> b c', x, x)
+                    return x
+                self.pooling_layers = _attention_pooling
+                self.out_channels = self.backbone.out_channels
+            else:
+                raise NotImplementedError(f"Backbone2D does not support {self.pooling} pooling")
+        else:
+            self.out_channels = self.backbone.out_channels
+
+    def forward(self, x):
+        if self.pooling:
+            x = self.backbone(x, flat_output=False)
+            x = self.pooling_layers(x).unsqueeze(1)
+            return x
+        else:
+            return self.backbone(x, flat_output=True)

leo/inference.py

@@ -0,0 +1,234 @@
+import json
+import os
+import torch
+import numpy as np
+from leo.model import SequentialGrounder
+from leo.utils import LabelConverter, convert_pc_to_box, obj_processing_post, pad_sequence
+from torch.utils.data import default_collate
+
+
+ASSET_DIR = os.path.join(os.getcwd(), 'assets')
+CKPT_DIR = os.path.join(os.getcwd(), 'checkpoint/leo')
+int2cat = json.load(open(os.path.join(ASSET_DIR, "meta/scannetv2_raw_categories.json"), 'r', encoding="utf-8"))
+cat2int = {w: i for i, w in enumerate(int2cat)}
+label_converter = LabelConverter(os.path.join(ASSET_DIR, "meta/scannetv2-labels.combined.tsv"))
+
+
+role_prompt = "You are an AI visual assistant situated in a 3D scene. "\
+    "You can perceive (1) an ego-view image (accessible when necessary) and (2) the objects (including yourself) in the scene (always accessible). "\
+        "You should properly respond to the USER's instruction according to the given visual information. "
+#role_prompt = " " 
+egoview_prompt = "Ego-view image:"
+objects_prompt = "Objects (including you) in the scene:"
+task_prompt = "USER: {instruction} ASSISTANT:"
+
+def get_prompt(instruction):
+    return {
+            'prompt_before_obj': role_prompt,
+            'prompt_middle_1': egoview_prompt,
+            'prompt_middle_2': objects_prompt,
+            'prompt_after_obj': task_prompt.format(instruction=instruction),
+        }
+
+def get_lang(task_item):
+    task_description = task_item['task_description']
+    sentence = task_description
+    data_dict = get_prompt(task_description)
+
+    # scan_id = task_item['scan_id']
+
+    if 'action_steps' in task_item:
+        action_steps = task_item['action_steps']
+        # tgt_object_id = [int(action['target_id']) for action in action_steps]
+        # tgt_object_name = [action['label'] for action in action_steps]
+    
+        for action in action_steps:
+            sentence += ' ' + action['action']
+
+        data_dict['output_gt'] = ' '.join([action['action'] + ' <s>' for action in action_steps])
+
+    # return scan_id, tgt_object_id, tgt_object_name, sentence, data_dict
+    return data_dict
+
+
+def load_data(scan_id):
+    one_scan = {}
+    # load scan
+    pcd_data = torch.load(os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_pcd.pth'))
+    inst_to_label = torch.load(os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_inst.pth')) 
+    points, colors, instance_labels  = pcd_data[0], pcd_data[1], pcd_data[-1]
+    colors = colors / 127.5 - 1
+    pcds = np.concatenate([points, colors], 1)
+    one_scan['pcds'] = pcds
+    one_scan['instance_labels'] = instance_labels
+    one_scan['inst_to_label'] = inst_to_label
+    # convert to gt object
+    obj_pcds = []
+    inst_ids = []
+    inst_labels = []
+    bg_indices = np.full((points.shape[0], ), 1, dtype=np.bool_)
+    for inst_id in inst_to_label.keys():
+        if inst_to_label[inst_id] in cat2int.keys():
+            mask = instance_labels == inst_id
+            if np.sum(mask) == 0:
+                continue
+            obj_pcds.append(pcds[mask])
+            inst_ids.append(inst_id)
+            inst_labels.append(cat2int[inst_to_label[inst_id]])
+            if inst_to_label[inst_id] not in ['wall', 'floor', 'ceiling']:
+                bg_indices[mask] = False
+    one_scan['obj_pcds'] = obj_pcds
+    one_scan['inst_labels'] = inst_labels
+    one_scan['inst_ids'] = inst_ids
+    one_scan['bg_pcds'] = pcds[bg_indices]
+    # calculate box for matching
+    obj_center = []
+    obj_box_size = []
+    for obj_pcd in obj_pcds:
+        _c, _b = convert_pc_to_box(obj_pcd)
+        obj_center.append(_c)
+        obj_box_size.append(_b)
+    one_scan['obj_loc'] = obj_center
+    one_scan['obj_box'] = obj_box_size
+    # load point feat
+    feat_pth = os.path.join(ASSET_DIR, f'inputs/{scan_id}', 'obj_feats.pth')
+    one_scan['obj_feats'] = torch.load(feat_pth).to('cpu')
+    #  convert to pq3d input
+    obj_labels = one_scan['inst_labels'] # N
+    obj_pcds = one_scan['obj_pcds']
+    obj_ids = one_scan['inst_ids'] 
+    # object filter
+    excluded_labels = ['wall', 'floor', 'ceiling']
+    def keep_obj(i, obj_label):
+        category = int2cat[obj_label]
+        # filter out background
+        if category in excluded_labels:
+            return False
+        # filter out objects not mentioned in the sentence
+        return True
+    selected_obj_idxs = [i for i, obj_label in enumerate(obj_labels) if keep_obj(i, obj_label)]
+    # crop objects to max_obj_len and reorganize ids ? # TODO
+    obj_labels = [obj_labels[i] for i in selected_obj_idxs]
+    obj_pcds = [obj_pcds[i] for i in selected_obj_idxs]
+    # subsample points
+    obj_pcds = np.array([obj_pcd[np.random.choice(len(obj_pcd), size=1024,
+                                        replace=len(obj_pcd) < 1024)] for obj_pcd in obj_pcds])
+    obj_fts, obj_locs, obj_boxes, rot_matrix = obj_processing_post(obj_pcds, rot_aug=False)
+    data_dict = {
+        "scan_id": scan_id,
+        "obj_fts": obj_fts.float(),
+        "obj_locs": obj_locs.float(),
+        "obj_labels": torch.LongTensor(obj_labels),
+        "obj_boxes": obj_boxes, 
+        "obj_pad_masks": torch.ones((len(obj_locs)), dtype=torch.bool), # used for padding in collate
+        "obj_ids": torch.LongTensor([obj_ids[i] for i in selected_obj_idxs])
+    }
+    # convert point feature
+    data_dict['obj_feats'] = one_scan['obj_feats'].squeeze(0)
+
+    useful_keys = ['tgt_object_id', 'scan_id', 'obj_labels', 'data_idx', 
+                   'obj_fts', 'obj_locs', 'obj_pad_masks', 'obj_ids', 
+                   'source', 'prompt_before_obj', 'prompt_middle_1', 
+                   'prompt_middle_2', 'prompt_after_obj', 'output_gt', 'obj_feats']
+    for k in list(data_dict.keys()):
+            if k not in useful_keys:
+                del data_dict[k]
+    # add new keys because of leo
+    data_dict['img_fts'] = torch.zeros(3, 224, 224)
+    data_dict['img_masks'] = torch.LongTensor([0]).bool()
+    data_dict['anchor_locs'] = torch.zeros(3)
+    data_dict['anchor_orientation'] = torch.zeros(4)
+    data_dict['anchor_orientation'][-1] = 1   # xyzw
+    # convert to leo format
+    data_dict['obj_masks'] = data_dict['obj_pad_masks']
+    del data_dict['obj_pad_masks']
+
+    return data_dict
+
+def form_batch(data_dict):
+    batch = [data_dict]
+    new_batch = {}
+
+    # pad
+    padding_keys = ['obj_fts', 'obj_locs', 'obj_masks', 'obj_labels', 'obj_ids']
+    for k in padding_keys:
+        tensors = [sample.pop(k) for sample in batch]
+        padded_tensor = pad_sequence(tensors, pad=0)
+        new_batch[k] = padded_tensor
+    # # list
+    # list_keys = ['tgt_object_id']
+    # for k in list_keys:
+    #     new_batch[k] = [sample.pop(k) for sample in batch]
+
+    # default collate
+    new_batch.update(default_collate(batch))
+    return new_batch
+
+    
+def inference(scan_id, task, predict_mode=False):
+    # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+    device = 'cpu' # ok for predict_mode=False, and both for Gradio demo
+
+    data_dict = load_data(scan_id)
+    data_dict.update(get_lang(task))
+    data_dict = form_batch(data_dict)
+
+    for key, value in data_dict.items():
+        if isinstance(value, torch.Tensor):
+            data_dict[key] = value.to(device)
+    
+    model = SequentialGrounder(predict_mode)
+    load_msg = model.load_state_dict(torch.load(os.path.join(CKPT_DIR, 'pytorch_model.bin'), map_location='cpu'), strict=False)
+    model.to(device)
+
+    data_dict = model(data_dict)
+
+    if predict_mode == False:
+        # calculate result id
+        result_id_list = [data_dict['obj_ids'][0][torch.argmax(data_dict['ground_logits'][i]).item()] 
+                        for i in range(len(data_dict['ground_logits']))]
+    else:
+        # calculate langauge
+        # tgt_object_id = data_dict['tgt_object_id']
+        if data_dict['ground_logits'] == None:
+            og_pred = []
+        else:
+            og_pred = torch.argmax(data_dict['ground_logits'], dim=1)
+        grd_batch_ind_list = data_dict['grd_batch_ind_list']
+        
+        response_pred = []
+        for i in range(1): # len(tgt_object_id)
+            # target_sequence = list(tgt_object_id[i].cpu().numpy())
+            predict_sequence = []
+            if og_pred != None:
+                for j in range(len(og_pred)):
+                    if grd_batch_ind_list[j] == i:
+                        predict_sequence.append(og_pred[j].item())
+
+            obj_ids = data_dict['obj_ids']
+            response_pred.append({
+                'predict_object_id' : [obj_ids[i][o].item() for o in predict_sequence],
+                'predict_object_id': [obj_ids[i][o].item() for o in predict_sequence],
+                'pred_plan_text': data_dict['output_txt'][i]
+            })
+
+    return result_id_list if predict_mode == False else response_pred
+
+if __name__ == '__main__':
+    inference("scene0050_00", {
+        "task_description": "Find the chair and move it to the table.",
+        "action_steps": [
+            {
+                "target_id": "1",
+                "label": "chair",
+                "action": "Find the chair."
+            },
+            {
+                "target_id": "2",
+                "label": "table",
+                "action": "Move the chair to the table."
+            }
+        ],
+        "scan_id": "scene0050_00"
+    }, predict_mode=True)
+    

leo/model.py

@@ -0,0 +1,477 @@
+import contextlib
+import math
+
+import clip
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM, LlamaTokenizer
+from leo.img_encoder import GridFeatureExtractor2D
+from leo.pcd_encoder import OSE3D
+from leo.grounding_head import SequentialGroundHead
+from leo.utils import get_mlp_head
+
+
+def maybe_autocast(model, dtype='float32', enabled=True): ### not-half mode
+    # if on cpu, don't use autocast
+    # if on gpu, use autocast with dtype if provided, otherwise use torch.float16
+    enable_autocast = model.device != torch.device('cpu')
+
+    if dtype == 'bf16':
+        dtype = torch.bfloat16
+    elif dtype == 'fp16':
+        dtype == torch.float16
+    else:
+        dtype = torch.float32
+
+    if enable_autocast:
+        return torch.cuda.amp.autocast(dtype=dtype, enabled=enabled)
+    else:
+        return contextlib.nullcontext()
+
+def disabled_train(self, mode=True):
+    """
+    Overwrite model.train with this function to make sure train/eval mode does not change anymore
+    """
+    return self
+
+
+class SequentialGrounder(torch.nn.Module):
+    def __init__(self,predict_mode=False):
+        super().__init__()
+        cfg = {
+            "model": {
+                "llm": {
+                    "name": "Vicuna7B",
+                    "cfg_path": "/scratch/generalvision/vicuna-7b",
+                    "truncation_side": "right",
+                    "max_context_len": 256,
+                    "max_out_len": 256,
+                    "lora": {
+                        "flag": True,
+                        "rank": 16,
+                        "alpha": 16,
+                        "dropout": 0.0,
+                        "target_modules": ['q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj'],
+                    },
+                },
+                "clip_txt_guidance": {
+                    "flag": False,
+                    "clip_out_dim": 1024,
+                },
+            },
+        }
+
+        self.predict_mode = predict_mode
+        
+        # LLM
+        llm_name = 'Vicuna7B'
+        llm_cfg_path = '/scratch/generalvision/vicuna-7b'
+        llm_truncation_side = 'right'
+        if 'vicuna' in llm_name.lower():
+            self.llm_tokenizer = LlamaTokenizer.from_pretrained(llm_cfg_path, truncation_side=llm_truncation_side)
+            self.llm_tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+            self.llm_model = LlamaForCausalLM.from_pretrained(llm_cfg_path, torch_dtype=torch.float32) # not-half mode torch_dtype=torch.float16
+            self.llm_model.resize_token_embeddings(len(self.llm_tokenizer))
+        else:
+            self.llm_tokenizer = AutoTokenizer.from_pretrained(llm_cfg_path, truncation_side=llm_truncation_side)
+            self.llm_model = AutoModelForCausalLM.from_pretrained(llm_cfg_path, torch_dtype=torch.float16)
+
+        for param in self.llm_model.parameters():
+            param.requires_grad = False
+        self.llm_model.eval()
+        self.llm_model.train = disabled_train
+
+        # 2D vision
+        self.img_encoder = GridFeatureExtractor2D()
+        self.img_proj = nn.Linear(
+            self.img_encoder.out_channels, self.llm_model.config.hidden_size
+        )
+
+        # 3D vision
+        self.pcd_encoder = OSE3D()
+        self.pcd_proj = nn.Linear(256, self.llm_model.config.hidden_size)
+
+        # type embedding
+        # self.img_type_embed = nn.Parameter(torch.zeros(self.llm_model.config.hidden_size), requires_grad=True)
+        # self.pcd_type_embed = nn.Parameter(torch.zeros(self.llm_model.config.hidden_size), requires_grad=True)
+
+        # LoRA
+        if cfg['model']['llm']['lora']['flag']:
+            lora_config = LoraConfig(
+                r=cfg['model']['llm']['lora']['rank'],
+                lora_alpha=cfg['model']['llm']['lora']['alpha'],
+                target_modules=cfg['model']['llm']['lora']['target_modules'],
+                lora_dropout=cfg['model']['llm']['lora']['dropout'],
+                bias='none',
+                modules_to_save=[],
+            )
+            self.llm_model = get_peft_model(self.llm_model, peft_config=lora_config)
+
+        self.max_context_len = 256
+        self.max_out_len = 256
+
+        # additional text x multi-modal tokens fusion
+        self.clip_txt_guidance = cfg['model']['clip_txt_guidance']['flag']
+        if self.clip_txt_guidance:
+            self.clip_model = clip.load('RN50')[0]
+            for param in self.clip_model.parameters():
+                param.requires_grad = False
+            self.clip_model.eval()
+            self.clip_model.train = disabled_train
+            self.clip_proj = nn.Linear(cfg['clip_txt_guidance']['clip_out_dim'], self.llm_model.config.hidden_size)
+        
+        # grounding head
+        self.ground_head = SequentialGroundHead()
+        self.obj_cls_head = get_mlp_head(4096, 768, 607, 0.3)
+        self.pre_grounding = True
+
+    @property
+    def device(self):
+        return list(self.parameters())[0].device
+
+    def build_right_justified_sequence(self, data_dict):
+        """
+        Concat six sequences: `prompt_before_obj`, `prompt_middle_1`, `img_tokens`, `prompt_middle_2`, `obj_tokens`, `prompt_after_obj`.
+        Return right justified sequence for causal LM: <pad>, <role/situation>, <img>, <objs>, <instruction>.
+        """
+        device = self.device
+        bs = len(data_dict['prompt_before_obj'])
+
+        self.llm_tokenizer.padding_side = 'left'
+        text_input_tokens_pre = self.llm_tokenizer(
+            data_dict['prompt_before_obj'],
+            return_tensors='pt',
+            padding='longest'
+        ).to(device)   # [PAD, BOS, tokens], (B, T1)
+
+        text_input_tokens_mid1 = self.llm_tokenizer(
+            data_dict['prompt_middle_1'],
+            return_tensors='pt',
+            padding='longest'
+        ).to(device)
+
+        img_tokens = data_dict['img_tokens'].to(device)
+        img_masks = data_dict['img_masks'].to(device)
+        img_masks = img_masks.reshape(-1, 1).repeat(1, img_tokens.size(1))
+
+        text_input_tokens_mid2 = self.llm_tokenizer(
+            data_dict['prompt_middle_2'],
+            return_tensors='pt',
+            padding='longest'
+        ).to(device)
+
+        obj_tokens = data_dict['obj_tokens'].to(device)
+        obj_masks = data_dict['obj_masks'].to(device)
+
+        # additional clip fusion
+        if self.clip_txt_guidance:
+            with torch.no_grad():
+                clip_fts = self.clip_model.encode_text(
+                    clip.tokenize(data_dict['prompt_after_obj'], truncate=True).to(device)
+                )
+            clip_fts = self.clip_proj(clip_fts)
+            # B, N, C
+            img_tokens = torch.einsum('bnc,bc->bnc', img_tokens, clip_fts)
+            obj_tokens = torch.einsum('bnc,bc->bnc', obj_tokens, clip_fts)
+
+        self.llm_tokenizer.padding_side = 'right'   # no need to be 'left', as padding tokens will be shifted
+        self.llm_tokenizer.truncation_side = 'left'   # truncate history
+        text_input_tokens_post = self.llm_tokenizer(
+            data_dict['prompt_after_obj'],
+            return_tensors='pt',
+            padding='longest',
+            truncation=True,
+            max_length=self.max_context_len,
+        ).to(device)   # [BOS, tokens, PAD], (B, T3)
+
+        assert text_input_tokens_mid1.attention_mask.all() and text_input_tokens_mid2.attention_mask.all(), \
+               "prompt_middle should be the same and thus no padding"
+
+        # remove bos, make "tokenize subseq and concat" equivalent to "tokenize the whole seq"
+        text_input_tokens_mid1.input_ids = text_input_tokens_mid1.input_ids[:, 1:]
+        text_input_tokens_mid1.attention_mask = text_input_tokens_mid1.attention_mask[:, 1:]
+        text_input_tokens_mid2.input_ids = text_input_tokens_mid2.input_ids[:, 1:]
+        text_input_tokens_mid2.attention_mask = text_input_tokens_mid2.attention_mask[:, 1:]
+        text_input_tokens_post.input_ids = text_input_tokens_post.input_ids[:, 1:]
+        text_input_tokens_post.attention_mask = text_input_tokens_post.attention_mask[:, 1:]
+        for i in range(bs):
+            if not img_masks[i].any():
+                # no image input, also mask the text prompt for image tokens
+                text_input_tokens_mid1.attention_mask[i].fill_(0)
+
+        inputs_embeds_pre = self.llm_model.get_input_embeddings()(text_input_tokens_pre.input_ids)
+        inputs_embeds_mid1 = self.llm_model.get_input_embeddings()(text_input_tokens_mid1.input_ids)
+        inputs_embeds_mid2 = self.llm_model.get_input_embeddings()(text_input_tokens_mid2.input_ids)
+        inputs_embeds_post = self.llm_model.get_input_embeddings()(text_input_tokens_post.input_ids)
+
+        # since img_tokens, prompt_mid, obj_tokens are fixed length without padding, we concat them first
+        inputs_embeds_mid = torch.cat([inputs_embeds_mid1, img_tokens, inputs_embeds_mid2, obj_tokens], dim=1)
+        attn_mask_mid = torch.cat(
+            [text_input_tokens_mid1.attention_mask, img_masks, text_input_tokens_mid2.attention_mask, obj_masks],
+            dim=1,
+        )
+
+        post_pad_length = torch.logical_not(text_input_tokens_post.attention_mask).sum(-1)
+
+        bs, l1, hidden_dim = inputs_embeds_pre.shape
+        _, l2, _ = inputs_embeds_mid.shape
+        _, l3, _ = inputs_embeds_post.shape
+
+        inputs_embeds = torch.zeros(bs, l1+l2+l3, hidden_dim).type(inputs_embeds_pre.dtype).to(device)
+        attention_mask = torch.zeros(bs, l1+l2+l3).type(obj_masks.dtype).to(device)
+
+        # assign by chunks
+        for i in range(bs):
+            post_pad_len = post_pad_length[i]
+
+            if post_pad_len > 0:
+                inputs_embeds[i, :post_pad_len] = inputs_embeds_post[i, -post_pad_len:]
+                attention_mask[i, :post_pad_len] = 0
+                inputs_embeds[i, post_pad_len+l1+l2:] = inputs_embeds_post[i, :-post_pad_len]
+                attention_mask[i, post_pad_len+l1+l2:] = 1
+            else:
+                # no padding
+                inputs_embeds[i, -l3:] = inputs_embeds_post[i]
+                attention_mask[i, -l3:] = 1
+
+            inputs_embeds[i, post_pad_len: post_pad_len+l1] = inputs_embeds_pre[i]
+            attention_mask[i, post_pad_len: post_pad_len+l1] = text_input_tokens_pre.attention_mask[i]
+
+            inputs_embeds[i, post_pad_len+l1: post_pad_len+l1+l2] = inputs_embeds_mid[i]
+            attention_mask[i, post_pad_len+l1: post_pad_len+l1+l2] = attn_mask_mid[i]
+
+        return inputs_embeds, attention_mask, (l1, l2, l3)
+
+    def forward(self, data_dict):
+        if self.predict_mode:
+            return self.generate(data_dict=data_dict)
+        """
+        data_dict requires keys:
+        # input
+        prompt_before_obj: list of str, (B,)
+        prompt_middle_1: list of str, (B,)
+        prompt_middle_2: list of str, (B,)
+        prompt_after_obj: list of str, (B,)
+        obj_fts: (B, N, P, 6), xyz + rgb
+        obj_masks: (B, N), 1 valid and 0 masked
+        obj_locs: (B, N, 6), xyz + whd
+        anchor_locs: (B, 3)
+        anchor_orientation: (B, C)
+        img_fts: (B, 3, H, W), rgb
+        img_masks: (B, 1), 1 valid and 0 masked
+        # output
+        output_gt: list of str, (B,)
+        """
+        device = self.device
+        bs = len(data_dict['prompt_after_obj'])
+        data_dict['bs'] = bs
+        if 'obj_tokens' not in data_dict:
+            # obtain obj tokens
+            data_dict = self.pcd_encoder(data_dict)
+            # TO CHANGE FOR DEBUG
+            #self.llm_model.float()
+            #data_dict['obj_tokens'] = torch.zeros((data_dict['obj_locs'].shape[0], data_dict['obj_locs'].shape[1], 256)).to(device=device)
+
+        data_dict['obj_tokens'] = self.pcd_proj(data_dict['obj_tokens'].to(device))
+        # data_dict['obj_tokens'] = data_dict['obj_tokens'] + self.pcd_type_embed
+
+        data_dict['img_tokens'] = self.img_proj(self.img_encoder(data_dict['img_fts']))
+        # data_dict['img_tokens'] = data_dict['img_tokens'] + self.img_type_embed
+        
+        # build input embdes and record prompt position
+        inputs_embeds, attention_mask, input_length = self.build_right_justified_sequence(data_dict=data_dict)
+        obj_token_length = data_dict['obj_masks'].shape[1]
+        # (B, T1+O+T2, D), (B, T1+O+T2)
+
+        self.llm_tokenizer.padding_side = 'right'
+        self.llm_tokenizer.truncation_side = 'right'
+        text_output_tokens = self.llm_tokenizer(
+            [t + self.llm_tokenizer.eos_token for t in data_dict['output_gt']],
+            return_tensors='pt',
+            padding='longest',
+            truncation=True,
+            max_length=self.max_out_len,
+        ).to(device)
+        # record position for special token [SOS]
+        grd_token_id = self.llm_tokenizer.convert_tokens_to_ids(['<s>'])[0]
+        out_input_ids_remove_first_sos = text_output_tokens.input_ids.clone()
+        out_input_ids_remove_first_sos[:, 0] = -100
+        grd_ind_0, grd_ind_1 = (out_input_ids_remove_first_sos == grd_token_id).nonzero(as_tuple=True)
+        
+
+        text_output_embeds = self.llm_model.get_input_embeddings()(text_output_tokens.input_ids)   # (B, T3, D)
+        inputs_embeds = torch.cat([inputs_embeds, text_output_embeds], dim=1)   # (B, T1+O+T2+T3, D)
+        attention_mask = torch.cat([attention_mask, text_output_tokens.attention_mask], dim=1)   # (B, T1+O+T2+T3)
+
+        # construct targets
+        targets = torch.zeros_like(attention_mask).long().fill_(-100)   # (B, T1+O+T2+T3)
+
+        # only apply loss to answer tokens
+        targets_idx = text_output_tokens.attention_mask.bool()
+        targets[:, -targets_idx.shape[1]:][targets_idx] = text_output_tokens.input_ids[targets_idx]
+
+        # do not predict bos token, regard it as condition instead
+        targets[:, -targets_idx.shape[1]] = -100
+
+        with maybe_autocast(self):
+            outputs = self.llm_model(
+                inputs_embeds=inputs_embeds.float(), # not-half mode
+                attention_mask=attention_mask,
+                return_dict=True,
+                output_hidden_states=True,
+            )
+
+        logits = outputs.logits.float()
+        last_hidden_state = outputs.hidden_states[-1]
+
+        # different from the loss inside `llm_model.forward`, here we take mean of each sequence instead of sum
+        shift_logits = logits[..., :-1, :].contiguous()
+        shift_labels = targets[..., 1:].contiguous()
+        num_tokens_for_loss = (shift_labels >= 0).int().sum(1)   # (B,)
+
+        shift_logits = rearrange(shift_logits, 'b t v -> (b t) v')
+        shift_labels = rearrange(shift_labels, 'b t -> (b t)')
+
+        shift_labels = shift_labels.to(shift_logits.device)
+        
+        # record for llm loss
+        data_dict['llm_logits'] = shift_logits
+        data_dict['llm_labels'] = shift_labels
+        data_dict['num_tokens_for_loss'] = num_tokens_for_loss
+        
+        # record for grounding loss
+        grd_list = []
+        obj_list = []
+        mask_list = []
+        for step in range(len(grd_ind_0)):
+            batch_ind = grd_ind_0[step]
+            grd_token_ind = grd_ind_1[step]
+            if self.pre_grounding:
+                output_obj_tokens = data_dict['obj_tokens'][batch_ind]
+            else:
+                output_obj_tokens = last_hidden_state[batch_ind, input_length[0] + input_length[1] - obj_token_length : input_length[0] + input_length[1], :]
+            output_grd_tokens = last_hidden_state[batch_ind, sum(input_length) + grd_token_ind:sum(input_length) + grd_token_ind + 1, :]
+            grd_list.append(output_grd_tokens)
+            obj_list.append(output_obj_tokens)
+            mask_list.append(data_dict['obj_masks'][batch_ind])
+        output_obj = torch.stack(obj_list).float()
+        output_grd = torch.stack(grd_list).float()
+        data_dict['ground_logits'] = self.ground_head(output_obj, output_grd, torch.stack(mask_list))
+        # data_dict['ground_label'] = torch.concat(data_dict['tgt_object_id'], dim=0)
+        
+        # record for cls loss
+        #obj_cls_post_embeds = last_hidden_state[:, input_length[0] + input_length[1] - obj_token_length : input_length[0] + input_length[1], :].float()
+        obj_cls_post_embeds = data_dict['obj_tokens'].float()
+        data_dict['obj_cls_post_logits'] = self.obj_cls_head(obj_cls_post_embeds)
+        return data_dict
+
+    @torch.no_grad()
+    def generate(
+        self,
+        data_dict,
+        use_nucleus_sampling=False,
+        num_beams=5,
+        max_length=256,
+        min_length=1,
+        top_p=0.9,
+        repetition_penalty=6.0,
+        length_penalty=1,
+        num_captions=1,
+        temperature=1,
+    ):
+        """
+        data_dict requires the same keys as forward() except output_gt
+        """
+        device = self.device
+        bs = len(data_dict['prompt_after_obj'])
+        data_dict['bs'] = bs
+        if 'obj_tokens' not in data_dict:
+            # obtain obj tokens
+            data_dict = self.pcd_encoder(data_dict)
+            # TO CHANGE FOR DEBUG
+            #self.llm_model.float()
+            #data_dict['obj_tokens'] = torch.zeros((data_dict['obj_locs'].shape[0], data_dict['obj_locs'].shape[1], 256)).to(device=device)
+
+        data_dict['obj_tokens'] = self.pcd_proj(data_dict['obj_tokens'].to(device))
+        # data_dict['obj_tokens'] = data_dict['obj_tokens'] + self.pcd_type_embed
+
+        data_dict['img_tokens'] = self.img_proj(self.img_encoder(data_dict['img_fts']))
+        # data_dict['img_tokens'] = data_dict['img_tokens'] + self.img_type_embed
+
+        inputs_embeds, attention_mask, input_length = self.build_right_justified_sequence(data_dict=data_dict)
+        obj_token_length = data_dict['obj_masks'].shape[1]
+        
+        # give bos token as condition
+        bos_tokens = self.llm_tokenizer(
+            [self.llm_tokenizer.bos_token] * bs,
+            return_tensors='pt',
+        ).to(device)
+        bos_tokens_ids = bos_tokens.input_ids[:, 0:1]   # (B, 1)
+        bos_tokens_attn = bos_tokens.attention_mask[:, 0:1]   # (B, 1)
+
+        # prepare a `bos_token`
+        bos_embeds = self.llm_model.get_input_embeddings()(bos_tokens_ids)   # (B, 1, D)
+        inputs_embeds = torch.cat([inputs_embeds, bos_embeds], dim=1)   # (B, T1+O+T2+1, D)
+        attention_mask = torch.cat([attention_mask, bos_tokens_attn], dim=1)   # (B, T1+O+T2+1)
+
+        with maybe_autocast(self):
+            outputs = self.llm_model.generate(
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                do_sample=use_nucleus_sampling,
+                top_p=top_p,
+                temperature=temperature,
+                num_beams=num_beams,
+                max_length=max_length,
+                min_length=min_length,
+                repetition_penalty=repetition_penalty,
+                length_penalty=length_penalty,
+                num_return_sequences=num_captions,
+                return_dict_in_generate=True,
+                output_hidden_states=True,
+                output_scores=True
+            )
+        # note output_ids_idx - 1 = step idx, because we do not preduct [BOS] 
+        beam_indices = outputs.beam_indices # bs x step, beam indices range (bsxbeam)
+        scores = outputs.scores # step x (bs x beam) x vocab
+        hidden_states = outputs.hidden_states # step x layer x (bs x beam) x token_num x hidden_dim
+        outputs = outputs.sequences # bs x output_ids
+        outputs[outputs == self.llm_tokenizer.unk_token_id] = self.llm_tokenizer.eos_token_id
+        # data_dict['output_tokens'] = outputs   # unable to gather variable-length tensors
+        
+        # record for grounding
+        grd_token_id = self.llm_tokenizer.convert_tokens_to_ids(['<s>'])[0]
+        out_input_ids_remove_first_sos = outputs.clone()
+        out_input_ids_remove_first_sos[:, 0] = -100
+        grd_ind_0, grd_ind_1 = (out_input_ids_remove_first_sos == grd_token_id).nonzero(as_tuple=True)
+        
+        grd_list = []
+        grd_batch_ind_list = []
+        obj_list = []
+        mask_list = []
+        if len(grd_ind_0) > 0:
+            for step in range(len(grd_ind_0)):
+                batch_ind = grd_ind_0[step]
+                grd_token_ind = grd_ind_1[step]
+                #output_obj_tokens = last_hidden_state[batch_ind, input_length[0] + input_length[1] - obj_token_length : input_length[0] + input_length[1], :]
+                output_obj_tokens = data_dict['obj_tokens'][batch_ind]
+                output_grd_tokens = hidden_states[grd_token_ind-1][-1][beam_indices[batch_ind, grd_token_ind-1]][-1].unsqueeze(0) # grd_token_ind - 1 because first token is sos
+                grd_list.append(output_grd_tokens)
+                grd_batch_ind_list.append(batch_ind)
+                obj_list.append(output_obj_tokens)
+                mask_list.append(data_dict['obj_masks'][batch_ind])
+            output_obj = torch.stack(obj_list).float()
+            output_grd = torch.stack(grd_list).float()
+            data_dict['ground_logits'] = self.ground_head(output_obj, output_grd, torch.stack(mask_list)) 
+        else:
+            data_dict['ground_logits'] = None
+        # data_dict['ground_label'] = torch.concat(data_dict['tgt_object_id'], dim=0)
+        data_dict['grd_batch_ind_list'] = grd_batch_ind_list
+        
+        output_txt = self.llm_tokenizer.batch_decode(outputs, skip_special_tokens=True)
+        output_txt = [txt.strip() for txt in output_txt]
+        data_dict['output_txt'] = output_txt
+        return data_dict

leo/pcd_encoder.py

@@ -0,0 +1,406 @@
+import torch
+import einops
+import numpy as np
+import torch.nn.functional as F
+from torch import Tensor, nn
+from typing import Optional
+from leo.utils import get_activation_fn, layer_repeat, calc_pairwise_locs
+
+
+def disabled_train(self, mode=True):
+    """
+    Overwrite model.train with this function to make sure train/eval mode does not change anymore
+    """
+    return self
+
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, dim_feedforward=2048, batch_first=True, dropout=0.1, activation="relu", prenorm=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout, batch_first=batch_first
+        )
+        # Implementation of Feedforward modules
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = get_activation_fn(activation)
+        self.prenorm = prenorm
+
+    def forward(
+            self, tgt, tgt_mask: Optional[Tensor] = None,
+            tgt_key_padding_mask: Optional[Tensor] = None,
+    ):
+        tgt2 = tgt
+        if self.prenorm:
+            tgt2 = self.norm1(tgt2)
+        tgt2, self_attn_matrices = self.self_attn(
+            query=tgt2, key=tgt2, value=tgt2, attn_mask=tgt_mask,
+            key_padding_mask=tgt_key_padding_mask
+        )
+        tgt = tgt + self.dropout1(tgt2)
+        if not self.prenorm:
+            tgt = self.norm1(tgt)
+        if self.prenorm:
+            tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout2(tgt2)
+        if not self.prenorm:
+            tgt = self.norm2(tgt)
+        return tgt, self_attn_matrices
+
+
+class MultiHeadAttentionSpatial(nn.Module):
+    def __init__(
+            self, d_model, n_head, dropout=0.1, spatial_multihead=True, spatial_dim=5,
+            spatial_attn_fusion='mul',
+    ):
+        super().__init__()
+        assert d_model % n_head == 0, 'd_model: %d, n_head: %d' % (d_model, n_head)
+
+        self.n_head = n_head
+        self.d_model = d_model
+        self.d_per_head = d_model // n_head
+        self.spatial_multihead = spatial_multihead
+        self.spatial_dim = spatial_dim
+        self.spatial_attn_fusion = spatial_attn_fusion
+
+        self.w_qs = nn.Linear(d_model, d_model)
+        self.w_ks = nn.Linear(d_model, d_model)
+        self.w_vs = nn.Linear(d_model, d_model)
+
+        self.fc = nn.Linear(d_model, d_model)
+        self.dropout = nn.Dropout(p=dropout)
+        self.layer_norm = nn.LayerNorm(d_model)
+
+        self.spatial_n_head = n_head if spatial_multihead else 1
+        if self.spatial_attn_fusion in ['mul', 'bias', 'add']:
+            self.pairwise_loc_fc = nn.Linear(spatial_dim, self.spatial_n_head)
+        elif self.spatial_attn_fusion == 'ctx':
+            self.pairwise_loc_fc = nn.Linear(spatial_dim, d_model)
+        elif self.spatial_attn_fusion == 'cond':
+            self.lang_cond_fc = nn.Linear(d_model, self.spatial_n_head * (spatial_dim + 1))
+        else:
+            raise NotImplementedError('unsupported spatial_attn_fusion %s' % (self.spatial_attn_fusion))
+
+    def forward(self, q, k, v, pairwise_locs, key_padding_mask=None, txt_embeds=None):
+        residual = q
+        q = einops.rearrange(self.w_qs(q), 'b l (head k) -> head b l k', head=self.n_head)
+        k = einops.rearrange(self.w_ks(k), 'b t (head k) -> head b t k', head=self.n_head)
+        v = einops.rearrange(self.w_vs(v), 'b t (head v) -> head b t v', head=self.n_head)
+        attn = torch.einsum('hblk,hbtk->hblt', q, k) / np.sqrt(q.shape[-1])
+
+        if self.spatial_attn_fusion in ['mul', 'bias', 'add']:
+            loc_attn = self.pairwise_loc_fc(pairwise_locs)
+            loc_attn = einops.rearrange(loc_attn, 'b l t h -> h b l t')
+            if self.spatial_attn_fusion == 'mul':
+                loc_attn = F.relu(loc_attn)
+            if not self.spatial_multihead:
+                loc_attn = einops.repeat(loc_attn, 'h b l t -> (h nh) b l t', nh=self.n_head)
+        elif self.spatial_attn_fusion == 'ctx':
+            loc_attn = self.pairwise_loc_fc(pairwise_locs)
+            loc_attn = einops.rearrange(loc_attn, 'b l t (h k) -> h b l t k', h=self.n_head)
+            loc_attn = torch.einsum('hblk,hbltk->hblt', q, loc_attn) / np.sqrt(q.shape[-1])
+        elif self.spatial_attn_fusion == 'cond':
+            spatial_weights = self.lang_cond_fc(residual)
+            spatial_weights = einops.rearrange(spatial_weights, 'b l (h d) -> h b l d', h=self.spatial_n_head,
+                                               d=self.spatial_dim + 1)
+            if self.spatial_n_head == 1:
+                spatial_weights = einops.repeat(spatial_weights, '1 b l d -> h b l d', h=self.n_head)
+            spatial_bias = spatial_weights[..., :1]
+            spatial_weights = spatial_weights[..., 1:]
+            loc_attn = torch.einsum('hbld,bltd->hblt', spatial_weights, pairwise_locs) + spatial_bias
+            loc_attn = torch.sigmoid(loc_attn)
+
+        if key_padding_mask is not None:
+            mask = einops.repeat(key_padding_mask, 'b t -> h b l t', h=self.n_head, l=q.size(2))
+            attn = attn.masked_fill(mask, -np.inf)
+            if self.spatial_attn_fusion in ['mul', 'cond']:
+                loc_attn = loc_attn.masked_fill(mask, 0)
+            else:
+                loc_attn = loc_attn.masked_fill(mask, -np.inf)
+
+        if self.spatial_attn_fusion == 'add':
+            fused_attn = (torch.softmax(attn, 3) + torch.softmax(loc_attn, 3)) / 2
+        else:
+            if self.spatial_attn_fusion in ['mul', 'cond']:
+                fused_attn = torch.log(torch.clamp(loc_attn, min=1e-6)) + attn
+            else:
+                fused_attn = loc_attn + attn
+            fused_attn = torch.softmax(fused_attn, 3)
+
+        assert torch.sum(torch.isnan(fused_attn) == 0), print(fused_attn)
+
+        output = torch.einsum('hblt,hbtv->hblv', fused_attn, v)
+        output = einops.rearrange(output, 'head b l v -> b l (head v)')
+        output = self.dropout(self.fc(output))
+        output = self.layer_norm(output + residual)
+        return output, fused_attn
+
+
+class TransformerSpatialEncoderLayer(TransformerEncoderLayer):
+    def __init__(
+            self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu",
+            spatial_multihead=True, spatial_dim=5, spatial_attn_fusion='mul'
+    ):
+        super().__init__(
+            d_model, nhead, dim_feedforward=dim_feedforward, dropout=dropout, activation=activation
+        )
+        del self.self_attn
+        self.self_attn = MultiHeadAttentionSpatial(
+            d_model, nhead, dropout=dropout,
+            spatial_multihead=spatial_multihead,
+            spatial_dim=spatial_dim,
+            spatial_attn_fusion=spatial_attn_fusion,
+        )
+
+    def forward(
+            self, tgt, tgt_pairwise_locs,
+            tgt_mask: Optional[Tensor] = None,
+            tgt_key_padding_mask: Optional[Tensor] = None,
+    ):
+        tgt2 = tgt
+        tgt2, self_attn_matrices = self.self_attn(
+            tgt2, tgt2, tgt2, tgt_pairwise_locs,
+            key_padding_mask=tgt_key_padding_mask
+        )
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        return tgt, self_attn_matrices
+
+
+def _init_weights_bert(module, std=0.02):
+    """
+        Huggingface transformer weight initialization,
+        most commonly for bert initialization
+    """
+    if isinstance(module, nn.Linear):
+        # Slightly different from the TF version which uses truncated_normal for initialization
+        # cf https://github.com/pytorch/pytorch/pull/5617
+        module.weight.data.normal_(mean=0.0, std=std)
+        if module.bias is not None:
+            module.bias.data.zero_()
+    elif isinstance(module, nn.Embedding):
+        module.weight.data.normal_(mean=0.0, std=std)
+        if module.padding_idx is not None:
+            module.weight.data[module.padding_idx].zero_()
+    elif isinstance(module, nn.LayerNorm):
+        module.bias.data.zero_()
+        module.weight.data.fill_(1.0)
+
+
+def generate_fourier_features(pos, num_bands=10, max_freq=15, concat_pos=True, sine_only=False):
+    # Input: B, N, C
+    # Output: B, N, C'
+    batch_size = pos.shape[0]
+    device = pos.device
+
+    min_freq = 1.0
+    # Nyquist frequency at the target resolution:
+    freq_bands = torch.linspace(start=min_freq, end=max_freq, steps=num_bands, device=device)
+
+    # Get frequency bands for each spatial dimension.
+    # Output is size [n, d * num_bands]
+    per_pos_features = pos.unsqueeze(-1).repeat(1, 1, 1, num_bands) * freq_bands
+    per_pos_features = torch.reshape(
+        per_pos_features, [batch_size, -1, np.prod(per_pos_features.shape[2:])])
+    if sine_only:
+        # Output is size [n, d * num_bands]
+        per_pos_features = torch.sin(np.pi * (per_pos_features))
+    else:
+        # Output is size [n, 2 * d * num_bands]
+        per_pos_features = torch.cat(
+            [torch.sin(np.pi * per_pos_features), torch.cos(np.pi * per_pos_features)], dim=-1
+        )
+    # Concatenate the raw input positions.
+    if concat_pos:
+        # Adds d bands to the encoding.
+        per_pos_features = torch.cat(
+            [pos, per_pos_features.expand(batch_size, -1, -1)], dim=-1)
+    return per_pos_features
+
+
+class OSE3D(nn.Module):
+    # Open-vocabulary, Spatial-attention, Embodied-token, 3D-agent
+    def __init__(self, use_spatial_attn=True, use_embodied_token=False, hidden_dim=256, fourier_size=84, spatial_encoder={
+        "num_attention_heads": 8,
+        "dim_feedforward": 2048,
+        "dropout": 0.1,
+        "activation": "gelu",
+        "spatial_dim": 5,
+        "spatial_multihead": True,
+        "spatial_attn_fusion": "cond",
+        "num_layers": 3,
+        "pairwise_rel_type": "center",
+        "spatial_dist_norm": True,
+        "obj_loc_encoding": "same_all",
+        "dim_loc": 6,
+        }):
+        super().__init__()
+        self.use_spatial_attn = use_spatial_attn   # spatial attention
+        self.use_embodied_token = use_embodied_token   # embodied token
+
+        # pcd backbone
+        # self.obj_encoder = PointcloudBackbone(backbone)
+        self.obj_proj = nn.Linear(768, hidden_dim)
+
+        # embodied token
+        if self.use_embodied_token:
+            self.anchor_feat = nn.Parameter(torch.zeros(1, 1, hidden_dim))
+            self.anchor_size = nn.Parameter(torch.ones(1, 1, 3))
+        self.orient_encoder = nn.Linear(fourier_size, hidden_dim)
+        self.obj_type_embed = nn.Embedding(2, hidden_dim)
+
+        # spatial encoder
+        if self.use_spatial_attn:
+            spatial_encoder_layer = TransformerSpatialEncoderLayer(
+                d_model=hidden_dim,
+                nhead=spatial_encoder['num_attention_heads'],
+                dim_feedforward=spatial_encoder['dim_feedforward'],
+                dropout=spatial_encoder['dropout'],
+                activation=spatial_encoder['activation'],
+                spatial_dim=spatial_encoder['spatial_dim'],
+                spatial_multihead=spatial_encoder['spatial_multihead'],
+                spatial_attn_fusion=spatial_encoder['spatial_attn_fusion'],
+            )
+        else:
+            spatial_encoder_layer = TransformerEncoderLayer(
+                d_model=hidden_dim,
+                nhead=spatial_encoder['num_attention_heads'],
+                dim_feedforward=spatial_encoder['dim_feedforward'],
+                dropout=spatial_encoder['dropout'],
+                activation=spatial_encoder['activation'],
+            )
+
+        self.spatial_encoder = layer_repeat(
+            spatial_encoder_layer,
+            spatial_encoder['num_layers'],
+        )
+        self.pairwise_rel_type = spatial_encoder['pairwise_rel_type']
+        self.spatial_dist_norm = spatial_encoder['spatial_dist_norm']
+        self.spatial_dim = spatial_encoder['spatial_dim']
+        self.obj_loc_encoding = spatial_encoder['obj_loc_encoding']
+
+        # location encoding
+        if self.obj_loc_encoding in ['same_0', 'same_all']:
+            num_loc_layers = 1
+        elif self.obj_loc_encoding == 'diff_all':
+            num_loc_layers = spatial_encoder['num_layers']
+
+        loc_layer = nn.Sequential(
+            nn.Linear(spatial_encoder['dim_loc'], hidden_dim),
+            nn.LayerNorm(hidden_dim),
+        )
+        self.loc_layers = layer_repeat(loc_layer, num_loc_layers)
+
+
+        # only initialize spatial encoder and loc layers
+        self.spatial_encoder.apply(_init_weights_bert)
+        self.loc_layers.apply(_init_weights_bert)
+
+        if self.use_embodied_token:
+            nn.init.normal_(self.anchor_feat, std=0.02)
+
+    @property
+    def device(self):
+        return list(self.parameters())[0].device
+
+    def forward(self, data_dict):
+        """
+        data_dict requires keys:
+            obj_fts: (B, N, P, 6), xyz + rgb
+            obj_masks: (B, N), 1 valid and 0 masked
+            obj_locs: (B, N, 6), xyz + whd
+            anchor_locs: (B, 3)
+            anchor_orientation: (B, C)
+        """
+
+        # obj_feats = self.obj_encoder(data_dict['obj_fts'])
+        obj_feats = data_dict['obj_feats']
+        obj_feats = self.obj_proj(obj_feats)
+        obj_masks = ~data_dict['obj_masks']   # flipped due to different convention of TransformerEncoder
+
+        B, N = obj_feats.shape[:2]
+        device = obj_feats.device
+
+        obj_type_ids = torch.zeros((B, N), dtype=torch.long, device=device)
+        obj_type_embeds = self.obj_type_embed(obj_type_ids)
+
+        if self.use_embodied_token:
+            # anchor feature
+            anchor_orient = data_dict['anchor_orientation'].unsqueeze(1)
+            anchor_orient_feat = self.orient_encoder(generate_fourier_features(anchor_orient))
+            anchor_feat = self.anchor_feat + anchor_orient_feat
+            anchor_mask = torch.zeros((B, 1), dtype=bool, device=device)
+
+            # anchor loc (3) + size (3)
+            anchor_loc = torch.cat(
+                [data_dict['anchor_locs'].unsqueeze(1), self.anchor_size.expand(B, -1, -1).to(device)], dim=-1
+            )
+
+            # anchor type
+            anchor_type_id = torch.ones((B, 1), dtype=torch.long, device=device)
+            anchor_type_embed = self.obj_type_embed(anchor_type_id)
+
+            # fuse anchor and objs
+            all_obj_feats = torch.cat([anchor_feat, obj_feats], dim=1)
+            all_obj_masks = torch.cat((anchor_mask, obj_masks), dim=1)
+
+            all_obj_locs = torch.cat([anchor_loc, data_dict['obj_locs']], dim=1)
+            all_obj_type_embeds = torch.cat((anchor_type_embed, obj_type_embeds), dim=1)
+
+        else:
+            all_obj_feats = obj_feats
+            all_obj_masks = obj_masks
+
+            all_obj_locs = data_dict['obj_locs']
+            all_obj_type_embeds = obj_type_embeds
+
+        all_obj_feats = all_obj_feats + all_obj_type_embeds
+
+        # call spatial encoder
+        if self.use_spatial_attn:
+            pairwise_locs = calc_pairwise_locs(
+                all_obj_locs[:, :, :3],
+                all_obj_locs[:, :, 3:],
+                pairwise_rel_type=self.pairwise_rel_type,
+                spatial_dist_norm=self.spatial_dist_norm,
+                spatial_dim=self.spatial_dim,
+            )
+
+        for i, pc_layer in enumerate(self.spatial_encoder):
+            if self.obj_loc_encoding == 'diff_all':
+                query_pos = self.loc_layers[i](all_obj_locs)
+            else:
+                query_pos = self.loc_layers[0](all_obj_locs)
+            if not (self.obj_loc_encoding == 'same_0' and i > 0):
+                all_obj_feats = all_obj_feats + query_pos
+
+            if self.use_spatial_attn:
+                all_obj_feats, _ = pc_layer(
+                    all_obj_feats, pairwise_locs,
+                    tgt_key_padding_mask=all_obj_masks
+                )
+            else:
+                all_obj_feats, _ = pc_layer(
+                    all_obj_feats,
+                    tgt_key_padding_mask=all_obj_masks
+                )
+
+        data_dict['obj_tokens'] = all_obj_feats
+        data_dict['obj_masks'] = ~all_obj_masks
+        
+        # ###feat_pth = os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_img_gt.pth')
+        # data_dict['obj_tokens'] = torch.load('assets/inputs/scene0350_00/obj_tokens.pth')
+        # data_dict['obj_masks'] = torch.load('assets/inputs/scene0350_00/obj_masks.pth')
+
+        return data_dict

leo/utils.py

@@ -0,0 +1,187 @@
+import csv
+import copy
+import torch
+import einops
+import numpy as np
+from torch import nn
+import torch.nn.functional as F
+
+
+
+def get_activation_fn(activation_type):
+    if activation_type not in ["relu", "gelu", "glu"]:
+        raise RuntimeError(f"activation function currently support relu/gelu, not {activation_type}")
+    return getattr(F, activation_type)
+
+def get_mlp_head(input_size, hidden_size, output_size, dropout=0):
+    return nn.Sequential(*[
+        nn.Linear(input_size, hidden_size),
+        nn.ReLU(),
+        nn.LayerNorm(hidden_size, eps=1e-12),
+        nn.Dropout(dropout),
+        nn.Linear(hidden_size, output_size)
+    ])
+
+def layer_repeat(module, N, share_layer=False):
+    if share_layer:
+        return nn.ModuleList([module] * N)
+    else:
+        return nn.ModuleList([copy.deepcopy(module) for _ in range(N - 1)] + [module])
+
+
+def calc_pairwise_locs(obj_centers, obj_whls, eps=1e-10, pairwise_rel_type='center', spatial_dist_norm=True,
+                       spatial_dim=5):
+    if pairwise_rel_type == 'mlp':
+        obj_locs = torch.cat([obj_centers, obj_whls], 2)
+        pairwise_locs = torch.cat(
+            [einops.repeat(obj_locs, 'b l d -> b l x d', x=obj_locs.size(1)),
+             einops.repeat(obj_locs, 'b l d -> b x l d', x=obj_locs.size(1))],
+            dim=3
+        )
+        return pairwise_locs
+
+    pairwise_locs = einops.repeat(obj_centers, 'b l d -> b l 1 d') \
+                    - einops.repeat(obj_centers, 'b l d -> b 1 l d')
+    pairwise_dists = torch.sqrt(torch.sum(pairwise_locs ** 2, 3) + eps)  # (b, l, l)
+    if spatial_dist_norm:
+        max_dists = torch.max(pairwise_dists.view(pairwise_dists.size(0), -1), dim=1)[0]
+        norm_pairwise_dists = pairwise_dists / einops.repeat(max_dists, 'b -> b 1 1')
+    else:
+        norm_pairwise_dists = pairwise_dists
+
+    if spatial_dim == 1:
+        return norm_pairwise_dists.unsqueeze(3)
+
+    pairwise_dists_2d = torch.sqrt(torch.sum(pairwise_locs[..., :2] ** 2, 3) + eps)
+    if pairwise_rel_type == 'center':
+        pairwise_locs = torch.stack(
+            [norm_pairwise_dists, pairwise_locs[..., 2] / pairwise_dists,
+             pairwise_dists_2d / pairwise_dists, pairwise_locs[..., 1] / pairwise_dists_2d,
+             pairwise_locs[..., 0] / pairwise_dists_2d],
+            dim=3
+        )
+    elif pairwise_rel_type == 'vertical_bottom':
+        bottom_centers = torch.clone(obj_centers)
+        bottom_centers[:, :, 2] -= obj_whls[:, :, 2]
+        bottom_pairwise_locs = einops.repeat(bottom_centers, 'b l d -> b l 1 d') \
+                               - einops.repeat(bottom_centers, 'b l d -> b 1 l d')
+        bottom_pairwise_dists = torch.sqrt(torch.sum(bottom_pairwise_locs ** 2, 3) + eps)  # (b, l, l)
+        bottom_pairwise_dists_2d = torch.sqrt(torch.sum(bottom_pairwise_locs[..., :2] ** 2, 3) + eps)
+        pairwise_locs = torch.stack(
+            [norm_pairwise_dists,
+             bottom_pairwise_locs[..., 2] / bottom_pairwise_dists,
+             bottom_pairwise_dists_2d / bottom_pairwise_dists,
+             pairwise_locs[..., 1] / pairwise_dists_2d,
+             pairwise_locs[..., 0] / pairwise_dists_2d],
+            dim=3
+        )
+
+    if spatial_dim == 4:
+        pairwise_locs = pairwise_locs[..., 1:]
+    return pairwise_locs
+
+def convert_pc_to_box(obj_pc):
+    xmin = np.min(obj_pc[:,0])
+    ymin = np.min(obj_pc[:,1])
+    zmin = np.min(obj_pc[:,2])
+    xmax = np.max(obj_pc[:,0])
+    ymax = np.max(obj_pc[:,1])
+    zmax = np.max(obj_pc[:,2])
+    center = [(xmin+xmax)/2, (ymin+ymax)/2, (zmin+zmax)/2]
+    box_size = [xmax-xmin, ymax-ymin, zmax-zmin]
+    return center, box_size
+
+class LabelConverter(object):
+    def __init__(self, file_path):
+        self.raw_name_to_id = {}
+        self.nyu40id_to_id = {}
+        self.nyu40_name_to_id = {}
+        self.scannet_name_to_scannet_id = {'cabinet':0, 'bed':1, 'chair':2, 'sofa':3, 'table':4,
+            'door':5, 'window':6,'bookshelf':7,'picture':8, 'counter':9, 'desk':10, 'curtain':11,
+            'refrigerator':12, 'shower curtain':13, 'toilet':14, 'sink':15, 'bathtub':16, 'others':17}  
+        self.id_to_scannetid = {}
+        self.scannet_raw_id_to_raw_name = {}
+        self.raw_name_to_scannet_raw_id = {}
+
+        with open(file_path, encoding='utf-8') as fd:
+            rd = list(csv.reader(fd, delimiter="\t", quotechar='"'))
+            for i in range(1, len(rd)):
+                raw_id = i - 1
+                scannet_raw_id = int(rd[i][0])
+                raw_name = rd[i][1]
+                nyu40_id = int(rd[i][4])
+                nyu40_name = rd[i][7]
+                self.raw_name_to_id[raw_name] = raw_id
+                self.scannet_raw_id_to_raw_name[scannet_raw_id] = raw_name
+                self.raw_name_to_scannet_raw_id[raw_name] = scannet_raw_id
+                self.nyu40id_to_id[nyu40_id] = raw_id
+                self.nyu40_name_to_id[nyu40_name] = raw_id
+                if nyu40_name not in self.scannet_name_to_scannet_id:
+                    self.id_to_scannetid[raw_id] = self.scannet_name_to_scannet_id['others']
+                else:
+                    self.id_to_scannetid[raw_id] = self.scannet_name_to_scannet_id[nyu40_name]
+
+def build_rotate_mat(split, rot_aug=True, rand_angle='axis'):
+    if rand_angle == 'random':
+        theta = np.random.rand() * np.pi * 2
+    else:
+        ROTATE_ANGLES = [0, np.pi/2, np.pi, np.pi*3/2]
+        theta_idx = np.random.randint(len(ROTATE_ANGLES))
+        theta = ROTATE_ANGLES[theta_idx]
+    if (theta is not None) and (theta != 0) and (split == 'train') and rot_aug:
+        rot_matrix = np.array([
+            [np.cos(theta), -np.sin(theta), 0],
+            [np.sin(theta), np.cos(theta), 0],
+            [0, 0, 1]
+        ], dtype=np.float32)
+    else:
+        rot_matrix = None
+    return rot_matrix
+
+def obj_processing_post(obj_pcds, rot_aug=True):
+        obj_pcds = torch.from_numpy(obj_pcds)
+        rot_matrix = build_rotate_mat('val', rot_aug)
+        if rot_matrix is not None:
+            rot_matrix = torch.from_numpy(rot_matrix.transpose())
+            obj_pcds[:, :, :3] @= rot_matrix
+        
+        xyz = obj_pcds[:, :, :3]
+        center = xyz.mean(1)
+        xyz_min = xyz.min(1).values
+        xyz_max = xyz.max(1).values
+        box_center = (xyz_min + xyz_max) / 2
+        size = xyz_max - xyz_min
+        obj_locs = torch.cat([center, size], dim=1)
+        obj_boxes = torch.cat([box_center, size], dim=1)
+
+        # centering
+        obj_pcds[:, :, :3].sub_(obj_pcds[:, :, :3].mean(1, keepdim=True))
+
+        # normalization
+        max_dist = (obj_pcds[:, :, :3]**2).sum(2).sqrt().max(1).values
+        max_dist.clamp_(min=1e-6)
+        obj_pcds[:, :, :3].div_(max_dist[:, None, None])
+        
+        return obj_pcds, obj_locs, obj_boxes, rot_matrix
+
+
+def pad_sequence(sequence_list, max_len=None, pad=0, return_mask=False):
+    lens = [x.shape[0] for x in sequence_list]
+    if max_len is None:
+        max_len = max(lens)
+        
+    shape = list(sequence_list[0].shape)
+    shape[0] = max_len
+    shape = [len(sequence_list)] + shape
+    dtype = sequence_list[0].dtype
+    device = sequence_list[0].device
+    padded_sequence = torch.ones(shape, dtype=dtype, device=device) * pad
+    for i, tensor in enumerate(sequence_list):
+        padded_sequence[i, :tensor.shape[0]] = tensor
+    padded_sequence = padded_sequence.to(dtype)
+
+    if return_mask:
+        mask = torch.arange(max_len).to(device)[None, :] >= torch.LongTensor(lens).to(device)[:, None] # True as masked.
+        return padded_sequence, mask
+    else:
+        return padded_sequence    

requirements.txt

@@ -0,0 +1,8 @@
+clip==0.2.0
+einops==0.8.0
+gradio==4.39.0
+numpy==1.24.3
+peft==0.12.0
+timm==1.0.8
+torch==2.3.1
+transformers==4.40.2