add inference files

assets/meta/scannetv2-labels.combined.tsv

@@ -0,0 +1,608 @@
+id	raw_category	category	count	nyu40id	eigen13id	nyuClass	nyu40class	eigen13class	ModelNet40	ModelNet10	ShapeNetCore55	synsetoffset	wnsynsetid	wnsynsetkey	mpcat40	mpcat40index
+1	wall	wall	8277	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+2	chair	chair	4646	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+22	books	book	1678	23	2	book	books	Books					n02870526	book.n.11	objects	39
+3	floor	floor	1553	2	5	floor	floor	Floor					n03365592	floor.n.01	floor	2
+5	door	door	1483	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+1163	object	object	1313	40	7		otherprop	Objects							objects	39
+16	window	window	1209	9	13	window	window	Window					n04587648	window.n.01	window	9
+4	table	table	1170	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+56	trash can	trash can	1090	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+13	pillow	pillow	937	18	7	pillow	pillow	Objects			pillow	3938244	n03938244	pillow.n.01	cushion	8
+15	picture	picture	862	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+41	ceiling	ceiling	806	22	3	ceiling	ceiling	Ceiling					n02990373	ceiling.n.01	ceiling	17
+26	box	box	775	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+161	doorframe	doorframe	768	8	12	door	door	Wall	door					doorframe.n.01	door	4
+19	monitor	monitor	765	40	7	monitor	otherprop	Objects	monitor	monitor	tv or monitor	3211117	n03782190	monitor.n.04	objects	39
+7	cabinet	cabinet	731	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+9	desk	desk	680	14	10	desk	desk	Table	desk	desk	table	4379243	n03179701	desk.n.01	table	5
+8	shelf	shelf	641	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+10	office chair	office chair	595	5	4	chair	chair	Chair	chair	chair	chair	3001627	n04373704	swivel_chair.n.01	chair	3
+31	towel	towel	570	27	7	towel	towel	Objects					n04459362	towel.n.01	towel	20
+6	couch	couch	502	6	9	sofa	sofa	Sofa	sofa	sofa	sofa	4256520	n04256520	sofa.n.01	sofa	10
+14	sink	sink	488	34	7	sink	sink	Objects	sink				n04223580	sink.n.01	sink	15
+48	backpack	backpack	479	40	7	backpack	otherprop	Objects					n02769748	backpack.n.01	objects	39
+28	lamp	lamp	419	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+11	bed	bed	370	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+18	bookshelf	bookshelf	360	10	6	bookshelf	bookshelf	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+71	mirror	mirror	349	19	7	mirror	mirror	Objects					n03773035	mirror.n.01	mirror	21
+21	curtain	curtain	347	16	13	curtain	curtain	Window	curtain				n03151077	curtain.n.01	curtain	12
+40	plant	plant	331	40	7	plant	otherprop	Objects	plant				n00017222	plant.n.02	plant	14
+52	whiteboard	whiteboard	327	30	7	whiteboard	whiteboard	Objects					n03211616	display_panel.n.01	board_panel	35
+96	radiator	radiator	322	39	6	radiator	otherfurniture	Furniture					n04041069	radiator.n.02	misc	40
+22	book	book	318	23	2	book	books	Books					n02870526	book.n.11	objects	39
+29	kitchen cabinet	kitchen cabinet	310	3	6	cabinet	cabinet	Furniture					n02933112	cabinet.n.01	cabinet	7
+49	toilet paper	toilet paper	291	40	7	toilet paper	otherprop	Objects					n15075141	toilet_tissue.n.01	objects	39
+29	kitchen cabinets	kitchen cabinet	289	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+23	armchair	armchair	281	5	4	chair	chair	Chair	chair	chair	chair	3001627	n02738535	armchair.n.01	chair	3
+63	shoes	shoe	272	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+24	coffee table	coffee table	258	7	10	coffee table	table	Table	table	table	table	4379243	n03063968	coffee_table.n.01	table	5
+17	toilet	toilet	256	33	7	toilet	toilet	Objects	toilet	toilet			n04446276	toilet.n.01	toilet	18
+47	bag	bag	252	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+32	clothes	clothes	248	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+46	keyboard	keyboard	246	40	7	keyboard	otherprop	Objects	keyboard		computer keyboard	3085013	n03085013	computer_keyboard.n.01	objects	39
+65	bottle	bottle	226	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+97	recycling bin	recycling bin	225	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+34	nightstand	nightstand	224	32	6	night stand	night stand	Furniture	night_stand	night_stand			n03015254	chest_of_drawers.n.01	chest_of_drawers	13
+38	stool	stool	221	40	7	stool	otherprop	Objects	stool				n04326896	stool.n.01	stool	19
+33	tv	tv	219	25	11	television	television	TV			tv or monitor	3211117	n03211117	display.n.06	tv_monitor	22
+75	file cabinet	file cabinet	217	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+36	dresser	dresser	213	17	6	dresser	dresser	Furniture	dresser	dresser			n03015254	chest_of_drawers.n.01	chest_of_drawers	13
+64	computer tower	computer tower	203	40	7	computer	otherprop	Objects					n03082979	computer.n.01	objects	39
+32	clothing	clothes	165	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+101	telephone	telephone	164	40	7	telephone	otherprop	Objects			telephone	4401088	n04401088	telephone.n.01	objects	39
+130	cup	cup	157	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+27	refrigerator	refrigerator	154	24	6	refridgerator	refridgerator	Furniture					n04070727	refrigerator.n.01	appliances	37
+44	end table	end table	147	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+131	jacket	jacket	146	40	7	jacket	otherprop	Objects					n03589791	jacket.n.01	clothes	38
+55	shower curtain	shower curtain	144	28	7	shower curtain	shower curtain	Objects	curtain				n04209239	shower_curtain.n.01	curtain	12
+42	bathtub	bathtub	144	36	7	bathtub	bathtub	Objects	bathtub	bathtub	tub	2808440	n02808440	bathtub.n.01	bathtub	25
+59	microwave	microwave	141	40	7	microwave	otherprop	Objects			microwave	3761084	n03761084	microwave.n.02	appliances	37
+159	kitchen counter	kitchen counter	140	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+74	sofa chair	sofa chair	129	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+82	paper towel dispenser	paper towel dispenser	129	40	7	paper towel dispenser	otherprop	Objects							objects	39
+1164	bathroom vanity	bathroom vanity	126	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	table	5
+93	suitcase	suitcase	118	40	7	luggage	otherprop	Objects					n02773838	bag.n.06	objects	39
+77	laptop	laptop	111	40	7	laptop	otherprop	Objects	laptop		laptop	3642806	n03642806	laptop.n.01	objects	39
+67	ottoman	ottoman	111	39	6	ottoman	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+128	shower walls	shower wall	109	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+50	printer	printer	106	40	7	printer	otherprop	Objects			printer	4004475	n04004475	printer.n.03	appliances	37
+35	counter	counter	104	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+69	board	board	100	38	7	board	otherstructure	Objects							board_panel	35
+100	soap dispenser	soap dispenser	99	40	7		otherprop	Objects					n04254120	soap_dispenser.n.01	objects	39
+62	stove	stove	95	38	7	stove	otherstructure	Objects			stove	4330267	n04330267	stove.n.02	appliances	37
+105	light	light	93	38	7	light	otherstructure	Objects					n03665366	light.n.02	lighting	28
+1165	closet wall	closet wall	90	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+165	mini fridge	mini fridge	87	24	6	refridgerator	refridgerator	Furniture					n03273913	electric_refrigerator.n.01	appliances	37
+7	cabinets	cabinet	79	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+5	doors	door	76	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+76	fan	fan	75	40	7	fan	otherprop	Objects					n03320046	fan.n.01	misc	40
+230	tissue box	tissue box	73	40	7	tissue box	otherprop	Objects					n02883344	box.n.01	objects	39
+54	blanket	blanket	72	40	7	blanket	otherprop	Objects					n02849154	blanket.n.01	objects	39
+125	bathroom stall	bathroom stall	71	38	7		otherstructure	Objects					n02873839	booth.n.02	misc	40
+72	copier	copier	70	40	7		otherprop	Objects					n03257586	duplicator.n.01	appliances	37
+68	bench	bench	66	39	6	bench	otherfurniture	Furniture	bench		bench	2828884	n02828884	bench.n.01	seating	34
+145	bar	bar	66	38	7	bar	otherstructure	Objects					n02788689	bar.n.03	misc	40
+157	soap dish	soap dish	65	40	7	soap dish	otherprop	Objects					n04254009	soap_dish.n.01	objects	39
+1166	laundry hamper	laundry hamper	65	40	7	laundry basket	otherprop	Objects							objects	39
+132	storage bin	storage bin	63	40	7	storage bin	otherprop	Objects							objects	39
+1167	bathroom stall door	bathroom stall door	62	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+232	light switch	light switch	61	38	7	light switch	otherstructure	Objects					n04372370	switch.n.01	misc	40
+134	coffee maker	coffee maker	61	40	7		otherprop	Objects					n03063338	coffee_maker.n.01	appliances	37
+51	tv stand	tv stand	61	39	6	tv stand	otherfurniture	Furniture	tv_stand				n03290653	entertainment_center.n.01	furniture	36
+250	decoration	decoration	60	40	7		otherprop	Objects					n03169390	decoration.n.01	misc	40
+1168	ceiling light	ceiling light	59	38	7	light	otherstructure	Objects					n03665366	light.n.02	lighting	28
+342	range hood	range hood	59	38	7	range hood	otherstructure	Objects	range_hood				n04053677	range_hood.n.01	misc	40
+89	blackboard	blackboard	58	38	7	blackboard	otherstructure	Objects					n02846511	blackboard.n.01	board_panel	35
+103	clock	clock	58	40	7	clock	otherprop	Objects			clock	3046257	n03046257	clock.n.01	objects	39
+99	wardrobe closet	wardrobe	54	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+95	rail	rail	53	38	7	railing	otherstructure	Objects					n04047401	railing.n.01	railing	30
+154	bulletin board	bulletin board	53	38	7	board	otherstructure	Objects					n03211616	display_panel.n.01	board_panel	35
+140	mat	mat	52	20	5	floor mat	floor mat	Floor					n03727837	mat.n.01	floor	2
+1169	trash bin	trash bin	52	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+193	ledge	ledge	51	38	7		otherstructure	Objects					n09337253	ledge.n.01	misc	40
+116	seat	seat	49	39	6	furniture	otherfurniture	Furniture					n04161981	seat.n.03	furniture	36
+202	mouse	mouse	49	40	7	mouse	otherprop	Objects					n03793489	mouse.n.04	objects	39
+73	basket	basket	48	40	7	basket	otherprop	Objects			basket	2801938	n02801938	basket.n.01	objects	39
+78	shower	shower	48	38	7		otherstructure	Objects					n04208936	shower.n.01	shower	23
+1170	dumbbell	dumbbell	48	40	7		otherprop	Objects					n03255030	dumbbell.n.01	objects	39
+79	paper	paper	46	26	7	paper	paper	Objects					n14974264	paper.n.01	objects	39
+80	person	person	46	31	7	person	person	Objects	person				n05217688	person.n.02	misc	40
+141	windowsill	windowsill	45	38	7		otherstructure	Objects					n04590263	windowsill.n.01	window	9
+57	closet	closet	45	39	6	wardrobe	otherfurniture	Furniture	wardrobe						misc	40
+102	bucket	bucket	45	40	7	bucket	otherprop	Objects					n02909870	bucket.n.01	misc	40
+261	sign	sign	44	40	7	sign	otherprop	Objects					n04217882	signboard.n.01	objects	39
+118	speaker	speaker	43	40	7	speaker	otherprop	Objects			speaker	3691459	n03691459	loudspeaker.n.01	objects	39
+136	dishwasher	dishwasher	43	38	7	dishwasher	otherstructure	Objects			dishwasher	3207941	n03207941	dishwasher.n.01	appliances	37
+98	container	container	43	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1171	stair rail	stair rail	42	38	7	banister	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+170	shower curtain rod	shower curtain rod	42	40	7		otherprop	Objects							curtain	12
+1172	tube	tube	41	40	7		otherprop	Objects							misc	40
+1173	bathroom cabinet	bathroom cabinet	39	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+79	papers	paper	39	26	7	paper	paper	Objects					n14974264	paper.n.01	objects	39
+221	storage container	storage container	39	40	7	container	otherprop	Objects							objects	39
+570	paper bag	paper bag	39	37	7	bag	bag	Objects					n04122825	sack.n.01	objects	39
+138	paper towel roll	paper towel roll	39	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+168	ball	ball	39	40	7	ball	otherprop	Objects							objects	39
+276	closet doors	closet door	38	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+106	laundry basket	laundry basket	37	40	7	laundry basket	otherprop	Objects			basket	2801938	n03050864	clothes_hamper.n.01	objects	39
+214	cart	cart	37	40	7	cart	otherprop	Objects					n03484083	handcart.n.01	shelving	31
+276	closet door	closet door	35	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+323	dish rack	dish rack	35	40	7	dish rack	otherprop	Objects					n03207630	dish_rack.n.01	objects	39
+58	stairs	stairs	35	38	7	stairs	otherstructure	Objects					n04298308	stairway.n.01	stairs	16
+86	blinds	blinds	35	13	13	blinds	blinds	Window					n02851099	blind.n.03	blinds	32
+2	stack of chairs	chair	35	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+399	purse	purse	34	40	7	purse	otherprop	Objects					n02774152	bag.n.04	objects	39
+121	bicycle	bicycle	33	40	7	bicycle	otherprop	Objects			bicycle	2834778	n02834778	bicycle.n.01	objects	39
+185	tray	tray	32	40	7	tray	otherprop	Objects					n04476259	tray.n.01	objects	39
+300	plunger	plunger	30	40	7		otherprop	Objects					n03970156	plunger.n.03	objects	39
+180	paper cutter	paper cutter	30	40	7	paper cutter	otherprop	Objects					n03886940	paper_cutter.n.01	objects	39
+163	toilet paper dispenser	toilet paper dispenser	29	40	7		otherprop	Objects							objects	39
+26	boxes	box	29	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+66	bin	bin	28	40	7	bin	otherprop	Objects					n02839910	bin.n.01	objects	39
+208	toilet seat cover dispenser	toilet seat cover dispenser	28	40	7		otherprop	Objects							objects	39
+112	guitar	guitar	28	40	7	guitar	otherprop	Objects	guitar		guitar	3467517	n03467517	guitar.n.01	objects	39
+540	mailboxes	mailbox	28	29	7	box	box	Objects			mailbox	3710193	n03710193	mailbox.n.01	misc	40
+395	handicap bar	handicap bar	27	38	7	bar	otherstructure	Objects							misc	40
+166	fire extinguisher	fire extinguisher	27	40	7	fire extinguisher	otherprop	Objects					n03345837	fire_extinguisher.n.01	misc	40
+122	ladder	ladder	27	39	6	ladder	otherfurniture	Furniture	stairs				n03632277	ladder.n.01	stairs	16
+120	column	column	26	38	7	column	otherstructure	Objects					n03074380	column.n.06	column	24
+107	pipe	pipe	25	40	7	pipe	otherprop	Objects					n03944672	pipe.n.02	misc	40
+283	vacuum cleaner	vacuum cleaner	25	40	7		otherprop	Objects					n04517823	vacuum.n.04	objects	39
+88	plate	plate	24	40	7	plate	otherprop	Objects					n03959485	plate.n.04	objects	39
+90	piano	piano	24	39	6	piano	otherfurniture	Furniture	piano		piano	3928116	n03928116	piano.n.01	furniture	36
+177	water cooler	water cooler	24	39	6	water cooler	otherfurniture	Furniture					n04559166	water_cooler.n.01	misc	40
+1174	cd case	cd case	24	40	7		otherprop	Objects							objects	39
+562	bowl	bowl	24	40	7	bowl	otherprop	Objects	bowl		bowl	2880940	n02880940	bowl.n.03	objects	39
+1175	closet rod	closet rod	24	40	7		otherprop	Objects					n04100174	rod.n.01	misc	40
+1156	bathroom counter	bathroom counter	24	12	6	counter	counter	Furniture	table	table	table	4379243	n03116530	counter.n.01	counter	26
+84	oven	oven	23	38	7	oven	otherstructure	Objects					n03862676	oven.n.01	appliances	37
+104	stand	stand	23	39	6	stand	otherfurniture	Furniture	table	table	table	4379243	n04301000	stand.n.04	table	5
+229	scale	scale	23	40	7	scale	otherprop	Objects					n04141975	scale.n.07	objects	39
+70	washing machine	washing machine	23	39	6	washing machine	otherfurniture	Furniture			washing_machine	4554684	n04554684	washer.n.03	appliances	37
+325	broom	broom	22	40	7	broom	otherprop	Objects					n02906734	broom.n.01	objects	39
+169	hat	hat	22	40	7	hat	otherprop	Objects					n03497657	hat.n.01	clothes	38
+128	shower wall	shower wall	22	1	12	wall	wall	Wall					n04208936	shower.n.01	wall	1
+331	guitar case	guitar case	21	40	7	guitar case	otherprop	Objects							objects	39
+87	rack	rack	21	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+488	water pitcher	water pitcher	21	40	7	pitcher	otherprop	Objects					n03950228	pitcher.n.02	objects	39
+776	laundry detergent	laundry detergent	21	40	7		otherprop	Objects							objects	39
+370	hair dryer	hair dryer	21	40	7	hair dryer	otherprop	Objects					n03483316	hand_blower.n.01	objects	39
+191	pillar	pillar	21	38	7	column	otherstructure	Objects					n03073977	column.n.07	column	24
+748	divider	divider	20	40	7		otherprop	Objects							wall	1
+242	power outlet	power outlet	19	40	7		otherprop	Objects							misc	40
+45	dining table	dining table	19	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+417	shower floor	shower floor	19	2	5	floor	floor	Floor					n04208936	shower.n.01	floor	2
+70	washing machines	washing machine	19	39	6	washing machine	otherfurniture	Furniture			washing_machine	4554684	n04554684	washer.n.03	appliances	37
+188	shower door	shower door	19	8	12	door	door	Wall	door				n04208936	shower.n.01	door	4
+1176	coffee kettle	coffee kettle	18	40	7	pot	otherprop	Objects					n03612814	kettle.n.01	objects	39
+1177	wardrobe cabinet	wardrobe 	18	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+1178	structure	structure	18	38	7		otherstructure	Objects							misc	40
+18	bookshelves	bookshelf	17	10	6	bookshelf	bookshelf	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+110	clothes dryer	clothes dryer	17	39	6		otherfurniture	Furniture					n03251766	dryer.n.01	appliances	37
+148	toaster	toaster	17	40	7	toaster	otherprop	Objects					n04442312	toaster.n.02	appliances	37
+63	shoe	shoe	17	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+155	ironing board	ironing board	16	39	6	ironing board	otherfurniture	Furniture					n03586090	ironing_board.n.01	objects	39
+572	alarm clock	alarm clock	16	40	7	alarm clock	otherprop	Objects			clock	3046257	n02694662	alarm_clock.n.01	objects	39
+1179	shower head	shower head	15	38	7		otherstructure	Objects							shower	23
+28	lamp base	lamp	15	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+392	water bottle	water bottle	15	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n04557648	water_bottle.n.01	objects	39
+1180	keyboard piano	keyboard piano	15	39	6	piano	otherfurniture	Furniture	piano		piano	3928116	n03928116	piano.n.01	furniture	36
+609	projector screen	projector screen	15	38	7	projector screen	otherstructure	Objects							misc	40
+1181	case of water bottles	case of water bottles	15	40	7		otherprop	Objects							objects	39
+195	toaster oven	toaster oven	14	40	7	toaster oven	otherprop	Objects					n04442441	toaster_oven.n.01	appliances	37
+581	music stand	music stand	14	39	6	music stand	otherfurniture	Furniture					n03801760	music_stand.n.01	furniture	36
+58	staircase	stairs	14	38	7	stairs	otherstructure	Objects					n04298308	stairway.n.01	stairs	16
+1182	coat rack	coat rack	14	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	3
+1183	storage organizer	storage organizer	14	40	7		otherprop	Objects							shelving	3
+139	machine	machine	14	40	7	machine	otherprop	Objects					n03699975	machine.n.01	appliances	37
+1184	folded chair	folded chair	14	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1185	fire alarm	fire alarm	14	40	7		otherprop	Objects					n03343737	fire_alarm.n.02	misc	40
+156	fireplace	fireplace	13	38	7	fireplace	otherstructure	Objects					n03346455	fireplace.n.01	fireplace	27
+408	vent	vent	13	40	7		otherprop	Objects					n04526241	vent.n.01	misc	40
+213	furniture	furniture	13	39	6	furniture	otherfurniture	Furniture					n03405725	furniture.n.01	furniture	36
+1186	power strip	power strip	13	40	7		otherprop	Objects							objects	39
+1187	calendar	calendar	13	40	7		otherprop	Objects							objects	39
+1188	poster	poster	13	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+115	toilet paper holder	toilet paper holder	13	40	7	toilet paper holder	otherprop	Objects							objects	39
+1189	potted plant	potted plant	12	40	7	plant	otherprop	Objects	plant				n00017222	plant.n.02	plant	14
+304	stuffed animal	stuffed animal	12	40	7	stuffed animal	otherprop	Objects					n04399382	teddy.n.01	objects	39
+1190	luggage	luggage	12	40	7	luggage	otherprop	Objects					n02774630	baggage.n.01	objects	39
+21	curtains	curtain	12	16	13	curtain	curtain	Window	curtain				n03151077	curtain.n.01	curtain	12
+312	headphones	headphones	12	40	7		otherprop	Objects					n03261776	earphone.n.01	objects	39
+233	crate	crate	12	39	6	crate	otherfurniture	Furniture					n03127925	crate.n.01	objects	39
+286	candle	candle	12	40	7	candle	otherprop	Objects	lamp				n02948072	candle.n.01	objects	39
+264	projector	projector	12	40	7	projector	otherprop	Objects					n04009552	projector.n.02	objects	39
+110	clothes dryers	clothes dryer	12	39	6		otherfurniture	Furniture					n03251766	dryer.n.01	appliances	37
+1191	mattress	mattress	12	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+356	dustpan	dustpan	12	40	7		otherprop	Objects					n03259009	dustpan.n.02	objects	39
+25	drawer	drawer	11	39	6	drawer	otherfurniture	Furniture					n03233905	drawer.n.01	furniture	36
+750	rod	rod	11	40	7		otherprop	Objects			pistol	3948459	n03427202	gat.n.01	misc	40
+269	globe	globe	11	40	7	globe	otherprop	Objects							objects	39
+307	footrest	footrest	11	39	6	foot rest	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+410	piano bench	piano bench	11	39	6	piano bench	otherfurniture	Furniture	bench		bench	2828884	n02828884	bench.n.01	seating	34
+730	breakfast bar	breakfast bar	11	38	7	bar	otherstructure	Objects							counter	26
+216	step stool	step stool	11	40	7	step stool	otherprop	Objects	stool				n04315713	step_stool.n.01	stool	19
+1192	hand rail	hand rail	11	38	7	railing	otherstructure	Objects							railing	30
+119	vending machine	vending machine	11	40	7	machine	otherprop	Objects					n04525305	vending_machine.n.01	appliances	37
+682	ceiling fan	ceiling fan	11	40	7	fan	otherprop	Objects					n03320046	fan.n.01	misc	40
+434	swiffer	swiffer	11	40	7		otherprop	Objects							objects	39
+126	foosball table	foosball table	11	39	6	foosball table	otherfurniture	Furniture	table	table	table	4379243	n04379243	table.n.02	table	5
+919	jar	jar	11	40	7	jar	otherprop	Objects			jar	3593526	n03593526	jar.n.01	objects	39
+85	footstool	footstool	11	39	6	ottoman	otherfurniture	Furniture	stool				n03380724	footstool.n.01	stool	19
+1193	folded table	folded table	10	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+108	round table	round table	10	7	10	table	table	Table	table	table	table	4379243	n04114554	round_table.n.02	table	5
+135	hamper	hamper	10	40	7	basket	otherprop	Objects			basket	2801938	n03482405	hamper.n.02	objects	39
+1194	poster tube	poster tube	10	40	7		otherprop	Objects							objects	39
+432	case	case	10	40	7	case	otherprop	Objects							objects	39
+53	carpet	carpet	10	40	7	rug	otherprop	Objects					n04118021	rug.n.01	floor	2
+1195	thermostat	thermostat	10	40	7		otherprop	Objects					n04422875	thermostat.n.01	misc	40
+111	coat	coat	10	40	7	jacket	otherprop	Objects					n03057021	coat.n.01	clothes	38
+305	water fountain	water fountain	10	38	7	water fountain	otherstructure	Objects					n03241335	drinking_fountain.n.01	misc	40
+1125	smoke detector	smoke detector	10	40	7		otherprop	Objects							misc	40
+13	pillows	pillow	9	18	7	pillow	pillow	Objects			pillow	3938244	n03938244	pillow.n.01	cushion	8
+1196	flip flops	flip flops	9	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+1197	cloth	cloth	9	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+1198	banner	banner	9	40	7		otherprop	Objects					n02788021	banner.n.01	misc	40
+1199	clothes hanger	clothes hanger	9	40	7		otherprop	Objects					n03057920	coat_hanger.n.01	objects	39
+1200	whiteboard eraser	whiteboard eraser	9	40	7		otherprop	Objects							objects	39
+378	iron	iron	9	40	7		otherprop	Objects					n03584829	iron.n.04	objects	39
+591	instrument case	instrument case	9	40	7	case	otherprop	Objects							objects	39
+49	toilet paper rolls	toilet paper	9	40	7	toilet paper	otherprop	Objects					n15075141	toilet_tissue.n.01	objects	39
+92	soap	soap	9	40	7	soap	otherprop	Objects					n04253437	soap.n.01	objects	39
+1098	block	block	9	40	7		otherprop	Objects							misc	40
+291	wall hanging	wall hanging	8	40	7		otherprop	Objects					n03491178	hanging.n.01	picture	6
+1063	kitchen island	kitchen island	8	38	7	kitchen island	otherstructure	Objects					n03620600	kitchen_island.n.01	counter	26
+107	pipes	pipe	8	38	7		otherstructure	Objects							misc	40
+1135	toothbrush	toothbrush	8	40	7	toothbrush	otherprop	Objects					n04453156	toothbrush.n.01	objects	39
+189	shirt	shirt	8	40	7		otherprop	Objects					n04197391	shirt.n.01	clothes	38
+245	cutting board	cutting board	8	40	7	cutting board	otherprop	Objects					n03025513	chopping_board.n.01	objects	39
+194	vase	vase	8	40	7	vase	otherprop	Objects	vase		jar	3593526	n04522168	vase.n.01	objects	39
+1201	shower control valve	shower control valve	8	38	7		otherstructure	Objects					n04208936	shower.n.01	shower	23
+386	exercise machine	exercise machine	8	40	7	machine	otherprop	Objects							gym_equipment	33
+1202	compost bin	compost bin	8	39	6	garbage bin	otherfurniture	Furniture			trash_bin	2747177	n02747177	ashcan.n.01	objects	39
+857	shorts	shorts	8	40	7	shorts	otherprop	Objects							clothes	38
+452	tire	tire	8	40	7		otherprop	Objects					n04440749	tire.n.01	objects	39
+1203	teddy bear	teddy bear	7	40	7	stuffed animal	otherprop	Objects					n04399382	teddy.n.01	objects	39
+346	bathrobe	bathrobe	7	40	7		otherprop	Objects					n02807616	bathrobe.n.01	clothes	38
+152	handrail	handrail	7	38	7	railing	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+83	faucet	faucet	7	40	7	faucet	otherprop	Objects			faucet	3325088	n03325088	faucet.n.01	misc	40
+1204	pantry wall	pantry wall	7	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+726	thermos	thermos	7	40	7	flask	otherprop	Objects	bottle		bottle	2876657	n04422727	thermos.n.01	objects	39
+61	rug	rug	7	40	7	rug	otherprop	Objects					n04118021	rug.n.01	floor	2
+39	couch cushions	cushion	7	18	7	pillow	pillow	Objects					n03151500	cushion.n.03	cushion	8
+1117	tripod	tripod	7	39	6	stand	otherfurniture	Furniture					n04485082	tripod.n.01	objects	39
+540	mailbox	mailbox	7	29	7	box	box	Objects			mailbox	3710193	n03710193	mailbox.n.01	misc	40
+1205	tupperware	tupperware	7	40	7		otherprop	Objects							objects	39
+415	shoe rack	shoe rack	7	40	7	shoe rack	otherprop	Objects							shelving	31
+31	towels	towel	6	27	7	towel	towel	Objects					n04459362	towel.n.01	towel	20
+1206	beer bottles	beer bottle	6	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+153	treadmill	treadmill	6	39	6	treadmill	otherfurniture	Furniture					n04477387	treadmill.n.01	gym_equipment	33
+1207	salt	salt	6	40	7		otherprop	Objects							objects	39
+129	chest	chest	6	39	6	chest	otherfurniture	Furniture	dresser	dresser					chest_of_drawers	13
+220	dispenser	dispenser	6	40	7		otherprop	Objects					n03210683	dispenser.n.01	objects	39
+1208	mirror doors	mirror door	6	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+231	remote	remote	6	40	7		otherprop	Objects			remote_control	4074963	n04074963	remote_control.n.01	objects	39
+1209	folded ladder	folded ladder	6	39	6	ladder	otherfurniture	Furniture	stairs				n03632277	ladder.n.01	misc	40
+39	cushion	cushion	6	18	7	pillow	pillow	Objects					n03151500	cushion.n.03	cushion	8
+1210	carton	carton	6	40	7		otherprop	Objects							objects	39
+117	step	step	6	38	7		otherstructure	Objects					n04314914	step.n.04	misc	40
+822	drying rack	drying rack	6	39	6	drying rack	otherfurniture	Furniture							shelving	31
+238	slippers	slipper	6	40	7	shoe	otherprop	Objects					n04241394	slipper.n.01	clothes	38
+143	pool table	pool table	6	39	6	pool table	otherfurniture	Furniture	table	table	table	4379243	n03982430	pool_table.n.01	table	5
+1211	soda stream	soda stream	6	40	7		otherprop	Objects							objects	39
+228	toilet brush	toilet brush	6	40	7	toilet brush	otherprop	Objects							objects	39
+494	loft bed	loft bed	6	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+226	cooking pot	cooking pot	6	40	7	pot	otherprop	Objects							objects	39
+91	heater	heater	6	39	6	heater	otherfurniture	Furniture					n03508101	heater.n.01	misc	40
+1072	messenger bag	messenger bag	6	37	7	bag	bag	Objects							objects	39
+435	stapler	stapler	6	40	7	stapler	otherprop	Objects					n04303497	stapler.n.01	objects	39
+1165	closet walls	closet wall	5	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+345	scanner	scanner	5	40	7		otherprop	Objects							appliances	37
+893	elliptical machine	elliptical machine	5	40	7	machine	otherprop	Objects							gym_equipment	33
+621	kettle	kettle	5	40	7	pot	otherprop	Objects					n03612814	kettle.n.01	objects	39
+1212	metronome	metronome	5	40	7		otherprop	Objects					n03757604	metronome.n.01	objects	39
+297	dumbell	dumbell	5	40	7		otherprop	Objects							objects	39
+1213	music book	music book	5	23	2	book	books	Books					n02870526	book.n.11	objects	39
+1214	rice cooker	rice cooker	5	40	7		otherprop	Objects							objects	39
+1215	dart board	dart board	5	38	7	board	otherstructure	Objects					n03162940	dartboard.n.01	objects	39
+529	sewing machine	sewing machine	5	40	7	sewing machine	otherprop	Objects					n04179913	sewing_machine.n.01	objects	39
+1216	grab bar	grab bar	5	38	7	railing	otherstructure	Objects							railing	30
+1217	flowerpot	flowerpot	5	40	7	vase	otherprop	Objects	vase		jar	3593526	n04522168	vase.n.01	objects	39
+1218	painting	painting	5	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+1219	railing	railing	5	38	7	railing	otherstructure	Objects					n04047401	railing.n.01	railing	30
+1220	stair	stair	5	38	7	stairs	otherstructure	Objects	stairs				n04314914	step.n.04	stairs	16
+525	toolbox	toolbox	5	39	6	chest	otherfurniture	Furniture					n04452615	toolbox.n.01	objects	39
+204	nerf gun	nerf gun	5	40	7		otherprop	Objects							objects	39
+693	binders	binder	5	40	7	binder	otherprop	Objects							objects	39
+179	desk lamp	desk lamp	5	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+1221	quadcopter	quadcopter	5	40	7		otherprop	Objects							objects	39
+1222	pitcher	pitcher	5	40	7	pitcher	otherprop	Objects					n03950228	pitcher.n.02	objects	39
+1223	hanging	hanging	5	40	7		otherprop	Objects							misc	40
+1224	mail	mail	5	40	7		otherprop	Objects							misc	40
+1225	closet ceiling	closet ceiling	5	22	3	ceiling	ceiling	Ceiling					n02990373	ceiling.n.01	ceiling	17
+1226	hoverboard	hoverboard	5	40	7		otherprop	Objects							objects	39
+1227	beanbag chair	beanbag chair	5	39	6	bean bag	otherfurniture	Furniture					n02816656	beanbag.n.01	chair	3
+571	water heater	water heater	5	40	7	water heater	otherprop	Objects					n04560113	water_heater.n.01	misc	40
+1228	spray bottle	spray bottle	5	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+556	rope	rope	5	40	7	rope	otherprop	Objects					n04108268	rope.n.01	objects	39
+280	plastic container	plastic container	5	40	7	container	otherprop	Objects							objects	39
+1229	soap bottle	soap bottle	5	40	7	soap	otherprop	Objects							objects	39
+1230	ikea bag	ikea bag	4	37	7	bag	bag	Objects				2773838	n02773838	bag.n.06	objects	39
+1231	sleeping bag	sleeping bag	4	40	7		otherprop	Objects					n04235860	sleeping_bag.n.01	objects	39
+1232	duffel bag	duffel bag	4	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+746	frying pan	frying pan	4	40	7	frying pan	otherprop	Objects					n03400231	frying_pan.n.01	objects	39
+1233	oven mitt	oven mitt	4	40	7		otherprop	Objects							objects	39
+1234	pot	pot	4	40	7	pot	otherprop	Objects					n04235860	sleeping_bag.n.01	objects	39
+144	hand dryer	hand dryer	4	40	7		otherprop	Objects							objects	39
+282	dollhouse	dollhouse	4	39	6	doll house	otherfurniture	Furniture					n03219483	dollhouse.n.01	objects	39
+167	shampoo bottle	shampoo bottle	4	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1235	hair brush	hair brush	4	40	7		otherprop	Objects					n02908217	brush.n.02	objects	39
+1236	tennis racket	tennis racket	4	40	7		otherprop	Objects					n04409806	tennis_racket.n.01	objects	39
+1237	display case	display case	4	40	7	case	otherprop	Objects							objects	39
+234	ping pong table	ping pong table	4	39	6	ping pong table	otherfurniture	Furniture	table	table	table	4379243	n04379243	table.n.02	table	5
+563	boiler	boiler	4	40	7		otherprop	Objects							misc	40
+1238	bag of coffee beans	bag of coffee beans	4	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+1239	bananas	banana	4	40	7		otherprop	Objects					n00021265	food.n.01	objects	39
+1240	carseat	carseat	4	40	7		otherprop	Objects							misc	40
+366	helmet	helmet	4	40	7		otherprop	Objects			helmet	3513137	n03513137	helmet.n.02	clothes	38
+816	umbrella	umbrella	4	40	7	umbrella	otherprop	Objects					n04507155	umbrella.n.01	objects	39
+1241	coffee box	coffee box	4	40	7		otherprop	Objects							objects	39
+719	envelope	envelope	4	40	7	envelope	otherprop	Objects					n03291819	envelope.n.01	objects	39
+284	wet floor sign	wet floor sign	4	40	7	sign	otherprop	Objects							misc	40
+1242	clothing rack	clothing rack	4	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+247	controller	controller	4	40	7		otherprop	Objects					n03096960	control.n.09	objects	39
+1243	bath walls	bathroom wall	4	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+1244	podium	podium	4	39	6		otherfurniture	Furniture					n03159640	dais.n.01	furniture	36
+1245	storage box	storage box	4	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1246	dolly	dolly	4	40	7		otherprop	Objects							misc	40
+1247	shampoo	shampoo	3	40	7		otherprop	Objects					n04183516	shampoo.n.01	objects	39
+592	paper tray	paper tray	3	40	7	paper tray	otherprop	Objects							objects	39
+385	cabinet door	cabinet door	3	8	12	door	door	Wall	door						door	4
+1248	changing station	changing station	3	40	7		otherprop	Objects							misc	40
+1249	poster printer	poster printer	3	40	7	printer	otherprop	Objects			printer	4004475	n04004475	printer.n.03	appliances	37
+133	screen	screen	3	40	7		otherprop	Objects					n03151077	curtain.n.01	curtain	12
+301	soap bar	soap bar	3	38	7	bar	otherstructure	Objects							objects	39
+1250	crutches	crutches	3	40	7		otherprop	Objects					n03141823	crutch.n.01	objects	39
+379	studio light	studio light	3	38	7	light	otherstructure	Objects							lighting	28
+130	stack of cups	cup	3	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+1251	toilet flush button	toilet flush button	3	40	7		otherprop	Objects							objects	39
+450	trunk	trunk	3	40	7		otherprop	Objects							misc	40
+1252	grocery bag	grocery bag	3	37	7	bag	bag	Objects			suitcase	2773838	n03461288	grocery_bag.n.01	objects	39
+316	plastic bin	plastic bin	3	40	7	bin	otherprop	Objects							objects	39
+1253	pizza box	pizza box	3	29	7	box	box	Objects							objects	39
+385	cabinet doors	cabinet door	3	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	door	4
+1254	legs	legs	3	31	7	person	person	Objects	person				n05217688	person.n.02	misc	40
+461	car	car	3	40	7	car	otherprop	Objects	car		car	2958343	n02958343	car.n.01	misc	40
+1255	shaving cream	shaving cream	3	40	7		otherprop	Objects					n04186051	shaving_cream.n.01	objects	39
+1256	luggage stand	luggage stand	3	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+599	shredder	shredder	3	40	7		otherprop	Objects					n04210120	shredder.n.01	objects	39
+281	statue	statue	3	40	7	sculpture	otherprop	Objects					n04306847	statue.n.01	misc	40
+1257	urinal	urinal	3	33	7	toilet	toilet	Objects	toilet	toilet			n04515991	urinal.n.01	toilet	18
+1258	hose	hose	3	40	7		otherprop	Objects					n03539875	hose.n.03	misc	40
+1259	bike pump	bike pump	3	40	7		otherprop	Objects							objects	39
+319	coatrack	coatrack	3	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+1260	bear	bear	3	40	7		otherprop	Objects							objects	39
+28	wall lamp	lamp	3	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+1261	humidifier	humidifier	3	40	7		otherprop	Objects							objects	39
+546	toothpaste	toothpaste	3	40	7	toothpaste	otherprop	Objects							objects	39
+1262	mouthwash bottle	mouthwash bottle	3	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1263	poster cutter	poster cutter	3	40	7		otherprop	Objects							objects	39
+1264	golf bag	golf bag	3	37	7	bag	bag	Objects			suitcase	2773838	n03445617	golf_bag.n.01	objects	39
+1265	food container	food container	3	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1266	camera	camera	3	40	7		otherprop	Objects							objects	39
+28	table lamp	lamp	3	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n04380533	table_lamp.n.01	lighting	28
+1267	yoga mat	yoga mat	3	20	5	floor mat	floor mat	Floor					n03727837	mat.n.01	floor	2
+1268	card	card	3	40	7		otherprop	Objects							objects	39
+1269	mug	mug	3	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+188	shower doors	shower door	3	38	7		otherstructure	Objects					n04208936	shower.n.01	door	4
+689	cardboard	cardboard	3	40	7		otherprop	Objects							objects	39
+1270	rack stand	rack stand	3	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+1271	boxes of paper	boxes of paper	3	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1272	flag	flag	3	40	7		otherprop	Objects							misc	40
+354	futon	futon	3	39	6	mattress	otherfurniture	Furniture					n03408444	futon.n.01	sofa	10
+339	magazine	magazine	3	40	7	magazine	otherprop	Objects					n06595351	magazine.n.01	objects	39
+1009	exit sign	exit sign	3	40	7	exit sign	otherprop	Objects							misc	40
+1273	rolled poster	rolled poster	3	40	7		otherprop	Objects							objects	39
+1274	wheel	wheel	3	40	7		otherprop	Objects							objects	39
+15	pictures	picture	3	11	8	picture	picture	Picture					n03931044	picture.n.01	picture	6
+1275	blackboard eraser	blackboard eraser	3	40	7	eraser	otherprop	Objects					n03294833	eraser.n.01	objects	39
+361	organizer	organizer	3	40	7		otherprop	Objects					n03918737	personal_digital_assistant.n.01	objects	39
+1276	doll	doll	3	40	7	toy	otherprop	Objects					n03219135	doll.n.01	objects	39
+326	book rack	book rack	3	39	6	bookrack	otherfurniture	Furniture							objects	39
+1277	laundry bag	laundry bag	3	40	7	laundry basket	otherprop	Objects			basket	2801938	n03050864	clothes_hamper.n.01	objects	39
+1278	sponge	sponge	3	40	7		otherprop	Objects					n01906749	sponge.n.04	objects	39
+116	seating	seat	3	39	6	furniture	otherfurniture	Furniture					n04161981	seat.n.03	furniture	36
+1184	folded chairs	folded chair	2	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1279	lotion bottle	lotion bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+212	can	can	2	40	7	can	otherprop	Objects			can	2946921	n02946921	can.n.01	objects	39
+1280	lunch box	lunch box	2	40	7		otherprop	Objects							objects	39
+1281	food display	food display	2	40	7		otherprop	Objects							misc	40
+794	storage shelf	storage shelf	2	40	7		otherprop	Objects							shelving	31
+1282	sliding wood door	sliding wood door	2	40	7		otherprop	Objects							door	4
+955	pants	pants	2	40	7		otherprop	Objects					n04489008	trouser.n.01	clothes	38
+387	wood	wood	2	40	7		otherprop	Objects							misc	40
+69	boards	board	2	38	7	board	otherstructure	Objects							board_panel	35
+65	bottles	bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+523	washcloth	washcloth	2	40	7		otherprop	Objects					n04554523	washcloth.n.01	towel	20
+389	workbench	workbench	2	39	6	bench	otherfurniture	Furniture	bench		table	4379243	n04600486	workbench.n.01	table	5
+29	open kitchen cabinet	kitchen cabinet	2	3	6	cabinet	cabinet	Furniture					n02933112	cabinet.n.01	cabinet	7
+1283	organizer shelf	organizer shelf	2	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+146	frame	frame	2	38	7		otherstructure	Objects							misc	40
+130	cups	cup	2	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39
+372	exercise ball	exercise ball	2	40	7	ball	otherprop	Objects					n04285146	sports_equipment.n.01	gym_equipment	33
+289	easel	easel	2	39	6	stand	otherfurniture	Furniture					n03262809	easel.n.01	furniture	36
+440	garbage bag	garbage bag	2	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+321	roomba	roomba	2	40	7		otherprop	Objects							objects	39
+976	garage door	garage door	2	38	7	garage door	otherstructure	Objects	door						door	4
+1256	luggage rack	luggage stand	2	39	6	stand	otherfurniture	Furniture					n04038440		shelving	31
+1284	bike lock	bike lock	2	40	7		otherprop	Objects							objects	39
+1285	briefcase	briefcase	2	40	7		otherprop	Objects					n02900705	briefcase.n.01	objects	39
+357	hand towel	hand towel	2	27	7	towel	towel	Objects					n03490006	hand_towel.n.01	towel	20
+1286	bath products	bath product	2	40	7		otherprop	Objects							objects	39
+1287	star	star	2	40	7		otherprop	Objects					n09444783	star.n.03	misc	40
+365	map	map	2	40	7	map	otherprop	Objects					n03720163	map.n.01	misc	40
+1288	coffee bean bag	coffee bean bag	2	37	7	bag	bag	Objects			suitcase	2773838	n02773838	bag.n.06	objects	39
+81	headboard	headboard	2	39	6	headboard	otherfurniture	Furniture					n03502200	headboard.n.01	bed	11
+1289	ipad	ipad	2	40	7		otherprop	Objects							objects	39
+1290	display rack	display rack	2	39	6	stand	otherfurniture	Furniture					n04038440	rack.n.05	shelving	31
+948	traffic cone	traffic cone	2	40	7	cone	otherprop	Objects	cone						objects	39
+174	toiletry	toiletry	2	40	7		otherprop	Objects					n04447443	toiletry.n.01	objects	39
+1028	canopy	canopy	2	40	7		otherprop	Objects							misc	40
+1291	massage chair	massage chair	2	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1292	paper organizer	paper organizer	2	40	7		otherprop	Objects							objects	39
+1005	barricade	barricade	2	40	7		otherprop	Objects							misc	40
+235	platform	platform	2	38	7		otherstructure	Objects							misc	40
+1293	cap	cap	2	40	7	hat	otherprop	Objects					n03497657	hat.n.01	clothes	38
+1294	dumbbell plates	dumbbell plates	2	40	7		otherprop	Objects							objects	39
+1295	elevator	elevator	2	38	7		otherstructure	Objects							misc	40
+1296	cooking pan	cooking pan	2	40	7	pan	otherprop	Objects					n03880531	pan.n.01	objects	39
+1297	trash bag	trash bag	2	37	7	bag	bag	Objects							objects	39
+1298	santa	santa	2	40	7		otherprop	Objects							misc	40
+1299	jewelry box	jewelry box	2	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1300	boat	boat	2	40	7		otherprop	Objects							misc	40
+1301	sock	sock	2	21	7	clothes	clothes	Objects					n04254777	sock.n.01	clothes	38
+1051	kinect	kinect	2	40	7	kinect	otherprop	Objects							objects	39
+566	crib	crib	2	39	6	crib	otherfurniture	Furniture							furniture	36
+1302	plastic storage bin	plastic storage bin	2	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1062	cooler	cooler	2	24	6	refridgerator	refridgerator	Furniture					n03102654	cooler.n.01	appliances	37
+1303	kitchen apron	kitchen apron	2	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+1304	dishwashing soap bottle	dishwashing soap bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1305	xbox controller	xbox controller	2	40	7		otherprop	Objects							objects	39
+1306	banana holder	banana holder	2	40	7		otherprop	Objects							objects	39
+298	ping pong paddle	ping pong paddle	2	40	7		otherprop	Objects							table	5
+1307	airplane	airplane	2	40	7		otherprop	Objects							misc	40
+1308	conditioner bottle	conditioner bottle	2	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+1309	tea kettle	tea kettle	2	40	7	tea kettle	otherprop	Objects					n04397768	teakettle.n.01	objects	39
+43	bedframe	bedframe	2	39	6		otherfurniture	Furniture					n02822579	bedstead.n.01	bed	11
+1310	wood beam	wood beam	2	38	7		otherstructure	Objects							beam	29
+593	toilet paper package	toilet paper package	2	40	7		otherprop	Objects							objects	39
+1311	wall mounted coat rack	wall mounted coat rack	2	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+1312	film light	film light	2	40	7		otherprop	Objects							lighting	28
+749	ceiling lamp	ceiling lamp	1	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+623	chain	chain	1	40	7		otherprop	Objects							chair	3
+1313	sofa	sofa	1	6	9	sofa	sofa	Sofa	sofa	sofa	sofa	4256520	n04256520	sofa.n.01	sofa	10
+99	closet wardrobe	wardrobe	1	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+265	sweater	sweater	1	40	7		otherprop	Objects					n04370048	sweater.n.01	clothes	38
+1314	kitchen mixer	kitchen mixer	1	40	7		otherprop	Objects							appliances	37
+99	wardrobe	wardrobe	1	39	6	wardrobe	otherfurniture	Furniture	wardrobe				n04550184	wardrobe.n.01	furniture	36
+1315	water softener	water softener	1	40	7		otherprop	Objects							misc	40
+448	banister	banister	1	38	7	banister	otherstructure	Objects					n02788148	bannister.n.02	railing	30
+257	trolley	trolley	1	40	7	trolley	otherprop	Objects					n04335435	streetcar.n.01	misc	40
+1316	pantry shelf	pantry shelf	1	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+786	sofa bed	sofa bed	1	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02818832	bed.n.01	bed	11
+801	loofa	loofa	1	40	7		otherprop	Objects							objects	39
+972	shower faucet handle	shower faucet handle	1	40	7	handle	otherprop	Objects							shower	23
+1317	toy piano	toy piano	1	40	7	toy	otherprop	Objects					n03964744	plaything.n.01	objects	39
+1318	fish	fish	1	40	7		otherprop	Objects					n02512053	fish.n.01	objects	39
+75	file cabinets	file cabinet	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n03337140	file.n.03	cabinet	7
+657	cat litter box	cat litter box	1	29	7	box	box	Objects							objects	39
+561	electric panel	electric panel	1	40	7		otherprop	Objects							misc	40
+93	suitcases	suitcase	1	40	7	luggage	otherprop	Objects					n02774630	baggage.n.01	objects	39
+513	curtain rod	curtain rod	1	38	7	curtain rod	otherstructure	Objects							curtain	12
+411	bunk bed	bunk bed	1	39	6	bunk bed	otherfurniture	Furniture	bed	bed	bed	2818832	n02920259	bunk_bed.n.01	bed	11
+1122	chandelier	chandelier	1	38	7	chandelier	otherstructure	Objects					n03005285	chandelier.n.01	lighting	28
+922	tape	tape	1	40	7	tape	otherprop	Objects							objects	39
+88	plates	plate	1	40	7		otherprop	Objects					n03959485	plate.n.04	objects	39
+518	alarm	alarm	1	40	7	alarm	otherprop	Objects			clock	3046257	n02694662	alarm_clock.n.01	objects	39
+814	fire hose	fire hose	1	40	7		otherprop	Objects					n03346004	fire_hose.n.01	misc	40
+1319	toy dinosaur	toy dinosaur	1	40	7	toy	otherprop	Objects					n03964744	plaything.n.01	objects	39
+1320	cone	cone	1	40	7		otherprop	Objects							objects	39
+649	glass doors	glass door	1	8	12	door	door	Wall	door				n03221720	door.n.01	door	4
+607	hatrack	hatrack	1	40	7		otherprop	Objects					n03059103	coatrack.n.01	shelving	31
+819	subwoofer	subwoofer	1	40	7	speaker	otherprop	Objects			speaker	3691459	n04349401	subwoofer.n.01	objects	39
+1321	fire sprinkler	fire sprinkler	1	40	7		otherprop	Objects							misc	40
+1322	trash cabinet	trash cabinet	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+1204	pantry walls	pantry wall	1	1	12	wall	wall	Wall					n04546855	wall.n.01	wall	1
+227	photo	photo	1	40	7	photo	otherprop	Objects					n03925226	photograph.n.01	picture	6
+817	barrier	barrier	1	40	7		otherprop	Objects					n02796623	barrier.n.01	misc	40
+130	stacks of cups	cup	1	40	7		otherprop	Objects					n03147509	cup.n.01	objects	39
+712	beachball	beachball	1	40	7	ball	otherprop	Objects					n02814224	beach_ball.n.01	objects	39
+1323	folded boxes	folded boxes	1	40	7		otherprop	Objects							objects	39
+1324	contact lens solution bottle	contact lens solution bottle	1	40	7	bottle	otherprop	Objects	bottle		bottle	2876657	n02876657	bottle.n.01	objects	39
+673	covered box	covered box	1	29	7	box	box	Objects							objects	39
+459	folder	folder	1	40	7	folder	otherprop	Objects					n03376279	folder.n.02	objects	39
+643	mail trays	mail tray	1	40	7	mail tray	otherprop	Objects							objects	39
+238	slipper	slipper	1	40	7		otherprop	Objects					n04241394	slipper.n.01	clothes	38
+765	magazine rack	magazine rack	1	39	6	stand	otherfurniture	Furniture					n03704549	magazine_rack.n.01	shelving	31
+1008	sticker	sticker	1	40	7	sticker	otherprop	Objects					n07272545	gummed_label.n.01	objects	39
+225	lotion	lotion	1	40	7		otherprop	Objects					n03690938	lotion.n.01	objects	39
+1083	buddha	buddha	1	40	7		otherprop	Objects							objects	39
+813	file organizer	file organizer	1	40	7		otherprop	Objects							objects	39
+138	paper towel rolls	paper towel roll	1	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+1145	night lamp	night lamp	1	35	7	lamp	lamp	Objects	lamp		lamp	3636649	n03636649	lamp.n.02	lighting	28
+796	fuse box	fuse box	1	40	7		otherprop	Objects							misc	40
+1325	knife block	knife block	1	40	7		otherprop	Objects							objects	39
+363	furnace	furnace	1	39	6	furnace	otherfurniture	Furniture					n03404449	furnace.n.01		
+1174	cd cases	cd case	1	40	7		otherprop	Objects							objects	39
+38	stools	stool	1	40	7	stool	otherprop	Objects	stool				n04326896	stool.n.01	stool	19
+1326	hand sanitzer dispenser	hand sanitzer dispenser	1	40	7		otherprop	Objects					n04254120	soap_dispenser.n.01	objects	39
+997	teapot	teapot	1	40	7	tea pot	otherprop	Objects					n04398044	teapot.n.01	objects	39
+1327	pen holder	pen holder	1	40	7		otherprop	Objects							objects	39
+1328	tray rack	tray rack	1	40	7		otherprop	Objects							objects	39
+1329	wig	wig	1	40	7		otherprop	Objects					n04584207	wig.n.01	objects	39
+182	switch	switch	1	40	7		otherprop	Objects					n04372370	switch.n.01	misc	40
+280	plastic containers	plastic container	1	40	7	container	otherprop	Objects					n03094503	container.n.01	objects	39
+1330	night light	night light	1	40	7		otherprop	Objects							lighting	28
+1331	notepad	notepad	1	40	7		otherprop	Objects							objects	39
+1332	mail bin	mail bin	1	40	7		otherprop	Objects							misc	40
+1333	elevator button	elevator button	1	40	7		otherprop	Objects							misc	40
+939	gaming wheel	gaming wheel	1	40	7		otherprop	Objects							objects	39
+1334	drum set	drum set	1	40	7		otherprop	Objects							objects	39
+480	cosmetic bag	cosmetic bag	1	37	7	bag	bag	Objects							objects	39
+907	coffee mug	coffee mug	1	40	7	vessel	otherprop	Objects			cup or mug	3797390	n03063599	coffee_mug.n.01	objects	39
+1335	closet shelf	closet shelf	1	15	6	shelves	shelves	Furniture	bookshelf		bookshelf	2871439	n02871439	bookshelf.n.01	shelving	31
+1336	baby mobile	baby mobile	1	40	7		otherprop	Objects							objects	39
+829	diaper bin	diaper bin	1	40	7	bin	otherprop	Objects							objects	39
+947	door wall	door wall	1	1	12	wall	wall	Wall							wall	1
+1116	stepstool	stepstool	1	40	7	step stool	otherprop	Objects							objects	39
+599	paper shredder	shredder	1	40	7		otherprop	Objects					n04210120	shredder.n.01	objects	39
+733	dress rack	dress rack	1	40	7		otherprop	Objects					n03238762	dress_rack.n.01	misc	40
+123	cover	cover	1	40	7	blanket	otherprop	Objects							objects	39
+506	shopping bag	shopping bag	1	37	7	bag	bag	Objects					n04204081	shopping_bag.n.01	objects	39
+569	sliding door	sliding door	1	8	12	door	door	Wall	door				n04239074	sliding_door.n.01	door	4
+1337	exercise bike	exercise bike	1	40	7	machine	otherprop	Objects					n04210120	shredder.n.01	gym_equipment	33
+1338	recliner chair	recliner chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03238762	dress_rack.n.01	chair	3
+1314	kitchenaid mixer	kitchen mixer	1	40	7		otherprop	Objects							appliances	37
+1339	soda can	soda can	1	40	7	can	otherprop	Objects			can	2946921	n02946921	can.n.01	objects	39
+1340	stovetop	stovetop	1	38	7	stove	otherstructure	Objects			stove	4330267	n04330267	stove.n.02	appliances	37
+851	stepladder	stepladder	1	39	6	ladder	otherfurniture	Furniture	stairs				n04315599	step_ladder.n.01	stairs	16
+142	tap	tap	1	40	7	faucet	otherprop	Objects			faucet	3325088	n04559451	water_faucet.n.01	objects	39
+436	cable	cable	1	40	7	cables	otherprop	Objects							objects	39
+1341	baby changing station	baby changing station	1	39	6		otherfurniture	Furniture							furniture	36
+1342	costume	costume	1	21	7	clothes	clothes	Objects					n02728440	apparel.n.01	clothes	38
+885	rocking chair	rocking chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n04099969	rocking_chair.n.01	chair	3
+693	binder	binder	1	40	7	binder	otherprop	Objects							objects	39
+815	media center	media center	1	3	6	cabinet	cabinet	Furniture			cabinet	2933112	n02933112	cabinet.n.01	cabinet	7
+401	towel rack	towel rack	1	40	7		otherprop	Objects					n04459773	towel_rack.n.01	misc	40
+1343	medal	medal	1	40	7		otherprop	Objects							objects	39
+1184	stack of folded chairs	folded chair	1	5	4	chair	chair	Chair	chair	chair	chair	3001627	n03001627	chair.n.01	chair	3
+1344	telescope	telescope	1	40	7		otherprop	Objects					n04403638	telescope.n.01	objects	39
+1345	closet doorframe	closet doorframe	1	8	12	door	door	Wall	door						door	4
+160	glass	glass	1	38	7	glass	otherstructure	Objects					n03438257	glass.n.02	misc	40
+1126	baseball cap	baseball cap	1	40	7		otherprop	Objects			cap	2954340	n02799323	baseball_cap.n.01	clothes	38
+1346	battery disposal jar	battery disposal jar	1	40	7	jar	otherprop	Objects			jar	3593526	n03593526	jar.n.01	objects	39
+332	mop	mop	1	40	7		otherprop	Objects					n04367480	swab.n.02	objects	39
+397	tank	tank	1	40	7		otherprop	Objects							objects	39
+643	mail tray	mail tray	1	40	7	mail tray	otherprop	Objects							objects	39
+551	centerpiece	centerpiece	1	40	7	centerpiece	otherprop	Objects					n02994419	centerpiece.n.02	objects	39
+1163	stick	stick	1	40	7	stick	otherprop	Objects							objects	39
+1347	closet floor	closet floor	1	2	5	floor	floor	Floor					n03365592	floor.n.01	floor	2
+1348	dryer sheets	dryer sheets	1	40	7		otherprop	Objects							objects	39
+803	bycicle	bycicle	1	40	7		otherprop	Objects							misc	40
+484	flower stand	flower stand	1	39	6	stand	otherfurniture	Furniture							furniture	36
+1349	air mattress	air mattress	1	4	1	bed	bed	Bed	bed	bed	bed	2818832	n02690809	air_mattress.n.01	bed	11
+1350	clip	clip	1	40	7		otherprop	Objects							objects	39
+222	side table	side table	1	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+1253	pizza boxes	pizza box	1	29	7	box	box	Objects					n02883344	box.n.01	objects	39
+1351	display	display	1	39	7		otherfurniture	Furniture					n03211117	display.n.06	misc	40
+1352	postcard	postcard	1	40	7		otherprop	Objects							objects	39
+828	display sign	display sign	1	40	7	sign	otherprop	Objects							misc	40
+1353	paper towel	paper towel	1	40	7	paper towel	otherprop	Objects					n03887697	paper_towel.n.01	towel	20
+612	boots	boot	1	40	7	shoe	otherprop	Objects					n04199027	shoe.n.01	clothes	38
+1354	tennis racket bag	tennis racket bag	1	40	7		otherprop	Objects							objects	39
+1355	air hockey table	air hockey table	1	7	10	table	table	Table	table	table	table	4379243	n04379243	table.n.02	table	5
+1301	socks	sock	1	21	7	clothes	clothes	Objects					n04254777	sock.n.01	clothes	38
+1356	food bag	food bag	1	37	7	bag	bag	Objects							objects	39
+1199	clothes hangers	clothes hanger	1	40	7		otherprop	Objects					n03057920	coat_hanger.n.01	misc	40
+1357	starbucks cup	starbucks cup	1	40	7	cup	otherprop	Objects	cup		cup or mug	3797390	n03797390	mug.n.04	objects	39

assets/meta/scannetv2_raw_categories.json

@@ -0,0 +1,609 @@
+[
+  "wall",
+  "chair",
+  "books",
+  "floor",
+  "door",
+  "object",
+  "window",
+  "table",
+  "trash can",
+  "pillow",
+  "picture",
+  "ceiling",
+  "box",
+  "doorframe",
+  "monitor",
+  "cabinet",
+  "desk",
+  "shelf",
+  "office chair",
+  "towel",
+  "couch",
+  "sink",
+  "backpack",
+  "lamp",
+  "bed",
+  "bookshelf",
+  "mirror",
+  "curtain",
+  "plant",
+  "whiteboard",
+  "radiator",
+  "book",
+  "kitchen cabinet",
+  "toilet paper",
+  "kitchen cabinets",
+  "armchair",
+  "shoes",
+  "coffee table",
+  "toilet",
+  "bag",
+  "clothes",
+  "keyboard",
+  "bottle",
+  "recycling bin",
+  "nightstand",
+  "stool",
+  "tv",
+  "file cabinet",
+  "dresser",
+  "computer tower",
+  "clothing",
+  "telephone",
+  "cup",
+  "refrigerator",
+  "end table",
+  "jacket",
+  "shower curtain",
+  "bathtub",
+  "microwave",
+  "kitchen counter",
+  "sofa chair",
+  "paper towel dispenser",
+  "bathroom vanity",
+  "suitcase",
+  "laptop",
+  "ottoman",
+  "shower walls",
+  "printer",
+  "counter",
+  "board",
+  "soap dispenser",
+  "stove",
+  "light",
+  "closet wall",
+  "mini fridge",
+  "cabinets",
+  "doors",
+  "fan",
+  "tissue box",
+  "blanket",
+  "bathroom stall",
+  "copier",
+  "bench",
+  "bar",
+  "soap dish",
+  "laundry hamper",
+  "storage bin",
+  "bathroom stall door",
+  "light switch",
+  "coffee maker",
+  "tv stand",
+  "decoration",
+  "ceiling light",
+  "range hood",
+  "blackboard",
+  "clock",
+  "wardrobe closet",
+  "rail",
+  "bulletin board",
+  "mat",
+  "trash bin",
+  "ledge",
+  "seat",
+  "mouse",
+  "basket",
+  "shower",
+  "dumbbell",
+  "paper",
+  "person",
+  "windowsill",
+  "closet",
+  "bucket",
+  "sign",
+  "speaker",
+  "dishwasher",
+  "container",
+  "stair rail",
+  "shower curtain rod",
+  "tube",
+  "bathroom cabinet",
+  "papers",
+  "storage container",
+  "paper bag",
+  "paper towel roll",
+  "ball",
+  "closet doors",
+  "laundry basket",
+  "cart",
+  "closet door",
+  "dish rack",
+  "stairs",
+  "blinds",
+  "stack of chairs",
+  "purse",
+  "bicycle",
+  "tray",
+  "plunger",
+  "paper cutter",
+  "toilet paper dispenser",
+  "boxes",
+  "bin",
+  "toilet seat cover dispenser",
+  "guitar",
+  "mailboxes",
+  "handicap bar",
+  "fire extinguisher",
+  "ladder",
+  "column",
+  "pipe",
+  "vacuum cleaner",
+  "plate",
+  "piano",
+  "water cooler",
+  "cd case",
+  "bowl",
+  "closet rod",
+  "bathroom counter",
+  "oven",
+  "stand",
+  "scale",
+  "washing machine",
+  "broom",
+  "hat",
+  "shower wall",
+  "guitar case",
+  "rack",
+  "water pitcher",
+  "laundry detergent",
+  "hair dryer",
+  "pillar",
+  "divider",
+  "power outlet",
+  "dining table",
+  "shower floor",
+  "washing machines",
+  "shower door",
+  "coffee kettle",
+  "wardrobe cabinet",
+  "structure",
+  "bookshelves",
+  "clothes dryer",
+  "toaster",
+  "shoe",
+  "ironing board",
+  "alarm clock",
+  "shower head",
+  "lamp base",
+  "water bottle",
+  "keyboard piano",
+  "projector screen",
+  "case of water bottles",
+  "toaster oven",
+  "music stand",
+  "staircase",
+  "coat rack",
+  "storage organizer",
+  "machine",
+  "folded chair",
+  "fire alarm",
+  "fireplace",
+  "vent",
+  "furniture",
+  "power strip",
+  "calendar",
+  "poster",
+  "toilet paper holder",
+  "potted plant",
+  "stuffed animal",
+  "luggage",
+  "curtains",
+  "headphones",
+  "crate",
+  "candle",
+  "projector",
+  "clothes dryers",
+  "mattress",
+  "dustpan",
+  "drawer",
+  "rod",
+  "globe",
+  "footrest",
+  "piano bench",
+  "breakfast bar",
+  "step stool",
+  "hand rail",
+  "vending machine",
+  "ceiling fan",
+  "swiffer",
+  "foosball table",
+  "jar",
+  "footstool",
+  "folded table",
+  "round table",
+  "hamper",
+  "poster tube",
+  "case",
+  "carpet",
+  "thermostat",
+  "coat",
+  "water fountain",
+  "smoke detector",
+  "pillows",
+  "flip flops",
+  "cloth",
+  "banner",
+  "clothes hanger",
+  "whiteboard eraser",
+  "iron",
+  "instrument case",
+  "toilet paper rolls",
+  "soap",
+  "block",
+  "wall hanging",
+  "kitchen island",
+  "pipes",
+  "toothbrush",
+  "shirt",
+  "cutting board",
+  "vase",
+  "shower control valve",
+  "exercise machine",
+  "compost bin",
+  "shorts",
+  "tire",
+  "teddy bear",
+  "bathrobe",
+  "handrail",
+  "faucet",
+  "pantry wall",
+  "thermos",
+  "rug",
+  "couch cushions",
+  "tripod",
+  "mailbox",
+  "tupperware",
+  "shoe rack",
+  "towels",
+  "beer bottles",
+  "treadmill",
+  "salt",
+  "chest",
+  "dispenser",
+  "mirror doors",
+  "remote",
+  "folded ladder",
+  "cushion",
+  "carton",
+  "step",
+  "drying rack",
+  "slippers",
+  "pool table",
+  "soda stream",
+  "toilet brush",
+  "loft bed",
+  "cooking pot",
+  "heater",
+  "messenger bag",
+  "stapler",
+  "closet walls",
+  "scanner",
+  "elliptical machine",
+  "kettle",
+  "metronome",
+  "dumbell",
+  "music book",
+  "rice cooker",
+  "dart board",
+  "sewing machine",
+  "grab bar",
+  "flowerpot",
+  "painting",
+  "railing",
+  "stair",
+  "toolbox",
+  "nerf gun",
+  "binders",
+  "desk lamp",
+  "quadcopter",
+  "pitcher",
+  "hanging",
+  "mail",
+  "closet ceiling",
+  "hoverboard",
+  "beanbag chair",
+  "water heater",
+  "spray bottle",
+  "rope",
+  "plastic container",
+  "soap bottle",
+  "ikea bag",
+  "sleeping bag",
+  "duffel bag",
+  "frying pan",
+  "oven mitt",
+  "pot",
+  "hand dryer",
+  "dollhouse",
+  "shampoo bottle",
+  "hair brush",
+  "tennis racket",
+  "display case",
+  "ping pong table",
+  "boiler",
+  "bag of coffee beans",
+  "bananas",
+  "carseat",
+  "helmet",
+  "umbrella",
+  "coffee box",
+  "envelope",
+  "wet floor sign",
+  "clothing rack",
+  "controller",
+  "bath walls",
+  "podium",
+  "storage box",
+  "dolly",
+  "shampoo",
+  "paper tray",
+  "cabinet door",
+  "changing station",
+  "poster printer",
+  "screen",
+  "soap bar",
+  "crutches",
+  "studio light",
+  "stack of cups",
+  "toilet flush button",
+  "trunk",
+  "grocery bag",
+  "plastic bin",
+  "pizza box",
+  "cabinet doors",
+  "legs",
+  "car",
+  "shaving cream",
+  "luggage stand",
+  "shredder",
+  "statue",
+  "urinal",
+  "hose",
+  "bike pump",
+  "coatrack",
+  "bear",
+  "wall lamp",
+  "humidifier",
+  "toothpaste",
+  "mouthwash bottle",
+  "poster cutter",
+  "golf bag",
+  "food container",
+  "camera",
+  "table lamp",
+  "yoga mat",
+  "card",
+  "mug",
+  "shower doors",
+  "cardboard",
+  "rack stand",
+  "boxes of paper",
+  "flag",
+  "futon",
+  "magazine",
+  "exit sign",
+  "rolled poster",
+  "wheel",
+  "pictures",
+  "blackboard eraser",
+  "organizer",
+  "doll",
+  "book rack",
+  "laundry bag",
+  "sponge",
+  "seating",
+  "folded chairs",
+  "lotion bottle",
+  "can",
+  "lunch box",
+  "food display",
+  "storage shelf",
+  "sliding wood door",
+  "pants",
+  "wood",
+  "boards",
+  "bottles",
+  "washcloth",
+  "workbench",
+  "open kitchen cabinet",
+  "organizer shelf",
+  "frame",
+  "cups",
+  "exercise ball",
+  "easel",
+  "garbage bag",
+  "roomba",
+  "garage door",
+  "luggage rack",
+  "bike lock",
+  "briefcase",
+  "hand towel",
+  "bath products",
+  "star",
+  "map",
+  "coffee bean bag",
+  "headboard",
+  "ipad",
+  "display rack",
+  "traffic cone",
+  "toiletry",
+  "canopy",
+  "massage chair",
+  "paper organizer",
+  "barricade",
+  "platform",
+  "cap",
+  "dumbbell plates",
+  "elevator",
+  "cooking pan",
+  "trash bag",
+  "santa",
+  "jewelry box",
+  "boat",
+  "sock",
+  "kinect",
+  "crib",
+  "plastic storage bin",
+  "cooler",
+  "kitchen apron",
+  "dishwashing soap bottle",
+  "xbox controller",
+  "banana holder",
+  "ping pong paddle",
+  "airplane",
+  "conditioner bottle",
+  "tea kettle",
+  "bedframe",
+  "wood beam",
+  "toilet paper package",
+  "wall mounted coat rack",
+  "film light",
+  "ceiling lamp",
+  "chain",
+  "sofa",
+  "closet wardrobe",
+  "sweater",
+  "kitchen mixer",
+  "wardrobe",
+  "water softener",
+  "banister",
+  "trolley",
+  "pantry shelf",
+  "sofa bed",
+  "loofa",
+  "shower faucet handle",
+  "toy piano",
+  "fish",
+  "file cabinets",
+  "cat litter box",
+  "electric panel",
+  "suitcases",
+  "curtain rod",
+  "bunk bed",
+  "chandelier",
+  "tape",
+  "plates",
+  "alarm",
+  "fire hose",
+  "toy dinosaur",
+  "cone",
+  "glass doors",
+  "hatrack",
+  "subwoofer",
+  "fire sprinkler",
+  "trash cabinet",
+  "pantry walls",
+  "photo",
+  "barrier",
+  "stacks of cups",
+  "beachball",
+  "folded boxes",
+  "contact lens solution bottle",
+  "covered box",
+  "folder",
+  "mail trays",
+  "slipper",
+  "magazine rack",
+  "sticker",
+  "lotion",
+  "buddha",
+  "file organizer",
+  "paper towel rolls",
+  "night lamp",
+  "fuse box",
+  "knife block",
+  "furnace",
+  "cd cases",
+  "stools",
+  "hand sanitzer dispenser",
+  "teapot",
+  "pen holder",
+  "tray rack",
+  "wig",
+  "switch",
+  "plastic containers",
+  "night light",
+  "notepad",
+  "mail bin",
+  "elevator button",
+  "gaming wheel",
+  "drum set",
+  "cosmetic bag",
+  "coffee mug",
+  "closet shelf",
+  "baby mobile",
+  "diaper bin",
+  "door wall",
+  "stepstool",
+  "paper shredder",
+  "dress rack",
+  "cover",
+  "shopping bag",
+  "sliding door",
+  "exercise bike",
+  "recliner chair",
+  "kitchenaid mixer",
+  "soda can",
+  "stovetop",
+  "stepladder",
+  "tap",
+  "cable",
+  "baby changing station",
+  "costume",
+  "rocking chair",
+  "binder",
+  "media center",
+  "towel rack",
+  "medal",
+  "stack of folded chairs",
+  "telescope",
+  "closet doorframe",
+  "glass",
+  "baseball cap",
+  "battery disposal jar",
+  "mop",
+  "tank",
+  "mail tray",
+  "centerpiece",
+  "stick",
+  "closet floor",
+  "dryer sheets",
+  "bycicle",
+  "flower stand",
+  "air mattress",
+  "clip",
+  "side table",
+  "pizza boxes",
+  "display",
+  "postcard",
+  "display sign",
+  "paper towel",
+  "boots",
+  "tennis racket bag",
+  "air hockey table",
+  "socks",
+  "food bag",
+  "clothes hangers",
+  "starbucks cup"
+]

pq3d/inference.py

@@ -0,0 +1,180 @@
+import json
+import os
+import torch
+
+import numpy as np
+
+
+from pq3d.model import Query3DUnified
+from pq3d.utils import LabelConverter, convert_pc_to_box, obj_processing_post, pad_sequence
+from torch.utils.data import default_collate
+from transformers import AutoTokenizer
+
+ASSET_DIR = os.path.join(os.getcwd(), 'assets')
+CKPT_DIR = os.path.join(os.getcwd(), 'checkpoint')
+int2cat = json.load(open(os.path.join(ASSET_DIR, "meta/scannetv2_raw_categories.json"), 'r', encoding="utf-8"))
+cat2int = {w: i for i, w in enumerate(int2cat)}
+label_converter = LabelConverter(os.path.join(ASSET_DIR, "meta/scannetv2-labels.combined.tsv"))
+
+def load_data(scan_id):
+    one_scan = {}
+    # load scan
+    pcd_data = torch.load(os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_pcd.pth'))
+    inst_to_label = torch.load(os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_inst.pth')) 
+    points, colors, instance_labels  = pcd_data[0], pcd_data[1], pcd_data[-1]
+    colors = colors / 127.5 - 1
+    pcds = np.concatenate([points, colors], 1)
+    one_scan['pcds'] = pcds
+    one_scan['instance_labels'] = instance_labels
+    one_scan['inst_to_label'] = inst_to_label
+    # convert to gt object
+    obj_pcds = []
+    inst_ids = []
+    inst_labels = []
+    bg_indices = np.full((points.shape[0], ), 1, dtype=np.bool_)
+    for inst_id in inst_to_label.keys():
+        if inst_to_label[inst_id] in cat2int.keys():
+            mask = instance_labels == inst_id
+            if np.sum(mask) == 0:
+                continue
+            obj_pcds.append(pcds[mask])
+            inst_ids.append(inst_id)
+            inst_labels.append(cat2int[inst_to_label[inst_id]])
+            if inst_to_label[inst_id] not in ['wall', 'floor', 'ceiling']:
+                bg_indices[mask] = False
+    one_scan['obj_pcds'] = obj_pcds
+    one_scan['inst_labels'] = inst_labels
+    one_scan['inst_ids'] = inst_ids
+    one_scan['bg_pcds'] = pcds[bg_indices]
+    # calculate box for matching
+    obj_center = []
+    obj_box_size = []
+    for obj_pcd in obj_pcds:
+        _c, _b = convert_pc_to_box(obj_pcd)
+        obj_center.append(_c)
+        obj_box_size.append(_b)
+    one_scan['obj_loc'] = obj_center
+    one_scan['obj_box'] = obj_box_size
+    # load image feat
+    feat_pth = os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_img_gt.pth')
+    feat_dict = torch.load(feat_pth)
+    feat_dim = next(iter(feat_dict.values())).shape[0]
+    n_obj = len(one_scan['inst_ids'])  # the last one is for missing objects.
+    feat = torch.zeros((n_obj, feat_dim), dtype=torch.float32)
+    for i, cid in enumerate(one_scan['inst_ids']):
+        if cid in feat_dict.keys():
+            feat[i] =  feat_dict[cid]
+    one_scan['image_obj_feat_gt'] = feat
+    # load voxel feat
+    feat_pth = os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_voxel_gt.pth')
+    feat_dict = torch.load(feat_pth)
+    feat_dim = next(iter(feat_dict.values())).shape[0]
+    n_obj = len(one_scan['inst_ids'])  # the last one is for missing objects.
+    feat = torch.zeros((n_obj, feat_dim), dtype=torch.float32)
+    for i, cid in enumerate(one_scan['inst_ids']):
+        if cid in feat_dict.keys():
+            feat[i] =  feat_dict[cid]
+    one_scan['voxel_obj_feat_gt'] = feat
+    # load point feat
+    feat_pth = os.path.join(ASSET_DIR, f'inputs/{scan_id}', f'{scan_id}_point_gt.pth')
+    feat_dict = torch.load(feat_pth)
+    feat_dim = next(iter(feat_dict.values())).shape[0]
+    n_obj = len(one_scan['inst_ids'])  # the last one is for missing objects.
+    feat = torch.zeros((n_obj, feat_dim), dtype=torch.float32)
+    for i, cid in enumerate(one_scan['inst_ids']):
+        if cid in feat_dict.keys():
+            feat[i] =  feat_dict[cid]
+    one_scan['pc_obj_feat_gt'] = feat
+    #  convert to pq3d input
+    obj_labels = one_scan['inst_labels'] # N
+    obj_pcds = one_scan['obj_pcds']
+    obj_ids = one_scan['inst_ids'] 
+    # object filter
+    excluded_labels = ['wall', 'floor', 'ceiling']
+    def keep_obj(i, obj_label):
+        category = int2cat[obj_label]
+        # filter out background
+        if category in excluded_labels:
+            return False
+        # filter out objects not mentioned in the sentence
+        return True
+    selected_obj_idxs = [i for i, obj_label in enumerate(obj_labels) if keep_obj(i, obj_label)]
+    obj_labels = [obj_labels[i] for i in selected_obj_idxs]
+    obj_pcds = [obj_pcds[i] for i in selected_obj_idxs]
+    # subsample points
+    obj_pcds = np.array([obj_pcd[np.random.choice(len(obj_pcd), size=1024,
+                                        replace=len(obj_pcd) < 1024)] for obj_pcd in obj_pcds])
+    obj_fts, obj_locs, obj_boxes, rot_matrix = obj_processing_post(obj_pcds, rot_aug=False)
+    data_dict = {
+        "scan_id": scan_id,
+        "obj_fts": obj_fts.float(),
+        "obj_locs": obj_locs.float(),
+        "obj_labels": torch.LongTensor(obj_labels),
+        "obj_boxes": obj_boxes, 
+        "obj_pad_masks": torch.ones((len(obj_locs)), dtype=torch.bool), # used for padding in collate
+        "obj_ids": [obj_ids[i] for i in selected_obj_idxs]
+    }
+    # convert image feature
+    feats = one_scan['image_obj_feat_' + 'gt']
+    valid = selected_obj_idxs
+    data_dict['mv_seg_fts'] = feats[valid]
+    data_dict['mv_seg_pad_masks'] = torch.ones(len(data_dict['mv_seg_fts']), dtype=torch.bool)
+    # convert voxel feature
+    feats = one_scan['voxel_obj_feat_' + 'gt']
+    valid = selected_obj_idxs
+    data_dict['voxel_seg_fts'] = feats[valid]
+    data_dict['voxel_seg_pad_masks'] = torch.ones(len(data_dict['voxel_seg_fts']), dtype=torch.bool)
+    # convert point feature
+    feats = one_scan['pc_obj_feat_' + 'gt']
+    valid = selected_obj_idxs
+    data_dict['pc_seg_fts'] = feats[valid]
+    data_dict['pc_seg_pad_masks'] = torch.ones(len(data_dict['pc_seg_fts']), dtype=torch.bool)
+    # build other
+    data_dict['query_locs'] = data_dict['obj_locs'].clone()
+    data_dict['query_pad_masks'] = data_dict['obj_pad_masks'].clone()
+    data_dict['seg_center'] = obj_locs.float()
+    data_dict['seg_pad_masks'] = data_dict['obj_pad_masks']
+    return data_dict
+
+def form_batch(data_dict):
+    batch = [data_dict]
+    new_batch = {}
+    # merge list keys
+    list_keys = [k for k, v in batch[0].items() if isinstance(v, list)]
+    for k in list_keys:
+        new_batch[k] = [sample.pop(k) for sample in batch] 
+    # merge tensor
+    padding_keys = [k for k, v in batch[0].items() if isinstance(v, torch.Tensor) and v.ndim > 0]
+    for k in padding_keys:
+        tensors = [sample.pop(k) for sample in batch]
+        padding_value = -100 if k == 'obj_labels' else 0
+        padded_tensor = pad_sequence(tensors, pad=padding_value)
+        new_batch[k] = padded_tensor
+    # others
+    new_batch.update(default_collate(batch))
+    return new_batch
+    
+def tokenize_txt(text):
+    tokenizer_name = 'openai/clip-vit-large-patch14'
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+    encoded_input = tokenizer([text], add_special_tokens=True, truncation=True)
+    data_dict = {}
+    data_dict['prompt'] = torch.FloatTensor(encoded_input.input_ids[0])
+    data_dict['prompt_pad_masks'] = torch.ones((len(data_dict['prompt']))).bool()
+    data_dict['prompt_type'] = 1 # txt
+    return data_dict
+    
+def inference(scan_id, text):
+    data_dict = load_data(scan_id)
+    data_dict.update(tokenize_txt(text))
+    data_dict = form_batch(data_dict)
+    model = Query3DUnified()
+    load_msg = model.load_state_dict(torch.load(os.path.join(CKPT_DIR, 'pytorch_model.bin')), strict=False)
+    data_dict = model(data_dict)
+    result_id = data_dict['obj_ids'][0][torch.argmax(data_dict['og3d_logits'][0]).item()]
+    print(f"finish infernece result id is {result_id}")
+    return result_id
+    
+if __name__ == '__main__':
+    inference("scene0050_00", "chair")
+    

pq3d/model.py

@@ -0,0 +1,909 @@
+from enum import IntEnum
+from functools import partial
+import einops
+import numpy as np
+import torch
+
+from contextlib import nullcontext
+import torch
+import torch.nn as nn
+from transformers import CLIPTextModelWithProjection
+import copy
+from transformers import T5ForConditionalGeneration
+from transformers.modeling_outputs import BaseModelOutput
+import torch.nn.functional as F
+
+def get_mlp_head(input_size, hidden_size, output_size, dropout=0):
+    return nn.Sequential(*[
+        nn.Linear(input_size, hidden_size),
+        nn.ReLU(),
+        nn.LayerNorm(hidden_size, eps=1e-12),
+        nn.Dropout(dropout),
+        nn.Linear(hidden_size, output_size)
+    ])
+
+def layer_repeat(module, N, share_layer=False):
+    if share_layer:
+        return nn.ModuleList([module] * N)
+    else:
+        return nn.ModuleList([copy.deepcopy(module) for _ in range(N - 1)] + [module])
+
+class CLIPLanguageEncoder(nn.Module):
+    def __init__(self, weights="openai/clip-vit-large-patch14", output_dim=768, freeze_backbone=True, use_projection=False, projection_type='mlp', num_projection_layers=1, dropout=0.1):
+        super().__init__()
+        self.context = torch.no_grad if freeze_backbone else nullcontext
+        self.model = CLIPTextModelWithProjection.from_pretrained(weights)
+        self.use_projection = use_projection
+        self.projection_type = projection_type
+        if use_projection:
+            if projection_type == 'mlp':
+                self.projection = get_mlp_head(self.model.config.hidden_size, output_dim, output_dim, dropout=dropout)
+            else:
+                raise NotImplementedError
+        #self.attention = nn.MultiheadAttention(embed_dim=768, num_heads=12, batch_first=True)
+        
+    def forward(self, txt_ids, txt_masks):
+        with self.context():
+            txt = self.model(txt_ids, txt_masks).last_hidden_state
+            txt = self.model.text_projection(txt)
+            txt = torch.nn.functional.normalize(txt, p=2, dim=2)
+        #txt = self.attention(txt, txt, txt, key_padding_mask=txt_masks.logical_not())[0]
+        if self.use_projection:
+            if self.projection_type == 'mlp':
+                txt = self.projection(txt)
+            elif self.projection_type == 'attention':
+                for attention_layer in self.projection:
+                    txt = attention_layer(txt, tgt_key_padding_mask = txt_masks.logical_not())
+            else:
+                raise NotImplementedError
+        return txt
+
+def _init_weights_bert(module, std=0.02):
+    """
+        Huggingface transformer weight initialization,
+        most commonly for bert initialization
+    """
+    if isinstance(module, nn.Linear):
+        # Slightly different from the TF version which uses truncated_normal for initialization
+        # cf https://github.com/pytorch/pytorch/pull/5617
+        module.weight.data.normal_(mean=0.0, std=std)
+        if module.bias is not None:
+            module.bias.data.zero_()
+    elif isinstance(module, nn.Embedding):
+        module.weight.data.normal_(mean=0.0, std=std)
+        if module.padding_idx is not None:
+            module.weight.data[module.padding_idx].zero_()
+    elif isinstance(module, nn.LayerNorm):
+        module.bias.data.zero_()
+        module.weight.data.fill_(1.0)
+        
+
+def break_up_pc(pc):
+    """
+    Split the pointcloud into xyz positions and features tensors.
+    This method is taken from VoteNet codebase (https://github.com/facebookresearch/votenet)
+
+    @param pc: pointcloud [N, 3 + C]
+    :return: the xyz tensor and the feature tensor
+    """
+    xyz = pc[..., 0:3].contiguous()
+    features = (
+        pc[..., 3:].transpose(1, 2).contiguous()
+        if pc.size(-1) > 3 else None
+    )
+    return xyz, features
+    
+class ObjectEncoder(nn.Module):
+    def __init__(self, backbone='none', input_feat_size=768, hidden_size=768, freeze_backbone=False, use_projection=False,
+                 tgt_cls_num=607, pretrained=None, dropout=0.1, use_cls_head=True):
+        super().__init__()
+        self.freeze_backbone = freeze_backbone
+        self.context = torch.no_grad if freeze_backbone else nullcontext
+        # if backbone == 'pointnet++':
+        #     self.backbone = PointNetPP(
+        #         sa_n_points=[32, 16, None],
+        #         sa_n_samples=[32, 32, None],
+        #         sa_radii=[0.2, 0.4, None],
+        #         sa_mlps=[[3, 64, 64, 128], [128, 128, 128, 256], [256, 256, 512, 768]],
+        #     )
+        if use_cls_head:
+            self.cls_head = get_mlp_head(input_feat_size, input_feat_size // 2, tgt_cls_num, dropout=0.3)
+
+        self.use_projection = use_projection
+        if use_projection:
+            self.input_feat_proj = nn.Sequential(nn.Linear(input_feat_size, hidden_size), nn.LayerNorm(hidden_size))
+        else:
+            assert input_feat_size == hidden_size, "input_feat_size should be equal to hidden_size!"
+        if dropout > 0:
+            self.dropout = nn.Dropout(dropout)
+
+        # load weights
+        self.apply(_init_weights_bert)
+        if pretrained:
+            print("load pretrained weights from {}".format(pretrained))
+            pre_state_dict = torch.load(pretrained)
+            state_dict = {}
+            for k, v in pre_state_dict.items():
+                if k[0] in ['0', '2', '4']: # key mapping for voxel
+                    k = 'cls_head.' + k
+                k = k.replace('vision_encoder.vis_cls_head.', 'cls_head.') # key mapping for mv
+                k = k.replace('point_cls_head.', 'cls_head.') # key mapping for pc 
+                k = k.replace('point_feature_extractor.', 'backbone.')
+                state_dict[k] = v
+            warning = self.load_state_dict(state_dict, strict=False)
+            print(warning)
+
+    def freeze_bn(self, m):
+        for layer in m.modules():
+            if isinstance(layer, nn.BatchNorm2d):
+                layer.eval()
+
+    def forward(self, obj_feats, **kwargs):
+        if self.freeze_backbone and hasattr(self, 'backbone'):
+            self.freeze_bn(self.backbone)
+
+        batch_size, num_objs  = obj_feats.shape[:2]
+        with self.context():
+            if hasattr(self, 'backbone'):
+                obj_feats = self.backbone(einops.rearrange(obj_feats, 'b o p d -> (b o) p d'))
+                obj_feats = einops.rearrange(obj_feats, '(b o) d -> b o d', b=batch_size)
+
+        obj_embeds = self.input_feat_proj(obj_feats) if self.use_projection else obj_feats
+        if hasattr(self, 'dropout'):
+            obj_embeds = self.dropout(obj_embeds)
+
+        if hasattr(self, 'cls_head'):
+            obj_cls_logits = self.cls_head(obj_feats)
+            return obj_embeds, obj_cls_logits
+        else:
+            return obj_embeds
+
+class SelfAttentionLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        dropout=0.0,
+        activation="relu",
+        normalize_before=False,
+        batch_first=False,
+    ):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=batch_first)
+
+        self.norm = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+        self.activation = get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+        self._reset_parameters()
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def with_pos_embed(self, tensor, pos):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None
+    ):
+        q = k = self.with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(
+            q,
+            k,
+            value=tgt,
+            attn_mask=attn_mask,
+            key_padding_mask=tgt_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+        tgt = self.norm(tgt)
+
+        return tgt
+
+    def forward_pre(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None
+    ):
+        tgt2 = self.norm(tgt)
+        q = k = self.with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(
+            q,
+            k,
+            value=tgt2,
+            attn_mask=attn_mask,
+            key_padding_mask=tgt_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+
+        return tgt
+
+    def forward(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None
+    ):
+        if self.normalize_before:
+            return self.forward_pre(
+                tgt, attn_mask, tgt_key_padding_mask, query_pos
+            )
+        return self.forward_post(
+            tgt, attn_mask, tgt_key_padding_mask, query_pos
+        )
+
+
+class CrossAttentionLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        dropout=0.0,
+        activation="relu",
+        normalize_before=False,
+        batch_first=False,
+    ):
+        super().__init__()
+        self.multihead_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout, batch_first=batch_first, add_zero_attn=True
+        )
+
+        self.norm = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+        self.activation = get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+        self._reset_parameters()
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def with_pos_embed(self, tensor, pos):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self,
+        tgt,
+        memory,
+        attn_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        tgt2 = self.multihead_attn(
+            query=self.with_pos_embed(tgt, query_pos),
+            key=self.with_pos_embed(memory, pos),
+            value=memory,
+            attn_mask=attn_mask,
+            key_padding_mask=memory_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+        tgt = self.norm(tgt)
+
+        return tgt
+
+    def forward_pre(
+        self,
+        tgt,
+        memory,
+        attn_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        tgt2 = self.norm(tgt)
+
+        tgt2 = self.multihead_attn(
+            query=self.with_pos_embed(tgt2, query_pos),
+            key=self.with_pos_embed(memory, pos),
+            value=memory,
+            attn_mask=attn_mask,
+            key_padding_mask=memory_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+
+        return tgt
+
+    def forward(
+        self,
+        tgt,
+        memory,
+        attn_mask=None,
+        memory_key_padding_mask=None,
+        pos=None,
+        query_pos=None,
+    ):
+        if self.normalize_before:
+            return self.forward_pre(
+                tgt,
+                memory,
+                attn_mask,
+                memory_key_padding_mask,
+                pos,
+                query_pos,
+            )
+        return self.forward_post(
+            tgt, memory, attn_mask, memory_key_padding_mask, pos, query_pos
+        )
+
+
+class FFNLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        dim_feedforward=2048,
+        dropout=0.0,
+        activation="relu",
+        normalize_before=False,
+    ):
+        super().__init__()
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm = nn.LayerNorm(d_model)
+
+        self.activation = get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+        self._reset_parameters()
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def with_pos_embed(self, tensor, pos):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(self, tgt):
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout(tgt2)
+        tgt = self.norm(tgt)
+        return tgt
+
+    def forward_pre(self, tgt):
+        tgt2 = self.norm(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+        tgt = tgt + self.dropout(tgt2)
+        return tgt
+
+    def forward(self, tgt):
+        if self.normalize_before:
+            return self.forward_pre(tgt)
+        return self.forward_post(tgt)
+    
+def get_activation_fn(activation_type):
+    if activation_type not in ["relu", "gelu", "glu"]:
+        raise RuntimeError(f"activation function currently support relu/gelu, not {activation_type}")
+    return getattr(F, activation_type)
+
+class MultiHeadAttentionSpatial(nn.Module):
+    def __init__(
+            self, d_model, n_head, dropout=0.1, spatial_multihead=True, spatial_dim=5,
+            spatial_attn_fusion='mul',
+    ):
+        super().__init__()
+        assert d_model % n_head == 0, 'd_model: %d, n_head: %d' % (d_model, n_head)
+
+        self.n_head = n_head
+        self.d_model = d_model
+        self.d_per_head = d_model // n_head
+        self.spatial_multihead = spatial_multihead
+        self.spatial_dim = spatial_dim
+        self.spatial_attn_fusion = spatial_attn_fusion
+
+        self.w_qs = nn.Linear(d_model, d_model)
+        self.w_ks = nn.Linear(d_model, d_model)
+        self.w_vs = nn.Linear(d_model, d_model)
+
+        self.fc = nn.Linear(d_model, d_model)
+
+        self.spatial_n_head = n_head if spatial_multihead else 1
+        if self.spatial_attn_fusion in ['mul', 'bias', 'add']:
+            self.pairwise_loc_fc = nn.Linear(spatial_dim, self.spatial_n_head)
+        elif self.spatial_attn_fusion == 'ctx':
+            self.pairwise_loc_fc = nn.Linear(spatial_dim, d_model)
+        elif self.spatial_attn_fusion == 'cond':
+            self.lang_cond_fc = nn.Linear(d_model, self.spatial_n_head * (spatial_dim + 1))
+        else:
+            raise NotImplementedError('unsupported spatial_attn_fusion %s' % (self.spatial_attn_fusion))
+
+    def forward(self, q, k, v, pairwise_locs, key_padding_mask=None, txt_embeds=None):
+        residual = q
+        q = einops.rearrange(self.w_qs(q), 'b l (head k) -> head b l k', head=self.n_head)
+        k = einops.rearrange(self.w_ks(k), 'b t (head k) -> head b t k', head=self.n_head)
+        v = einops.rearrange(self.w_vs(v), 'b t (head v) -> head b t v', head=self.n_head)
+        attn = torch.einsum('hblk,hbtk->hblt', q, k) / np.sqrt(q.shape[-1])
+
+        if self.spatial_attn_fusion in ['mul', 'bias', 'add']:
+            loc_attn = self.pairwise_loc_fc(pairwise_locs)
+            loc_attn = einops.rearrange(loc_attn, 'b l t h -> h b l t')
+            if self.spatial_attn_fusion == 'mul':
+                loc_attn = F.relu(loc_attn)
+            if not self.spatial_multihead:
+                loc_attn = einops.repeat(loc_attn, 'h b l t -> (h nh) b l t', nh=self.n_head)
+        elif self.spatial_attn_fusion == 'ctx':
+            loc_attn = self.pairwise_loc_fc(pairwise_locs)
+            loc_attn = einops.rearrange(loc_attn, 'b l t (h k) -> h b l t k', h=self.n_head)
+            loc_attn = torch.einsum('hblk,hbltk->hblt', q, loc_attn) / np.sqrt(q.shape[-1])
+        elif self.spatial_attn_fusion == 'cond':
+            spatial_weights = self.lang_cond_fc(residual)
+            spatial_weights = einops.rearrange(spatial_weights, 'b l (h d) -> h b l d', h=self.spatial_n_head,
+                                               d=self.spatial_dim + 1)
+            if self.spatial_n_head == 1:
+                spatial_weights = einops.repeat(spatial_weights, '1 b l d -> h b l d', h=self.n_head)
+            spatial_bias = spatial_weights[..., :1]
+            spatial_weights = spatial_weights[..., 1:]
+            loc_attn = torch.einsum('hbld,bltd->hblt', spatial_weights, pairwise_locs) + spatial_bias
+            loc_attn = torch.sigmoid(loc_attn)
+
+        if key_padding_mask is not None:
+            mask = einops.repeat(key_padding_mask, 'b t -> h b l t', h=self.n_head, l=q.size(2))
+            attn = attn.masked_fill(mask, -np.inf)
+            if self.spatial_attn_fusion in ['mul', 'cond']:
+                loc_attn = loc_attn.masked_fill(mask, 0)
+            else:
+                loc_attn = loc_attn.masked_fill(mask, -np.inf)
+
+        if self.spatial_attn_fusion == 'add':
+            fused_attn = (torch.softmax(attn, 3) + torch.softmax(loc_attn, 3)) / 2
+        else:
+            if self.spatial_attn_fusion in ['mul', 'cond']:
+                fused_attn = torch.log(torch.clamp(loc_attn, min=1e-6)) + attn
+            else:
+                fused_attn = loc_attn + attn
+            fused_attn = torch.softmax(fused_attn, 3)
+
+        assert torch.sum(torch.isnan(fused_attn) == 0), print(fused_attn)
+
+        output = torch.einsum('hblt,hbtv->hblv', fused_attn, v)
+        output = einops.rearrange(output, 'head b l v -> b l (head v)')
+        output = self.fc(output)
+        return output, fused_attn
+    
+class SpatialSelfAttentionLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        dropout=0.0,
+        activation="relu",
+        normalize_before=False,
+        batch_first=False,
+        spatial_multihead=True, spatial_dim=5, spatial_attn_fusion='mul'
+    ):
+        super().__init__()
+        self.self_attn = MultiHeadAttentionSpatial(
+            d_model, nhead, dropout=dropout,
+            spatial_multihead=spatial_multihead,
+            spatial_dim=spatial_dim,
+            spatial_attn_fusion=spatial_attn_fusion,
+        )
+
+        self.norm = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+        self.activation = get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+        self._reset_parameters()
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def with_pos_embed(self, tensor, pos):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None,
+        pairwise_locs=None
+    ):
+        q = k = self.with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(
+            q,
+            k,
+            tgt,
+            key_padding_mask=tgt_key_padding_mask,
+            pairwise_locs=pairwise_locs,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+        tgt = self.norm(tgt)
+
+        return tgt
+
+    def forward_pre(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None,
+        pairwise_locs=None
+    ):
+        tgt2 = self.norm(tgt)
+        q = k = self.with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(
+            q,
+            k,
+            tgt,
+            key_padding_mask=tgt_key_padding_mask,
+            pairwise_locs=pairwise_locs,
+        )[0]
+        tgt = tgt + self.dropout(tgt2)
+
+        return tgt
+
+    def forward(
+        self, tgt, attn_mask=None, tgt_key_padding_mask=None, query_pos=None,
+        pairwise_locs=None
+    ):
+        if self.normalize_before:
+            return self.forward_pre(
+                tgt, attn_mask, tgt_key_padding_mask, query_pos,
+                pairwise_locs
+            )
+        return self.forward_post(
+            tgt, attn_mask, tgt_key_padding_mask, query_pos,
+            pairwise_locs
+        )
+        
+class QueryEncoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, memories, dim_feedforward=2048, dropout=0.1, activation="relu", prenorm=False, spatial_selfattn=False, structure='mixed', memory_dropout=0, drop_memories_test=[]):
+        super().__init__()
+        if spatial_selfattn:
+            self.self_attn = SpatialSelfAttentionLayer(d_model, nhead, dropout=dropout, activation=activation, normalize_before=prenorm, batch_first=True)
+        else:
+            self.self_attn = SelfAttentionLayer(d_model, nhead, dropout=dropout, activation=activation, normalize_before=prenorm, batch_first=True)
+        cross_attn_layer = CrossAttentionLayer(d_model, nhead, dropout=dropout, activation=activation, normalize_before=prenorm, batch_first=True) 
+        self.cross_attn_list = layer_repeat(cross_attn_layer, len(memories))
+        self.memory2ca = {memory:ca for memory, ca in zip(memories, self.cross_attn_list)}
+        self.ffn = FFNLayer(d_model, dim_feedforward, dropout=dropout, activation=activation, normalize_before=prenorm)
+        self.structure = structure
+        self.memories = memories
+        self.memory_dropout = memory_dropout
+        self.drop_memories_test = drop_memories_test
+        if structure == 'gate':
+            self.gate_proj = nn.Linear(d_model, d_model)
+
+    def forward(self, query, input_dict, pairwise_locs=None):
+        _, query_masks, query_pos = input_dict['query']
+
+        def sequential_ca(query, memories):
+            for memory in memories:
+                cross_attn = self.memory2ca[memory]
+                feat, mask, pos = input_dict[memory] 
+                if mask.ndim == 2:
+                    memory_key_padding_mask = mask
+                    attn_mask = None
+                else:
+                    memory_key_padding_mask = None
+                    attn_mask = mask
+                query = cross_attn(tgt=query, memory=feat, attn_mask=attn_mask, memory_key_padding_mask = memory_key_padding_mask, query_pos = query_pos, pos = pos)
+            return query
+
+        def parallel_ca(query, memories):
+            assert 'prompt' not in memories
+            query_list = []
+            for memory in memories:
+                cross_attn = self.memory2ca[memory]
+                feat, mask, pos = input_dict[memory] 
+                if mask.ndim == 2:
+                    memory_key_padding_mask = mask
+                    attn_mask = None
+                else:
+                    memory_key_padding_mask = None
+                    attn_mask = mask
+                update = cross_attn(tgt=query, memory=feat, attn_mask=attn_mask, memory_key_padding_mask = memory_key_padding_mask, query_pos = query_pos, pos = pos)
+                query_list.append(update)
+            # training time memory dropout
+            if self.training and self.memory_dropout > 0.0:
+                dropout_mask = torch.rand(query.shape[0], len(memories), device=query.device) > self.memory_dropout
+                num_remained_memories = dropout_mask.sum(dim=1)
+                dropout_mask = torch.logical_or(dropout_mask, num_remained_memories.unsqueeze(-1) == 0)
+                num_remained_memories = dropout_mask.sum(dim=1)
+                query_tensor = torch.stack(query_list, dim=1)
+                query = (query_tensor * dropout_mask.unsqueeze(-1).unsqueeze(-1)).sum(dim=1) / num_remained_memories.unsqueeze(-1).unsqueeze(-1).float()
+            else:
+                query = torch.stack(query_list, dim=1).mean(dim=1)
+            return query
+        
+        memories = self.memories if self.training else [m for m in self.memories if m not in self.drop_memories_test]
+        
+        if self.structure == 'sequential':
+            query = sequential_ca(query, memories)
+        elif self.structure == 'parallel':
+            query = parallel_ca(query, memories)
+        elif self.structure == 'mixed':
+            # [mv,pc,vx] + prompt
+            query = parallel_ca(query, [m for m in memories if m != 'prompt'])
+            query = sequential_ca(query, ['prompt'])
+        elif self.structure == 'gate':
+            prompt = sequential_ca(query, ['prompt'])
+            gate = torch.sigmoid(self.gate_proj(prompt))
+            update = parallel_ca(query, [m for m in self.memories if m != 'prompt'])
+            query = (1. - gate) * query + gate * update
+        else:
+            raise NotImplementedError(f"Unknow structure type: {self.structure}")
+
+        if isinstance(self.self_attn, SpatialSelfAttentionLayer):
+            query = self.self_attn(query, tgt_key_padding_mask = query_masks, query_pos = query_pos, 
+                                   pairwise_locs = pairwise_locs)
+        else:
+            query = self.self_attn(query, tgt_key_padding_mask = query_masks, query_pos = query_pos)
+        query = self.ffn(query)
+
+        return query
+    
+class QueryMaskEncoder(nn.Module):
+    def __init__(self, memories=[], memory_dropout=0.0, hidden_size=768, num_attention_heads=12, num_layers=4,
+                share_layer=False, spatial_selfattn=False, structure='sequential', drop_memories_test=[], use_self_mask=False, num_blocks=1):
+        super().__init__()
+
+        self.spatial_selfattn = spatial_selfattn
+        query_encoder_layer = QueryEncoderLayer(hidden_size, num_attention_heads, memories, spatial_selfattn=spatial_selfattn, structure=structure, memory_dropout=memory_dropout, drop_memories_test=drop_memories_test)
+        self.unified_encoder = layer_repeat(query_encoder_layer, num_layers, share_layer)
+
+        self.apply(_init_weights_bert)
+        self.memory_dropout = memory_dropout
+        self.scene_meomories = [x for x in memories if x != 'prompt']
+        self.drop_memories_test = drop_memories_test
+        self.use_self_mask = use_self_mask
+        self.num_heads = num_attention_heads
+        self.num_blocks = num_blocks
+
+    def forward(self, input_dict, pairwise_locs, mask_head=None):
+            
+        predictions_class, predictions_mask = [], []
+        
+        query = input_dict['query'][0]
+        voxel_feat = input_dict['voxel'][0] if 'voxel' in input_dict.keys() else None
+
+        for block_counter in range(self.num_blocks):
+            for i, layer in enumerate(self.unified_encoder):
+                if mask_head is not None:
+                    output_class, outputs_mask, attn_mask = mask_head(query)
+                    predictions_class.append(output_class)
+                    predictions_mask.append(outputs_mask)  
+                if self.use_self_mask:
+                    attn_mask[attn_mask.all(-1)] = False # prevent query to attend to no point
+                    attn_mask = attn_mask.repeat_interleave(self.num_heads, 0)
+                    for memory in input_dict.keys():
+                        if memory in ['query', 'prompt']:
+                            continue
+                        input_dict[memory][1] = attn_mask
+                        
+                if isinstance(voxel_feat, list):
+                    input_dict['voxel'][0] = voxel_feat[i]  # select voxel features from multi-scale
+                query = layer(query, input_dict, pairwise_locs)
+
+        return query, predictions_class, predictions_mask
+    
+class PromptType(IntEnum):
+    TXT = 1
+    IMAGE = 2
+    LOC = 3
+
+class GroundHead(nn.Module):
+    def __init__(self, input_size=768, hidden_size=768, dropout=0.3):
+        super().__init__()
+        self.og3d_head = get_mlp_head(
+            input_size, hidden_size,
+            1, dropout=dropout
+        )
+
+    def forward(self, obj_embeds, obj_masks=None, **kwargs):
+        og3d_logits = self.og3d_head(obj_embeds).squeeze(2)
+        if obj_masks is not None:
+            og3d_logits = og3d_logits.masked_fill_(obj_masks.logical_not(), -float('inf'))
+        return og3d_logits
+    
+class T5(nn.Module):
+    def __init__(self, variant='t5-small', input_size=768, use_projection=True, **kwargs):
+        super().__init__()
+        self.model = T5ForConditionalGeneration.from_pretrained(variant)
+        self.model.config.update(kwargs)
+        hidden_size = self.model.config.d_model
+        self.use_projection = use_projection
+        if use_projection:
+            self.input_proj = nn.Sequential(nn.Linear(input_size, hidden_size), nn.LayerNorm(hidden_size))
+        else:
+            assert input_size == hidden_size, "input_feat_size should be equal to hidden_size!"
+
+    def forward(self, query_embeds, attention_masks, labels=None):
+        if self.use_projection:
+            query_embeds = self.input_proj(query_embeds)
+
+        if labels is not None:
+            outputs = self.model(encoder_outputs=[query_embeds], attention_mask=attention_masks, labels=labels)
+            outputs = outputs.logits
+        else:
+            outputs = self.model.generate(encoder_outputs=BaseModelOutput(last_hidden_state=query_embeds), attention_mask=attention_masks, do_sample=False)
+            outputs = outputs[:, 1:] # remove the decoder start token for T5 generation output.
+        return outputs
+
+def calc_pairwise_locs(obj_centers, obj_whls, eps=1e-10, pairwise_rel_type='center', spatial_dist_norm=True,
+                       spatial_dim=5):
+    if pairwise_rel_type == 'mlp':
+        obj_locs = torch.cat([obj_centers, obj_whls], 2)
+        pairwise_locs = torch.cat(
+            [einops.repeat(obj_locs, 'b l d -> b l x d', x=obj_locs.size(1)),
+             einops.repeat(obj_locs, 'b l d -> b x l d', x=obj_locs.size(1))],
+            dim=3
+        )
+        return pairwise_locs
+
+    pairwise_locs = einops.repeat(obj_centers, 'b l d -> b l 1 d') \
+                    - einops.repeat(obj_centers, 'b l d -> b 1 l d')
+    pairwise_dists = torch.sqrt(torch.sum(pairwise_locs ** 2, 3) + eps)  # (b, l, l)
+    if spatial_dist_norm:
+        max_dists = torch.max(pairwise_dists.view(pairwise_dists.size(0), -1), dim=1)[0]
+        norm_pairwise_dists = pairwise_dists / einops.repeat(max_dists, 'b -> b 1 1')
+    else:
+        norm_pairwise_dists = pairwise_dists
+
+    if spatial_dim == 1:
+        return norm_pairwise_dists.unsqueeze(3)
+
+    pairwise_dists_2d = torch.sqrt(torch.sum(pairwise_locs[..., :2] ** 2, 3) + eps)
+    if pairwise_rel_type == 'center':
+        pairwise_locs = torch.stack(
+            [norm_pairwise_dists, pairwise_locs[..., 2] / pairwise_dists,
+             pairwise_dists_2d / pairwise_dists, pairwise_locs[..., 1] / pairwise_dists_2d,
+             pairwise_locs[..., 0] / pairwise_dists_2d],
+            dim=3
+        )
+    elif pairwise_rel_type == 'vertical_bottom':
+        bottom_centers = torch.clone(obj_centers)
+        bottom_centers[:, :, 2] -= obj_whls[:, :, 2]
+        bottom_pairwise_locs = einops.repeat(bottom_centers, 'b l d -> b l 1 d') \
+                               - einops.repeat(bottom_centers, 'b l d -> b 1 l d')
+        bottom_pairwise_dists = torch.sqrt(torch.sum(bottom_pairwise_locs ** 2, 3) + eps)  # (b, l, l)
+        bottom_pairwise_dists_2d = torch.sqrt(torch.sum(bottom_pairwise_locs[..., :2] ** 2, 3) + eps)
+        pairwise_locs = torch.stack(
+            [norm_pairwise_dists,
+             bottom_pairwise_locs[..., 2] / bottom_pairwise_dists,
+             bottom_pairwise_dists_2d / bottom_pairwise_dists,
+             pairwise_locs[..., 1] / pairwise_dists_2d,
+             pairwise_locs[..., 0] / pairwise_dists_2d],
+            dim=3
+        )
+
+    if spatial_dim == 4:
+        pairwise_locs = pairwise_locs[..., 1:]
+    return pairwise_locs
+
+class Query3DUnified(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        # record parameters
+        self.memories = ['mv', 'pc', 'voxel', 'prompt']
+        self.heads = ['ground', 'generation']
+        self.use_offline_voxel_fts = True
+        self.use_offline_attn_mask = False
+        self.inputs = self.memories[:]
+        self.pairwise_rel_type = 'center'
+        self.spatial_dim = 5
+        self.num_heads = 12
+        self.skip_query_encoder_mask_pred = True
+        # build prompt type
+        self.prompt_types = ['txt', 'loc']
+        # build feature encoder
+        self.txt_encoder = CLIPLanguageEncoder(use_projection=True, projection_type='mlp', num_projection_layers=1)
+        self.mv_encoder = ObjectEncoder(input_feat_size=768, hidden_size=768, use_projection=True, dropout=0.1, use_cls_head=False)
+        self.voxel_encoder = ObjectEncoder(input_feat_size=128,hidden_size=768, use_projection=True, dropout=0.1, use_cls_head=False)
+        self.pc_encoder = ObjectEncoder(input_feat_size=768, hidden_size=768, dropout=0.1,use_cls_head=False)
+        # build location encoder
+        dim_loc = 6
+        hidden_size = 768
+        self.dim_loc = dim_loc
+        self.hidden_size = hidden_size
+        self.coord_encoder = nn.Sequential(
+            nn.Linear(3, hidden_size),
+            nn.LayerNorm(hidden_size),
+        )
+        self.box_encoder = nn.Sequential(
+            nn.Linear(3, hidden_size),
+            nn.LayerNorm(hidden_size),
+        )
+        # build unified encoder    
+        self.unified_encoder = QueryMaskEncoder(hidden_size=768, num_attention_heads=12, num_layers=4, spatial_selfattn=True, memories=self.memories, drop_memories_test=[], memory_dropout=0.6, structure='mixed', use_self_mask=False, num_blocks=1)
+        # build task head
+        self.ground_head = GroundHead(hidden_size=384, input_size=768, dropout=0.3)
+        self.generation_head = T5(variant='t5-small', input_size=768, use_projection=True, max_new_tokens=50)
+        
+    def prompt_encoder(self, data_dict):
+        prompt = data_dict['prompt']
+        prompt_pad_masks = data_dict['prompt_pad_masks']
+        prompt_type = data_dict['prompt_type']
+        prompt_feat = torch.zeros(prompt.shape + (self.hidden_size,), device=prompt.device)
+        for type in self.prompt_types:
+            # get idx
+            idx = prompt_type == getattr(PromptType, type.upper())
+            if idx.sum() == 0:
+                continue
+            input = prompt[idx]
+            mask = prompt_pad_masks[idx]
+            # encode
+            if type == 'txt':
+                encoder = self.txt_encoder
+                feat = encoder(input.long(), mask)
+            elif type == 'loc':
+                loc_prompts = input[:, :self.dim_loc]
+                if self.dim_loc > 3:
+                    feat = self.coord_encoder(loc_prompts[:, :3]).unsqueeze(1) + self.box_encoder(loc_prompts[:, 3:6]).unsqueeze(1)
+                mask[:, 1:] = False
+            else:
+                raise NotImplementedError(f'{type} is not implemented')
+            # put back to orignal prompt
+            prompt_feat[idx] = feat
+            prompt_pad_masks[idx] = mask
+        return prompt_feat, prompt_pad_masks.logical_not()
+        
+    def forward(self, data_dict):
+        input_dict = {}
+        # build query
+        mask = data_dict['query_pad_masks'].logical_not()
+        query_locs = data_dict['query_locs'][:, :, :self.dim_loc]
+        if self.dim_loc > 3:
+            query_pos  = self.coord_encoder(query_locs[:, :, :3]) + self.box_encoder(query_locs[:, :, 3:6])
+        feat = torch.zeros_like(query_pos)
+        pos = query_pos
+        input_dict['query'] = (feat, mask, pos)
+        # encode fts including point, voxel, image, and prompt
+        # the semantics of the attention mask in pytorch (True as masked) is the opposite as Huggingface Transformers (False as masked)  
+        fts_locs = data_dict['seg_center']
+        if self.dim_loc > 3:
+            fts_pos = self.coord_encoder(fts_locs[:, :, :3]) + self.box_encoder(fts_locs[:, :,  3:6])
+        if self.dim_loc > 3:
+            fts_pos += self.box_encoder(fts_locs[:, :, 3:6])
+        for input in self.inputs:
+            feat, mask, pos = None, None, None
+            if input == 'prompt':
+                feat, mask = self.prompt_encoder(data_dict)
+            elif input == 'mv':
+                feat = self.mv_encoder(obj_feats = data_dict['mv_seg_fts'])
+                mask = data_dict['mv_seg_pad_masks'].logical_not()
+                pos = fts_pos
+            elif input == 'pc':
+                feat = self.pc_encoder(obj_feats = data_dict['pc_seg_fts'])
+                mask = data_dict['pc_seg_pad_masks'].logical_not()
+                pos = fts_pos
+            elif input == 'voxel':
+                feat = self.voxel_encoder(data_dict['voxel_seg_fts'])
+                mask = data_dict['voxel_seg_pad_masks'].logical_not()                
+                pos = fts_pos
+            else:
+                raise NotImplementedError(f"Unknow input type: {input}")
+            input_dict[input] = [feat, mask, pos]
+        # build offline attention mask for guided mask training
+        if self.use_offline_attn_mask:
+            offline_attn_masks = data_dict['offline_attn_mask']
+        else:
+            offline_attn_masks = None
+        mask_head_partial = None
+        # generate features for spatial attention
+        if self.unified_encoder.spatial_selfattn:
+            pairwise_locs = calc_pairwise_locs(query_locs[:, :, :3], None, 
+                                           pairwise_rel_type=self.pairwise_rel_type, spatial_dist_norm=True,
+                                           spatial_dim=self.spatial_dim)
+        else:
+            pairwise_locs = None
+            
+        # unified encoding                           
+        query, predictions_class, predictions_mask = self.unified_encoder(input_dict, pairwise_locs, mask_head_partial)
+        
+        # task head
+        for head in self.heads:
+            if head == 'ground':
+                inputs = [query, data_dict['query_pad_masks']]
+                logits = getattr(self, head + '_head')(*inputs)
+                data_dict[head + '_logits'] = logits
+                data_dict['og3d_logits'] = logits
+            elif head == 'generation':
+                inputs = [query, data_dict['query_pad_masks']] + [None]
+                logits = getattr(self, head + '_head')(*inputs)
+                data_dict[head + '_logits'] = logits
+            else:
+                raise NotImplementedError(f"Unknow head type: {head}")
+       
+        return data_dict

pq3d/utils.py

@@ -0,0 +1,110 @@
+import csv
+import numpy as np
+import torch
+
+
+def convert_pc_to_box(obj_pc):
+    xmin = np.min(obj_pc[:,0])
+    ymin = np.min(obj_pc[:,1])
+    zmin = np.min(obj_pc[:,2])
+    xmax = np.max(obj_pc[:,0])
+    ymax = np.max(obj_pc[:,1])
+    zmax = np.max(obj_pc[:,2])
+    center = [(xmin+xmax)/2, (ymin+ymax)/2, (zmin+zmax)/2]
+    box_size = [xmax-xmin, ymax-ymin, zmax-zmin]
+    return center, box_size
+
+class LabelConverter(object):
+    def __init__(self, file_path):
+        self.raw_name_to_id = {}
+        self.nyu40id_to_id = {}
+        self.nyu40_name_to_id = {}
+        self.scannet_name_to_scannet_id = {'cabinet':0, 'bed':1, 'chair':2, 'sofa':3, 'table':4,
+            'door':5, 'window':6,'bookshelf':7,'picture':8, 'counter':9, 'desk':10, 'curtain':11,
+            'refrigerator':12, 'shower curtain':13, 'toilet':14, 'sink':15, 'bathtub':16, 'others':17}  
+        self.id_to_scannetid = {}
+        self.scannet_raw_id_to_raw_name = {}
+        self.raw_name_to_scannet_raw_id = {}
+
+        with open(file_path, encoding='utf-8') as fd:
+            rd = list(csv.reader(fd, delimiter="\t", quotechar='"'))
+            for i in range(1, len(rd)):
+                raw_id = i - 1
+                scannet_raw_id = int(rd[i][0])
+                raw_name = rd[i][1]
+                nyu40_id = int(rd[i][4])
+                nyu40_name = rd[i][7]
+                self.raw_name_to_id[raw_name] = raw_id
+                self.scannet_raw_id_to_raw_name[scannet_raw_id] = raw_name
+                self.raw_name_to_scannet_raw_id[raw_name] = scannet_raw_id
+                self.nyu40id_to_id[nyu40_id] = raw_id
+                self.nyu40_name_to_id[nyu40_name] = raw_id
+                if nyu40_name not in self.scannet_name_to_scannet_id:
+                    self.id_to_scannetid[raw_id] = self.scannet_name_to_scannet_id['others']
+                else:
+                    self.id_to_scannetid[raw_id] = self.scannet_name_to_scannet_id[nyu40_name]
+
+def build_rotate_mat(split, rot_aug=True, rand_angle='axis'):
+    if rand_angle == 'random':
+        theta = np.random.rand() * np.pi * 2
+    else:
+        ROTATE_ANGLES = [0, np.pi/2, np.pi, np.pi*3/2]
+        theta_idx = np.random.randint(len(ROTATE_ANGLES))
+        theta = ROTATE_ANGLES[theta_idx]
+    if (theta is not None) and (theta != 0) and (split == 'train') and rot_aug:
+        rot_matrix = np.array([
+            [np.cos(theta), -np.sin(theta), 0],
+            [np.sin(theta), np.cos(theta), 0],
+            [0, 0, 1]
+        ], dtype=np.float32)
+    else:
+        rot_matrix = None
+    return rot_matrix
+
+def obj_processing_post(obj_pcds, rot_aug=True):
+        obj_pcds = torch.from_numpy(obj_pcds)
+        rot_matrix = build_rotate_mat('val', rot_aug)
+        if rot_matrix is not None:
+            rot_matrix = torch.from_numpy(rot_matrix.transpose())
+            obj_pcds[:, :, :3] @= rot_matrix
+        
+        xyz = obj_pcds[:, :, :3]
+        center = xyz.mean(1)
+        xyz_min = xyz.min(1).values
+        xyz_max = xyz.max(1).values
+        box_center = (xyz_min + xyz_max) / 2
+        size = xyz_max - xyz_min
+        obj_locs = torch.cat([center, size], dim=1)
+        obj_boxes = torch.cat([box_center, size], dim=1)
+
+        # centering
+        obj_pcds[:, :, :3].sub_(obj_pcds[:, :, :3].mean(1, keepdim=True))
+
+        # normalization
+        max_dist = (obj_pcds[:, :, :3]**2).sum(2).sqrt().max(1).values
+        max_dist.clamp_(min=1e-6)
+        obj_pcds[:, :, :3].div_(max_dist[:, None, None])
+        
+        return obj_pcds, obj_locs, obj_boxes, rot_matrix
+
+
+def pad_sequence(sequence_list, max_len=None, pad=0, return_mask=False):
+    lens = [x.shape[0] for x in sequence_list]
+    if max_len is None:
+        max_len = max(lens)
+        
+    shape = list(sequence_list[0].shape)
+    shape[0] = max_len
+    shape = [len(sequence_list)] + shape
+    dtype = sequence_list[0].dtype
+    device = sequence_list[0].device
+    padded_sequence = torch.ones(shape, dtype=dtype, device=device) * pad
+    for i, tensor in enumerate(sequence_list):
+        padded_sequence[i, :tensor.shape[0]] = tensor
+    padded_sequence = padded_sequence.to(dtype)
+
+    if return_mask:
+        mask = torch.arange(max_len).to(device)[None, :] >= torch.LongTensor(lens).to(device)[:, None] # True as masked.
+        return padded_sequence, mask
+    else:
+        return padded_sequence    

requirements.txt

@@ -0,0 +1,106 @@
+aiofiles==23.2.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+attrs==23.2.0
+Brotli @ file:///croot/brotli-split_1714483155106/work
+certifi @ file:///croot/certifi_1720453481653/work/certifi
+chamfer==2.0.0
+chardet @ file:///tmp/build/80754af9/chardet_1607706775000/work
+charset-normalizer @ file:///tmp/build/80754af9/charset-normalizer_1630003229654/work
+click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
+dnspython==2.6.1
+einops==0.8.0
+email_validator==2.2.0
+exceptiongroup==1.2.1
+fastapi==0.111.0
+fastapi-cli==0.0.4
+ffmpy==0.3.2
+filelock @ file:///croot/filelock_1700591183607/work
+fonttools==4.53.1
+fsspec==2024.6.1
+gmpy2 @ file:///tmp/build/80754af9/gmpy2_1645438755360/work
+gradio==4.37.2
+gradio_client==1.0.2
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.4
+idna @ file:///croot/idna_1714398848350/work
+importlib_resources==6.4.0
+Jinja2 @ file:///croot/jinja2_1716993405101/work
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe @ file:///croot/markupsafe_1704205993651/work
+matplotlib==3.9.1
+mdurl==0.1.2
+mkl-fft @ file:///croot/mkl_fft_1695058164594/work
+mkl-random @ file:///croot/mkl_random_1695059800811/work
+mkl-service==2.4.0
+mpmath @ file:///croot/mpmath_1690848262763/work
+networkx @ file:///croot/networkx_1717597493534/work
+numpy @ file:///croot/numpy_and_numpy_base_1708638617955/work/dist/numpy-1.26.4-cp39-cp39-linux_x86_64.whl#sha256=6094eeedd869502faa0fd0a8c5ad3a70c5779be06ddd1feb7627e5c212fac420
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+orjson==3.10.6
+packaging==24.1
+pandas==2.2.2
+pillow @ file:///croot/pillow_1714398848491/work
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+PySocks @ file:///tmp/build/80754af9/pysocks_1605305812635/work
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.0
+ruff==0.5.1
+safetensors==0.4.3
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.37.2
+sympy @ file:///croot/sympy_1701397643339/work
+tokenizers==0.19.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.0.0
+torchaudio==2.0.0
+torchvision==0.15.0
+tqdm==4.66.4
+transformers==4.42.3
+triton==2.0.0
+typer==0.12.3
+typing_extensions @ file:///croot/typing_extensions_1715268824938/work
+tzdata==2024.1
+ujson==5.10.0
+urllib3 @ file:///croot/urllib3_1718912636303/work
+uvicorn==0.30.1
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==11.0.3
+zipp==3.19.2