{"id":"https://openalex.org/W4321160919","doi":"https://doi.org/10.48550/arxiv.2302.07241","title":"ConceptFusion: Open-set Multimodal 3D Mapping","display_name":"ConceptFusion: Open-set Multimodal 3D Mapping","publication_year":2023,"publication_date":"2023-02-14","ids":{"openalex":"https://openalex.org/W4321160919","doi":"https://doi.org/10.48550/arxiv.2302.07241"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2302.07241","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.07241","pdf_url":"https://arxiv.org/pdf/2302.07241","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2302.07241","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047370393","display_name":"Krishna Murthy Jatavallabhula","orcid":"https://orcid.org/0000-0003-4927-3387"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Jatavallabhula, Krishna Murthy","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001304008","display_name":"Alihusein Kuwajerwala","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kuwajerwala, Alihusein","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100602746","display_name":"Qiao Gu","orcid":"https://orcid.org/0000-0001-9431-6716"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Qiao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084670506","display_name":"Mohd Omama","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Omama, Mohd","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075177714","display_name":"Tao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Tao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Maalouf, Alaa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maalouf, Alaa","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100415973","display_name":"Shuang Li","orcid":"https://orcid.org/0000-0003-4258-3163"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Shuang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070768105","display_name":"Ganesh Iyer","orcid":"https://orcid.org/0000-0001-9331-1278"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Iyer, Ganesh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112862090","display_name":"Soroush Saryazdi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saryazdi, Soroush","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050116265","display_name":"Nikhil Keetha","orcid":"https://orcid.org/0000-0003-2770-0835"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Keetha, Nikhil","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075797855","display_name":"Ayush Tewari","orcid":"https://orcid.org/0000-0002-3805-4421"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tewari, Ayush","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071093940","display_name":"Joshua B. Tenenbaum","orcid":"https://orcid.org/0000-0002-1925-2035"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tenenbaum, Joshua B.","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064929876","display_name":"Celso Miguel de Melo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"de Melo, Celso Miguel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113194940","display_name":"Madhava Krishna","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krishna, Madhava","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037065865","display_name":"Liam Paull","orcid":"https://orcid.org/0000-0003-2492-6660"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Paull, Liam","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010648258","display_name":"Florian Shkurti","orcid":"https://orcid.org/0000-0002-5672-5831"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shkurti, Florian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5085020955","display_name":"Antonio Torralba","orcid":"https://orcid.org/0000-0003-4915-0256"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Torralba, Antonio","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":17,"corresponding_author_ids":["https://openalex.org/A5047370393"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8094700574874878},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6694053411483765},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5804874300956726},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4945182502269745},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.47989046573638916},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.4684344232082367},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.37123680114746094}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8094700574874878},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6694053411483765},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5804874300956726},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4945182502269745},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.47989046573638916},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.4684344232082367},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.37123680114746094},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2302.07241","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.07241","pdf_url":"https://arxiv.org/pdf/2302.07241","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2302.07241","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2302.07241","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2302.07241","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2302.07241","pdf_url":"https://arxiv.org/pdf/2302.07241","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11","score":0.7599999904632568}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2062195135","https://openalex.org/W8302103","https://openalex.org/W3171631314","https://openalex.org/W2795079307","https://openalex.org/W2674584172","https://openalex.org/W4288076938","https://openalex.org/W2967743314","https://openalex.org/W1990892711","https://openalex.org/W2360858150","https://openalex.org/W370365947"],"abstract_inverted_index":{"Building":[0],"3D":[1,27,103,112,154,192,226],"maps":[2,28,52,155],"of":[3,44,88,98,123,201],"the":[4,33,102,120],"environment":[5],"is":[6,78],"central":[7],"to":[8,32,101,107,109,111,131],"robot":[9],"navigation,":[10],"planning,":[11],"and":[12,90,142,159,176,212],"interaction":[13],"with":[14,26,72,225],"objects":[15],"in":[16,61,116],"a":[17,41,74,85,95,199,207],"scene.":[18],"Most":[19],"existing":[20],"approaches":[21],"that":[22,77,146],"integrate":[23],"semantic":[24],"concepts":[25,89,134,179],"largely":[29],"remain":[30],"confined":[31],"closed-set":[34],"setting:":[35],"they":[36],"can":[37,53,150],"only":[38,54],"reason":[39,132],"about":[40,133],"finite":[42],"set":[43,87],"concepts,":[45],"pre-defined":[46],"at":[47],"training":[48,173],"time.":[49],"Further,":[50],"these":[51,70],"be":[55,151],"queried":[56],"using":[57,64],"class":[58],"labels,":[59],"or":[60,174,238],"recent":[62],"work,":[63],"text":[65],"prompts.":[66],"We":[67,144,194,217],"address":[68],"both":[69],"issues":[71],"ConceptFusion,":[73],"scene":[75],"representation":[76],"(1)":[79],"fundamentally":[80],"open-set,":[81],"enabling":[82,94],"reasoning":[83],"beyond":[84],"closed":[86],"(ii)":[91],"inherently":[92],"multimodal,":[93],"diverse":[96],"range":[97],"possible":[99],"queries":[100],"map,":[104],"from":[105],"language,":[106,140],"images,":[108,141],"audio,":[110],"geometry,":[113],"all":[114],"working":[115],"concert.":[117],"ConceptFusion":[118,197],"leverages":[119],"open-set":[121,148,227],"capabilities":[122],"today's":[124],"foundation":[125,223],"models":[126,224],"pre-trained":[127],"on":[128,191,198],"internet-scale":[129],"data":[130],"across":[135],"modalities":[136],"such":[137],"as":[138],"natural":[139],"audio.":[143],"demonstrate":[145],"pixel-aligned":[147],"features":[149],"fused":[152],"into":[153],"via":[156],"traditional":[157],"SLAM":[158],"multi-view":[160],"fusion":[161],"approaches.":[162],"This":[163],"enables":[164],"effective":[165],"zero-shot":[166],"spatial":[167],"reasoning,":[168],"not":[169],"needing":[170],"any":[171],"additional":[172],"finetuning,":[175],"retains":[177],"long-tailed":[178],"better":[180],"than":[181,188],"supervised":[182],"approaches,":[183],"outperforming":[184],"them":[185],"by":[186],"more":[187,231],"40%":[189],"margin":[190],"IoU.":[193],"extensively":[195],"evaluate":[196],"number":[200],"real-world":[202,208],"datasets,":[203],"simulated":[204],"home":[205],"environments,":[206],"tabletop":[209],"manipulation":[210],"task,":[211],"an":[213],"autonomous":[214],"driving":[215],"platform.":[216],"showcase":[218],"new":[219],"avenues":[220],"for":[221],"blending":[222],"multimodal":[228],"mapping.":[229],"For":[230],"information,":[232],"visit":[233],"our":[234,240],"project":[235],"page":[236],"https://concept-fusion.github.io":[237],"watch":[239],"5-minute":[241],"explainer":[242],"video":[243],"https://www.youtube.com/watch?v=rkXgws8fiDs":[244]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2025-10-10T00:00:00"}