{"id":"https://openalex.org/W4388444503","doi":"https://doi.org/10.48550/arxiv.2311.01755","title":"Towards a Unified Transformer-based Framework for Scene Graph Generation and Human-object Interaction Detection","display_name":"Towards a Unified Transformer-based Framework for Scene Graph Generation and Human-object Interaction Detection","publication_year":2023,"publication_date":"2023-11-03","ids":{"openalex":"https://openalex.org/W4388444503","doi":"https://doi.org/10.48550/arxiv.2311.01755"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2311.01755","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01755","pdf_url":"https://arxiv.org/pdf/2311.01755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2311.01755","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052920929","display_name":"Tao He","orcid":"https://orcid.org/0000-0002-7566-7649"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"He, Tao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066645546","display_name":"Lianli Gao","orcid":"https://orcid.org/0000-0002-2522-6394"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Lianli","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036987388","display_name":"Jingkuan Song","orcid":"https://orcid.org/0000-0002-2549-8322"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Jingkuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5017943466","display_name":"Yuan-Fang Li","orcid":"https://orcid.org/0000-0003-4651-2821"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yuan-Fang","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5052920929"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9955000281333923,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.73417067527771},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6571304798126221},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6511533856391907},{"id":"https://openalex.org/keywords/scene-graph","display_name":"Scene graph","score":0.576634407043457},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5432248711585999},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.47754085063934326},{"id":"https://openalex.org/keywords/object-detection","display_name":"Object detection","score":0.4743187129497528},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.45029258728027344},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.43601712584495544},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42142045497894287},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.28159618377685547},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.23160269856452942}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.73417067527771},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6571304798126221},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6511533856391907},{"id":"https://openalex.org/C179372163","wikidata":"https://www.wikidata.org/wiki/Q1406181","display_name":"Scene graph","level":3,"score":0.576634407043457},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5432248711585999},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.47754085063934326},{"id":"https://openalex.org/C2776151529","wikidata":"https://www.wikidata.org/wiki/Q3045304","display_name":"Object detection","level":3,"score":0.4743187129497528},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.45029258728027344},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.43601712584495544},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42142045497894287},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.28159618377685547},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.23160269856452942},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2311.01755","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01755","pdf_url":"https://arxiv.org/pdf/2311.01755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2311.01755","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2311.01755","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2311.01755","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2311.01755","pdf_url":"https://arxiv.org/pdf/2311.01755","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4388444503.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4231704780","https://openalex.org/W2083794993","https://openalex.org/W352609212","https://openalex.org/W1511772879","https://openalex.org/W4379115841","https://openalex.org/W4200340037","https://openalex.org/W4206178588","https://openalex.org/W3094491777","https://openalex.org/W3214715529","https://openalex.org/W4287635093"],"abstract_inverted_index":{"Scene":[0],"graph":[1],"generation":[2],"(SGG)":[3],"and":[4,17,22,26,61,131,181,221],"human-object":[5,71,99,159],"interaction":[6],"(HOI)":[7],"detection":[8],"are":[9],"two":[10,87,120],"important":[11],"visual":[12,55,149],"tasks":[13,33,128,223,233],"aiming":[14],"at":[15],"localising":[16],"recognising":[18],"relationships":[19,56],"between":[20,24,85],"objects,":[21,27],"interactions":[23,160],"humans":[25],"respectively.":[28],"Prevailing":[29],"works":[30],"treat":[31],"these":[32],"as":[34,94],"distinct":[35],"tasks,":[36,88],"leading":[37],"to":[38,45,76,124,157,193,206,235],"the":[39,52,68,86,114,127,163,184],"development":[40],"of":[41,54,70,103,129,148,170,187],"task-specific":[42],"models":[43],"tailored":[44],"individual":[46],"datasets.":[47],"However,":[48],"we":[49,105,135,152,211],"posit":[50],"that":[51,65,213],"presence":[53],"can":[57,92],"furnish":[58],"crucial":[59],"contextual":[60],"intricate":[62],"relational":[63],"cues":[64],"significantly":[66],"augment":[67],"inference":[69],"interactions.":[72,100],"This":[73],"motivates":[74],"us":[75],"think":[77],"if":[78],"there":[79],"is":[80],"a":[81,95,108,137,146],"natural":[82],"intrinsic":[83],"relationship":[84],"where":[89],"scene":[90],"graphs":[91],"serve":[93],"source":[96],"for":[97,231],"inferring":[98],"In":[101],"light":[102],"this,":[104],"introduce":[106],"SG2HOI+,":[107],"unified":[109],"one-step":[110],"model":[111,190],"based":[112,161],"on":[113,162,218],"Transformer":[115,139],"architecture.":[116],"Our":[117],"approach":[118,200],"employs":[119],"interactive":[121],"hierarchical":[122],"Transformers":[123],"seamlessly":[125],"unify":[126],"SGG":[130,196,220],"HOI":[132,208,222],"detection.":[133],"Concretely,":[134],"initiate":[136],"relation":[138,143,165],"tasked":[140],"with":[141],"generating":[142],"triples":[144],"from":[145],"suite":[147],"features.":[150],"Subsequently,":[151],"employ":[153],"another":[154],"transformer-based":[155],"decoder":[156],"predict":[158],"generated":[164],"triples.":[166],"A":[167],"comprehensive":[168],"series":[169],"experiments":[171],"conducted":[172],"across":[173],"established":[174],"benchmark":[175],"datasets":[176],"including":[177],"Visual":[178],"Genome,":[179],"V-COCO,":[180],"HICO-DET":[182],"demonstrates":[183],"compelling":[185],"performance":[186,203],"our":[188,199,214],"SG2HOI+":[189,215],"in":[191,224],"comparison":[192],"prevalent":[194],"one-stage":[195],"models.":[197],"Remarkably,":[198],"achieves":[201],"competitive":[202],"when":[204],"compared":[205,234],"state-of-the-art":[207],"methods.":[209],"Additionally,":[210],"observe":[212],"jointly":[216],"trained":[217],"both":[219,232],"an":[225],"end-to-end":[226],"manner":[227],"yields":[228],"substantial":[229],"improvements":[230],"individualized":[236],"training":[237],"paradigms.":[238]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2023-11-07T00:00:00"}