{"id":"https://openalex.org/W4391376291","doi":"https://doi.org/10.48550/arxiv.2401.15708","title":"Object-Driven One-Shot Fine-tuning of Text-to-Image Diffusion with Prototypical Embedding","display_name":"Object-Driven One-Shot Fine-tuning of Text-to-Image Diffusion with Prototypical Embedding","publication_year":2024,"publication_date":"2024-01-28","ids":{"openalex":"https://openalex.org/W4391376291","doi":"https://doi.org/10.48550/arxiv.2401.15708"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2401.15708","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.15708","pdf_url":"https://arxiv.org/pdf/2401.15708","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2401.15708","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042325439","display_name":"Jianxiang Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Lu, Jianxiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Xie, Cong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xie, Cong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5023177983","display_name":"Hui Guo","orcid":"https://orcid.org/0000-0002-7874-7273"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Hui","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5042325439"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.7390999794006348,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12357","display_name":"Digital Media Forensic Detection","score":0.7390999794006348,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12859","display_name":"Cell Image Analysis Techniques","score":0.7071999907493591,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T13114","display_name":"Image Processing Techniques and Applications","score":0.6948000192642212,"subfield":{"id":"https://openalex.org/subfields/2214","display_name":"Media Technology"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.7216163873672485},{"id":"https://openalex.org/keywords/shot","display_name":"Shot (pellet)","score":0.6614673137664795},{"id":"https://openalex.org/keywords/object","display_name":"Object (grammar)","score":0.6428155303001404},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.6287207007408142},{"id":"https://openalex.org/keywords/single-shot","display_name":"Single shot","score":0.6217201352119446},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6014367341995239},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.587946355342865},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.5709993243217468},{"id":"https://openalex.org/keywords/diffusion","display_name":"Diffusion","score":0.46558111906051636},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.3832142949104309},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.2160039246082306},{"id":"https://openalex.org/keywords/optics","display_name":"Optics","score":0.17185765504837036},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.13578474521636963}],"concepts":[{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.7216163873672485},{"id":"https://openalex.org/C2778344882","wikidata":"https://www.wikidata.org/wiki/Q278938","display_name":"Shot (pellet)","level":2,"score":0.6614673137664795},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.6428155303001404},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.6287207007408142},{"id":"https://openalex.org/C3019835501","wikidata":"https://www.wikidata.org/wiki/Q1310130","display_name":"Single shot","level":2,"score":0.6217201352119446},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6014367341995239},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.587946355342865},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5709993243217468},{"id":"https://openalex.org/C69357855","wikidata":"https://www.wikidata.org/wiki/Q163214","display_name":"Diffusion","level":2,"score":0.46558111906051636},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.3832142949104309},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.2160039246082306},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.17185765504837036},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.13578474521636963},{"id":"https://openalex.org/C97355855","wikidata":"https://www.wikidata.org/wiki/Q11473","display_name":"Thermodynamics","level":1,"score":0.0},{"id":"https://openalex.org/C191897082","wikidata":"https://www.wikidata.org/wiki/Q11467","display_name":"Metallurgy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2401.15708","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.15708","pdf_url":"https://arxiv.org/pdf/2401.15708","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2401.15708","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2401.15708","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2401.15708","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2401.15708","pdf_url":"https://arxiv.org/pdf/2401.15708","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391376291.pdf","grobid_xml":"https://content.openalex.org/works/W4391376291.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W3142396426","https://openalex.org/W2471333042","https://openalex.org/W2955491601","https://openalex.org/W146529714","https://openalex.org/W2316500695","https://openalex.org/W2017914143","https://openalex.org/W294285819","https://openalex.org/W2390764861","https://openalex.org/W1667634839","https://openalex.org/W2327067209"],"abstract_inverted_index":{"As":[0],"large-scale":[1],"text-to-image":[2,13],"generation":[3],"models":[4,23],"have":[5,18],"made":[6],"remarkable":[7],"progress":[8],"in":[9,45,67],"the":[10,39,56,77,85],"field":[11],"of":[12,41,59,100],"generation,":[14],"many":[15],"fine-tuning":[16,84],"methods":[17],"been":[19],"proposed.":[20],"However,":[21],"these":[22],"often":[24],"struggle":[25],"with":[26,30,131,137],"novel":[27],"objects,":[28],"especially":[29],"one-shot":[31],"scenarios.":[32],"Our":[33,142],"proposed":[34,121],"method":[35,123,143],"aims":[36],"to":[37,96,115],"address":[38],"challenges":[40],"generalizability":[42,63],"and":[43,55,64,80,140],"fidelity":[44,139],"an":[46],"object-driven":[47,122],"way,":[48],"using":[49],"only":[50],"a":[51,70,93],"single":[52],"input":[53],"image":[54],"object-specific":[57,109],"regions":[58],"interest.":[60],"To":[61,103],"improve":[62,105],"mitigate":[65],"overfitting,":[66],"our":[68,120],"paradigm,":[69],"prototypical":[71],"embedding":[72],"is":[73],"initialized":[74],"based":[75],"on":[76],"object's":[78],"appearance":[79],"its":[81],"class,":[82],"before":[83],"diffusion":[86],"model.":[87],"And":[88],"during":[89],"fine-tuning,":[90],"we":[91,107],"propose":[92],"class-characterizing":[94],"regularization":[95],"preserve":[97],"prior":[98],"knowledge":[99],"object":[101],"classes.":[102],"further":[104],"fidelity,":[106],"introduce":[108],"loss,":[110],"which":[111],"can":[112,128],"also":[113],"use":[114],"implant":[116],"multiple":[117],"objects.":[118],"Overall,":[119],"for":[124],"implanting":[125],"new":[126],"objects":[127],"integrate":[129],"seamlessly":[130],"existing":[132,146],"concepts":[133],"as":[134,136],"well":[135],"high":[138],"generalization.":[141],"outperforms":[144],"several":[145],"works.":[147],"The":[148],"code":[149],"will":[150],"be":[151],"released.":[152]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2024-01-31T00:00:00"}