{"id":"https://openalex.org/W4403797131","doi":"https://doi.org/10.48550/arxiv.2409.18071","title":"FreeEdit: Mask-free Reference-based Image Editing with Multi-modal Instruction","display_name":"FreeEdit: Mask-free Reference-based Image Editing with Multi-modal Instruction","publication_year":2024,"publication_date":"2024-09-26","ids":{"openalex":"https://openalex.org/W4403797131","doi":"https://doi.org/10.48550/arxiv.2409.18071"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2409.18071","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18071","pdf_url":"https://arxiv.org/pdf/2409.18071","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2409.18071","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101275668","display_name":"Runze He","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"He, Runze","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103172392","display_name":"Kai Ma","orcid":"https://orcid.org/0000-0003-1810-6658"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Kai","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103005183","display_name":"Linjiang Huang","orcid":"https://orcid.org/0000-0001-9701-6487"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Linjiang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024226895","display_name":"Shaofei Huang","orcid":"https://orcid.org/0000-0001-8996-9907"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Shaofei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076814111","display_name":"Jialin Gao","orcid":"https://orcid.org/0000-0002-8554-7827"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gao, Jialin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102862658","display_name":"Xiaoming Wei","orcid":"https://orcid.org/0000-0002-6788-4790"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Xiaoming","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031794241","display_name":"Jiao Dai","orcid":"https://orcid.org/0000-0003-3559-8009"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Jiao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112539471","display_name":"Jizhong Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Jizhong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100330138","display_name":"Si Liu","orcid":"https://orcid.org/0000-0002-9180-2935"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Si","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5101275668"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9853000044822693,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7061999440193176},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6625980138778687},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.6191474795341492},{"id":"https://openalex.org/keywords/image-editing","display_name":"Image editing","score":0.598028838634491},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.49805736541748047},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4393148422241211},{"id":"https://openalex.org/keywords/image-manipulation","display_name":"Image manipulation","score":0.4262133836746216},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3894311189651489},{"id":"https://openalex.org/keywords/materials-science","display_name":"Materials science","score":0.08218821883201599}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7061999440193176},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6625980138778687},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.6191474795341492},{"id":"https://openalex.org/C2776674983","wikidata":"https://www.wikidata.org/wiki/Q545981","display_name":"Image editing","level":3,"score":0.598028838634491},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.49805736541748047},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4393148422241211},{"id":"https://openalex.org/C2987933465","wikidata":"https://www.wikidata.org/wiki/Q141130","display_name":"Image manipulation","level":3,"score":0.4262133836746216},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3894311189651489},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.08218821883201599},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2409.18071","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18071","pdf_url":"https://arxiv.org/pdf/2409.18071","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2409.18071","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2409.18071","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2409.18071","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2409.18071","pdf_url":"https://arxiv.org/pdf/2409.18071","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4403797131.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2144279858","https://openalex.org/W4319453795","https://openalex.org/W4205104112","https://openalex.org/W1974870326","https://openalex.org/W2770776392","https://openalex.org/W3176454756","https://openalex.org/W2772330423","https://openalex.org/W3163523050","https://openalex.org/W4387969181","https://openalex.org/W2988576913"],"abstract_inverted_index":{"Introducing":[0],"user-specified":[1],"visual":[2,39],"concepts":[3,12],"in":[4,115,141],"image":[5,32,44,112,133,143,178],"editing":[6,64,72,79,113,134,171,210],"is":[7,98],"highly":[8],"practical":[9],"as":[10,173,175,190],"these":[11],"convey":[13],"the":[14,38,42,53,63,71,75,83,90,111,122,139,164,181,184,221],"user's":[15],"intent":[16],"more":[17],"precisely":[18],"than":[19],"text-based":[20],"descriptions.":[21],"We":[22,215],"propose":[23],"FreeEdit,":[24],"a":[25,107,116,147,152,157,176],"novel":[26],"approach":[27,51],"for":[28,77,131],"achieving":[29],"such":[30,189],"reference-based":[31,132],"editing,":[33,169],"which":[34],"can":[35],"accurately":[36],"reproduce":[37],"concept":[40],"from":[41],"reference":[43,86,103,148,177],"based":[45],"on":[46,200],"user-friendly":[47],"language":[48,59,213],"instructions.":[49,214],"Our":[50],"leverages":[52],"multi-modal":[54],"instruction":[55],"encoder":[56],"to":[57,61,100,138,219],"encode":[58],"instructions":[60],"guide":[62],"process.":[65],"This":[66,96],"implicit":[67],"way":[68,118],"of":[69,85,183,223],"locating":[70],"area":[73],"eliminates":[74],"need":[76],"manual":[78],"masks.":[80],"To":[81],"enhance":[82],"reconstruction":[84],"details,":[87],"we":[88,150],"introduce":[89],"Decoupled":[91],"Residual":[92],"ReferAttention":[93],"(DRRA)":[94],"module.":[95],"module":[97],"designed":[99],"integrate":[101],"fine-grained":[102],"features":[104],"extracted":[105],"by":[106,203],"detail":[108],"extractor":[109],"into":[110],"process":[114],"residual":[117],"without":[119],"interfering":[120],"with":[121],"original":[123],"self-attention.":[124],"Given":[125],"that":[126,145,179],"existing":[127,233],"datasets":[128],"are":[129],"unsuitable":[130],"tasks,":[135],"particularly":[136],"due":[137],"difficulty":[140],"constructing":[142],"triplets":[144],"include":[146],"image,":[149],"curate":[151],"high-quality":[153,208],"dataset,":[154],"FreeBench,":[155],"using":[156],"newly":[158],"developed":[159],"twice-repainting":[160],"scheme.":[161],"FreeBench":[162,201],"comprises":[163],"images":[165],"before":[166],"and":[167,194],"after":[168],"detailed":[170],"instructions,":[172],"well":[174],"maintains":[180],"identity":[182],"edited":[185],"object,":[186],"encompassing":[187],"tasks":[188],"object":[191],"addition,":[192],"replacement,":[193],"deletion.":[195],"By":[196],"conducting":[197],"phased":[198],"training":[199],"followed":[202],"quality":[204],"tuning,":[205],"FreeEdit":[206,224],"achieves":[207],"zero-shot":[209],"through":[211],"convenient":[212],"conduct":[216],"extensive":[217],"experiments":[218],"evaluate":[220],"effectiveness":[222],"across":[225],"multiple":[226],"task":[227],"types,":[228],"demonstrating":[229],"its":[230],"superiority":[231],"over":[232],"methods.":[234],"The":[235],"code":[236],"will":[237],"be":[238],"available":[239],"at:":[240],"https://freeedit.github.io/.":[241]},"counts_by_year":[],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2025-10-10T00:00:00"}