{"id":"https://openalex.org/W4407012547","doi":"https://doi.org/10.48550/arxiv.2501.18533","title":"Rethinking Bottlenecks in Safety Fine-Tuning of Vision Language Models","display_name":"Rethinking Bottlenecks in Safety Fine-Tuning of Vision Language Models","publication_year":2025,"publication_date":"2025-01-30","ids":{"openalex":"https://openalex.org/W4407012547","doi":"https://doi.org/10.48550/arxiv.2501.18533"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2501.18533","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.18533","pdf_url":"https://arxiv.org/pdf/2501.18533","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2501.18533","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030009218","display_name":"Yi Ding","orcid":"https://orcid.org/0000-0003-3406-9770"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ding, Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103281142","display_name":"Lijun Li","orcid":"https://orcid.org/0000-0002-4665-4216"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Lijun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083124454","display_name":"Bing Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Bing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103068380","display_name":"Jing Shao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shao, Jing","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5030009218"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9858999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9858999848365784,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.973800003528595,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5015709400177002},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33141571283340454},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32127803564071655}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5015709400177002},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33141571283340454},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32127803564071655}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2501.18533","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.18533","pdf_url":"https://arxiv.org/pdf/2501.18533","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:doi:10.48550/arxiv.2501.18533","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2501.18533","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2501.18533","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2501.18533","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2501.18533","pdf_url":"https://arxiv.org/pdf/2501.18533","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Large":[0],"Vision-Language":[1],"Models":[2],"(VLMs)":[3],"have":[4],"achieved":[5],"remarkable":[6],"performance":[7,156],"across":[8,174],"a":[9,51,82,190],"wide":[10],"range":[11],"of":[12,119],"tasks.":[13],"However,":[14],"their":[15],"deployment":[16],"in":[17,36,77,141],"safety-critical":[18,78],"domains":[19],"poses":[20],"significant":[21],"challenges.":[22],"Existing":[23],"safety":[24,52,58,90,116,155,187],"fine-tuning":[25,128,166],"methods,":[26],"which":[27],"focus":[28],"on":[29,185],"textual":[30],"or":[31,40],"multimodal":[32],"content,":[33],"fall":[34],"short":[35],"addressing":[37],"challenging":[38,142],"cases":[39],"disrupt":[41],"the":[42,105,180],"balance":[43],"between":[44],"helpfulness":[45],"and":[46,70,75,121,138,178],"harmlessness.":[47],"Our":[48,124],"evaluation":[49],"highlights":[50],"reasoning":[53,60,76,96],"gap:":[54],"these":[55],"methods":[56],"lack":[57],"visual":[59,73,147],"ability,":[61],"leading":[62],"to":[63,98],"such":[64],"bottlenecks.":[65],"To":[66],"address":[67],"this":[68],"limitation":[69],"enhance":[71],"both":[72,134],"perception":[74],"contexts,":[79],"we":[80,103],"propose":[81],"novel":[83],"dataset":[84,112],"that":[85,127],"integrates":[86],"multi-image":[87,115,143],"inputs":[88],"with":[89,130,167],"Chain-of-Thought":[91],"(CoT)":[92],"labels":[93],"as":[94],"fine-grained":[95],"logic":[97],"improve":[99],"model":[100],"performance.":[101],"Specifically,":[102,165],"introduce":[104],"Multi-Image":[106],"Safety":[107],"(MIS)":[108],"dataset,":[109],"an":[110],"instruction-following":[111],"tailored":[113],"for":[114],"scenarios,":[117],"consisting":[118],"training":[120],"test":[122],"splits.":[123],"experiments":[125],"demonstrate":[126],"InternVL2.5-8B":[129],"MIS":[131,168],"significantly":[132],"outperforms":[133],"powerful":[135],"open-source":[136],"models":[137,140],"API-based":[139],"tasks":[144],"requiring":[145],"safety-related":[146],"reasoning.":[148],"This":[149],"approach":[150],"not":[151],"only":[152],"delivers":[153],"exceptional":[154],"but":[157],"also":[158],"preserves":[159],"general":[160,176],"capabilities":[161],"without":[162],"any":[163],"trade-offs.":[164],"increases":[169],"average":[170],"accuracy":[171],"by":[172,189],"0.83%":[173],"five":[175],"benchmarks":[177,188],"reduces":[179],"Attack":[181],"Success":[182],"Rate":[183],"(ASR)":[184],"multiple":[186],"large":[191],"margin.":[192]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}