{"id":"https://openalex.org/W4402671895","doi":"https://doi.org/10.18653/v1/2024.acl-long.123","title":"VariErr NLI: Separating Annotation Error from Human Label Variation","display_name":"VariErr NLI: Separating Annotation Error from Human Label Variation","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402671895","doi":"https://doi.org/10.18653/v1/2024.acl-long.123"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.acl-long.123","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.acl-long.123","pdf_url":"https://aclanthology.org/2024.acl-long.123.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2024.acl-long.123.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041680406","display_name":"Leon Weber","orcid":"https://orcid.org/0000-0002-2499-472X"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Leon Weber-Genzel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008224824","display_name":"Siyao Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siyao Peng","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112862689","display_name":"Marie-Catherine de Marneffe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marie-Catherine De Marneffe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088832285","display_name":"Barbara Plank","orcid":"https://orcid.org/0000-0002-4394-1965"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barbara Plank","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5041680406"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.669,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.75313321,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"2256","last_page":"2269"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9490000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9490000009536743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.7214709520339966},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7012341618537903},{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.574554443359375},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44215139746665955},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33311203122138977}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.7214709520339966},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7012341618537903},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.574554443359375},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44215139746665955},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33311203122138977},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2024.acl-long.123","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.acl-long.123","pdf_url":"https://aclanthology.org/2024.acl-long.123.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},{"id":"pmh:oai:dial.uclouvain.be:boreal:298679","is_oa":true,"landing_page_url":"http://hdl.handle.net/2078.1/298679","pdf_url":null,"source":{"id":"https://openalex.org/S4306400166","display_name":"Digital Access to Libraries","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800384882","host_organization_name":"Harris County Public Library","host_organization_lineage":["https://openalex.org/I2800384882"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), Vol. 1, no.-, p. 2256\u20132269 (2024)","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.acl-long.123","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2024.acl-long.123","pdf_url":"https://aclanthology.org/2024.acl-long.123.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321390","display_name":"Fonds De La Recherche Scientifique - FNRS","ror":"https://ror.org/03q83t159"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402671895.pdf","grobid_xml":"https://content.openalex.org/works/W4402671895.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2361861616","https://openalex.org/W2386430105","https://openalex.org/W2263699433","https://openalex.org/W2377979023","https://openalex.org/W2218034408","https://openalex.org/W2392921965","https://openalex.org/W2356521405","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Human":[0],"label":[1,100,138],"variation":[2],"arises":[3],"when":[4,20],"annotators":[5,97],"assign":[6],"different":[7],"labels":[8,21],"to":[9],"the":[10,42,86,104,122,152],"same":[11],"item":[12],"for":[13,24,116,170],"valid":[14],"reasons,":[15],"while":[16],"annotation":[17,94],"errors":[18,135],"occur":[19],"are":[22,29],"assigned":[23],"invalid":[25],"reasons.These":[26],"two":[27],"issues":[28],"prevalent":[30],"in":[31,40,61,89,133,179],"NLP":[32,187],"benchmarks,":[33],"yet":[34],"existing":[35],"research":[36,172],"has":[37],"studied":[38],"them":[39],"isolation.To":[41],"best":[43,153],"of":[44,106,124,159],"our":[45],"knowledge,":[46],"there":[47],"exists":[48],"no":[49],"prior":[50],"work":[51],"that":[52,141],"focuses":[53],"on":[54,85,113,173],"teasing":[55],"apart":[56],"error":[57,127,174],"from":[58],"signal,":[59],"especially":[60],"cases":[62],"where":[63],"signal":[64],"is":[65,151,163],"beyond":[66,165],"black-and-white.To":[67],"fill":[68],"this":[69],"gap,":[70],"we":[71],"introduce":[72],"a":[73,77,92],"systematic":[74],"methodology":[75,162],"and":[76,101,131,148,184],"new":[78],"dataset,":[79],"VARIERR":[80],"(variation":[81],"versus":[82,136,175],"error),":[83],"focusing":[84],"NLI":[87],"task":[88],"English.We":[90],"propose":[91],"2-round":[93],"procedure":[95],"with":[96],"explaining":[98],"each":[99],"subsequently":[102],"judging":[103],"validity":[105,111],"label-explanation":[107],"pairs.VARIERR":[108],"contains":[109],"7,732":[110],"judgments":[112],"1,933":[114],"explanations":[115],"500":[117],"re-annotated":[118],"MNLI":[119],"items.We":[120],"assess":[121],"effectiveness":[123],"various":[125],"automatic":[126],"detection":[128],"(AED)":[129],"methods":[130,144],"GPTs":[132,147],"uncovering":[134],"human":[137,160],"variation.We":[139],"find":[140],"state-of-the-art":[142],"AED":[143],"significantly":[145],"underperform":[146],"humans.While":[149],"GPT-4":[150],"system,":[154],"it":[155],"still":[156],"falls":[157],"short":[158],"performance.Our":[161],"applicable":[164],"NLI,":[166],"offering":[167],"fertile":[168],"ground":[169],"future":[171],"plausible":[176],"variation,":[177],"which":[178],"turn":[180],"can":[181],"yield":[182],"better":[183],"more":[185],"trustworthy":[186],"systems.":[188]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}