{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T22:21:09Z","timestamp":1777501269641,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":60,"publisher":"ACM","license":[{"start":{"date-parts":[[2018,10,15]],"date-time":"2018-10-15T00:00:00Z","timestamp":1539561600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"funder":[{"DOI":"10.13039\/100000001","name":"National Science Foundation","doi-asserted-by":"publisher","award":["CNS-1563722"],"award-info":[{"award-number":["CNS-1563722"]}],"id":[{"id":"10.13039\/100000001","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100000185","name":"Defense Advanced Research Projects Agency","doi-asserted-by":"publisher","award":["FA8750-15-2-0104"],"award-info":[{"award-number":["FA8750-15-2-0104"]}],"id":[{"id":"10.13039\/100000185","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2018,10,15]]},"DOI":"10.1145\/3243734.3243804","type":"proceedings-article","created":{"date-parts":[[2018,10,16]],"date-time":"2018-10-16T17:38:33Z","timestamp":1539711513000},"page":"2123-2138","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":469,"title":["Evaluating Fuzz Testing"],"prefix":"10.1145","author":[{"given":"George","family":"Klees","sequence":"first","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}]},{"given":"Andrew","family":"Ruef","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}]},{"given":"Benji","family":"Cooper","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}]},{"given":"Shiyi","family":"Wei","sequence":"additional","affiliation":[{"name":"University of Texas at Dallas, Dallas, TX, USA"}]},{"given":"Michael","family":"Hicks","sequence":"additional","affiliation":[{"name":"University of Maryland, College Park, MD, USA"}]}],"member":"320","published-online":{"date-parts":[[2018,10,15]]},"reference":[{"key":"e_1_3_2_2_1_1","unstructured":"AFL 2018. American Fuzzing Lop (AFL). http:\/\/lcamtuf.coredump.cx\/afl\/.  AFL 2018. American Fuzzing Lop (AFL). http:\/\/lcamtuf.coredump.cx\/afl\/."},{"key":"e_1_3_2_2_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1985793.1985795"},{"key":"e_1_3_2_2_3_1","unstructured":"BFF 2018. CERT Basic Fuzzing Framework (BFF). https:\/\/github.com\/CERTCC\/certfuzz.  BFF 2018. CERT Basic Fuzzing Framework (BFF). https:\/\/github.com\/CERTCC\/certfuzz."},{"key":"e_1_3_2_2_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1167473.1167488"},{"key":"e_1_3_2_2_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134020"},{"key":"e_1_3_2_2_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/2976749.2978428"},{"key":"e_1_3_2_2_7_1","doi-asserted-by":"publisher","DOI":"10.1113\/jphysiol.2012.239376"},{"key":"e_1_3_2_2_8_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2012.31"},{"key":"e_1_3_2_2_9_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2015.50"},{"key":"e_1_3_2_2_10_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2018.00046"},{"key":"e_1_3_2_2_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/2491956.2462173"},{"key":"e_1_3_2_2_12_1","unstructured":"ConfIntv 2018. Confidence Intervals for a Median. http:\/\/www.ucl.ac.uk\/ich\/short-courses-events\/about-stats-courses\/stats-rm\/Chapter_8_Content\/confidence_interval_single_median.  ConfIntv 2018. Confidence Intervals for a Median. http:\/\/www.ucl.ac.uk\/ich\/short-courses-events\/about-stats-courses\/stats-rm\/Chapter_8_Content\/confidence_interval_single_median."},{"key":"e_1_3_2_2_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134069"},{"key":"e_1_3_2_2_14_1","volume-title":"CGC","author":"DARPA","year":"2018","unstructured":"DARPA CGC 2018 . Darpa Cyber Grand Challenge (CGC) Binaries. https:\/\/github.com\/CyberGrandChallenge\/. DARPA CGC 2018. Darpa Cyber Grand Challenge (CGC) Binaries. https:\/\/github.com\/CyberGrandChallenge\/."},{"key":"e_1_3_2_2_15_1","unstructured":"Brendan Dolan-Gavitt. 2018. Of Bugs and Baselines. http:\/\/moyix.blogspot.com\/2018\/03\/of-bugs-and-baselines.html.  Brendan Dolan-Gavitt. 2018. Of Bugs and Baselines. http:\/\/moyix.blogspot.com\/2018\/03\/of-bugs-and-baselines.html."},{"key":"e_1_3_2_2_16_1","volume-title":"LAVA: Large- Scale Automated Vulnerability Addition. In IEEE Symposium on Security and Privacy (S&P).","author":"Dolan-Gavitt Brendan","year":"2016","unstructured":"Brendan Dolan-Gavitt , Patrick Hulin , Engin Kirda , Tim Leek , Andrea Mambretti , William K. Robertson , Frederick Ulrich , and Ryan Whelan . 2016 . LAVA: Large- Scale Automated Vulnerability Addition. In IEEE Symposium on Security and Privacy (S&P). Brendan Dolan-Gavitt, Patrick Hulin, Engin Kirda, Tim Leek, Andrea Mambretti, William K. Robertson, Frederick Ulrich, and Ryan Whelan. 2016. LAVA: Large- Scale Automated Vulnerability Addition. In IEEE Symposium on Security and Privacy (S&P)."},{"key":"e_1_3_2_2_17_1","doi-asserted-by":"publisher","DOI":"10.1111\/j.1476-5381.2011.01809.x"},{"key":"e_1_3_2_2_18_1","unstructured":"FuzzerTestSuite 2018. Fuzzer Test Suite. https:\/\/github.com\/google\/fuzzer-test-suite.  FuzzerTestSuite 2018. Fuzzer Test Suite. https:\/\/github.com\/google\/fuzzer-test-suite."},{"key":"e_1_3_2_2_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568278"},{"key":"e_1_3_2_2_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/2976002.2976017"},{"key":"e_1_3_2_2_21_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2017.09.018"},{"key":"e_1_3_2_2_22_1","volume-title":"USENIX Security Symposium.","author":"Haller Istvan","year":"2013","unstructured":"Istvan Haller , Asia Slowinska , Matthias Neugschwandtner , and Herbert Bos . 2013 . Dowsing for Overflows: A Guided Fuzzer to Find Buffer Boundary Violations . In USENIX Security Symposium. Istvan Haller, Asia Slowinska, Matthias Neugschwandtner, and Herbert Bos. 2013. Dowsing for Overflows: A Guided Fuzzer to Find Buffer Boundary Violations. In USENIX Security Symposium."},{"key":"e_1_3_2_2_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134103"},{"key":"e_1_3_2_2_24_1","volume-title":"Enhancing Memory Error Detection for Large-Scale Applications and Fuzz Testing. In Network and Distributed System Security Symposium (NDSS).","author":"Han Wookhyun","year":"2018","unstructured":"Wookhyun Han , Byunggill Joe , Byoungyoung Lee , Chengyu Song , and Insik Shin . 2018 . Enhancing Memory Error Detection for Large-Scale Applications and Fuzz Testing. In Network and Distributed System Security Symposium (NDSS). Wookhyun Han, Byunggill Joe, Byoungyoung Lee, Chengyu Song, and Insik Shin. 2018. Enhancing Memory Error Detection for Large-Scale Applications and Fuzz Testing. In Network and Distributed System Security Symposium (NDSS)."},{"key":"e_1_3_2_2_25_1","volume-title":"VDF: Targeted Evolutionary Fuzz Testing of Virtual Devices. In Research in Attacks, Intrusions, and Defenses (RAID).","author":"Henderson Andrew","year":"2017","unstructured":"Andrew Henderson , Heng Yin , Guang Jin , Hao Han , and Hongmei Deng . 2017 . VDF: Targeted Evolutionary Fuzz Testing of Virtual Devices. In Research in Attacks, Intrusions, and Defenses (RAID). Andrew Henderson, Heng Yin, Guang Jin, Hao Han, and Hongmei Deng. 2017. VDF: Targeted Evolutionary Fuzz Testing of Virtual Devices. In Research in Attacks, Intrusions, and Defenses (RAID)."},{"key":"e_1_3_2_2_26_1","unstructured":"Michael Hicks. 2015. What is a bug? http:\/\/www.pl-enthusiast.net\/2015\/09\/08\/what-is-a-bug\/.  Michael Hicks. 2015. What is a bug? http:\/\/www.pl-enthusiast.net\/2015\/09\/08\/what-is-a-bug\/."},{"key":"e_1_3_2_2_27_1","volume-title":"Chizpurfle: A Gray-Box Android Fuzzer for Vendor Service Customizations. In IEEE International Symposium on Software Reliability Engineering (ISSRE).","author":"Iannillo Antonio Ken","year":"2017","unstructured":"Antonio Ken Iannillo , Roberto Natella , Domenico Cotroneo , and Cristina Nita-Rotaru . 2017 . Chizpurfle: A Gray-Box Android Fuzzer for Vendor Service Customizations. In IEEE International Symposium on Software Reliability Engineering (ISSRE). Antonio Ken Iannillo, Roberto Natella, Domenico Cotroneo, and Cristina Nita-Rotaru. 2017. Chizpurfle: A Gray-Box Android Fuzzer for Vendor Service Customizations. In IEEE International Symposium on Software Reliability Engineering (ISSRE)."},{"key":"e_1_3_2_2_28_1","doi-asserted-by":"publisher","DOI":"10.1145\/2568225.2568271"},{"key":"e_1_3_2_2_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2786805.2786844"},{"key":"e_1_3_2_2_30_1","volume-title":"IEEE International Conference on Software Analysis, Evolution, and Reengineering (SANER).","author":"Kochhar P. S.","unstructured":"P. S. Kochhar , F. Thung , and D. Lo . 2015. Code coverage and test suite effectiveness: Empirical study with real bugs in large systems . In IEEE International Conference on Software Analysis, Evolution, and Reengineering (SANER). P. S. Kochhar, F. Thung, and D. Lo. 2015. Code coverage and test suite effectiveness: Empirical study with real bugs in large systems. In IEEE International Conference on Software Analysis, Evolution, and Reengineering (SANER)."},{"key":"e_1_3_2_2_31_1","unstructured":"lcamtuf. 2018. AFL quick start guide. http:\/\/lcamtuf.coredump.cx\/afl\/QuickStartGuide.txt.  lcamtuf. 2018. AFL quick start guide. http:\/\/lcamtuf.coredump.cx\/afl\/QuickStartGuide.txt."},{"key":"e_1_3_2_2_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238176"},{"key":"e_1_3_2_2_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3106295"},{"key":"e_1_3_2_2_34_1","unstructured":"libFuzzer 2018. libFuzzer. https:\/\/llvm.org\/docs\/LibFuzzer.html.  libFuzzer 2018. libFuzzer. https:\/\/llvm.org\/docs\/LibFuzzer.html."},{"key":"e_1_3_2_2_35_1","doi-asserted-by":"publisher","DOI":"10.1109\/CCST.2015.7389677"},{"key":"e_1_3_2_2_36_1","volume-title":"USENIX Security Symposium.","author":"Molnar David","unstructured":"David Molnar , Xue Cong Li , and David A. Wagner . 2009. Dynamic Test Generation to Find Integer Bugs in x86 Binary Linux Programs . In USENIX Security Symposium. David Molnar, Xue Cong Li, and David A. Wagner. 2009. Dynamic Test Generation to Find Integer Bugs in x86 Binary Linux Programs. In USENIX Security Symposium."},{"key":"e_1_3_2_2_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/1508244.1508275"},{"key":"e_1_3_2_2_38_1","volume-title":"Longnecker","author":"Lyman Ott R.","year":"2006","unstructured":"R. Lyman Ott and Micheal T . Longnecker . 2006 . Introduction to Statistical Methods and Data Analysis (with CD-ROM) . R. Lyman Ott and Micheal T. Longnecker. 2006. Introduction to Statistical Methods and Data Analysis (with CD-ROM)."},{"key":"e_1_3_2_2_39_1","doi-asserted-by":"publisher","DOI":"10.1109\/SP.2018.00056"},{"key":"e_1_3_2_2_40_1","volume-title":"NEZHA: Efficient Domain-Independent Differential Testing. In IEEE Symposium on Security and Privacy (S&P).","author":"Petsios Theofilos","year":"2017","unstructured":"Theofilos Petsios , Adrian Tang , Salvatore J. Stolfo , Angelos D. Keromytis , and Suman Jana . 2017 . NEZHA: Efficient Domain-Independent Differential Testing. In IEEE Symposium on Security and Privacy (S&P). Theofilos Petsios, Adrian Tang, Salvatore J. Stolfo, Angelos D. Keromytis, and Suman Jana. 2017. NEZHA: Efficient Domain-Independent Differential Testing. In IEEE Symposium on Security and Privacy (S&P)."},{"key":"e_1_3_2_2_41_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134073"},{"key":"e_1_3_2_2_42_1","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-662-54494-5_3"},{"key":"e_1_3_2_2_43_1","unstructured":"Radamsa 2018. Radamsa. https:\/\/github.com\/aoh\/radamsa.  Radamsa 2018. Radamsa. https:\/\/github.com\/aoh\/radamsa."},{"key":"e_1_3_2_2_44_1","volume-title":"Vuzzer: Application-aware evolutionary fuzzing. In NDSS.","author":"Rawat Sanjay","year":"2017","unstructured":"Sanjay Rawat , Vivek Jain , Ashish Kumar , Lucian Cojocar , Cristiano Giuffrida , and Herbert Bos . 2017 . Vuzzer: Application-aware evolutionary fuzzing. In NDSS. Sanjay Rawat, Vivek Jain, Ashish Kumar, Lucian Cojocar, Cristiano Giuffrida, and Herbert Bos. 2017. Vuzzer: Application-aware evolutionary fuzzing. In NDSS."},{"key":"e_1_3_2_2_45_1","volume-title":"Optimizing Seed Selection for Fuzzing. In USENIX Security Symposium.","author":"Rebert Alexandre","year":"2014","unstructured":"Alexandre Rebert , Sang Kil Cha , Thanassis Avgerinos , Jonathan Foote , David Warren , Gustavo Grieco , and David Brumley . 2014 . Optimizing Seed Selection for Fuzzing. In USENIX Security Symposium. Alexandre Rebert, Sang Kil Cha, Thanassis Avgerinos, Jonathan Foote, David Warren, Gustavo Grieco, and David Brumley. 2014. Optimizing Seed Selection for Fuzzing. In USENIX Security Symposium."},{"key":"e_1_3_2_2_46_1","volume-title":"Local Restarts. In International Conference on Theory and Applications of Satisfiability Testing (SAT).","author":"Ryvchin Vadim","year":"2008","unstructured":"Vadim Ryvchin and Ofer Strichman . 2008 . Local Restarts. In International Conference on Theory and Applications of Satisfiability Testing (SAT). Vadim Ryvchin and Ofer Strichman. 2008. Local Restarts. In International Conference on Theory and Applications of Satisfiability Testing (SAT)."},{"key":"e_1_3_2_2_47_1","volume-title":"Security Symposium.","author":"Schumilo Sergej","year":"2017","unstructured":"Sergej Schumilo , Cornelius Aschermann , Robert Gawlik , Sebastian Schinzel , and Thorsten Holz . 2017 . kAFL: Hardware-Assisted Feedback Fuzzing for OS Kernels . Security Symposium. Sergej Schumilo, Cornelius Aschermann, Robert Gawlik, Sebastian Schinzel, and Thorsten Holz. 2017. kAFL: Hardware-Assisted Feedback Fuzzing for OS Kernels. Security Symposium."},{"key":"e_1_3_2_2_48_1","volume-title":"USENIX Annual Technical Conference.","author":"Serebryany Konstantin","year":"2012","unstructured":"Konstantin Serebryany , Derek Bruening , Alexander Potapenko , and Dmitriy Vyukov . 2012 . AddressSanitizer: A Fast Address Sanity Checker .. In USENIX Annual Technical Conference. Konstantin Serebryany, Derek Bruening, Alexander Potapenko, and Dmitriy Vyukov. 2012. AddressSanitizer: A Fast Address Sanity Checker.. In USENIX Annual Technical Conference."},{"key":"e_1_3_2_2_49_1","doi-asserted-by":"crossref","unstructured":"Bhargava Shastry Markus Leutner Tobias Fiebig Kashyap Thimmaraju Fabian Yamaguchi Konrad Rieck Stefan Schmid Jean-Pierre Seifert and Anja Feldmann. 2017. Static Program Analysis as a Fuzzing Aid. In Research in Attacks Intrusions and Defenses (RAID).  Bhargava Shastry Markus Leutner Tobias Fiebig Kashyap Thimmaraju Fabian Yamaguchi Konrad Rieck Stefan Schmid Jean-Pierre Seifert and Anja Feldmann. 2017. Static Program Analysis as a Fuzzing Aid. In Research in Attacks Intrusions and Defenses (RAID).","DOI":"10.1007\/978-3-319-66332-6_2"},{"key":"e_1_3_2_2_50_1","doi-asserted-by":"publisher","DOI":"10.14722\/ndss.2016.23368"},{"key":"e_1_3_2_2_51_1","doi-asserted-by":"publisher","DOI":"10.1145\/3238147.3238200"},{"key":"e_1_3_2_2_52_1","article-title":"A Critique and Improvement of the CL Common Language Effect Size Statistics of McGraw and Wong","volume":"25","author":"Vargha Andr\u00e1s","year":"2000","unstructured":"Andr\u00e1s Vargha and Harold D. Delaney . 2000 . A Critique and Improvement of the CL Common Language Effect Size Statistics of McGraw and Wong . Journal of Educational and Behavioral Statistics 25 , 2 (2000). Andr\u00e1s Vargha and Harold D. Delaney. 2000. A Critique and Improvement of the CL Common Language Effect Size Statistics of McGraw and Wong. Journal of Educational and Behavioral Statistics 25, 2 (2000).","journal-title":"Journal of Educational and Behavioral Statistics"},{"key":"e_1_3_2_2_53_1","volume-title":"Skyfire: Data-Driven Seed Generation for Fuzzing. In IEEE Symposium on Security and Privacy (S&P).","author":"Wang Junjie","year":"2017","unstructured":"Junjie Wang , Bihuan Chen , Lei Wei , and Yang Liu . 2017 . Skyfire: Data-Driven Seed Generation for Fuzzing. In IEEE Symposium on Security and Privacy (S&P). Junjie Wang, Bihuan Chen, Lei Wei, and Yang Liu. 2017. Skyfire: Data-Driven Seed Generation for Fuzzing. In IEEE Symposium on Security and Privacy (S&P)."},{"key":"e_1_3_2_2_54_1","volume-title":"SeededFuzz: Selecting and Generating Seeds for Directed Fuzzing. In International Symposium on Theoretical Aspects of Software Engineering (TASE).","author":"Wang Weiguang","year":"2016","unstructured":"Weiguang Wang , Hao Sun , and Qingkai Zeng . 2016 . SeededFuzz: Selecting and Generating Seeds for Directed Fuzzing. In International Symposium on Theoretical Aspects of Software Engineering (TASE). Weiguang Wang, Hao Sun, and Qingkai Zeng. 2016. SeededFuzz: Selecting and Generating Seeds for Directed Fuzzing. In International Symposium on Theoretical Aspects of Software Engineering (TASE)."},{"key":"e_1_3_2_2_55_1","doi-asserted-by":"publisher","DOI":"10.1145\/2508859.2516736"},{"key":"e_1_3_2_2_56_1","doi-asserted-by":"publisher","DOI":"10.1145\/3133956.3134046"},{"key":"e_1_3_2_2_57_1","volume-title":"IEEE International Conference on Smart Grid Communications.","author":"Yoo Hyunguk","year":"2016","unstructured":"Hyunguk Yoo and Taeshik Shon . 2016 . Grammar-based adaptive fuzzing: Evaluation on SCADA modbus protocol . In IEEE International Conference on Smart Grid Communications. Hyunguk Yoo and Taeshik Shon. 2016. Grammar-based adaptive fuzzing: Evaluation on SCADA modbus protocol. In IEEE International Conference on Smart Grid Communications."},{"key":"e_1_3_2_2_58_1","doi-asserted-by":"publisher","DOI":"10.1109\/CIS.2017.00127"},{"key":"e_1_3_2_2_59_1","doi-asserted-by":"publisher","DOI":"10.1109\/TENCON.2017.8227972"},{"key":"e_1_3_2_2_60_1","unstructured":"Zzuf 2018. Zzuf. http:\/\/caca.zoy.org\/wiki\/zzuf.  Zzuf 2018. Zzuf. http:\/\/caca.zoy.org\/wiki\/zzuf."}],"event":{"name":"CCS '18: 2018 ACM SIGSAC Conference on Computer and Communications Security","location":"Toronto Canada","acronym":"CCS '18","sponsor":["SIGSAC ACM Special Interest Group on Security, Audit, and Control"]},"container-title":["Proceedings of the 2018 ACM SIGSAC Conference on Computer and Communications Security"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3243734.3243804","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3243734.3243804","content-type":"application\/pdf","content-version":"vor","intended-application":"syndication"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3243734.3243804","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T02:08:19Z","timestamp":1750212499000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3243734.3243804"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,10,15]]},"references-count":60,"alternative-id":["10.1145\/3243734.3243804","10.1145\/3243734"],"URL":"https:\/\/doi.org\/10.1145\/3243734.3243804","relation":{},"subject":[],"published":{"date-parts":[[2018,10,15]]},"assertion":[{"value":"2018-10-15","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}