{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,1,29]],"date-time":"2026-01-29T22:48:04Z","timestamp":1769726884108,"version":"3.49.0"},"reference-count":55,"publisher":"Springer Science and Business Media LLC","issue":"5","license":[{"start":{"date-parts":[[2020,8,28]],"date-time":"2020-08-28T00:00:00Z","timestamp":1598572800000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"},{"start":{"date-parts":[[2020,8,28]],"date-time":"2020-08-28T00:00:00Z","timestamp":1598572800000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.springer.com\/tdm"}],"funder":[{"DOI":"10.13039\/501100000038","name":"Natural Sciences and Engineering Research Council of Canada","doi-asserted-by":"publisher","award":["CRDPJ-517460-17"],"award-info":[{"award-number":["CRDPJ-517460-17"]}],"id":[{"id":"10.13039\/501100000038","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Empir Software Eng"],"published-print":{"date-parts":[[2020,9]]},"DOI":"10.1007\/s10664-020-09866-z","type":"journal-article","created":{"date-parts":[[2020,8,28]],"date-time":"2020-08-28T15:04:10Z","timestamp":1598627050000},"page":"4130-4160","update-policy":"https:\/\/doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":19,"title":["Using black-box performance models to detect performance regressions under varying workloads: an empirical study"],"prefix":"10.1007","volume":"25","author":[{"ORCID":"https:\/\/orcid.org\/0000-0001-9920-5855","authenticated-orcid":false,"given":"Lizhi","family":"Liao","sequence":"first","affiliation":[]},{"given":"Jinfu","family":"Chen","sequence":"additional","affiliation":[]},{"given":"Heng","family":"Li","sequence":"additional","affiliation":[]},{"given":"Yi","family":"Zeng","sequence":"additional","affiliation":[]},{"given":"Weiyi","family":"Shang","sequence":"additional","affiliation":[]},{"given":"Jianmei","family":"Guo","sequence":"additional","affiliation":[]},{"given":"Catalin","family":"Sporea","sequence":"additional","affiliation":[]},{"given":"Andrei","family":"Toma","sequence":"additional","affiliation":[]},{"given":"Sarah","family":"Sajedi","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2020,8,28]]},"reference":[{"key":"9866_CR1","unstructured":"Apache James (2019) Project-apache james server 3-release notes. http:\/\/james.apache.org\/server\/release-notes.html. Last accessed 10\/09\/2019"},{"key":"9866_CR2","unstructured":"Gridsearchc (2019) https:\/\/scikit-learn.org\/stable\/modules\/generated\/sklearn.model_selection.GridSearchCV.html. Last accessed 10\/11\/2019"},{"key":"9866_CR3","unstructured":"Pidstat (2019) Report statistics for tasks - linux man page. https:\/\/linux.die.net\/man\/1\/pidstat. Last accessed 10\/11\/2019"},{"key":"9866_CR4","doi-asserted-by":"crossref","unstructured":"Alcocer JPS, Bergel A (2015) Tracking down performance variation against source code evolution. In: Proceedings of the 11th symposium on dynamic languages, DLS 2015. Association for Computing Machinery, New York, pp 129\u2013139","DOI":"10.1145\/2816707.2816718"},{"key":"9866_CR5","doi-asserted-by":"crossref","unstructured":"Barna C, Litoiu M, Ghanbari H (2011) Autonomic load-testing framework. In: Proceedings of the 8th international conference on autonomic computing, ICAC 2011, Karlsruhe, Germany, June 14-18, 2011, pp 91\u2013100","DOI":"10.1145\/1998582.1998598"},{"key":"9866_CR6","doi-asserted-by":"crossref","unstructured":"Benesty J, Chen J, Huang Y, Cohen I (2009) Pearson correlation coefficient. In Noise reduction in speech processing, pp 1\u20134. Springer","DOI":"10.1007\/978-3-642-00296-0_5"},{"key":"9866_CR7","unstructured":"Breiman L, Cutler A, Liaw A, Wiener M (2018) Breiman and cutler\u2019s random forests for classification and regression. R Package Version 4.6\u201314"},{"key":"9866_CR8","unstructured":"Cannon J (2019) Performance degradation affecting salesforce clients. https:\/\/marketingland.com\/performance-degradation-affecting-salesforce-clients-267699. Last accessed 10\/11\/2019"},{"key":"9866_CR9","doi-asserted-by":"crossref","unstructured":"Chen T, Shang W, Hassan AE, Nasser MN, Flora P (2016) Cacheoptimizer: helping developers configure caching frameworks for hibernate-based database-centric web applications. In: Proceedings of the 24th ACM SIGSOFT international symposium on foundations of software engineering, FSE 2016, Seattle, WA, USA, November 13-18, 2016, pp 666\u2013677","DOI":"10.1145\/2950290.2950303"},{"key":"9866_CR10","unstructured":"Cliff N (1996) Ordinal methods for behavioral data analysis"},{"key":"9866_CR11","unstructured":"Cohen I, Chase JS, Goldszmidt M, Kelly T, Symons J (2004) Correlating instrumentation data to system states: A building block for automated diagnosis and control. In: 6th symposium on operating system design and implementation (OSDI 2004), San Francisco, California, USA, December 6-8, 2004, pp 231\u2013244"},{"key":"9866_CR12","doi-asserted-by":"crossref","unstructured":"Cohen I, Zhang S, Goldszmidt M, Symons J, Kelly T, Fox A (2005) Capturing, indexing, clustering, and retrieving system history. In: Proceedings of the 20th ACM symposium on operating systems principles 2005, SOSP 2005, Brighton, UK, October 23-26, 2005, pp 105\u2013118","DOI":"10.1145\/1095809.1095821"},{"key":"9866_CR13","doi-asserted-by":"crossref","unstructured":"Cortez E, Bonde A, Muzio A, Russinovich M, Fontoura M, Bianchini R (2017) Resource central: Understanding and predicting workloads for improved resource management in large cloud platforms. In: Proceedings of the 26th symposium on operating systems principles, Shanghai, China, October 28-31, 2017, pp 153\u2013167","DOI":"10.1145\/3132747.3132772"},{"key":"9866_CR14","doi-asserted-by":"crossref","unstructured":"Dacrema MF, Cremonesi P, Jannach D (2019) Are we really making much progress? A worrying analysis of recent neural recommendation approaches. In: Proceedings of the 13th ACM conference on recommender systems, RecSys 2019, Copenhagen, Denmark, September 16-20, 2019., pp 101\u2013109","DOI":"10.1145\/3298689.3347058"},{"key":"9866_CR15","doi-asserted-by":"crossref","unstructured":"de Oliveira AB, Fischmeister S, Diwan A, Hauswirth M, Sweeney PF (2013) Why you should care about quantile regression. In: Architectural support for programming languages and operating systems, ASPLOS \u201913, Houston, TX, USA - March 16 - 20, 2013, pp 207\u2013218","DOI":"10.1145\/2451116.2451140"},{"key":"9866_CR16","doi-asserted-by":"crossref","unstructured":"Didona D, Quaglia F, Romano P, Torre E (2015) Enhancing performance prediction robustness by combining analytical modeling and machine learning. In: Proceedings of the 6th ACM\/SPEC international conference on performance engineering, Austin, TX, USA, Jan 31 - Feb 4, 2015, pp 145\u2013156","DOI":"10.1145\/2668930.2688047"},{"key":"9866_CR17","doi-asserted-by":"crossref","unstructured":"Farshchi M, Schneider J, Weber I, Grundy JC (2015) Experience report: Anomaly detection of cloud application operations using log and cloud metric correlation analysis. In: 26th IEEE international symposium on software reliability engineering, ISSRE 2015, Gaithersbury, MD, USA, November 2-5, 2015, pp 24\u201334","DOI":"10.1109\/ISSRE.2015.7381796"},{"key":"9866_CR18","doi-asserted-by":"crossref","unstructured":"Foo KC, Jiang ZM, Adams B, Hassan AE, Zou Y, Flora P (2010) Mining performance regression testing repositories for automated performance analysis. In: Proceedings of the 2010 10th international conference on quality software, QSIC \u201910, pp 32\u201341","DOI":"10.1109\/QSIC.2010.35"},{"key":"9866_CR19","doi-asserted-by":"crossref","unstructured":"Foo KC, Jiang ZMJ, Adams B, Hassan AE, Zou Y, Flora P (2015) An industrial case study on the automated detection of performance regressions in heterogeneous environments. In: Proceedings of the 37th international conference on software engineering - vol 2, ICSE \u201915, pp 159\u2013168","DOI":"10.1109\/ICSE.2015.144"},{"key":"9866_CR20","doi-asserted-by":"crossref","unstructured":"Gao R, Jiang ZM, Barna C, Litoiu M (2016) A framework to evaluate the effectiveness of different load testing analysis techniques. In: 2016 IEEE International conference on software testing, verification and validation, ICST 2016, chicago, IL, USA, April 11-15, 2016, pp 22\u201332","DOI":"10.1109\/ICST.2016.9"},{"issue":"1","key":"9866_CR21","doi-asserted-by":"publisher","first-page":"4","DOI":"10.1002\/stvr.1573","volume":"26","author":"S Ghaith","year":"2016","unstructured":"Ghaith S, Wang M, Perry P, Jiang ZM, O\u2019Sullivan P, Murphy J (2016) Anomaly detection in performance regression testing by transaction profile estimation. Softw Test Verif Reliab 26(1):4\u201339","journal-title":"Softw Test Verif Reliab"},{"key":"9866_CR22","unstructured":"Gong Z, Gu X, Wilkes J (2010) PRESS: Predictive elastic resource scaling for cloud systems. In: Proceedings of the 6th international conference on network and service management, CNSM 2010, Niagara Falls, Canada, October 25-29, 2010, pp 9\u201316"},{"issue":"1","key":"9866_CR23","doi-asserted-by":"publisher","first-page":"68","DOI":"10.1145\/1496091.1496103","volume":"39","author":"A Greenberg","year":"2008","unstructured":"Greenberg A, Hamilton J, Maltz DA, Patel P (2008) The cost of a cloud: Research problems in data center networks. SIGCOMM Comput Commun Rev 39(1):68\u201373","journal-title":"SIGCOMM Comput Commun Rev"},{"key":"9866_CR24","doi-asserted-by":"crossref","unstructured":"Guo J, Czarnecki K, Apel S, Siegmund N, Wasowski A (2013) Variability-aware performance prediction: a statistical learning approach. In: 2013 28Th IEEE\/ACM international conference on automated software engineering, ASE 2013, silicon valley, CA, USA, November 11-15, 2013, pp 301\u2013311","DOI":"10.1109\/ASE.2013.6693089"},{"issue":"3","key":"9866_CR25","doi-asserted-by":"publisher","first-page":"1826","DOI":"10.1007\/s10664-017-9573-6","volume":"23","author":"J Guo","year":"2018","unstructured":"Guo J, Yang D, Siegmund N, Apel S, Sarkar A, Valov P, Czarnecki K, Wasowski A, Yu H (2018) Data-efficient performance learning for configurable systems. Empir Softw Eng 23(3):1826\u20131867","journal-title":"Empir Softw Eng"},{"key":"9866_CR26","doi-asserted-by":"crossref","unstructured":"He S, Lin Q, Lou J, Zhang H, Lyu MR, Zhang D (2018) Identifying impactful service system problems via log analysis. In: Proceedings of the 2018 ACM joint meeting on european software engineering conference and symposium on the foundations of software engineering, ESEC\/SIGSOFT FSE 2018, Lake Buena Vista, FL, USA, November 04-09, 2018, pp 60-70","DOI":"10.1145\/3236024.3236083"},{"issue":"1","key":"9866_CR27","doi-asserted-by":"publisher","first-page":"4:1","DOI":"10.1145\/2791120","volume":"48","author":"O Ibidunmoye","year":"2015","unstructured":"Ibidunmoye O, Hern\u00e1ndez-rodriguez F, Elmroth E (2015) Performance anomaly detection and bottleneck identification. ACM Comput Surv 48 (1):4:1\u20134:35","journal-title":"ACM Comput Surv"},{"issue":"11","key":"9866_CR28","doi-asserted-by":"publisher","first-page":"1091","DOI":"10.1109\/TSE.2015.2445340","volume":"41","author":"ZM Jiang","year":"2015","unstructured":"Jiang ZM, Hassan AE (2015) A survey on load testing of large-scale software systems. IEEE Trans Software Eng 41(11):1091\u20131118","journal-title":"IEEE Trans Software Eng"},{"key":"9866_CR29","doi-asserted-by":"crossref","unstructured":"Jiang ZM, Hassan AE, Hamann G, Flora P (2009) Automated performance analysis of load tests. In: 25Th IEEE international conference on software maintenance (ICSM 2009), September 20-26, 2009, Edmonton, Alberta, Canada, pp 125\u2013134","DOI":"10.1109\/ICSM.2009.5306331"},{"key":"9866_CR30","doi-asserted-by":"crossref","unstructured":"Krasic C, Sinha A, Kirsh L (2007) Priority-progress CPU adaptation for elastic real-time applications. In: Zimmermann R, Griwodz C (eds) Multimedia computing and networking 2007, vol 6504, International Society for Optics and Photonics, SPIE, pp 172\u2013183","DOI":"10.1117\/12.705735"},{"issue":"11","key":"9866_CR31","doi-asserted-by":"publisher","first-page":"868","DOI":"10.1109\/TSE.2006.106","volume":"32","author":"D Krishnamurthy","year":"2006","unstructured":"Krishnamurthy D, Rolia JA, Majumdar S (2006) A synthetic workload generation technique for stress testing session-based systems. IEEE Trans Software Eng 32(11):868\u2013882","journal-title":"IEEE Trans Software Eng"},{"key":"9866_CR32","unstructured":"Lazowska ED, Zahorjan J, Graham GS, Sevcik KC (1984) Quantitative system performance - computer system analysis using queueing network models. Prentice Hall"},{"key":"9866_CR33","doi-asserted-by":"crossref","unstructured":"Lim M, Lou J, Zhang H, Fu Q, Teoh ABJ, Lin Q, Ding R, Zhang D (2014) Identifying recurrent and unknown performance issues. In: 2014 IEEE International conference on data mining, ICDM 2014, Shenzhen, China, December 14-17, 2014, pp 320\u2013329","DOI":"10.1109\/ICDM.2014.96"},{"key":"9866_CR34","doi-asserted-by":"crossref","unstructured":"Malik H, Jiang ZM, Adams B, Hassan AE, Flora P, Hamann G (2010) Automatic comparison of load tests to support the performance analysis of large enterprise systems. In: 14Th european conference on software maintenance and reengineering, CSMR 2010, 15-18 March 2010, Madrid, Spain, pp 222\u2013231","DOI":"10.1109\/CSMR.2010.39"},{"key":"9866_CR35","doi-asserted-by":"crossref","unstructured":"Malik H, Hemmati H, Hassan AE (2013) Automatic detection of performance deviations in the load testing of large scale systems. In: 35Th international conference on software engineering, ICSE \u201913, san francisco, CA, USA, May 18-26, 2013, pp 1012\u20131021","DOI":"10.1109\/ICSE.2013.6606651"},{"issue":"1","key":"9866_CR36","doi-asserted-by":"publisher","first-page":"13","DOI":"10.20982\/tqmp.04.1.p013","volume":"4","author":"N Nachar","year":"2008","unstructured":"Nachar N et al (2008) The mann-whitney u: A test for assessing whether two independent samples come from the same distribution. Tutorials in Quantitative Methods for Psychology 4(1):13\u201320","journal-title":"Tutorials in Quantitative Methods for Psychology"},{"key":"9866_CR37","doi-asserted-by":"crossref","unstructured":"Nguyen THD, Adams B, Jiang ZM, Hassan AE, Nasser MN, Flora P (2011) Automated verification of load tests using control charts. In: 18Th asia pacific software engineering conference, APSEC 2011, ho chi minh, Vietnam, December 5-8, 2011, pp 282\u2013289","DOI":"10.1109\/APSEC.2011.59"},{"key":"9866_CR38","doi-asserted-by":"crossref","unstructured":"Nguyen THD, Adams B, Jiang ZM, Hassan AE, Nasser MN, Flora P (2012) Automated detection of performance regressions using statistical process control techniques. In: Third joint WOSP\/SIPEW international conference on performance engineering, ICPE\u201912, boston, MA, USA - April 22 - 25, 2012, pp 299\u2013310","DOI":"10.1145\/2188286.2188344"},{"key":"9866_CR39","unstructured":"Romano J, Kromrey JD, Coraggio J, Skowronek J (2006) Appropriate statistics for ordinal level data: Should we really be using t-test and cohen\u2019sd for evaluating group differences on the nsse and other surveys. In: annual meeting of the Florida association of institutional research, pp 1\u201333"},{"key":"9866_CR40","unstructured":"Sato D (2014) Canary release. MartinFowler. com"},{"key":"9866_CR41","doi-asserted-by":"crossref","unstructured":"Shang W, Hassan AE, Nasser MN, Flora P (2015) Automated detection of performance regressions using regression models on clustered performance counters. In: Proceedings of the 6th ACM\/SPEC international conference on performance engineering, Austin, TX, USA, January 31 - February 4, 2015, pp 15\u201326","DOI":"10.1145\/2668930.2688052"},{"issue":"3","key":"9866_CR42","doi-asserted-by":"publisher","first-page":"279","DOI":"10.4300\/JGME-D-12-00156.1","volume":"4","author":"GM Sullivan","year":"2012","unstructured":"Sullivan GM, Feinn R (2012) Using effect size\u2014or why the p value is not enough. Journal of Graduate Medical Education 4(3):279\u2013282","journal-title":"Journal of Graduate Medical Education"},{"key":"9866_CR43","doi-asserted-by":"crossref","unstructured":"Syer MD, Jiang ZM, Nagappan M, Hassan AE, Nasser MN, Flora P (2013) Leveraging performance counters and execution logs to diagnose memory-related performance issues. In: 2013 IEEE International conference on software maintenance, eindhoven, The Netherlands, September 22-28, 2013, pp 110\u2013119","DOI":"10.1109\/ICSM.2013.22"},{"key":"9866_CR44","doi-asserted-by":"crossref","unstructured":"Syer MD, Jiang ZM, Nagappan M, Hassan AE, Nasser MN, Flora P (2014) Continuous validation of load test suites. In: ACM\/SPEC International conference on performance engineering, ICPE\u201914, dublin, ireland, March 22-26, 2014, pp 259\u2013270","DOI":"10.1145\/2568088.2568101"},{"issue":"1","key":"9866_CR45","doi-asserted-by":"publisher","first-page":"189","DOI":"10.1007\/s10515-016-0196-8","volume":"24","author":"MD Syer","year":"2017","unstructured":"Syer MD, Shang W, Jiang ZM, Hassan AE (2017) Continuous validation of performance test workloads. Autom Softw Eng 24(1):189\u2013231","journal-title":"Autom Softw Eng"},{"key":"9866_CR46","unstructured":"Syncsort (2018) White paper: Assessing the financial impact of downtime"},{"key":"9866_CR47","doi-asserted-by":"crossref","unstructured":"Tan J, Kavulya S, Gandhi R, Narasimhan P (2010) Visual, log-based causal tracing for performance debugging of mapreduce systems. In: 2010 International conference on distributed computing systems, ICDCS 2010, genova, italy, june 21-25, 2010, pp 795\u2013806","DOI":"10.1109\/ICDCS.2010.63"},{"key":"9866_CR48","doi-asserted-by":"crossref","unstructured":"Valov P, Petkovich J, Guo J, Fischmeister S, Czarnecki K (2017) Transferring performance prediction models across different hardware platforms. In: Proceedings of the 8th ACM\/SPEC on international conference on performance engineering, ICPE 2017, L\u2019Aquila, Italy, April 22-26, 2017, pp 39\u201350","DOI":"10.1145\/3030207.3030216"},{"issue":"12","key":"9866_CR49","doi-asserted-by":"publisher","first-page":"1147","DOI":"10.1109\/32.888628","volume":"26","author":"EJ Weyuker","year":"2000","unstructured":"Weyuker EJ, Vokolos FI (2000) Experience with performance testing of software systems: Issues, an approach, and case study. IEEE Trans Software Eng 26(12):1147\u20131156","journal-title":"IEEE Trans Software Eng"},{"key":"9866_CR50","doi-asserted-by":"crossref","unstructured":"Xiong P, Pu C, Zhu X, Griffith R (2013) vperfguard: an automated model-driven framework for application performance diagnosis in consolidated cloud environments. In: ACM\/SPEC international conference on performance engineering, ICPE\u201913, Prague, Czech Republic, pp 271\u2013282","DOI":"10.1145\/2479871.2479909"},{"key":"9866_CR51","doi-asserted-by":"crossref","unstructured":"Xu W, Huang L, Fox A, Patterson DA, Jordan MI (2009) Detecting large-scale system problems by mining console logs. In: Proceedings of the 22nd ACM Symposium on Operating Systems Principles 2009, SOSP 2009, Big Sky, Montana, USA, October 11-14, 2009, pp 117\u2013132","DOI":"10.1145\/1629575.1629587"},{"key":"9866_CR52","doi-asserted-by":"crossref","unstructured":"Xu Y, Chen N, Fernandez A, Sinno O, Bhasin A (2015) From infrastructure to culture: A\/b testing challenges in large scale social networks. In: Proceedings of the 21th ACM SIGKDD international conference on knowledge discovery and data mining, KDD \u201915. Association for Computing Machinery, New York, pp 2227\u20132236","DOI":"10.1145\/2783258.2788602"},{"key":"9866_CR53","unstructured":"Yadwadkar NJ, Bhattacharyya C, Gopinath K, Niranjan T, Susarla S (2010) Discovery of application workloads from network file traces. In: 8Th USENIX conference on file and storage technologies, san jose, CA, USA, February 23-26, 2010, pp 183\u2013196"},{"key":"9866_CR54","doi-asserted-by":"crossref","unstructured":"Yao KB, de P\u00e1dua G, Shang W, Sporea S, Toma A, Sajedi S (2018) Log4perf: Suggesting logging locations for web-based systems\u2019 performance monitoring. In: Proceedings of the 2018 ACM\/SPEC international conference on performance engineering, ICPE \u201918, pp 127\u2013138","DOI":"10.1145\/3184407.3184416"},{"key":"9866_CR55","doi-asserted-by":"crossref","unstructured":"Zhou M, Chen J, Hu H, Yu J, Li Z, Hu H (2019) Deeptle: Learning code-level features to predict code performance before it runs. In: 2019 26th Asia-Pacific software engineering conference (APSEC), pp 252\u2013259","DOI":"10.1109\/APSEC48747.2019.00042"}],"container-title":["Empirical Software Engineering"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09866-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10664-020-09866-z\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10664-020-09866-z.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2021,8,28]],"date-time":"2021-08-28T00:16:34Z","timestamp":1630109794000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10664-020-09866-z"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,8,28]]},"references-count":55,"journal-issue":{"issue":"5","published-print":{"date-parts":[[2020,9]]}},"alternative-id":["9866"],"URL":"https:\/\/doi.org\/10.1007\/s10664-020-09866-z","relation":{},"ISSN":["1382-3256","1573-7616"],"issn-type":[{"value":"1382-3256","type":"print"},{"value":"1573-7616","type":"electronic"}],"subject":[],"published":{"date-parts":[[2020,8,28]]},"assertion":[{"value":"28 August 2020","order":1,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}}]}}