{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T01:59:52Z","timestamp":1777341592742,"version":"3.51.4"},"publisher-location":"New York, NY, USA","reference-count":47,"publisher":"ACM","license":[{"start":{"date-parts":[[2016,7,7]],"date-time":"2016-07-07T00:00:00Z","timestamp":1467849600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2016,7,7]]},"DOI":"10.1145\/2911451.2911507","type":"proceedings-article","created":{"date-parts":[[2016,7,8]],"date-time":"2016-07-08T15:03:00Z","timestamp":1467990180000},"page":"463-472","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":78,"title":["When does Relevance Mean Usefulness and User Satisfaction in Web Search?"],"prefix":"10.1145","author":[{"given":"Jiaxin","family":"Mao","sequence":"first","affiliation":[{"name":"Tsinghua University, Beijing, China"}]},{"given":"Yiqun","family":"Liu","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}]},{"given":"Ke","family":"Zhou","sequence":"additional","affiliation":[{"name":"Yahoo! Research, London, England UK"}]},{"given":"Jian-Yun","family":"Nie","sequence":"additional","affiliation":[{"name":"Universit\u00e9 de Montr\u00e9al, Montreal, Canada"}]},{"given":"Jingtao","family":"Song","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}]},{"given":"Min","family":"Zhang","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}]},{"given":"Shaoping","family":"Ma","sequence":"additional","affiliation":[{"name":"Tsinghua University, Beijing, China"}]},{"given":"Jiashen","family":"Sun","sequence":"additional","affiliation":[{"name":"Samsung R&D Institute China - Beijing, Beijing, China"}]},{"given":"Hengliang","family":"Luo","sequence":"additional","affiliation":[{"name":"Samsung R&D Institute China - Beijing, Beijing, China"}]}],"member":"320","published-online":{"date-parts":[[2016,7,7]]},"reference":[{"key":"e_1_3_2_1_1_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277902"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390450"},{"key":"e_1_3_2_1_3_1","volume-title":"Proc. SIGIR '08 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments.","author":"Belkin N. J.","year":"2008","unstructured":"N. J. Belkin, M. Cole, and R. Bierig. Is relevance the right criterion for evaluating interactive information retrieval. In Proc. SIGIR '08 Workshop on Beyond Binary Relevance: Preferences, Diversity, and Set-Level Judgments., 2008."},{"key":"e_1_3_2_1_4_1","doi-asserted-by":"publisher","DOI":"10.1145\/1571941.1571955"},{"key":"e_1_3_2_1_5_1","doi-asserted-by":"publisher","DOI":"10.1145\/290941.291025"},{"key":"e_1_3_2_1_6_1","volume-title":"Overview of the trec 2014 session track","author":"Carterette B.","year":"2013","unstructured":"B. Carterette, E. Kanoulas, M. Hall, and P. Clough. Overview of the trec 2014 session track. 2013."},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.1145\/1645953.1646033"},{"key":"e_1_3_2_1_8_1","doi-asserted-by":"publisher","DOI":"10.1145\/1526709.1526711"},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/1390334.1390446"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1108\/eb050097"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1037\/h0026256"},{"key":"e_1_3_2_1_12_1","first-page":"53","volume-title":"Applied multiple regression\/correlation analysis for the behavioral sciences","author":"Cohen J.","year":"1975","unstructured":"J. Cohen and P. Cohen. Applied multiple regression\/correlation analysis for the behavioral sciences, chapter 2, pages 53--54. Lawrence Erlbaum Associates, 1975."},{"key":"e_1_3_2_1_13_1","first-page":"1","volume-title":"Proc. HCIR","author":"Cole M.","year":"2009","unstructured":"M. Cole, J. Liu, N. Belkin, R. Bierig, J. Gwizdka, C. Liu, J. Zhang, and X. Zhang. Usefulness as the criterion for evaluation of interactive information retrieval. Proc. HCIR, pages 1--4, 2009."},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.4630240204"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242651"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1145\/1059981.1059982"},{"key":"e_1_3_2_1_17_1","first-page":"1189","volume-title":"Greedy function approximation: a gradient boosting machine. Annals of statistics","author":"Friedman J. H.","year":"2001","unstructured":"J. H. Friedman. Greedy function approximation: a gradient boosting machine. Annals of statistics, pages 1189--1232, 2001."},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/1718487.1718515"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556221"},{"key":"e_1_3_2_1_20_1","doi-asserted-by":"publisher","DOI":"10.1145\/1277741.1277839"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/582415.582418"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.5555\/1793274.1793280"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/2684822.2685319"},{"key":"e_1_3_2_1_24_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.ipm.2006.07.021"},{"key":"e_1_3_2_1_25_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010056"},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.10137"},{"key":"e_1_3_2_1_27_1","doi-asserted-by":"publisher","DOI":"10.1561\/1500000012"},{"key":"e_1_3_2_1_28_1","volume-title":"Effects of rank and precision of search results on users' evaluations of system performance","author":"Kelly D.","year":"2007","unstructured":"D. Kelly, X. Fu, and C. Shah. Effects of rank and precision of search results on users' evaluations of system performance. University of North Carolina, 2007."},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/2556195.2556220"},{"key":"e_1_3_2_1_30_1","first-page":"159","volume-title":"The measurement of observer agreement for categorical data. biometrics","author":"Landis J. R.","year":"1977","unstructured":"J. R. Landis and G. G. Koch. The measurement of observer agreement for categorical data. biometrics, pages 159--174, 1977."},{"issue":"1","key":"e_1_3_2_1_31_1","first-page":"1","article-title":"Using dwell time as an implicit measure of usefulness in different task types","volume":"48","author":"Liu C.","year":"2011","unstructured":"C. Liu, J. Liu, N. Belkin, M. Cole, and J. Gwizdka. Using dwell time as an implicit measure of usefulness in different task types. Proc. ASIST, 48(1):1--4, 2011.","journal-title":"Proc. ASIST"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767721"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/2505515.2507665"},{"key":"e_1_3_2_1_34_1","doi-asserted-by":"publisher","DOI":"10.1145\/2806416.2806488"},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2009916.2010055"},{"key":"e_1_3_2_1_36_1","doi-asserted-by":"publisher","DOI":"10.1145\/1835449.1835542"},{"key":"e_1_3_2_1_37_1","first-page":"201","volume-title":"the Second Conference on Conceptions of Library and Information Science","volume":"1","author":"Saracevic T.","year":"1996","unstructured":"T. Saracevic. Relevance reconsidered. In the Second Conference on Conceptions of Library and Information Science, volume 1, pages 201--218, 1996."},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/2484028.2484075"},{"key":"e_1_3_2_1_39_1","doi-asserted-by":"publisher","DOI":"10.1145\/1076034.1076111"},{"key":"e_1_3_2_1_40_1","doi-asserted-by":"publisher","DOI":"10.1145\/1148170.1148176"},{"key":"e_1_3_2_1_41_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.20046"},{"key":"e_1_3_2_1_42_1","doi-asserted-by":"publisher","DOI":"10.1002\/asi.22948"},{"key":"e_1_3_2_1_43_1","doi-asserted-by":"publisher","DOI":"10.5555\/648264.753539"},{"key":"e_1_3_2_1_44_1","volume-title":"Overview of trec","author":"Voorhees E. M.","year":"2001","unstructured":"E. M. Voorhees and D. Harman. Overview of trec 2001. In Trec, 2001."},{"key":"e_1_3_2_1_45_1","doi-asserted-by":"publisher","DOI":"10.1145\/1242572.1242576"},{"key":"e_1_3_2_1_46_1","doi-asserted-by":"publisher","DOI":"10.1145\/1183614.1183659"},{"key":"e_1_3_2_1_47_1","doi-asserted-by":"publisher","DOI":"10.1145\/2661829.2661953"}],"event":{"name":"SIGIR '16: The 39th International ACM SIGIR conference on research and development in Information Retrieval","location":"Pisa Italy","acronym":"SIGIR '16","sponsor":["SIGIR ACM Special Interest Group on Information Retrieval"]},"container-title":["Proceedings of the 39th International ACM SIGIR conference on Research and Development in Information Retrieval"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2911451.2911507","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/2911451.2911507","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,18]],"date-time":"2025-06-18T04:55:02Z","timestamp":1750222502000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/2911451.2911507"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2016,7,7]]},"references-count":47,"alternative-id":["10.1145\/2911451.2911507","10.1145\/2911451"],"URL":"https:\/\/doi.org\/10.1145\/2911451.2911507","relation":{},"subject":[],"published":{"date-parts":[[2016,7,7]]},"assertion":[{"value":"2016-07-07","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}