{"corpus_id":268510462,"paper_sha":"d749810bb631947fbd9d21fa312a8bd9e8496ceb","doi":"10.48550/arXiv.2403.09738","arxiv_id":"2403.09738","pmid":null,"pmcid":null,"mag_id":null,"dblp_id":"journals/corr/abs-2403-09738","acl_id":"2024.naacl-long.83","title":"Evaluating Large Language Models as Generative User Simulators for Conversational Recommendation","year":2024,"publication_date":"2024-03-13","venue":"North American Chapter of the Association for Computational Linguistics","journal":{"name":"ArXiv","pages":null,"volume":"abs/2403.09738"},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle","Conference"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":38,"citation_count":44,"influential_citation_count":2,"is_open_access":false,"arxiv_categories":["cs.CL","cs.AI","cs.IR"],"arxiv_license":"http://creativecommons.org/licenses/by/4.0/","arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":null,"s2_open_access_landing_url":null,"s2_open_access_license":null,"s2_open_access_status":null,"pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"Synthetic users are cost-effective proxies for real users in the evaluation of conversational recommender systems. Large language models show promise in simulating human-like behavior, raising the question of their ability to represent a diverse population of users. We introduce a new protocol to measure the degree to which language models can accurately emulate human behavior in conversational recommendation. This protocol is comprised of five tasks, each designed to evaluate a key property that a synthetic user should exhibit: choosing which items to talk about, expressing binary preferences, expressing open-ended preferences, requesting recommendations, and giving feedback. Through evaluation of baseline simulators, we demonstrate these tasks effectively reveal deviations of language models from human behavior, and offer insights on how to reduce the deviations with model selection and prompting strategies.","claims":[{"public_id":"cl_9e0e3676df27727221610f6fce34b9cd","status":"active","text":"A new protocol is introduced to measure how accurately language models emulate human behavior in conversational recommendation.","confidence":0.98,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_9e0e3676df27727221610f6fce34b9cd"},{"public_id":"cl_f466ddc586881b3e8a9374fec34cbd57","status":"active","text":"Evaluations of baseline simulators show that the tasks reveal deviations of language models from human behavior.","confidence":0.95,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_f466ddc586881b3e8a9374fec34cbd57"},{"public_id":"cl_9347d32c4a09fd110812eff87e626e54","status":"active","text":"Model selection and prompting strategies help reduce the observed deviations between language models and human behavior.","confidence":0.9,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_9347d32c4a09fd110812eff87e626e54"},{"public_id":"cl_1807b393d970a19b6ac4618162285131","status":"active","text":"The protocol consists of five tasks covering item selection, binary preferences, open-ended preferences, recommendation requests, and feedback generation.","confidence":0.99,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_1807b393d970a19b6ac4618162285131"}],"concepts":[{"public_id":"co_0c41353137ca2eb336d24511517d5477","status":"active","name":"conversational recommendation","description":"A recommendation setting in which users interact with a system through dialogue.","types":["task setting"],"aliases":["conversational recommender systems","CRS"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_0c41353137ca2eb336d24511517d5477"},{"public_id":"co_0c82efe2d3735cb2d30a21016179b17c","status":"active","name":"item selection","description":"The task of choosing which items to talk about in a conversation.","types":["task"],"aliases":["choosing which items to talk about"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_0c82efe2d3735cb2d30a21016179b17c"},{"public_id":"co_15eeaf36ed1398fac4b692f917596ce9","status":"active","name":"feedback generation","description":"The task of providing feedback about recommended items or system responses.","types":["task"],"aliases":["giving feedback"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_15eeaf36ed1398fac4b692f917596ce9"},{"public_id":"co_3c8ca37c55d30820343196b9daba6dab","status":"active","name":"prompting strategies","description":"Prompt designs used to steer language model behavior in simulation tasks.","types":["strategy"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_3c8ca37c55d30820343196b9daba6dab"},{"public_id":"co_42e76110d9575831d19604ffc0639641","status":"active","name":"language models","description":"Large language models used here as synthetic user simulators in conversational recommendation.","types":["model"],"aliases":["LLMs"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_42e76110d9575831d19604ffc0639641"},{"public_id":"co_5de97cd3052e7d0d33029e3b439dc424","status":"active","name":"human behavior","description":"The conversational behaviors of real users that synthetic users aim to emulate.","types":["behavior"],"aliases":["human-like behavior"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_5de97cd3052e7d0d33029e3b439dc424"},{"public_id":"co_602af3bb3e47d421444657f18aa3d84e","status":"active","name":"binary preferences","description":"The task of expressing preferences in a yes/no or like/dislike form.","types":["task"],"aliases":["expressing binary preferences"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_602af3bb3e47d421444657f18aa3d84e"},{"public_id":"co_aec7e290af070ff8c916df6fb5c39465","status":"active","name":"protocol","description":"An evaluation procedure designed to measure how well synthetic users emulate human behavior.","types":["evaluation protocol"],"aliases":["new protocol"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_aec7e290af070ff8c916df6fb5c39465"},{"public_id":"co_b205e058f377903c41999748ddc98a32","status":"active","name":"baseline simulators","description":"Reference user simulators evaluated as comparison systems in the protocol.","types":["baseline systems"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_b205e058f377903c41999748ddc98a32"},{"public_id":"co_d1dd1bfb7f323146c20cf1c4d4c64880","status":"active","name":"recommendation requests","description":"The task of asking for recommendations during a dialogue.","types":["task"],"aliases":["requesting recommendations"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_d1dd1bfb7f323146c20cf1c4d4c64880"},{"public_id":"co_de4e5b9aa7e5db2b11c73498f7a14265","status":"active","name":"five tasks","description":"The set of five evaluation tasks used in the protocol to assess synthetic user behavior.","types":["task set"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_de4e5b9aa7e5db2b11c73498f7a14265"},{"public_id":"co_efbc038daaa775c4aebcf0ab5d1c81f5","status":"active","name":"open-ended preferences","description":"The task of expressing preferences in free-form language.","types":["task"],"aliases":["expressing open-ended preferences"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_efbc038daaa775c4aebcf0ab5d1c81f5"},{"public_id":"co_f20339c302a03167ff4675ddc51b3242","status":"active","name":"model selection","description":"Choosing among language models to improve synthetic user simulation performance.","types":["strategy"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_f20339c302a03167ff4675ddc51b3242"}],"external_ids":{"DOI":"10.48550/arXiv.2403.09738","ArXiv":"2403.09738","PubMed":null,"PubMedCentral":null,"MAG":null,"DBLP":"journals/corr/abs-2403-09738","ACL":"2024.naacl-long.83"},"open_access":{"is_open_access":true,"pdf_url":"https://arxiv.org/pdf/2403.09738","landing_url":"https://arxiv.org/abs/2403.09738","source":"arxiv","pdf_url_source":"derived_arxiv","license":"http://creativecommons.org/licenses/by/4.0/","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":true,"full_text_source":"arxiv","count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":636961,"paper_uid":"9f35353b-592f-41c7-9244-15e0a0e0c0cb","canonical_identity":{"paper_id":636961,"paper_uid":"9f35353b-592f-41c7-9244-15e0a0e0c0cb","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/268510462"}