{"corpus_id":244462814,"paper_sha":"79fe917aaa8f50fb82cae814035bd4cb7edcea33","doi":"10.1609/aaai.v36i11.21562","arxiv_id":"2111.10058","pmid":null,"pmcid":null,"mag_id":null,"dblp_id":"journals/corr/abs-2111-10058","acl_id":null,"title":"DeepQR: Neural-based Quality Ratings for Learnersourced Multiple-Choice Questions","year":2021,"publication_date":"2021-11-19","venue":"AAAI Conference on Artificial Intelligence","journal":{"name":"ArXiv","pages":null,"volume":"abs/2111.10058"},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle","Conference"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":52,"citation_count":16,"influential_citation_count":0,"is_open_access":true,"arxiv_categories":["cs.CL","cs.AI","cs.LG"],"arxiv_license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/21562/21311","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/79fe917aaa8f50fb82cae814035bd4cb7edcea33","s2_open_access_license":null,"s2_open_access_status":"GOLD","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"Automated question quality rating (AQQR) aims to evaluate question quality through computational means, thereby addressing emerging challenges in online learnersourced question repositories. Existing methods for AQQR rely solely on explicitly-defined criteria such as readability and word count, while not fully utilising the power of state-of-the-art deep-learning techniques. We propose DeepQR, a novel neural-network model for AQQR that is trained using multiple-choice-question (MCQ) datasets collected from PeerWise, a widely-used learnersourcing platform. Along with designing DeepQR, we investigate models based on explicitly-defined features, or semantic features, or both. We also introduce a self-attention mechanism to capture semantic correlations between MCQ components, and a contrastive-learning approach to acquire question representations using quality ratings. Extensive experiments on datasets collected from eight university-level courses illustrate that DeepQR has superior performance over six comparative models.","claims":[{"public_id":"cl_f5974c7f43aa5e016d9374550e56cade","status":"active","text":"DeepQR achieves superior performance over six comparative models on datasets from eight university-level courses.","confidence":0.95,"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_f5974c7f43aa5e016d9374550e56cade"},{"public_id":"cl_35b03da4d97ba8b2795cff76988a44a5","status":"active","text":"DeepQR employs a contrastive-learning approach to acquire question representations using quality ratings.","confidence":0.9,"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_35b03da4d97ba8b2795cff76988a44a5"},{"public_id":"cl_8af81f5732b1839c49d687fdf426ac72","status":"active","text":"DeepQR is a novel neural-network model for automated question quality rating trained on multiple-choice-question datasets from PeerWise.","confidence":0.95,"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_8af81f5732b1839c49d687fdf426ac72"},{"public_id":"cl_6f98a97ef03cd759e19edc4b370372d6","status":"active","text":"DeepQR uses a self-attention mechanism to capture semantic correlations between MCQ components.","confidence":0.9,"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_6f98a97ef03cd759e19edc4b370372d6"}],"concepts":[{"public_id":"co_04ebe2bad3ba4da7f7c077761679bd63","status":"active","name":"PeerWise","description":"A widely-used learnersourcing platform from which multiple-choice question datasets were collected.","types":["platform"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_04ebe2bad3ba4da7f7c077761679bd63"},{"public_id":"co_36db6b42333c92095112df35db563b88","status":"active","name":"datasets from eight university-level courses","description":"The datasets collected from eight university courses used for evaluation.","types":["dataset"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_36db6b42333c92095112df35db563b88"},{"public_id":"co_43387d540cef538ec7f1e09d1953c414","status":"active","name":"DeepQR","description":"A neural-network model for automated question quality rating using multiple-choice-question datasets from PeerWise.","types":["method"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_43387d540cef538ec7f1e09d1953c414"},{"public_id":"co_5d927219408c76524f4526f957d6f24e","status":"active","name":"MCQ components","description":"The components of a multiple-choice question, such as the stem and answer options.","types":["component"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_5d927219408c76524f4526f957d6f24e"},{"public_id":"co_611290e6f2d2259e84dfa89461237440","status":"active","name":"question representations","description":"The encoded representations of questions learned by DeepQR for quality rating.","types":["representation"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_611290e6f2d2259e84dfa89461237440"},{"public_id":"co_731f637f20e62bc5e1295f855dc89f4e","status":"active","name":"automated question quality rating","description":"The task of automatically evaluating the quality of questions.","types":["task"],"aliases":["AQQR"],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_731f637f20e62bc5e1295f855dc89f4e"},{"public_id":"co_a6306e5a672f4db40e39c94463e239c7","status":"active","name":"self-attention mechanism","description":"A mechanism used in DeepQR to capture semantic correlations between MCQ components.","types":["technique"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_a6306e5a672f4db40e39c94463e239c7"},{"public_id":"co_a6459f728f696e3ee4bac8ed5133551a","status":"active","name":"quality ratings","description":"The ratings or labels indicating the quality of questions.","types":["data"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_a6459f728f696e3ee4bac8ed5133551a"},{"public_id":"co_d67998f6921da72425c260e1cd79010b","status":"active","name":"comparative models","description":"The six models used as baselines for comparison against DeepQR.","types":["baseline"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_d67998f6921da72425c260e1cd79010b"},{"public_id":"co_e77d7e098694638a71dd1ccb0cc089d8","status":"active","name":"contrastive-learning approach","description":"A learning approach that uses quality ratings to acquire question representations via contrastive objectives.","types":["method"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_e77d7e098694638a71dd1ccb0cc089d8"},{"public_id":"co_e8a06a6e24006bda362c6d93ce0f2c75","status":"active","name":"superior performance","description":"The result that DeepQR outperforms six comparative models on evaluation datasets.","types":["result"],"aliases":[],"contributors":[{"id":1165,"public_id":"ezd9qvkvax","public_label":"The Reverser‮ (ezd9qvkvax)","roles":["extraction"],"url":"https://sah.borca.ai/u/ezd9qvkvax"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_e8a06a6e24006bda362c6d93ce0f2c75"}],"external_ids":{"DOI":"10.1609/aaai.v36i11.21562","ArXiv":"2111.10058","PubMed":null,"PubMedCentral":null,"MAG":null,"DBLP":"journals/corr/abs-2111-10058","ACL":null},"open_access":{"is_open_access":true,"pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/21562/21311","landing_url":"https://www.semanticscholar.org/paper/79fe917aaa8f50fb82cae814035bd4cb7edcea33","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":null,"status":"GOLD","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":true,"full_text_source":"arxiv","count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":632188,"paper_uid":"82cf641a-f152-40e2-a7e6-51411e80c439","canonical_identity":{"paper_id":632188,"paper_uid":"82cf641a-f152-40e2-a7e6-51411e80c439","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/244462814"}