{"corpus_id":221819581,"paper_sha":"9e67b9758520e49016ab66bafb974d2e1ed762d1","doi":"10.18653/v1/2020.emnlp-main.213","arxiv_id":"2009.09025","pmid":null,"pmcid":null,"mag_id":3087597081,"dblp_id":"journals/corr/abs-2009-09025","acl_id":"2020.emnlp-main.213","title":"COMET: A Neural Framework for MT Evaluation","year":2020,"publication_date":"2020-09-18","venue":"Conference on Empirical Methods in Natural Language Processing","journal":{"name":"ArXiv","pages":null,"volume":"abs/2009.09025"},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle","Conference"],"pubmed_pub_types":null,"s2_fields_of_study":["Linguistics","Computer Science"],"reference_count":59,"citation_count":1439,"influential_citation_count":407,"is_open_access":true,"arxiv_categories":["cs.CL"],"arxiv_license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://www.aclweb.org/anthology/2020.emnlp-main.213.pdf","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/9e67b9758520e49016ab66bafb974d2e1ed762d1","s2_open_access_license":"CCBY","s2_open_access_status":"HYBRID","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"We present COMET, a neural framework for training multilingual machine translation evaluation models which obtains new state-of-the-art levels of correlation with human judgements. Our framework leverages recent breakthroughs in cross-lingual pretrained language modeling resulting in highly multilingual and adaptable MT evaluation models that exploit information from both the source input and a target-language reference translation in order to more accurately predict MT quality. To showcase our framework, we train three models with different types of human judgements: Direct Assessments, Human-mediated Translation Edit Rate and Multidimensional Quality Metrics. Our models achieve new state-of-the-art performance on the WMT 2019 Metrics shared task and demonstrate robustness to high-performing systems.","claims":[{"public_id":"cl_785d2a49c47074cefe462a66c16bee30","status":"active","text":"COMET is a neural framework for training multilingual machine translation evaluation models that achieves new state-of-the-art levels of correlation with human judgements.","confidence":0.95,"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_785d2a49c47074cefe462a66c16bee30"},{"public_id":"cl_a8cfb2daabb171d6e389be7d21d31840","status":"active","text":"The COMET models achieve new state-of-the-art performance on the WMT 2019 Metrics shared task and demonstrate robustness to high-performing systems.","confidence":0.9,"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_a8cfb2daabb171d6e389be7d21d31840"},{"public_id":"cl_8c155b5efa5d94b8ff7f3fa151d6d681","status":"active","text":"The framework leverages cross-lingual pretrained language modeling to produce highly multilingual and adaptable MT evaluation models that exploit information from both the source input and a target-language reference translation.","confidence":0.9,"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_8c155b5efa5d94b8ff7f3fa151d6d681"},{"public_id":"cl_12089428680837f647ec66546cdefac9","status":"active","text":"Three models are trained with different types of human judgements: Direct Assessments, Human-mediated Translation Edit Rate, and Multidimensional Quality Metrics.","confidence":0.95,"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_12089428680837f647ec66546cdefac9"}],"concepts":[{"public_id":"co_18ec83a5b2bc9d8ac37f9214cb89389b","status":"active","name":"robustness to high-performing systems","description":"The property of COMET models maintaining good correlation with human judgements even for high-quality MT outputs.","types":["property"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_18ec83a5b2bc9d8ac37f9214cb89389b"},{"public_id":"co_25ba896f92b39f005c85b58ca56d0318","status":"active","name":"Direct Assessments","description":"A type of human judgement used to train one of the COMET models.","types":["judgement type"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_25ba896f92b39f005c85b58ca56d0318"},{"public_id":"co_2be8443dd17452abdf991ae60cf4db7e","status":"active","name":"cross-lingual pretrained language modeling","description":"Pretrained language models that work across languages, used here to improve MT evaluation.","types":["technique"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_2be8443dd17452abdf991ae60cf4db7e"},{"public_id":"co_3148f0cc02ffe2a55d36db87c7c7c2bb","status":"active","name":"multilingual machine translation evaluation models","description":"Models that evaluate machine translation quality across multiple languages.","types":["model"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_3148f0cc02ffe2a55d36db87c7c7c2bb"},{"public_id":"co_41f3790e26382d5a19c0d1539a029a39","status":"active","name":"Human-mediated Translation Edit Rate","description":"A human-mediated edit rate metric used as a training judgement type.","types":["judgement type"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_41f3790e26382d5a19c0d1539a029a39"},{"public_id":"co_4f39730bc1b46209cfce22f230d80905","status":"active","name":"source input","description":"The original source-language text used as input to the MT evaluation model.","types":["data"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_4f39730bc1b46209cfce22f230d80905"},{"public_id":"co_51dd7c5da387de8af4124d5e6e4014aa","status":"active","name":"WMT 2019 Metrics shared task","description":"The evaluation benchmark on which COMET models achieve state-of-the-art performance.","types":["benchmark"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_51dd7c5da387de8af4124d5e6e4014aa"},{"public_id":"co_64090646c8dd7ebf8288c4376d352563","status":"active","name":"Multidimensional Quality Metrics","description":"A multidimensional quality metric used as a training judgement type.","types":["judgement type"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_64090646c8dd7ebf8288c4376d352563"},{"public_id":"co_d405a2bee3353aee8c3cf0de16c89dfb","status":"active","name":"COMET","description":"A neural framework for training multilingual machine translation evaluation models.","types":["framework"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_d405a2bee3353aee8c3cf0de16c89dfb"},{"public_id":"co_fda72e4896cb1ccc442041cde32605c3","status":"active","name":"target-language reference translation","description":"A human-written reference translation in the target language used for evaluation.","types":["data"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_fda72e4896cb1ccc442041cde32605c3"},{"public_id":"co_fdfed2a1fb03e07dfb7127d1eb9fa69f","status":"active","name":"neural framework","description":"The overall neural architecture used to build the MT evaluation models.","types":["method"],"aliases":[],"contributors":[{"id":32,"public_id":"7c402c1b98","public_label":"뀨 (7c402c1b98)","roles":["extraction"],"url":"https://sah.borca.ai/u/7c402c1b98"},{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["review"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_fdfed2a1fb03e07dfb7127d1eb9fa69f"}],"external_ids":{"DOI":"10.18653/v1/2020.emnlp-main.213","ArXiv":"2009.09025","PubMed":null,"PubMedCentral":null,"MAG":3087597081,"DBLP":"journals/corr/abs-2009-09025","ACL":"2020.emnlp-main.213"},"open_access":{"is_open_access":true,"pdf_url":"https://www.aclweb.org/anthology/2020.emnlp-main.213.pdf","landing_url":"https://www.semanticscholar.org/paper/9e67b9758520e49016ab66bafb974d2e1ed762d1","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":"CCBY","status":"HYBRID","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":true,"full_text_source":"arxiv","count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":638937,"paper_uid":"56109b2f-f64c-4b71-8821-239db4beb1e7","canonical_identity":{"paper_id":638937,"paper_uid":"56109b2f-f64c-4b71-8821-239db4beb1e7","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/221819581"}