{"corpus_id":212725677,"paper_sha":"d19bc4735cdb26935dbf50d2ac6d6b82fcc20302","doi":"10.1109/cvpr42600.2020.01424","arxiv_id":"2003.06692","pmid":null,"pmcid":null,"mag_id":3012317973,"dblp_id":"journals/corr/abs-2003-06692","acl_id":null,"title":"EmotiCon: Context-Aware Multimodal Emotion Recognition Using Frege’s Principle","year":2020,"publication_date":"2020-03-14","venue":"Computer Vision and Pattern Recognition","journal":{"name":"2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","pages":"14222-14231","volume":null},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle","Conference"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":63,"citation_count":160,"influential_citation_count":27,"is_open_access":true,"arxiv_categories":["cs.CV","cs.HC","eess.IV"],"arxiv_license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://arxiv.org/pdf/2003.06692","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/d19bc4735cdb26935dbf50d2ac6d6b82fcc20302","s2_open_access_license":null,"s2_open_access_status":"GREEN","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"We present EmotiCon, a learning-based algorithm for context-aware perceived human emotion recognition from videos and images. Motivated by Frege's Context Principle from psychology, our approach combines three interpretations of context for emotion recognition. Our first interpretation is based on using multiple modalities (e.g.faces and gaits) for emotion recognition. For the second interpretation, we gather semantic context from the input image and use a self-attention-based CNN to encode this information. Finally, we use depth maps to model the third interpretation related to socio-dynamic interactions and proximity among agents. We demonstrate the efficiency of our network through experiments on EMOTIC, a benchmark dataset. We report an Average Precision (AP) score of 35.48 across 26 classes, which is an improvement of 7-8 over prior methods. We also introduce a new dataset, GroupWalk, which is a collection of videos captured in multiple real-world settings of people walking. We report an AP of 65.83 across 4 categories on GroupWalk, which is also an improvement over prior methods.","claims":[{"public_id":"cl_9982038597d7b1a4e266b2ed7c109b27","status":"active","text":"EmotiCon combines multimodal, semantic, and depth-based context to perform context-aware perceived human emotion recognition from videos and images.","confidence":0.98,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_9982038597d7b1a4e266b2ed7c109b27"},{"public_id":"cl_0ee9e534537c5cca60a3778d17b39902","status":"active","text":"GroupWalk is introduced as a new dataset of videos captured in multiple real-world settings of people walking.","confidence":0.96,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_0ee9e534537c5cca60a3778d17b39902"},{"public_id":"cl_962772ff2d6eff07e17c07dedc187363","status":"active","text":"The model achieves an Average Precision of 35.48 across 26 classes on EMOTIC, improving by 7-8 points over prior methods.","confidence":0.97,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_962772ff2d6eff07e17c07dedc187363"},{"public_id":"cl_13d3ada4c3450f90ac6ab4bd29dd7443","status":"active","text":"The model achieves an Average Precision of 65.83 across 4 categories on GroupWalk, improving over prior methods.","confidence":0.96,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_13d3ada4c3450f90ac6ab4bd29dd7443"}],"concepts":[{"public_id":"co_111c98fab39f091b05603dc0b7676682","status":"active","name":"context-aware perceived human emotion recognition","description":"Recognition of perceived human emotions from video or image inputs while incorporating contextual information.","types":["task"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_111c98fab39f091b05603dc0b7676682"},{"public_id":"co_2ead1ab4f76d46bce3f97af39fdd110e","status":"active","name":"self-attention-based CNN","description":"A convolutional neural network augmented with self-attention to encode semantic information.","types":["method"],"aliases":["self-attention-based convolutional neural network"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_2ead1ab4f76d46bce3f97af39fdd110e"},{"public_id":"co_338e36826800c4bf2df1866f76123e06","status":"active","name":"prior methods","description":"Previously published methods used as comparison baselines in the experiments.","types":["baseline"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_338e36826800c4bf2df1866f76123e06"},{"public_id":"co_37be4b0690ddd3f7c34cb5f28a83e740","status":"active","name":"depth maps","description":"Depth representations used to model spatial relationships and interactions among people.","types":["input modality"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_37be4b0690ddd3f7c34cb5f28a83e740"},{"public_id":"co_66b978741bd8bbe79da67ff410593b37","status":"active","name":"GroupWalk","description":"A new video dataset of people walking in multiple real-world settings.","types":["dataset"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_66b978741bd8bbe79da67ff410593b37"},{"public_id":"co_8a4ded440d93ac2fa130e129883b0bab","status":"active","name":"multimodal context","description":"Use of multiple input modalities such as faces and gaits to support emotion recognition.","types":["concept"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_8a4ded440d93ac2fa130e129883b0bab"},{"public_id":"co_92ee728067e18f2da550f3ceffd8b8e0","status":"active","name":"videos captured in multiple real-world settings","description":"Video recordings collected in diverse natural environments and scenes.","types":["data source"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_92ee728067e18f2da550f3ceffd8b8e0"},{"public_id":"co_b66bc0266da2f9b52873837621c4fb7f","status":"active","name":"Average Precision","description":"An evaluation metric used to report recognition performance across classes.","types":["metric"],"aliases":["AP"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_b66bc0266da2f9b52873837621c4fb7f"},{"public_id":"co_b71a744f3ce96cfbd0fe985aadb0fb7a","status":"active","name":"EmotiCon","description":"A learning-based algorithm for context-aware perceived human emotion recognition from visual data.","types":["method"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_b71a744f3ce96cfbd0fe985aadb0fb7a"},{"public_id":"co_c22ec0d03d09372f6da9b28b2379bd00","status":"active","name":"semantic context","description":"Information derived from the input image that provides scene-level or object-level meaning.","types":["concept"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_c22ec0d03d09372f6da9b28b2379bd00"},{"public_id":"co_ced9498fb63fe8d8b189fe5e714d9d33","status":"active","name":"Frege's Context Principle","description":"A principle from psychology motivating the use of context in interpreting emotion.","types":["theoretical principle"],"aliases":["Context Principle"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_ced9498fb63fe8d8b189fe5e714d9d33"},{"public_id":"co_cf2bf04dce9fa982a91d838ec788dba0","status":"active","name":"EMOTIC","description":"A benchmark dataset used to evaluate perceived human emotion recognition.","types":["dataset","benchmark"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_cf2bf04dce9fa982a91d838ec788dba0"}],"external_ids":{"DOI":"10.1109/cvpr42600.2020.01424","ArXiv":"2003.06692","PubMed":null,"PubMedCentral":null,"MAG":3012317973,"DBLP":"journals/corr/abs-2003-06692","ACL":null},"open_access":{"is_open_access":true,"pdf_url":"https://arxiv.org/pdf/2003.06692","landing_url":"https://www.semanticscholar.org/paper/d19bc4735cdb26935dbf50d2ac6d6b82fcc20302","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":null,"status":"GREEN","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":true,"full_text_source":"arxiv","count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":636401,"paper_uid":"f42798dd-cae6-46fd-bc2f-58e0388235c8","canonical_identity":{"paper_id":636401,"paper_uid":"f42798dd-cae6-46fd-bc2f-58e0388235c8","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/212725677"}