{"corpus_id":1310556,"paper_sha":"0ad8672b627382c1f5b31358c7928f02964ed6e1","doi":"10.1145/2998181.2998196","arxiv_id":"1611.02682","pmid":null,"pmcid":null,"mag_id":3102875147,"dblp_id":"journals/corr/KimSCB16","acl_id":null,"title":"Mechanical Novel: Crowdsourcing Complex Work through Reflection and Revision","year":2016,"publication_date":"2016-11-08","venue":"Conference on Computer Supported Cooperative Work","journal":{"name":"Proceedings of the 2017 ACM Conference on Computer Supported Cooperative Work and Social Computing","pages":null,"volume":null},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle","Book","Conference"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":43,"citation_count":94,"influential_citation_count":7,"is_open_access":true,"arxiv_categories":["cs.HC"],"arxiv_license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://arxiv.org/pdf/1611.02682","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/0ad8672b627382c1f5b31358c7928f02964ed6e1","s2_open_access_license":null,"s2_open_access_status":"GREEN","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"Crowdsourcing systems accomplish large tasks with scale and speed by breaking work down into independent parts. However, many types of complex creative work, such as fiction writing, have remained out of reach for crowds because work is tightly interdependent: changing one part of a story may trigger changes to the overall plot and vice versa. Taking inspiration from how expert authors write, we propose a technique for achieving interdependent complex goals with crowds. With this technique, the crowd loops between reflection, to select a high-level goal, and revision, to decompose that goal into low-level, actionable tasks. We embody this approach in Mechanical Novel, a system that crowdsources short fiction stories on Amazon Mechanical Turk. In a field experiment, Mechanical Novel resulted in higher-quality stories than an iterative crowdsourcing workflow. Our findings suggest that orienting crowd work around high-level goals may enable workers to coordinate their effort to accomplish complex work.","claims":[{"public_id":"cl_5d3d0a2deab7d47680d5890a42b1ab82","status":"active","text":"A crowd workflow that loops between reflection to select a high-level goal and revision to decompose that goal into low-level actionable tasks is proposed for achieving interdependent complex goals.","confidence":0.93,"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_5d3d0a2deab7d47680d5890a42b1ab82"},{"public_id":"cl_f4c9ba4495194f3598fd95b26adcdded","status":"active","text":"In a field experiment, Mechanical Novel produced higher-quality stories than an iterative crowdsourcing workflow.","confidence":0.95,"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_f4c9ba4495194f3598fd95b26adcdded"},{"public_id":"cl_eb204dc593b54cf576ec6c7edfe0a575","status":"active","text":"Mechanical Novel embodies the reflection-and-revision approach to crowdsource short fiction stories on Amazon Mechanical Turk.","confidence":0.94,"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_eb204dc593b54cf576ec6c7edfe0a575"},{"public_id":"cl_ed0089b3ae81684969f2a85f263818a1","status":"active","text":"Orienting crowd work around high-level goals may enable workers to coordinate their effort on complex work.","confidence":0.86,"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/claims/cl_ed0089b3ae81684969f2a85f263818a1"}],"concepts":[{"public_id":"co_004bc56e4e4c22c8f857cf88d0d69bc0","status":"active","name":"short fiction stories","description":"The creative writing outputs crowdsourced by Mechanical Novel.","types":["output"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_004bc56e4e4c22c8f857cf88d0d69bc0"},{"public_id":"co_154fc9b3c22cb4330fea7575fb80d90d","status":"active","name":"complex work","description":"Work with tightly interdependent parts, exemplified by fiction writing in the abstract.","types":["task category"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_154fc9b3c22cb4330fea7575fb80d90d"},{"public_id":"co_2b83281dfceaa29b3e3028b55ab4f49e","status":"active","name":"crowd work","description":"Work performed by distributed crowd workers in a crowdsourcing system.","types":["work process"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_2b83281dfceaa29b3e3028b55ab4f49e"},{"public_id":"co_4a9d9226f48791c83317eaf3f2198d51","status":"active","name":"high-level goals","description":"Abstract objectives used to orient and coordinate crowd work in the proposed approach.","types":["coordination mechanism"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_4a9d9226f48791c83317eaf3f2198d51"},{"public_id":"co_546363c05572f82d3688ef957ab1b36a","status":"active","name":"iterative crowdsourcing workflow","description":"A comparison workflow for crowdsourcing stories through repeated iterations.","types":["baseline"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_546363c05572f82d3688ef957ab1b36a"},{"public_id":"co_7ff6da5e6f0a6ce19e428cb1e49fff9b","status":"active","name":"field experiment","description":"The evaluation setting used to compare Mechanical Novel with another crowdsourcing workflow.","types":["evaluation setting"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_7ff6da5e6f0a6ce19e428cb1e49fff9b"},{"public_id":"co_808ea72582ade837d6f4bb0cc4e62134","status":"active","name":"reflection","description":"A workflow phase in which the crowd selects a high-level goal for the creative work.","types":["workflow phase"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_808ea72582ade837d6f4bb0cc4e62134"},{"public_id":"co_8465f3d0990b57839c4fe7a836ace834","status":"active","name":"Amazon Mechanical Turk","description":"The crowdsourcing platform used to recruit workers for Mechanical Novel.","types":["platform"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_8465f3d0990b57839c4fe7a836ace834"},{"public_id":"co_b9c6d48dc6e2b91ecd5e8842a8db45b1","status":"active","name":"interdependent complex goals","description":"Goals in complex creative work where changes to one part can affect other parts and the overall structure.","types":["problem setting"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_b9c6d48dc6e2b91ecd5e8842a8db45b1"},{"public_id":"co_bb8ddc25aa46f65dccbf6107e201718e","status":"active","name":"revision","description":"A workflow phase in which the crowd decomposes a selected goal into low-level actionable tasks.","types":["workflow phase"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_bb8ddc25aa46f65dccbf6107e201718e"},{"public_id":"co_ec7afb4b6ce7c0590800e1513a50fb8f","status":"active","name":"Mechanical Novel","description":"A system that crowdsources short fiction stories on Amazon Mechanical Turk using a reflection-and-revision workflow.","types":["system"],"aliases":[],"contributors":[{"id":136,"public_id":"3c2apqe3ut","public_label":"Anonymous (3c2apqe3ut)","roles":["extraction"],"url":"https://sah.borca.ai/u/3c2apqe3ut"},{"id":2,"public_id":"4715169a40","public_label":"AK (4715169a40)","roles":["review"],"url":"https://sah.borca.ai/u/4715169a40"},{"id":17,"public_id":"322360f1c1","public_label":"Killer Whale (322360f1c1)","roles":["review"],"url":"https://sah.borca.ai/u/322360f1c1"}],"url":"https://sah.borca.ai/concepts/co_ec7afb4b6ce7c0590800e1513a50fb8f"}],"external_ids":{"DOI":"10.1145/2998181.2998196","ArXiv":"1611.02682","PubMed":null,"PubMedCentral":null,"MAG":3102875147,"DBLP":"journals/corr/KimSCB16","ACL":null},"open_access":{"is_open_access":true,"pdf_url":"https://arxiv.org/pdf/1611.02682","landing_url":"https://www.semanticscholar.org/paper/0ad8672b627382c1f5b31358c7928f02964ed6e1","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":null,"status":"GREEN","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":true,"full_text_source":"arxiv","count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":632137,"paper_uid":"4e17902a-e95b-43b0-850c-14ca0eac2498","canonical_identity":{"paper_id":632137,"paper_uid":"4e17902a-e95b-43b0-850c-14ca0eac2498","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/1310556"}