{"corpus_id":34408811,"paper_sha":"62fec5cbb951d6f1074c2de0748be8035c470816","doi":"10.1002/cpe.3786","arxiv_id":null,"pmid":null,"pmcid":null,"mag_id":2288369134,"dblp_id":"journals/concurrency/KhanHLTK17","acl_id":null,"title":"Optimizing hadoop parameter settings with gene expression programming guided PSO","year":2017,"publication_date":"2017-02-10","venue":"Concurrency and Computation","journal":{"name":"Concurrency and Computation: Practice and Experience","pages":null,"volume":"29"},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":59,"citation_count":30,"influential_citation_count":2,"is_open_access":true,"arxiv_categories":null,"arxiv_license":null,"arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.3786","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/62fec5cbb951d6f1074c2de0748be8035c470816","s2_open_access_license":"CCBY","s2_open_access_status":"HYBRID","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"Hadoop MapReduce has become a major computing technology in support of big data analytics. The Hadoop framework has over 190 configuration parameters, and some of them can have a significant effect on the performance of a Hadoop job. Manually tuning the optimum or near optimum values of these parameters is a challenging task and also a time consuming process. This paper optimizes the performance of Hadoop by automatically tuning its configuration parameter settings. The proposed work first employs gene expression programming technique to build an objective function based on historical job running records, which represents a correlation among the Hadoop configuration parameters. It then employs particle swarm optimization technique, which makes use of the objective function to search for optimal or near optimal parameter settings. Experimental results show that the proposed work enhances the performance of Hadoop significantly compared with the default settings. Moreover, it outperforms both rule‐of‐thumb settings and the Starfish model in Hadoop performance optimization. © 2016 The Authors. Concurrency and Computation: Practice and Experience Published by John Wiley & Sons Ltd.","claims":[{"public_id":"cl_81c526a9b00db59e38c8f39a5bb747be","status":"active","text":"Automatic tuning with the proposed approach significantly improves Hadoop performance relative to default settings.","confidence":0.95,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_81c526a9b00db59e38c8f39a5bb747be"},{"public_id":"cl_22761736807ed47e458deb0b23b6e5de","status":"active","text":"Gene expression programming is used to build an objective function from historical Hadoop job running records that captures correlations among configuration parameters.","confidence":0.97,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_22761736807ed47e458deb0b23b6e5de"},{"public_id":"cl_64267511355fedffab45f6d0d6d89bac","status":"active","text":"Particle swarm optimization uses the learned objective function to search for optimal or near optimal Hadoop parameter settings.","confidence":0.96,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_64267511355fedffab45f6d0d6d89bac"},{"public_id":"cl_4e3372f3f970136818ed43f3388762d6","status":"active","text":"The proposed approach outperforms both rule-of-thumb settings and the Starfish model in Hadoop performance optimization.","confidence":0.95,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_4e3372f3f970136818ed43f3388762d6"}],"concepts":[{"public_id":"co_19d5db12d1ddd8f4e4c419935cd4d44e","status":"active","name":"particle swarm optimization","description":"A population-based optimization method used here to search for favorable Hadoop configuration settings.","types":["optimization method"],"aliases":["PSO"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_19d5db12d1ddd8f4e4c419935cd4d44e"},{"public_id":"co_213a690c7d636de2ae5397e8dfc0ff73","status":"active","name":"rule-of-thumb settings","description":"Heuristic Hadoop configuration values based on common practical guidance.","types":["baseline"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_213a690c7d636de2ae5397e8dfc0ff73"},{"public_id":"co_261639ce198f653feed6ffe7cec9175b","status":"active","name":"Starfish model","description":"A Hadoop performance optimization model used as a comparison baseline.","types":["model","baseline"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_261639ce198f653feed6ffe7cec9175b"},{"public_id":"co_2b7a4f1fbd5834411c6fb82ff29dae69","status":"active","name":"Hadoop performance","description":"The execution performance of Hadoop jobs as affected by configuration choices.","types":["outcome"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_2b7a4f1fbd5834411c6fb82ff29dae69"},{"public_id":"co_44ee585200af0ea30be3e7ee0fa50ab1","status":"active","name":"objective function","description":"A function derived from historical job records that encodes relationships among Hadoop configuration parameters.","types":["model"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_44ee585200af0ea30be3e7ee0fa50ab1"},{"public_id":"co_4f872c8165d284e9fd746bc8e8242c9d","status":"active","name":"gene expression programming","description":"An evolutionary computation technique used here to construct an objective function from prior Hadoop execution data.","types":["method"],"aliases":["GEP"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_4f872c8165d284e9fd746bc8e8242c9d"},{"public_id":"co_531509b8a88c12d5c40e69a54a5cc04c","status":"active","name":"Hadoop performance optimization","description":"The task of improving Hadoop job performance through configuration selection.","types":["optimization task"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_531509b8a88c12d5c40e69a54a5cc04c"},{"public_id":"co_5e91f10af98a48b91e3b2a6b13745a14","status":"active","name":"automatic tuning","description":"The automated selection of configuration values without manual trial-and-error.","types":["method"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_5e91f10af98a48b91e3b2a6b13745a14"},{"public_id":"co_65b4d2a7d1f2ee5f9d28fca1ff44a466","status":"active","name":"default settings","description":"The out-of-the-box Hadoop configuration values used as a baseline for comparison.","types":["baseline"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_65b4d2a7d1f2ee5f9d28fca1ff44a466"},{"public_id":"co_6ff9efffc256d34e5740528089354e9d","status":"active","name":"Hadoop parameter settings","description":"Specific values assigned to Hadoop configuration parameters for a job or workload.","types":["configuration settings"],"aliases":["parameter settings"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_6ff9efffc256d34e5740528089354e9d"},{"public_id":"co_7f1fc4676e2bcd2c13b2cfa1cc37ec25","status":"active","name":"Hadoop configuration parameters","description":"The tunable settings of the Hadoop framework that affect job performance.","types":["configuration parameters"],"aliases":["Hadoop parameters"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_7f1fc4676e2bcd2c13b2cfa1cc37ec25"},{"public_id":"co_d46b9e875317fcd8f035be9f035e8c2b","status":"active","name":"historical job running records","description":"Past Hadoop job execution records used as input for learning the parameter relationship model.","types":["data source"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_d46b9e875317fcd8f035be9f035e8c2b"}],"external_ids":{"DOI":"10.1002/cpe.3786","ArXiv":null,"PubMed":null,"PubMedCentral":null,"MAG":2288369134,"DBLP":"journals/concurrency/KhanHLTK17","ACL":null},"open_access":{"is_open_access":true,"pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cpe.3786","landing_url":"https://www.semanticscholar.org/paper/62fec5cbb951d6f1074c2de0748be8035c470816","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":"CCBY","status":"HYBRID","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":false,"full_text_source":null,"count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":635585,"paper_uid":"e2e29972-0bfa-4dfc-8155-48a1b5a07dad","canonical_identity":{"paper_id":635585,"paper_uid":"e2e29972-0bfa-4dfc-8155-48a1b5a07dad","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/34408811"}