{"corpus_id":69365995,"paper_sha":"b16cddcb084356c5fa175d666561b69bf3fbfa12","doi":"10.12783/dtcse/iece2018/26631","arxiv_id":null,"pmid":null,"pmcid":null,"mag_id":2904469652,"dblp_id":null,"acl_id":null,"title":"Design of Data Capture Program Based on Web Crawler Technology","year":2018,"publication_date":"2018-12-07","venue":"DEStech Transactions on Computer Science and Engineering","journal":{"name":"DEStech Transactions on Computer Science and Engineering","pages":null,"volume":null},"journal_issn":null,"journal_title":null,"publication_types":[],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":6,"citation_count":4,"influential_citation_count":0,"is_open_access":false,"arxiv_categories":null,"arxiv_license":null,"arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":null,"s2_open_access_landing_url":null,"s2_open_access_license":null,"s2_open_access_status":null,"pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"With the advent of the Internet of things era and the vigorous development of electronic information era, the network information resources are growing exponentially. Faced with the demand of obtaining useful information, based on the general web crawler technology, this paper uses Python software to design a deep and optimized web crawler data fetching program. This crawler program can effectively solve a series of problems, such as waiting time, information overlapping and information incompleteness, so that the crawler has good performance.","claims":[{"public_id":"cl_24d3224ac64f1fa0454b9ce067120c3f","status":"active","text":"A deep and optimized web crawler data-fetching program is designed in Python based on general web crawler technology.","confidence":0.97,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_24d3224ac64f1fa0454b9ce067120c3f"},{"public_id":"cl_fbfe703c990f66c32921cf4cba5ebbf7","status":"active","text":"The crawler program achieves good performance.","confidence":0.8,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_fbfe703c990f66c32921cf4cba5ebbf7"},{"public_id":"cl_3b5fa20668a567ed5698e2e858db8122","status":"active","text":"The crawler program addresses waiting time, information overlapping, and information incompleteness.","confidence":0.93,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_3b5fa20668a567ed5698e2e858db8122"}],"concepts":[{"public_id":"co_6e46668b1a4406f5c15866f4af3e5ec2","status":"active","name":"good performance","description":"A favorable performance outcome attributed to the crawler program in the abstract.","types":["result"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_6e46668b1a4406f5c15866f4af3e5ec2"},{"public_id":"co_911d0626f4dba8fa21ae87a77eb2ef3f","status":"active","name":"information incompleteness","description":"Missing or incomplete information retrieved during web crawling.","types":["problem"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_911d0626f4dba8fa21ae87a77eb2ef3f"},{"public_id":"co_cb2600cc2518195f5ab608ed65df16a2","status":"active","name":"waiting time","description":"The delay involved in fetching or processing web information.","types":["performance issue"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_cb2600cc2518195f5ab608ed65df16a2"},{"public_id":"co_cd51fa799833fa5aaa64d87267ad921c","status":"active","name":"information overlapping","description":"Redundant duplication of retrieved information in web crawling.","types":["problem"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_cd51fa799833fa5aaa64d87267ad921c"},{"public_id":"co_d6013eedb06e3af55e102483205d7c9a","status":"active","name":"Python","description":"The programming language used to implement the crawler program.","types":["programming language"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_d6013eedb06e3af55e102483205d7c9a"},{"public_id":"co_ded95e9c5729b038b27024d7f0d7ddc4","status":"active","name":"general web crawler technology","description":"Standard web crawling technology used as the basis for the proposed program.","types":["method"],"aliases":["web crawler technology"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_ded95e9c5729b038b27024d7f0d7ddc4"},{"public_id":"co_f84e2d1837c576bb2d06dea2cfad6aee","status":"active","name":"deep and optimized web crawler data fetching program","description":"A Python-based crawler program designed to collect web data efficiently while reducing common fetching problems.","types":["method","software"],"aliases":["crawler program","data fetching program"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_f84e2d1837c576bb2d06dea2cfad6aee"}],"external_ids":{"DOI":"10.12783/dtcse/iece2018/26631","ArXiv":null,"PubMed":null,"PubMedCentral":null,"MAG":2904469652,"DBLP":null,"ACL":null},"open_access":{"is_open_access":false,"pdf_url":null,"landing_url":"https://sah.borca.ai/papers/69365995","source":null,"pdf_url_source":null,"license":null,"reason":"pdf_url_not_indexed"},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":false,"full_text_source":null,"count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":633294,"paper_uid":"6e61559b-8e6f-4d2d-ace6-677bcb8f5b41","canonical_identity":{"paper_id":633294,"paper_uid":"6e61559b-8e6f-4d2d-ace6-677bcb8f5b41","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/69365995"}