{"corpus_id":222278988,"paper_sha":"f8a1a60dad2c8cff9c197bc36f3763fa07251a4a","doi":"10.1002/cav.1939","arxiv_id":null,"pmid":null,"pmcid":null,"mag_id":3081020705,"dblp_id":"journals/jvca/XiangWJWLYZ20","acl_id":null,"title":"Sketch‐based modeling with a differentiable renderer","year":2020,"publication_date":"2020-07-01","venue":"Comput. Animat. Virtual Worlds","journal":{"name":"Computer Animation and Virtual Worlds","pages":null,"volume":"31"},"journal_issn":null,"journal_title":null,"publication_types":["JournalArticle"],"pubmed_pub_types":null,"s2_fields_of_study":["Computer Science"],"reference_count":37,"citation_count":11,"influential_citation_count":0,"is_open_access":true,"arxiv_categories":null,"arxiv_license":null,"arxiv_journal_ref":null,"mesh_headings":null,"chemicals":null,"comments_corrections":null,"source_flags":1,"s2_open_access_pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cav.1939","s2_open_access_landing_url":"https://www.semanticscholar.org/paper/f8a1a60dad2c8cff9c197bc36f3763fa07251a4a","s2_open_access_license":"CCBY","s2_open_access_status":"HYBRID","pmc_open_access_pdf_url":null,"pmc_open_access_landing_url":null,"pmc_open_access_license":null,"pmc_open_access_status":null,"unpaywall_open_access_pdf_url":null,"unpaywall_open_access_landing_url":null,"unpaywall_open_access_license":null,"unpaywall_open_access_status":null,"abstract":"Sketch‐based modeling aims to recover three‐dimensional (3D) shape from two‐dimensional line drawings. However, due to the sparsity and ambiguity of the sketch, it is extremely challenging for computers to interpret line drawings of physical objects. Most conventional systems are restricted to specific scenarios such as recovering for specific shapes, which are not conducive to generalize. Recent progress of deep learning methods have sparked new ideas for solving computer vision and pattern recognition issues. In this work, we present an end‐to‐end learning framework to predict 3D shape from line drawings. Our approach is based on a two‐steps strategy, it converts the sketch image to its normal image, then recover the 3D shape subsequently. A differentiable renderer is proposed and incorporated into this framework, it allows the integration of the rendering pipeline with neural networks. Experimental results show our method outperforms the state‐of‐art, which demonstrates that our framework is able to cope with the challenges in single sketch‐based 3D shape modeling.","claims":[{"public_id":"cl_07406f81e0d30f92758cc8216c6a197a","status":"active","text":"A differentiable renderer is integrated into the framework to connect the rendering pipeline with neural networks.","confidence":0.97,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_07406f81e0d30f92758cc8216c6a197a"},{"public_id":"cl_f8e3b57f965b4f5b7425acb1d040f965","status":"active","text":"An end-to-end learning framework predicts 3D shape from line drawings by first converting the sketch image to a normal image and then recovering the 3D shape.","confidence":0.98,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_f8e3b57f965b4f5b7425acb1d040f965"},{"public_id":"cl_b5f30c0a2765706e1b93413939091d0b","status":"active","text":"The proposed method outperforms the state of the art on experimental evaluation.","confidence":0.93,"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/claims/cl_b5f30c0a2765706e1b93413939091d0b"}],"concepts":[{"public_id":"co_0ba1159f979fdc17c3e7a861c4302dfe","status":"active","name":"normal image","description":"An intermediate image representation describing surface orientation information derived from the sketch.","types":["intermediate representation"],"aliases":["normal map"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_0ba1159f979fdc17c3e7a861c4302dfe"},{"public_id":"co_0ddcb7391924338c56e3e95e85a1ad88","status":"active","name":"end-to-end learning framework","description":"A training and inference framework that maps sketch inputs directly to 3D shape predictions through learned components.","types":["method"],"aliases":["framework"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_0ddcb7391924338c56e3e95e85a1ad88"},{"public_id":"co_0fe02e84a6b7dcedf6a1e2053c889b56","status":"active","name":"neural networks","description":"Learned function approximators used here to model the mapping from sketches to shapes.","types":["model"],"aliases":["NNs"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_0fe02e84a6b7dcedf6a1e2053c889b56"},{"public_id":"co_119b6339eb95d19ef445f60e1fdf2489","status":"active","name":"state of the art","description":"The strongest previously reported methods used as comparison in evaluation.","types":["baseline","comparison standard"],"aliases":["SOTA"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_119b6339eb95d19ef445f60e1fdf2489"},{"public_id":"co_167b0217903dd957c30280f0f9b19ca2","status":"active","name":"3D shape","description":"The three-dimensional geometry of an object recovered from the input drawing.","types":["output","geometry"],"aliases":["three-dimensional shape"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_167b0217903dd957c30280f0f9b19ca2"},{"public_id":"co_27c25609d7d869e6d9ba280f8b610c03","status":"active","name":"rendering pipeline","description":"The sequence of rendering operations used to generate images from 3D representations.","types":["process"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_27c25609d7d869e6d9ba280f8b610c03"},{"public_id":"co_66e79a0c82dd589f8cede9cd4f1df8b6","status":"active","name":"proposed method","description":"The specific sketch-based modeling approach introduced in the abstract.","types":["method"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_66e79a0c82dd589f8cede9cd4f1df8b6"},{"public_id":"co_6fb8df9f8a9f74c6afb922d4c212506b","status":"active","name":"differentiable renderer","description":"A renderer with gradients that can be optimized jointly with neural network components.","types":["method","module"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_6fb8df9f8a9f74c6afb922d4c212506b"},{"public_id":"co_a8f424f93146311f10240c2be09e341f","status":"active","name":"line drawings","description":"Two-dimensional sketch-like images composed of lines that serve as input for 3D modeling.","types":["input data"],"aliases":["sketches","sketch image"],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_a8f424f93146311f10240c2be09e341f"},{"public_id":"co_cc4b817bd23b4c0ada3e7d9bc004de37","status":"active","name":"experimental results","description":"The empirical evaluation results reported for the method.","types":["evaluation results"],"aliases":[],"contributors":[{"id":1,"public_id":"12632b8b5f","public_label":"Anonymous (12632b8b5f)","roles":["extraction"],"url":"https://sah.borca.ai/u/12632b8b5f"}],"url":"https://sah.borca.ai/concepts/co_cc4b817bd23b4c0ada3e7d9bc004de37"}],"external_ids":{"DOI":"10.1002/cav.1939","ArXiv":null,"PubMed":null,"PubMedCentral":null,"MAG":3081020705,"DBLP":"journals/jvca/XiangWJWLYZ20","ACL":null},"open_access":{"is_open_access":true,"pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/cav.1939","landing_url":"https://www.semanticscholar.org/paper/f8a1a60dad2c8cff9c197bc36f3763fa07251a4a","source":"semantic_scholar","pdf_url_source":"semantic_scholar_open_access_pdf","license":"CCBY","status":"HYBRID","reason":null},"reference_availability":{"status":"available","references_indexed":true,"full_text_available":false,"full_text_source":null,"count_basis":"semantic_scholar_metadata","extraction_status":"not_applicable","reason":null},"source":{"provider":"episteme2","base_corpus":"semantic_scholar_dump","freshness_mode":"unknown","basis":["semantic_scholar_metadata","postgres_metadata"],"limits":["paper metadata is based on indexed upstream scholarly datasets","claims and concepts are available only for extracted papers","absence of claims or concepts means no extracted graph data is available in this response"],"status":"available","degraded":false,"degraded_reasons":[],"diagnostics":{"status":"available","degraded":false,"degraded_reasons":[],"metadata_status":"available","graph_status":"available","abstract_status":"available"},"source_flags":1},"paper_id":632564,"paper_uid":"671edb9d-db1f-4005-8609-2eb8aaad6f65","canonical_identity":{"paper_id":632564,"paper_uid":"671edb9d-db1f-4005-8609-2eb8aaad6f65","identity_status":"available","lookup_basis":"semantic_scholar_external_id","compatibility_path":"corpus_id"},"url":"https://sah.borca.ai/papers/222278988"}