{ "schema_version": 1, "canonical_profile": "https://dylantao.github.io/", "last_reviewed": "2026-07-13", "source_note": "Bibliographic facts come from _bibliography/papers.bib; source-reviewed editorial context comes from _data/publication_context.yml; display classifications and authorship-role cross-checks use _data/publication_lens.yml. Volatile citation counts and Google Scholar publication IDs are intentionally excluded.", "papers": [ { "key": "tao2026whw", "slug": "what-happened-and-why", "entry_type": "inproceedings", "title": "What Happened and Why? Trace-Guided Micro-Episodes with Elicited User Explanations for Product Iteration", "authors": [ { "name": "Sirui Tao", "given_name": "Sirui", "family_name": "Tao", "bibtex_name": "Tao, Sirui" }, { "name": "William P. McCarthy", "given_name": "William P.", "family_name": "McCarthy", "bibtex_name": "McCarthy, William P." }, { "name": "Steven P. Dow", "given_name": "Steven P.", "family_name": "Dow", "bibtex_name": "Dow, Steven P." } ], "venue": "Herding CATs: Making Sense of Creative Activity Traces (CHI 2026 Workshop)", "series": null, "editors": [], "volume": null, "number": null, "pages": null, "page_start": null, "page_end": null, "publisher": null, "isbn": null, "abbreviation": "CHI WS", "note": "Workshop Position Paper", "month": "apr", "month_numeric": 4, "year": 2026, "selected": true, "preview": "herding_cats_why_what.png", "doi": null, "arxiv": null, "work_type": "workshop-paper", "work_group": "short-form", "abstract": "Teams shipping AI workflows in design tools can measure usage yet often struggle to explain why features fail. In creative work, standard metrics are ambiguous: a long session could imply productive exploration or frustrating struggle with stochastic outputs. We argue for trace-guided micro-episodes, a unit of analysis binding interaction logs—what users did—to their intent. Rather than relying on disruptive surveys, we propose a “utility-for-rationale” paradigm: systems offer optional, context-aware controls at likely friction points, capturing user explanations as a byproduct of real-time error recovery. This approach converts ambiguous telemetry into causal evidence without breaking flow. We posit this methodology serves a dual purpose: equipping teams with diagnostic clarity to iterate on vague failure modes (e.g., controllability vs. quality) while generating the grounded alignment data required to train future agents.", "tldr": "A position paper proposing trace-guided micro-episodes that pair short interaction-trace windows and interface state with optional, in-flow user clarification so teams can interpret ambiguous behavior in creative AI tools.", "why_cite": { "statement": "Cite this position paper when motivating rationale-enriched telemetry: short, trace-guided windows paired with optional in-flow clarification to diagnose ambiguous moments in creative or AI-supported work.", "cite_when": [ "Discussing rationale-enriched interaction logging or post-deployment diagnosis in creative and AI tools.", "Designing feedback interventions at likely friction points without moving users into a separate survey flow." ], "contributions": [ "Defines a micro-episode as a bounded trace window joined with interface state and a lightweight user explanation.", "Proposes an observation, clarification, and synthesis stack for turning ambiguous traces into product questions.", "Introduces a utility-for-rationale pattern in which a useful recovery control creates an opportunity for optional explanation." ], "evidence": [ "This is a four-page CHI 2026 workshop position paper; it proposes a framework and research agenda rather than reporting an empirical evaluation.", "Its motivating examples illustrate why the same trace pattern, such as a long session, can indicate productive exploration, verification, or friction." ], "boundaries": [ "The proposed interventions have not yet been validated for insight quality, analysis time, interruption cost, or downstream agent training.", "Traces and clarifications do not by themselves establish causality; trigger selection may distract users or bias later behavior.", "Friction signals and clarification schemas must be adapted to the domain and the interaction touchpoints available in a particular tool." ] }, "authorship": { "role": "first-author", "role_label": "First and corresponding author", "statement": "Sirui Tao is the first and corresponding author. This guide does not assign unverified individual task contributions." }, "topics": [ "creative activity traces", "rationale-enriched telemetry", "product iteration", "generative AI tools", "post-deployment evaluation" ], "related_project": "https://dylantao.github.io/projects/what-happened-and-why/", "provenance": { "reviewed_on": "2026-07-13", "basis": "Author page and paper reviewed; conceptual claims are labeled as proposed rather than evaluated.", "sources": [ "https://dylantao.github.io/projects/what-happened-and-why/", "https://dylantao.github.io/projects/what-happened-and-why/what-happened-and-why.pdf", "https://herding-cats-ws.github.io/" ] }, "links": { "citation_page": "https://dylantao.github.io/publications/what-happened-and-why/", "citation_page_path": "/publications/what-happened-and-why/", "markdown": "https://dylantao.github.io/ai/papers/what-happened-and-why.md", "markdown_path": "/ai/papers/what-happened-and-why.md", "bibtex": "https://dylantao.github.io/ai/papers/what-happened-and-why.bib", "bibtex_path": "/ai/papers/what-happened-and-why.bib", "ris": "https://dylantao.github.io/ai/papers/what-happened-and-why.ris", "ris_path": "/ai/papers/what-happened-and-why.ris", "pdf": "https://dylantao.github.io/projects/what-happened-and-why/what-happened-and-why.pdf", "website": "https://dylantao.github.io/projects/what-happened-and-why/", "source_url": "https://herding-cats-ws.github.io/2026/papers/p02.pdf" }, "citation": { "bibtex": "@inproceedings{tao2026whw,\n title = {What Happened and Why? Trace-Guided Micro-Episodes with Elicited User Explanations for Product Iteration},\n author = {Tao, Sirui and McCarthy, William P. and Dow, Steven P.},\n booktitle = {Herding CATs: Making Sense of Creative Activity Traces (CHI 2026 Workshop)},\n url = {https://herding-cats-ws.github.io/2026/papers/p02.pdf},\n note = {Workshop Position Paper},\n month = {apr},\n year = {2026}\n}", "ris": "TY - CPAPER\nTI - What Happened and Why? Trace-Guided Micro-Episodes with Elicited User Explanations for Product Iteration\nAU - Tao, Sirui\nAU - McCarthy, William P.\nAU - Dow, Steven P.\nPY - 2026\nT2 - Herding CATs: Making Sense of Creative Activity Traces (CHI 2026 Workshop)\nUR - https://dylantao.github.io/projects/what-happened-and-why/\nN1 - Workshop Position Paper\nER -" } }, { "key": "wang2025hotspot", "slug": "hotspot", "entry_type": "inproceedings", "title": "HotSpot: Signed Distance Function Optimization with an Asymptotically Sufficient Condition", "authors": [ { "name": "Zimo Wang", "given_name": "Zimo", "family_name": "Wang", "bibtex_name": "Wang, Zimo" }, { "name": "Cheng Wang", "given_name": "Cheng", "family_name": "Wang", "bibtex_name": "Wang, Cheng" }, { "name": "Taiki Yoshino", "given_name": "Taiki", "family_name": "Yoshino", "bibtex_name": "Yoshino, Taiki" }, { "name": "Sirui Tao", "given_name": "Sirui", "family_name": "Tao", "bibtex_name": "Tao, Sirui" }, { "name": "Ziyang Fu", "given_name": "Ziyang", "family_name": "Fu", "bibtex_name": "Fu, Ziyang" }, { "name": "Tzu-Mao Li", "given_name": "Tzu-Mao", "family_name": "Li", "bibtex_name": "Li, Tzu-Mao" } ], "venue": "Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)", "series": null, "editors": [], "volume": null, "number": null, "pages": "1276--1286", "page_start": "1276", "page_end": "1286", "publisher": "IEEE", "isbn": null, "abbreviation": "CVPR", "note": "Highlight", "month": "jun", "month_numeric": 6, "year": 2025, "selected": true, "preview": "hotspot.png", "doi": "10.1109/CVPR52734.2025.00127", "arxiv": "2411.14628", "work_type": "full-paper", "work_group": "full-paper", "abstract": "We propose HotSpot, a method for optimizing neural signed distance functions by using the solution of a screened Poisson equation, which provides an asymptotically sufficient condition to ensure the output converges to a true distance function. In contrast, existing losses, such as the eikonal loss, act as necessary but insufficient constraints and cannot guarantee that the recovered implicit function represents a true distance function, even if the output minimizes these losses almost everywhere. Furthermore, the eikonal loss suffers from stability issues in optimization. Finally, in conventional optimizations, area loss is indispensable but distorts the output. We address these challenges by designing a loss function that, when minimized, converges to the true distance function, ensures stability, and naturally penalizes large surface area. We present theoretical analysis and experiments on both challenging 2D and 3D datasets and show that our method provides better surface reconstruction and more accurate distance approximation.", "tldr": "HotSpot uses a screened-Poisson heat loss as an asymptotically sufficient condition for neural signed-distance-function optimization, improving stability while naturally penalizing excess surface area.", "why_cite": { "statement": "Cite HotSpot for neural implicit surface reconstruction from unoriented points when the argument concerns sufficient SDF constraints, optimization stability, topology, or surface-area regularization.", "cite_when": [ "Comparing losses for neural signed distance functions rather than treating the eikonal condition as sufficient.", "Discussing stable reconstruction, distance accuracy, or topology from unoriented point observations, with sparse-boundary failure modes made explicit." ], "contributions": [ "Derives a screened-Poisson heat loss whose limiting solution supplies an asymptotically sufficient SDF condition.", "Provides convergence, approximation-error, and optimization-stability analysis.", "Connects the objective to a natural surface-area penalty and evaluates it across 2D and 3D reconstruction tasks." ], "evidence": [ "On the reported 2D benchmark, HotSpot reached 0.9870 IoU and 0.0014 Chamfer distance, compared with 0.7882 and 0.0055 for DiGS and 0.6620 and 0.0073 for StEik.", "On the evaluated ShapeNet subset, HotSpot reached 0.9796 IoU and 0.0029 Chamfer distance; SAL retained slightly lower overall RMSE and MAE, so the results do not show universal metric dominance.", "The topology ablation reports correct topology on all 14 tested 2D shapes." ], "boundaries": [ "The experiments target 2D and 3D reconstruction from unoriented point positions, including a 260-shape, 13-category ShapeNet subset; they do not establish superiority for every implicit-representation task.", "Sparse boundary sampling, high absorption, or an over-strong heat term can tear boundaries or collapse a signed solution toward an unsigned distance.", "Boundary weight and spatial scaling remain tuning considerations." ] }, "authorship": { "role": "coauthor", "role_label": "Coauthor", "statement": "Sirui Tao is a coauthor; the first two authors are marked as equal contributors. This guide does not assign Sirui an unverified individual task contribution." }, "topics": [ "signed distance functions", "neural implicit surfaces", "screened Poisson equation", "surface reconstruction", "computer graphics" ], "related_project": "https://dylantao.github.io/projects/hotspot/", "provenance": { "reviewed_on": "2026-07-13", "basis": "Official CVPR paper, project page, and DOI record reviewed.", "sources": [ "https://doi.org/10.1109/CVPR52734.2025.00127", "https://openaccess.thecvf.com/content/CVPR2025/papers/Wang_HotSpot_Signed_Distance_Function_Optimization_with_an_Asymptotically_Sufficient_Condition_CVPR_2025_paper.pdf", "https://zeamoxwang.github.io/HotSpot-CVPR25/", "https://arxiv.org/abs/2411.14628" ] }, "links": { "citation_page": "https://dylantao.github.io/publications/hotspot/", "citation_page_path": "/publications/hotspot/", "markdown": "https://dylantao.github.io/ai/papers/hotspot.md", "markdown_path": "/ai/papers/hotspot.md", "bibtex": "https://dylantao.github.io/ai/papers/hotspot.bib", "bibtex_path": "/ai/papers/hotspot.bib", "ris": "https://dylantao.github.io/ai/papers/hotspot.ris", "ris_path": "/ai/papers/hotspot.ris", "doi": "https://doi.org/10.1109/CVPR52734.2025.00127", "arxiv": "https://arxiv.org/abs/2411.14628", "pdf": "https://openaccess.thecvf.com/content/CVPR2025/papers/Wang_HotSpot_Signed_Distance_Function_Optimization_with_an_Asymptotically_Sufficient_Condition_CVPR_2025_paper.pdf", "website": "https://zeamoxwang.github.io/HotSpot-CVPR25/", "code": "https://github.com/Galaxeaaa/HotSpot", "video": "https://www.youtube.com/watch?v=v-OeGOxgqRM", "source_url": "https://openaccess.thecvf.com/content/CVPR2025/html/Wang_HotSpot_Signed_Distance_Function_Optimization_with_an_Asymptotically_Sufficient_Condition_CVPR_2025_paper.html" }, "citation": { "bibtex": "@inproceedings{wang2025hotspot,\n title = {HotSpot: Signed Distance Function Optimization with an Asymptotically Sufficient Condition},\n author = {Wang, Zimo and Wang, Cheng and Yoshino, Taiki and Tao, Sirui and Fu, Ziyang and Li, Tzu-Mao},\n booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n pages = {1276--1286},\n publisher = {IEEE},\n doi = {10.1109/CVPR52734.2025.00127},\n arxiv = {2411.14628},\n url = {https://openaccess.thecvf.com/content/CVPR2025/html/Wang_HotSpot_Signed_Distance_Function_Optimization_with_an_Asymptotically_Sufficient_Condition_CVPR_2025_paper.html},\n note = {Highlight},\n month = {jun},\n year = {2025}\n}", "ris": "TY - CPAPER\nTI - HotSpot: Signed Distance Function Optimization with an Asymptotically Sufficient Condition\nAU - Wang, Zimo\nAU - Wang, Cheng\nAU - Yoshino, Taiki\nAU - Tao, Sirui\nAU - Fu, Ziyang\nAU - Li, Tzu-Mao\nPY - 2025\nT2 - Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)\nSP - 1276\nEP - 1286\nPB - IEEE\nDO - 10.1109/CVPR52734.2025.00127\nUR - https://zeamoxwang.github.io/HotSpot-CVPR25/\nN1 - Highlight\nER -" } }, { "key": "tao2024designweaver", "slug": "designweaver", "entry_type": "inproceedings", "title": "DesignWeaver: Dimensional Scaffolding for Text-to-Image Product Design", "authors": [ { "name": "Sirui Tao", "given_name": "Sirui", "family_name": "Tao", "bibtex_name": "Tao, Sirui" }, { "name": "Ivan Liang", "given_name": "Ivan", "family_name": "Liang", "bibtex_name": "Liang, Ivan" }, { "name": "Cindy Peng", "given_name": "Cindy", "family_name": "Peng", "bibtex_name": "Peng, Cindy" }, { "name": "Zhiqing Wang", "given_name": "Zhiqing", "family_name": "Wang", "bibtex_name": "Wang, Zhiqing" }, { "name": "Srishti Palani", "given_name": "Srishti", "family_name": "Palani", "bibtex_name": "Palani, Srishti" }, { "name": "Steven P. Dow", "given_name": "Steven P.", "family_name": "Dow", "bibtex_name": "Dow, Steven P." } ], "venue": "Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems", "series": "CHI '25", "editors": [], "volume": null, "number": null, "pages": "1--26", "page_start": "1", "page_end": "26", "publisher": "ACM", "isbn": null, "abbreviation": "CHI", "note": null, "month": "apr", "month_numeric": 4, "year": 2025, "selected": true, "preview": "designweaver.png", "doi": "10.1145/3706598.3714211", "arxiv": "2502.09867", "work_type": "full-paper", "work_group": "full-paper", "abstract": "Generative AI has enabled novice designers to quickly create professional-looking visual representations for product concepts. However, novices have limited domain knowledge that could constrain their ability to write prompts that effectively explore a product design space. To understand how experts explore and communicate about design spaces, we conducted a formative study with 12 experienced product designers and found that experts—and their less-versed clients—often use visual references to guide co-design discussions rather than written descriptions. These insights inspired DesignWeaver, an interface that helps novices generate prompts for a text-to-image model by surfacing key product design dimensions from generated images into a palette for quick selection. In a study with 52 novices, DesignWeaver enabled participants to craft longer prompts with more domain-specific vocabularies, resulting in more diverse, innovative product designs. However, the nuanced prompts heightened participants' expectations beyond what current text-to-image models could deliver. We discuss implications for AI-based product design support tools.", "tldr": "DesignWeaver turns dimensions derived from design briefs and generated images into a selectable palette that helps novice product designers write richer prompts and explore a broader text-to-image design space.", "why_cite": { "statement": "Cite DesignWeaver when discussing dimensional scaffolding, novice prompt construction, or interface support for richer and more varied exploration in text-to-image product design.", "cite_when": [ "Designing interfaces that externalize product-design dimensions instead of relying on a blank prompt box.", "Studying how prompt scaffolds change vocabulary, iteration, visual diversity, novelty, and user expectations." ], "contributions": [ "Reports a formative study with 12 experienced product designers about how experts and clients communicate across a design space.", "Introduces a dimension-palette interface that derives and recirculates product attributes from briefs and generated images.", "Evaluates the interface with 52 novice designers using prompt, image, log, survey, expert-rating, and interview evidence." ], "evidence": [ "Participants wrote longer prompts (48.22 versus 23.73 words) and used more unique terms per prompt (24.48 versus 10.59); both Mann-Whitney comparisons report p < .001.", "Generated images were more diverse by CLIP similarity (0.863 versus 0.903, where lower means more diverse; p < .001), and expert-rated novelty was higher (4.09 versus 3.54; p = .002).", "Requirement alignment (p = .059), image satisfaction (p = .579), and expectation alignment (p = .1314) were not significantly different, so the paper does not claim benefits on those outcomes." ], "boundaries": [ "The controlled study involved 52 novices, ages 19–31, completing a chair-design task; experienced-designer, collaborative, and other-domain use remain open questions.", "Preset dimensions may constrain creativity as well as scaffold it.", "Richer prompts can raise expectations beyond what current text-to-image models deliver reliably." ] }, "authorship": { "role": "first-author", "role_label": "First and corresponding author", "statement": "Sirui Tao is the first and corresponding author. This guide does not assign unverified individual task contributions." }, "topics": [ "dimensional scaffolding", "text-to-image interaction", "product design", "creativity support tools", "novice designers" ], "related_project": "https://dylantao.github.io/projects/designweaver/", "provenance": { "reviewed_on": "2026-07-13", "basis": "Published CHI paper, arXiv record, project page, and code repository reviewed.", "sources": [ "https://doi.org/10.1145/3706598.3714211", "https://arxiv.org/abs/2502.09867", "https://dylantao.github.io/projects/designweaver/", "https://github.com/slimykat/DesignWeaver" ] }, "links": { "citation_page": "https://dylantao.github.io/publications/designweaver/", "citation_page_path": "/publications/designweaver/", "markdown": "https://dylantao.github.io/ai/papers/designweaver.md", "markdown_path": "/ai/papers/designweaver.md", "bibtex": "https://dylantao.github.io/ai/papers/designweaver.bib", "bibtex_path": "/ai/papers/designweaver.bib", "ris": "https://dylantao.github.io/ai/papers/designweaver.ris", "ris_path": "/ai/papers/designweaver.ris", "doi": "https://doi.org/10.1145/3706598.3714211", "arxiv": "https://arxiv.org/abs/2502.09867", "pdf": "https://dl.acm.org/doi/pdf/10.1145/3706598.3714211", "website": "https://dylantao.github.io/projects/designweaver/", "code": "https://github.com/slimykat/DesignWeaver", "video": "https://youtu.be/04s9TpR3KBg?si=KILidfm13yFI13Ya" }, "citation": { "bibtex": "@inproceedings{tao2024designweaver,\n title = {DesignWeaver: Dimensional Scaffolding for Text-to-Image Product Design},\n author = {Tao, Sirui and Liang, Ivan and Peng, Cindy and Wang, Zhiqing and Palani, Srishti and Dow, Steven P.},\n booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},\n series = {CHI '25},\n pages = {1--26},\n publisher = {ACM},\n doi = {10.1145/3706598.3714211},\n arxiv = {2502.09867},\n month = {apr},\n year = {2025}\n}", "ris": "TY - CPAPER\nTI - DesignWeaver: Dimensional Scaffolding for Text-to-Image Product Design\nAU - Tao, Sirui\nAU - Liang, Ivan\nAU - Peng, Cindy\nAU - Wang, Zhiqing\nAU - Palani, Srishti\nAU - Dow, Steven P.\nPY - 2025\nT2 - Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems\nT3 - CHI '25\nSP - 1\nEP - 26\nPB - ACM\nDO - 10.1145/3706598.3714211\nUR - https://dylantao.github.io/projects/designweaver/\nER -" } }, { "key": "tung2023physion++", "slug": "physion-plus-plus", "entry_type": "inproceedings", "title": "Physion++: Evaluating Physical Scene Understanding with Objects Consisting of Different Physical Attributes in Humans and Machines", "authors": [ { "name": "Hsiao-Yu Tung", "given_name": "Hsiao-Yu", "family_name": "Tung", "bibtex_name": "Tung, Hsiao-Yu" }, { "name": "Mingyu Ding", "given_name": "Mingyu", "family_name": "Ding", "bibtex_name": "Ding, Mingyu" }, { "name": "Zhenfang Chen", "given_name": "Zhenfang", "family_name": "Chen", "bibtex_name": "Chen, Zhenfang" }, { "name": "Sirui Tao", "given_name": "Sirui", "family_name": "Tao", "bibtex_name": "Tao, Sirui" }, { "name": "Vedang Lad", "given_name": "Vedang", "family_name": "Lad", "bibtex_name": "Lad, Vedang" }, { "name": "Daniel Bear", "given_name": "Daniel", "family_name": "Bear", "bibtex_name": "Bear, Daniel" }, { "name": "Chuang Gan", "given_name": "Chuang", "family_name": "Gan", "bibtex_name": "Gan, Chuang" }, { "name": "Josh Tenenbaum", "given_name": "Josh", "family_name": "Tenenbaum", "bibtex_name": "Tenenbaum, Josh" }, { "name": "Daniel Yamins", "given_name": "Daniel", "family_name": "Yamins", "bibtex_name": "Yamins, Daniel" }, { "name": "Judith Fan", "given_name": "Judith", "family_name": "Fan", "bibtex_name": "Fan, Judith" }, { "name": "Kevin Smith", "given_name": "Kevin", "family_name": "Smith", "bibtex_name": "Smith, Kevin" } ], "venue": "Proceedings of the Annual Meeting of the Cognitive Science Society", "series": null, "editors": [], "volume": null, "number": null, "pages": null, "page_start": null, "page_end": null, "publisher": null, "isbn": null, "abbreviation": "CogSci", "note": "Poster with abstract", "month": "jul", "month_numeric": 7, "year": 2023, "selected": false, "preview": "physion++.gif", "doi": null, "arxiv": null, "work_type": "poster-abstract", "work_group": "short-form", "abstract": "Human physical scene understanding requires more than simply localizing and recognizing objects—we can quickly adapt our predictions about how a scene will unfold by incorporating objects' latent physics properties, such as the masses of the objects in the scene. What are the underlying computational mechanisms that allow humans to infer these physical properties and adapt their physical predictions so efficiently from visual inputs? One hypothesis is that general intuitive physics knowledge can be learned from enough raw data, instantiated as computational models that predict future video frames in large datasets of complex scenes. To test this hypothesis, we evaluate existing state-of-the-art video models. We measured both model and human performance on Physion++, a novel dataset and benchmark that rigorously evaluates visual physical prediction in humans and machines, under circumstances where accurate physical prediction relies on accurate estimates of the latent physical properties of objects in the scene. Specifically, we tested scenarios where accurate prediction relied on accurate estimates of objects' mechanical properties, including masses, friction, elasticity and deformability, and the values of these mechanical properties could only be inferred by observing how these objects moved and interacted with other objects and/or fluids. We found that models that encode objectness and physical states tend to perform better, yet there is still a huge gap compared to human performance. We also found most models' predictions correlate poorly with that made by humans. These results show that current deep learning models that succeed in some settings nevertheless fail to achieve human-level physical prediction in other cases, especially those where latent property inference is required.", "tldr": "Physion++ evaluates physical prediction when mass, friction, elasticity, and deformability must be inferred online from how objects move and interact.", "why_cite": { "statement": "Cite this CogSci Physion++ record when motivating benchmarks for latent physical-property inference or human–model gaps in physical scene prediction.", "cite_when": [ "Studying physical prediction where key mechanical properties are not given and must be inferred from observed motion or interaction.", "Comparing human judgments with video, object-centric, or physical-state model predictions under changing latent properties." ], "contributions": [ "Introduces a benchmark focused on four latent mechanical-property families—mass, friction, elasticity, and deformability.", "Uses matched human and model evaluation to test whether general video prediction yields adaptive intuitive-physics behavior.", "Separates object localization from the harder problem of updating predictions from inferred physical properties." ], "evidence": [ "The exact CogSci abstract reports that models encoding objectness and physical state tend to perform better, while remaining far from human performance.", "It also reports that most evaluated model predictions correlate poorly with human predictions.", "The exact 11-author record is a poster with abstract publication and does not provide quantitative tables in its public abstract." ], "boundaries": [ "The public CogSci record supports qualitative, not paper-table-level quantitative, claims.", "A separate nine-author NeurIPS technical paper has a different title and author list; its numbers and DOI must not be silently attributed to this 11-author CogSci record.", "The benchmark concerns prediction settings where properties are inferred from observed motion and interaction, not all forms of physical reasoning." ] }, "authorship": { "role": "coauthor", "role_label": "Coauthor", "statement": "Sirui Tao is a coauthor on the 11-author CogSci record. This guide does not assign an unverified individual task contribution." }, "topics": [ "intuitive physics", "latent physical properties", "physical scene understanding", "human-model comparison", "video prediction" ], "related_project": "https://escholarship.org/uc/item/3x9960zn", "provenance": { "reviewed_on": "2026-07-13", "basis": "Exact eScholarship OAI metadata and public abstract reviewed; the PDF link was recorded; companion-paper claims were excluded.", "sources": [ "https://escholarship.org/uc/item/3x9960zn", "https://escholarship.org/content/qt3x9960zn/qt3x9960zn.pdf", "https://escholarship.org/oai?verb=GetRecord&metadataPrefix=oai_dc&identifier=oai:escholarship.org:ark:/13030/qt3x9960zn" ] }, "links": { "citation_page": "https://dylantao.github.io/publications/physion-plus-plus/", "citation_page_path": "/publications/physion-plus-plus/", "markdown": "https://dylantao.github.io/ai/papers/physion-plus-plus.md", "markdown_path": "/ai/papers/physion-plus-plus.md", "bibtex": "https://dylantao.github.io/ai/papers/physion-plus-plus.bib", "bibtex_path": "/ai/papers/physion-plus-plus.bib", "ris": "https://dylantao.github.io/ai/papers/physion-plus-plus.ris", "ris_path": "/ai/papers/physion-plus-plus.ris", "pdf": "https://escholarship.org/content/qt3x9960zn/qt3x9960zn.pdf", "website": "https://escholarship.org/uc/item/3x9960zn", "source_url": "https://escholarship.org/uc/item/3x9960zn" }, "citation": { "bibtex": "@inproceedings{tung2023physion++,\n title = {Physion++: Evaluating Physical Scene Understanding with Objects Consisting of Different Physical Attributes in Humans and Machines},\n author = {Tung, Hsiao-Yu and Ding, Mingyu and Chen, Zhenfang and Tao, Sirui and Lad, Vedang and Bear, Daniel and Gan, Chuang and Tenenbaum, Josh and Yamins, Daniel and Fan, Judith and Smith, Kevin},\n booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society},\n url = {https://escholarship.org/uc/item/3x9960zn},\n note = {Poster with abstract},\n month = {jul},\n year = {2023}\n}", "ris": "TY - CPAPER\nTI - Physion++: Evaluating Physical Scene Understanding with Objects Consisting of Different Physical Attributes in Humans and Machines\nAU - Tung, Hsiao-Yu\nAU - Ding, Mingyu\nAU - Chen, Zhenfang\nAU - Tao, Sirui\nAU - Lad, Vedang\nAU - Bear, Daniel\nAU - Gan, Chuang\nAU - Tenenbaum, Josh\nAU - Yamins, Daniel\nAU - Fan, Judith\nAU - Smith, Kevin\nPY - 2023\nT2 - Proceedings of the Annual Meeting of the Cognitive Science Society\nUR - https://escholarship.org/uc/item/3x9960zn\nN1 - Poster with abstract\nER -" } }, { "key": "bear2021physion", "slug": "physion", "entry_type": "inproceedings", "title": "Physion: Evaluating Physical Prediction from Vision in Humans and Machines", "authors": [ { "name": "Daniel Bear", "given_name": "Daniel", "family_name": "Bear", "bibtex_name": "Bear, Daniel" }, { "name": "Elias Wang", "given_name": "Elias", "family_name": "Wang", "bibtex_name": "Wang, Elias" }, { "name": "Damian Mrowca", "given_name": "Damian", "family_name": "Mrowca", "bibtex_name": "Mrowca, Damian" }, { "name": "Felix Binder", "given_name": "Felix", "family_name": "Binder", "bibtex_name": "Binder, Felix" }, { "name": "Hsiao-Yu Tung", "given_name": "Hsiao-Yu", "family_name": "Tung", "bibtex_name": "Tung, Hsiao-Yu" }, { "name": "Pramod RT", "given_name": "Pramod", "family_name": "RT", "bibtex_name": "RT, Pramod" }, { "name": "Cameron Holdaway", "given_name": "Cameron", "family_name": "Holdaway", "bibtex_name": "Holdaway, Cameron" }, { "name": "Sirui Tao", "given_name": "Sirui", "family_name": "Tao", "bibtex_name": "Tao, Sirui" }, { "name": "Kevin Smith", "given_name": "Kevin", "family_name": "Smith", "bibtex_name": "Smith, Kevin" }, { "name": "Fan-Yun Sun", "given_name": "Fan-Yun", "family_name": "Sun", "bibtex_name": "Sun, Fan-Yun" }, { "name": "Fei-Fei Li", "given_name": "Fei-Fei", "family_name": "Li", "bibtex_name": "Li, Fei-Fei" }, { "name": "Nancy Kanwisher", "given_name": "Nancy", "family_name": "Kanwisher", "bibtex_name": "Kanwisher, Nancy" }, { "name": "Josh Tenenbaum", "given_name": "Josh", "family_name": "Tenenbaum", "bibtex_name": "Tenenbaum, Josh" }, { "name": "Dan Yamins", "given_name": "Dan", "family_name": "Yamins", "bibtex_name": "Yamins, Dan" }, { "name": "Judith Fan", "given_name": "Judith", "family_name": "Fan", "bibtex_name": "Fan, Judith" } ], "venue": "Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks", "series": null, "editors": [ { "name": "J. Vanschoren", "given_name": "J.", "family_name": "Vanschoren", "bibtex_name": "Vanschoren, J." }, { "name": "S. Yeung", "given_name": "S.", "family_name": "Yeung", "bibtex_name": "Yeung, S." } ], "volume": "1", "number": null, "pages": null, "page_start": null, "page_end": null, "publisher": null, "isbn": null, "abbreviation": "NeurIPS", "note": null, "month": "dec", "month_numeric": 12, "year": 2021, "selected": true, "preview": "physion.gif", "doi": null, "arxiv": "2106.08261", "work_type": "full-paper", "work_group": "full-paper", "abstract": "While current vision algorithms excel at many challenging tasks, it is unclear how well they understand the physical dynamics of real-world environments. Here we introduce Physion, a dataset and benchmark for rigorously evaluating the ability to predict how physical scenarios will evolve over time. Our dataset features realistic simulations of a wide range of physical phenomena, including rigid and soft-body collisions, stable multi-object configurations, rolling, sliding, and projectile motion, thus providing a more comprehensive challenge than previous benchmarks. We used Physion to benchmark a suite of models varying in their architecture, learning objective, input-output structure, and training data. In parallel, we obtained precise measurements of human prediction behavior on the same set of scenarios, allowing us to directly evaluate how well any model could approximate human behavior. We found that vision algorithms that learn object-centric representations generally outperform those that do not, yet still fall far short of human performance. On the other hand, graph neural networks with direct access to physical state information both perform substantially better and make predictions that are more similar to those made by humans. These results suggest that extracting physical representations of scenes is the main bottleneck to achieving human-level and human-like physical understanding in vision algorithms. We have publicly released all data and code to facilitate the use of Physion to benchmark additional models in a fully reproducible manner, enabling systematic evaluation of progress towards vision algorithms that understand physical environments as robustly as people do.", "tldr": "Physion provides eight simulated scenario families and a model-agnostic object-contact prediction task for directly comparing human and model physical prediction.", "why_cite": { "statement": "Cite Physion for human-aligned intuitive-physics benchmarking, object-centric physical prediction, or generalization across diverse simulated scenario families.", "cite_when": [ "Evaluating whether a vision model predicts physical outcomes in ways that approach human accuracy and error patterns.", "Motivating object-centric representations, object-contact prediction, or transfer across physical scenario families." ], "contributions": [ "Releases a dataset spanning Dominoes, Support, Collide, Contain, Drop, Link, Roll, and Drape scenarios.", "Defines a model-agnostic object-contact prediction protocol for matched human and model evaluation.", "Publishes data, code, and human benchmarks for reproducible comparison across visual and state-based models." ], "evidence": [ "The human study recruited 800 participants, excluded 112 by a preregistered criterion, and analyzed 688; reported human accuracy was 0.71 (t = 27.5, p < 1e-7).", "Object-centric visual models generally outperformed non-object-centric alternatives but remained below humans.", "Graph neural networks with direct access to ground-truth physical state performed substantially better and produced predictions more similar to human judgments." ], "boundaries": [ "Physion uses synthetic ThreeDWorld scenes and a binary contact-prediction task; it does not cover all real-world materials, fluid behavior, jointed multipart objects, or property ranges.", "Particle-based state models receive ground-truth 3D physical state that a vision system must otherwise infer.", "Strong within-benchmark performance does not by itself establish broad real-world physical understanding or transfer from a single scenario family." ] }, "authorship": { "role": "coauthor", "role_label": "Coauthor", "statement": "Sirui Tao is a coauthor. This guide does not assign an unverified individual task contribution." }, "topics": [ "intuitive physics", "physical prediction", "object-centric vision", "human-model comparison", "benchmark datasets" ], "related_project": "https://dylantao.github.io/projects/physion/", "provenance": { "reviewed_on": "2026-07-13", "basis": "Official NeurIPS abstract and PDF, project page, and released code reviewed.", "sources": [ "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/d09bf41544a3365a46c9077ebb5e35c3-Abstract-round1.html", "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/file/d09bf41544a3365a46c9077ebb5e35c3-Paper-round1.pdf", "https://physion-benchmark.github.io/", "https://github.com/cogtoolslab/physics-benchmarking-neurips2021" ] }, "links": { "citation_page": "https://dylantao.github.io/publications/physion/", "citation_page_path": "/publications/physion/", "markdown": "https://dylantao.github.io/ai/papers/physion.md", "markdown_path": "/ai/papers/physion.md", "bibtex": "https://dylantao.github.io/ai/papers/physion.bib", "bibtex_path": "/ai/papers/physion.bib", "ris": "https://dylantao.github.io/ai/papers/physion.ris", "ris_path": "/ai/papers/physion.ris", "arxiv": "https://arxiv.org/abs/2106.08261", "pdf": "https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/d09bf41544a3365a46c9077ebb5e35c3-Paper-round1.pdf", "website": "https://physion-benchmark.github.io/", "code": "https://github.com/cogtoolslab/physics-benchmarking-neurips2021", "video": "https://www.youtube.com/watch?v=Jz7ImDazcJI&ab_channel=FelixBinder", "source_url": "https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/d09bf41544a3365a46c9077ebb5e35c3-Abstract-round1.html" }, "citation": { "bibtex": "@inproceedings{bear2021physion,\n title = {Physion: Evaluating Physical Prediction from Vision in Humans and Machines},\n author = {Bear, Daniel and Wang, Elias and Mrowca, Damian and Binder, Felix and Tung, Hsiao-Yu and RT, Pramod and Holdaway, Cameron and Tao, Sirui and Smith, Kevin and Sun, Fan-Yun and Li, Fei-Fei and Kanwisher, Nancy and Tenenbaum, Josh and Yamins, Dan and Fan, Judith},\n booktitle = {Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks},\n editor = {Vanschoren, J. and Yeung, S.},\n volume = {1},\n arxiv = {2106.08261},\n url = {https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/d09bf41544a3365a46c9077ebb5e35c3-Abstract-round1.html},\n month = {dec},\n year = {2021}\n}", "ris": "TY - CPAPER\nTI - Physion: Evaluating Physical Prediction from Vision in Humans and Machines\nAU - Bear, Daniel\nAU - Wang, Elias\nAU - Mrowca, Damian\nAU - Binder, Felix\nAU - Tung, Hsiao-Yu\nAU - RT, Pramod\nAU - Holdaway, Cameron\nAU - Tao, Sirui\nAU - Smith, Kevin\nAU - Sun, Fan-Yun\nAU - Li, Fei-Fei\nAU - Kanwisher, Nancy\nAU - Tenenbaum, Josh\nAU - Yamins, Dan\nAU - Fan, Judith\nPY - 2021\nT2 - Proceedings of the Neural Information Processing Systems Track on Datasets and Benchmarks\nVL - 1\nED - Vanschoren, J.\nED - Yeung, S.\nUR - https://physion-benchmark.github.io/\nER -" } } ] }