{ "meta": { "generated": "2026-09-11", "total": 109, "categories": { "ego": { "name": "Egocentric human", "blurb": "First-person capture of people doing things — the largest and cheapest source of manipulation data." }, "exo": { "name": "Third-person video", "blurb": "Exocentric human video: abundant, noisy, and the hardest viewpoint to transfer from." }, "umi": { "name": "UMI & handheld", "blurb": "Robot-free capture rigs that produce directly deployable demonstrations." }, "teleop": { "name": "Teleoperation", "blurb": "Demonstrations collected by a human driving the robot, through leader-follower, VR or exoskeleton." }, "robot": { "name": "Robot manipulation at scale", "blurb": "Large pooled corpora of real robot trajectories, single- and cross-embodiment." }, "dex": { "name": "Hand-object & dexterous", "blurb": "Fine-grained hand pose, contact and grasping — human and synthetic." }, "tactile": { "name": "Tactile", "blurb": "Touch data, paired with vision and language where it exists." }, "hri": { "name": "Human-robot interaction", "blurb": "Robots perceiving, navigating around, and physically exchanging objects with people." }, "multi": { "name": "Multi-robot & collaboration", "blurb": "Two or more agents coordinating on a shared task. The thinnest category here, which is itself the finding." }, "motion": { "name": "Human motion & humanoid", "blurb": "Mocap and pose corpora that humanoid whole-body controllers are trained on." }, "sim": { "name": "Simulation benchmarks", "blurb": "Task suites that define what progress is measured against." } }, "order": [ "ego", "umi", "teleop", "robot", "dex", "tactile", "hri", "multi", "exo", "motion", "sim" ] }, "datasets": [ { "id": "egocentric1m", "name": "Egocentric-1M", "title": "Egocentric-1M", "cat": "ego", "org": "industry", "one": "Roughly one million hours of factory-floor egocentric video — larger than every prior egocentric dataset combined, released open under Apache 2.0.", "device": "Build AI custom head-mounted glasses", "modal": "RGB (streamable, no full download required)", "hours": "~1,000,000 h", "eps": "10.8 billion frames · gated on Hugging Face", "tasks": "assembly, sorting, packaging, machining", "scenes": "real production floors, Southeast Asia", "subj": "14,228 factory workers", "embod": "human", "note": "Build AI's scaling ladder went 10K hours (Nov 2025) → 100K (Dec 2025) → 1M (Apr 2026). The framing to hold onto: this is industrial repetitive labour, not household diversity, so it is enormous and narrow at the same time. Whether that narrowness matters is an open empirical question and a very cheap one to test.", "arxiv": "", "authors": [], "year": 2026, "date": "2026-04-08", "institutions": [], "paper": "", "site": "https://huggingface.co/builddotai", "fig": "", "hours_num": 1000000.0 }, { "id": "humannet", "name": "HumanNet", "title": "HumanNet: Scaling Human-centric Video Learning to One Million Hours", "cat": "ego", "org": "industry", "one": "A human-centric video learning stack built to scale to one million hours, with controlled VLA ablations on the value of the data.", "device": "aggregated human video", "modal": "RGB, language, action labels", "hours": "1,000,000 h (stated)", "eps": "—", "tasks": "—", "scenes": "—", "subj": "—", "embod": "human → robot", "note": "Read it for the ablation rather than the corpus: it is one of the few papers that isolates what an extra order of magnitude of human video actually buys a VLA. Caveat: the million-hour figure is the paper's stated scale — its controlled ablations run at far smaller sizes, so treat the headline as a design target rather than a corpus you can download today.", "arxiv": "2605.06747", "authors": [ "Yufan Deng", "Daquan Zhou" ], "year": 2026, "date": "2026-05-07", "institutions": [], "paper": "https://arxiv.org/abs/2605.06747", "site": "https://arxiv.org/abs/2605.06747", "fig": "figures/ds-humannet.webp", "figs": [ "figures/ds-humannet-1.webp", "figures/ds-humannet-2.webp" ], "hours_num": 1000000.0 }, { "id": "egocentric100k", "name": "Egocentric-100K", "title": "Egocentric-100K", "cat": "ego", "org": "industry", "one": "The 100,000-hour rung of Build AI's ladder — the release that made million-hour egocentric look inevitable.", "device": "Build AI head-mounted glasses", "modal": "RGB", "hours": "100,000 h", "eps": "—", "tasks": "industrial manual work", "scenes": "factory floors", "subj": "—", "embod": "human", "note": "", "arxiv": "", "authors": [], "year": 2025, "date": "2025-12-01", "institutions": [], "paper": "", "site": "https://huggingface.co/datasets/builddotai/Egocentric-100K", "fig": "", "hours_num": 100000.0 }, { "id": "egoscale", "name": "EgoScale", "title": "EgoScale: Scaling Dexterous Manipulation with Diverse Egocentric Human Data", "cat": "ego", "org": "industry", "one": "NVIDIA GEAR's scaling study: a VLA trained on 20,854 hours of action-labelled egocentric human video, with a measured log-linear scaling law.", "device": "aggregated egocentric + retargeting pipeline", "modal": "RGB, wrist motion, retargeted dexterous hand actions", "hours": "20,854 h", "eps": "—", "tasks": "dexterous manipulation", "scenes": "—", "subj": "—", "embod": "human → dexterous robot", "note": "The most important number in this table for your ego-dexterity idea: validation loss falls log-linearly with human data scale, and that loss correlates with real-robot performance. It turns 'more human video helps' from a hope into a predictable curve.", "arxiv": "2602.16710", "authors": [ "Ruijie Zheng", "Dantong Niu", "Yuqi Xie", "Jing Wang", "Mengda Xu", "Yunfan Jiang", "Fernando Castañeda", "Fengyuan Hu", "You Liang Tan", "Letian Fu", "Trevor Darrell", "Furong Huang", "Yuke Zhu", "Danfei Xu", "Linxi Fan" ], "year": 2026, "date": "2026-02-18", "institutions": [], "paper": "https://arxiv.org/abs/2602.16710", "site": "https://research.nvidia.com/labs/gear/egoscale/", "fig": "figures/ds-egoscale.webp", "figs": [ "figures/ds-egoscale-1.webp" ], "hours_num": 20854.0 }, { "id": "ego2robot", "name": "Ego2Robot", "title": "Ego2Robot: Scalable Robot Data Synthesis from Egocentric Human Data", "cat": "ego", "org": "academia", "one": "A pipeline that converts egocentric human video into robot training data, producing 18,561 hours across 15 robot morphologies — the largest ego-to-robot corpus to date.", "device": "egocentric video → retargeting → visual synthesis", "modal": "RGB (synthesised robot arm), retargeted actions", "hours": "18,561 h", "eps": "—", "tasks": "manipulation", "scenes": "curated + in-the-wild", "subj": "15 robot morphologies", "embod": "15 robot morphologies", "note": "The other half of the million-hour story: raw human hours are useless until something turns them into actions. Action retargeting + robot-arm visual synthesis + multi-level quality curation is the recipe.", "arxiv": "2608.02580", "authors": [ "Ye Wang", "Pei Lin", "Xiong-Hui Chen", "Haoqi Yuan", "Zhixuan Liang", "Yiyang Huang", "Anzhe Chen", "Zixing Lei", "Jie Zhang", "Tao Zhang", "Haoyang Li", "Tong Zhang", "Chenxi Xiao", "Ziyuan Jiao", "Qin Jin" ], "year": 2026, "date": "2026-08-03", "institutions": [], "paper": "https://arxiv.org/abs/2608.02580", "site": "https://arxiv.org/abs/2608.02580", "fig": "figures/ds-ego2robot.webp", "figs": [ "figures/ds-ego2robot-1.webp" ], "hours_num": 18561.0 }, { "id": "xperience10m", "name": "Xperience-10M", "title": "Xperience-10M", "cat": "ego", "org": "industry", "one": "Ten thousand hours of egocentric recording with six synchronised RGB streams, stereo depth, SLAM pose and full hand-plus-body motion capture — roughly a petabyte.", "device": "Ropedia multi-camera capture rig", "modal": "6× RGB, audio, stereo depth, camera pose/SLAM, hand mocap, full-body mocap, IMU, hierarchical language", "hours": "10,000 h", "eps": "10M experiences · 2.88B RGB + 720M depth + 576M pose/mocap frames", "tasks": "hierarchical task/sub-task annotation", "scenes": "—", "subj": "—", "embod": "human", "note": "The richest egocentric annotation stack that exists — depth, SLAM and mocap all synchronised. Where Egocentric-1M has scale and nothing else, this has everything a real-to-sim or retargeting pipeline needs. If you only adopt one corpus for the Real2Sim2Real project, it is probably this one.", "arxiv": "", "authors": [], "year": 2026, "date": "2026-01-01", "institutions": [], "paper": "", "site": "https://huggingface.co/datasets/ropedia-ai/xperience-10m", "fig": "", "hours_num": 10000.0 }, { "id": "egocentric10k", "name": "Egocentric-10K", "title": "Egocentric-10K", "cat": "ego", "org": "industry", "one": "The first rung: 10,000 hours of factory egocentric video, November 2025.", "device": "Build AI head-mounted glasses", "modal": "RGB", "hours": "10,000 h", "eps": "—", "tasks": "industrial manual work", "scenes": "factory floors", "subj": "—", "embod": "human", "note": "", "arxiv": "", "authors": [], "year": 2025, "date": "2025-11-01", "institutions": [], "paper": "", "site": "https://huggingface.co/datasets/builddotai/Egocentric-10K", "fig": "", "hours_num": 10000.0 }, { "id": "aceego0", "name": "ACE-Ego-0", "title": "ACE-Ego-0: Unifying Egocentric Human and Robotic Data for VLA Pretraining", "cat": "ego", "org": "academia", "one": "A VLA pretraining framework that mixes robot, simulation and pseudo-action-labelled egocentric human data in one recipe.", "device": "aggregated", "modal": "RGB, robot actions, pseudo-action labels", "hours": "4.53K h robot+sim · 1.48K h human", "eps": "—", "tasks": "manipulation", "scenes": "—", "subj": "—", "embod": "multiple", "note": "The mixing ratio between robot, sim and human data is the real content here — the question everyone training a VLA has to answer and almost nobody reports.", "arxiv": "2606.17200", "authors": [ "Hao Li", "Ganlong Zhao", "Yufei Liu", "Haotian Hou", "Guoquan Ye", "Tongyan Fang", "Chunxiao Liu", "Sicong Huang", "Jianbo Liu", "Xiaogang Wang", "Hongsheng Li" ], "year": 2026, "date": "2026-06-15", "institutions": [], "paper": "https://arxiv.org/abs/2606.17200", "site": "https://arxiv.org/abs/2606.17200", "fig": "figures/ds-aceego0.webp", "figs": [ "figures/ds-aceego0-1.webp", "figures/ds-aceego0-2.webp" ], "hours_num": 4530.0 }, { "id": "ego4d", "name": "Ego4D", "title": "Ego4D: Around the World in 3,000 Hours of Egocentric Video", "cat": "ego", "org": "consortium", "one": "The foundational egocentric corpus: daily-life video from 931 wearers in 9 countries, with benchmark suites for memory, forecasting and social understanding.", "device": "Head-mounted cameras (Vuzix, GoPro, Aria, Weeview, Pupil)", "modal": "RGB, audio, 3D scans, eye gaze, IMU, stereo (subset)", "hours": "3,670 h", "eps": "3,670 h of video", "tasks": "hundreds of scenarios", "scenes": "74 locations", "subj": "931 wearers", "embod": "human", "note": "The substrate under almost every 'learn from human video' result. No native hand-pose labels — that gap is why EgoDex and OpenEgo exist.", "arxiv": "2110.07058", "authors": [ "Kristen Grauman", "Andrew Westbury", "Eugene Byrne", "Zachary Chavis", "Antonino Furnari", "Rohit Girdhar", "Jackson Hamburger", "Hao Jiang", "Miao Liu", "Xingyu Liu", "Miguel Martin", "Tushar Nagarajan", "Ilija Radosavovic", "Santhosh Kumar Ramakrishnan", "Fiona Ryan", "Jayant Sharma", "Michael Wray", "Mengmeng Xu", "Eric Zhongcong Xu", "Chen Zhao", "Siddhant Bansal", "Dhruv Batra", "Vincent Cartillier", "Sean Crane", "Tien Do", "Morrie Doulaty", "Akshay Erapalli", "Christoph Feichtenhofer", "Adriano Fragomeni", "Qichen Fu", "Abrham Gebreselasie", "Cristina Gonzalez", "James Hillis", "Xuhua Huang", "Yifei Huang", "Wenqi Jia", "Weslie Khoo", "Jachym Kolar", "Satwik Kottur", "Anurag Kumar", "Federico Landini", "Chao Li", "Yanghao Li", "Zhenqiang Li", "Karttikeya Mangalam", "Raghava Modhugu", "Jonathan Munro", "Tullie Murrell", "Takumi Nishiyasu", "Will Price", "Paola Ruiz Puentes", "Merey Ramazanova", "Leda Sari", "Kiran Somasundaram", "Audrey Southerland", "Yusuke Sugano", "Ruijie Tao", "Minh Vo", "Yuchen Wang", "Xindi Wu", "Takuma Yagi", "Ziwei Zhao", "Yunyi Zhu", "Pablo Arbelaez", "David Crandall", "Dima Damen", "Giovanni Maria Farinella", "Christian Fuegen", "Bernard Ghanem", "Vamsi Krishna Ithapu", "C. V. Jawahar", "Hanbyul Joo", "Kris Kitani", "Haizhou Li", "Richard Newcombe", "Aude Oliva", "Hyun Soo Park", "James M. Rehg", "Yoichi Sato", "Jianbo Shi", "Mike Zheng Shou", "Antonio Torralba", "Lorenzo Torresani", "Mingfei Yan", "Jitendra Malik" ], "year": 2021, "date": "2021-10-13", "institutions": [ "Meta AI", "UT Austin", "88 institutions" ], "paper": "https://arxiv.org/abs/2110.07058", "site": "https://ego4d-data.org/", "fig": "figures/ds-ego4d.webp", "figs": [ "figures/ds-ego4d-1.webp", "figures/ds-ego4d-2.webp" ], "hours_num": 3670.0 }, { "id": "egoverse", "name": "EgoVerse", "title": "EgoVerse: An Egocentric Human Dataset for Robot Learning from Around the World", "cat": "ego", "org": "consortium", "one": "A collaborative, standardised egocentric platform pooling human demonstrations from labs and industry worldwide.", "device": "mixed (community contributed)", "modal": "RGB, hand pose, manipulation annotations", "hours": "1,362 h", "eps": "80K episodes", "tasks": "1,965 tasks", "scenes": "240 scenes", "subj": "2,087 demonstrators", "embod": "human", "note": "The largest single egocentric manipulation release, and the first serious attempt at a shared contribution standard rather than another siloed corpus.", "arxiv": "2604.07607", "authors": [ "Ryan Punamiya", "Simar Kareer", "Zeyi Liu", "Josh Citron", "Ri-Zhao Qiu", "Xiongyi Cai", "Alexey Gavryushin", "Jiaqi Chen", "Davide Liconti", "Lawrence Y. Zhu", "Patcharapong Aphiwetsa", "Baoyu Li", "Aniketh Cheluva", "Pranav Kuppili", "Yangcen Liu", "Dhruv Patel", "Aidan Gao", "Hye-Young Chung", "Ryan Co", "Renee Zbizika", "Jeff Liu", "Xiaomeng Xu", "Haoyu Xiong", "Geng Chen", "Sebastiano Oliani", "Wenkai Xuan", "Chenyu Yang", "Xi Wang", "James Fort", "Richard Newcombe", "Josh Gao", "Jason Chong", "Garrett Matsuda", "Aseem Doriwala", "Marc Pollefeys", "Robert Katzschmann", "Xiaolong Wang", "Shuran Song", "Judy Hoffman", "Danfei Xu" ], "year": 2026, "date": "2026-04-08", "institutions": [], "paper": "https://arxiv.org/abs/2604.07607", "site": "https://arxiv.org/abs/2604.07607", "fig": "figures/ds-egoverse.webp", "figs": [ "figures/ds-egoverse-1.webp", "figures/ds-egoverse-2.webp" ], "hours_num": 1362.0 }, { "id": "egoexo4d", "name": "Ego-Exo4D", "title": "Ego-Exo4D: Understanding Skilled Human Activity from First- and Third-Person Perspectives", "cat": "ego", "org": "consortium", "one": "Time-synchronised first- and third-person video of the same skilled activity — the dataset that turns 'is ego better than exo?' into an experiment.", "device": "Aria glasses (ego) + 4-5 static GoPros (exo)", "modal": "RGB, multi-view, audio, eye gaze, IMU, 3D pose, expert commentary", "hours": "1,286 h", "eps": "5,035 takes", "tasks": "8 skilled domains", "scenes": "13 cities", "subj": "740 participants", "embod": "human", "note": "The only large corpus with genuinely paired ego/exo views. Central to the third-person-learning idea.", "arxiv": "2311.18259", "authors": [ "Kristen Grauman", "Andrew Westbury", "Lorenzo Torresani", "Kris Kitani", "Jitendra Malik", "Triantafyllos Afouras", "Kumar Ashutosh", "Vijay Baiyya", "Siddhant Bansal", "Bikram Boote", "Eugene Byrne", "Zach Chavis", "Joya Chen", "Feng Cheng", "Fu-Jen Chu", "Sean Crane", "Avijit Dasgupta", "Jing Dong", "Maria Escobar", "Cristhian Forigua", "Abrham Gebreselasie", "Sanjay Haresh", "Jing Huang", "Md Mohaiminul Islam", "Suyog Jain", "Rawal Khirodkar", "Devansh Kukreja", "Kevin J Liang", "Jia-Wei Liu", "Sagnik Majumder", "Yongsen Mao", "Miguel Martin", "Effrosyni Mavroudi", "Tushar Nagarajan", "Francesco Ragusa", "Santhosh Kumar Ramakrishnan", "Luigi Seminara", "Arjun Somayazulu", "Yale Song", "Shan Su", "Zihui Xue", "Edward Zhang", "Jinxu Zhang", "Angela Castillo", "Changan Chen", "Xinzhu Fu", "Ryosuke Furuta", "Cristina Gonzalez", "Prince Gupta", "Jiabo Hu", "Yifei Huang", "Yiming Huang", "Weslie Khoo", "Anush Kumar", "Robert Kuo", "Sach Lakhavani", "Miao Liu", "Mi Luo", "Zhengyi Luo", "Brighid Meredith", "Austin Miller", "Oluwatumininu Oguntola", "Xiaqing Pan", "Penny Peng", "Shraman Pramanick", "Merey Ramazanova", "Fiona Ryan", "Wei Shan", "Kiran Somasundaram", "Chenan Song", "Audrey Southerland", "Masatoshi Tateno", "Huiyu Wang", "Yuchen Wang", "Takuma Yagi", "Mingfei Yan", "Xitong Yang", "Zecheng Yu", "Shengxin Cindy Zha", "Chen Zhao", "Ziwei Zhao", "Zhifan Zhu", "Jeff Zhuo", "Pablo Arbelaez", "Gedas Bertasius", "David Crandall", "Dima Damen", "Jakob Engel", "Giovanni Maria Farinella", "Antonino Furnari", "Bernard Ghanem", "Judy Hoffman", "C. V. Jawahar", "Richard Newcombe", "Hyun Soo Park", "James M. Rehg", "Yoichi Sato", "Manolis Savva", "Jianbo Shi", "Mike Zheng Shou", "Michael Wray" ], "year": 2023, "date": "2023-11-30", "institutions": [ "Meta AI", "15 universities" ], "paper": "https://arxiv.org/abs/2311.18259", "site": "https://ego-exo4d-data.org/", "fig": "figures/ds-egoexo4d.webp", "figs": [ "figures/ds-egoexo4d-1.webp" ], "hours_num": 1286.0 }, { "id": "openego", "name": "OpenEgo", "title": "OpenEgo: A Large-Scale Multimodal Egocentric Dataset for Dexterous Manipulation", "cat": "ego", "org": "academia", "one": "Six public egocentric datasets unified into one schema with hand-pose and manipulation annotations.", "device": "mixed (aggregated)", "modal": "RGB, 3D hand pose, language", "hours": "1,107 h", "eps": "—", "tasks": "290 manipulation tasks", "scenes": "600+ environments", "subj": "—", "embod": "human", "note": "Aggregation, not new capture — but the harmonised schema saves weeks of data engineering.", "arxiv": "2509.05513", "authors": [ "Ahad Jawaid", "Yu Xiang" ], "year": 2025, "date": "2025-09-05", "institutions": [], "paper": "https://arxiv.org/abs/2509.05513", "site": "https://arxiv.org/abs/2509.05513", "fig": "figures/ds-openego.webp", "hours_num": 1107.0 }, { "id": "egodex", "name": "EgoDex", "title": "EgoDex: Learning Dexterous Manipulation from Large-Scale Egocentric Video", "cat": "ego", "org": "industry", "one": "Apple's large-scale egocentric video with paired 3D hand and finger tracking captured at record time, aimed squarely at dexterous manipulation.", "device": "Apple Vision Pro", "modal": "RGB 1080p/30 Hz, 3D hand + finger pose, camera pose", "hours": "829 h", "eps": "338K episodes", "tasks": "194 tasks", "scenes": "—", "subj": "—", "embod": "human", "note": "Solves Ego4D's biggest gap — native, accurate hand pose at scale. The most useful single corpus for the ego-dexterity idea.", "arxiv": "2505.11709", "authors": [ "Ryan Hoque", "Peide Huang", "David J. Yoon", "Mouli Sivapurapu", "Jian Zhang" ], "year": 2025, "date": "2025-05-16", "institutions": [], "paper": "https://arxiv.org/abs/2505.11709", "site": "https://github.com/apple/ml-egodex", "fig": "figures/ds-egodex.webp", "figs": [ "figures/ds-egodex-1.webp", "figures/ds-egodex-2.webp" ], "hours_num": 829.0 }, { "id": "nymeria", "name": "Nymeria", "title": "Nymeria: A Massive Collection of Multimodal Egocentric Daily Motion in the Wild", "cat": "ego", "org": "industry", "one": "Egocentric daily motion in the wild with full-body ground truth — the largest paired ego-video-plus-motion-capture collection.", "device": "Project Aria glasses + wrist Aria + XSens body suit", "modal": "RGB, SLAM, eye gaze, IMU, full-body 3D motion, language narration", "hours": "300 h", "eps": "1,200 sequences · 399 km travelled", "tasks": "daily activity", "scenes": "50 locations", "subj": "264 participants", "embod": "human", "note": "The body-motion counterpart to Ego4D. If you want humanoid whole-body targets from ego video, this is the supervision.", "arxiv": "2406.09905", "authors": [ "Lingni Ma", "Yuting Ye", "Fangzhou Hong", "Vladimir Guzov", "Yifeng Jiang", "Rowan Postyeni", "Luis Pesqueira", "Alexander Gamino", "Vijay Baiyya", "Hyo Jin Kim", "Kevin Bailey", "David Soriano Fosas", "C. Karen Liu", "Ziwei Liu", "Jakob Engel", "Renzo De Nardi", "Richard Newcombe" ], "year": 2024, "date": "2024-06-14", "institutions": [], "paper": "https://arxiv.org/abs/2406.09905", "site": "https://github.com/facebookresearch/nymeria_dataset", "fig": "figures/ds-nymeria.webp", "figs": [ "figures/ds-nymeria-1.webp" ], "hours_num": 300.0 }, { "id": "egoexolearn", "name": "EgoExoLearn", "title": "EgoExoLearn: A Dataset for Bridging Asynchronous Ego- and Exo-centric View of Procedural Activities in Real World", "cat": "ego", "org": "academia", "one": "Asynchronous ego and exo video of the same procedures — humans watching a demonstration and then doing the task themselves.", "device": "Head-mounted camera + fixed demonstration video", "modal": "RGB, gaze, audio, procedure annotations", "hours": "120 h", "eps": "—", "tasks": "daily + laboratory procedures", "scenes": "daily life + labs", "subj": "—", "embod": "human", "note": "Asynchronous rather than simultaneous ego/exo — closer to how a robot would actually learn by watching.", "arxiv": "2403.16182", "authors": [ "Yifei Huang", "Guo Chen", "Jilan Xu", "Mingfang Zhang", "Lijin Yang", "Baoqi Pei", "Hongjie Zhang", "Lu Dong", "Yali Wang", "Limin Wang", "Yu Qiao" ], "year": 2024, "date": "2024-03-24", "institutions": [], "paper": "https://arxiv.org/abs/2403.16182", "site": "https://github.com/OpenGVLab/EgoExoLearn", "fig": "figures/ds-egoexolearn.webp", "figs": [ "figures/ds-egoexolearn-1.webp", "figures/ds-egoexolearn-2.webp" ], "hours_num": 120.0 }, { "id": "epic100", "name": "Rescaling Egocentric Vision", "title": "Rescaling Egocentric Vision", "cat": "ego", "org": "academia", "one": "Long-term unscripted kitchen activity from head-mounted cameras, densely annotated with fine-grained actions.", "device": "Head-mounted GoPro", "modal": "RGB, audio, action segments", "hours": "100 h", "eps": "700 videos · 20M frames", "tasks": "90K action segments", "scenes": "45 kitchens", "subj": "45 participants", "embod": "human", "note": "The precision standard for egocentric action annotation; small next to Ego4D but far more densely labelled.", "arxiv": "2006.13256", "authors": [ "Dima Damen", "Hazel Doughty", "Giovanni Maria Farinella", "Antonino Furnari", "Evangelos Kazakos", "Jian Ma", "Davide Moltisanti", "Jonathan Munro", "Toby Perrett", "Will Price", "Michael Wray" ], "year": 2020, "date": "2020-06-23", "institutions": [], "paper": "https://arxiv.org/abs/2006.13256", "site": "https://epic-kitchens.github.io/", "fig": "figures/ds-epic100.webp", "figs": [ "figures/ds-epic100-1.webp", "figures/ds-epic100-2.webp" ], "hours_num": 100.0 }, { "id": "epicfields", "name": "EPIC Fields", "title": "EPIC Fields: Marrying 3D Geometry and Video Understanding", "cat": "ego", "org": "academia", "one": "EPIC-KITCHENS re-registered in 3D — camera poses and neural-rendering-ready reconstructions for 96% of its videos.", "device": "Head-mounted GoPro + COLMAP reconstruction", "modal": "RGB, camera poses, 3D geometry", "hours": "99 h", "eps": "19M registered frames", "tasks": "—", "scenes": "45 kitchens", "subj": "—", "embod": "human", "note": "Directly relevant to real-to-sim: it is egocentric video already solved for geometry.", "arxiv": "2306.08731", "authors": [ "Vadim Tschernezki", "Ahmad Darkhalil", "Zhifan Zhu", "David Fouhey", "Iro Laina", "Diane Larlus", "Dima Damen", "Andrea Vedaldi" ], "year": 2023, "date": "2023-06-14", "institutions": [], "paper": "https://arxiv.org/abs/2306.08731", "site": "https://epic-kitchens.github.io/epic-fields/", "fig": "figures/ds-epicfields.webp", "figs": [ "figures/ds-epicfields-1.webp", "figures/ds-epicfields-2.webp" ], "hours_num": 99.0 }, { "id": "hot3d", "name": "HOT3D", "title": "HOT3D: Hand and Object Tracking in 3D from Egocentric Multi-View Videos", "cat": "ego", "org": "industry", "one": "Egocentric multi-view hand and object tracking in 3D, with laser-scanned object models and ground-truth poses.", "device": "Project Aria + Quest 3", "modal": "RGB, monochrome, 3D hand pose, object 6D pose, eye gaze", "hours": "13.9 h (833 min)", "eps": "3.7M+ images", "tasks": "hand-object tracking", "scenes": "—", "subj": "19 subjects", "embod": "human", "note": "The precision benchmark for egocentric hand-object tracking — use it to validate your pose pipeline before trusting it on Ego4D.", "arxiv": "2411.19167", "authors": [ "Prithviraj Banerjee", "Sindi Shkodrani", "Pierre Moulon", "Shreyas Hampali", "Shangchen Han", "Fan Zhang", "Linguang Zhang", "Jade Fountain", "Edward Miller", "Selen Basol", "Richard Newcombe", "Robert Wang", "Jakob Julian Engel", "Tomas Hodan" ], "year": 2024, "date": "2024-11-28", "institutions": [], "paper": "https://arxiv.org/abs/2411.19167", "site": "https://facebookresearch.github.io/hot3d/", "fig": "figures/ds-hot3d.webp", "figs": [ "figures/ds-hot3d-1.webp", "figures/ds-hot3d-2.webp" ], "hours_num": 13.9 }, { "id": "egoemg", "name": "EgoEMG", "title": "EgoEMG: A Multimodal Egocentric Dataset with Bilateral EMG and Vision for Hand Pose Estimation", "cat": "ego", "org": "academia", "one": "Egocentric video paired with bilateral wrist EMG — muscle activity as a modality vision cannot see.", "device": "2× 8-channel wrist EMG @2 kHz, IMU, ego RGB, external RGB-D, mocap", "modal": "EMG, IMU, RGB, RGB-D, hand mocap", "hours": "10+ h", "eps": "—", "tasks": "60 gesture classes (30 single-hand, 30 bimanual)", "scenes": "lab", "subj": "41 participants", "embod": "human", "note": "Unusual and under-explored: EMG carries intent and force before motion is visible, which is exactly the gap a vision-only world model has.", "arxiv": "2605.05712", "authors": [ "Ziheng Xi", "Jiayi Yu", "Yitao Wang", "Yanbo Duan", "Jianjiang Feng", "Jie Zhou" ], "year": 2026, "date": "2026-05-07", "institutions": [], "paper": "https://arxiv.org/abs/2605.05712", "site": "https://arxiv.org/abs/2605.05712", "fig": "figures/ds-egoemg.webp", "figs": [ "figures/ds-egoemg-1.webp", "figures/ds-egoemg-2.webp" ], "hours_num": 10.0 }, { "id": "humanego", "name": "HumanEgo", "title": "HumanEgo: Zero-Shot Robot Learning from Minutes of Human Egocentric Videos", "cat": "ego", "org": "academia", "one": "The opposite bet from scaling: zero-shot robot learning from as little as 30 minutes of human egocentric video per task.", "device": "head-mounted camera", "modal": "RGB, hand pose", "hours": "0.5 h per task", "eps": "—", "tasks": "4 real-world tasks", "scenes": "—", "subj": "—", "embod": "robot arm", "note": "92.5% success from 30 minutes of human video, beating matched-time robot teleoperation by 41%. The sharpest counterpoint in this whole table to the million-hour thesis — worth taking seriously before you commit to a collection campaign.", "arxiv": "2605.24934", "authors": [ "Zhi", "Botao He", "Kelin Yu", "Seungjae Lee", "Ruohan Gao", "Furong Huang", "Yiannis Aloimonos" ], "year": 2026, "date": "2026-05-24", "institutions": [], "paper": "https://arxiv.org/abs/2605.24934", "site": "https://arxiv.org/abs/2605.24934", "fig": "figures/ds-humanego.webp", "figs": [ "figures/ds-humanego-1.webp" ], "hours_num": 0.5 }, { "id": "egokit", "name": "EgoKit", "title": "EgoKit: Towards Unified Low-Cost Egocentric Data Collection with Heterogeneous Devices", "cat": "ego", "org": "academia", "one": "An open, low-cost toolkit for collecting egocentric data across heterogeneous devices, including XR headsets with 26-joint hand tracking.", "device": "phones, action cams, XR headsets (unified)", "modal": "RGB, head pose, OpenXR 26-joint hand tracking", "hours": "—", "eps": "—", "tasks": "—", "scenes": "—", "subj": "—", "embod": "human", "note": "If you want to run your own egocentric collection rather than consume someone else's, this is the cheapest credible starting point.", "arxiv": "2605.16797", "authors": [ "Liuchuan Yu", "Erdem Murat", "Beichen Wang", "Yan Zeng", "Tingting Luo", "Huizhen Zhou", "Shanghao Li", "Huining Feng", "Zhigen Zhao", "Ning Yang", "Ke Jing", "Yunhao Liu", "Ruoya Sheng" ], "year": 2026, "date": "2026-05-16", "institutions": [], "paper": "https://arxiv.org/abs/2605.16797", "site": "https://arxiv.org/abs/2605.16797", "fig": "figures/ds-egokit.webp", "figs": [ "figures/ds-egokit-1.webp", "figures/ds-egokit-2.webp" ], "hours_num": -1 }, { "id": "egolive", "name": "EgoLive", "title": "EgoLive: A Large-Scale Egocentric Dataset from Real-World Human Tasks", "cat": "ego", "org": "industry", "one": "Large-scale egocentric capture from real-world human tasks, positioned on breadth of semantic coverage rather than raw hours.", "device": "head-mounted capture", "modal": "RGB, task annotations", "hours": "—", "eps": "—", "tasks": "real-world human tasks", "scenes": "in-the-wild", "subj": "—", "embod": "human", "note": "Argues that the long tail of semantics matters more than hour count, and benchmarks itself against EgoDex and Xperience-10M on exactly that. Worth reading as the counter-argument to pure scaling.", "arxiv": "2604.23570", "authors": [ "Yihang Li", "Xuelong Wei", "Jingzhou Luo", "Yingjing Xiao", "Yibo Bai", "Guangyuan Zhou", "Teng Zou", "Chenguang Gui", "Jiajun Wen", "He Zhang", "Kangliang Chen", "Xing Pan", "Shuaiyan Liu", "Daming Wang", "Tao An", "Jiayi Li", "Shibo Jin", "Wanwan Zhang", "Tianyu Wang", "Boren Wei", "Zhixuan Huang", "Fangsheng Liu", "Ruodai Li", "Hui Zhang", "Anson Li", "Yicheng Gong", "Peng Cao", "Jiaming Liang", "Liang Lin" ], "year": 2026, "date": "2026-04-26", "institutions": [], "paper": "https://arxiv.org/abs/2604.23570", "site": "https://arxiv.org/abs/2604.23570", "fig": "figures/ds-egolive.webp", "figs": [ "figures/ds-egolive-1.webp" ], "hours_num": -1 }, { "id": "egovla", "name": "EgoVLA", "title": "EgoVLA: Learning Vision-Language-Action Models from Egocentric Human Videos", "cat": "ego", "org": "academia", "one": "Not a dataset but the reference recipe: train a VLA on egocentric human video, then transfer it to a robot.", "device": "—", "modal": "RGB, hand pose", "hours": "—", "eps": "—", "tasks": "—", "scenes": "—", "subj": "—", "embod": "human → robot", "note": "Read alongside the ego datasets — it tells you what preprocessing they actually need to be usable.", "arxiv": "2507.12440", "authors": [ "Ruihan Yang", "Qinxi Yu", "Yecheng Wu", "Rui Yan", "Borui Li", "An-Chieh Cheng", "Xueyan Zou", "Yunhao Fang", "Xuxin Cheng", "Ri-Zhao Qiu", "Hongxu Yin", "Sifei Liu", "Song Han", "Yao Lu", "Xiaolong Wang" ], "year": 2025, "date": "2025-07-16", "institutions": [], "paper": "https://arxiv.org/abs/2507.12440", "site": "https://arxiv.org/abs/2507.12440", "fig": "figures/ds-egovla.webp", "figs": [ "figures/ds-egovla-1.webp", "figures/ds-egovla-2.webp" ], "hours_num": -1 }, { "id": "hrdt", "name": "H-RDT", "title": "H-RDT: Human Manipulation Enhanced Bimanual Robotic Manipulation", "cat": "ego", "org": "academia", "one": "Bimanual manipulation improved by pretraining on large-scale egocentric human video with paired 3D hand poses.", "device": "aggregated egocentric + bimanual robot", "modal": "RGB, 3D hand pose, robot actions", "hours": "—", "eps": "—", "tasks": "bimanual manipulation", "scenes": "sim + real", "subj": "—", "embod": "bimanual", "note": "", "arxiv": "2507.23523", "authors": [ "Hongzhe Bi", "Lingxuan Wu", "Tianwei Lin", "Hengkai Tan", "Zhizhong Su", "Hang Su", "Jun Zhu" ], "year": 2025, "date": "2025-07-31", "institutions": [], "paper": "https://arxiv.org/abs/2507.23523", "site": "https://arxiv.org/abs/2507.23523", "fig": "figures/ds-hrdt.webp", "figs": [ "figures/ds-hrdt-1.webp" ], "hours_num": -1 }, { "id": "aea", "name": "Aria Everyday Activities Dataset", "title": "Aria Everyday Activities Dataset", "cat": "ego", "org": "industry", "one": "Everyday activity sequences from Aria glasses with globally aligned 3D trajectories, gaze and speech.", "device": "Project Aria glasses", "modal": "RGB, SLAM, eye gaze, point cloud, speech transcript", "hours": "—", "eps": "143 sequences", "tasks": "daily activity", "scenes": "5 indoor locations", "subj": "multiple wearers", "embod": "human", "note": "Small but the cleanest machine-perception annotations of the Aria family.", "arxiv": "2402.13349", "authors": [ "Zhaoyang Lv", "Nicholas Charron", "Pierre Moulon", "Alexander Gamino", "Cheng Peng", "Chris Sweeney", "Edward Miller", "Huixuan Tang", "Jeff Meissner", "Jing Dong", "Kiran Somasundaram", "Luis Pesqueira", "Mark Schwesinger", "Omkar Parkhi", "Qiao Gu", "Renzo De Nardi", "Shangyi Cheng", "Steve Saarinen", "Vijay Baiyya", "Yuyang Zou", "Richard Newcombe", "Jakob Julian Engel", "Xiaqing Pan", "Carl Ren" ], "year": 2024, "date": "2024-02-20", "institutions": [], "paper": "https://arxiv.org/abs/2402.13349", "site": "https://facebookresearch.github.io/projectaria_tools/docs/open_datasets/aria_everyday_activities_dataset", "fig": "figures/ds-aea.webp", "figs": [ "figures/ds-aea-1.webp", "figures/ds-aea-2.webp" ], "hours_num": -1 }, { "id": "egovid", "name": "EgoVid-5M", "title": "EgoVid-5M: A Large-Scale Video-Action Dataset for Egocentric Video Generation", "cat": "ego", "org": "academia", "one": "Five million egocentric clips curated specifically for training egocentric video generation with action conditioning.", "device": "curated from existing corpora", "modal": "RGB, kinematic action labels, text", "hours": "—", "eps": "5M clips", "tasks": "—", "scenes": "—", "subj": "—", "embod": "human", "note": "If you want to train a world model on the ego stream, this is the pre-curated pretraining set.", "arxiv": "2411.08380", "authors": [ "Xiaofeng Wang", "Kang Zhao", "Feng Liu", "Jiayu Wang", "Guosheng Zhao", "Xiaoyi Bao", "Zheng Zhu", "Yingya Zhang", "Xingang Wang" ], "year": 2024, "date": "2024-11-13", "institutions": [], "paper": "https://arxiv.org/abs/2411.08380", "site": "https://egovid.github.io/", "fig": "figures/ds-egovid.webp", "figs": [ "figures/ds-egovid-1.webp", "figures/ds-egovid-2.webp" ], "hours_num": -1 }, { "id": "aria", "name": "Project Aria", "title": "Project Aria: A New Tool for Egocentric Multi-Modal AI Research", "cat": "ego", "org": "industry", "one": "The glasses themselves: the research platform and open tooling that most modern egocentric datasets are recorded on.", "device": "Project Aria glasses", "modal": "RGB, SLAM cameras, eye tracking, IMU, magnetometer, barometer, audio", "hours": "—", "eps": "platform + tools", "tasks": "—", "scenes": "—", "subj": "—", "embod": "human", "note": "Not a dataset — the capture platform. Worth knowing the sensor suite before designing your own rig.", "arxiv": "2308.13561", "authors": [ "Jakob Engel", "Kiran Somasundaram", "Michael Goesele", "Albert Sun", "Alexander Gamino", "Andrew Turner", "Arjang Talattof", "Arnie Yuan", "Bilal Souti", "Brighid Meredith", "Cheng Peng", "Chris Sweeney", "Cole Wilson", "Dan Barnes", "Daniel DeTone", "David Caruso", "Derek Valleroy", "Dinesh Ginjupalli", "Duncan Frost", "Edward Miller", "Elias Mueggler", "Evgeniy Oleinik", "Fan Zhang", "Guruprasad Somasundaram", "Gustavo Solaira", "Harry Lanaras", "Henry Howard-Jenkins", "Huixuan Tang", "Hyo Jin Kim", "Jaime Rivera", "Ji Luo", "Jing Dong", "Julian Straub", "Kevin Bailey", "Kevin Eckenhoff", "Lingni Ma", "Luis Pesqueira", "Mark Schwesinger", "Maurizio Monge", "Nan Yang", "Nick Charron", "Nikhil Raina", "Omkar Parkhi", "Peter Borschowa", "Pierre Moulon", "Prince Gupta", "Raul Mur-Artal", "Robbie Pennington", "Sachin Kulkarni", "Sagar Miglani", "Santosh Gondi", "Saransh Solanki", "Sean Diener", "Shangyi Cheng", "Simon Green", "Steve Saarinen", "Suvam Patra", "Tassos Mourikis", "Thomas Whelan", "Tripti Singh", "Vasileios Balntas", "Vijay Baiyya", "Wilson Dreewes", "Xiaqing Pan", "Yang Lou", "Yipu Zhao", "Yusuf Mansour", "Yuyang Zou", "Zhaoyang Lv", "Zijian Wang", "Mingfei Yan", "Carl Ren", "Renzo De Nardi", "Richard Newcombe" ], "year": 2023, "date": "2023-08-24", "institutions": [], "paper": "https://arxiv.org/abs/2308.13561", "site": "https://facebookresearch.github.io/projectaria_tools/", "fig": "figures/ds-aria.webp", "figs": [ "figures/ds-aria-1.webp", "figures/ds-aria-2.webp" ], "hours_num": -1 }, { "id": "hoi4d", "name": "HOI4D", "title": "HOI4D: A 4D Egocentric Dataset for Category-Level Human-Object Interaction", "cat": "ego", "org": "academia", "one": "4D egocentric human-object interaction with per-frame panoptic segmentation, object poses and hand poses.", "device": "Head-mounted RGB-D", "modal": "RGB-D, 3D hand pose, object 6D pose, panoptic masks", "hours": "—", "eps": "4,000 sequences · 2.4M frames", "tasks": "16 object categories", "scenes": "610 indoor rooms", "subj": "4 participants", "embod": "human", "note": "Very dense annotation, very few subjects — the opposite tradeoff from Ego4D.", "arxiv": "2203.01577", "authors": [ "Yunze Liu", "Yun Liu", "Che Jiang", "Kangbo Lyu", "Weikang Wan", "Hao Shen", "Boqiang Liang", "Zhoujie Fu", "He Wang", "Li Yi" ], "year": 2022, "date": "2022-03-03", "institutions": [], "paper": "https://arxiv.org/abs/2203.01577", "site": "https://hoi4d.github.io/", "fig": "figures/ds-hoi4d.webp", "figs": [ "figures/ds-hoi4d-1.webp", "figures/ds-hoi4d-2.webp" ], "hours_num": -1 }, { "id": "egobody", "name": "EgoBody", "title": "EgoBody: Human Body Shape and Motion of Interacting People from Head-Mounted Devices", "cat": "ego", "org": "academia", "one": "Egocentric recordings of two-person social interaction with accurate 3D body shape and motion for the interactee.", "device": "HoloLens2 + multi-Kinect rig", "modal": "RGB-D, SMPL-X body meshes, eye gaze, scene mesh", "hours": "—", "eps": "125 sequences", "tasks": "social interaction", "scenes": "15 indoor scenes", "subj": "36 subjects", "embod": "human", "note": "The human-partner perception data an HRI system needs, from the wearer's viewpoint.", "arxiv": "2112.07642", "authors": [ "Siwei Zhang", "Qianli Ma", "Yan Zhang", "Zhiyin Qian", "Taein Kwon", "Marc Pollefeys", "Federica Bogo", "Siyu Tang" ], "year": 2021, "date": "2021-12-14", "institutions": [], "paper": "https://arxiv.org/abs/2112.07642", "site": "https://sanweiliti.github.io/egobody/egobody.html", "fig": "figures/ds-egobody.webp", "hours_num": -1 }, { "id": "h2o-ego", "name": "H2O", "title": "H2O: Two Hands Manipulating Objects for First Person Interaction Recognition", "cat": "ego", "org": "academia", "one": "Two hands manipulating objects from a first-person view, with markerless 3D annotations for both hands and object 6D pose.", "device": "Head-mounted RGB-D", "modal": "RGB-D, two-hand 3D pose, object 6D pose, interaction labels", "hours": "—", "eps": "—", "tasks": "bimanual interaction recognition", "scenes": "—", "subj": "4 subjects", "embod": "human", "note": "", "arxiv": "2104.11181", "authors": [ "Taein Kwon", "Bugra Tekin", "Jan Stuhmer", "Federica Bogo", "Marc Pollefeys" ], "year": 2021, "date": "2021-04-22", "institutions": [], "paper": "https://arxiv.org/abs/2104.11181", "site": "https://taeinkwon.com/projects/h2o/", "fig": "figures/ds-h2o-ego.webp", "figs": [ "figures/ds-h2o-ego-1.webp" ], "hours_num": -1 }, { "id": "egoexo-handrig", "name": "egoexo-handrig", "title": "egoexo-handrig", "cat": "ego", "org": "industry", "one": "In-the-wild 3D hand tracking from a mobile multi-camera ego-exo rig, built to get ground truth outside the lab.", "device": "mobile multi-camera ego+exo rig", "modal": "RGB multi-view, 3D hand pose", "hours": "—", "eps": "—", "tasks": "hand tracking", "scenes": "in-the-wild", "subj": "—", "embod": "human", "note": "", "arxiv": "2510.02601", "authors": [], "year": null, "date": "", "institutions": [], "paper": "https://arxiv.org/abs/2510.02601", "site": "https://arxiv.org/abs/2510.02601", "fig": "figures/ds-egoexo-handrig.webp", "figs": [ "figures/ds-egoexo-handrig-1.webp" ], "hours_num": -1 }, { "id": "yubi", "name": "YUBI", "title": "YUBI: Yielding Universal Bidigital Interface for Bimanual Dexterous Manipulation at Scale", "cat": "umi", "org": "academia", "one": "Bidigital handheld interface for bimanual dexterity, with a UMI-style dataset of genuinely unprecedented scale.", "device": "YUBI bidigital device + VR 6-DoF tracking", "modal": "RGB, 6-DoF pose, bimanual finger state", "hours": "8,434 h", "eps": "1.20M episodes", "tasks": "119 tasks", "scenes": "—", "subj": "—", "embod": "bimanual dexterous", "note": "8,434 hours from a handheld interface — an order of magnitude past anything teleoperated. This is the strongest existing evidence for the UMI-scaling thesis.", "arxiv": "2606.10244", "authors": [ "Takehiko Ohkawa", "Jumpei Arima", "Yuki Noguchi", "Masatoshi Tateno", "Makoto Sugiura", "Takuya Okubo", "Kengo Ikeuchi", "Yuma Shin", "Hiroki Nishizawa", "Naoaki Kanazawa", "Yuki Wakayama", "Daiki Fukunaga", "Koshi Makihara", "Tomohiro Motoda", "Floris Erich", "Yukiyasu Domae", "Tatsuya Matsushima", "Yohishiro Okumatsu", "Kei Ota" ], "year": 2026, "date": "2026-06-08", "institutions": [], "paper": "https://arxiv.org/abs/2606.10244", "site": "https://arxiv.org/abs/2606.10244", "fig": "figures/ds-yubi.webp", "figs": [ "figures/ds-yubi-1.webp" ], "hours_num": 8434.0 }, { "id": "openaoe", "name": "Open-AoE", "title": "Open-AoE: An Open Egocentric Manipulation Dataset and Toolchain for Embodied Learning", "cat": "umi", "org": "academia", "one": "An open egocentric manipulation dataset shipped with its capture toolchain rather than as a frozen release.", "device": "egocentric capture rig (open toolchain)", "modal": "RGB, hand pose, action", "hours": "—", "eps": "—", "tasks": "manipulation", "scenes": "—", "subj": "—", "embod": "human → robot", "note": "The toolchain is the valuable half — it means you can extend it rather than only consume it.", "arxiv": "2607.14183", "authors": [ "(see PDF)" ], "year": 2026, "date": "2026-07-14", "institutions": [], "paper": "https://arxiv.org/abs/2607.14183", "site": "https://arxiv.org/abs/2607.14183", "fig": "figures/ds-openaoe.webp", "figs": [ "figures/ds-openaoe-1.webp", "figures/ds-openaoe-2.webp" ], "hours_num": -1 }, { "id": "realdexumi", "name": "RealDexUMI", "title": "RealDexUMI: A Wearable Universal Manipulation Interface for Dexterous Robot Learning", "cat": "umi", "org": "academia", "one": "A wearable universal manipulation interface built specifically for dexterous robot learning.", "device": "Wearable UMI (dexterous)", "modal": "RGB, hand pose, EE pose", "hours": "—", "eps": "—", "tasks": "dexterous manipulation", "scenes": "in-the-wild", "subj": "—", "embod": "dexterous hand", "note": "", "arxiv": "2606.06033", "authors": [ "Chaoyi Xu", "Yixuan Jiang", "Jiahui Huan", "Yuhui Fu", "Haoyu Zhou", "Weitian Yuan", "Jiayi Yu", "Wanpeng Zhang", "Haoqi Yuan", "Zongqing Lu" ], "year": 2026, "date": "2026-06-04", "institutions": [], "paper": "https://arxiv.org/abs/2606.06033", "site": "https://arxiv.org/abs/2606.06033", "fig": "figures/ds-realdexumi.webp", "figs": [ "figures/ds-realdexumi-1.webp", "figures/ds-realdexumi-2.webp" ], "hours_num": -1 }, { "id": "ume-data", "name": "Universal Manipulation Exoskeleton", "title": "Universal Manipulation Exoskeleton: Learning Compliant Whole-body Policies with Real-time Torque Feedback", "cat": "umi", "org": "academia", "one": "Whole-body exoskeleton capture with compliance — the force dimension UMI-style rigs leave out.", "device": "Universal Manipulation Exoskeleton", "modal": "RGB, joint state, contact force, whole-body pose", "hours": "—", "eps": "—", "tasks": "compliant whole-body manipulation", "scenes": "—", "subj": "—", "embod": "humanoid / mobile manipulator", "note": "", "arxiv": "2606.14218", "authors": [ "(see PDF)" ], "year": 2026, "date": "2026-06-14", "institutions": [], "paper": "https://arxiv.org/abs/2606.14218", "site": "https://arxiv.org/abs/2606.14218", "fig": "figures/ds-ume-data.webp", "figs": [ "figures/ds-ume-data-1.webp", "figures/ds-ume-data-2.webp" ], "hours_num": -1 }, { "id": "dexumi-data", "name": "DexUMI", "title": "DexUMI: Using Human Hand as the Universal Manipulation Interface for Dexterous Manipulation", "cat": "umi", "org": "academia", "one": "Dexterous-hand demonstrations captured with a wearable exoskeleton adapter plus video inpainting of the robot hand.", "device": "Wearable hand exoskeleton + camera", "modal": "RGB (inpainted), finger joint state, tactile", "hours": "—", "eps": "—", "tasks": "dexterous manipulation", "scenes": "in-the-wild", "subj": "—", "embod": "XHand / Inspire hand", "note": "The two-part gap-closing recipe — hardware for kinematics, inpainting for appearance.", "arxiv": "2505.21864", "authors": [ "Mengda Xu", "Han Zhang", "Yifan Hou", "Zhenjia Xu", "Linxi Fan", "Manuela Veloso", "Shuran Song" ], "year": 2025, "date": "2025-05-27", "institutions": [ "Stanford University", "Columbia University", "NVIDIA", "JP Morgan AI Research" ], "paper": "https://arxiv.org/abs/2505.21864", "site": "https://dex-umi.github.io/", "fig": "figures/ds-dexumi-data.webp", "figs": [ "figures/ds-dexumi-data-1.webp", "figures/ds-dexumi-data-2.webp" ], "hours_num": -1 }, { "id": "fastumi100k", "name": "FastUMI-100K", "title": "FastUMI-100K: Advancing Data-driven Robotic Manipulation with a Large-scale UMI-style Dataset", "cat": "umi", "org": "academia", "one": "The largest open UMI-style corpus: 100K+ handheld demonstrations in household environments, released in LeRobot format.", "device": "FastUMI handheld rig (hardware-decoupled, lightweight tracker)", "modal": "RGB multi-view wrist fisheye, EE state, text annotations", "hours": "—", "eps": "100K+ trajectories", "tasks": "54 tasks", "scenes": "household", "subj": "—", "embod": "embodiment-agnostic", "note": "LeRobot v2.1 formatted, so it drops straight into an existing training stack. The obvious starting corpus for a UMI world model.", "arxiv": "2510.08022", "authors": [ "Kehui Liu", "Zhongjie Jia", "Yang Li", "Zhaxizhuoma", "Pengan Chen", "Song Liu", "Xin Liu", "Pingrui Zhang", "Haoming Song", "Xinyi Ye", "Nieqing Cao", "Zhigang Wang", "Jia Zeng", "Dong Wang", "Yan Ding", "Bin Zhao", "Xuelong Li" ], "year": 2025, "date": "2025-10-09", "institutions": [], "paper": "https://arxiv.org/abs/2510.08022", "site": "https://github.com/MrKeee/FastUMI-100K", "fig": "figures/ds-fastumi100k.webp", "figs": [ "figures/ds-fastumi100k-1.webp", "figures/ds-fastumi100k-2.webp" ], "hours_num": -1 }, { "id": "umi-data", "name": "Universal Manipulation Interface", "title": "Universal Manipulation Interface: In-The-Wild Robot Teaching Without In-The-Wild Robots", "cat": "umi", "org": "academia", "one": "The original handheld-gripper demonstrations: deployable robot data collected in the wild with no robot present.", "device": "UMI handheld gripper + fisheye GoPro + side mirrors", "modal": "RGB fisheye, 6-DoF EE pose (SLAM), gripper width", "hours": "—", "eps": "~1,400 demos across tasks", "tasks": "dynamic, bimanual, precise, long-horizon", "scenes": "in-the-wild", "subj": "—", "embod": "UR5 / Franka / ARX (transferable)", "note": "Small by modern standards, but it defined the format everything below follows.", "arxiv": "2402.10329", "authors": [ "Cheng Chi", "Zhenjia Xu", "Chuer Pan", "Eric Cousineau", "Benjamin Burchfiel", "Siyuan Feng", "Russ Tedrake", "Shuran Song" ], "year": 2024, "date": "2024-02-15", "institutions": [ "Stanford University", "Columbia University", "Toyota Research Institute" ], "paper": "https://arxiv.org/abs/2402.10329", "site": "https://umi-gripper.github.io/", "fig": "figures/ds-umi-data.webp", "figs": [ "figures/ds-umi-data-1.webp", "figures/ds-umi-data-2.webp" ], "hours_num": -1 }, { "id": "roboturk", "name": "RoboTurk", "title": "RoboTurk: A Crowdsourcing Platform for Robotic Skill Learning through Imitation", "cat": "teleop", "org": "academia", "one": "Crowdsourced 6-DoF teleoperation through ordinary phones — the first attempt to scale demonstration collection past a single lab.", "device": "smartphone-based teleoperation, crowdsourced", "modal": "RGB, actions", "hours": "137.5 h", "eps": "2,100+ demonstrations", "tasks": "3 tasks", "scenes": "sim + real", "subj": "54 users", "embod": "Sawyer / sim", "note": "Historically important: the argument that operator access, not robot count, is the bottleneck.", "arxiv": "1811.02790", "authors": [ "Ajay Mandlekar", "Yuke Zhu", "Animesh Garg", "Jonathan Booher", "Max Spero", "Albert Tung", "Julian Gao", "John Emmons", "Anchit Gupta", "Emre Orbay", "Silvio Savarese", "Li Fei-Fei" ], "year": 2018, "date": "2018-11-07", "institutions": [], "paper": "https://arxiv.org/abs/1811.02790", "site": "https://roboturk.stanford.edu/", "fig": "figures/ds-roboturk.webp", "figs": [ "figures/ds-roboturk-1.webp", "figures/ds-roboturk-2.webp" ], "hours_num": 137.5 }, { "id": "airexo2", "name": "AirExo-2", "title": "AirExo-2: Scaling up Generalizable Robotic Imitation Learning with Low-Cost Exoskeletons", "cat": "teleop", "org": "academia", "one": "In-the-wild demonstrations from a low-cost exoskeleton, designed to transfer without robot data.", "device": "AirExo-2 exoskeleton", "modal": "RGB-D, joint state", "hours": "—", "eps": "—", "tasks": "generalisable manipulation", "scenes": "in-the-wild", "subj": "—", "embod": "dual-arm", "note": "The main competitor framing for UME-style exoskeleton capture.", "arxiv": "2503.03081", "authors": [ "Hongjie Fang", "Chenxi Wang", "Yiming Wang", "Jingjing Chen", "Shangning Xia", "Jun Lv", "Hao-Shu Fang", "Cewu Lu" ], "year": 2025, "date": "2025-03-05", "institutions": [ "Shanghai Jiao Tong University" ], "paper": "https://arxiv.org/abs/2503.03081", "site": "https://airexo.tech/airexo2/", "fig": "figures/ds-airexo2.webp", "figs": [ "figures/ds-airexo2-1.webp", "figures/ds-airexo2-2.webp" ], "hours_num": -1 }, { "id": "dexcap-data", "name": "DexCap", "title": "DexCap: Scalable and Portable Mocap Data Collection System for Dexterous Manipulation", "cat": "teleop", "org": "academia", "one": "Portable glove-plus-SLAM mocap for dexterous manipulation, collected without a robot in the loop.", "device": "Rokoko glove + LiDAR chest rig + SLAM", "modal": "RGB-D, finger joint angles, 6-DoF wrist pose, point cloud", "hours": "—", "eps": "—", "tasks": "dexterous manipulation", "scenes": "in-the-wild", "subj": "—", "embod": "LEAP hand + Franka", "note": "The most direct hardware precedent for a world-action model from glove capture.", "arxiv": "2403.07788", "authors": [ "Chen Wang", "Haochen Shi", "Weizhuo Wang", "Ruohan Zhang", "Li Fei-Fei", "C. Karen Liu" ], "year": 2024, "date": "2024-03-12", "institutions": [ "Stanford University" ], "paper": "https://arxiv.org/abs/2403.07788", "site": "https://dex-cap.github.io/", "fig": "figures/ds-dexcap-data.webp", "figs": [ "figures/ds-dexcap-data-1.webp", "figures/ds-dexcap-data-2.webp" ], "hours_num": -1 }, { "id": "humanplus", "name": "HumanPlus", "title": "HumanPlus: Humanoid Shadowing and Imitation from Humans", "cat": "teleop", "org": "academia", "one": "Humanoid shadowing from a single RGB camera, plus the imitation data it produces on a 33-DoF humanoid.", "device": "single RGB camera (shadowing) + 40 h of human mocap", "modal": "RGB, whole-body pose, joint state", "hours": "—", "eps": "40 demos per skill", "tasks": "wearing shoes, warehouse unloading, folding, typing", "scenes": "lab", "subj": "—", "embod": "33-DoF custom humanoid", "note": "", "arxiv": "2406.10454", "authors": [ "Zipeng Fu", "Qingqing Zhao", "Qi Wu", "Gordon Wetzstein", "Chelsea Finn" ], "year": 2024, "date": "2024-06-15", "institutions": [], "paper": "https://arxiv.org/abs/2406.10454", "site": "https://humanoid-ai.github.io/", "fig": "figures/ds-humanplus.webp", "figs": [ "figures/ds-humanplus-1.webp", "figures/ds-humanplus-2.webp" ], "hours_num": -1 }, { "id": "mobile-aloha-data", "name": "Mobile ALOHA", "title": "Mobile ALOHA: Learning Bimanual Mobile Manipulation with Low-Cost Whole-Body Teleoperation", "cat": "teleop", "org": "academia", "one": "Mobile bimanual demonstrations with whole-body teleoperation, plus the co-training result that static data lifts mobile performance.", "device": "Mobile ALOHA (tethered whole-body teleop)", "modal": "RGB 4-camera, joint + base velocity", "hours": "—", "eps": "50 demos per task", "tasks": "7 mobile manipulation tasks", "scenes": "real home/kitchen", "subj": "—", "embod": "Mobile ALOHA", "note": "The co-training finding — up to 90% improvement from static data — is the reusable result, not the dataset.", "arxiv": "2401.02117", "authors": [ "Zipeng Fu", "Tony Z. Zhao", "Chelsea Finn" ], "year": 2024, "date": "2024-01-04", "institutions": [ "Stanford University" ], "paper": "https://arxiv.org/abs/2401.02117", "site": "https://mobile-aloha.github.io/", "fig": "figures/ds-mobile-aloha-data.webp", "figs": [ "figures/ds-mobile-aloha-data-1.webp" ], "hours_num": -1 }, { "id": "television", "name": "Open-TeleVision", "title": "Open-TeleVision: Teleoperation with Immersive Active Visual Feedback", "cat": "teleop", "org": "academia", "one": "Open-source immersive VR teleoperation with active stereo vision — cheap, high-quality bimanual demonstrations.", "device": "VR headset + active stereo head", "modal": "stereo RGB, hand/arm pose, robot state", "hours": "—", "eps": "—", "tasks": "bimanual dexterous", "scenes": "lab", "subj": "—", "embod": "H1 / GR-1 / bimanual arms", "note": "", "arxiv": "2407.01512", "authors": [ "Xuxin Cheng", "Jialong Li", "Shiqi Yang", "Ge Yang", "Xiaolong Wang" ], "year": 2024, "date": "2024-07-01", "institutions": [ "UC San Diego", "MIT" ], "paper": "https://arxiv.org/abs/2407.01512", "site": "https://robot-tv.github.io/", "fig": "figures/ds-television.webp", "figs": [ "figures/ds-television-1.webp", "figures/ds-television-2.webp" ], "hours_num": -1 }, { "id": "aloha-data", "name": "Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware", "title": "Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware", "cat": "teleop", "org": "academia", "one": "Fine-grained bimanual demonstrations from the low-cost leader-follower rig that made bimanual IL reproducible.", "device": "ALOHA leader-follower (ViperX/WidowX)", "modal": "RGB 4-camera, joint positions", "hours": "—", "eps": "~50 demos per task", "tasks": "6 fine-grained tasks", "scenes": "lab", "subj": "—", "embod": "ViperX 6-DoF bimanual", "note": "Tiny, and that is the point — 50 demos per task was enough for threading a zip tie.", "arxiv": "2304.13705", "authors": [ "Tony Z. Zhao", "Vikash Kumar", "Sergey Levine", "Chelsea Finn" ], "year": 2023, "date": "2023-04-23", "institutions": [ "Stanford University", "UC Berkeley", "Meta AI" ], "paper": "https://arxiv.org/abs/2304.13705", "site": "https://tonyzhaozh.github.io/aloha/", "fig": "figures/ds-aloha-data.webp", "figs": [ "figures/ds-aloha-data-1.webp", "figures/ds-aloha-data-2.webp" ], "hours_num": -1 }, { "id": "qwen-robotmanip", "name": "Qwen-RobotManip Technical Report", "title": "Qwen-RobotManip Technical Report: Alignment Unlocks Scale for Robotic Manipulation Foundation Models", "cat": "robot", "org": "industry", "one": "A ~38,100-hour pretraining corpus assembled entirely from open datasets and human video, with a human-to-robot synthesis pipeline across 15 platforms.", "device": "aggregated open data + egocentric human video", "modal": "RGB, proprioception, language, synthesised actions", "hours": "~38,100 h", "eps": "—", "tasks": "—", "scenes": "—", "subj": "15 platforms", "embod": "15 platforms", "note": "Notable for using no proprietary collection at all — a direct test of whether open data plus human video is enough.", "arxiv": "2606.17846", "authors": [ "Haoqi Yuan", "Zhixuan Liang", "Anzhe Chen", "Ye Wang", "Haoyang Li", "Pei Lin", "Yiyang Huang", "Zixing Lei", "Tong Zhang", "Jiazhao Zhang", "Jie Zhang", "Jingyang Fan", "Gengze Zhou", "Qihang Peng", "Chenxu Lv", "Xiaoyue Chen", "An Yang", "Fei Huang", "Junyang Lin", "Dayiheng Liu", "Jingren Zhou", "Chenfei Wu", "Xiong-Hui Chen" ], "year": 2026, "date": "2026-06-16", "institutions": [], "paper": "https://arxiv.org/abs/2606.17846", "site": "https://arxiv.org/abs/2606.17846", "fig": "figures/ds-qwen-robotmanip.webp", "figs": [ "figures/ds-qwen-robotmanip-1.webp" ], "hours_num": 38100.0 }, { "id": "galaxea", "name": "Galaxea Open-World Dataset and G0 Dual-System VLA Model", "title": "Galaxea Open-World Dataset and G0 Dual-System VLA Model", "cat": "robot", "org": "industry", "one": "500+ hours of mobile manipulation in real homes and workplaces, all on one embodiment.", "device": "Galaxea R1-Lite mobile bimanual robot, teleoperation", "modal": "RGB, proprioception, language, subtask annotations", "hours": "500+ h", "eps": "~100K trajectories", "tasks": "150 task categories", "scenes": "50 real scenes", "subj": "—", "embod": "Galaxea R1-Lite", "note": "Authentic living and working environments rather than lab mock-ups — closest in spirit to what MABEL would need.", "arxiv": "2509.00576", "authors": [ "Tao Jiang", "Tianyuan Yuan", "Yicheng Liu", "Chenhao Lu", "Jianning Cui", "Xiao Liu", "Shuiqi Cheng", "Jiyang Gao", "Huazhe Xu", "Hang Zhao" ], "year": 2025, "date": "2025-08-30", "institutions": [], "paper": "https://arxiv.org/abs/2509.00576", "site": "https://opengalaxea.github.io/G0/", "fig": "figures/ds-galaxea.webp", "figs": [ "figures/ds-galaxea-1.webp" ], "hours_num": 500.0 }, { "id": "droid", "name": "DROID", "title": "DROID: A Large-Scale In-The-Wild Robot Manipulation Dataset", "cat": "robot", "org": "consortium", "one": "In-the-wild manipulation collected on one standardised hardware setup across 564 scenes — diversity with consistency.", "device": "Franka Panda + Robotiq, VR teleoperation, 3 cameras (2 stereo + wrist)", "modal": "RGB stereo, depth, proprioception, language, camera calibration", "hours": "350 h", "eps": "76K trajectories", "tasks": "86 tasks", "scenes": "564 scenes", "subj": "50 data collectors · 13 institutions", "embod": "Franka Panda", "note": "Standardised hardware is why it is more useful per-episode than OXE. The stereo calibration makes it usable for 3D policies.", "arxiv": "2403.12945", "authors": [ "Alexander Khazatsky", "Karl Pertsch", "Suraj Nair", "Ashwin Balakrishna", "Sudeep Dasari", "Siddharth Karamcheti", "Soroush Nasiriany", "Mohan Kumar Srirama", "Lawrence Yunliang Chen", "Kirsty Ellis", "Peter David Fagan", "Joey Hejna", "Masha Itkina", "Marion Lepert", "Yecheng Jason Ma", "Patrick Tree Miller", "Jimmy Wu", "Suneel Belkhale", "Shivin Dass", "Huy Ha", "Arhan Jain", "Abraham Lee", "Youngwoon Lee", "Marius Memmel", "Sungjae Park", "Ilija Radosavovic", "Kaiyuan Wang", "Albert Zhan", "Kevin Black", "Cheng Chi", "Kyle Beltran Hatch", "Shan Lin", "Jingpei Lu", "Jean Mercat", "Abdul Rehman", "Pannag R Sanketi", "Archit Sharma", "Cody Simpson", "Quan Vuong", "Homer Rich Walke", "Blake Wulfe", "Ted Xiao", "Jonathan Heewon Yang", "Arefeh Yavary", "Tony Z. Zhao", "Christopher Agia", "Rohan Baijal", "Mateo Guaman Castro", "Daphne Chen", "Qiuyu Chen", "Trinity Chung", "Jaimyn Drake", "Ethan Paul Foster", "Jensen Gao", "Vitor Guizilini", "David Antonio Herrera", "Minho Heo", "Kyle Hsu", "Jiaheng Hu", "Muhammad Zubair Irshad", "Donovon Jackson", "Charlotte Le", "Yunshuang Li", "Kevin Lin", "Roy Lin", "Zehan Ma", "Abhiram Maddukuri", "Suvir Mirchandani", "Daniel Morton", "Tony Nguyen", "Abigail O'Neill", "Rosario Scalise", "Derick Seale", "Victor Son", "Stephen Tian", "Emi Tran", "Andrew E. Wang", "Yilin Wu", "Annie Xie", "Jingyun Yang", "Patrick Yin", "Yunchu Zhang", "Osbert Bastani", "Glen Berseth", "Jeannette Bohg", "Ken Goldberg", "Abhinav Gupta", "Abhishek Gupta", "Dinesh Jayaraman", "Joseph J Lim", "Jitendra Malik", "Roberto Martín-Martín", "Subramanian Ramamoorthy", "Dorsa Sadigh", "Shuran Song", "Jiajun Wu", "Michael C. Yip", "Yuke Zhu", "Thomas Kollar", "Sergey Levine", "Chelsea Finn" ], "year": 2024, "date": "2024-03-19", "institutions": [ "Stanford University", "UC Berkeley", "13 institutions" ], "paper": "https://arxiv.org/abs/2403.12945", "site": "https://droid-dataset.github.io/", "fig": "figures/ds-droid.webp", "figs": [ "figures/ds-droid-1.webp", "figures/ds-droid-2.webp" ], "hours_num": 350.0 }, { "id": "agibot2026", "name": "AgiBot World 2026", "title": "AgiBot World 2026", "cat": "robot", "org": "industry", "one": "AgiBot's 2026 release, aimed at reinforcement learning rather than imitation: real-world trajectories across industrial and household tasks.", "device": "AgiBot fleet, teleoperation", "modal": "RGB-D, proprioception, reward/task annotations", "hours": "—", "eps": "11,430 real-world trajectories", "tasks": "14 industrial and household tasks", "scenes": "—", "subj": "—", "embod": "AgiBot G1", "note": "Small next to the original million-trajectory AgiBot World, and deliberately so — it is structured for RL, which almost no large robot dataset is. Released with Shanghai AI Lab and the National-Local Humanoid Robotics Innovation Centre.", "arxiv": "", "authors": [], "year": 2026, "date": "2026-04-01", "institutions": [], "paper": "", "site": "https://agibot-world.com/", "fig": "", "hours_num": -1 }, { "id": "agibot", "name": "AgiBot World Colosseo", "title": "AgiBot World Colosseo: A Large-scale Manipulation Platform for Scalable and Intelligent Embodied Systems", "cat": "robot", "org": "industry", "one": "A million-plus trajectories from a fleet of 100+ identical humanoid mobile manipulators — uniformity as the design principle.", "device": "AgiBot G1 fleet, teleoperation", "modal": "RGB-D, proprioception, tactile (subset), language", "hours": "—", "eps": "1M+ trajectories", "tasks": "217 tasks", "scenes": "106 scenes · 5 deployment settings", "subj": "—", "embod": "AgiBot G1", "note": "The industrial answer to DROID: hardware uniformity instead of institutional diversity. Which bet is right is still open.", "arxiv": "2503.06669", "authors": [ "AgiBot-World-Contributors", "Qingwen Bu", "Jisong Cai", "Li Chen", "Xiuqi Cui", "Yan Ding", "Siyuan Feng", "Shenyuan Gao", "Xindong He", "Xuan Hu", "Xu Huang", "Shu Jiang", "Yuxin Jiang", "Cheng Jing", "Hongyang Li", "Jialu Li", "Chiming Liu", "Yi Liu", "Yuxiang Lu", "Jianlan Luo", "Ping Luo", "Yao Mu", "Yuehan Niu", "Yixuan Pan", "Jiangmiao Pang", "Yu Qiao", "Guanghui Ren", "Cheng Ruan", "Jiaqi Shan", "Yongjian Shen", "Chengshi Shi", "Mingkang Shi", "Modi Shi", "Chonghao Sima", "Jianheng Song", "Huijie Wang", "Wenhao Wang", "Dafeng Wei", "Chengen Xie", "Guo Xu", "Junchi Yan", "Cunbiao Yang", "Lei Yang", "Shukai Yang", "Maoqing Yao", "Jia Zeng", "Chi Zhang", "Qinglin Zhang", "Bin Zhao", "Chengyue Zhao", "Jiaqi Zhao", "Jianchao Zhu" ], "year": 2025, "date": "2025-03-09", "institutions": [ "AgiBot", "Shanghai AI Laboratory" ], "paper": "https://arxiv.org/abs/2503.06669", "site": "https://agibot-world.com/", "fig": "figures/ds-agibot.webp", "figs": [ "figures/ds-agibot-1.webp", "figures/ds-agibot-2.webp" ], "hours_num": -1 }, { "id": "robocoin", "name": "RoboCOIN", "title": "RoboCOIN: An Open-Sourced Bimanual Robotic Data Collection for Integrated Manipulation", "cat": "robot", "org": "consortium", "one": "Large-scale bimanual manipulation across 15 robot platforms, categorised by the type of two-arm coordination required.", "device": "15 robot platforms, teleoperation", "modal": "RGB multi-view, proprioception, language", "hours": "—", "eps": "180,000+ demonstrations", "tasks": "421 bimanual tasks · 39 coordination types", "scenes": "16 environments", "subj": "432 objects", "embod": "15 bimanual platforms", "note": "The taxonomy of 39 bimanual coordination actions is genuinely useful — it is the closest thing to a vocabulary for two-arm collaboration.", "arxiv": "2511.17441", "authors": [ "Shihan Wu", "Xuecheng Liu", "Shaoxuan Xie", "Pengwei Wang", "Xinghang Li", "Bowen Yang", "Zhe Li", "Kai Zhu", "Hongyu Wu", "Yiheng Liu", "Zhaoye Long", "Runtian Xu", "Yue Wang", "Chong Liu", "Dihan Wang", "Ziqiang Ni", "Xiang Yang", "You Liu", "Ruoxuan Feng", "Lei Zhang", "Denghang Huang", "Chenghao Jin", "Anlan Yin", "Xinlong Wang", "Zhenguo Sun", "Junkai Zhao", "Mengfei Du", "Mingyu Cao", "Xiansheng Chen", "Hongyang Cheng", "Xiaojie Zhang", "Yankai Fu", "Ning Chen", "Cheng Chi", "Sixiang Chen", "Huaihai Lyu", "Xiaoshuai Hao", "Yequan Wang", "Bo Lei", "Dong Liu", "Xi Yang", "Yance Jiao", "Tengfei Pan", "Yunyan Zhang", "Songjing Wang", "Ziqian Zhang", "Xu Liu", "Ji Zhang", "Caowei Meng", "Zhizheng Zhang", "Jiyang Gao", "Song Wang", "Xiaokun Leng", "Zhiqiang Xie", "Zhenzhen Zhou", "Peng Huang", "Wu Yang", "Yandong Guo", "Yichao Zhu", "Suibing Zheng", "Hao Cheng", "Xinmin Ding", "Yang Yue", "Huanqian Wang", "Chi Chen", "Jingrui Pang", "YuXi Qian", "Haoran Geng", "Lianli Gao", "Haiyuan Li", "Bin Fang", "Gao Huang", "Yaodong Yang", "Hao Dong", "He Wang", "Hang Zhao", "Yadong Mu", "Di Hu", "Hao Zhao", "Tiejun Huang", "Shanghang Zhang", "Yonghua Lin", "Zhongyuan Wang", "Guocai Yao" ], "year": 2025, "date": "2025-11-21", "institutions": [], "paper": "https://arxiv.org/abs/2511.17441", "site": "https://arxiv.org/abs/2511.17441", "fig": "figures/ds-robocoin.webp", "figs": [ "figures/ds-robocoin-1.webp", "figures/ds-robocoin-2.webp" ], "hours_num": -1 }, { "id": "ario", "name": "All Robots in One", "title": "All Robots in One: A New Standard and Unified Dataset for Versatile, General-Purpose Embodied Agents", "cat": "robot", "org": "consortium", "one": "A unified standard plus the largest aggregated embodied dataset — roughly 3 million episodes across 258 series.", "device": "mixed (aggregated + new collection)", "modal": "RGB-D, proprioception, language, tactile (partial)", "hours": "—", "eps": "~3M episodes", "tasks": "321,064 tasks", "scenes": "—", "subj": "—", "embod": "many", "note": "Enormous, and the data-format standard is the more durable contribution.", "arxiv": "2408.10899", "authors": [ "Zhiqiang Wang", "Hao Zheng", "Yunshuang Nie", "Wenjun Xu", "Qingwei Wang", "Hua Ye", "Zhe Li", "Kaidong Zhang", "Xuewen Cheng", "Wanxi Dong", "Chang Cai", "Liang Lin", "Feng Zheng", "Xiaodan Liang" ], "year": 2024, "date": "2024-08-20", "institutions": [], "paper": "https://arxiv.org/abs/2408.10899", "site": "https://imaei.github.io/project_pages/ario/", "fig": "figures/ds-ario.webp", "figs": [ "figures/ds-ario-1.webp", "figures/ds-ario-2.webp" ], "hours_num": -1 }, { "id": "fmb", "name": "FMB", "title": "FMB: a Functional Manipulation Benchmark for Generalizable Robotic Learning", "cat": "robot", "org": "academia", "one": "A functional manipulation benchmark built from 3D-printed objects so other labs can replicate the physical setup exactly.", "device": "Franka + teleoperation", "modal": "RGB-D multi-view, proprioception", "hours": "—", "eps": "22,550 trajectories", "tasks": "assembly + single-object manipulation", "scenes": "lab", "subj": "—", "embod": "Franka Panda", "note": "Reproducible physical hardware is rare and underrated — this is one of the few real-robot benchmarks you can actually rebuild.", "arxiv": "2401.08553", "authors": [ "Jianlan Luo", "Charles Xu", "Fangchen Liu", "Liam Tan", "Zipeng Lin", "Jeffrey Wu", "Pieter Abbeel", "Sergey Levine" ], "year": 2024, "date": "2024-01-16", "institutions": [], "paper": "https://arxiv.org/abs/2401.08553", "site": "https://functional-manipulation-benchmark.github.io/", "fig": "figures/ds-fmb.webp", "figs": [ "figures/ds-fmb-1.webp", "figures/ds-fmb-2.webp" ], "hours_num": -1 }, { "id": "robomind", "name": "RoboMIND", "title": "RoboMIND: Benchmark on Multi-embodiment Intelligence Normative Data for Robot Manipulation", "cat": "robot", "org": "consortium", "one": "Multi-embodiment manipulation collected under a single standardised protocol, spanning single-arm, dual-arm and humanoid.", "device": "unified collection platform, teleoperation", "modal": "RGB-D multi-view, proprioception, language", "hours": "—", "eps": "107K trajectories", "tasks": "479 tasks", "scenes": "—", "subj": "96 object classes", "embod": "Franka, UR5e, AgileX dual-arm, humanoid", "note": "The standardised protocol is the contribution; most pooled datasets are heterogeneous messes by comparison.", "arxiv": "2412.13877", "authors": [ "Kun Wu", "Chengkai Hou", "Jiaming Liu", "Zhengping Che", "Xiaozhu Ju", "Zhuqin Yang", "Meng Li", "Yinuo Zhao", "Zhiyuan Xu", "Guang Yang", "Shichao Fan", "Xinhua Wang", "Fei Liao", "Zhen Zhao", "Guangyu Li", "Zhao Jin", "Lecheng Wang", "Jilei Mao", "Ning Liu", "Pei Ren", "Qiang Zhang", "Yaoxu Lyu", "Mengzhen Liu", "Jingyang He", "Yulin Luo", "Zeyu Gao", "Chenxuan Li", "Chenyang Gu", "Yankai Fu", "Di Wu", "Xingyu Wang", "Sixiang Chen", "Zhenyu Wang", "Pengju An", "Siyuan Qian", "Shanghang Zhang", "Jian Tang" ], "year": 2024, "date": "2024-12-18", "institutions": [], "paper": "https://arxiv.org/abs/2412.13877", "site": "https://x-humanoid-robomind.github.io/", "fig": "figures/ds-robomind.webp", "figs": [ "figures/ds-robomind-1.webp" ], "hours_num": -1 }, { "id": "bridgev2", "name": "BridgeData V2", "title": "BridgeData V2: A Dataset for Robot Learning at Scale", "cat": "robot", "org": "academia", "one": "Diverse low-cost-arm manipulation designed for studying generalisation, on hardware any lab can buy.", "device": "WidowX 250 + VR/leader teleoperation", "modal": "RGB multi-view, depth, proprioception, language", "hours": "—", "eps": "60,096 trajectories", "tasks": "24 environments", "scenes": "24 environments", "subj": "—", "embod": "WidowX 250", "note": "The generalisation benchmark of the pre-DROID era, and still the cheapest to reproduce.", "arxiv": "2308.12952", "authors": [ "Homer Walke", "Kevin Black", "Abraham Lee", "Moo Jin Kim", "Max Du", "Chongyi Zheng", "Tony Zhao", "Philippe Hansen-Estruch", "Quan Vuong", "Andre He", "Vivek Myers", "Kuan Fang", "Chelsea Finn", "Sergey Levine" ], "year": 2023, "date": "2023-08-24", "institutions": [ "UC Berkeley", "Stanford University", "Google DeepMind" ], "paper": "https://arxiv.org/abs/2308.12952", "site": "https://rail-berkeley.github.io/bridgedata/", "fig": "figures/ds-bridgev2.webp", "figs": [ "figures/ds-bridgev2-1.webp", "figures/ds-bridgev2-2.webp" ], "hours_num": -1 }, { "id": "oxe", "name": "Open X-Embodiment", "title": "Open X-Embodiment: Robotic Learning Datasets and RT-X Models", "cat": "robot", "org": "consortium", "one": "The pooled cross-embodiment corpus: 22 robot types from 21 institutions in one schema — the ImageNet moment for robot data.", "device": "teleoperation (mixed: VR, leader-follower, kinesthetic)", "modal": "RGB, multi-view, proprioception, language instructions", "hours": "—", "eps": "1M+ trajectories", "tasks": "527 skills · 160,266 task instances", "scenes": "mixed", "subj": "21 institutions", "embod": "22 embodiments", "note": "The training set under Octo, OpenVLA and most open generalist policies. Heterogeneous to a fault — the per-dataset quality varies enormously.", "arxiv": "2310.08864", "authors": [ "Open X-Embodiment Collaboration", "Abby O'Neill", "Abdul Rehman", "Abhinav Gupta", "Abhiram Maddukuri", "Abhishek Gupta", "Abhishek Padalkar", "Abraham Lee", "Acorn Pooley", "Agrim Gupta", "Ajay Mandlekar", "Ajinkya Jain", "Albert Tung", "Alex Bewley", "Alex Herzog", "Alex Irpan", "Alexander Khazatsky", "Anant Rai", "Anchit Gupta", "Andrew Wang", "Andrey Kolobov", "Anikait Singh", "Animesh Garg", "Aniruddha Kembhavi", "Annie Xie", "Anthony Brohan", "Antonin Raffin", "Archit Sharma", "Arefeh Yavary", "Arhan Jain", "Ashwin Balakrishna", "Ayzaan Wahid", "Ben Burgess-Limerick", "Beomjoon Kim", "Bernhard Schölkopf", "Blake Wulfe", "Brian Ichter", "Cewu Lu", "Charles Xu", "Charlotte Le", "Chelsea Finn", "Chen Wang", "Chenfeng Xu", "Cheng Chi", "Chenguang Huang", "Christine Chan", "Christopher Agia", "Chuer Pan", "Chuyuan Fu", "Coline Devin", "Danfei Xu", "Daniel Morton", "Danny Driess", "Daphne Chen", "Deepak Pathak", "Dhruv Shah", "Dieter Büchler", "Dinesh Jayaraman", "Dmitry Kalashnikov", "Dorsa Sadigh", "Edward Johns", "Ethan Foster", "Fangchen Liu", "Federico Ceola", "Fei Xia", "Feiyu Zhao", "Felipe Vieira Frujeri", "Freek Stulp", "Gaoyue Zhou", "Gaurav S. Sukhatme", "Gautam Salhotra", "Ge Yan", "Gilbert Feng", "Giulio Schiavi", "Glen Berseth", "Gregory Kahn", "Guangwen Yang", "Guanzhi Wang", "Hao Su", "Hao-Shu Fang", "Haochen Shi", "Henghui Bao", "Heni Ben Amor", "Henrik I Christensen", "Hiroki Furuta", "Homanga Bharadhwaj", "Homer Walke", "Hongjie Fang", "Huy Ha", "Igor Mordatch", "Ilija Radosavovic", "Isabel Leal", "Jacky Liang", "Jad Abou-Chakra", "Jaehyung Kim", "Jaimyn Drake", "Jan Peters", "Jan Schneider", "Jasmine Hsu", "Jay Vakil", "Jeannette Bohg", "Jeffrey Bingham", "Jeffrey Wu", "Jensen Gao", "Jiaheng Hu", "Jiajun Wu", "Jialin Wu", "Jiankai Sun", "Jianlan Luo", "Jiayuan Gu", "Jie Tan", "Jihoon Oh", "Jimmy Wu", "Jingpei Lu", "Jingyun Yang", "Jitendra Malik", "João Silvério", "Joey Hejna", "Jonathan Booher", "Jonathan Tompson", "Jonathan Yang", "Jordi Salvador", "Joseph J. Lim", "Junhyek Han", "Kaiyuan Wang", "Kanishka Rao", "Karl Pertsch", "Karol Hausman", "Keegan Go", "Keerthana Gopalakrishnan", "Ken Goldberg", "Kendra Byrne", "Kenneth Oslund", "Kento Kawaharazuka", "Kevin Black", "Kevin Lin", "Kevin Zhang", "Kiana Ehsani", "Kiran Lekkala", "Kirsty Ellis", "Krishan Rana", "Krishnan Srinivasan", "Kuan Fang", "Kunal Pratap Singh", "Kuo-Hao Zeng", "Kyle Hatch", "Kyle Hsu", "Laurent Itti", "Lawrence Yunliang Chen", "Lerrel Pinto", "Li Fei-Fei", "Liam Tan", "Linxi \"Jim\" Fan", "Lionel Ott", "Lisa Lee", "Luca Weihs", "Magnum Chen", "Marion Lepert", "Marius Memmel", "Masayoshi Tomizuka", "Masha Itkina", "Mateo Guaman Castro", "Max Spero", "Maximilian Du", "Michael Ahn", "Michael C. Yip", "Mingtong Zhang", "Mingyu Ding", "Minho Heo", "Mohan Kumar Srirama", "Mohit Sharma", "Moo Jin Kim", "Muhammad Zubair Irshad", "Naoaki Kanazawa", "Nicklas Hansen", "Nicolas Heess", "Nikhil J Joshi", "Niko Suenderhauf", "Ning Liu", "Norman Di Palo", "Nur Muhammad Mahi Shafiullah", "Oier Mees", "Oliver Kroemer", "Osbert Bastani", "Pannag R Sanketi", "Patrick \"Tree\" Miller", "Patrick Yin", "Paul Wohlhart", "Peng Xu", "Peter David Fagan", "Peter Mitrano", "Pierre Sermanet", "Pieter Abbeel", "Priya Sundaresan", "Qiuyu Chen", "Quan Vuong", "Rafael Rafailov", "Ran Tian", "Ria Doshi", "Roberto Martín-Martín", "Rohan Baijal", "Rosario Scalise", "Rose Hendrix", "Roy Lin", "Runjia Qian", "Ruohan Zhang", "Russell Mendonca", "Rutav Shah", "Ryan Hoque", "Ryan Julian", "Samuel Bustamante", "Sean Kirmani", "Sergey Levine", "Shan Lin", "Sherry Moore", "Shikhar Bahl", "Shivin Dass", "Shubham Sonawani", "Shubham Tulsiani", "Shuran Song", "Sichun Xu", "Siddhant Haldar", "Siddharth Karamcheti", "Simeon Adebola", "Simon Guist", "Soroush Nasiriany", "Stefan Schaal", "Stefan Welker", "Stephen Tian", "Subramanian Ramamoorthy", "Sudeep Dasari", "Suneel Belkhale", "Sungjae Park", "Suraj Nair", "Suvir Mirchandani", "Takayuki Osa", "Tanmay Gupta", "Tatsuya Harada", "Tatsuya Matsushima", "Ted Xiao", "Thomas Kollar", "Tianhe Yu", "Tianli Ding", "Todor Davchev", "Tony Z. Zhao", "Travis Armstrong", "Trevor Darrell", "Trinity Chung", "Vidhi Jain", "Vikash Kumar", "Vincent Vanhoucke", "Vitor Guizilini", "Wei Zhan", "Wenxuan Zhou", "Wolfram Burgard", "Xi Chen", "Xiangyu Chen", "Xiaolong Wang", "Xinghao Zhu", "Xinyang Geng", "Xiyuan Liu", "Xu Liangwei", "Xuanlin Li", "Yansong Pang", "Yao Lu", "Yecheng Jason Ma", "Yejin Kim", "Yevgen Chebotar", "Yifan Zhou", "Yifeng Zhu", "Yilin Wu", "Ying Xu", "Yixuan Wang", "Yonatan Bisk", "Yongqiang Dou", "Yoonyoung Cho", "Youngwoon Lee", "Yuchen Cui", "Yue Cao", "Yueh-Hua Wu", "Yujin Tang", "Yuke Zhu", "Yunchu Zhang", "Yunfan Jiang", "Yunshuang Li", "Yunzhu Li", "Yusuke Iwasawa", "Yutaka Matsuo", "Zehan Ma", "Zhuo Xu", "Zichen Jeff Cui", "Zichen Zhang", "Zipeng Fu", "Zipeng Lin" ], "year": 2023, "date": "2023-10-13", "institutions": [ "34 labs worldwide" ], "paper": "https://arxiv.org/abs/2310.08864", "site": "https://robotics-transformer-x.github.io/", "fig": "figures/ds-oxe.webp", "figs": [ "figures/ds-oxe-1.webp", "figures/ds-oxe-2.webp" ], "hours_num": -1 }, { "id": "rh20t", "name": "RH20T", "title": "RH20T: A Comprehensive Robotic Dataset for Learning Diverse Skills in One-Shot", "cat": "robot", "org": "academia", "one": "Contact-rich manipulation at scale with force-torque, across many skills, robots and viewpoints — the most under-used corpus in this table.", "device": "Multiple arms + teleoperation (haptic + VR)", "modal": "RGB-D multi-view, force-torque, tactile, audio, proprioception", "hours": "—", "eps": "110,000+ sequences", "tasks": "147 tasks", "scenes": "diverse", "subj": "—", "embod": "4 robot configurations", "note": "One of very few large datasets with real force-torque throughout. If you care about contact-rich learning, this is the corpus to start from.", "arxiv": "2307.00595", "authors": [ "Hao-Shu Fang", "Hongjie Fang", "Zhenyu Tang", "Jirong Liu", "Chenxi Wang", "Junbo Wang", "Haoyi Zhu", "Cewu Lu" ], "year": 2023, "date": "2023-07-02", "institutions": [], "paper": "https://arxiv.org/abs/2307.00595", "site": "https://rh20t.github.io/", "fig": "figures/ds-rh20t.webp", "figs": [ "figures/ds-rh20t-1.webp", "figures/ds-rh20t-2.webp" ], "hours_num": -1 }, { "id": "roboagent", "name": "RoboAgent", "title": "RoboAgent: Generalization and Efficiency in Robot Manipulation via Semantic Augmentations and Action Chunking", "cat": "robot", "org": "academia", "one": "A sample-efficiency demonstration: 12 skills and 38 tasks from only 7,500 demonstrations via semantic augmentation.", "device": "Franka + teleoperation", "modal": "RGB multi-view, proprioception, language", "hours": "—", "eps": "7,500 demonstrations", "tasks": "38 tasks · 12 skills", "scenes": "kitchen scenes", "subj": "—", "embod": "Franka Panda", "note": "", "arxiv": "2309.01918", "authors": [ "Homanga Bharadhwaj", "Jay Vakil", "Mohit Sharma", "Abhinav Gupta", "Shubham Tulsiani", "Vikash Kumar" ], "year": 2023, "date": "2023-09-05", "institutions": [], "paper": "https://arxiv.org/abs/2309.01918", "site": "https://robopen.github.io/", "fig": "figures/ds-roboagent.webp", "figs": [ "figures/ds-roboagent-1.webp", "figures/ds-roboagent-2.webp" ], "hours_num": -1 }, { "id": "bcz", "name": "BC-Z", "title": "BC-Z: Zero-Shot Task Generalization with Robotic Imitation Learning", "cat": "robot", "org": "industry", "one": "Robot demonstrations paired with human videos and language, built for zero-shot task generalisation.", "device": "Everyday Robots + teleoperation + human video", "modal": "RGB, language, human demo video", "hours": "—", "eps": "25,877 episodes", "tasks": "100+ tasks", "scenes": "—", "subj": "—", "embod": "Everyday Robots", "note": "", "arxiv": "2202.02005", "authors": [ "Eric Jang", "Alex Irpan", "Mohi Khansari", "Daniel Kappler", "Frederik Ebert", "Corey Lynch", "Sergey Levine", "Chelsea Finn" ], "year": 2022, "date": "2022-02-04", "institutions": [], "paper": "https://arxiv.org/abs/2202.02005", "site": "https://sites.google.com/view/bc-z/home", "fig": "figures/ds-bcz.webp", "figs": [ "figures/ds-bcz-1.webp", "figures/ds-bcz-2.webp" ], "hours_num": -1 }, { "id": "langtable", "name": "Interactive Language", "title": "Interactive Language: Talking to Robots in Real Time", "cat": "robot", "org": "industry", "one": "Nearly 600K language-labelled trajectories for real-time language-conditioned control on a tabletop.", "device": "tabletop arm, teleoperation", "modal": "RGB, language annotations, actions", "hours": "—", "eps": "~600K language-labelled trajectories", "tasks": "long-horizon rearrangement", "scenes": "tabletop", "subj": "—", "embod": "xArm", "note": "An order of magnitude more language labels than anything before it.", "arxiv": "2210.06407", "authors": [ "Corey Lynch", "Ayzaan Wahid", "Jonathan Tompson", "Tianli Ding", "James Betker", "Robert Baruch", "Travis Armstrong", "Pete Florence" ], "year": 2022, "date": "2022-10-12", "institutions": [], "paper": "https://arxiv.org/abs/2210.06407", "site": "https://interactive-language.github.io/", "fig": "figures/ds-langtable.webp", "figs": [ "figures/ds-langtable-1.webp", "figures/ds-langtable-2.webp" ], "hours_num": -1 }, { "id": "rt1data", "name": "RT-1", "title": "RT-1: Robotics Transformer for Real-World Control at Scale", "cat": "robot", "org": "industry", "one": "Google's 17-month kitchen-scale collection — the dataset that first showed robot policies follow a data-scaling curve.", "device": "13 Everyday Robots mobile manipulators, teleoperation", "modal": "RGB, proprioception, language instructions", "hours": "—", "eps": "~130K episodes", "tasks": "700+ instructions", "scenes": "office kitchens", "subj": "—", "embod": "Everyday Robots", "note": "Not fully open, but the scaling result is the reason everything after it exists.", "arxiv": "2212.06817", "authors": [ "Anthony Brohan", "Noah Brown", "Justice Carbajal", "Yevgen Chebotar", "Joseph Dabis", "Chelsea Finn", "Keerthana Gopalakrishnan", "Karol Hausman", "Alex Herzog", "Jasmine Hsu", "Julian Ibarz", "Brian Ichter", "Alex Irpan", "Tomas Jackson", "Sally Jesmonth", "Nikhil J Joshi", "Ryan Julian", "Dmitry Kalashnikov", "Yuheng Kuang", "Isabel Leal", "Kuang-Huei Lee", "Sergey Levine", "Yao Lu", "Utsav Malla", "Deeksha Manjunath", "Igor Mordatch", "Ofir Nachum", "Carolina Parada", "Jodilyn Peralta", "Emily Perez", "Karl Pertsch", "Jornell Quiambao", "Kanishka Rao", "Michael Ryoo", "Grecia Salazar", "Pannag Sanketi", "Kevin Sayed", "Jaspiar Singh", "Sumedh Sontakke", "Austin Stone", "Clayton Tan", "Huong Tran", "Vincent Vanhoucke", "Steve Vega", "Quan Vuong", "Fei Xia", "Ted Xiao", "Peng Xu", "Sichun Xu", "Tianhe Yu", "Brianna Zitkovich" ], "year": 2022, "date": "2022-12-13", "institutions": [ "Google" ], "paper": "https://arxiv.org/abs/2212.06817", "site": "https://robotics-transformer1.github.io/", "fig": "figures/ds-rt1data.webp", "figs": [ "figures/ds-rt1data-1.webp" ], "hours_num": -1 }, { "id": "robonet", "name": "RoboNet", "title": "RoboNet: Large-Scale Multi-Robot Learning", "cat": "robot", "org": "academia", "one": "The first serious multi-robot pooled dataset — video of many robots interacting with objects, shared across institutions.", "device": "mixed (7 robot platforms)", "modal": "RGB video, actions, states", "hours": "—", "eps": "15M+ video frames", "tasks": "—", "scenes": "—", "subj": "4 institutions", "embod": "7 robot platforms", "note": "The intellectual ancestor of Open X-Embodiment, five years early.", "arxiv": "1910.11215", "authors": [ "Sudeep Dasari", "Frederik Ebert", "Stephen Tian", "Suraj Nair", "Bernadette Bucher", "Karl Schmeckpeper", "Siddharth Singh", "Sergey Levine", "Chelsea Finn" ], "year": 2019, "date": "2019-10-24", "institutions": [], "paper": "https://arxiv.org/abs/1910.11215", "site": "https://www.robonet.wiki/", "fig": "figures/ds-robonet.webp", "figs": [ "figures/ds-robonet-1.webp" ], "hours_num": -1 }, { "id": "dexgraspnet2", "name": "DexGraspNet 2.0", "title": "DexGraspNet 2.0: Learning Generative Dexterous Grasping in Large-scale Synthetic Cluttered Scenes", "cat": "dex", "org": "academia", "one": "Dexterous grasping in cluttered scenes at scale — 427 million grasps with demonstrated zero-shot sim-to-real.", "device": "synthetic", "modal": "grasp poses, depth, cluttered scenes", "hours": "—", "eps": "427M grasps", "tasks": "cluttered grasping", "scenes": "8,270 scenes", "subj": "1,319 objects", "embod": "dexterous hand (sim)", "note": "Reports 90.7% real-world success in clutter from synthetic data alone — a strong data point for the soft/sim2real debate.", "arxiv": "2410.23004", "authors": [ "Jialiang Zhang", "Haoran Liu", "Danshi Li", "Xinqiang Yu", "Haoran Geng", "Yufei Ding", "Jiayi Chen", "He Wang" ], "year": 2024, "date": "2024-10-30", "institutions": [], "paper": "https://arxiv.org/abs/2410.23004", "site": "https://pku-epic.github.io/DexGraspNet2.0/", "fig": "figures/ds-dexgraspnet2.webp", "figs": [ "figures/ds-dexgraspnet2-1.webp", "figures/ds-dexgraspnet2-2.webp" ], "hours_num": -1 }, { "id": "oakink2", "name": "OAKINK2", "title": "OAKINK2: A Dataset of Bimanual Hands-Object Manipulation in Complex Task Completion", "cat": "dex", "org": "academia", "one": "Bimanual hand-object manipulation for complex multi-step daily tasks, organised into affordance / primitive / complex task levels.", "device": "Multi-view mocap", "modal": "RGB, 3D hand-object pose, task hierarchy", "hours": "—", "eps": "—", "tasks": "complex daily activities", "scenes": "lab", "subj": "—", "embod": "human", "note": "The task-hierarchy abstraction is unusual and useful if you want long-horizon structure rather than single grasps.", "arxiv": "2403.19417", "authors": [ "Xinyu Zhan", "Lixin Yang", "Yifei Zhao", "Kangrui Mao", "Hanlin Xu", "Zenan Lin", "Kailin Li", "Cewu Lu" ], "year": 2024, "date": "2024-03-28", "institutions": [], "paper": "https://arxiv.org/abs/2403.19417", "site": "https://oakink.net/v2/", "fig": "figures/ds-oakink2.webp", "figs": [ "figures/ds-oakink2-1.webp", "figures/ds-oakink2-2.webp" ], "hours_num": -1 }, { "id": "taco", "name": "TACO", "title": "TACO: Benchmarking Generalizable Bimanual Tool-ACtion-Object Understanding", "cat": "dex", "org": "academia", "one": "Bimanual tool-action-object interaction with paired third-person and egocentric views.", "device": "Multi-view + head-mounted", "modal": "RGB, ego+exo, 3D hand-object meshes, action labels", "hours": "—", "eps": "2.5K motion sequences", "tasks": "tool use", "scenes": "lab", "subj": "—", "embod": "human", "note": "Tool use specifically — the task family where functional correspondence matters most.", "arxiv": "2401.08399", "authors": [ "Yun Liu", "Haolin Yang", "Xu Si", "Ling Liu", "Zipeng Li", "Yuxiang Zhang", "Yebin Liu", "Li Yi" ], "year": 2024, "date": "2024-01-16", "institutions": [], "paper": "https://arxiv.org/abs/2401.08399", "site": "https://taco2024.github.io/", "fig": "figures/ds-taco.webp", "figs": [ "figures/ds-taco-1.webp", "figures/ds-taco-2.webp" ], "hours_num": -1 }, { "id": "dexart", "name": "DexArt", "title": "DexArt: Benchmarking Generalizable Dexterous Manipulation with Articulated Objects", "cat": "dex", "org": "academia", "one": "Benchmark for dexterous manipulation of articulated objects with generalisation to unseen instances.", "device": "simulation", "modal": "point cloud, articulation state", "hours": "—", "eps": "—", "tasks": "articulated manipulation", "scenes": "sim", "subj": "—", "embod": "Allegro hand (sim)", "note": "", "arxiv": "2305.05706", "authors": [ "Chen Bao", "Helin Xu", "Yuzhe Qin", "Xiaolong Wang" ], "year": 2023, "date": "2023-05-09", "institutions": [], "paper": "https://arxiv.org/abs/2305.05706", "site": "https://www.chenbao.tech/dexart/", "fig": "figures/ds-dexart.webp", "figs": [ "figures/ds-dexart-1.webp", "figures/ds-dexart-2.webp" ], "hours_num": -1 }, { "id": "arctic", "name": "ARCTIC", "title": "ARCTIC: A Dataset for Dexterous Bimanual Hand-Object Manipulation", "cat": "dex", "org": "academia", "one": "Two hands dexterously manipulating articulated objects, with synchronised 3D meshes and dense contact annotation.", "device": "Multi-view mocap (8 cameras) + head-mounted", "modal": "RGB, 3D hand+object meshes, contact maps", "hours": "—", "eps": "2.1M video frames", "tasks": "bimanual articulated manipulation", "scenes": "lab", "subj": "10 subjects", "embod": "human", "note": "The contact annotation is the part that matters — it is what contact-consistent retargeting needs as supervision.", "arxiv": "2204.13662", "authors": [ "Zicong Fan", "Omid Taheri", "Dimitrios Tzionas", "Muhammed Kocabas", "Manuel Kaufmann", "Michael J. Black", "Otmar Hilliges" ], "year": 2022, "date": "2022-04-28", "institutions": [], "paper": "https://arxiv.org/abs/2204.13662", "site": "https://arctic.is.tue.mpg.de/", "fig": "figures/ds-arctic.webp", "figs": [ "figures/ds-arctic-1.webp" ], "hours_num": -1 }, { "id": "dexgraspnet", "name": "DexGraspNet", "title": "DexGraspNet: A Large-Scale Robotic Dexterous Grasp Dataset for General Objects Based on Simulation", "cat": "dex", "org": "academia", "one": "A million-plus simulated ShadowHand grasps across thousands of objects, all validated in physics.", "device": "synthetic (Isaac Gym validated)", "modal": "grasp poses, object meshes", "hours": "—", "eps": "1.32M grasps", "tasks": "grasping", "scenes": "sim", "subj": "5,355 objects", "embod": "ShadowHand (sim)", "note": "Synthetic, but the validation-in-physics step is what makes it usable rather than decorative.", "arxiv": "2210.02697", "authors": [ "Ruicheng Wang", "Jialiang Zhang", "Jiayi Chen", "Yinzhen Xu", "Puhao Li", "Tengyu Liu", "He Wang" ], "year": 2022, "date": "2022-10-06", "institutions": [], "paper": "https://arxiv.org/abs/2210.02697", "site": "https://pku-epic.github.io/DexGraspNet/", "fig": "figures/ds-dexgraspnet.webp", "figs": [ "figures/ds-dexgraspnet-1.webp", "figures/ds-dexgraspnet-2.webp" ], "hours_num": -1 }, { "id": "dexycb", "name": "DexYCB", "title": "DexYCB: A Benchmark for Capturing Hand Grasping of Objects", "cat": "dex", "org": "industry", "one": "The standard benchmark for capturing human hand grasping of YCB objects, with 6D object and 3D hand pose.", "device": "8-camera RGB-D rig", "modal": "RGB-D, 3D hand pose, object 6D pose", "hours": "—", "eps": "1,000 sequences", "tasks": "grasping", "scenes": "lab", "subj": "10 subjects", "embod": "human", "note": "NVIDIA. Small and clean; still the default evaluation set for hand-pose and handover work.", "arxiv": "2104.04631", "authors": [ "Yu-Wei Chao", "Wei Yang", "Yu Xiang", "Pavlo Molchanov", "Ankur Handa", "Jonathan Tremblay", "Yashraj S. Narang", "Karl Van Wyk", "Umar Iqbal", "Stan Birchfield", "Jan Kautz", "Dieter Fox" ], "year": 2021, "date": "2021-04-09", "institutions": [], "paper": "https://arxiv.org/abs/2104.04631", "site": "https://dex-ycb.github.io/", "fig": "figures/ds-dexycb.webp", "figs": [ "figures/ds-dexycb-1.webp", "figures/ds-dexycb-2.webp" ], "hours_num": -1 }, { "id": "grab", "name": "GRAB", "title": "GRAB: A Dataset of Whole-Body Human Grasping of Objects", "cat": "dex", "org": "academia", "one": "Whole-body human grasping — not just hands, but the full body approaching and manipulating objects.", "device": "Optical mocap (Vicon) + object markers", "modal": "SMPL-X whole-body, hand pose, contact, object pose", "hours": "—", "eps": "1,334 sequences", "tasks": "grasp + intent", "scenes": "lab", "subj": "10 subjects · 51 objects", "embod": "human", "note": "Whole-body context for grasping — relevant once your manipulator is attached to a mobile base rather than a table.", "arxiv": "2008.11200", "authors": [ "Omid Taheri", "Nima Ghorbani", "Michael J. Black", "Dimitrios Tzionas" ], "year": 2020, "date": "2020-08-25", "institutions": [], "paper": "https://arxiv.org/abs/2008.11200", "site": "https://grab.is.tue.mpg.de/", "fig": "figures/ds-grab.webp", "hours_num": -1 }, { "id": "robotacdex", "name": "RoboTacDex", "title": "RoboTacDex: A Dexterous Visual-Tactile-Action Dataset for Humanoid Manipulation", "cat": "tactile", "org": "academia", "one": "Paired visual-tactile action data for humanoid dexterous manipulation.", "device": "dexterous hand with tactile skin", "modal": "RGB, tactile, joint state, actions", "hours": "—", "eps": "—", "tasks": "dexterous manipulation", "scenes": "—", "subj": "—", "embod": "humanoid dexterous hand", "note": "Exactly the paired modality a tactile world model needs for pretraining.", "arxiv": "2606.31836", "authors": [ "(see PDF)" ], "year": 2026, "date": "2026-06-31", "institutions": [ "Unitree Robotics" ], "paper": "https://arxiv.org/abs/2606.31836", "site": "https://arxiv.org/abs/2606.31836", "fig": "figures/ds-robotacdex.webp", "figs": [ "figures/ds-robotacdex-1.webp", "figures/ds-robotacdex-2.webp" ], "hours_num": -1 }, { "id": "tvl", "name": "A Touch, Vision, and Language Dataset for Multimodal Alignment", "title": "A Touch, Vision, and Language Dataset for Multimodal Alignment", "cat": "tactile", "org": "academia", "one": "In-the-wild vision-touch pairs with language labels — the dataset behind touch-vision-language alignment.", "device": "handheld DIGIT + camera", "modal": "tactile images, RGB, language", "hours": "—", "eps": "44K vision-touch pairs", "tasks": "—", "scenes": "in-the-wild", "subj": "—", "embod": "handheld", "note": "90% of labels are GPT-4V pseudo-labels — useful, but know what you are training on.", "arxiv": "2402.13232", "authors": [ "Letian Fu", "Gaurav Datta", "Huang Huang", "William Chung-Ho Panitch", "Jaimyn Drake", "Joseph Ortiz", "Mustafa Mukadam", "Mike Lambeta", "Roberto Calandra", "Ken Goldberg" ], "year": 2024, "date": "2024-02-20", "institutions": [], "paper": "https://arxiv.org/abs/2402.13232", "site": "https://tactile-vlm.github.io/", "fig": "figures/ds-tvl.webp", "figs": [ "figures/ds-tvl-1.webp", "figures/ds-tvl-2.webp" ], "hours_num": -1 }, { "id": "sparsh", "name": "Sparsh", "title": "Sparsh: Self-supervised touch representations for vision-based tactile sensing", "cat": "tactile", "org": "industry", "one": "Self-supervised touch representations plus TacBench — the closest thing tactile sensing has to a foundation model.", "device": "DIGIT, GelSight, Soft Bubbles", "modal": "tactile images", "hours": "—", "eps": "460K+ tactile images", "tasks": "TacBench: 6 tasks", "scenes": "lab", "subj": "—", "embod": "multiple sensors", "note": "Meta. Reports large gains over task-specific training — the tactile analogue of R3M, and the encoder to start from for a tactile world model.", "arxiv": "2410.24090", "authors": [ "Carolina Higuera", "Akash Sharma", "Chaithanya Krishna Bodduluri", "Taosha Fan", "Patrick Lancaster", "Mrinal Kalakrishnan", "Michael Kaess", "Byron Boots", "Mike Lambeta", "Tingfan Wu", "Mustafa Mukadam" ], "year": 2024, "date": "2024-10-31", "institutions": [], "paper": "https://arxiv.org/abs/2410.24090", "site": "https://sparsh-ssl.github.io/", "fig": "figures/ds-sparsh.webp", "figs": [ "figures/ds-sparsh-1.webp" ], "hours_num": -1 }, { "id": "tlv", "name": "Towards Comprehensive Multimodal Perception", "title": "Towards Comprehensive Multimodal Perception: Introducing the Touch-Language-Vision Dataset", "cat": "tactile", "org": "academia", "one": "A touch-language-vision dataset with human-annotated tactile descriptions for semantic alignment.", "device": "tactile sensor + camera", "modal": "tactile, RGB, language", "hours": "—", "eps": "20K pairs", "tasks": "—", "scenes": "lab", "subj": "—", "embod": "—", "note": "", "arxiv": "2403.09813", "authors": [ "Ning Cheng", "You Li", "Jing Gao", "Bin Fang", "Jinan Xu", "Wenjuan Han" ], "year": 2024, "date": "2024-03-14", "institutions": [], "paper": "https://arxiv.org/abs/2403.09813", "site": "https://arxiv.org/abs/2403.09813", "fig": "figures/ds-tlv.webp", "hours_num": -1 }, { "id": "objectfolder-real", "name": "The ObjectFolder Benchmark", "title": "The ObjectFolder Benchmark: Multisensory Learning with Neural and Real Objects", "cat": "tactile", "org": "academia", "one": "The real-world counterpart: measured vision, sound and touch for 100 physical household objects, with a 10-task benchmark.", "device": "scanning rig + GelSight + impact sound capture", "modal": "3D mesh, video, impact sounds, tactile", "hours": "—", "eps": "100 real objects", "tasks": "10 benchmark tasks", "scenes": "lab", "subj": "100 objects", "embod": "—", "note": "The sim-and-real pairing is what makes this useful for multisensory sim-to-real.", "arxiv": "2306.00956", "authors": [ "Ruohan Gao", "Yiming Dou", "Hao Li", "Tanmay Agarwal", "Jeannette Bohg", "Yunzhu Li", "Li Fei-Fei", "Jiajun Wu" ], "year": 2023, "date": "2023-06-01", "institutions": [], "paper": "https://arxiv.org/abs/2306.00956", "site": "https://objectfolder.stanford.edu/", "fig": "figures/ds-objectfolder-real.webp", "figs": [ "figures/ds-objectfolder-real-1.webp", "figures/ds-objectfolder-real-2.webp" ], "hours_num": -1 }, { "id": "objectfolder2", "name": "ObjectFolder 2.0", "title": "ObjectFolder 2.0: A Multisensory Object Dataset for Sim2Real Transfer", "cat": "tactile", "org": "academia", "one": "Virtualised objects with visual, acoustic and tactile neural representations, built for sim-to-real transfer.", "device": "simulated multisensory (neural implicit)", "modal": "visual, acoustic, tactile", "hours": "—", "eps": "1,000 objects", "tasks": "—", "scenes": "sim", "subj": "1,000 objects", "embod": "sim", "note": "", "arxiv": "2204.02389", "authors": [ "Ruohan Gao", "Zilin Si", "Yen-Yu Chang", "Samuel Clarke", "Jeannette Bohg", "Li Fei-Fei", "Wenzhen Yuan", "Jiajun Wu" ], "year": 2022, "date": "2022-04-05", "institutions": [], "paper": "https://arxiv.org/abs/2204.02389", "site": "https://objectfolder.stanford.edu/", "fig": "figures/ds-objectfolder2.webp", "figs": [ "figures/ds-objectfolder2-1.webp", "figures/ds-objectfolder2-2.webp" ], "hours_num": -1 }, { "id": "touchandgo", "name": "Touch and Go", "title": "Touch and Go: Learning from Human-Collected Vision and Touch", "cat": "tactile", "org": "academia", "one": "Human-collected paired vision and touch in the wild — someone walking around probing objects with a tactile sensor.", "device": "GelSight + handheld camera", "modal": "RGB, tactile images", "hours": "—", "eps": "13,900 touches", "tasks": "—", "scenes": "indoor + outdoor", "subj": "3,971 objects", "embod": "handheld", "note": "In-the-wild rather than lab-bench tactile data, which is rare and what pretraining needs.", "arxiv": "2211.12498", "authors": [ "Fengyu Yang", "Chenyang Ma", "Jiacheng Zhang", "Jing Zhu", "Wenzhen Yuan", "Andrew Owens" ], "year": 2022, "date": "2022-11-22", "institutions": [], "paper": "https://arxiv.org/abs/2211.12498", "site": "https://touch-and-go.github.io/", "fig": "figures/ds-touchandgo.webp", "figs": [ "figures/ds-touchandgo-1.webp" ], "hours_num": -1 }, { "id": "musohu", "name": "Toward Human-Like Social Robot Navigation", "title": "Toward Human-Like Social Robot Navigation: A Large-Scale, Multi-Modal, Social Human Navigation Dataset", "cat": "hri", "org": "academia", "one": "Multi-modal recordings of human navigation behaviour, captured to teach robots to move like people do.", "device": "human-worn sensor rig", "modal": "RGB-D, 3D lidar, IMU, GPS, audio", "hours": "20 h", "eps": "300 trials · ~100 km", "tasks": "social navigation", "scenes": "public spaces", "subj": "13 humans", "embod": "human (for robot transfer)", "note": "", "arxiv": "2303.14880", "authors": [ "Duc M. Nguyen", "Mohammad Nazeri", "Amirreza Payandeh", "Aniket Datar", "Xuesu Xiao" ], "year": 2023, "date": "2023-03-27", "institutions": [], "paper": "https://arxiv.org/abs/2303.14880", "site": "https://arxiv.org/abs/2303.14880", "fig": "figures/ds-musohu.webp", "figs": [ "figures/ds-musohu-1.webp", "figures/ds-musohu-2.webp" ], "hours_num": 20.0 }, { "id": "scand", "name": "Socially Compliant Navigation Dataset (SCAND)", "title": "Socially Compliant Navigation Dataset (SCAND): A Large-Scale Dataset of Demonstrations for Social Navigation", "cat": "hri", "org": "academia", "one": "Socially compliant navigation demonstrations — humans teleoperating robots through real crowds.", "device": "Spot + Jackal, joystick teleoperation", "modal": "3D lidar, RGB, joystick commands, odometry, IMU", "hours": "8.7 h", "eps": "138 trajectories · 25 miles", "tasks": "social navigation", "scenes": "public campus spaces", "subj": "—", "embod": "Spot, Clearpath Jackal", "note": "", "arxiv": "2203.15041", "authors": [ "Haresh Karnan", "Anirudh Nair", "Xuesu Xiao", "Garrett Warnell", "Soeren Pirk", "Alexander Toshev", "Justin Hart", "Joydeep Biswas", "Peter Stone" ], "year": 2022, "date": "2022-03-28", "institutions": [], "paper": "https://arxiv.org/abs/2203.15041", "site": "https://www.cs.utexas.edu/~xiao/SCAND/SCAND.html", "fig": "figures/ds-scand.webp", "figs": [ "figures/ds-scand-1.webp", "figures/ds-scand-2.webp" ], "hours_num": 8.7 }, { "id": "thormagni", "name": "THÖR-MAGNI", "title": "THÖR-MAGNI: A Large-scale Indoor Motion Capture Recording of Human Movement and Robot Interaction", "cat": "hri", "org": "academia", "one": "Large-scale indoor motion capture of humans moving and interacting with robots in a shared workspace.", "device": "Qualisys mocap + robot + eye-tracking glasses", "modal": "6-DoF human trajectories, robot trajectories, gaze, video", "hours": "3.5 h", "eps": "—", "tasks": "shared-space navigation + interaction", "scenes": "indoor lab", "subj": "40 participants", "embod": "differential-drive + mobile manipulator", "note": "One of very few datasets with millimetre-accurate ground truth for both the human and the robot — the right substrate for measuring who adapts to whom.", "arxiv": "2403.09285", "authors": [ "Tim Schreiter", "Tiago Rodrigues de Almeida", "Yufei Zhu", "Eduardo Gutierrez Maestro", "Lucas Morillo-Mendez", "Andrey Rudenko", "Luigi Palmieri", "Tomasz P. Kucner", "Martin Magnusson", "Achim J. Lilienthal" ], "year": 2024, "date": "2024-03-14", "institutions": [], "paper": "https://arxiv.org/abs/2403.09285", "site": "http://thor.oru.se/", "fig": "figures/ds-thormagni.webp", "hours_num": 3.5 }, { "id": "jrdb", "name": "JRDB", "title": "JRDB: A Dataset and Benchmark of Egocentric Robot Visual Perception of Humans in Built Environments", "cat": "hri", "org": "academia", "one": "A social-navigation perception dataset recorded from a robot moving among people indoors and outdoors.", "device": "JackRabbot robot: 360° stereo, 2× Velodyne, 2× Sick lidar, audio", "modal": "360° RGB, 3D point clouds, audio, IMU, odometry", "hours": "1.07 h (64 min)", "eps": "2.3M bounding boxes · 1.8M 3D cuboids", "tasks": "detection + tracking", "scenes": "indoor + outdoor campus", "subj": "3,500+ person trajectories", "embod": "JackRabbot", "note": "The standard for robot-centric human perception. Short but exhaustively annotated.", "arxiv": "1910.11792", "authors": [ "Roberto Martín-Martín", "Mihir Patel", "Hamid Rezatofighi", "Abhijeet Shenoi", "JunYoung Gwak", "Eric Frankel", "Amir Sadeghian", "Silvio Savarese" ], "year": 2019, "date": "2019-10-25", "institutions": [], "paper": "https://arxiv.org/abs/1910.11792", "site": "https://jrdb.erc.monash.edu/", "fig": "figures/ds-jrdb.webp", "figs": [ "figures/ds-jrdb-1.webp", "figures/ds-jrdb-2.webp" ], "hours_num": 1.07 }, { "id": "r2handoversim", "name": "R2HandoverSim", "title": "R2HandoverSim: A Simulation Framework and Benchmark for Robot-to-Human Object Handovers", "cat": "hri", "org": "academia", "one": "The reverse direction: a simulation benchmark for robot-to-human handovers, which had no standard protocol.", "device": "simulation", "modal": "RGB-D, hand + object pose", "hours": "—", "eps": "—", "tasks": "robot-to-human handover", "scenes": "sim", "subj": "—", "embod": "arm + gripper", "note": "", "arxiv": "2606.21011", "authors": [ "Hanxin Zhang", "Abdulqader Dhafer", "Hongbiao Dong", "Zhou Daniel Hao" ], "year": 2026, "date": "2026-06-19", "institutions": [], "paper": "https://arxiv.org/abs/2606.21011", "site": "https://arxiv.org/abs/2606.21011", "fig": "figures/ds-r2handoversim.webp", "figs": [ "figures/ds-r2handoversim-1.webp", "figures/ds-r2handoversim-2.webp" ], "hours_num": -1 }, { "id": "dexh2r", "name": "DexH2R", "title": "DexH2R: A Benchmark for Dynamic Dexterous Grasping in Human-to-Robot Handover", "cat": "hri", "org": "academia", "one": "Real-world human-to-robot handovers on a dexterous hand, with dense frame-level annotation.", "device": "dexterous robotic hand + multi-camera", "modal": "RGB-D, hand + object pose, robot state", "hours": "—", "eps": "4,282 handover trials · 456K frames", "tasks": "dynamic handover", "scenes": "lab", "subj": "39 participants · 56 objects", "embod": "dexterous hand", "note": "The largest real handover dataset on a multi-finger hand — the physical-HRI data that barely exists elsewhere.", "arxiv": "2506.23152", "authors": [ "Youzhuo Wang", "Jiayi Ye", "Chuyang Xiao", "Yiming Zhong", "Heng Tao", "Hang Yu", "Yumeng Liu", "Jingyi Yu", "Yuexin Ma" ], "year": 2025, "date": "2025-06-29", "institutions": [], "paper": "https://arxiv.org/abs/2506.23152", "site": "https://arxiv.org/abs/2506.23152", "fig": "figures/ds-dexh2r.webp", "figs": [ "figures/ds-dexh2r-1.webp", "figures/ds-dexh2r-2.webp" ], "hours_num": -1 }, { "id": "kaiwu", "name": "Kaiwu", "title": "Kaiwu: A Multimodal Manipulation Dataset and Framework for Robot Learning and Human-Robot Interaction", "cat": "hri", "org": "academia", "one": "A multimodal manipulation dataset that records the human, the environment and the robot together — built for robot learning and HRI at once.", "device": "mocap + multi-view cameras + robot", "modal": "RGB-D, human pose, object pose, robot state, force", "hours": "—", "eps": "11,664 integrated instances", "tasks": "manipulation + interaction", "scenes": "lab", "subj": "20 subjects · 30 objects", "embod": "robot arm", "note": "Unusual in capturing all three streams synchronously. If you want to measure who adapts to whom in a collaboration, this is the closest existing data.", "arxiv": "2503.05231", "authors": [ "Jiang, Shuo", "Li, Haonan", "Ren, Ruochen", "Zhou, Yanmin", "Wang, Zhipeng", "He, Bin" ], "year": 2025, "date": "2025-03-07", "institutions": [], "paper": "https://arxiv.org/abs/2503.05231", "site": "https://arxiv.org/abs/2503.05231", "fig": "figures/ds-kaiwu.webp", "figs": [ "figures/ds-kaiwu-1.webp", "figures/ds-kaiwu-2.webp" ], "hours_num": -1 }, { "id": "interact", "name": "InteRACT", "title": "InteRACT: Transformer Models for Human Intent Prediction Conditioned on Robot Actions", "cat": "hri", "org": "academia", "one": "Human-robot collaborative manipulation data where human intent is predicted *conditioned on what the robot does* — with the teleoperation setup open-sourced.", "device": "7-DoF arm teleoperation + human motion capture", "modal": "RGB-D, human pose, robot state", "hours": "—", "eps": "—", "tasks": "collaborative manipulation", "scenes": "tabletop", "subj": "—", "embod": "7-DoF arm", "note": "The conditioning is the point: most intent-prediction datasets treat the human as if the robot were not there. This one does not, which is exactly the framing your HRI idea needs.", "arxiv": "2311.12943", "authors": [ "Kushal Kedia", "Atiksh Bhardwaj", "Prithwish Dan", "Sanjiban Choudhury" ], "year": 2023, "date": "2023-11-21", "institutions": [], "paper": "https://arxiv.org/abs/2311.12943", "site": "https://arxiv.org/abs/2311.12943", "fig": "figures/ds-interact.webp", "figs": [ "figures/ds-interact-1.webp", "figures/ds-interact-2.webp" ], "hours_num": -1 }, { "id": "handoversim", "name": "HandoverSim", "title": "HandoverSim: A Simulation Framework and Benchmark for Human-to-Robot Object Handovers", "cat": "hri", "org": "industry", "one": "Simulation benchmark for human-to-robot handovers, driven by DexYCB human motion.", "device": "simulation (DexYCB-driven human)", "modal": "RGB-D, hand + object pose", "hours": "—", "eps": "1,000 handover scenarios", "tasks": "human-to-robot handover", "scenes": "sim", "subj": "—", "embod": "Panda + gripper", "note": "NVIDIA. The standard evaluation protocol for handovers, and the reason later handover work is comparable at all.", "arxiv": "2205.09747", "authors": [ "Yu-Wei Chao", "Chris Paxton", "Yu Xiang", "Wei Yang", "Balakumar Sundaralingam", "Tao Chen", "Adithyavairavan Murali", "Maya Cakmak", "Dieter Fox" ], "year": 2022, "date": "2022-05-19", "institutions": [], "paper": "https://arxiv.org/abs/2205.09747", "site": "https://handover-sim.github.io/", "fig": "figures/ds-handoversim.webp", "figs": [ "figures/ds-handoversim-1.webp" ], "hours_num": -1 }, { "id": "jrdbact", "name": "JRDB-Act", "title": "JRDB-Act: A Large-scale Dataset for Spatio-temporal Action, Social Group and Activity Detection", "cat": "hri", "org": "academia", "one": "JRDB extended with spatio-temporal action, social grouping and activity labels.", "device": "JackRabbot", "modal": "360° RGB, 3D, action + social group labels", "hours": "—", "eps": "2.8M action labels", "tasks": "action + social group detection", "scenes": "campus", "subj": "—", "embod": "JackRabbot", "note": "", "arxiv": "2106.08827", "authors": [ "Mahsa Ehsanpour", "Fatemeh Saleh", "Silvio Savarese", "Ian Reid", "Hamid Rezatofighi" ], "year": 2021, "date": "2021-06-16", "institutions": [], "paper": "https://arxiv.org/abs/2106.08827", "site": "https://jrdb.erc.monash.edu/", "fig": "figures/ds-jrdbact.webp", "figs": [ "figures/ds-jrdbact-1.webp", "figures/ds-jrdbact-2.webp" ], "hours_num": -1 }, { "id": "mogaze", "name": "MoGaze", "title": "MoGaze: A Dataset of Full-Body Motions that Includes Workspace Geometry\\n and Eye-Gaze", "cat": "hri", "org": "academia", "one": "Long manipulation sequences with full-body motion, the 3D geometry of the workspace, and eye gaze — three things rarely captured together.", "device": "optical mocap + eye tracker + scanned workspace", "modal": "full-body 3D motion, eye gaze, workspace geometry, object poses", "hours": "—", "eps": "~3 h of manipulation sequences", "tasks": "pick-and-place sequences", "scenes": "scanned lab", "subj": "6 participants", "embod": "human (for robot prediction)", "note": "Eye gaze is the strongest early signal of human intent and almost nothing else records it alongside full-body motion and scene geometry.", "arxiv": "2011.11552", "authors": [ "Philipp Kratzer", "Simon Bihlmaier", "Niteesh Balachandra Midlagajni", "Rohit Prakash", "Marc Toussaint", "Jim Mainprice" ], "year": 2020, "date": "2020-11-23", "institutions": [], "paper": "https://arxiv.org/abs/2011.11552", "site": "https://humans-to-robots-motion.github.io/mogaze/", "fig": "figures/ds-mogaze.webp", "figs": [ "figures/ds-mogaze-1.webp", "figures/ds-mogaze-2.webp" ], "hours_num": -1 }, { "id": "ga3t", "name": "GA3T", "title": "GA3T: A Ground-Aerial Terrain Traversability Dataset for Heterogeneous Robot Teams in Unstructured Environments", "cat": "multi", "org": "academia", "one": "A real-world ground-aerial collaborative perception dataset: a Husky UGV and an Autel UAV working the same unstructured terrain together.", "device": "Clearpath Husky UGV + Autel EVO II UAV", "modal": "RGB, LiDAR, IMU, GPS, cross-agent views", "hours": "—", "eps": "—", "tasks": "terrain traversability, collaborative perception", "scenes": "unstructured outdoor", "subj": "2 heterogeneous agents", "embod": "UGV + UAV", "note": "Rare: an actual real-world multi-robot dataset rather than a simulator. Aerial-ground rather than manipulation, but the cross-agent perception problem is the same one M2 has.", "arxiv": "2605.06478", "authors": [ "Siwei Cai", "Knut Peterson", "Quan Tran", "Christian Ricks", "Dhanush Parthasarathy", "Amir Kaidarov", "Neil Deshpande", "Sukaina Najm", "David Han", "Lifeng Zhou" ], "year": 2026, "date": "2026-05-07", "institutions": [], "paper": "https://arxiv.org/abs/2605.06478", "site": "https://arxiv.org/abs/2605.06478", "fig": "figures/ds-ga3t.webp", "hours_num": -1 }, { "id": "hercules", "name": "HERCULES", "title": "HERCULES: An Open-Source Simulation Framework for Heterogeneous Multi-Robot SLAM, Collaborative Perception, and Exploration", "cat": "multi", "org": "academia", "one": "An open simulation framework and benchmark for heterogeneous multi-robot SLAM, collaborative perception and exploration.", "device": "simulation (Unreal Engine 5 / AirSim)", "modal": "RGB, depth, LiDAR, IMU, ground-truth poses", "hours": "—", "eps": "—", "tasks": "collaborative SLAM, exploration", "scenes": "large-scale sim", "subj": "concurrent aerial + ground agents", "embod": "UAV + UGV", "note": "", "arxiv": "2606.22756", "authors": [ "Sandilya Sai Garimella", "Daniel Chase Butterfield", "Sean Wilson", "Lu Gan" ], "year": 2026, "date": "2026-06-22", "institutions": [], "paper": "https://arxiv.org/abs/2606.22756", "site": "https://arxiv.org/abs/2606.22756", "fig": "figures/ds-hercules.webp", "figs": [ "figures/ds-hercules-1.webp", "figures/ds-hercules-2.webp" ], "hours_num": -1 }, { "id": "hribench", "name": "HRIBench", "title": "HRIBench: Benchmarking Interaction-Centric Human-Robot Collaboration", "cat": "multi", "org": "academia", "one": "Interaction-centric human-robot collaboration benchmark with role-conditioned tasks requiring synchronised multi-agent coordination.", "device": "simulation", "modal": "RGB, state, role conditioning", "hours": "—", "eps": "650+ evaluation episodes", "tasks": "13 role-conditioned tasks", "scenes": "sim", "subj": "—", "embod": "humanoid + human agent", "note": "Reports lifting GR00T N1.5's physical-task success from 0.10 to 0.43 using its simulated data — the most direct evidence that collaboration data is a real gap.", "arxiv": "2607.13056", "authors": [ "Chang Liu", "Jiawei Zhang", "Tao Zhang", "Ye Wang", "Hongyu Zhou", "Qin Jin" ], "year": 2026, "date": "2026-07-05", "institutions": [], "paper": "https://arxiv.org/abs/2607.13056", "site": "https://arxiv.org/abs/2607.13056", "fig": "figures/ds-hribench.webp", "figs": [ "figures/ds-hribench-1.webp", "figures/ds-hribench-2.webp" ], "hours_num": -1 }, { "id": "rocochallenge", "name": "RoCo Challenge at AAAI 2026", "title": "RoCo Challenge at AAAI 2026: Benchmarking Robotic Collaborative Manipulation for Assembly Towards Industrial Automation", "cat": "multi", "org": "consortium", "one": "An AAAI 2026 challenge on collaborative assembly, released with a teleoperated dual-arm mobile manipulation dataset.", "device": "dual-arm mobile platform, teleoperation", "modal": "RGB, depth, proprioception, action streams (synchronised)", "hours": "—", "eps": "300+ teleoperated demonstrations", "tasks": "collaborative assembly", "scenes": "industrial", "subj": "60+ teams, 170+ participants, 10+ countries", "embod": "dual-arm mobile manipulator", "note": "One of the very few real-robot collaborative-manipulation datasets that exists, and it comes with a leaderboard. Directly usable for M2 as a baseline setting rather than a from-scratch build.", "arxiv": "2603.15469", "authors": [ "Haichao Liu", "Yuheng Zhou", "Zhenyu Wu", "Ziheng Ji", "Ziyu Shan", "Qianzhun Wang", "Ruixuan Liu", "Zhiyuan Yang", "Yejun Gu", "Shalman Khan", "Shijun Yan", "Jun Liu", "Haiyue Zhu", "Changliu Liu", "Jianfei Yang", "Jingbing Zhang", "Ziwei Wang" ], "year": 2026, "date": "2026-03-16", "institutions": [], "paper": "https://arxiv.org/abs/2603.15469", "site": "https://arxiv.org/abs/2603.15469", "fig": "figures/ds-rocochallenge.webp", "figs": [ "figures/ds-rocochallenge-1.webp", "figures/ds-rocochallenge-2.webp" ], "hours_num": -1 }, { "id": "comuros", "name": "LLM-Based Generalizable Hierarchical Task Planning and Execution for Heterogeneous Robot Teams with Event-Driven Replanning", "title": "LLM-Based Generalizable Hierarchical Task Planning and Execution for Heterogeneous Robot Teams with Event-Driven Replanning", "cat": "multi", "org": "academia", "one": "A heterogeneous-team benchmark of scenarios and tasks spanning roughly twenty robots, with a task-manager LLM allocating subtasks to robot-level LLMs.", "device": "simulation + hardware", "modal": "state, language, ROS2 skill traces", "hours": "—", "eps": "22 scenarios", "tasks": "54 tasks", "scenes": "—", "subj": "~20 robots", "embod": "heterogeneous team", "note": "Interesting for M2 because it reports recovery from disruptive events and coordinated transport with emergent human-robot cooperation — i.e. it measures robustness, not just success.", "arxiv": "2511.22354", "authors": [ "Suraj Borate", "B Bhavish", "Vipul Pardeshi", "Madhu Vadali" ], "year": 2025, "date": "2025-11-27", "institutions": [], "paper": "https://arxiv.org/abs/2511.22354", "site": "https://arxiv.org/abs/2511.22354", "fig": "figures/ds-comuros.webp", "figs": [ "figures/ds-comuros-1.webp" ], "hours_num": -1 }, { "id": "humanthor", "name": "Demonstrating HumanTHOR", "title": "Demonstrating HumanTHOR: A Simulation Platform and Benchmark for Human-Robot Collaboration in a Shared Workspace", "cat": "multi", "org": "academia", "one": "Simulation platform and benchmark for human-robot collaboration in a shared workspace, with realistic human agents.", "device": "simulation (AI2-THOR based)", "modal": "RGB-D, scene state", "hours": "—", "eps": "—", "tasks": "shared-workspace collaboration", "scenes": "sim households", "subj": "simulated humans", "embod": "mobile manipulator", "note": "", "arxiv": "2406.06498", "authors": [ "Chenxu Wang", "Boyuan Du", "Jiaxin Xu", "Peiyan Li", "Di Guo", "Huaping Liu" ], "year": 2024, "date": "2024-06-10", "institutions": [], "paper": "https://arxiv.org/abs/2406.06498", "site": "https://sites.google.com/view/humanthor", "fig": "figures/ds-humanthor.webp", "figs": [ "figures/ds-humanthor-1.webp", "figures/ds-humanthor-2.webp" ], "hours_num": -1 }, { "id": "rocobench", "name": "RoCo", "title": "RoCo: Dialectic Multi-Robot Collaboration with Large Language Models", "cat": "multi", "org": "academia", "one": "The reference multi-robot collaboration benchmark: six tabletop tasks that need both high-level communication and coordinated multi-arm motion planning.", "device": "simulation + real multi-arm", "modal": "state, RGB, language dialogue", "hours": "—", "eps": "text dataset for agent reasoning", "tasks": "6 collaboration tasks", "scenes": "tabletop", "subj": "—", "embod": "multi-arm (sim + real)", "note": "RoCoBench is what almost every LLM multi-robot paper now reports on. It also supports human-in-the-loop, so a person can be one of the agents — the bridge to your HRI idea.", "arxiv": "2307.04738", "authors": [ "Zhao Mandi", "Shreeya Jain", "Shuran Song" ], "year": 2023, "date": "2023-07-10", "institutions": [], "paper": "https://arxiv.org/abs/2307.04738", "site": "https://project-roco.github.io/", "fig": "figures/ds-rocobench.webp", "figs": [ "figures/ds-rocobench-1.webp", "figures/ds-rocobench-2.webp" ], "hours_num": -1 }, { "id": "assembly101", "name": "Assembly101", "title": "Assembly101: A Large-Scale Multi-View Video Dataset for Understanding Procedural Activities", "cat": "exo", "org": "academia", "one": "Multi-view procedural assembly and disassembly with both fixed and egocentric cameras and dense hand poses.", "device": "8 static RGB + 4 head-mounted monochrome", "modal": "RGB, multi-view, 3D hand poses, action segments", "hours": "513 h", "eps": "4,321 videos", "tasks": "101 toy vehicles · 1M fine-grained segments", "scenes": "lab", "subj": "53 participants", "embod": "human (ego + exo)", "note": "Both viewpoints and 18M hand poses — a good testbed for the ego-versus-exo comparison at task level.", "arxiv": "2203.14712", "authors": [ "Fadime Sener", "Dibyadip Chatterjee", "Daniel Shelepov", "Kun He", "Dipika Singhania", "Robert Wang", "Angela Yao" ], "year": 2022, "date": "2022-03-28", "institutions": [], "paper": "https://arxiv.org/abs/2203.14712", "site": "https://assembly-101.github.io/", "fig": "figures/ds-assembly101.webp", "figs": [ "figures/ds-assembly101-1.webp", "figures/ds-assembly101-2.webp" ], "hours_num": 513.0 }, { "id": "howto100m", "name": "HowTo100M", "title": "HowTo100M: Learning a Text-Video Embedding by Watching Hundred Million Narrated Video Clips", "cat": "exo", "org": "academia", "one": "136 million clips from 1.22M narrated instructional web videos — the largest source of humans explaining tasks while doing them.", "device": "web video (YouTube)", "modal": "RGB, ASR narration", "hours": "15 years of video", "eps": "136M clips", "tasks": "23K visual tasks", "scenes": "in-the-wild", "subj": "—", "embod": "human (third-person)", "note": "Noisy and enormous. The natural pretraining corpus for third-person learning, if you can handle the noise.", "arxiv": "1906.03327", "authors": [ "Antoine Miech", "Dimitri Zhukov", "Jean-Baptiste Alayrac", "Makarand Tapaswi", "Ivan Laptev", "Josef Sivic" ], "year": 2019, "date": "2019-06-07", "institutions": [], "paper": "https://arxiv.org/abs/1906.03327", "site": "https://www.di.ens.fr/willow/research/howto100m/", "fig": "figures/ds-howto100m.webp", "figs": [ "figures/ds-howto100m-1.webp" ], "hours_num": 15.0 }, { "id": "ssv2", "name": "The \"something something\" video database for learning and evaluating visual common sense", "title": "The \"something something\" video database for learning and evaluating visual common sense", "cat": "exo", "org": "industry", "one": "Crowd-acted third-person clips of basic physical interactions, built to test whether models understand physical causality rather than object identity.", "device": "crowdsourced handheld video", "modal": "RGB, action class", "hours": "—", "eps": "220,847 videos", "tasks": "174 action templates", "scenes": "in-the-wild", "subj": "crowd workers", "embod": "human (third-person)", "note": "Still the sharpest test of physical rather than semantic video understanding.", "arxiv": "1706.04261", "authors": [ "Raghav Goyal", "Samira Ebrahimi Kahou", "Vincent Michalski", "Joanna Materzyńska", "Susanne Westphal", "Heuna Kim", "Valentin Haenel", "Ingo Fruend", "Peter Yianilos", "Moritz Mueller-Freitag", "Florian Hoppe", "Christian Thurau", "Ingo Bax", "Roland Memisevic" ], "year": 2017, "date": "2017-06-13", "institutions": [], "paper": "https://arxiv.org/abs/1706.04261", "site": "https://www.qualcomm.com/developer/software/something-something-v-2-dataset", "fig": "figures/ds-ssv2.webp", "figs": [ "figures/ds-ssv2-1.webp" ], "hours_num": -1 }, { "id": "amass", "name": "AMASS", "title": "AMASS: Archive of Motion Capture as Surface Shapes", "cat": "motion", "org": "academia", "one": "Fifteen optical mocap datasets unified into one SMPL body parameterisation — the backbone of humanoid motion imitation.", "device": "optical marker-based mocap (15 sources)", "modal": "SMPL body pose + shape", "hours": "40+ h", "eps": "11,000+ motions", "tasks": "—", "scenes": "mocap studios", "subj": "300+ subjects", "embod": "human (retargetable)", "note": "Everything from DeepMimic descendants through PHC, H2O and HOVER is trained on retargeted AMASS.", "arxiv": "1904.03278", "authors": [ "Naureen Mahmood", "Nima Ghorbani", "Nikolaus F. Troje", "Gerard Pons-Moll", "Michael J. Black" ], "year": 2019, "date": "2019-04-05", "institutions": [], "paper": "https://arxiv.org/abs/1904.03278", "site": "https://amass.is.tue.mpg.de/", "fig": "figures/ds-amass.webp", "figs": [ "figures/ds-amass-1.webp", "figures/ds-amass-2.webp" ], "hours_num": 40.0 }, { "id": "humanoidx", "name": "Learning from Massive Human Videos for Universal Humanoid Pose Control", "title": "Learning from Massive Human Videos for Universal Humanoid Pose Control", "cat": "motion", "org": "academia", "one": "Twenty million humanoid robot poses with paired text descriptions, mined from massive human video.", "device": "human video → pose → retargeting pipeline", "modal": "humanoid joint poses, text", "hours": "—", "eps": "20M+ humanoid poses", "tasks": "text-conditioned pose control", "scenes": "in-the-wild", "subj": "—", "embod": "Unitree H1", "note": "Already retargeted to a real humanoid, which saves the step everyone else redoes.", "arxiv": "2412.14172", "authors": [ "Jiageng Mao", "Siheng Zhao", "Siqi Song", "Tianheng Shi", "Junjie Ye", "Mingtong Zhang", "Haoran Geng", "Jitendra Malik", "Vitor Guizilini", "Yue Wang" ], "year": 2024, "date": "2024-12-18", "institutions": [], "paper": "https://arxiv.org/abs/2412.14172", "site": "https://usc-gvl.github.io/UH-1/", "fig": "figures/ds-humanoidx.webp", "figs": [ "figures/ds-humanoidx-1.webp", "figures/ds-humanoidx-2.webp" ], "hours_num": -1 }, { "id": "motionx", "name": "Motion-X", "title": "Motion-X: A Large-scale 3D Expressive Whole-body Human Motion Dataset", "cat": "motion", "org": "academia", "one": "Expressive whole-body motion including hands and face, annotated at frame level with text.", "device": "multi-source video + annotation pipeline", "modal": "SMPL-X whole-body pose, text descriptions", "hours": "—", "eps": "81.1K motion sequences · 15.6M pose annotations", "tasks": "—", "scenes": "in-the-wild", "subj": "—", "embod": "human (retargetable)", "note": "Hands and face, which AMASS lacks — relevant if your humanoid has fingers.", "arxiv": "2307.00818", "authors": [ "Jing Lin", "Ailing Zeng", "Shunlin Lu", "Yuanhao Cai", "Ruimao Zhang", "Haoqian Wang", "Lei Zhang" ], "year": 2023, "date": "2023-07-03", "institutions": [], "paper": "https://arxiv.org/abs/2307.00818", "site": "https://motion-x-dataset.github.io/", "fig": "figures/ds-motionx.webp", "figs": [ "figures/ds-motionx-1.webp", "figures/ds-motionx-2.webp" ], "hours_num": -1 }, { "id": "roboverse", "name": "RoboVerse", "title": "RoboVerse: Towards a Unified Platform, Dataset and Benchmark for Scalable and Generalizable Robot Learning", "cat": "sim", "org": "consortium", "one": "A unified platform, dataset and benchmark that bridges many existing simulators under one interface.", "device": "multi-simulator", "modal": "RGB-D, state, language", "hours": "—", "eps": "—", "tasks": "—", "scenes": "sim", "subj": "—", "embod": "many (sim)", "note": "", "arxiv": "2504.18904", "authors": [ "Haoran Geng", "Feishi Wang", "Songlin Wei", "Yuyang Li", "Bangjun Wang", "Boshi An", "Charlie Tianyue Cheng", "Haozhe Lou", "Peihao Li", "Yen-Jen Wang", "Yutong Liang", "Dylan Goetting", "Chaoyi Xu", "Haozhe Chen", "Yuxi Qian", "Yiran Geng", "Jiageng Mao", "Weikang Wan", "Mingtong Zhang", "Jiangran Lyu", "Siheng Zhao", "Jiazhao Zhang", "Jialiang Zhang", "Chengyang Zhao", "Haoran Lu", "Yufei Ding", "Ran Gong", "Yuran Wang", "Yuxuan Kuang", "Ruihai Wu", "Baoxiong Jia", "Carlo Sferrazza", "Hao Dong", "Siyuan Huang", "Yue Wang", "Jitendra Malik", "Pieter Abbeel" ], "year": 2025, "date": "2025-04-26", "institutions": [], "paper": "https://arxiv.org/abs/2504.18904", "site": "https://roboverseorg.github.io/", "fig": "figures/ds-roboverse.webp", "figs": [ "figures/ds-roboverse-1.webp", "figures/ds-roboverse-2.webp" ], "hours_num": -1 }, { "id": "behavior1k", "name": "BEHAVIOR-1K", "title": "BEHAVIOR-1K: A Human-Centered, Embodied AI Benchmark with 1,000 Everyday Activities and Realistic Simulation", "cat": "sim", "org": "academia", "one": "A thousand everyday household activities chosen by surveying what people actually want robots to do.", "device": "simulation (OmniGibson)", "modal": "RGB-D, segmentation, physics state", "hours": "—", "eps": "1,000 activities", "tasks": "1,000 activities", "scenes": "50 scenes", "subj": "9,000+ objects", "embod": "multiple (sim)", "note": "Includes fluids, soft bodies and cloth, which almost no other benchmark simulates at all.", "arxiv": "2403.09227", "authors": [ "Chengshu Li", "Ruohan Zhang", "Josiah Wong", "Cem Gokmen", "Sanjana Srivastava", "Roberto Martín-Martín", "Chen Wang", "Gabrael Levine", "Wensi Ai", "Benjamin Martinez", "Hang Yin", "Michael Lingelbach", "Minjune Hwang", "Ayano Hiranaka", "Sujay Garlanka", "Arman Aydin", "Sharon Lee", "Jiankai Sun", "Mona Anvari", "Manasi Sharma", "Dhruva Bansal", "Samuel Hunter", "Kyu-Young Kim", "Alan Lou", "Caleb R Matthews", "Ivan Villa-Renteria", "Jerry Huayang Tang", "Claire Tang", "Fei Xia", "Yunzhu Li", "Silvio Savarese", "Hyowon Gweon", "C. Karen Liu", "Jiajun Wu", "Li Fei-Fei" ], "year": 2024, "date": "2024-03-14", "institutions": [], "paper": "https://arxiv.org/abs/2403.09227", "site": "https://behavior.stanford.edu/", "fig": "figures/ds-behavior1k.webp", "figs": [ "figures/ds-behavior1k-1.webp", "figures/ds-behavior1k-2.webp" ], "hours_num": -1 }, { "id": "humanoidbench", "name": "HumanoidBench", "title": "HumanoidBench: Simulated Humanoid Benchmark for Whole-Body Locomotion and Manipulation", "cat": "sim", "org": "academia", "one": "Whole-body humanoid locomotion and manipulation tasks — and an honest finding that current RL solves the first and fails the second.", "device": "simulation (MuJoCo)", "modal": "state, RGB", "hours": "—", "eps": "—", "tasks": "27 tasks", "scenes": "sim", "subj": "—", "embod": "Unitree H1 (sim)", "note": "", "arxiv": "2403.10506", "authors": [ "Carmelo Sferrazza", "Dun-Ming Huang", "Xingyu Lin", "Youngwoon Lee", "Pieter Abbeel" ], "year": 2024, "date": "2024-03-15", "institutions": [ "UC Berkeley" ], "paper": "https://arxiv.org/abs/2403.10506", "site": "https://humanoid-bench.github.io/", "fig": "figures/ds-humanoidbench.webp", "figs": [ "figures/ds-humanoidbench-1.webp", "figures/ds-humanoidbench-2.webp" ], "hours_num": -1 }, { "id": "maniskill3", "name": "ManiSkill3", "title": "ManiSkill3: GPU Parallelized Robotics Simulation and Rendering for Generalizable Embodied AI", "cat": "sim", "org": "academia", "one": "GPU-parallel physics and rendering together, for vision-based policies that need throughput on both.", "device": "simulation (SAPIEN)", "modal": "RGB-D, point cloud, state", "hours": "—", "eps": "—", "tasks": "12 task families", "scenes": "sim", "subj": "—", "embod": "many (sim)", "note": "", "arxiv": "2410.00425", "authors": [ "Stone Tao", "Fanbo Xiang", "Arth Shukla", "Yuzhe Qin", "Xander Hinrichsen", "et al." ], "year": 2024, "date": "2024-10-01", "institutions": [ "UC San Diego" ], "paper": "https://arxiv.org/abs/2410.00425", "site": "https://maniskill.readthedocs.io/", "fig": "figures/ds-maniskill3.webp", "figs": [ "figures/ds-maniskill3-1.webp", "figures/ds-maniskill3-2.webp" ], "hours_num": -1 }, { "id": "robocasa", "name": "RoboCasa", "title": "RoboCasa: Large-Scale Simulation of Everyday Tasks for Generalist Robots", "cat": "sim", "org": "academia", "one": "Large-scale simulated household kitchens with generative scene and task variation.", "device": "simulation (MuJoCo/robosuite)", "modal": "RGB-D, proprioception, language", "hours": "—", "eps": "100K+ trajectories", "tasks": "100 tasks", "scenes": "120 kitchen scenes", "subj": "2,500+ objects", "embod": "mobile manipulator (sim)", "note": "", "arxiv": "2406.02523", "authors": [ "Soroush Nasiriany", "Abhiram Maddukuri", "Lance Zhang", "Adeet Parikh", "Aaron Lo", "Abhishek Joshi", "Ajay Mandlekar", "Yuke Zhu" ], "year": 2024, "date": "2024-06-04", "institutions": [ "UT Austin", "NVIDIA" ], "paper": "https://arxiv.org/abs/2406.02523", "site": "https://robocasa.ai/", "fig": "figures/ds-robocasa.webp", "figs": [ "figures/ds-robocasa-1.webp", "figures/ds-robocasa-2.webp" ], "hours_num": -1 }, { "id": "libero", "name": "LIBERO", "title": "LIBERO: Benchmarking Knowledge Transfer for Lifelong Robot Learning", "cat": "sim", "org": "academia", "one": "Lifelong-learning benchmark probing knowledge transfer across task suites — the default VLA evaluation.", "device": "simulation (robosuite)", "modal": "RGB, proprioception, language", "hours": "—", "eps": "130 tasks × 50 demos", "tasks": "130 tasks", "scenes": "sim", "subj": "—", "embod": "Franka (sim)", "note": "Everyone reports on it, which makes saturation weak evidence of anything real.", "arxiv": "2306.03310", "authors": [ "Bo Liu", "Yifeng Zhu", "Chongkai Gao", "Yihao Feng", "Qiang Liu", "Yuke Zhu", "Peter Stone" ], "year": 2023, "date": "2023-06-05", "institutions": [ "UT Austin" ], "paper": "https://arxiv.org/abs/2306.03310", "site": "https://libero-project.github.io/", "fig": "figures/ds-libero.webp", "figs": [ "figures/ds-libero-1.webp" ], "hours_num": -1 }, { "id": "calvin", "name": "CALVIN", "title": "CALVIN: A Benchmark for Language-Conditioned Policy Learning for Long-Horizon Robot Manipulation Tasks", "cat": "sim", "org": "academia", "one": "Long-horizon language-conditioned manipulation with a strict evaluation protocol for instruction chaining.", "device": "simulation (PyBullet)", "modal": "RGB-D, proprioception, language", "hours": "—", "eps": "24 h of teleoperated play per environment", "tasks": "34 tasks", "scenes": "4 environments", "subj": "—", "embod": "Franka (sim)", "note": "", "arxiv": "2112.03227", "authors": [ "Oier Mees", "Lukas Hermann", "Erick Rosete-Beas", "Wolfram Burgard" ], "year": 2021, "date": "2021-12-06", "institutions": [], "paper": "https://arxiv.org/abs/2112.03227", "site": "http://calvin.cs.uni-freiburg.de/", "fig": "figures/ds-calvin.webp", "figs": [ "figures/ds-calvin-1.webp" ], "hours_num": -1 }, { "id": "metaworld", "name": "Meta-World", "title": "Meta-World: A Benchmark and Evaluation for Multi-Task and Meta Reinforcement Learning", "cat": "sim", "org": "academia", "one": "Fifty distinct simulated manipulation tasks for multi-task and meta-RL — the standard generalisation suite.", "device": "simulation (MuJoCo)", "modal": "state, RGB", "hours": "—", "eps": "—", "tasks": "50 tasks", "scenes": "sim tabletop", "subj": "—", "embod": "Sawyer (sim)", "note": "", "arxiv": "1910.10897", "authors": [ "Tianhe Yu", "Deirdre Quillen", "Zhanpeng He", "Ryan Julian", "Avnish Narayan", "Hayden Shively", "Adithya Bellathur", "Karol Hausman", "Chelsea Finn", "Sergey Levine" ], "year": 2019, "date": "2019-10-24", "institutions": [], "paper": "https://arxiv.org/abs/1910.10897", "site": "https://meta-world.github.io/", "fig": "figures/ds-metaworld.webp", "figs": [ "figures/ds-metaworld-1.webp" ], "hours_num": -1 } ] }