{"meta": {"generated": "2026-10-05", "counts": {"total": 613, "curated": 426, "library": 197, "attached": 63, "dropped": 35}, "trees": {"rl": {"name": "Reinforcement Learning", "blurb": "Policies learned by interaction and reward — from algorithmic foundations through legged locomotion, dexterous manipulation, multi-agent coordination, and RL as a post-training stage on top of foundation models.", "branches": {"foundations": ["Algorithmic foundations", "Policy-gradient, actor-critic, off-policy and model-based cores that everything below is built on."], "locomotion": ["Locomotion", "Legged and humanoid locomotion: blind and perceptive, flat to parkour, sim-to-real throughout."], "motion-imitation": ["Motion imitation & characters", "Reward-shaped tracking of reference motion — DeepMimic and its descendants, now the standard humanoid whole-body recipe."], "manipulation": ["Dexterous manipulation", "In-hand reorientation, vision-based dexterity, contact-rich and force-aware RL, grasping in clutter."], "loco-manip": ["Whole-body loco-manipulation", "Arms and legs on one policy — force-adaptive carrying, mobile pick-and-place, door opening."], "marl": ["Multi-agent RL", "Cooperative transport, decentralized coordination, opponent modelling, emergent team behaviour."], "post-training": ["RL post-training on foundation models", "Turning a pretrained VLA or generalist policy into a reliable one with on-robot or offline RL."], "model-based": ["Model-based & world-model RL", "Learning a dynamics model and planning or dreaming inside it."], "sim2real": ["Sim-to-real & real-to-sim", "Domain randomization, adaptation, system identification, real2sim2real loops."]}}, "il": {"name": "Imitation Learning", "blurb": "Policies learned from demonstration — behaviour cloning and its modern descendants: action-chunking transformers, diffusion and flow policies, vision-language-action models, world/world-action models, and the growing effort to learn from human video instead of robot data.", "branches": {"foundations": ["Foundations of imitation", "BC, DAgger, inverse RL, adversarial imitation, implicit policies."], "visuomotor": ["Visuomotor policy architectures", "ACT/CVAE, diffusion policy, flow matching, 3D policies — the action-decoder zoo."], "vla": ["Vision-Language-Action models", "Generalist instruction-following policies built on VLM backbones."], "in-context": ["In-context & few-shot", "Prompting a policy with demonstrations at test time instead of fine-tuning."], "world-models": ["World models & WAM", "Predicting futures — video, latent, or joint world-action — and acting through the prediction."], "human-video": ["Learning from human video", "Egocentric and third-person human demonstrations as a data source for robots."], "retargeting": ["Retargeting & teleoperation", "Mapping human motion onto robot embodiments; the interfaces that collect the data."], "data-scaling": ["Data scaling & cross-embodiment", "Large multi-robot datasets and the policies trained across them."], "tactile-learn": ["Tactile & contact learning", "Policies that use touch, and the representation learning behind them."]}}, "systems": {"name": "Hardware & Systems", "blurb": "The machines and the plumbing — open-source arms and hands, mobile manipulators, data-collection interfaces, tactile skins, humanoid platforms, co-design, and the simulators and benchmarks the field runs on.", "branches": {"bimanual": ["Open-source arms & bimanual", "Low-cost, reproducible manipulator platforms that made the data flywheel possible."], "mobile-manip": ["Mobile manipulation platforms", "Bases + arms: TidyBot lineage, Stretch, whole-body household systems."], "interfaces": ["Data-collection interfaces", "UMI-style handheld grippers, exoskeletons, gloves, VR teleop rigs."], "hands": ["Dexterous hands", "Tendon- and direct-drive anthropomorphic hands, open and commercial."], "tactile-hw": ["Tactile sensing hardware", "Vision-based and magnetic skins, fingertip sensors, full-hand coverage."], "humanoid-hw": ["Humanoid & legged platforms", "The robots themselves — commercial, research, and open-source."], "codesign": ["Co-design & morphology", "Optimizing body and brain together, increasingly with learning in the loop."], "sim-bench": ["Simulators & benchmarks", "MuJoCo, Isaac, Genesis, and the task suites that define progress."], "planning-control": ["Planning & control", "MPC, trajectory optimization, sampling-based planning, collision avoidance — the model-based stack RL is often measured against."], "perception": ["Perception & state estimation", "SLAM, odometry, calibration, pose estimation — the layer everything above assumes works."], "agentic": ["Agentic & auto-research systems", "LLM agents that design, code, run and iterate robotics experiments."]}}}, "projects": {"mabel": {"name": "MABEL", "kind": "current", "order": 1, "tag": "Mobile bimanual humanoid", "blurb": "Wheeled-humanoid platform: bimanual arms on a swerve base, whole-body teleoperation, learned retargeting, and an open hardware stack.", "path": "~/Desktop/MABEL", "color": "#23577E"}, "m2": {"name": "M2", "kind": "current", "order": 2, "tag": "Multi-robot collaboration", "blurb": "Two (and more) mobile manipulators cooperating — collaborative transport, distributed MPC, multi-agent RL, and the planning stack that keeps them from fighting each other.", "path": "~/Desktop/M2", "color": "#2E7D4F"}, "r2s2r": {"name": "Real2Sim2Real", "kind": "current", "order": 3, "tag": "Egocentric capture → sim → policy", "blurb": "Learn from egocentric video: reconstruct the scene as a simulatable asset, pretrain the policy by imitation inside it, then fine-tune with RL and deploy back to the real scene. The ego view is the interface that holds the loop together.", "path": "", "color": "#C6301A"}, "wam-tactile": {"name": "Tactile WAM", "kind": "current", "order": 4, "tag": "World-action model · touch + UMI glove", "blurb": "A world-action model whose observation stream carries contact, trained on UMI-glove capture — predict the future and the action that causes it, with touch in the loop rather than inferred from pixels.", "path": "", "color": "#D9A13F"}, "geodex": {"name": "GeoDex", "kind": "past", "order": 5, "tag": "Geometry-driven dexterity", "blurb": "Dexterous hand work — geometry-aware grasping and in-hand manipulation, rollout studies, and the hardware behind them.", "path": "~/Desktop/dexterous_hand", "color": "#C88A2E"}, "iris": {"name": "IRIS / MPR", "kind": "past", "order": 6, "tag": "Cinema robot arm", "blurb": "6-DOF cinema robot arm with visuomotor imitation learning (ACT/CVAE, diffusion), plus the next-generation Motion Picture Robotics platform on a swerve base.", "path": "~/Documents/GitHub/MEng_project", "color": "#7B5EA8"}, "exo": {"name": "Exoskeleton / Sim-to-real", "kind": "past", "order": 7, "tag": "Wearable robotics", "blurb": "End-to-end assistive torque control, mixture-of-experts exoskeleton policies, and musculoskeletal sim-to-real (ExoGym).", "path": "", "color": "#B0566A"}}}, "papers": [{"id": "being-h07", "title": "Being-H0.7: A Latent World-Action Model from Egocentric Videos", "tree": "il", "branch": "world-models", "authors": ["Hao Luo", "Wanpeng Zhang", "Yicheng Feng", "Sipeng Zheng", "Haiweng Xu", "Chaoyi Xu", "Ziheng Xi", "Yuhui Fu", "Zongqing Lu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-30", "topics": ["latent world-action model", "egocentric video pretraining"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["dream-tac", "dreamdojo", "ego2robot", "egoscale", "motionwam", "vjepa2"], "arxiv": "2605.00078", "doi": "10.48550/arxiv.2605.00078", "code": "", "site": "", "note": "A latent world-action model learned from egocentric human video — the Being-H0 line moved from VLA pretraining to world-action modelling. This is the closest published system to your UMI-glove WAM idea and it works from ordinary human video rather than instrumented capture.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Visual-Language-Action models (VLAs) have advanced generalist robot control by mapping multimodal observations and language instructions directly to actions, but sparse action supervision often encourages shortcut mappings rather than representations of dynamics, contact, and task progress. Recent world-action models introduce future prediction through video rollouts, yet pixel-space prediction is a costly and indirect substrate for control, as it may model visual details irrelevant to action generation and introduces substantial training or inference overhead. We present Being-H0.7, a latent world-action model that brings future-aware reasoning into VLA-style policies without generating future frames. Being-H0.7 inserts learnable latent queries between perception and action as a compact reasoning interface, and trains them with a future-informed dual-branch design: a deployable prior branch infers latent states from the current context, while a training-only posterior branch replaces the queries with embeddings from future observations. Jointly aligning the two branches at the latent reasoning space leads the prior branch to reason future-aware, action-useful structure from current observations alone. At inference, Being-H0.7 discards the posterior branch and performs no visual rollout. Experiments across six simulation benchmarks and diverse real-world tasks show that Being-H0.7 achieves state-of-the-art or comparable performance, combining the predictive benefits of world models with the efficiency and deployability of direct VLA policies.", "source": "curated", "first_author": "Hao Luo", "url": "https://arxiv.org/abs/2605.00078", "fig": "figures/being-h07.webp", "figs": ["figures/being-h07-1.webp"]}, {"id": "c2dex", "title": "C2Dex: Contact-Consistent Reconstruction and Retargeting for Dexterous Manipulation from Monocular Video", "tree": "il", "branch": "retargeting", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-07", "topics": ["contact-consistent retargeting", "hand reconstruction"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "m2", "r2s2r"], "related": ["anyteleop", "dex-retargeting", "dexmv", "dexpilot", "ego2robot", "gmr", "human2humanoid", "retarget-objectives", "x-op"], "arxiv": "2608.07045", "doi": "", "code": "", "site": "", "note": "From your M2 library — and almost exactly your learned-retargeting idea (retarget so that contacts are preserved, not joint angles). Read this before you write that proposal.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.07045v1_C2Dex_Contact_Consistent_Reconstruction_and_Retargeting_for.pdf", "abstract": "High-quality demonstrations for dexterous robot manipulation are costly and difficult to collect, whereas monocular human videos provide a scalable source of diverse manipulation behaviors. However, transferring such demonstrations to dexterous robots remains challenging: monocular hand-object interaction (HOI) reconstruction often produces temporally unstable contacts and physically implausible interactions, while conventional retargeting methods struggle to preserve task-relevant contacts and local interaction geometry across different hand embodiments. We present C2Dex, a video-to-dexterous-manipulation framework built around a shared interaction representation: stable object-side contacts recovered by aggregating noisy frame-wise observations in the canonical object space. These stable contacts serve a dual role: as trajectory-level constraints that guide reconstruction toward temporally coherent and physically plausible human HOI trajectories, and as explicit transfer targets for the dexterous hand, where Laplacian interaction optimization preserves the local hand-object geometry across embodiments and residual reinforcement learning refines the trajectory in simulation. Experiments on DexYCB and TACO show that C2Dex achieves end-to-end trajectory success rates of 57.78% and 26.67%, respectively, substantially outperforming the strongest baselines (17.78% and 10.00%) under identical evaluation criteria. Real-robot replay experiments further demonstrate physical feasibility across diverse contact-rich manipulation tasks. Project page: https://k-jie.github.io/C2Dex/", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2608.07045", "fig": "figures/c2dex.webp"}, {"id": "chorus", "title": "CHORUS: Decentralized Multi-Embodiment Collaboration with One VLA Policy", "tree": "rl", "branch": "marl", "authors": ["Ria Doshi", "Tian Gao", "Annie Chen", "Chelsea Finn", "Jeannette Bohg"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-10", "topics": ["decentralized collaboration", "multi-embodiment", "single policy"], "paradigm": ["VLA", "MARL"], "method": [], "projects": ["m2"], "related": ["climrs", "cross-humanoid-wbc", "crossformer", "decentralized-aerial", "h2compact", "mappo", "roco", "wholebodyvla"], "arxiv": "2606.12352", "doi": "10.48550/arxiv.2606.12352", "code": "", "site": "", "note": "One VLA policy driving decentralized collaboration across different embodiments. This is the most direct answer yet to M2's core question — whether two heterogeneous robots need two policies or one — and it says one. Read it before designing M2's policy architecture.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Multi-robot collaboration allows robots to efficiently take on a wide range of tasks, from moving a couch through a doorway to assembling structures on a construction site. However, achieving such coordination in mobile multi-robot settings remains challenging: centralized methods conditioned on the combined observations of a team scale poorly with team size, and decentralized methods that train one policy per robot often require explicit alignment procedures or information sharing at inference time to overcome partial observability. Our key insight is that the visuomotor priors of pretrained vision-language-action (VLA) models should enable reactive, decentralized collaboration from each robot's local observations alone, without these inference-time assumptions. We propose CHORUS, a framework that adapts a single VLA backbone to control diverse, multi-robot teams. At inference time, each robot runs an independent copy of CHORUS, conditioned only on its own observations and a robot-identifying prompt. In real-world experiments including mobile tape measurement, library book handovers, and laundry basket lifting, CHORUS achieves a 64% point improvement over decentralized, from-scratch models, improves reactivity to teammate behavior by 40% points, and outperforms centralized baselines. Together, these results show that a shared VLA backbone is capable of achieving decentralized multi-robot collaboration, without per-robot policies or inter-robot communication at inference.", "source": "curated", "first_author": "Ria Doshi", "url": "https://arxiv.org/abs/2606.12352", "fig": "figures/chorus.webp", "figs": ["figures/chorus-1.webp", "figures/chorus-2.webp"]}, {"id": "cosmos-policy", "title": "Cosmos Policy: Fine-Tuning Video Models for Visuomotor Control and Planning", "tree": "il", "branch": "world-models", "authors": ["Moo Jin Kim", "Yihuai Gao", "Tsung-Yi Lin", "Yen-Chen Lin", "Yunhao Ge", "Grace Lam", "Percy Liang", "Shuran Song", "Ming-Yu Liu", "Chelsea Finn", "Jinwei Gu"], "corresponding": [], "institutions": ["NVIDIA", "Stanford University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-22", "topics": ["fine-tuning a video foundation model for control", "visuomotor policy and planning"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["cosmos", "dreamdojo", "dreamgen", "dreamzero", "genie-envisioner", "wla-model"], "arxiv": "2601.16163", "doi": "10.48550/arxiv.2601.16163", "code": "", "site": "", "note": "<b>Cosmos Policy</b> (NVIDIA) — the cleanest statement of the reuse argument: take a pretrained video world foundation model and fine-tune it into a visuomotor policy, so the same weights serve control <em>and</em> planning. The practical reason to care is that it tells you what the minimum viable adaptation is — you almost certainly do not need to pretrain a world model to build one.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Recent video generation models demonstrate remarkable ability to capture complex physical interactions and scene evolution over time. To leverage their spatiotemporal priors, robotics works have adapted video models for policy learning but introduce complexity by requiring multiple stages of post-training and new architectural components for action generation. In this work, we introduce Cosmos Policy, a simple approach for adapting a large pretrained video model (Cosmos-Predict2) into an effective robot policy through a single stage of post-training on the robot demonstration data collected on the target platform, with no architectural modifications. Cosmos Policy learns to directly generate robot actions encoded as latent frames within the video model's latent diffusion process, harnessing the model's pretrained priors and core learning algorithm to capture complex action distributions. Additionally, Cosmos Policy generates future state images and values (expected cumulative rewards), which are similarly encoded as latent frames, enabling test-time planning of action trajectories with higher likelihood of success. In our evaluations, Cosmos Policy achieves state-of-the-art performance on the LIBERO and RoboCasa simulation benchmarks (98.5% and 67.1% average success rates, respectively) and the highest average score in challenging real-world bimanual manipulation tasks, outperforming strong diffusion policies trained from scratch, video model-based policies, and state-of-the-art vision-language-action models fine-tuned on the same robot demonstrations. Furthermore, given policy rollout data, Cosmos Policy can learn from experience to refine its world model and value function and leverage model-based planning to achieve even higher success rates in challenging tasks. We release code, models, and training data at https://research.nvidia.com/labs/dir/cosmos-policy/", "source": "curated", "first_author": "Moo Jin Kim", "url": "https://arxiv.org/abs/2601.16163", "fig": "figures/cosmos-policy.webp", "figs": ["figures/cosmos-policy-1.webp"]}, {"id": "dream-tac", "title": "Dream-Tac: A Unified Tactile World Action Model for Contact-Rich Robot Manipulation", "tree": "il", "branch": "world-models", "authors": ["Yunfan Lou", "Yifan Ye", "Yankai Fu", "Jun Cen", "Xiaowei Chi", "Yaoxu Lyu", "Peidong Jia", "Sirui Han", "Zhihe Lu", "Shanghang Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-07", "topics": ["tactile", "world-action model", "contact-rich"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["audio-vla", "being-h07", "deform360", "dexskin", "diamond", "dreamzero", "egoemg", "ev-wm", "lawam", "motionwam", "oa-wam", "omnivta", "tacforesight", "tactile-wam", "vjepa2", "vt-wam"], "arxiv": "2606.08737", "doi": "", "code": "", "site": "", "note": "The paper that occupies your tactile-WAM idea most directly: it jointly models actions, future visual observations AND tactile dynamics, on a Tac-UMI gripper, over five contact-rich tasks (peg insertion, nut threading, gear meshing, bulb insertion, power insertion). Read it first and decide what you would do that it does not — the obvious gaps are multi-finger hands and slip prediction as an explicit metric.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "World action models inherit the predictive capability of world models, enabling action generation to be guided by anticipated future observations. However, they rely primarily on vision and often fail in contact-rich manipulation, where critical cues arise from physical interaction. In this paper, we propose Dream-Tac, a unified Tactile-World Action Model that jointly models actions, future visual observations, and tactile dynamics. Specifically, Dream-Tac introduces (i) contact-gated visuotactile fusion to selectively integrate tactile signals and (ii) a contact-aware attention bias to better regulate cross-modal interactions during manipulation. To support real-time deployment, we further design a dual-level acceleration strategy, reformulating the contact-aware bias to preserve the fused attention path during training and introducing cache-based diffusion acceleration at inference, achieving up to 2.9$\\times$ faster training and 1.8$\\times$ faster inference. Across six contact-rich manipulation tasks, Dream-Tac improves action accuracy by 31.7\\% on average, demonstrating the effectiveness of unified visuotactile world modeling.Code is available at https://github.com/LYFCLOUDFAN/Dream-Tac.", "source": "curated", "first_author": "Yunfan Lou", "url": "https://arxiv.org/abs/2606.08737", "fig": "figures/dream-tac.webp", "figs": ["figures/dream-tac-1.webp"]}, {"id": "dreamdojo", "title": "DreamDojo: A Generalist Robot World Model from Large-Scale Human Videos", "tree": "il", "branch": "world-models", "authors": ["Shenyuan Gao", "William Liang", "Kaiyuan Zheng", "Ayaan Malik", "Seonghyeon Ye", "Sihyun Yu", "Wei-Cheng Tseng", "Yuzhu Dong", "Kaichun Mo", "Chen-Hsuan Lin", "Qianli Ma", "Seungjun Nah", "Loic Magne", "Jiannan Xiang", "Yuqi Xie", "Ruijie Zheng", "Dantong Niu", "You Liang Tan", "K. R. Zentner", "George Kurian", "Suneel Indupuru", "Pooya Jannaty", "Jinwei Gu", "Jun Zhang", "Jitendra Malik", "Pieter Abbeel", "Ming-Yu Liu", "Yuke Zhu", "Joel Jang", "Linxi \"Jim\" Fan"], "corresponding": [], "institutions": ["NVIDIA", "Stanford University", "UC Berkeley", "UC San Diego", "HKUST", "KAIST"], "lab": "NVIDIA GEAR", "venue": "ICML", "venue_type": "conference", "year": 2026, "date": "2026-02-06", "topics": ["44k hours egocentric human video", "latent actions", "real-time distillation", "policy evaluation"], "paradigm": ["world model"], "method": [], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["adaworld", "being-h07", "cosmos-policy", "dreamgen", "dreamzero", "egophys", "genie-envisioner", "geniworld", "himem-wam", "interactive-world-sim", "ivideogpt", "lapa", "lawm", "motionwam", "tactile-wam", "vista-wm"], "arxiv": "2602.06949", "doi": "10.48550/arxiv.2602.06949", "code": "", "site": "", "note": "<b>DreamDojo</b> (NVIDIA, ICML 2026) — the one you asked for, and the most important single entry in this batch. A generalist robot world model pretrained on <b>44,000 hours of egocentric human video</b>, the largest video corpus used for world-model pretraining to date. Two things make it matter to you. First, the action-label problem is solved the way you would want it solved: <b>continuous latent actions as a unified proxy</b>, so unlabelled human video contributes interaction knowledge, and a small amount of target-robot data post-trains it into something controllable. Second, they distil it to <b>10.81 FPS real time</b>, which is what converts a world model from an offline curiosity into something you can teleoperate through, evaluate policies in, and plan with. That triple — live teleoperation, policy evaluation, model-based planning — is very close to the stated scope of your Tactile WAM project, so read this before you freeze that design.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Being able to simulate the outcomes of actions in varied environments will revolutionize the development of generalist agents at scale. However, modeling these world dynamics, especially for dexterous robotics tasks, poses significant challenges due to limited data coverage and scarce action labels. As an endeavor towards this end, we introduce DreamDojo, a foundation world model that learns diverse interactions and dexterous controls from 44k hours of egocentric human videos. Our data mixture represents the largest video dataset to date for world model pretraining, spanning a wide range of daily scenarios with diverse objects and skills. To address the scarcity of action labels, we introduce continuous latent actions as unified proxy actions, enhancing interaction knowledge transfer from unlabeled videos. After post-training on small-scale target robot data, DreamDojo demonstrates a strong understanding of physics and precise action controllability. We also devise a distillation pipeline that accelerates DreamDojo to a real-time speed of 10.81 FPS and further improves context consistency. Our work enables several important applications based on generative world models, including live teleoperation, policy evaluation, and model-based planning. Systematic evaluation on multiple challenging out-of-distribution (OOD) benchmarks verifies the significance of our method for simulating open-world, contact-rich tasks, paving the way for general-purpose robot world models.", "source": "curated", "first_author": "Shenyuan Gao", "url": "https://arxiv.org/abs/2602.06949", "fig": "figures/dreamdojo.webp", "figs": ["figures/dreamdojo-1.webp", "figures/dreamdojo-2.webp"]}, {"id": "ego2robot", "title": "Ego2Robot: Scalable Robot Data Synthesis from Egocentric Human Data", "tree": "il", "branch": "human-video", "authors": ["Ye Wang", "Pei Lin", "Xiong-Hui Chen", "Haoqi Yuan", "Zhixuan Liang", "Yiyang Huang", "Anzhe Chen", "Zixing Lei", "Jie Zhang", "Tao Zhang", "Haoyang Li", "Tong Zhang", "Chenxi Xiao", "Ziyuan Jiao", "Qin Jin"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-03", "topics": ["ego-to-robot synthesis", "action retargeting", "data curation"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["being-h07", "c2dex", "dexumi", "egoscale", "humannet", "intention-priors", "lapa", "lawm-3d", "phantom", "simdex"], "arxiv": "2608.02580", "doi": "10.48550/arxiv.2608.02580", "code": "", "site": "", "note": "The conversion step that makes a million hours of human video usable: retarget the actions, synthesise the robot arm into the pixels, and curate at multiple levels of quality. Produces <b>18,561 hours</b> of robot training data across 15 morphologies — the largest ego-to-robot corpus published. Raw human hours are worthless until something does this; read it as the other half of EgoScale.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Learning generalizable robot manipulation policies requires large-scale and diverse demonstration data. Egocentric human manipulation videos offer rich scene and task diversity, and prior work has shown that retargeting and rendering such videos into robot-format data can yield effective per-task policies at small scale. However, whether this approach can provide pretraining benefits for vision-language-action models at scale remains unexplored. We present \\textbf{Ego2Robot}, a scalable pipeline that converts egocentric human manipulation videos into robot training data through action retargeting, robot-arm visual synthesis, and multi-level quality curation. Ego2Robot supports both curated datasets and in-the-wild videos, producing 18,561 hours of robot training data spanning 15 robot morphologies, making it the largest ego-to-robot dataset to date. To evaluate generalization, we extend RoboTwin2.0 with disentangled perturbation axes covering visual appearance, scene layout, embodiment morphology, and task semantics. Experiments show that joint pretraining on Ego2Robot-synthesized and robot data consistently improves out-of-distribution generalization across multiple perturbation types, with benefits validated on real-robot deployment. Project page: https://www-ye.github.io/ego2robot_blog/", "source": "curated", "first_author": "Ye Wang", "url": "https://arxiv.org/abs/2608.02580", "fig": "figures/ego2robot.webp", "figs": ["figures/ego2robot-1.webp"]}, {"id": "egoengine", "title": "EgoEngine: From Egocentric Human Videos to High-Fidelity Dexterous Robot Demonstrations", "tree": "il", "branch": "human-video", "authors": ["Yangcen Liu", "Shuo Cheng", "Xinchen Yin", "Woo Chul Shin", "Alfred Cueva", "Yiran Yang", "Zhenyang Chen", "Chuye Zhang", "Danfei Xu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-10", "topics": ["egocentric video", "digital twin", "dexterous demonstrations"], "paradigm": ["real2sim", "IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["acdc", "egodemogen", "egomimic", "egophys", "phantom", "polaris", "r2s-ego", "simfoundry", "video2sim2real", "videomimic"], "arxiv": "2606.12604", "doi": "", "code": "", "site": "", "note": "EgoEngine builds a simulation scene from egocentric video that preserves task-relevant geometry and object layout, recovering hand pose and scene entities together, then trains in the reconstructed twin. This is your Real2Sim2Real project's thesis, already executed. The remaining gap is the RL fine-tuning stage and closing the loop back to the real scene.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Dexterous manipulation is limited by the cost of collecting large-scale robot demonstrations. Egocentric human videos offer a scalable source of diverse manipulation behaviors, but directly using them for robot learning requires bridging two gaps: the visual gap between human and robot observations, and the action gap between human motion and robot-executable action. We propose EgoEngine, a scalable framework for transforming egocentric human manipulation videos into high-fidelity robot data. Given an egocentric RGB video, EgoEngine produces: (i) a high-fidelity robot observation video replacing human with robot while preserving scene context and temporal alignment, and (ii) a task-aligned, executable robot action trajectory under feasibility constraints. Experiments in simulation and on real robots show that EgoEngine enables scalable conversion of human videos into robot data and, to our knowledge, demonstrates the first zero-shot visuomotor dexterous policy learning from egocentric human videos without real-robot demonstrations. Project website: https://egoengine.github.io.", "source": "curated", "first_author": "Yangcen Liu", "url": "https://arxiv.org/abs/2606.12604", "fig": "figures/egoengine.webp"}, {"id": "egophys", "title": "EgoPhys: Learning Generalizable Physics Models of Deformable Objects from Egocentric Video", "tree": "il", "branch": "world-models", "authors": ["Hyunjin Kim", "Ri-Zhao Qiu", "Guangqi Jiang", "Xiaolong Wang"], "corresponding": [], "institutions": ["UC San Diego"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-15", "topics": ["egocentric video", "generalizable physics models", "deformable objects"], "paradigm": ["world model"], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": ["adaptigraph", "deform360", "dreamdojo", "egoengine", "phystwin", "physworld", "r2s-ego"], "arxiv": "2606.16202", "doi": "10.48550/arxiv.2606.16202", "code": "", "site": "", "note": "<b>EgoPhys</b> (Xiaolong Wang's lab, UCSD — not RoboPIL, despite the subject matter) — learning generalizable physics models of deformable objects <em>from egocentric video</em>. This is the most direct collision with your own Real2Sim2Real-from-egocentric-video project of anything in the library, and you should read it early: it is either a strong baseline or a partial scoop, and which one it is determines whether your novelty claim needs to move toward the tactile/UMI-glove axis.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humans naturally understand object physics through everyday interactions, but faithfully predicting complex deformable dynamics, such as elastic materials and fabrics, remains a major challenge for computer vision and robotics. We present EgoPhys, a framework that constructs deformable physical digital twins from egocentric RGB-only video using generalizable priors. EgoPhys overcomes the limitations of existing methods to enable controllable deformable digital twin generation from egocentric videos by distilling per-object inverse-physics solutions into a compact codebook, enabling prediction of dense spring stiffness fields for unseen objects without per-spring test-time optimization. Trained with generalizable priors from diverse egocentric interactions, EgoPhys outperforms baselines in reconstruction, future prediction, and zero-shot generalization. To support training and evaluation, we curate an egocentric interaction dataset covering diverse deformable objects, scenes, and manipulation styles. We deploy EgoPhys on a real xArm6 robot, demonstrating that a digital twin initialized from a single egocentric human play video can serve as an internal world representation to aid in deformable-object planning, highlighting egocentric RGB observations as a scalable path toward real-to-sim pipelines.", "source": "curated", "first_author": "Hyunjin Kim", "url": "https://arxiv.org/abs/2606.16202", "fig": "figures/egophys.webp", "figs": ["figures/egophys-1.webp", "figures/egophys-2.webp"]}, {"id": "egoscale", "title": "EgoScale: Scaling Dexterous Manipulation with Diverse Egocentric Human Data", "tree": "il", "branch": "human-video", "authors": ["Ruijie Zheng", "Dantong Niu", "Yuqi Xie", "Jing Wang", "Mengda Xu", "Yunfan Jiang", "Fernando Castañeda", "Fengyuan Hu", "You Liang Tan", "Letian Fu", "Trevor Darrell", "Furong Huang", "Yuke Zhu", "Danfei Xu", "Linxi Fan"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-18", "topics": ["egocentric scaling law", "dexterous manipulation", "action-labelled human video"], "paradigm": ["IL", "VLA", "flow matching"], "method": [], "projects": ["geodex", "r2s2r", "wam-tactile"], "related": ["ace-ego-0", "being-h07", "dreamgen", "ego2robot", "egolive", "egomimic", "hrdt", "humanego", "humannet", "humanoid-policy", "intention-priors", "phantom", "r3m"], "arxiv": "2602.16710", "doi": "10.48550/arxiv.2602.16710", "code": "", "site": "", "note": "The scaling result the human-data thesis needed. NVIDIA GEAR trains a flow-based VLA on <b>20,854 hours</b> of action-labelled egocentric human video — 20× any prior effort — and finds a <b>log-linear scaling law</b> between human data scale and validation loss, with that loss correlating with real-robot performance. Before this, 'more human video helps' was a hope. Now it is a curve you can budget against. Two-stage: pretrain on human video with wrist motion and retargeted hand actions, then a light mid-training stage on aligned human-robot play data.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Human behavior is among the most scalable sources of data for learning physical intelligence, yet how to effectively leverage it for dexterous manipulation remains unclear. While prior work demonstrates human to robot transfer in constrained settings, it is unclear whether large scale human data can support fine grained, high degree of freedom dexterous manipulation. We present EgoScale, a human to dexterous manipulation transfer framework built on large scale egocentric human data. We train a Vision Language Action (VLA) model on over 20,854 hours of action labeled egocentric human video, more than 20 times larger than prior efforts, and uncover a log linear scaling law between human data scale and validation loss. This validation loss strongly correlates with downstream real robot performance, establishing large scale human data as a predictable supervision source. Beyond scale, we introduce a simple two stage transfer recipe: large scale human pretraining followed by lightweight aligned human robot mid training. This enables strong long horizon dexterous manipulation and one shot task adaptation with minimal robot supervision. Our final policy improves average success rate by 54% over a no pretraining baseline using a 22 DoF dexterous robotic hand, and transfers effectively to robots with lower DoF hands, indicating that large scale human motion provides a reusable, embodiment agnostic motor prior.", "source": "curated", "first_author": "Ruijie Zheng", "url": "https://arxiv.org/abs/2602.16710", "fig": "figures/egoscale.webp", "figs": ["figures/egoscale-1.webp"]}, {"id": "far-dex", "title": "FAR-Dex: Few-shot Data Augmentation and Adaptive Residual Policy Refinement for Dexterous Manipulation", "tree": "rl", "branch": "manipulation", "authors": ["Yushan Bai", "Fulin Chen", "Hongzheng Sun", "Yuchuang Tong", "En Li", "Zhengtao Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-11", "topics": ["few-shot augmentation", "adaptive residual policy refinement", "dexterity"], "paradigm": ["IL", "RL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["asap", "humanoid-dex-s2r", "omnitactune", "pld", "residual-assembly", "video2sim2real"], "arxiv": "2603.10451", "doi": "10.48550/arxiv.2603.10451", "code": "", "site": "", "note": "Few-shot data augmentation plus an <b>adaptive residual</b> policy for dexterous manipulation — the 2026 instance of the pattern that keeps recurring in this library: a base policy from imitation, a residual from RL, and the residual is where all the contact lives.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Achieving human-like dexterous manipulation through the collaboration of multi-fingered hands with robotic arms remains a longstanding challenge in robotics, primarily due to the scarcity of high-quality demonstrations and the complexity of high-dimensional action spaces. To address these challenges, we propose FAR-Dex, a hierarchical framework that integrates few-shot data augmentation with adaptive residual refinement to enable robust and precise arm-hand coordination in dexterous tasks. First, FAR-DexGen leverages the IsaacLab simulator to generate diverse and physically constrained trajectories from a few demonstrations, providing a data foundation for policy training. Second, FAR-DexRes introduces an adaptive residual module that refines policies by combining multi-step trajectory segments with observation features, thereby enhancing accuracy and robustness in manipulation scenarios. Experiments in both simulation and real-world demonstrate that FAR-Dex improves data quality by 13.4% and task success rates by 7% over state-of-the-art methods. It further achieves over 80% success in real-world tasks, enabling fine-grained dexterous manipulation with strong positional generalization.", "source": "curated", "first_author": "Yushan Bai", "url": "https://arxiv.org/abs/2603.10451", "fig": "figures/far-dex.webp", "figs": ["figures/far-dex-1.webp", "figures/far-dex-2.webp", "figures/far-dex-3.webp"]}, {"id": "wam-tutorial", "title": "From World Models to World Action Models: A Concise Tutorial for Robotics", "tree": "il", "branch": "world-models", "authors": ["Xiaoxiong Zhang", "Xiong Zeng", "Wei Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-01", "topics": ["tutorial", "world models to world-action models", "formalism"], "paradigm": ["survey"], "method": [], "projects": ["wam-tactile"], "related": ["beyond-vla-wm", "dreamzero", "lawam", "wm-survey-manip", "wm-survey-robot"], "arxiv": "2607.00836", "doi": "10.48550/arxiv.2607.00836", "code": "", "site": "", "note": "<b>From World Models to World Action Models: A Concise Tutorial for Robotics</b> — the short, definitional piece that pins down what the <em>action</em> in world-action model buys you over a plain world model. Read this first of the four surveys; it is the one that will make the rest legible, and it gives you the vocabulary your own WAM paper should use.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Rather than providing an exhaustive survey, this paper presents a concise tutorial on world models and world action models for robotics. After reading the tutorial, readers should have a clear understanding of what constitutes a \"world\", how world models and world action models are defined, and what roles they play within robotic AI systems. The tutorial also develops a unified perspective for comparing representative approaches, such as World Labs' spatial intelligence models, Yann LeCun's JEPA framework, and NVIDIA's Cosmos platform, and clarifies how these models differ in their representations, predictive capabilities, and interaction mechanisms.", "source": "curated", "first_author": "Xiaoxiong Zhang", "url": "https://arxiv.org/abs/2607.00836", "fig": "figures/wam-tutorial.webp", "figs": ["figures/wam-tutorial-1.webp", "figures/wam-tutorial-2.webp", "figures/wam-tutorial-3.webp"]}, {"id": "haic", "title": "HAIC: Humanoid Agile Object Interaction Control via Dynamics-Aware World Model", "tree": "rl", "branch": "loco-manip", "authors": ["Dongting Li", "Xingyu Chen", "Qianyang Wu", "Bo Chen", "Sikai Wu", "Hanyu Wu", "Guoyao Zhang", "Liang Li", "Mingliang Zhou", "Diyun Xiang", "Jianzhu Ma", "Qiang Zhang", "Renjing Xu"], "corresponding": [], "institutions": ["Tsinghua University", "HKUST (Guangzhou)", "HKUST", "ETH Zurich", "Xiaomi Robotics Lab"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-12", "topics": ["dynamics-aware world model", "agile object interaction", "humanoid whole-body"], "paradigm": ["world model", "RL"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["amo", "dreamzero", "hugwbc", "motionwam", "resmimic", "wholebodyvla"], "arxiv": "2602.11758", "doi": "", "code": "", "site": "", "note": "<b>HAIC</b> — humanoid agile object interaction control driven by a <b>dynamics-aware world model</b>. The entry in this batch closest to MABEL: most world-model work is tabletop manipulation, and this one is about a humanoid interacting with objects <em>while moving</em>, where the model has to capture the coupling between whole-body momentum and the object. If a world model is ever going to help MABEL, it will look more like this than like a video WAM.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid robots show promise for complex whole-body tasks in unstructured environments. Although Human-Object Interaction (HOI) has advanced, most methods focus on fully actuated objects rigidly coupled to the robot, ignoring underactuated objects with independent dynamics and non-holonomic constraints. These introduce control challenges from coupling forces and occlusions. We present HAIC, a unified framework for robust interaction across diverse object dynamics without external state estimation. Our key contribution is a dynamics predictor that estimates high-order object states (velocity, acceleration) solely from proprioceptive history. These predictions are projected onto static geometric priors to form a spatially grounded dynamic occupancy map, enabling the policy to infer collision boundaries and contact affordances in blind spots. We use asymmetric fine-tuning, where a world model continuously adapts to the student policy's exploration, ensuring robust state estimation under distribution shifts. Experiments on a humanoid robot show HAIC achieves high success rates in agile tasks (skateboarding, cart pushing/pulling under various loads) by proactively compensating for inertial perturbations, and also masters multi-object long-horizon tasks like carrying a box across varied terrain by predicting the dynamics of multiple objects.", "source": "curated", "first_author": "Dongting Li", "url": "https://arxiv.org/abs/2602.11758", "fig": "figures/haic.webp", "figs": ["figures/haic-1.webp", "figures/haic-2.webp", "figures/haic-3.webp"]}, {"id": "harbor", "title": "HARBOR: A Harness Framework for Agentic Robot Reinforcement Learning", "tree": "systems", "branch": "agentic", "authors": ["Zechu Li", "Yufeng Jin", "Xiaoyang Liu", "Puze Liu", "Vignesh Prasad", "Carlo D'Eramo", "Georgia Chalvatzaki"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-07", "topics": ["RL harness", "agentic workflow automation", "benchmark"], "paradigm": ["LLM", "RL"], "method": [], "projects": ["geodex"], "related": ["agent-laboratory", "embo-team", "eureka", "polaris", "rf-agent", "serl", "voyager"], "arxiv": "2606.08610", "doi": "", "code": "", "site": "", "note": "HARBOR automates the whole simulation-RL workflow across 6 benchmarks and 16 tasks spanning manipulation, locomotion and bimanual dexterous control, with policies that transfer to real robots. This is both your 'RL harness' and your 'agentic auto-research for dexterous manipulation' idea in one paper. Read it immediately — it either kills the idea or tells you exactly which half is still open (the real-world harness, not the simulated one).", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Reinforcement learning (RL) has become a powerful paradigm for robot learning, particularly in sim-to-real settings, but its broader adoption remains limited by the engineering pipeline surrounding the algorithms. Building tasks, shaping rewards, and tuning hyperparameters require substantial expert effort, making RL workflows costly and difficult to scale. We introduce HARBOR, an agentic framework that frames robot RL automation as a harness-engineering problem: given a simulator codebase and a task specification, it automates the workflow from environment setup to policy training in simulation. HARBOR decomposes such high-level objectives into bounded stages executed by specialized agents through standardized commands, persistent artifacts, executable gates, and reusable knowledge, and scales iteration via decentralized parallel trials and experience learning across runs. We evaluate HARBOR across 6 benchmarks and 16 tasks in total, spanning manipulation, locomotion, and bimanual dexterous control. We demonstrate that HARBOR automates the simulation RL workflow end-to-end, designs rewards, tunes algorithms to match or improve over default configurations, and reduces engineering effort at practical token and wall-clock cost; the resulting policies can also be transferred to real robots.", "source": "curated", "first_author": "Zechu Li", "url": "https://arxiv.org/abs/2606.08610", "fig": "figures/harbor.webp", "figs": ["figures/harbor-1.webp"]}, {"id": "human2humanoid", "title": "Human2Humanoid: Physics-Aware Cross-Morphology Motion Retargeting for Humanoid Robots", "tree": "il", "branch": "retargeting", "authors": ["Tianqi Huang", "Feiyang Yuan", "Junchi Gu", "Shurui Fang", "Xiaohu Zhang", "Yu Wang", "Wei Gao", "Shiwu Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-02", "topics": ["physics-aware retargeting", "cross-morphology"], "paradigm": ["teleop"], "method": [], "projects": ["mabel"], "related": ["c2dex", "gmr", "h2o", "retarget-objectives", "x-op"], "arxiv": "2606.03476", "doi": "10.48550/arxiv.2606.03476", "code": "", "site": "", "note": "Physics-aware cross-morphology retargeting: respect dynamics, not just kinematics, when mapping a human onto a robot with different mass distribution. Together with [[gmr]] and [[c2dex]] this is now a real sub-field — and the three of them bound what is left of your learned-retargeting idea.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Retargeting human motion to humanoid robots is critical for teleoperation, imitation learning and human-robot interaction. However, it remains challenging because of substantial morphological discrepancies between humans and robots, including differences in skeletal topology, limb proportions and degrees of freedom, as well as the scarcity of paired motion data. This paper presents Human2Humanoid, an unsupervised motion retargeting framework that transfers human motions to humanoid robot behaviors with high fidelity. To bridge the domain gap under unpaired data, we adopt a CycleGAN-based architecture equipped with a skeleton-aware graph convolutional network to capture topology-dependent motion features. To address cross-domain scale mismatches, we introduce a morphology-invariant end-effector consistency loss that aligns normalized end-effector trajectories to preserve motion semantics across embodiments. To improve physical plausibility and reduce contact artifacts, we impose explicit physics-aware feasibility constraints to encourage reproduction of the contact patterns in the source motion. Experimental results show that the proposed method successfully retargets human motion to the Unitree G1 humanoid robot without paired data, and outperforms existing methods in both downstream controllability and physical feasibility.", "source": "curated", "first_author": "Tianqi Huang", "url": "https://arxiv.org/abs/2606.03476", "fig": "figures/human2humanoid.webp"}, {"id": "humanego", "title": "HumanEgo: Zero-Shot Robot Learning from Minutes of Human Egocentric Videos", "tree": "il", "branch": "human-video", "authors": ["Zhi", "Botao He", "Kelin Yu", "Seungjae Lee", "Ruohan Gao", "Furong Huang", "Yiannis Aloimonos"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-24", "topics": ["few-minute human video", "zero-shot transfer", "sample efficiency"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["demohlm", "egomimic", "egoscale", "one-demo", "phantom"], "arxiv": "2605.24934", "doi": "10.48550/arxiv.2605.24934", "code": "", "site": "", "note": "The sharpest counterpoint to the scaling thesis in this whole library. <b>30 minutes</b> of human egocentric video per task gives 92.5% average success across four real tasks — 75% with only 15 minutes — and beats matched-time robot teleoperation by 41%. If a half-hour is enough, the argument for a million-hour campaign is about generality, not about any single task. Read it before committing to collection.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Human egocentric video captures rich manipulation demonstrations without any robot hardware, yet transferring these skills to robots remains challenging due to the embodiment gap between human and robot in both visual appearance and kinematics. We present HumanEgo, a framework that bridges the embodiment gap by lifting each human demonstration to an entity-level representation of hand-object interaction, and training a flow matching policy with dense auxiliary objectives that amplify supervision from every trajectory. HumanEgo is robot-data-free, hardware-agnostic, data-efficient, and zero-shot human-to-robot transferable. With only 30 minutes of human videos per task, HumanEgo achieves 92.5% average success across four real-world tasks (75% with just 15 minutes), outperforms matched-time robot teleoperation by 41%, and robustly transfers zero-shot across novel robots, cameras, and environments. We release HumanEgo as an easy-to-use, open-source framework for learning robot policies directly from human data: https://github.com/TX-Leo/HumanEgo", "source": "curated", "first_author": "Zhi", "url": "https://arxiv.org/abs/2605.24934", "fig": "figures/humanego.webp", "figs": ["figures/humanego-1.webp"]}, {"id": "hmi-robotfree", "title": "Humanoid Manipulation Interface: Humanoid Whole-Body Manipulation from Robot-Free Demonstrations", "tree": "systems", "branch": "interfaces", "authors": ["Ruiqian Nai", "Boyuan Zheng", "Junming Zhao", "Haodong Zhu", "Sicong Dai", "Zunhao Chen", "Yihang Hu", "Yingdong Hu", "Tong Zhang", "Chuan Wen", "Yang Gao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-06", "topics": ["robot-free demonstrations", "whole-body manipulation"], "paradigm": ["teleop", "IL"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dexumi", "homie", "realdexumi", "ume", "umi", "umi-on-legs"], "arxiv": "2602.06643", "doi": "10.48550/arxiv.2602.06643", "code": "", "site": "", "note": "Humanoid Manipulation Interface — UMI's argument extended to the <b>whole body</b>: collect whole-body manipulation demonstrations with no robot present. This is the single most MABEL-relevant paper in this batch, and it is close enough to your mobile-dexterous-UMI idea that you should read it before writing that proposal.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Current approaches for humanoid whole-body manipulation, primarily relying on teleoperation or visual sim-to-real reinforcement learning, are hindered by hardware logistics and complex reward engineering. Consequently, demonstrated autonomous skills remain limited and are typically restricted to controlled environments. In this paper, we present the Humanoid Manipulation Interface (HuMI), a portable and efficient framework for learning diverse whole-body manipulation tasks across various environments. HuMI enables robot-free data collection by capturing rich whole-body motion using portable hardware. This data drives a hierarchical learning pipeline that translates human motions into dexterous and feasible humanoid skills. Extensive experiments across five whole-body tasks--including kneeling, squatting, tossing, walking, and bimanual manipulation--demonstrate that HuMI achieves a 3x increase in data collection efficiency compared to teleoperation and attains a 70% success rate in unseen environments.", "source": "curated", "first_author": "Ruiqian Nai", "url": "https://arxiv.org/abs/2602.06643", "fig": "figures/hmi-robotfree.webp", "figs": ["figures/hmi-robotfree-1.webp", "figures/hmi-robotfree-2.webp", "figures/hmi-robotfree-3.webp"]}, {"id": "interactive-world-sim", "title": "Interactive World Simulator for Robot Policy Training and Evaluation", "tree": "systems", "branch": "sim-bench", "authors": ["Yixuan Wang", "Rhythm Syed", "Fangyu Wu", "Mengchao Zhang", "Aykut Onol", "Jose Barreiros", "Hooshang Nayyeri", "Tony Dear", "Huan Zhang", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign", "Toyota Research Institute"], "lab": "RoboPIL (Yunzhu Li)", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-09", "topics": ["consistency models for latent dynamics", "15 FPS on one 4090", "10-minute rollouts"], "paradigm": ["world model", "simulator"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["dreamdojo", "lawam", "roboworld", "worldeval"], "arxiv": "2603.08546", "doi": "", "code": "", "site": "", "note": "<b>Interactive World Simulator</b> — builds an interactive world model from a <em>moderate-sized</em> robot interaction dataset using consistency models for both image decoding and latent dynamics, and reports stable interaction for <b>over 10 minutes at 15 FPS on a single RTX 4090</b>. That combination — modest data, one consumer GPU, long stable rollouts — makes this the most directly reproducible entry in this section for a lab-scale project.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Action-conditioned video prediction models (often referred to as world models) have shown strong potential for robotics applications, but existing approaches are often slow and struggle to capture physically consistent interactions over long horizons, limiting their usefulness for scalable robot policy training and evaluation. We present Interactive World Simulator, a framework for building interactive world models from a moderate-sized robot interaction dataset. Our approach leverages consistency models for both image decoding and latent-space dynamics prediction, enabling fast and stable simulation of physical interactions. In our experiments, the learned world models produce interaction-consistent pixel-level predictions and support stable long-horizon interactions for more than 10 minutes at 15 FPS on a single RTX 4090 GPU. Our framework enables scalable demonstration collection solely within the world models to train state-of-the-art imitation policies. Through extensive real-world evaluation across diverse tasks involving rigid objects, deformable objects, object piles, and their interactions, we find that policies trained on world-model-generated data perform comparably to those trained on the same amount of real-world data. Additionally, we evaluate policies both within the world models and in the real world across diverse tasks, and observe a strong correlation between simulated and real-world performance. Together, these results establish the Interactive World Simulator as a stable and physically consistent surrogate for scalable robotic data generation and faithful, reproducible policy evaluation.", "source": "curated", "first_author": "Yixuan Wang", "url": "https://arxiv.org/abs/2603.08546", "fig": "figures/interactive-world-sim.webp", "figs": ["figures/interactive-world-sim-1.webp", "figures/interactive-world-sim-2.webp"]}, {"id": "lawam", "title": "LaWAM: Latent World Action Models for Efficient Dynamics-Aware Robot Policies", "tree": "il", "branch": "world-models", "authors": ["Jialei Chen", "Kai Wang", "Kang Chen", "Shuaihang Chen", "Feng Gao", "Wenhao Tang", "Zhiyuan Li", "Weilin Liu", "Zhuyu Yao", "Boxun Li", "Yuanbo Xu", "Chao Yu"], "corresponding": [], "institutions": ["Tsinghua University", "Peking University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-14", "topics": ["latent visual subgoals instead of reconstructed video", "efficient dynamics-aware policy"], "paradigm": ["world model", "IL"], "method": [], "projects": ["wam-tactile"], "related": ["dream-tac", "dreamzero", "flare", "interactive-world-sim", "lapa", "lawm", "lawm-3d", "vjepa2", "vla-jepa", "wam-tutorial"], "arxiv": "2606.15768", "doi": "10.48550/arxiv.2606.15768", "code": "", "site": "", "note": "<b>LaWAM</b> — the efficiency argument against video WAMs, and a good one: expose predictive dynamics to the policy as <b>compact latent visual subgoals</b> rather than decoding future frames at all. Generating pixels you immediately throw away is the single largest waste in the WAM stack, and this is the paper that says so. For a tactile WAM that must run at contact rates, this architecture is more plausible than a video diffusion backbone.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Vision-Language-Action models (VLAs) leverage large-scale vision-language pretraining for semantic robot control, but often lack explicit foresight into how robot actions change the scene. World-Action Models (WAMs) address this limitation by conditioning policies on predicted futures, yet existing approaches typically rely on computationally expensive video generation with substantial pixel-level redundancy. We present LaWAM, a Latent World Action Model that exposes predictive dynamics to robot policies through compact latent visual subgoals instead of reconstructed future video. At the core of LaWAM is a latent-action-conditioned Latent World Model (LaWM). We obtain LaWM by training a latent action model in the latent space of a pretrained vision foundation model and repurposing its forward decoder to predict future observation features for scene evolution. LaWAM then conditions action generation on these predicted latent visual subgoals to enable dynamics-aware robot control. LaWAM achieves state-of-the-art or competitive success rates (SRs) across LIBERO (98.6% SR), RoboTwin (91.22% SR), and real-world manipulation tasks while retaining low-latency inference. LaWAM runs in 187 ms per action-chunk prediction and achieves up to 24x lower wall-clock latency than pixel-space WAMs.", "source": "curated", "first_author": "Jialei Chen", "url": "https://arxiv.org/abs/2606.15768", "fig": "figures/lawam.webp", "figs": ["figures/lawam-1.webp", "figures/lawam-2.webp"]}, {"id": "mm-hand", "title": "MM-Hand: A 21-DOF Multi-modal Modular Dexterous Robotic Hand with Remote Actuation", "tree": "systems", "branch": "hands", "authors": ["Zhuoheng Li", "Qingquan Lin", "Checheng Yu", "Qiangyu Chen", "Zhiqian Lan", "Lutong Zhang", "Hongyang Li", "Ping Luo"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-19", "topics": ["21-DOF modular hand", "remote actuation", "multimodal sensing"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["aero-hand", "dexskin", "orca-hand", "ruka-v2"], "arxiv": "2604.17245", "doi": "", "code": "", "site": "", "note": "MM-Hand: 21 DOF, remote tendon actuation, modular printed fingers, and joint-angle + tactile + motor + in-palm stereo sensing. Essentially the specification you wrote down, already built. The differentiator left to you is the learning stack and the sensing-for-world-modelling angle, not the DOF count.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "High-DOF dexterous hands require compact actuation, rich sensing, and reliable thermal behavior, but conventional designs often occupy valuable in-hand space, increase end-effector mass, and suffer from heat accumulation near the hand. Remote tendon-driven actuation offers an alternative by relocating motors to the robot base or an external motor hub, thereby freeing the fingers and palm for additional degrees of freedom, sensing modules, and maintainable mechanical structures. This paper presents MM-Hand, a 21-DOF Multimodal Modular dexterous hand based on remote tendon-driven actuation. The hand integrates spring-return tendon-driven fingers, modular 3D-printed finger and palm structures, quick tendon connectors for maintenance, and a multimodal sensing system including joint angle sensors, tactile sensors, motor-side feedback, and in-palm stereo vision. We further analyze tendon-sheath length variation and friction loss to guide the design of the routing, motor hub, and closed-loop joint control. Experiments validate the transmission, output force, sensing, and control capability of the system. The fingertip force reaches 25N under a 1m remote sheath transmission, demonstrating practical load capacity despite long-distance tendon routing. Closed-loop joint-level experiments further evaluate command tracking with a static arm and during arm motion. These results show that MM-Hand provides a lightweight, sensor-rich, and maintainable hardware platform for dexterous manipulation research. To support the community, all hardware designs and software frameworks are made fully open-source at https://mmlab.hk/research/MM-Hand.", "source": "curated", "first_author": "Zhuoheng Li", "url": "https://arxiv.org/abs/2604.17245", "fig": "figures/mm-hand.webp", "figs": ["figures/mm-hand-1.webp", "figures/mm-hand-2.webp", "figures/mm-hand-3.webp"]}, {"id": "motionwam", "title": "MotionWAM: Towards Foundation World Action Models for Real-Time Humanoid Loco-Manipulation", "tree": "il", "branch": "world-models", "authors": ["Jia Zheng (140297)", "Teli Ma", "Yudong Fan", "Zifan Wang", "Shuo Yang", "Junwei Liang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-08", "topics": ["foundation world-action model", "real-time loco-manipulation"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["being-h07", "decowam", "dream-tac", "dreamdojo", "haic", "mobilewam", "vjepa2", "wholebodyvla"], "arxiv": "2606.09215", "doi": "10.48550/arxiv.2606.09215", "code": "", "site": "", "note": "A world-action model aimed at <b>real-time</b> humanoid loco-manipulation — the constraint most WAM work ducks, since a world model that needs 200 ms per step cannot close a balance loop. Read with [[decowam]] and [[mobilewam]].", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "World Action Models (WAMs) couple a video dynamics prior to the policy and have shown encouraging results on tabletop manipulation, but iterative denoising over high-dimensional video-action latents leaves them too slow for real-time humanoid loco-manipulation. The problem is compounded by the dominant hierarchical paradigm, in which a high-level manipulation policy controls only the upper body while a low-level controller tracks coarse base commands -- placing upper and lower body in inconsistent action spaces and reducing the legs to balance-preserving locomotion. We present MotionWAM, a real-time WAM that drives autonomous humanoid loco-manipulation from a single egocentric camera by conditioning the policy on the intermediate denoising features of a video world model. MotionWAM replaces the upper-lower split with a unified motion latent and predicts whole-body motion tokens that jointly cover locomotion, torso motion, height regulation, foot interaction, and hand manipulation in a single action space. A three-stage learning framework progressively adapts the video world model to egocentric visual dynamics and to the target humanoid embodiment. On nine real-world Unitree G1 tasks, MotionWAM runs in real time, substantially outperforms Vision-Language-Action (VLA) baselines fine-tuned on the same demonstrations by over 30% in overall success rate, and executes task-driven foot interaction that decoupled upper-lower policies cannot reach. Our results suggest that video-pretrained WAMs can be lifted from tabletop manipulation to coordinated, human-like whole-body humanoid control.", "source": "curated", "first_author": "Jia Zheng (140297)", "url": "https://arxiv.org/abs/2606.09215", "fig": "figures/motionwam.webp", "figs": ["figures/motionwam-1.webp"]}, {"id": "spine-phase", "title": "Phase Relationship between Spinal Motion and Limb Support Determines High-speed Running Performance in a Cheetah Model with Asymmetric Spinal Stiffness", "tree": "rl", "branch": "locomotion", "authors": ["Tomoya Kamimura", "Yuya Oshita", "Mau Adachi", "Yuichi Ambe", "Akihito Sano", "Naomi Wada", "Fumitoshi Matsuno", "Shinya Aoi"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-01", "topics": ["spinal motion phase", "asymmetric stiffness", "high-speed running"], "paradigm": ["control"], "method": [], "projects": [], "related": ["coros-codesign", "mit-cheetah", "s-cheetah", "twisting-waist"], "arxiv": "2604.00329", "doi": "10.48550/arxiv.2604.00329", "code": "", "site": "", "note": "The mechanism paper: it is the <b>phase relationship</b> between spinal motion and limb support — not spinal range of motion — that determines high-speed running performance, with asymmetric spinal stiffness. This is the variable a co-design search should actually be optimizing, and it is the kind of insight that tells you what to parameterise.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Cheetahs are characterized by large spinal flexion and extension during high-speed running, yet the dynamical role of the phase relationship between spinal motion and limb support remains unclear. We aimed to clarify how this phase relationship affects running performance, focusing on the effect of asymmetric spinal stiffness. Using a simple planar cheetah model with asymmetric torsional spinal stiffness, we numerically searched for periodic bounding solutions over a range of stiffness parameters and compared their ground reaction forces, horizontal velocities, and stability. We obtained both cheetah-like solutions, in which the spine extends after hindlimb liftoff and flexes after forelimb liftoff, and non-cheetah-like solutions, in which the spine flexes after hindlimb liftoff and extends after forelimb liftoff. Under asymmetric spinal stiffness, cheetah-like solutions reduced ground reaction forces while maintaining horizontal velocity more effectively than non-cheetah-like solutions. The phase relationship between spinal motion and stance timing is a key determinant of high-speed running performance. These findings provide a dynamical understanding of cheetah locomotion and suggest design principles for spined legged robots.", "source": "curated", "first_author": "Tomoya Kamimura", "url": "https://arxiv.org/abs/2604.00329", "fig": "figures/spine-phase.webp"}, {"id": "realdexumi", "title": "RealDexUMI: A Wearable Universal Manipulation Interface for Dexterous Robot Learning", "tree": "systems", "branch": "interfaces", "authors": ["Chaoyi Xu", "Yixuan Jiang", "Jiahui Huan", "Yuhui Fu", "Haoyu Zhou", "Weitian Yuan", "Jiayi Yu", "Wanpeng Zhang", "Haoqi Yuan", "Zongqing Lu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-04", "topics": ["wearable UMI", "dexterous capture", "in-the-wild data"], "paradigm": ["hardware", "IL"], "method": [], "projects": ["geodex", "mabel", "r2s2r", "wam-tactile"], "related": ["dexcap", "dexumi", "hmi-robotfree", "ume", "umi", "umi-on-legs", "wt-umi"], "arxiv": "2606.06033", "doi": "", "code": "", "site": "", "note": "RealDexUMI — a wearable universal manipulation interface built for dexterous hands. This is the 'custom dexterous UMI' idea as a published system. Combined with DexUMI and UME it means the interface itself is no longer the contribution; what you do with the data is.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Learning dexterous manipulation requires demonstrations that preserve fine hand-object interactions while remaining executable at deployment. Existing pipelines either lose deployable dexterity through retargeting or embodiment conversion, or rely on robot-specific teleoperation that is costly to scale and often lacks intuitive, contact-aware control for dexterous data collection. We present RealDexUMI, a wearable universal manipulation interface built around a shared dexterous end-effector module that integrates a lightweight dexterous hand, in-hand vision, and fingertip tactile sensing. A palm-side isomorphic teleoperation glove maps human finger inputs to robot-hand joint commands, enabling real-time, retargeting-free, intuitive, and precise hand control. The shared hand and sensing modules yield zero-gap end-effector data, with matched in-hand observations, tactile signals, contacts, and hand actions between collection and deployment. Across eight real-robot tasks spanning fine-grained, contact-rich, long-horizon, and bimanual manipulation, policies trained on RealDexUMI data achieve an average success rate of 88.75%, generalize to unseen initial poses, and transfer across three embodiments. Website: https://research.beingbeyond.com/realdexumi", "source": "curated", "first_author": "Chaoyi Xu", "url": "https://arxiv.org/abs/2606.06033", "fig": "figures/realdexumi.webp", "figs": ["figures/realdexumi-1.webp", "figures/realdexumi-2.webp"]}, {"id": "roboworld", "title": "RoboWorld: Fast and Reliable Neural Simulators for Generalist Robot Policy Evaluation", "tree": "systems", "branch": "sim-bench", "authors": ["Byeongguk Jeon", "Seonghyeon Ye", "JaeHyeok Doo", "Sungdong Kim", "Minjoon Seo", "Hyungmok Son", "Kimin Lee"], "corresponding": [], "institutions": ["KAIST"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-01", "topics": ["neural simulator trained on DROID", "RoboArena replication", "100 H100-hours"], "paradigm": ["world model", "benchmark"], "method": [], "projects": ["m2", "r2s2r"], "related": ["droid", "genie-envisioner", "interactive-world-sim", "robogaze", "worldarena2", "worldeval"], "arxiv": "2607.01060", "doi": "10.48550/arxiv.2607.01060", "code": "", "site": "", "note": "<b>RoboWorld</b> — a video world model trained on DROID, used as a fast neural simulator for generalist policy evaluation; replicating the RoboArena benchmark inside it costs <b>100 H100 hours</b> instead of months of real evaluation. The number is the point: it makes the cost argument for neural simulation concrete and checkable.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Video world models are emerging as a scalable alternative for evaluating generalist robot policies, bypassing the physical constraints and engineering burdens of real-world deployment. However, evaluating policies with video world models remains challenging, as world-model errors can make generated rollouts unreliable and slow inference limits large-scale throughput. We introduce RoboWorld, an automated evaluation pipeline that pairs a fast autoregressive video world model with a task-progress-aware vision-language model scoring. To enable reliable long-horizon autoregressive world-model rollouts, we propose Step Forcing, which combines anchored and one-step self-forwarded contexts to reduce train-test mismatch while preserving action-observation dynamics. Together, these components enable RoboWorld to align strongly with real-world robot evaluation across tasks and environments, achieving Pearson's r = 0.989 and Spearman's $ρ$ = 0.970.", "source": "curated", "first_author": "Byeongguk Jeon", "url": "https://arxiv.org/abs/2607.01060", "fig": "figures/roboworld.webp", "figs": ["figures/roboworld-1.webp", "figures/roboworld-2.webp"]}, {"id": "human-video-survey", "title": "Robot Learning from Human Videos: A Survey", "tree": "il", "branch": "human-video", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Intrinsic", "Generalist AI"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-27", "topics": ["survey", "human video", "robot learning"], "paradigm": ["survey"], "method": [], "projects": ["m2", "r2s2r"], "related": ["do-as-i-do", "ego4d", "egomimic", "manip-survey", "open-aoe", "phantom", "simdex", "vid2robot"], "arxiv": "2604.27621", "doi": "", "code": "", "site": "", "note": "In your M2 literature. Start your egocentric-video literature review here — it is the most recent map of the area and will give you the citation skeleton for free.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2604.27621v1_Robot_Learning_from_Human_Videos_A_Survey.pdf", "abstract": "A critical bottleneck hindering further advancement in embodied AI and robotics is the challenge of scaling robot data. To address this, the field of learning robot manipulation skills from human video data has attracted rapidly growing attention in recent years, driven by the abundance of human activity videos and advances in computer vision. This line of research promises to enable robots to acquire skills passively from the vast and readily available resource of human demonstrations, substantially favoring scalable learning for generalist robotic systems. Therefore, we present this survey to provide a comprehensive and up-to-date review of human-video-based learning techniques in robotics, focusing on both human-robot skill transfer and data foundations. We first review the policy learning foundations in robotics, and then describe the fundamental interfaces to incorporate human videos. Subsequently, we introduce a hierarchical taxonomy of transferring human videos to robot skills, covering task-, observation-, and action-oriented pathways, along with a cross-family analysis of their couplings with different data configurations and learning paradigms. In addition, we investigate the data foundations including widely-used human video datasets and video generation schemes, and provide large-scale statistical trends in dataset development and utilization. Ultimately, we emphasize the challenges and limitations intrinsic to this field, and delineate potential avenues for future research. The paper list of our survey is available at https://github.com/IRMVLab/awesome-robot-learning-from-human-videos.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2604.27621", "fig": "figures/human-video-survey.webp", "figs": ["figures/human-video-survey-1.webp", "figures/human-video-survey-2.webp", "figures/human-video-survey-3.webp"]}, {"id": "ruka-v2", "title": "Ruka-v2: Tendon Driven Open-Source Dexterous Hand with Wrist and Abduction for Robot Learning", "tree": "systems", "branch": "hands", "authors": ["Xinqi Lucas Liu", "Ruoxi Hu", "Alejandro Ojeda Olarte", "Zhuoran Chen", "Kenny Ma", "Charles Cheng Ji", "Lerrel Pinto", "Raunaq Bhirangi", "Irmak Guzey"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-27", "topics": ["tendon-driven hand", "wrist and abduction", "open source"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["aero-hand", "faive-ball", "leap-hand", "mm-hand", "orca-hand", "ruka"], "arxiv": "2603.26660", "doi": "", "code": "", "site": "", "note": "RUKA-v2 — the direct successor to the RUKA hand in your library, now with a decoupled 2-DOF parallel wrist and finger abduction, fully open-sourced, demonstrated on 13 teleoperated dexterous tasks. This is the current state of the art for your open 22-DOF hand idea and the thing your design has to improve on. NYU lineage, so the people are reachable.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Lack of accessible and dexterous robot hardware has been a significant bottleneck to achieving human-level dexterity in robots. Last year, we released Ruka, a fully open-sourced, tendon-driven humanoid hand with 11 degrees of freedom - 2 per finger and 3 at the thumb - buildable for under $1,300. It was one of the first fully open-sourced humanoid hands, and introduced a novel data-driven approach to finger control that captures tendon dynamics within the control system. Despite these contributions, Ruka lacked two degrees of freedom essential for closely imitating human behavior: wrist mobility and finger adduction/abduction. In this paper, we introduce Ruka-v2: a fully open-sourced, tendon-driven humanoid hand featuring a decoupled 2-DOF parallel wrist and abduction/adduction at the fingers. The parallel wrist adds smooth, independent flexion/extension and radial/ulnar deviation, enabling manipulation in confined environments such as cabinets. Abduction enables motions such as grasping thin objects, in-hand rotation, and calligraphy. We present the design of Ruka-v2 and evaluate it against Ruka through user studies on teleoperated tasks, finding a 51.3% reduction in completion time and a 21.2% increase in success rate. We further demonstrate its full range of applications for robot learning: bimanual and single-arm teleoperation across 13 dexterous tasks, and autonomous policy learning on 3 tasks. All 3D print files, assembly instructions, controller software, and videos are available at https://ruka-hand-v2.github.io/ .", "source": "curated", "first_author": "Xinqi Lucas Liu", "url": "https://arxiv.org/abs/2603.26660", "fig": "figures/ruka-v2.webp", "figs": ["figures/ruka-v2-1.webp", "figures/ruka-v2-2.webp"]}, {"id": "s-cheetah", "title": "S-Cheetah: A Novel Quadrupedal Robot with a 3-DOF Active Spine Learning Agile Locomotion", "tree": "systems", "branch": "codesign", "authors": ["Zimu Li", "Weibang Bai"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-27", "topics": ["3-DOF active spine", "rotary gallop", "RL locomotion"], "paradigm": ["hardware", "RL"], "method": [], "projects": [], "related": ["dogmo", "dyret", "mit-cheetah", "spine-phase", "transform2act", "twisting-waist"], "arxiv": "2605.27909", "doi": "10.48550/arxiv.2605.27909", "code": "", "site": "", "note": "A quadruped with a bio-inspired serial <b>3-DOF active spine</b> giving tri-axial rotation, trained with RL, reporting 6.9 m/s on a rotary G2 gallop, 7.2 rad/s in-place turning, and emergent feline aerial self-righting. <b>Important caveat: it is simulation-only.</b> The paper states it “currently focuses on the design and simulation phases prior to hardware deployment”; both headline numbers were measured in Isaac Sim and then checked in MuJoCo, and the 20 kg / 625 mm / 33.5 N·m leg / 50 N·m spine figures are design parameters rather than measurements. So it establishes the <em>hypothesis</em> that a 3-DOF spine helps, at simulated scale, and leaves the hardware, the sim-to-real gap, and the choice of spine structure entirely open.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The biological spine of quadrupeds enables sagittal flexion/extension, lateral bending, and axial rotation, playing a crucial role in highly agile and dexterous locomotion. While numerous studies have integrated active spinal joints into quadrupedal robots to enhance agility, most designs simplify control complexity by reducing spinal degrees of freedom (DOF), failing to achieve the spatial tri-axial rotation characteristic of biological spines. Consequently, replicating a multi-DOF biomimetic spine and effectively leveraging it to empower the agile locomotion of quadrupedal robots remains a significant research challenge. In this study, we present S-Cheetah, a quadrupedal robot featuring a 3-DOF bio-inspired serial active spine capable of biomimetic spatial tri-axial rotation. To empower the robot to fully utilize this active spine, we developed a specialized reinforcement learning framework to actively promote the engagement of the introduced spine and maximize the robot's locomotive capabilities by integrating an acceleration curriculum learning strategy with tailored reward functions, such as a gallop gait reward, a spine undulation reward, and a spine steering reward. Experimental results demonstrate that S-Cheetah can achieve a peak speed of 6.9 m/s using the rotary G2 gallop gait and an in-place turning rate of 7.2 rad/s. Besides, the system exhibits an emergent, feline-inspired aerial self-righting capability, allowing it to land stably on four feet from arbitrary orientations during free fall. Finally, through extensive evaluations across diverse locomotion tasks, we prove that the introduction of the proposed 3-DOF spine comprehensively enhances the locomotive agility of quadrupedal robots. Project website: himmy-robotics.github.io/scheetah", "source": "curated", "first_author": "Zimu Li", "url": "https://arxiv.org/abs/2605.27909", "fig": "figures/s-cheetah.webp", "figs": ["figures/s-cheetah-1.webp", "figures/s-cheetah-2.webp", "figures/s-cheetah-3.webp"]}, {"id": "simfoundry", "title": "SimFoundry: Modular and Automated Scene Generation for Policy Learning and Evaluation", "tree": "systems", "branch": "sim-bench", "authors": ["Nadun Ranawaka", "Josiah Wong", "Wei-Lin Pai", "Wei-Teng Chu", "Tianyuan Dai", "Masoud Moghani", "Hang Yin", "Yi Jiang", "Wesley Durbano", "Brandon Huynh", "Y D Fang", "Danfei Xu", "Ruohan Zhang", "Li Fei-Fei", "Linxi Fan", "Bowen Wen", "Ajay Mandlekar", "Yuke Zhu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-26", "topics": ["automated scene generation from video", "sim-ready twins", "editing"], "paradigm": ["simulator"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "egoengine", "prism", "r2s-ego", "robodream", "robosnap"], "arxiv": "2606.28276", "doi": "10.48550/arxiv.2606.28276", "code": "", "site": "", "note": "Zero-shot real-to-sim from a video, producing sim-ready digital twins with object, scene and task editing, and automated generation of cousins. This is the closest thing to the 'agent constructs the scene' component you want — it is a pipeline rather than an LLM loop, but it is the substrate an agent would drive.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Training and evaluating robot policies in the real world is costly and difficult to scale. We introduce SimFoundry, a modular and automated system for zero-shot real-to-sim scene construction from a video. SimFoundry generates sim-ready digital twins and supports object, scene, and task editing, enabling the automated generation of diverse digital cousins: affordance-preserving variations of reconstructed real-world scenes. Policies trained on SimFoundry data transfer zero-shot to challenging real tasks involving multi-step manipulation, articulated object interaction, and bimanual interaction, and its digital cousins (variations of the original scene, objects, and tasks) facilitate generalization to new real-world conditions. Across 7 manipulation tasks and 5 policy architectures, SimFoundry simulation evaluations strongly predict real-world performance, with mean Pearson correlation 0.911 and mean maximum ranking violation 0.018. When evaluating sim-trained policies zero-shot in the real world, policies trained with object, scene, and task cousins in simulation show average task success rate improvements of 17%, 21%, and 40%, respectively. Additional details at https://research.nvidia.com/labs/gear/simfoundry/ .", "source": "curated", "first_author": "Nadun Ranawaka", "url": "https://arxiv.org/abs/2606.28276", "fig": "figures/simfoundry.webp", "figs": ["figures/simfoundry-1.webp"]}, {"id": "ume", "title": "Universal Manipulation Exoskeleton: Learning Compliant Whole-body Policies with Real-time Torque Feedback", "tree": "systems", "branch": "interfaces", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-14", "topics": ["exoskeleton capture", "compliant whole-body manipulation"], "paradigm": ["hardware", "IL"], "method": [], "projects": ["exo", "m2", "mabel", "wam-tactile"], "related": ["airexo", "dexumi", "exo-viha", "fast-umi", "hmi-robotfree", "homie", "realdexumi", "umi", "wheeled-posforce", "wt-umi"], "arxiv": "2606.14218", "doi": "", "code": "", "site": "", "note": "You named this one — it is in your M2 humanoid_platforms and mobile_bimanual_platforms folders. Exoskeleton-based whole-body capture with compliance, i.e. the force half that UMI leaves out.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2606.14218v1_Universal_Manipulation_Exoskeleton_Learning_Compliant_Whole.pdf", "abstract": "For robots to work safely in household environments, they need to be compliant and react to torque and force feedback during contact. However, the majority of existing data collection pipelines still lack the ability to capture force and torque data for learning active compliant policies. In this paper, we present Universal Manipulation Exoskeleton (UME), an upper-limb exoskeleton that provides real-time haptic torque feedback while recording whole-arm configurations and joint torque signals for teleoperation. With transparent torque feedback, human operators can even unsheathe kinematically constrained objects while blindfolded. UME is low-cost, lightweight, and portable. Equipped with an embedded IMU, it enables teleoperation for mobile manipulation. With our proposed universal retargeting algorithm, UME can teleoperate a range of robots, including the 7DoF OpenArm, 7DoF Franka, and 6DoF X-ARM. We demonstrate that this combination of capabilities enables learning bimanual, whole-body, and active compliant policies that operate effectively in highly constrained spaces. The learned robust autonomous policies achieve high success rates across a variety of tasks, including long-horizon mobile manipulation, force-mediated box flipping, visually occluded box pushing, and space-constrained tabletop manipulation. Videos, code, and additional information can be found at https://ume-exo.github.io.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2606.14218", "fig": "figures/ume.webp", "figs": ["figures/ume-1.webp", "figures/ume-2.webp"]}, {"id": "video2sim2real", "title": "Video2Sim2Real: Full-Stack Autonomous Dexterous Skill Acquisition from a Single Human Video", "tree": "rl", "branch": "sim2real", "authors": ["Yunhai Han", "Jianuo Qiu", "Linhao Bai", "Ziyu Xiao", "Zihang Zeng", "Yangcen Liu", "Zhaodong Yang", "Shalin Jain", "Wenrui Ma", "Jiaqi Fu", "Yuqian Zheng", "Manisha Natarajan", "Muhammad Zubair Irshad", "Kenneth Shaw", "Matthew Gombolay", "Zsolt Kira", "Harish Ravichandar"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-07", "topics": ["single human video", "dexterous skill acquisition", "full stack"], "paradigm": ["real2sim", "RL", "IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["asap", "dist-r2s2r", "egoengine", "far-dex", "hil-serl", "humanoid-dex-s2r", "r2s-ego", "real2sim", "residual-assembly", "videomimic"], "arxiv": "2606.08828", "doi": "", "code": "", "site": "", "note": "Video2Sim2Real: one human video in, a dexterous skill out, with imitation producing the transferable base trajectory and residual RL handling the contact discrepancies. That IL-then-residual-RL split is exactly the architecture your project proposes — read it before writing the proposal.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Human manipulation videos are a convenient and intuitive source for robot learning. However, directly transferring human dexterity to robots remains challenging due to perception errors and embodiment gap. To address this, we introduce Video2Sim2Real, a full-stack framework for autonomous skill acquisition from a single human manipulation video. Our framework first uses off-the-shelf foundation models to reconstruct a simulator-ready digital twin and extract robot and object motion priors. Rather than treating the extracted robot motion as a reliable reference throughout execution, our key idea is to recover and leverage the most fundamental sources of supervision from the demonstrated skill: We identify object-centric keyframes to optimize the corresponding robot configurations using object information from the simulator, and use these configurations as anchors that refine the robot motion such that it ultimately has the desired impact on the environment. To bridge the remaining sim-to-real gap, we introduce a sim-to-real strategy that decouples robustness to noisy and incomplete perception from variations in hand-object interaction dynamics. Specifically, we learn to recalibrate robot configurations from noisy real-world point clouds via IL, and leverage residual RL to perform local finger-level adaptations to ensure for robust and effective interactions. Finally, a collision-aware motion planning module enables spatial generalization to novel object configurations. Across several everyday manipulation tasks, Video2Sim2Real improves simulated task success, safety, and trajectory coherence over numerous baselines, and achieves better sim-to-real transfer than existing techniques. These results demonstrate a promising path toward autonomous dexterous skill acquisition from human videos.", "source": "curated", "first_author": "Yunhai Han", "url": "https://arxiv.org/abs/2606.08828", "fig": "figures/video2sim2real.webp", "figs": ["figures/video2sim2real-1.webp"]}, {"id": "dreamzero", "title": "World Action Models are Zero-shot Policies", "tree": "il", "branch": "world-models", "authors": ["Seonghyeon Ye", "Yunhao Ge", "Kaiyuan Zheng", "Shenyuan Gao", "Sihyun Yu", "George Kurian", "Suneel Indupuru", "You Liang Tan", "Chuning Zhu", "Jiannan Xiang", "Ayaan Malik", "Kyungmin Lee", "William Liang", "Nadun Ranawaka", "Jiasheng Gu", "Yinzhen Xu", "Guanzhi Wang", "Fengyuan Hu", "Avnish Narayan", "Johan Bjorck", "Jing Wang", "Gwanghyun Kim", "Dantong Niu", "Ruijie Zheng", "Yuqi Xie", "Jimmy Wu", "Qi Wang", "Ryan Julian", "Danfei Xu", "Yilun Du", "Yevgen Chebotar", "Scott Reed", "Jan Kautz", "Yuke Zhu", "Linxi \"Jim\" Fan", "Joel Jang"], "corresponding": [], "institutions": ["NVIDIA"], "lab": "NVIDIA GEAR", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-17", "topics": ["14B autoregressive diffusion transformer", "joint video+action prediction", "zero-shot policy"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": ["aim-wam", "cosmos-policy", "dream-tac", "dreamdojo", "geniworld", "groot-n1", "haic", "lawam", "oa-wam", "pi05", "vla-jepa", "wam-tutorial", "wla-model"], "arxiv": "2602.15922", "doi": "10.48550/arxiv.2602.15922", "code": "", "site": "", "note": "<b>DreamZero</b> (NVIDIA) — 'World Action Models are Zero-shot Policies'. A 14B autoregressive diffusion transformer trained with chunk-wise teacher-forced video denoising, jointly decoding <em>future frames and actions</em> from visual context, language and proprioception. The headline claim is the one worth arguing with: <b>&gt;2× better generalization to unseen tasks and environments than GR00T N1.6 and π0.5</b> on real robots, plus few-shot embodiment transfer from 30 minutes of play data. The engineering result is just as useful — <b>38× speedup, 5.7 s → 150 ms</b> — because a WAM that cannot close a control loop is not a policy. If your Tactile WAM is going to claim anything, it has to claim it against this.", "aka": "DreamZero", "stars": 3, "curated": true, "local": "", "abstract": "State-of-the-art Vision-Language-Action (VLA) models excel at semantic generalization but struggle to generalize to unseen physical motions in novel environments. We introduce DreamZero, a World Action Model (WAM) built upon a pretrained video diffusion backbone. Unlike VLAs, WAMs learn physical dynamics by predicting future world states and actions, using video as a dense representation of how the world evolves. By jointly modeling video and action, DreamZero learns diverse skills effectively from heterogeneous robot data without relying on repetitive demonstrations. This results in over 2x improvement in generalization to new tasks and environments compared to state-of-the-art VLAs in real robot experiments. Crucially, through model and system optimizations, we enable a 14B autoregressive video diffusion model to perform real-time closed-loop control at 7Hz. Finally, we demonstrate two forms of cross-embodiment transfer: video-only demonstrations from other robots or humans yield a relative improvement of over 42% on unseen task performance with just 10-20 minutes of data. More surprisingly, DreamZero enables few-shot embodiment adaptation, transferring to a new embodiment with only 30 minutes of play data while retaining zero-shot generalization.", "source": "curated", "first_author": "Seonghyeon Ye", "url": "https://arxiv.org/abs/2602.15922", "fig": "figures/dreamzero.webp", "figs": ["figures/dreamzero-1.webp", "figures/dreamzero-2.webp"], "arxiv_url": "https://arxiv.org/abs/2602.15922"}, {"id": "wm-survey-robot", "title": "World Model for Robot Learning: A Comprehensive Survey", "tree": "il", "branch": "world-models", "authors": ["Bohan Hou", "Gen Li", "Jindou Jia", "Tuo An", "Xinying Guo", "Sicong Leng", "Haoran Geng", "Yanjie Ze", "Tatsuya Harada", "Philip Torr", "Oier Mees", "Marc Pollefeys", "Zhuang Liu", "Jiajun Wu", "Pieter Abbeel", "Jitendra Malik", "Yilun Du", "Jianfei Yang"], "corresponding": [], "institutions": ["Stanford University", "UC Berkeley", "ETH Zurich", "University of Oxford", "Princeton University", "NTU Singapore"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-30", "topics": ["comprehensive survey", "taxonomy of world models for robot learning"], "paradigm": ["survey"], "method": [], "projects": ["wam-tactile"], "related": ["beyond-vla-wm", "embodied-sim-survey", "wam-tutorial", "wm-survey-manip", "worldarena2"], "arxiv": "2605.00080", "doi": "10.48550/arxiv.2605.00080", "code": "", "site": "", "note": "<b>World Model for Robot Learning: A Comprehensive Survey</b> — the most complete map of this area as of 2026. Start here if you want the taxonomy before the papers; it is also the fastest way to check whether a subarea you are about to claim as novel already has a name.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "World models, which are predictive representations of how environments evolve under actions, have become a central component of robot learning. They support policy learning, planning, simulation, evaluation, data generation, and have advanced rapidly with the rise of foundation models and large-scale video generation. However, the literature remains fragmented across architectures, functional roles, and embodied application domains. To address this gap, we present a comprehensive review of world models from a robot-learning perspective. We examine how world models are coupled with robot policies, how they serve as learned simulators for reinforcement learning and evaluation, and how robotic video world models have progressed from imagination-based generation to controllable, structured, and foundation-scale formulations. We further connect these ideas to navigation and autonomous driving, and summarize representative datasets, benchmarks, and evaluation protocols. Overall, this survey systematically reviews the rapidly growing literature on world models for robot learning, clarifies key paradigms and applications, and highlights major challenges and future directions for predictive modeling in embodied agents. To facilitate continued access to newly emerging works, benchmarks, and resources, we will maintain and regularly update the accompanying GitHub repository alongside this survey.", "source": "curated", "first_author": "Bohan Hou", "url": "https://arxiv.org/abs/2605.00080", "fig": "figures/wm-survey-robot.webp", "figs": ["figures/wm-survey-robot-1.webp"]}, {"id": "x2n", "title": "X2-N: A Transformable Wheel-legged Humanoid Robot with Dual-mode Locomotion and Manipulation", "tree": "systems", "branch": "codesign", "authors": ["Yan Ning", "Xingzhou Chen", "Delong Li", "Hao Zhang", "Hanfu Gai", "Jinlei Zhao", "Tongyuan Li", "Cheng Zhang", "Zhihui Peng", "Ling Shi"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-23", "topics": ["transformable wheel-legged humanoid", "dual-mode locomotion", "RL whole-body control"], "paradigm": ["hardware", "RL"], "method": [], "projects": [], "related": ["berkeley-humanoid-lite", "changing-morph-dyn", "dyret", "getup-morphologies", "humanup", "jdeite", "reconfig-wheelleg", "swheg", "var-topology-truss"], "arxiv": "2604.21541", "doi": "10.48550/arxiv.2604.21541", "code": "", "site": "", "note": "X2-N is a high-DoF robot that operates as both a humanoid and a wheel-legged vehicle and <b>transforms between them by joint reconfiguration</b>, with a single RL whole-body control framework covering hybrid locomotion, the transformation itself, and manipulation. This is your transformer-robot idea, built and published in April 2026. Read it first — it defines what is left to claim.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Wheel-legged robots combine the efficiency of wheeled locomotion with the versatility of legged systems, enabling rapid traversal over both continuous and discrete terrains. However, conventional designs typically employ fixed wheels as feet and limited degrees of freedom (DoFs) at the hips, resulting in reduced stability and mobility during legged locomotion compared to humanoids with flat feet. In addition, most existing platforms lack a full upper body with arms, which limits their ability to perform dexterous manipulation tasks. In this letter, we present X2-N, a high-DoF transformable robot with dual-mode locomotion and manipulation. X2-N can operate in both humanoid and wheel-legged forms and transform seamlessly between them through joint reconfiguration. We further propose a reinforcement learning (RL)-based whole-body control framework tailored to this morphology, enabling control across hybrid locomotion, transformation, and manipulation. We validate X2-N in a range of challenging locomotion and manipulation tasks, including dynamic skating-like motion, stair climbing, and package delivery. Results demonstrate high", "source": "curated", "first_author": "Yan Ning", "url": "https://arxiv.org/abs/2604.21541", "fig": "figures/x2n.webp", "figs": ["figures/x2n-1.webp", "figures/x2n-2.webp", "figures/x2n-3.webp"]}, {"id": "pi06", "title": "$π^{*}_{0.6}$: a VLA That Learns From Experience", "tree": "il", "branch": "vla", "authors": ["Physical Intelligence"], "corresponding": [], "institutions": ["Physical Intelligence"], "lab": "Physical Intelligence (π)", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-05", "topics": ["RL from experience", "advantage-conditioned policy", "real-world improvement"], "paradigm": ["VLA", "RL"], "method": ["RECAP: RL with experience and corrections via advantage-conditioned policies"], "projects": [], "related": ["dyna", "dyna2", "hand-in-loop", "hil-serl", "iRe-VLA", "pi0", "pi05", "ript-vla"], "arxiv": "2511.14759", "doi": "", "code": "", "site": "", "note": "The direct precedent for your RL-post-training idea, from the people with the most robot data. Demonstrations get you a policy; on-robot experience plus corrections is what roughly halves the failure rate. Verify the exact numbers against the paper before citing.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We study how vision-language-action (VLA) models can improve through real-world deployments via reinforcement learning (RL). We present a general-purpose method, RL with Experience and Corrections via Advantage-conditioned Policies (RECAP), that provides for RL training of VLAs via advantage conditioning. Our method incorporates heterogeneous data into the self-improvement process, including demonstrations, data from on-policy collection, and expert teleoperated interventions provided during autonomous execution. RECAP starts by pre-training a generalist VLA with offline RL, which we call $π^{*}_{0.6}$, that can then be specialized to attain high performance on downstream tasks through on-robot data collection. We show that the $π^{*}_{0.6}$ model trained with the full RECAP method can fold laundry in real homes, reliably assemble boxes, and make espresso drinks using a professional espresso machine. On some of the hardest tasks, RECAP more than doubles task throughput and roughly halves the task failure rate.", "source": "curated", "first_author": "Physical Intelligence", "url": "https://arxiv.org/abs/2511.14759", "fig": "figures/pi06.webp", "figs": ["figures/pi06-1.webp", "figures/pi06-2.webp", "figures/pi06-3.webp"]}, {"id": "pi05", "title": "$π_{0.5}$: a Vision-Language-Action Model with Open-World Generalization", "tree": "il", "branch": "vla", "authors": ["Kevin Black", "Noah Brown", "James Darpinian", "Karan Dhabalia", "Danny Driess", "Adnan Esmail", "et al."], "corresponding": [], "institutions": ["Physical Intelligence"], "lab": "Physical Intelligence (π)", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-04-22", "topics": ["open-world generalization", "heterogeneous co-training", "mobile manipulation"], "paradigm": ["VLA", "flow matching"], "method": ["co-training on robot data + web data + high-level subtask prediction", "discrete then flow decoding"], "projects": ["iris"], "related": ["agibot-world", "cosmos-reason", "dreamzero", "gemini-robotics", "generalist-ai", "knowledge-insulation", "pi0", "pi06", "rt2", "skild", "sunday-gelato", "tidybot2"], "arxiv": "2504.16054", "doi": "", "code": "", "site": "", "note": "In your CoRL 2025 folder. Cleans an unseen kitchen it has never entered. The lesson is the data recipe, not the architecture: heterogeneous co-training (web VQA, subtask labels, other robots) is what produces open-world behaviour.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/439_pi_0_5_a_Vision_Language_A.pdf", "abstract": "In order for robots to be useful, they must perform practically relevant tasks in the real world, outside of the lab. While vision-language-action (VLA) models have demonstrated impressive results for end-to-end robot control, it remains an open question how far such models can generalize in the wild. We describe $π_{0.5}$, a new model based on $π_{0}$ that uses co-training on heterogeneous tasks to enable broad generalization. $π_{0.5}$\\ uses data from multiple robots, high-level semantic prediction, web data, and other sources to enable broadly generalizable real-world robotic manipulation. Our system uses a combination of co-training and hybrid multi-modal examples that combine image observations, language commands, object detections, semantic subtask prediction, and low-level actions. Our experiments show that this kind of knowledge transfer is essential for effective generalization, and we demonstrate for the first time that an end-to-end learning-enabled robotic system can perform long-horizon and dexterous manipulation skills, such as cleaning a kitchen or bedroom, in entirely new homes.", "source": "curated", "first_author": "Kevin Black", "url": "https://arxiv.org/abs/2504.16054", "fig": "figures/pi05.webp", "figs": ["figures/pi05-1.webp", "figures/pi05-2.webp"]}, {"id": "hugwbc", "title": "A Unified and General Humanoid Whole-Body Controller for Versatile Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Yufei Xue", "Wentao Dong", "Minghuan Liu", "Weinan Zhang", "Jiangmiao Pang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-05", "topics": ["unified whole-body controller", "gait parameterisation", "upper-body intervention"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["amo", "cross-humanoid-wbc", "exbody2", "haic", "homie", "hover", "unirelo"], "arxiv": "2502.03206", "doi": "10.48550/arxiv.2502.03206", "code": "", "site": "", "note": "One policy producing walking, running, jumping, standing and hopping with <b>tunable</b> frequency, swing height, body height, waist rotation and pitch — and it accepts real-time upper-body intervention from a teleoperator while holding the gait. That last property is what makes it a loco-manipulation controller rather than a locomotion one, and it is the interface MABEL wants underneath its arms.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Locomotion is a fundamental skill for humanoid robots. However, most existing works make locomotion a single, tedious, unextendable, and unconstrained movement. This limits the kinematic capabilities of humanoid robots. In contrast, humans possess versatile athletic abilities-running, jumping, hopping, and finely adjusting gait parameters such as frequency and foot height. In this paper, we investigate solutions to bring such versatility into humanoid locomotion and thereby propose HugWBC: a unified and general humanoid whole-body controller for versatile locomotion. By designing a general command space in the aspect of tasks and behaviors, along with advanced techniques like symmetrical loss and intervention training for learning a whole-body humanoid controlling policy in simulation, HugWBC enables real-world humanoid robots to produce various natural gaits, including walking, jumping, standing, and hopping, with customizable parameters such as frequency, foot swing height, further combined with different body height, waist rotation, and body pitch. Beyond locomotion, HugWBC also supports real-time interventions from external upper-body controllers like teleoperation, enabling loco-manipulation with precision under any locomotive behavior. Extensive experiments validate the high tracking accuracy and robustness of HugWBC with/without upper-body intervention for all commands, and we further provide an in-depth analysis of how the various commands affect humanoid movement and offer insights into the relationships between these commands. To our knowledge, HugWBC is the first humanoid whole-body controller that supports such versatile locomotion behaviors with high robustness and flexibility.", "source": "curated", "first_author": "Yufei Xue", "url": "https://arxiv.org/abs/2502.03206", "fig": "figures/hugwbc.webp", "figs": ["figures/hugwbc-1.webp", "figures/hugwbc-2.webp", "figures/hugwbc-3.webp"]}, {"id": "amo", "title": "AMO: Adaptive Motion Optimization for Hyper-Dexterous Humanoid Whole-Body Control", "tree": "rl", "branch": "loco-manip", "authors": ["Jialong Li", "Xuxin Cheng", "Tianshu Huang", "Shiqi Yang", "Ri-Zhao Qiu", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-06", "topics": ["adaptive motion optimization", "hyper-dexterous whole-body control"], "paradigm": ["RL", "control"], "method": [], "projects": ["mabel"], "related": ["exbody2", "falcon", "haic", "handoff", "hover", "hugwbc", "kinematics-multipolicy", "resmimic"], "arxiv": "2505.03738", "doi": "10.48550/arxiv.2505.03738", "code": "", "site": "", "note": "Sim-to-real RL fused with online trajectory optimization for real-time whole-body control on a 29-DoF Unitree G1, reporting a materially expanded workspace. The hybrid — learned policy for robustness, optimizer for reach — is the pragmatic answer when pure RL cannot hit the corner of the workspace.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid robots derive much of their dexterity from hyper-dexterous whole-body movements, enabling tasks that require a large operational workspace: such as picking objects off the ground. However, achieving these capabilities on real humanoids remains challenging due to their high degrees of freedom (DoF) and nonlinear dynamics. We propose Adaptive Motion Optimization (AMO), a framework that integrates sim-to-real reinforcement learning (RL) with trajectory optimization for real-time, adaptive whole-body control. To mitigate distribution bias in motion imitation RL, we construct a hybrid AMO dataset and train a network capable of robust, on-demand adaptation to potentially O.O.D. commands. We validate AMO in simulation and on a 29-DoF Unitree G1 humanoid robot, demonstrating superior stability and an expanded workspace compared to strong baselines. Finally, we show that AMO's consistent performance supports autonomous task execution via imitation learning, underscoring the system's versatility and robustness.", "source": "curated", "first_author": "Jialong Li", "url": "https://arxiv.org/abs/2505.03738", "fig": "figures/amo.webp", "figs": ["figures/amo-1.webp", "figures/amo-2.webp"]}, {"id": "asap", "title": "ASAP: Aligning Simulation and Real-World Physics for Learning Agile Humanoid Whole-Body Skills", "tree": "rl", "branch": "sim2real", "authors": ["Tairan He", "Jiawei Gao", "Wenli Xiao", "Yuanhang Zhang", "Zi Wang", "Jiashun Wang", "Zhengyi Luo", "Guanya Shi", "et al."], "corresponding": ["Tairan He"], "institutions": ["Carnegie Mellon University", "NVIDIA"], "lab": "CMU LeCAR Lab", "venue": "RSS", "venue_type": "conference", "year": 2025, "date": "2025-02-03", "topics": ["sim-to-real", "delta action model", "agile humanoid skills"], "paradigm": ["RL"], "method": ["learn a residual 'delta action' model from real rollouts", "fine-tune the policy against it"], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["beyond-mimic", "deepmimic", "domain-rand", "dreureka", "far-dex", "humanoid-dex-s2r", "real2sim", "resmimic", "rma", "unitree-g1", "video2sim2real", "videomimic", "wheeled-multistage", "wheeled-payload"], "arxiv": "2502.01143", "doi": "", "code": "", "site": "", "note": "In your library. The cleanest modern take on closing the dynamics gap: collect real rollouts, learn a delta-action correction, retrain in the corrected sim. This is residual physics applied to sim-to-real, and it is the template for your residual world-model idea.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/asap.pdf", "abstract": "Humanoid robots hold the potential for unparalleled versatility in performing human-like, whole-body skills. However, achieving agile and coordinated whole-body motions remains a significant challenge due to the dynamics mismatch between simulation and the real world. Existing approaches, such as system identification (SysID) and domain randomization (DR) methods, often rely on labor-intensive parameter tuning or result in overly conservative policies that sacrifice agility. In this paper, we present ASAP (Aligning Simulation and Real-World Physics), a two-stage framework designed to tackle the dynamics mismatch and enable agile humanoid whole-body skills. In the first stage, we pre-train motion tracking policies in simulation using retargeted human motion data. In the second stage, we deploy the policies in the real world and collect real-world data to train a delta (residual) action model that compensates for the dynamics mismatch. Then, ASAP fine-tunes pre-trained policies with the delta action model integrated into the simulator to align effectively with real-world dynamics. We evaluate ASAP across three transfer scenarios: IsaacGym to IsaacSim, IsaacGym to Genesis, and IsaacGym to the real-world Unitree G1 humanoid robot. Our approach significantly improves agility and whole-body coordination across various dynamic motions, reducing tracking error compared to SysID, DR, and delta dynamics learning baselines. ASAP enables highly agile motions that were previously difficult to achieve, demonstrating the potential of delta action learning in bridging simulation and real-world dynamics. These results suggest a promising sim-to-real direction for developing more expressive and agile humanoids.", "source": "curated", "first_author": "Tairan He", "url": "https://arxiv.org/abs/2502.01143", "fig": "figures/asap.webp", "figs": ["figures/asap-1.webp", "figures/asap-2.webp"]}, {"id": "retarget-objectives", "title": "Analyzing Key Objectives in Human-to-Robot Retargeting for Dexterous Manipulation", "tree": "il", "branch": "retargeting", "authors": ["Xin", "Chendong", "Mingrui Yu", "Yongpeng Jiang", "Zhefeng Zhang", "Xiang Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-06-11", "topics": ["retargeting objectives", "ablation", "dexterous manipulation"], "paradigm": ["teleop"], "method": [], "projects": ["geodex", "mabel"], "related": ["anyteleop", "c2dex", "dexpilot", "gmr", "human2humanoid"], "arxiv": "2506.09384", "doi": "10.48550/arxiv.2506.09384", "code": "", "site": "", "note": "An ablation of what the retargeting objective should actually optimize for dexterous manipulation. This is the closest existing thing to the <b>benchmark</b> your learned-retargeting idea proposes — read it first and decide whether to extend it rather than start over.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Kinematic retargeting from human hands to robot hands is essential for transferring dexterity from humans to robots in manipulation teleoperation and imitation learning. However, due to mechanical differences between human and robot hands, completely reproducing human motions on robot hands is impossible. Existing works on retargeting incorporate various optimization objectives, focusing on different aspects of hand configuration. However, the lack of experimental comparative studies leaves the significance and effectiveness of these objectives unclear. This work aims to analyze these retargeting objectives for dexterous manipulation through extensive real-world comparative experiments. Specifically, we propose a comprehensive retargeting objective formulation that integrates intuitively crucial factors appearing in recent approaches. The significance of each factor is evaluated through experimental ablation studies on the full objective in kinematic posture retargeting and real-world teleoperated manipulation tasks. Experimental results and conclusions provide valuable insights for designing more accurate and effective retargeting algorithms for real-world dexterous manipulation.", "source": "curated", "first_author": "Xin", "url": "https://arxiv.org/abs/2506.09384", "fig": "figures/retarget-objectives.webp", "figs": ["figures/retarget-objectives-1.webp", "figures/retarget-objectives-2.webp", "figures/retarget-objectives-3.webp"]}, {"id": "behavior-robot-suite", "title": "BEHAVIOR Robot Suite: Streamlining Real-World Whole-Body Manipulation for Everyday Household Activities", "tree": "systems", "branch": "mobile-manip", "authors": ["Yunfan Jiang", "Ruohan Zhang", "Josiah Wong", "Chen Wang", "Yanjie Ze", "Hang Yin", "Cem Gokmen", "Shuran Song", "Jiajun Wu", "Li Fei-Fei"], "corresponding": [], "institutions": ["Stanford University"], "lab": "Stanford Vision & Learning (SVL)", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-03-07", "topics": ["whole-body manipulation", "household benchmark", "open platform"], "paradigm": ["hardware", "IL"], "method": [], "projects": ["mabel"], "related": ["homer", "mobile-aloha", "rekep", "robocasa", "tidybot", "tidybot2", "voxposer", "yor"], "arxiv": "2503.05652", "doi": "", "code": "", "site": "", "note": "In your MABEL hardware folder. Identifies the three capabilities household tasks actually need — bimanual coordination, stable navigation, extensive reach — and builds the platform (JoyLo interface + WB-VIMA policy) around them. The capability decomposition is the useful part for MABEL's spec.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/707_BEHAVIOR_Robot_Suite_Strea.pdf", "abstract": "Real-world household tasks present significant challenges for mobile manipulation robots. An analysis of existing robotics benchmarks reveals that successful task performance hinges on three key whole-body control capabilities: bimanual coordination, stable and precise navigation, and extensive end-effector reachability. Achieving these capabilities requires careful hardware design, but the resulting system complexity further complicates visuomotor policy learning. To address these challenges, we introduce the BEHAVIOR Robot Suite (BRS), a comprehensive framework for whole-body manipulation in diverse household tasks. Built on a bimanual, wheeled robot with a 4-DoF torso, BRS integrates a cost-effective whole-body teleoperation interface for data collection and a novel algorithm for learning whole-body visuomotor policies. We evaluate BRS on five challenging household tasks that not only emphasize the three core capabilities but also introduce additional complexities, such as long-range navigation, interaction with articulated and deformable objects, and manipulation in confined spaces. We believe that BRS's integrated robotic embodiment, data collection interface, and learning framework mark a significant step toward enabling real-world whole-body manipulation for everyday household tasks. BRS is open-sourced at https://behavior-robot-suite.github.io/", "source": "curated", "first_author": "Yunfan Jiang", "url": "https://arxiv.org/abs/2503.05652", "fig": "figures/behavior-robot-suite.webp", "figs": ["figures/behavior-robot-suite-1.webp", "figures/behavior-robot-suite-2.webp"]}, {"id": "cosmos-reason", "title": "Cosmos-Reason1: From Physical Common Sense To Embodied Reasoning", "tree": "il", "branch": "vla", "authors": ["NVIDIA", ":", "Alisson Azzolini", "Junjie Bai", "Hannah Brandon", "Jiaxin Cao", "Prithvijit Chattopadhyay", "Huayu Chen", "Jinju Chu", "Yin Cui", "Jenna Diamond", "Yifan Ding", "Liang Feng", "Francesco Ferroni", "Rama Govindaraju", "Jinwei Gu", "Siddharth Gururani", "Imad El Hanafi", "Zekun Hao", "Jacob Huffman", "Jingyi Jin", "Brendan Johnson", "Rizwan Khan", "George Kurian", "Elena Lantz", "Nayeon Lee", "Zhaoshuo Li", "Xuan Li", "Maosheng Liao", "Tsung-Yi Lin", "Yen-Chen Lin", "Ming-Yu Liu", "Xiangyu Lu", "Alice Luo", "Andrew Mathau", "Yun Ni", "Lindsey Pavao", "Wei Ping", "David W. Romero", "Misha Smelyanskiy", "Shuran Song", "Lyne Tchapmi", "Andrew Z. Wang", "Boxin Wang", "Haoxiang Wang", "Fangyin Wei", "Jiashu Xu", "Yao Xu", "Dinghao Yang", "Xiaodong Yang", "Zhuolin Yang", "Jingxu Zhang", "Xiaohui Zeng", "Zhe Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-18", "topics": ["physical common sense", "embodied reasoning", "chain-of-thought"], "paradigm": ["VLA", "LLM"], "method": [], "projects": [], "related": ["cosmos", "dvla", "gemini-robotics", "groot-n1", "groot-n15", "groot-n16", "pi05", "spatial-brain", "voyager"], "arxiv": "2503.15558", "doi": "", "code": "", "site": "", "note": "Cosmos-Reason1 — NVIDIA's reasoning VLM for physical AI, and the 'System 2' that later GR00T models plug in. Two models (7B, 56B) that watch video, reason in long chains of thought about physics and common sense, and emit an embodied decision in language. Read it for the ontology and the benchmarks: it is the clearest statement of what 'physical reasoning' is being defined to mean.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Physical AI systems need to perceive, understand, and perform complex actions in the physical world. In this paper, we present the Cosmos-Reason1 models that can understand the physical world and generate appropriate embodied decisions (e.g., next step action) in natural language through long chain-of-thought reasoning processes. We begin by defining key capabilities for Physical AI reasoning, with a focus on physical common sense and embodied reasoning. To represent physical common sense, we use a hierarchical ontology that captures fundamental knowledge about space, time, and physics. For embodied reasoning, we rely on a two-dimensional ontology that generalizes across different physical embodiments. Building on these capabilities, we develop two multimodal large language models, Cosmos-Reason1-7B and Cosmos-Reason1-56B. We curate data and train our models in two stages: Physical AI supervised fine-tuning (SFT) and Physical AI reinforcement learning (RL). To evaluate our models, we build comprehensive benchmarks for physical common sense and embodied reasoning according to our ontologies. Evaluation results show that Physical AI SFT and RL bring significant improvements. To facilitate the development of Physical AI, we make our code and pre-trained models available under the NVIDIA Open Model License at https://github.com/nvidia-cosmos/cosmos-reason1.", "source": "curated", "first_author": "NVIDIA", "url": "https://arxiv.org/abs/2503.15558", "fig": "figures/cosmos-reason.webp", "figs": ["figures/cosmos-reason-1.webp"]}, {"id": "doglove", "title": "DOGlove: Dexterous Manipulation with a Low-Cost Open-Source Haptic Force Feedback Glove", "tree": "systems", "branch": "interfaces", "authors": ["Han Zhang", "Songbo Hu", "Zhecheng Yuan", "Huazhe Xu"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Qi Zhi Institute"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2025, "date": "2025-02-11", "topics": ["haptic glove", "force feedback", "open source"], "paradigm": ["hardware", "teleop"], "method": ["~$600 glove with 21-DOF tracking and 5-finger force feedback"], "projects": ["geodex", "wam-tactile"], "related": ["bunny-visionpro", "dexcap", "dexumi", "egoemg", "exo-viha", "orca-hand", "ruka"], "arxiv": "2502.07730", "doi": "", "code": "", "site": "", "note": "In your Gloves/ folder. Force feedback at $600 changes what glove teleop can collect — you get contact-aware demonstrations rather than free-space ones. Relevant to both the 22-DOF hand and the tactile-world-model ideas.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/Gloves/DOGlove.pdf", "abstract": "Dexterous hand teleoperation plays a pivotal role in enabling robots to achieve human-level manipulation dexterity. However, current teleoperation systems often rely on expensive equipment and lack multi-modal sensory feedback, restricting human operators' ability to perceive object properties and perform complex manipulation tasks. To address these limitations, we present DOGlove, a low-cost, precise, and haptic force feedback glove system for teleoperation and manipulation. DoGlove can be assembled in hours at a cost under 600 USD. It features a customized joint structure for 21-DoF motion capture, a compact cable-driven torque transmission mechanism for 5-DoF multidirectional force feedback, and a linear resonate actuator for 5-DoF fingertip haptic feedback. Leveraging action and haptic force retargeting, DOGlove enables precise and immersive teleoperation of dexterous robotic hands, achieving high success rates in complex, contact-rich tasks. We further evaluate DOGlove in scenarios without visual feedback, demonstrating the critical role of haptic force feedback in task performance. In addition, we utilize the collected demonstrations to train imitation learning policies, highlighting the potential and effectiveness of DOGlove. DOGlove's hardware and software system will be fully open-sourced at https://do-glove.github.io/.", "source": "curated", "first_author": "Han Zhang", "url": "https://arxiv.org/abs/2502.07730", "fig": "figures/doglove.webp", "figs": ["figures/doglove-1.webp", "figures/doglove-2.webp", "figures/doglove-3.webp"]}, {"id": "dexskin", "title": "DexSkin: High-Coverage Conformable Robotic Skin for Learning Contact-Rich Manipulation", "tree": "systems", "branch": "tactile-hw", "authors": ["Suzannah Wistreich", "Baiyu Shi", "Stephen Tian", "Samuel Clarke", "Michael Nath", "Chengyi Xu", "Zhenan Bao", "Jiajun Wu"], "corresponding": [], "institutions": ["Stanford University"], "lab": "Stanford Vision & Learning", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-09-23", "topics": ["conformable skin", "contact-rich learning", "high coverage", "proceedings 2025"], "paradigm": ["hardware", "IL"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "av-contact", "digit", "dream-tac", "egoemg", "gelsight", "mm-hand", "omnitactune", "robot-synesthesia", "robotacdex", "rotate-without-seeing", "sonicsense"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. High-coverage conformable skin — the missing piece for a tactile world model, because fingertip-only sensing misses most of the contact that matters.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/510_DexSkin_High_Coverage_Conf.pdf", "abstract": "", "source": "curated", "first_author": "Suzannah Wistreich", "url": "https://arxiv.org/abs/2509.18830", "fig": "figures/x-dexskinhighcoverageconformableroboticski.webp"}, {"id": "dexumi", "title": "DexUMI: Using Human Hand as the Universal Manipulation Interface for Dexterous Manipulation", "tree": "systems", "branch": "interfaces", "authors": ["Mengda Xu", "Han Zhang", "Yifan Hou", "Zhenjia Xu", "Linxi Fan", "Manuela Veloso", "Shuran Song"], "corresponding": ["Mengda Xu", "Shuran Song"], "institutions": ["Stanford University", "Columbia University", "NVIDIA", "JP Morgan AI Research"], "lab": "Stanford REALab", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-05-27", "topics": ["dexterous UMI", "wearable exoskeleton", "hand embodiment gap"], "paradigm": ["hardware", "IL"], "method": ["wearable hand exoskeleton as the adapter + software inpainting of the human hand into robot pixels"], "projects": ["exo", "geodex", "mabel", "r2s2r", "wam-tactile"], "related": ["ace-teleop", "airexo", "dexcap", "doglove", "ego2robot", "egosteer", "fast-umi", "hmi-robotfree", "maniwav", "mimic-robotics", "phantom", "realdexumi", "ruka", "sunday-gelato", "ume", "umi", "wt-umi"], "arxiv": "2505.21864", "doi": "", "code": "", "site": "https://dex-umi.github.io/", "note": "In your library twice (Gloves/ and the CoRL 2025 folder). This is the paper standing between you and the 'custom dexterous UMI' idea — read its two-part gap-closing strategy (hardware exoskeleton for kinematics, video inpainting for appearance) and decide what you would do differently.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/Gloves/DexUMI.pdf", "abstract": "We present DexUMI - a data collection and policy learning framework that uses the human hand as the natural interface to transfer dexterous manipulation skills to various robot hands. DexUMI includes hardware and software adaptations to minimize the embodiment gap between the human hand and various robot hands. The hardware adaptation bridges the kinematics gap using a wearable hand exoskeleton. It allows direct haptic feedback in manipulation data collection and adapts human motion to feasible robot hand motion. The software adaptation bridges the visual gap by replacing the human hand in video data with high-fidelity robot hand inpainting. We demonstrate DexUMI's capabilities through comprehensive real-world experiments on two different dexterous robot hand hardware platforms, achieving an average task success rate of 86%.", "source": "curated", "first_author": "Mengda Xu", "url": "https://arxiv.org/abs/2505.21864", "fig": "figures/dexumi.webp", "figs": ["figures/dexumi-1.webp", "figures/dexumi-2.webp"]}, {"id": "dogmo", "title": "DogMo: A Large-Scale Multi-View RGB-D Dataset for 4D Canine Motion Recovery", "tree": "il", "branch": "human-video", "authors": ["Wang", "Zan", "Chen", "Siyu", "Mo", "Luya", "Gao", "Xinfeng", "Shen", "Yuxin", "Ding", "Lebin", "Liang", "Wei"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-28", "topics": ["4D canine motion", "multi-view RGB-D", "1.2k sequences"], "paradigm": ["dataset"], "method": [], "projects": ["r2s2r"], "related": ["animal-avatars", "barc", "corgi", "hsmal", "s-cheetah"], "arxiv": "2510.24117", "doi": "10.48550/arxiv.2510.24117", "code": "", "site": "", "note": "1,200 motion sequences from 10 dogs, multi-view RGB-D with real 3D — the first dog dataset with enough fidelity to measure how a real animal's trunk actually bends while running. If you want to derive a spine from data rather than assume one, this is the measurement.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We present DogMo, a large-scale multi-view RGB-D video dataset capturing diverse canine movements for the task of motion recovery from images. DogMo comprises 1.2k motion sequences collected from 10 unique dogs, offering rich variation in both motion and breed. It addresses key limitations of existing dog motion datasets, including the lack of multi-view and real 3D data, as well as limited scale and diversity. Leveraging DogMo, we establish four motion recovery benchmark settings that support systematic evaluation across monocular and multi-view, RGB and RGB-D inputs. To facilitate accurate motion recovery, we further introduce a three-stage, instance-specific optimization pipeline that fits the SMAL model to the motion sequences. Our method progressively refines body shape and pose through coarse alignment, dense correspondence supervision, and temporal regularization. Our dataset and method provide a principled foundation for advancing research in dog motion recovery and open up new directions at the intersection of computer vision, computer graphics, and animal behavior modeling.", "source": "curated", "first_author": "Wang", "url": "https://arxiv.org/abs/2510.24117", "fig": "figures/dogmo.webp", "figs": ["figures/dogmo-1.webp", "figures/dogmo-2.webp"]}, {"id": "dreamgen", "title": "DreamGen: Unlocking Generalization in Robot Learning through Video World Models", "tree": "il", "branch": "world-models", "authors": ["Joel Jang", "Seonghyeon Ye", "Zongyu Lin", "Jiannan Xiang", "Johan Bjorck", "Yu Fang", "Fengyuan Hu", "Spencer Huang", "Kaushil Kundalia", "Yen-Chen Lin", "Loic Magne", "Ajay Mandlekar", "Avnish Narayan", "You Liang Tan", "Guanzhi Wang", "Jing Wang", "Qi Wang", "Yinzhen Xu", "Xiaohui Zeng", "Kaiyuan Zheng", "Ruijie Zheng", "Ming-Yu Liu", "Luke Zettlemoyer", "Dieter Fox", "Jan Kautz", "Scott Reed", "Yuke Zhu", "Linxi Fan"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-19", "topics": ["neural trajectories", "video world model", "synthetic data"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["wam-tactile"], "related": ["ace-ego-0", "cosmos", "cosmos-policy", "dreamdojo", "egoscale", "genie", "gigaworld-0", "gr2", "groot-n1", "groot-n15", "humannet", "prism", "rigvid", "robocurate", "robodream", "vjepa2"], "arxiv": "2505.12705", "doi": "", "code": "", "site": "", "note": "DreamGen, from NVIDIA GEAR. Adapt an image-to-video world model to your robot's embodiment, generate photorealistic 'neural trajectories' of tasks it has never done, extract pseudo-actions, and train on them. The headline is the data ratio: 22 new behaviours in seen and unseen environments from teleop on a single pick-and-place task. This is the strongest existing argument that a video world model is a data engine rather than a planner, and it is directly relevant to the UMI-scale-up thesis.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce DreamGen, a simple yet highly effective 4-stage pipeline for training robot policies that generalize across behaviors and environments through neural trajectories - synthetic robot data generated from video world models. DreamGen leverages state-of-the-art image-to-video generative models, adapting them to the target robot embodiment to produce photorealistic synthetic videos of familiar or novel tasks in diverse environments. Since these models generate only videos, we recover pseudo-action sequences using either a latent action model or an inverse-dynamics model (IDM). Despite its simplicity, DreamGen unlocks strong behavior and environment generalization: a humanoid robot can perform 22 new behaviors in both seen and unseen environments, while requiring teleoperation data from only a single pick-and-place task in one environment. To evaluate the pipeline systematically, we introduce DreamGen Bench, a video generation benchmark that shows a strong correlation between benchmark performance and downstream policy success. Our work establishes a promising new axis for scaling robot learning well beyond manual data collection. Code available at https://github.com/NVIDIA/GR00T-Dreams.", "source": "curated", "first_author": "Joel Jang", "url": "https://arxiv.org/abs/2505.12705", "fig": "figures/dreamgen.webp", "figs": ["figures/dreamgen-1.webp", "figures/dreamgen-2.webp", "figures/dreamgen-3.webp"]}, {"id": "egomimic", "title": "EgoMimic: Scaling Imitation Learning via Egocentric Video", "tree": "il", "branch": "human-video", "authors": ["Simar Kareer", "Dhruv Patel", "Ryan Punamiya", "Pranay Mathur", "Shuo Cheng", "Chen Wang", "Judy Hoffman", "Danfei Xu"], "corresponding": ["Simar Kareer"], "institutions": ["Georgia Tech"], "lab": "Georgia Tech RIPL (Danfei Xu)", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2024-10-31", "topics": ["egocentric human data", "co-training", "Aria glasses"], "paradigm": ["IL"], "method": ["Project Aria glasses capture + humanoid co-training with normalized hand/robot action spaces"], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["dexcap", "ego4d", "egodemogen", "egoengine", "egokit", "egoscale", "egosteer", "human-video-survey", "humanego", "humanoid-policy", "mimicplay", "phantom", "simdex"], "arxiv": "2410.24221", "doi": "", "code": "", "site": "", "note": "Human wears Aria glasses, robot learns from it. The interesting engineering is the action-space normalization that lets human hand trajectories and robot trajectories go in the same batch. Closest published thing to your egocentric-dexterity plan.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The scale and diversity of demonstration data required for imitation learning is a significant challenge. We present EgoMimic, a full-stack framework which scales manipulation via human embodiment data, specifically egocentric human videos paired with 3D hand tracking. EgoMimic achieves this through: (1) a system to capture human embodiment data using the ergonomic Project Aria glasses, (2) a low-cost bimanual manipulator that minimizes the kinematic gap to human data, (3) cross-domain data alignment techniques, and (4) an imitation learning architecture that co-trains on human and robot data. Compared to prior works that only extract high-level intent from human videos, our approach treats human and robot data equally as embodied demonstration data and learns a unified policy from both data sources. EgoMimic achieves significant improvement on a diverse set of long-horizon, single-arm and bimanual manipulation tasks over state-of-the-art imitation learning methods and enables generalization to entirely new scenes. Finally, we show a favorable scaling trend for EgoMimic, where adding 1 hour of additional hand data is significantly more valuable than 1 hour of additional robot data. Videos and additional information can be found at https://egomimic.github.io/", "source": "curated", "first_author": "Simar Kareer", "url": "https://arxiv.org/abs/2410.24221", "fig": "figures/egomimic.webp", "figs": ["figures/egomimic-1.webp", "figures/egomimic-2.webp", "figures/egomimic-3.webp"]}, {"id": "changing-morph-dyn", "title": "Fast and Modular Whole-Body Lagrangian Dynamics of Legged Robots with Changing Morphology", "tree": "systems", "branch": "codesign", "authors": [], "corresponding": [], "institutions": [], "lab": "", "venue": "Nonlinear Dynamics", "venue_type": "journal", "year": 2025, "date": "2025-04-23", "topics": ["whole-body dynamics under changing morphology", "modular assembly"], "paradigm": ["control"], "method": [], "projects": [], "related": ["gcnt", "jdeite", "morph-hypernet", "opt-legged-control", "var-topology-truss", "x2n"], "arxiv": "2504.16383", "doi": "", "code": "", "site": "", "note": "Closed-form, singularity-free whole-body Lagrangian dynamics for legged robots whose <b>morphology changes</b> — each limb modelled independently (Boltzmann-Hamel equations plus screw theory) and reassembled according to the robot's current configuration, so you do not re-derive the dynamics per mode. For a machine with a different kinematic tree in car form and robot form, this is the modelling tool. Author list not yet verified against the arXiv record — check before citing.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "", "url": "https://arxiv.org/abs/2504.16383", "fig": "figures/changing-morph-dyn.webp"}, {"id": "groot-n1", "title": "GR00T N1: An Open Foundation Model for Generalist Humanoid Robots", "tree": "il", "branch": "vla", "authors": ["NVIDIA: Johan Bjorck", "Fernando Castañeda", "Nikita Cherniadev", "Xingye Da", "Runyu Ding", "et al."], "corresponding": [], "institutions": ["NVIDIA"], "lab": "NVIDIA GEAR Lab", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-18", "topics": ["humanoid foundation model", "dual-system architecture"], "paradigm": ["VLA", "diffusion"], "method": ["System-2 VLM for reasoning + System-1 diffusion transformer for 120 Hz actions"], "projects": ["mabel"], "related": ["cosmos-reason", "dexvla", "dreamgen", "dreamzero", "gemini-robotics", "generalist-ai", "groot-n15", "groot-n16", "helix", "humanoid-policy", "pi0", "rdt1b", "spatial-brain", "unitree-g1", "vlm4vla", "wholebodyvla"], "arxiv": "2503.14734", "doi": "", "code": "", "site": "", "note": "The open humanoid foundation model, and the clearest statement of the dual-system (slow reasoning / fast control) split that most 2025-26 VLAs converged on.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "General-purpose robots need a versatile body and an intelligent mind. Recent advancements in humanoid robots have shown great promise as a hardware platform for building generalist autonomy in the human world. A robot foundation model, trained on massive and diverse data sources, is essential for enabling the robots to reason about novel situations, robustly handle real-world variability, and rapidly learn new tasks. To this end, we introduce GR00T N1, an open foundation model for humanoid robots. GR00T N1 is a Vision-Language-Action (VLA) model with a dual-system architecture. The vision-language module (System 2) interprets the environment through vision and language instructions. The subsequent diffusion transformer module (System 1) generates fluid motor actions in real time. Both modules are tightly coupled and jointly trained end-to-end. We train GR00T N1 with a heterogeneous mixture of real-robot trajectories, human videos, and synthetically generated datasets. We show that our generalist robot model GR00T N1 outperforms the state-of-the-art imitation learning baselines on standard simulation benchmarks across multiple robot embodiments. Furthermore, we deploy our model on the Fourier GR-1 humanoid robot for language-conditioned bimanual manipulation tasks, achieving strong performance with high data efficiency.", "source": "curated", "first_author": "NVIDIA: Johan Bjorck", "url": "https://arxiv.org/abs/2503.14734", "fig": "figures/groot-n1.webp", "figs": ["figures/groot-n1-1.webp", "figures/groot-n1-2.webp"]}, {"id": "gemini-robotics", "title": "Gemini Robotics: Bringing AI into the Physical World", "tree": "il", "branch": "vla", "authors": ["Google DeepMind Robotics Team"], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind Robotics", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-25", "topics": ["VLA", "embodied reasoning", "dexterity"], "paradigm": ["VLA"], "method": ["Gemini 2.0 backbone + embodied reasoning (ER) variant + action decoder"], "projects": [], "related": ["cosmos-reason", "groot-n1", "pi05", "rt2"], "arxiv": "2503.20020", "doi": "", "code": "", "site": "", "note": "The frontier-lab entry. The interesting half is Gemini Robotics-ER: spatial and physical reasoning evaluated separately from control, which is the right way to attribute VLA failures.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Recent advancements in large multimodal models have led to the emergence of remarkable generalist capabilities in digital domains, yet their translation to physical agents such as robots remains a significant challenge. This report introduces a new family of AI models purposefully designed for robotics and built upon the foundation of Gemini 2.0. We present Gemini Robotics, an advanced Vision-Language-Action (VLA) generalist model capable of directly controlling robots. Gemini Robotics executes smooth and reactive movements to tackle a wide range of complex manipulation tasks while also being robust to variations in object types and positions, handling unseen environments as well as following diverse, open vocabulary instructions. We show that with additional fine-tuning, Gemini Robotics can be specialized to new capabilities including solving long-horizon, highly dexterous tasks, learning new short-horizon tasks from as few as 100 demonstrations and adapting to completely novel robot embodiments. This is made possible because Gemini Robotics builds on top of the Gemini Robotics-ER model, the second model we introduce in this work. Gemini Robotics-ER (Embodied Reasoning) extends Gemini's multimodal reasoning capabilities into the physical world, with enhanced spatial and temporal understanding. This enables capabilities relevant to robotics including object detection, pointing, trajectory and grasp prediction, as well as multi-view correspondence and 3D bounding box predictions. We show how this novel combination can support a variety of robotics applications. We also discuss and address important safety considerations related to this new class of robotics foundation models. The Gemini Robotics family marks a substantial step towards developing general-purpose robots that realizes AI's potential in the physical world.", "source": "curated", "first_author": "Google DeepMind Robotics Team", "url": "https://arxiv.org/abs/2503.20020", "fig": "figures/gemini-robotics.webp", "figs": ["figures/gemini-robotics-1.webp", "figures/gemini-robotics-2.webp", "figures/gemini-robotics-3.webp"]}, {"id": "genie-envisioner", "title": "Genie Envisioner: A Unified World Foundation Platform for Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Yue Liao", "Pengfei Zhou", "Siyuan Huang", "Donglin Yang", "Shengcong Chen", "Yuxin Jiang", "Yue Hu", "Jingbin Cai", "Si Liu", "Jianlan Luo", "Liliang Chen", "Shuicheng Yan", "Maoqing Yao", "Guanghui Ren"], "corresponding": [], "institutions": ["AgiBot", "NUS"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-08-07", "topics": ["unified world foundation platform", "policy learning + simulation + evaluation in one model"], "paradigm": ["world model"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["cosmos-policy", "dreamdojo", "genie", "geniworld", "iris", "ivideogpt", "roboworld", "worldeval"], "arxiv": "2508.05635", "doi": "10.48550/arxiv.2508.05635", "code": "", "site": "", "note": "<b>Genie Envisioner</b> — one video world model serving <em>three</em> jobs: generating policy actions, acting as a neural simulator, and evaluating policies. Worth reading as the platform-level statement of what a WAM is for, which is the framing your Tactile WAM project will need in its introduction.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce Genie Envisioner (GE), a unified world foundation platform for robotic manipulation that integrates policy learning, evaluation, and simulation within a single video-generative framework. At its core, GE-Base is a large-scale, instruction-conditioned video diffusion model that captures the spatial, temporal, and semantic dynamics of real-world robotic interactions in a structured latent space. Built upon this foundation, GE-Act maps latent representations to executable action trajectories through a lightweight, flow-matching decoder, enabling precise and generalizable policy inference across diverse embodiments with minimal supervision. To support scalable evaluation and training, GE-Sim serves as an action-conditioned neural simulator, producing high-fidelity rollouts for closed-loop policy development. The platform is further equipped with EWMBench, a standardized benchmark suite measuring visual fidelity, physical consistency, and instruction-action alignment. Together, these components establish Genie Envisioner as a scalable and practical foundation for instruction-driven, general-purpose embodied intelligence. All code, models, and benchmarks will be released publicly.", "source": "curated", "first_author": "Yue Liao", "url": "https://arxiv.org/abs/2508.05635", "fig": "figures/genie-envisioner.webp", "figs": ["figures/genie-envisioner-1.webp", "figures/genie-envisioner-2.webp"]}, {"id": "homie", "title": "HOMIE: Humanoid Loco-Manipulation with Isomorphic Exoskeleton Cockpit", "tree": "systems", "branch": "interfaces", "authors": ["Qingwei Ben", "Jia", "Feiyu", "Jia Zeng", "Junting Dong", "Dahua Lin", "Jiangmiao Pang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-18", "topics": ["isomorphic exoskeleton cockpit", "pedal locomotion", "data flywheel"], "paradigm": ["teleop", "RL"], "method": [], "projects": ["exo", "mabel"], "related": ["ace-teleop", "exo-viha", "hmi-robotfree", "hover", "hugwbc", "omnih2o", "telemoma", "twist", "ume"], "arxiv": "2502.13013", "doi": "10.48550/arxiv.2502.13013", "code": "", "site": "", "note": "The most complete teleoperation cockpit published: an RL body policy driven by a <b>pedal</b>, an isomorphic exoskeleton arm, and motion gloves for the hands — one operator, whole robot. The pedal is the design insight worth stealing: it frees both arms for manipulation while still giving continuous locomotion command, which every VR-based humanoid teleop rig struggles with. Open-sourced as OpenHomie.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Generalizable humanoid loco-manipulation poses significant challenges, requiring coordinated whole-body control and precise, contact-rich object manipulation. To address this, this paper introduces HOMIE, a semi-autonomous teleoperation system that combines a reinforcement learning policy for body control mapped to a pedal, an isomorphic exoskeleton arm for arm control, and motion-sensing gloves for hand control, forming a unified cockpit to freely operate humanoids and establish a data flywheel. The policy incorporates novel designs, including an upper-body pose curriculum, a height-tracking reward, and symmetry utilization. These features enable the system to perform walking and squatting to specific heights while seamlessly adapting to arbitrary upper-body poses. The exoskeleton, by eliminating the reliance on inverse dynamics, delivers faster and more precise arm control. The gloves utilize Hall sensors instead of servos, allowing even compact devices to achieve 15 or more degrees of freedom and freely adapt to any model of dexterous hands. Compared to previous teleoperation systems, HOMIE stands out for its exceptional efficiency, completing tasks in half the time; its expanded working range, allowing users to freely reach high and low areas as well as interact with any objects; and its affordability, with a price of just $500. The system is fully open-source, demos and code can be found in our https://homietele.github.io/.", "source": "curated", "first_author": "Qingwei Ben", "url": "https://arxiv.org/abs/2502.13013", "fig": "figures/homie.webp", "figs": ["figures/homie-1.webp", "figures/homie-2.webp", "figures/homie-3.webp"]}, {"id": "hover", "title": "HOVER: Versatile Neural Whole-Body Controller for Humanoid Robots", "tree": "il", "branch": "retargeting", "authors": ["Tairan He", "Wenli Xiao", "Toru Lin", "Zhengyi Luo", "Zhenjia Xu", "Zhenyu Jiang", "Jan Kautz", "Changliu Liu", "Guanya Shi", "Xiaolong Wang", "Yuke Zhu", "Zhen Fu"], "corresponding": [], "institutions": ["NVIDIA", "Carnegie Mellon University", "UC Berkeley", "UT Austin", "UC San Diego"], "lab": "NVIDIA GEAR; CMU LeCAR", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2024-10-28", "topics": ["unified whole-body controller", "mode switching"], "paradigm": ["RL"], "method": ["distill multiple control modes into one masked policy"], "projects": ["mabel"], "related": ["adaptive-tracking", "amo", "exbody", "exbody2", "gmr", "h2o", "handoff", "homie", "hugwbc", "maskedmimic", "omnih2o", "twist", "unirelo", "unitree-g1", "x-op"], "arxiv": "2410.21229", "doi": "", "code": "", "site": "", "note": "In your library (twice). One controller that covers joint-angle, root-velocity and kinematic-target modes by masking the command at train time — MaskedMimic's trick on real hardware. Direct blueprint for a MABEL whole-body controller.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/MABEL/literatures/teleop_and_retargeting/HOVER_paper.pdf", "abstract": "Humanoid whole-body control requires adapting to diverse tasks such as navigation, loco-manipulation, and tabletop manipulation, each demanding a different mode of control. For example, navigation relies on root velocity tracking, while tabletop manipulation prioritizes upper-body joint angle tracking. Existing approaches typically train individual policies tailored to a specific command space, limiting their transferability across modes. We present the key insight that full-body kinematic motion imitation can serve as a common abstraction for all these tasks and provide general-purpose motor skills for learning multiple modes of whole-body control. Building on this, we propose HOVER (Humanoid Versatile Controller), a multi-mode policy distillation framework that consolidates diverse control modes into a unified policy. HOVER enables seamless transitions between control modes while preserving the distinct advantages of each, offering a robust and scalable solution for humanoid control across a wide range of modes. By eliminating the need for policy retraining for each control mode, our approach improves efficiency and flexibility for future humanoid applications.", "source": "curated", "first_author": "Tairan He", "url": "https://arxiv.org/abs/2410.21229", "fig": "figures/hover.webp", "figs": ["figures/hover-1.webp", "figures/hover-2.webp"]}, {"id": "humanoid-policy", "title": "Humanoid Policy ~ Human Policy", "tree": "il", "branch": "vla", "authors": ["Ri-Zhao Qiu", "Shiqi Yang", "Xuxin Cheng", "Chaitanya Chawla", "Jialong Li", "Tairan He", "Ge Yan", "et al."], "corresponding": [], "institutions": ["UC San Diego", "Carnegie Mellon University", "University of Washington"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-03-17", "topics": ["human-humanoid co-training", "egocentric data"], "paradigm": ["VLA", "IL"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["ace-ego-0", "ego4d", "egoexo4d", "egomimic", "egoscale", "groot-n1", "intention-priors", "phantom"], "arxiv": "2503.13441", "doi": "", "code": "", "site": "", "note": "In your library twice (RL/corl_final.pdf and the CoRL folder). Trains one policy on human egocentric video and humanoid teleop jointly by making the action spaces commensurate. Central to your egocentric-video-for-dexterity idea.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/94_Humanoid_Policy_Human_Polic.pdf", "abstract": "Training manipulation policies for humanoid robots with diverse data enhances their robustness and generalization across tasks and platforms. However, learning solely from robot demonstrations is labor-intensive, requiring expensive tele-operated data collection which is difficult to scale. This paper investigates a more scalable data source, egocentric human demonstrations, to serve as cross-embodiment training data for robot learning. We mitigate the embodiment gap between humanoids and humans from both the data and modeling perspectives. We collect an egocentric task-oriented dataset (PH2D) that is directly aligned with humanoid manipulation demonstrations. We then train a human-humanoid behavior policy, which we term Human Action Transformer (HAT). The state-action space of HAT is unified for both humans and humanoid robots and can be differentiably retargeted to robot actions. Co-trained with smaller-scale robot data, HAT directly models humanoid robots and humans as different embodiments without additional supervision. We show that human data improves both generalization and robustness of HAT with significantly better data collection efficiency. Code and data: https://human-as-robot.github.io/", "source": "curated", "first_author": "Ri-Zhao Qiu", "url": "https://arxiv.org/abs/2503.13441", "fig": "figures/humanoid-policy.webp", "figs": ["figures/humanoid-policy-1.webp"]}, {"id": "icrt", "title": "In-Context Imitation Learning via Next-Token Prediction", "tree": "il", "branch": "in-context", "authors": ["Letian Fu", "Huang Huang", "Gaurav Datta", "Lawrence Yunliang Chen", "William Chung-Ho Panitch", "Fangchen Liu", "Hui Li", "Ken Goldberg"], "corresponding": ["Letian Fu"], "institutions": ["UC Berkeley"], "lab": "AUTOLAB", "venue": "CVPR", "venue_type": "conference", "year": 2025, "date": "2024-08-28", "topics": ["in-context learning", "next-token prediction", "few-shot"], "paradigm": ["IL", "transformer"], "method": ["prompt the policy with a handful of demo trajectories at test time", "no weight updates"], "projects": [], "related": ["one-demo", "robocat", "rt1", "skild-scale", "vid2robot"], "arxiv": "2408.15980", "doi": "", "code": "", "site": "", "note": "You asked for in-context learning specifically — this is the cleanest robot instantiation. Give the transformer a few demos as context and it performs the new task without fine-tuning.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We explore how to enhance next-token prediction models to perform in-context imitation learning on a real robot, where the robot executes new tasks by interpreting contextual information provided during the input phase, without updating its underlying policy parameters. We propose In-Context Robot Transformer (ICRT), a causal transformer that performs autoregressive prediction on sensorimotor trajectories without relying on any linguistic data or reward function. This formulation enables flexible and training-free execution of new tasks at test time, achieved by prompting the model with sensorimotor trajectories of the new task composing of image observations, actions and states tuples, collected through human teleoperation. Experiments with a Franka Emika robot demonstrate that the ICRT can adapt to new tasks specified by prompts, even in environment configurations that differ from both the prompt and the training data. In a multitask environment setup, ICRT significantly outperforms current state-of-the-art next-token prediction models in robotics on generalizing to unseen tasks. Code, checkpoints and data are available on https://icrt.dev/", "source": "curated", "first_author": "Letian Fu", "url": "https://arxiv.org/abs/2408.15980", "fig": "figures/icrt.webp", "figs": ["figures/icrt-1.webp", "figures/icrt-2.webp", "figures/icrt-3.webp"]}, {"id": "knowledge-insulation", "title": "Knowledge Insulating Vision-Language-Action Models: Train Fast, Run Fast, Generalize Better", "tree": "il", "branch": "vla", "authors": ["Danny Driess", "Jost Tobias Springenberg", "Brian Ichter", "Lili Yu", "Adrian Li-Bell", "Karl Pertsch", "Allen Z. Ren", "Homer Walke", "Quan Vuong", "Lucy Xiaoyang Shi", "Sergey Levine"], "corresponding": [], "institutions": [], "lab": "", "venue": "NeurIPS", "venue_type": "preprint", "year": 2025, "date": "2025-05-29", "topics": ["knowledge insulation", "VLA training", "generalization"], "paradigm": ["VLA", "flow matching"], "method": [], "projects": ["iris"], "related": ["iRe-VLA", "openvla-oft", "pi0", "pi05", "pld"], "arxiv": "2505.23705", "doi": "10.52202/085713-3439", "code": "", "site": "", "note": "Physical Intelligence's answer to the problem that training a VLA's action expert degrades the VLM's language and semantic knowledge. Insulate the backbone from the action gradients: train fast, run fast, generalize better. Read alongside iRe-VLA and PLD — all three are attacking the same destabilization problem from different directions, and this is the cleanest diagnosis of it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Vision-language-action (VLA) models provide a powerful approach to training control policies for physical systems, such as robots, by combining end-to-end learning with transfer of semantic knowledge from web-scale vision-language model (VLM) training. However, the constraints of real-time control are often at odds with the design of VLMs: the most powerful VLMs have tens or hundreds of billions of parameters, presenting an obstacle to real-time inference, and operate on discrete tokens rather than the continuous-valued outputs that are required for controlling robots. To address this challenge, recent VLA models have used specialized modules for efficient continuous control, such as action experts or continuous output heads, which typically require adding new untrained parameters to the pretrained VLM backbone. While these modules improve real-time and control capabilities, it remains an open question whether they preserve or degrade the semantic knowledge contained in the pretrained VLM, and what effect they have on the VLA training dynamics. In this paper, we study this question in the context of VLAs that include a continuous diffusion or flow matching action expert, showing that naively including such experts significantly harms both training speed and knowledge transfer. We provide an extensive analysis of various design choices, their impact on performance and knowledge transfer, and propose a technique for insulating the VLM backbone during VLA training that mitigates this issue. Videos are available at https://pi.website/research/knowledge_insulation.", "source": "curated", "first_author": "Danny Driess", "url": "https://doi.org/10.52202/085713-3439", "arxiv_url": "https://arxiv.org/abs/2505.23705", "fig": "figures/knowledge-insulation.webp"}, {"id": "humanup", "title": "Learning Getting-Up Policies for Real-World Humanoid Robots", "tree": "rl", "branch": "locomotion", "authors": ["Xialin He", "Runpei Dong", "Zixuan Chen", "Saurabh Gupta"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-17", "topics": ["getting-up policies", "real-world humanoid", "curriculum"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["deepmimic", "fall-safety", "getup-morphologies", "kungfu-bot", "stablemimic", "unirelo", "x2n"], "arxiv": "2502.12152", "doi": "10.48550/arxiv.2502.12152", "code": "", "site": "", "note": "The reference getting-up paper: learned policies that stand a real humanoid back up from varied fallen configurations, later extended to deformable, slippery and inclined terrain. Any robot that falls needs this, and a transforming robot will fall in postures nobody anticipated.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Automatic fall recovery is a crucial prerequisite before humanoid robots can be reliably deployed. Hand-designing controllers for getting up is difficult because of the varied configurations a humanoid can end up in after a fall and the challenging terrains humanoid robots are expected to operate on. This paper develops a learning framework to produce controllers that enable humanoid robots to get up from varying configurations on varying terrains. Unlike previous successful applications of learning to humanoid locomotion, the getting-up task involves complex contact patterns (which necessitates accurately modeling of the collision geometry) and sparser rewards. We address these challenges through a two-phase approach that induces a curriculum. The first stage focuses on discovering a good getting-up trajectory under minimal constraints on smoothness or speed / torque limits. The second stage then refines the discovered motions into deployable (i.e. smooth and slow) motions that are robust to variations in initial configuration and terrains. We find these innovations enable a real-world G1 humanoid robot to get up from two main situations that we considered: a) lying face up and b) lying face down, both tested on flat, deformable, slippery surfaces and slopes (e.g., sloppy grass and snowfield). This is one of the first successful demonstrations of learned getting-up policies for human-sized humanoid robots in the real world.", "source": "curated", "first_author": "Xialin He", "url": "https://arxiv.org/abs/2502.12152", "fig": "figures/humanup.webp", "figs": ["figures/humanup-1.webp"]}, {"id": "getup-morphologies", "title": "Learning to Get Up Across Morphologies: Zero-Shot Recovery with a Unified Humanoid Policy", "tree": "rl", "branch": "locomotion", "authors": ["Jonathan Spraggett"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-13", "topics": ["cross-morphology recovery", "unified policy", "zero-shot"], "paradigm": ["RL"], "method": [], "projects": [], "related": ["cross-humanoid-wbc", "dyret", "gcnt", "humanup", "morph-hypernet", "unirelo", "x2n"], "arxiv": "2512.12230", "doi": "10.48550/arxiv.2512.12230", "code": "", "site": "", "note": "One policy that gets seven different humanoids (0.48–0.81 m, 2.8–7.9 kg) back on their feet, transferring zero-shot to unseen morphologies at ~86%. Directly relevant to a transforming robot: the policy has to survive the body changing, which is the same problem as the body being different.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Fall recovery is a critical skill for humanoid robots in dynamic environments such as RoboCup, where prolonged downtime often decides the match. Recent techniques using deep reinforcement learning (DRL) have produced robust get-up behaviors, yet existing methods require training of separate policies for each robot morphology. This paper presents a single DRL policy capable of recovering from falls across seven humanoid robots with diverse heights (0.48 - 0.81 m), weights (2.8 - 7.9 kg), and dynamics. Trained with CrossQ, the unified policy transfers zero-shot up to 86 +/- 7% (95% CI [81, 89]) on unseen morphologies, eliminating the need for robot-specific training. Comprehensive leave-one-out experiments, morph scaling analysis, and diversity ablations show that targeted morphological coverage improves zero-shot generalization. In some cases, the shared policy even surpasses the specialist baselines. These findings illustrate the practicality of morphology-agnostic control for fall recovery, laying the foundation for generalist humanoid control. The software is open-source and available at: https://github.com/utra-robosoccer/unified-humanoid-getup", "source": "curated", "first_author": "Jonathan Spraggett", "url": "https://arxiv.org/abs/2512.12230", "fig": "figures/getup-morphologies.webp"}, {"id": "dreamerv3", "title": "Mastering Diverse Domains through World Models", "tree": "rl", "branch": "model-based", "authors": ["Danijar Hafner", "Jurgis Pasukonis", "Jimmy Ba", "Timothy Lillicrap"], "corresponding": ["Danijar Hafner"], "institutions": ["Google DeepMind", "University of Toronto"], "lab": "DeepMind", "venue": "Nature", "venue_type": "journal", "year": 2025, "date": "2023-01-10", "topics": ["world models", "latent imagination", "generality"], "paradigm": ["model-based RL"], "method": ["RSSM latent dynamics", "symlog returns", "fixed hyperparameters across 150+ tasks"], "projects": ["wam-tactile"], "related": ["causal-wm", "daydreamer", "dino-wm", "dreamer-v1", "dreamer-v2", "dreamerv4", "muzero", "planet", "pwm", "storm", "tdmpc2", "worldmodels"], "arxiv": "2301.04104", "doi": "", "code": "", "site": "", "note": "One hyperparameter set, 150+ domains, diamonds in Minecraft from scratch. The reason 'world model' became a default design choice rather than an exotic one.", "aka": "Dreamer V3", "stars": 3, "curated": true, "local": "", "abstract": "Developing a general algorithm that learns to solve tasks across a wide range of applications has been a fundamental challenge in artificial intelligence. Although current reinforcement learning algorithms can be readily applied to tasks similar to what they have been developed for, configuring them for new application domains requires significant human expertise and experimentation. We present DreamerV3, a general algorithm that outperforms specialized methods across over 150 diverse tasks, with a single configuration. Dreamer learns a model of the environment and improves its behavior by imagining future scenarios. Robustness techniques based on normalization, balancing, and transformations enable stable learning across domains. Applied out of the box, Dreamer is the first algorithm to collect diamonds in Minecraft from scratch without human data or curricula. This achievement has been posed as a significant challenge in artificial intelligence that requires exploring farsighted strategies from pixels and sparse rewards in an open world. Our work allows solving challenging control problems without extensive experimentation, making reinforcement learning broadly applicable.", "source": "curated", "first_author": "Danijar Hafner", "url": "https://arxiv.org/abs/2301.04104", "fig": "figures/dreamerv3.webp", "figs": ["figures/dreamerv3-1.webp", "figures/dreamerv3-2.webp", "figures/dreamerv3-3.webp"], "arxiv_url": "https://arxiv.org/abs/2301.04104"}, {"id": "newton", "title": "Newton: An Open-Source GPU-Accelerated Physics Engine for Robot Learning", "tree": "systems", "branch": "sim-bench", "authors": ["NVIDIA", "Google DeepMind", "Disney Research"], "corresponding": [], "institutions": ["NVIDIA", "Google DeepMind", "Disney Research"], "lab": "", "venue": "open source", "venue_type": "industry", "year": 2025, "date": "2025-09-01", "topics": ["physics engine", "differentiable simulation", "rigid and soft bodies"], "paradigm": ["simulator"], "method": [], "projects": [], "related": ["genesis", "isaaclab", "mujoco", "soft-gentle"], "arxiv": "", "doi": "", "code": "https://github.com/newton-physics/newton", "site": "https://github.com/newton-physics/newton", "note": "The three groups with the most to gain from better contact simulation building one engine together, with explicit support for rigid and flexible parts in the same scene. If the soft-body claims hold, this is the most important development for your soft-object sim-to-real idea — benchmark it against MuJoCo and Genesis before believing the marketing.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "NVIDIA", "url": "https://github.com/newton-physics/newton"}, {"id": "orca-hand", "title": "ORCA: An Open-Source, Reliable, Cost-Effective, Anthropomorphic Robotic Hand for Uninterrupted Dexterous Task Learning", "tree": "systems", "branch": "hands", "authors": ["Clemens Schwarke", "Victor Klemm", "Jesus Tordesillas", "Jean-Pierre Sleiman", "Marco Hutter", "et al."], "corresponding": [], "institutions": ["ETH Zürich"], "lab": "ETH RSL", "venue": "IROS", "venue_type": "conference", "year": 2025, "date": "2025-04-05", "topics": ["open-source tendon hand", "17 DOF", "reliability"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["1x-hand", "aero-hand", "dexwrist", "dlr-hand", "doglove", "faive-ball", "leap-hand", "midas-hand", "mimic-robotics", "mm-hand", "pds-joint", "ruka", "ruka-v2"], "arxiv": "2504.04259", "doi": "", "code": "", "site": "https://www.orcahand.com/", "note": "In your Robot Hands/ folder and your M2 library. Tendon-driven, 17 DOF, ~$2k, and explicitly engineered for reliability and repairability rather than peak spec. The closest existing reference design for your 22-DOF open hand — read its tendon routing and maintenance sections carefully.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/Robot Hands/ORCA.pdf", "abstract": "General-purpose robots should possess human-like dexterity and agility to perform tasks with the same versatility as us. A human-like form factor further enables the use of vast datasets of human-hand interactions. However, the primary bottleneck in dexterous manipulation lies not only in software but arguably even more in hardware. Robotic hands that approach human capabilities are often prohibitively expensive, bulky, or require enterprise-level maintenance, limiting their accessibility for broader research and practical applications. What if the research community could get started with reliable dexterous hands within a day? We present the open-source ORCA hand, a reliable and anthropomorphic 17-DoF tendon-driven robotic hand with integrated tactile sensors, fully assembled in less than eight hours and built for a material cost below 2,000 CHF. We showcase ORCA's key design features such as popping joints, auto-calibration, and tensioning systems that significantly reduce complexity while increasing reliability, accuracy, and robustness. We benchmark the ORCA hand across a variety of tasks, ranging from teleoperation and imitation learning to zero-shot sim-to-real reinforcement learning. Furthermore, we demonstrate its durability, withstanding more than 10,000 continuous operation cycles - equivalent to approximately 20 hours - without hardware failure, the only constraint being the duration of the experiment itself. Video is here: https://youtu.be/kUbPSYMmOds. Design files, source code, and documentation are available at https://srl.ethz.ch/orcahand.", "source": "curated", "first_author": "Clemens Schwarke", "url": "https://arxiv.org/abs/2504.04259", "fig": "figures/orca-hand.webp", "figs": ["figures/orca-hand-1.webp", "figures/orca-hand-2.webp", "figures/orca-hand-3.webp"]}, {"id": "phantom", "title": "Phantom: Training Robots Without Robots Using Only Human Videos", "tree": "il", "branch": "human-video", "authors": ["Marion Lepert", "Jiaying Fang", "Jeannette Bohg"], "corresponding": ["Marion Lepert"], "institutions": ["Stanford University"], "lab": "Stanford IPRL (Bohg)", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-01", "topics": ["human video only", "zero robot data", "inpainting"], "paradigm": ["IL"], "method": ["edit human hands out of video and paint the robot in", "then train BC on the edited video"], "projects": ["geodex", "r2s2r"], "related": ["dexumi", "do-as-i-do", "ego2robot", "egoengine", "egomimic", "egoscale", "human-video-survey", "humanego", "humanoid-policy", "learn-any-images", "mimicplay", "rigvid"], "arxiv": "2503.00779", "doi": "", "code": "", "site": "", "note": "Zero robot demonstrations — close the embodiment gap by *editing the video* rather than by learning a correspondence. Cheap, slightly absurd, and it works. Worth trying for GeoDex.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Training general-purpose robots requires learning from large and diverse data sources. Current approaches rely heavily on teleoperated demonstrations which are difficult to scale. We present a scalable framework for training manipulation policies directly from human video demonstrations, requiring no robot data. Our method converts human demonstrations into robot-compatible observation-action pairs using hand pose estimation and visual data editing. We inpaint the human arm and overlay a rendered robot to align the visual domains. This enables zero-shot deployment on real hardware without any fine-tuning. We demonstrate strong success rates-up to 92%-on a range of tasks including deformable object manipulation, multi-object sweeping, and insertion. Our approach generalizes to novel environments and supports closed-loop execution. By demonstrating that effective policies can be trained using only human videos, our method broadens the path to scalable robot learning.", "source": "curated", "first_author": "Marion Lepert", "url": "https://arxiv.org/abs/2503.00779", "fig": "figures/phantom.webp", "figs": ["figures/phantom-1.webp", "figures/phantom-2.webp"]}, {"id": "phystwin", "title": "PhysTwin: Physics-Informed Reconstruction and Simulation of Deformable Objects from Videos", "tree": "systems", "branch": "sim-bench", "authors": ["Hanxiao Jiang", "Hao-Yu Hsu", "Kaifeng Zhang", "Hsin-Ni Yu", "Shenlong Wang", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "ICCV", "venue_type": "conference", "year": 2025, "date": "2025-03-23", "topics": ["physics-informed reconstruction", "spring-mass + Gaussian splats", "inverse modelling"], "paradigm": ["simulator"], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": ["acdc", "adaptigraph", "deform360", "egophys", "embodied-sim-survey", "gs-dynamics", "particle-grid-dynamics", "physgen3d", "physworld", "robocraft"], "arxiv": "2503.17973", "doi": "10.1109/ICCV51701.2025.00678", "code": "", "site": "", "note": "<b>PhysTwin</b> (ICCV 2025) — from sparse video of a deformable object being manipulated to a <b>real-time interactive replica</b> that is both photo-realistic and physically realistic. The recipe is worth copying wholesale: spring-mass for physics, generative shape model for the unseen geometry, Gaussian splats for appearance, and a multi-stage inverse-modelling optimization that recovers dense physical properties from the video. This is the strongest existing answer to 'what does a digital twin mean for a non-rigid object', and the natural comparison point for the reconstruction leg of Real2Sim2Real.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Creating a physical digital twin of a real-world object has immense potential in robotics, content creation, and XR. In this paper, we present PhysTwin, a novel framework that uses sparse videos of dynamic objects under interaction to produce a photo- and physically realistic, real-time interactive virtual replica. Our approach centers on two key components: (1) a physics-informed representation that combines spring-mass models for realistic physical simulation, generative shape models for geometry, and Gaussian splats for rendering; and (2) a novel multi-stage, optimization-based inverse modeling framework that reconstructs complete geometry, infers dense physical properties, and replicates realistic appearance from videos. Our method integrates an inverse physics framework with visual perception cues, enabling high-fidelity reconstruction even from partial, occluded, and limited viewpoints. PhysTwin supports modeling various deformable objects, including ropes, stuffed animals, cloth, and delivery packages. Experiments show that PhysTwin outperforms competing methods in reconstruction, rendering, future prediction, and simulation under novel interactions. We further demonstrate its applications in interactive real-time simulation and model-based robotic motion planning.", "source": "curated", "first_author": "Hanxiao Jiang", "url": "https://arxiv.org/abs/2503.17973", "fig": "figures/phystwin.webp", "figs": ["figures/phystwin-1.webp"]}, {"id": "physworld", "title": "PhysWorld: From Real Videos to World Models of Deformable Objects via Physics-Aware Demonstration Synthesis", "tree": "systems", "branch": "sim-bench", "authors": ["Yu Yang", "Zhilu Zhang", "Xiang Zhang", "Yihan Zeng", "Hui Li", "Wangmeng Zuo"], "corresponding": [], "institutions": ["Huawei"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-24", "topics": ["physics-aware demonstration synthesis", "real video to world model", "deformables"], "paradigm": ["simulator", "IL"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "adaptigraph", "egophys", "embodied-sim-survey", "physgen3d", "phystwin", "prism", "rigvid"], "arxiv": "2510.21447", "doi": "10.48550/arxiv.2510.21447", "code": "", "site": "", "note": "<b>PhysWorld</b> (Huawei, Wangmeng Zuo's group — a parallel line to RoboPIL's, arriving at the same place) — real videos in, a world model of the deformable object out, and then <b>physics-aware demonstration synthesis</b> on top of it. Note what this is: the full real2sim2real loop for soft objects, where the sim is not authored but reconstructed, and its payoff is measured in synthesized demonstrations rather than in reconstruction error. That is the framing your project should adopt — reconstruction quality is a means, demonstration yield is the metric.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Interactive world models that simulate object dynamics are crucial for robotics, VR, and AR. However, it remains a significant challenge to learn physics-consistent dynamics models from limited real-world video data, especially for deformable objects with spatially-varying physical properties. To overcome the challenge of data scarcity, we propose PhysWorld, a novel framework that utilizes a simulator to synthesize physically plausible and diverse demonstrations to learn efficient world models. Specifically, we first construct a physics-consistent digital twin within MPM simulator via constitutive model selection and global-to-local optimization of physical properties. Subsequently, we apply part-aware perturbations to the physical properties and generate various motion patterns for the digital twin, synthesizing extensive and diverse demonstrations. Finally, using these demonstrations, we train a lightweight GNN-based world model that is embedded with physical properties. The real video can be used to further refine the physical properties. PhysWorld achieves accurate and fast future predictions for various deformable objects, and also generalizes well to novel interactions. Experiments show that PhysWorld has competitive performance while enabling inference speeds 47 times faster than the recent state-of-the-art method, i.e., PhysTwin.", "source": "curated", "first_author": "Yu Yang", "url": "https://arxiv.org/abs/2510.21447", "fig": "figures/physworld.webp"}, {"id": "ruka", "title": "RUKA: Rethinking the Design of Humanoid Hands with Learning", "tree": "systems", "branch": "hands", "authors": ["Anya Zorin", "Irmak Guzey", "Billy Yan", "Aadhithya Iyer", "Lisa Kondrich", "Nikhil X. Bhattasali", "Lerrel Pinto"], "corresponding": ["Anya Zorin"], "institutions": ["New York University"], "lab": "NYU General-Purpose Robotics Lab (Lerrel Pinto)", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-04-17", "topics": ["tendon-driven hand", "learned kinematics", "low cost"], "paradigm": ["hardware", "IL"], "method": ["learn the tendon-to-joint map from MANUS glove data instead of modelling it analytically"], "projects": ["geodex"], "related": ["1x-hand", "dexcap", "dexumi", "doglove", "leap-hand", "midas-hand", "mimic-robotics", "orca-hand", "ruka-v2"], "arxiv": "2504.13165", "doi": "", "code": "", "site": "https://ruka-hand.github.io/", "note": "In your Robot Hands/ folder. The key move for your 22-DOF idea: stop trying to analytically model tendon coupling and just *learn* the motor-to-joint map from data. That removes the hardest part of building a high-DOF tendon hand. NYU lab — relevant to your PhD context.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/Robot Hands/RUKA.pdf", "abstract": "Dexterous manipulation is a fundamental capability for robotic systems, yet progress has been limited by hardware trade-offs between precision, compactness, strength, and affordability. Existing control methods impose compromises on hand designs and applications. However, learning-based approaches present opportunities to rethink these trade-offs, particularly to address challenges with tendon-driven actuation and low-cost materials. This work presents RUKA, a tendon-driven humanoid hand that is compact, affordable, and capable. Made from 3D-printed parts and off-the-shelf components, RUKA has 5 fingers with 15 underactuated degrees of freedom enabling diverse human-like grasps. Its tendon-driven actuation allows powerful grasping in a compact, human-sized form factor. To address control challenges, we learn joint-to-actuator and fingertip-to-actuator models from motion-capture data collected by the MANUS glove, leveraging the hand's morphological accuracy. Extensive evaluations demonstrate RUKA's superior reachability, durability, and strength compared to other robotic hands. Teleoperation tasks further showcase RUKA's dexterous movements. The open-source design and assembly instructions of RUKA, code, and data are available at https://ruka-hand.github.io/.", "source": "curated", "first_author": "Anya Zorin", "url": "https://arxiv.org/abs/2504.13165", "fig": "figures/ruka.webp", "figs": ["figures/ruka-1.webp", "figures/ruka-2.webp", "figures/ruka-3.webp"]}, {"id": "resmimic", "title": "ResMimic: From General Motion Tracking to Humanoid Whole-body Loco-Manipulation via Residual Learning", "tree": "rl", "branch": "loco-manip", "authors": ["Siheng Zhao", "Yanjie Ze", "Yue Wang", "C. Karen Liu", "Pieter Abbeel", "Guanya Shi", "Rocky Duan"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-06", "topics": ["residual learning", "motion tracking to loco-manipulation"], "paradigm": ["RL"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["adaptive-tracking", "amo", "asap", "demohlm", "exbody2", "falcon", "haic", "kinematics-multipolicy", "pld"], "arxiv": "2510.05070", "doi": "10.48550/arxiv.2510.05070", "code": "", "site": "", "note": "Takes a general motion-tracking policy and adds a <b>residual</b> to turn it into whole-body loco-manipulation, rather than retraining from scratch. The residual pattern shows up everywhere in this library ([[asap]], [[pld]], [[omnitactune]]) and this is its whole-body instance.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid whole-body loco-manipulation promises transformative capabilities for daily service and warehouse tasks. While recent advances in general motion tracking (GMT) have enabled humanoids to reproduce diverse human motions, these policies lack the precision and object awareness required for loco-manipulation. To this end, we introduce ResMimic, a two-stage residual learning framework for precise and expressive humanoid control from human motion data. First, a GMT policy, trained on large-scale human-only motion, serves as a task-agnostic base for generating human-like whole-body movements. An efficient but precise residual policy is then learned to refine the GMT outputs to improve locomotion and incorporate object interaction. To further facilitate efficient training, we design (i) a point-cloud-based object tracking reward for smoother optimization, (ii) a contact reward that encourages accurate humanoid body-object interactions, and (iii) a curriculum-based virtual object controller to stabilize early training. We evaluate ResMimic in both simulation and on a real Unitree G1 humanoid. Results show substantial gains in task success, training efficiency, and robustness over strong baselines. Videos are available at https://resmimic.github.io/ .", "source": "curated", "first_author": "Siheng Zhao", "url": "https://arxiv.org/abs/2510.05070", "fig": "figures/resmimic.webp", "figs": ["figures/resmimic-1.webp", "figures/resmimic-2.webp", "figures/resmimic-3.webp"]}, {"id": "gmr", "title": "Retargeting Matters: General Motion Retargeting for Humanoid Motion Tracking", "tree": "il", "branch": "retargeting", "authors": ["João P. Araújo", "Yanjie Ze", "Pei Xu", "Jiajun Wu", "C. Karen Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-02", "topics": ["general motion retargeting", "tracking fidelity", "real-time CPU"], "paradigm": ["teleop"], "method": [], "projects": ["mabel"], "related": ["c2dex", "exbody2", "h2o", "hover", "human2humanoid", "retarget-objectives", "twist"], "arxiv": "2510.02252", "doi": "10.48550/arxiv.2510.02252", "code": "", "site": "", "note": "Retargeting Matters — and the title is the finding. It shows that retargeting quality, not policy quality, often determines whether whole-body tracking works, and closes most of the gap to closed-source baselines. Runs in real time on CPU and is the retargeter behind [[twist]]. Your learned-retargeting idea should treat this as the baseline to beat.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid motion tracking policies are central to building teleoperation pipelines and hierarchical controllers, yet they face a fundamental challenge: the embodiment gap between humans and humanoid robots. Current approaches address this gap by retargeting human motion data to humanoid embodiments and then training reinforcement learning (RL) policies to imitate these reference trajectories. However, artifacts introduced during retargeting, such as foot sliding, self-penetration, and physically infeasible motion are often left in the reference trajectories for the RL policy to correct. While prior work has demonstrated motion tracking abilities, they often require extensive reward engineering and domain randomization to succeed. In this paper, we systematically evaluate how retargeting quality affects policy performance when excessive reward tuning is suppressed. To address issues that we identify with existing retargeting methods, we propose a new retargeting method, General Motion Retargeting (GMR). We evaluate GMR alongside two open-source retargeters, PHC and ProtoMotions, as well as with a high-quality closed-source dataset from Unitree. Using BeyondMimic for policy training, we isolate retargeting effects without reward tuning. Our experiments on a diverse subset of the LAFAN1 dataset reveal that while most motions can be tracked, artifacts in retargeted data significantly reduce policy robustness, particularly for dynamic or long sequences. GMR consistently outperforms existing open-source methods in both tracking performance and faithfulness to the source motion, achieving perceptual fidelity and policy success rates close to the closed-source baseline. Website: https://jaraujo98.github.io/retargeting_matters. Code: https://github.com/YanjieZe/GMR.", "source": "curated", "first_author": "João P. Araújo", "url": "https://arxiv.org/abs/2510.02252", "fig": "figures/gmr.webp", "figs": ["figures/gmr-1.webp"]}, {"id": "rigvid", "title": "Robotic Manipulation by Imitating Generated Videos Without Physical Demonstrations", "tree": "il", "branch": "human-video", "authors": ["Shivansh Patel", "Shraddhaa Mohan", "Hanlin Mai", "Unnat Jain", "Svetlana Lazebnik", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "ICLR", "venue_type": "conference", "year": 2025, "date": "2025-07-01", "topics": ["imitating AI-generated video", "no physical demonstrations", "monocular 4D extraction"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["dreamgen", "phantom", "physworld", "prism", "robodream", "unipi"], "arxiv": "2507.00990", "doi": "10.48550/arxiv.2507.00990", "code": "", "site": "", "note": "<b>RIGVid</b> (ICLR 2026) — Robots Imitating Generated Videos. Generate a video of the task with a video diffusion model, extract 4D object motion from it, retarget that to the robot, and execute — pouring, wiping, mixing, with <b>zero physical demonstrations and no robot-specific training</b>. The reason this belongs in your library is that it inverts the usual data argument: the world model is not a simulator you plan in, it is a <em>demonstration generator</em>, and the only thing you need from it is that the object trajectory be physically plausible.", "aka": "RIGVid", "stars": 3, "curated": true, "local": "", "abstract": "This work introduces Robots Imitating Generated Videos (RIGVid), a system that enables robots to perform complex manipulation tasks--such as pouring, wiping, and mixing--purely by imitating AI-generated videos, without requiring any physical demonstrations or robot-specific training. Given a language command and an initial scene image, a video diffusion model generates potential demonstration videos, and a vision-language model (VLM) automatically filters out results that do not follow the command. A 6D pose tracker then extracts object trajectories from the video, and the trajectories are retargeted to the robot in an embodiment-agnostic fashion. Through extensive real-world evaluations, we show that filtered generated videos are as effective as real demonstrations, and that performance improves with generation quality. We also show that relying on generated videos outperforms more compact alternatives such as keypoint prediction using VLMs, and that strong 6D pose tracking outperforms other ways to extract trajectories, such as dense feature point tracking. These findings suggest that videos produced by a state-of-the-art off-the-shelf model can offer an effective source of supervision for robotic manipulation.", "source": "curated", "first_author": "Shivansh Patel", "url": "https://arxiv.org/abs/2507.00990", "fig": "figures/rigvid.webp", "figs": ["figures/rigvid-1.webp", "figures/rigvid-2.webp", "figures/rigvid-3.webp"], "arxiv_url": "https://arxiv.org/abs/2507.00990"}, {"id": "pld", "title": "Self-Improving Vision-Language-Action Models with Data Generation via Residual RL", "tree": "rl", "branch": "post-training", "authors": ["Wenli Xiao", "Haotian Lin", "Andy Peng", "Haoru Xue", "Tairan He", "Yuqi Xie", "Fengyuan Hu", "Jimmy Wu", "Zhengyi Luo", "Linxi \"Jim\" Fan", "Guanya Shi", "Yuke Zhu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-30", "topics": ["residual RL", "self-improving VLA", "data generation"], "paradigm": ["RL", "VLA"], "method": [], "projects": ["iris"], "related": ["ace-ego-0", "dlr-vla", "expo-ft", "far-dex", "hand-in-loop", "hil-serl", "iRe-VLA", "knowledge-insulation", "residual-assembly", "resmimic", "ript-vla", "rlinf-vla", "robocurate", "stare-vla", "vla-jumpstart"], "arxiv": "2511.00091", "doi": "", "code": "", "site": "", "note": "PLD — probe, learn, distill. Lightweight residual actors trained off-policy fix the base VLA's failures, then the successful rollouts are distilled back into it. The most practical recipe published for your RL-post-training idea, and it avoids destabilising the language backbone entirely.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Supervised fine-tuning (SFT) has become the de facto post-training strategy for large vision-language-action (VLA) models, but its reliance on costly human demonstrations limits scalability and generalization. We propose Probe, Learn, Distill (PLD), a three-stage plug-and-play framework that improves VLAs through residual reinforcement learning (RL) and distribution-aware data collection. In Stage 1, we train lightweight residual actors to probe failure regions of the VLA generalist. In Stage 2, we use a hybrid rollout scheme that aligns collected trajectories with the generalist's deployment distribution while capturing recovery behaviors. In Stage 3, we distill the curated trajectories back into the generalist with standard SFT. PLD achieves near-saturated 99% task success on LIBERO, over 50% gains in SimplerEnv, and 100% success on real-world Franka and YAM arm manipulation tasks. Ablations show that residual probing and distribution-aware replay are key to collecting deployment-aligned data that improves both seen and unseen tasks, offering a scalable path toward self-improving VLA models.", "source": "curated", "first_author": "Wenli Xiao", "url": "https://arxiv.org/abs/2511.00091", "fig": "figures/pld.webp", "figs": ["figures/pld-1.webp", "figures/pld-2.webp"]}, {"id": "humanoid-dex-s2r", "title": "Sim-to-Real Reinforcement Learning for Vision-Based Dexterous Manipulation on Humanoids", "tree": "rl", "branch": "sim2real", "authors": ["Lin", "Toru", "Sachdev", "Kartik", "Fan", "Linxi", "Malik", "Jitendra", "Zhu", "Yuke"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-27", "topics": ["automated real-to-sim tuning", "vision-based dexterity", "policy distillation"], "paradigm": ["RL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["asap", "chen-visual-inhand", "dexndm", "far-dex", "video2sim2real"], "arxiv": "2502.20396", "doi": "10.48550/arxiv.2502.20396", "code": "", "site": "", "note": "Vision-based dexterous manipulation on humanoids with an <b>automated real-to-sim tuning module</b>, a contact-and-object-goal reward formulation, and divide-and-conquer distillation. The automated tuning module is the piece to steal: it is the closest published thing to closing the loop, even though the paper does not frame it that way.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Learning generalizable robot manipulation policies, especially for complex multi-fingered humanoids, remains a significant challenge. Existing approaches primarily rely on extensive data collection and imitation learning, which are expensive, labor-intensive, and difficult to scale. Sim-to-real reinforcement learning (RL) offers a promising alternative, but has mostly succeeded in simpler state-based or single-hand setups. How to effectively extend this to vision-based, contact-rich bimanual manipulation tasks remains an open question. In this paper, we introduce a practical sim-to-real RL recipe that trains a humanoid robot to perform three challenging dexterous manipulation tasks: grasp-and-reach, box lift and bimanual handover. Our method features an automated real-to-sim tuning module, a generalized reward formulation based on contact and object goals, a divide-and-conquer policy distillation framework, and a hybrid object representation strategy with modality-specific augmentation. We demonstrate high success rates on unseen objects and robust, adaptive policy behaviors -- highlighting that vision-based dexterous manipulation via sim-to-real RL is not only viable, but also scalable and broadly applicable to real-world humanoid manipulation tasks.", "source": "curated", "first_author": "Lin", "url": "https://arxiv.org/abs/2502.20396", "fig": "figures/humanoid-dex-s2r.webp", "figs": ["figures/humanoid-dex-s2r-1.webp"]}, {"id": "twist", "title": "TWIST: Teleoperated Whole-Body Imitation System", "tree": "systems", "branch": "interfaces", "authors": ["Yanjie Ze", "Zixuan Chen", "João Pedro Araújo", "Zi-ang Cao", "Xue Bin Peng", "Jiajun Wu", "C. Karen Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-05", "topics": ["whole-body imitation teleoperation", "real-time retargeting"], "paradigm": ["teleop", "RL"], "method": [], "projects": ["mabel"], "related": ["exbody2", "gmr", "h2o", "homie", "hover", "omnih2o"], "arxiv": "2505.02833", "doi": "10.48550/arxiv.2505.02833", "code": "", "site": "", "note": "Teleoperated whole-body imitation from a single human demonstrator, with GMR as its retargeter. The pairing matters: TWIST is the system, [[gmr]] is the retargeting quality it depends on, and the two together are the cleanest current recipe for 'human moves, humanoid follows' at full body scale.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Teleoperating humanoid robots in a whole-body manner marks a fundamental step toward developing general-purpose robotic intelligence, with human motion providing an ideal interface for controlling all degrees of freedom. Yet, most current humanoid teleoperation systems fall short of enabling coordinated whole-body behavior, typically limiting themselves to isolated locomotion or manipulation tasks. We present the Teleoperated Whole-Body Imitation System (TWIST), a system for humanoid teleoperation through whole-body motion imitation. We first generate reference motion clips by retargeting human motion capture data to the humanoid robot. We then develop a robust, adaptive, and responsive whole-body controller using a combination of reinforcement learning and behavior cloning (RL+BC). Through systematic analysis, we demonstrate how incorporating privileged future motion frames and real-world motion capture (MoCap) data improves tracking accuracy. TWIST enables real-world humanoid robots to achieve unprecedented, versatile, and coordinated whole-body motor skills--spanning whole-body manipulation, legged manipulation, locomotion, and expressive movement--using a single unified neural network controller. Our project website: https://humanoid-teleop.github.io", "source": "curated", "first_author": "Yanjie Ze", "url": "https://arxiv.org/abs/2505.02833", "fig": "figures/twist.webp", "figs": ["figures/twist-1.webp", "figures/twist-2.webp"]}, {"id": "text2robot", "title": "Text2Robot: Evolutionary Robot Design from Text Descriptions", "tree": "systems", "branch": "codesign", "authors": ["Ryan P. Ringel", "Zachary S. Charlick", "Jiaxun Liu", "Boxi Xia", "Boyuan Chen"], "corresponding": ["Boyuan Chen"], "institutions": ["Duke University"], "lab": "Duke General Robotics Lab", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2024-06-28", "topics": ["text-to-morphology", "generative design", "quadrupeds"], "paradigm": ["co-design", "LLM"], "method": [], "projects": [], "related": ["diffusebot", "dyret", "eureka", "lipson2000", "robomorph"], "arxiv": "2406.19963", "doi": "", "code": "", "site": "", "note": "Text prompt → 3D generative model → printable quadruped → evolved controller, in under a day. This is the closest published thing to your agentic co-design idea, and it is on quadrupeds, so it touches the fast-dog idea too. The gap it leaves: it optimizes for locomotion, not for dexterity.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Robot design has traditionally been costly and labor-intensive. Despite advancements in automated processes, it remains challenging to navigate a vast design space while producing physically manufacturable robots. We introduce Text2Robot, a framework that converts user text specifications and performance preferences into physical quadrupedal robots. Within minutes, Text2Robot can use text-to-3D models to provide strong initializations of diverse morphologies. Within a day, our geometric processing algorithms and body-control co-optimization produce a walking robot by explicitly considering real-world electronics and manufacturability. Text2Robot enables rapid prototyping and opens new opportunities for robot design with generative models.", "source": "curated", "first_author": "Ryan P. Ringel", "url": "https://arxiv.org/abs/2406.19963", "fig": "figures/text2robot.webp", "figs": ["figures/text2robot-1.webp", "figures/text2robot-2.webp", "figures/text2robot-3.webp"]}, {"id": "tidybot2", "title": "TidyBot++: An Open-Source Holonomic Mobile Manipulator for Robot Learning", "tree": "systems", "branch": "mobile-manip", "authors": ["Jimmy Wu", "William Chong", "Robert Holmberg", "Aaditya Prasad", "Yihuai Gao", "Oussama Khatib", "Shuran Song", "Szymon Rusinkiewicz", "Jeannette Bohg"], "corresponding": ["Jimmy Wu"], "institutions": ["Princeton University", "Stanford University"], "lab": "Princeton / Stanford IPRL", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2024-12-16", "topics": ["holonomic base", "open hardware", "mobile manipulation"], "paradigm": ["hardware"], "method": ["powered-caster holonomic drive", "~$6k base", "phone-based whole-body teleop interface"], "projects": ["m2", "mabel"], "related": ["behavior-robot-suite", "homer", "mobile-aloha", "pi05", "stretch", "telemoma", "tidybot", "umi", "umi-on-legs", "yor"], "arxiv": "2412.10447", "doi": "", "code": "", "site": "https://tidybot2.github.io/", "note": "You named this one. The powered-caster holonomic base is the design lesson for MABEL: full planar mobility without a swerve module's complexity, and a base that can be driven during demonstration collection rather than parked. The phone-as-teleop-interface trick is worth stealing outright.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Exploiting the promise of recent advances in imitation learning for mobile manipulation will require the collection of large numbers of human-guided demonstrations. This paper proposes an open-source design for an inexpensive, robust, and flexible mobile manipulator that can support arbitrary arms, enabling a wide range of real-world household mobile manipulation tasks. Crucially, our design uses powered casters to enable the mobile base to be fully holonomic, able to control all planar degrees of freedom independently and simultaneously. This feature makes the base more maneuverable and simplifies many mobile manipulation tasks, eliminating the kinematic constraints that create complex and time-consuming motions in nonholonomic bases. We equip our robot with an intuitive mobile phone teleoperation interface to enable easy data acquisition for imitation learning. In our experiments, we use this interface to collect data and show that the resulting learned policies can successfully perform a variety of common household mobile manipulation tasks.", "source": "curated", "first_author": "Jimmy Wu", "url": "https://arxiv.org/abs/2412.10447", "fig": "figures/tidybot2.webp", "figs": ["figures/tidybot2-1.webp", "figures/tidybot2-2.webp"]}, {"id": "phhi-survey", "title": "Toward Seamless Physical Human-Humanoid Interaction: Insights from Control, Intent, and Modeling with a Vision for What Comes Next", "tree": "rl", "branch": "loco-manip", "authors": ["Cardona", "Gustavo A.", "Kumbhar", "Shubham S.", "Artemiadis", "Panagiotis"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-08", "topics": ["physical human-humanoid interaction", "survey", "control and intent"], "paradigm": ["survey"], "method": [], "projects": ["m2", "mabel"], "related": ["falcon", "h2compact", "intention-tracking", "interact", "workspace-opt"], "arxiv": "2512.07765", "doi": "10.48550/arxiv.2512.07765", "code": "", "site": "", "note": "A survey of physical human-humanoid interaction across control, intent and modelling, with an explicit agenda for what comes next. Start your HRI literature review here — it is the most recent map, and it covers the physical-contact case rather than the social-navigation case most HRI surveys default to.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Physical Human-Humanoid Interaction (pHHI) is a rapidly advancing field with significant implications for deploying robots in unstructured, human-centric environments. In this review, we examine the current state of the art in pHHI through three core pillars: (i) humanoid modeling and control, (ii) human intent estimation, and (iii) computational human models. For each pillar, we survey representative approaches, identify open challenges, and analyze current limitations that hinder robust, scalable, and adaptive interaction. These include the need for whole-body control strategies capable of handling uncertain human dynamics, real-time intent inference under limited sensing, and modeling techniques that account for variability in human physical states. Although significant progress has been made within each domain, integration across pillars remains limited. We propose pathways for unifying methods across these areas to enable cohesive interaction frameworks. This structure enables us not only to map the current landscape but also to propose concrete directions for future research that aim to bridge these domains. Additionally, we introduce a unified taxonomy of interaction types based on modality, distinguishing between direct interactions (e.g., physical contact) and indirect interactions (e.g., object-mediated), and on the level of robot engagement, ranging from assistance to cooperation and collaboration. For each category in this taxonomy, we provide the three core pillars that highlight opportunities for cross-pillar unification. Our goal is to suggest avenues to advance robust, safe, and intuitive physical interaction, providing a roadmap for future research that will allow humanoid systems to effectively understand, anticipate, and collaborate with human partners in diverse real-world settings.", "source": "curated", "first_author": "Cardona", "url": "https://arxiv.org/abs/2512.07765", "fig": "figures/phhi-survey.webp"}, {"id": "dreamerv4", "title": "Training Agents Inside of Scalable World Models", "tree": "rl", "branch": "model-based", "authors": ["Danijar Hafner", "Wilson Yan", "Timothy Lillicrap"], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-29", "topics": ["scalable world models", "offline-to-online", "imagination training"], "paradigm": ["model-based RL"], "method": ["shortcut forcing", "efficient transformer world model trained on offline data"], "projects": ["wam-tactile"], "related": ["cosmos", "daydreamer", "dreamerv3", "genie3", "vjepa2"], "arxiv": "2509.24527", "doi": "", "code": "", "site": "", "note": "In your library. Learns Minecraft diamonds from offline data alone by training the agent purely inside the world model. The clearest signal yet for 'auto-research inside a world model'.", "aka": "Dreamer 4", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/dreamer_v4.pdf", "abstract": "World models learn general knowledge from videos and simulate experience for training behaviors in imagination, offering a path towards intelligent agents. However, previous world models have been unable to accurately predict object interactions in complex environments. We introduce Dreamer 4, a scalable agent that learns to solve control tasks by reinforcement learning inside of a fast and accurate world model. In the complex video game Minecraft, the world model accurately predicts object interactions and game mechanics, outperforming previous world models by a large margin. The world model achieves real-time interactive inference on a single GPU through a shortcut forcing objective and an efficient transformer architecture. Moreover, the world model learns general action conditioning from only a small amount of data, allowing it to extract the majority of its knowledge from diverse unlabeled videos. We propose the challenge of obtaining diamonds in Minecraft from only offline data, aligning with practical applications such as robotics where learning from environment interaction can be unsafe and slow. This task requires choosing sequences of over 20,000 mouse and keyboard actions from raw pixels. By learning behaviors in imagination, Dreamer 4 is the first agent to obtain diamonds in Minecraft purely from offline data, without environment interaction. Our work provides a scalable recipe for imagination training, marking a step towards intelligent agents.", "source": "curated", "first_author": "Danijar Hafner", "url": "https://arxiv.org/abs/2509.24527", "fig": "figures/dreamerv4.webp", "figs": ["figures/dreamerv4-1.webp"], "arxiv_url": "https://arxiv.org/abs/2509.24527"}, {"id": "vjepa2", "title": "V-JEPA 2: Self-Supervised Video Models Enable Understanding, Prediction and Planning", "tree": "il", "branch": "world-models", "authors": ["Mido Assran", "Adrien Bardes", "Nicolas Ballas", "Michael Rabbat", "Yann LeCun", "et al."], "corresponding": [], "institutions": ["Meta AI"], "lab": "FAIR", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-06-11", "topics": ["self-supervised video", "latent prediction", "zero-shot planning"], "paradigm": ["world model"], "method": ["joint-embedding predictive architecture", "1M+ hours of video", "action-conditioned head"], "projects": ["wam-tactile"], "related": ["being-h07", "cosmos", "dino-wm", "dream-tac", "dreamerv4", "dreamgen", "flare", "genie", "gr2", "lawam", "mobilewam", "motionwam", "navigation-world-models", "oa-wam", "r3m", "vla-jepa"], "arxiv": "2506.09985", "doi": "", "code": "", "site": "", "note": "Predicts in latent space rather than pixels — cheaper and less distracted by texture. Zero-shot pick-and-place on a new lab's robot from 62 hours of unlabelled robot video. The strongest current argument for the latent (not generative) branch of world models.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "A major challenge for modern AI is to learn to understand the world and learn to act largely by observation. This paper explores a self-supervised approach that combines internet-scale video data with a small amount of interaction data (robot trajectories), to develop models capable of understanding, predicting, and planning in the physical world. We first pre-train an action-free joint-embedding-predictive architecture, V-JEPA 2, on a video and image dataset comprising over 1 million hours of internet video. V-JEPA 2 achieves strong performance on motion understanding (77.3 top-1 accuracy on Something-Something v2) and state-of-the-art performance on human action anticipation (39.7 recall-at-5 on Epic-Kitchens-100) surpassing previous task-specific models. Additionally, after aligning V-JEPA 2 with a large language model, we demonstrate state-of-the-art performance on multiple video question-answering tasks at the 8 billion parameter scale (e.g., 84.0 on PerceptionTest, 76.9 on TempCompass). Finally, we show how self-supervised learning can be applied to robotic planning tasks by post-training a latent action-conditioned world model, V-JEPA 2-AC, using less than 62 hours of unlabeled robot videos from the Droid dataset. We deploy V-JEPA 2-AC zero-shot on Franka arms in two different labs and enable picking and placing of objects using planning with image goals. Notably, this is achieved without collecting any data from the robots in these environments, and without any task-specific training or reward. This work demonstrates how self-supervised learning from web-scale data and a small amount of robot interaction data can yield a world model capable of planning in the physical world.", "source": "curated", "first_author": "Mido Assran", "url": "https://arxiv.org/abs/2506.09985", "fig": "figures/vjepa2.webp", "figs": ["figures/vjepa2-1.webp", "figures/vjepa2-2.webp", "figures/vjepa2-3.webp"]}, {"id": "videomimic", "title": "Visual Imitation Enables Contextual Humanoid Control", "tree": "rl", "branch": "motion-imitation", "authors": ["Arthur Allshire", "Hongsuk Choi", "Junyi Zhang", "David McAllister", "Anthony Zhang", "Chung Min Kim", "Trevor Darrell", "Pieter Abbeel", "Jitendra Malik", "Angjoo Kanazawa"], "corresponding": ["Arthur Allshire"], "institutions": ["UC Berkeley"], "lab": "BAIR / KAIR", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-06", "topics": ["real-to-sim", "human video", "terrain-aware humanoid control"], "paradigm": ["RL", "real2sim"], "method": ["monocular video -> 4D reconstruction of human+scene -> RL tracking in reconstructed terrain"], "projects": ["mabel", "r2s2r"], "related": ["animal-avatars", "asap", "deepmimic", "egoengine", "real2sim", "video2sim2real"], "arxiv": "2505.03729", "doi": "", "code": "", "site": "", "note": "Directly on your real-to-sim-to-real-from-egocentric-views idea. Reconstructs both the human motion and the terrain from ordinary video, then trains a policy in that reconstruction. The scene is the part everyone else drops.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/videomimic.pdf", "abstract": "How can we teach humanoids to climb staircases and sit on chairs using the surrounding environment context? Arguably, the simplest way is to just show them-casually capture a human motion video and feed it to humanoids. We introduce VIDEOMIMIC, a real-to-sim-to-real pipeline that mines everyday videos, jointly reconstructs the humans and the environment, and produces whole-body control policies for humanoid robots that perform the corresponding skills. We demonstrate the results of our pipeline on real humanoid robots, showing robust, repeatable contextual control such as staircase ascents and descents, sitting and standing from chairs and benches, as well as other dynamic whole-body skills-all from a single policy, conditioned on the environment and global root commands. VIDEOMIMIC offers a scalable path towards teaching humanoids to operate in diverse real-world environments.", "source": "curated", "first_author": "Arthur Allshire", "url": "https://arxiv.org/abs/2505.03729", "fig": "figures/videomimic.webp", "figs": ["figures/videomimic-1.webp", "figures/videomimic-2.webp"]}, {"id": "wholebodyvla", "title": "WholeBodyVLA: Towards Unified Latent VLA for Whole-Body Loco-Manipulation Control", "tree": "il", "branch": "vla", "authors": ["Jiang", "Haoran", "Chen", "Jin", "Bu", "Qingwen", "Chen", "Li", "Shi", "Modi", "Zhang", "Yanjie", "Li", "Delong", "Suo", "Chuanzhe", "Wang", "Chuang", "Peng", "Zhihui", "Li", "Hongyang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-11", "topics": ["unified latent VLA", "whole-body loco-manipulation"], "paradigm": ["VLA"], "method": [], "projects": ["m2", "mabel", "wam-tactile"], "related": ["chorus", "decowam", "groot-n1", "haic", "motionwam", "umi-on-legs"], "arxiv": "2512.11047", "doi": "10.48550/arxiv.2512.11047", "code": "", "site": "", "note": "ICLR 2026. A unified <b>latent</b> VLA for whole-body loco-manipulation — one model issuing both base and arm commands through a shared latent action space rather than two stacked controllers. If it holds, it collapses the manipulation/locomotion interface that [[umi-on-legs]] carefully separates, which is a real architectural fork for MABEL.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid robots require precise locomotion and dexterous manipulation to perform challenging loco-manipulation tasks. Yet existing approaches, modular or end-to-end, are deficient in manipulation-aware locomotion. This confines the robot to a limited workspace, preventing it from performing large-space loco-manipulation. We attribute this to: (1) the challenge of acquiring loco-manipulation knowledge due to the scarcity of humanoid teleoperation data, and (2) the difficulty of faithfully and reliably executing locomotion commands, stemming from the limited precision and stability of existing RL controllers. To acquire richer loco-manipulation knowledge, we propose a unified latent learning framework that enables Vision-Language-Action (VLA) system to learn from low-cost action-free egocentric videos. Moreover, an efficient human data collection pipeline is devised to augment the dataset and scale the benefits. To execute the desired locomotion commands more precisely, we present a loco-manipulation-oriented (LMO) RL policy specifically tailored for accurate and stable core loco-manipulation movements, such as advancing, turning, and squatting. Building on these components, we introduce WholeBodyVLA, a unified framework for humanoid loco-manipulation. To the best of our knowledge, WholeBodyVLA is one of its kind enabling large-space humanoid loco-manipulation. It is verified via comprehensive experiments on the AgiBot X2 humanoid, outperforming prior baseline by 21.3%. It also demonstrates strong generalization and high extensibility across a broad range of tasks.", "source": "curated", "first_author": "Jiang", "url": "https://arxiv.org/abs/2512.11047", "fig": "figures/wholebodyvla.webp", "figs": ["figures/wholebodyvla-1.webp"]}, {"id": "worldeval", "title": "WorldEval: World Model as Real-World Robot Policies Evaluator", "tree": "systems", "branch": "sim-bench", "authors": ["Yaxuan Li", "Yichen Zhu", "Junjie Wen", "Chaomin Shen", "Yi Xu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-25", "topics": ["world model as policy evaluator", "real-world policy ranking without a robot"], "paradigm": ["world model", "benchmark"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["ev-wm", "genie-envisioner", "interactive-world-sim", "robogaze", "roboworld", "worldarena2"], "arxiv": "2505.19017", "doi": "10.48550/arxiv.2505.19017", "code": "", "site": "", "note": "<b>WorldEval</b> — use the world model to <em>evaluate</em> real-world robot policies, so ranking candidate policies no longer costs robot hours. This is the application of world models most likely to pay off for you soonest: you have more policy variants than you have time on hardware, and this is the literature on whether the ranking transfers.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The field of robotics has made significant strides toward developing generalist robot manipulation policies. However, evaluating these policies in real-world scenarios remains time-consuming and challenging, particularly as the number of tasks scales and environmental conditions change. In this work, we demonstrate that world models can serve as a scalable, reproducible, and reliable proxy for real-world robot policy evaluation. A key challenge is generating accurate policy videos from world models that faithfully reflect the robot actions. We observe that directly inputting robot actions or using high-dimensional encoding methods often fails to generate action-following videos. To address this, we propose Policy2Vec, a simple yet effective approach to turn a video generation model into a world simulator that follows latent action to generate the robot video. We then introduce WorldEval, an automated pipeline designed to evaluate real-world robot policies entirely online. WorldEval effectively ranks various robot policies and individual checkpoints within a single policy, and functions as a safety detector to prevent dangerous actions by newly developed robot models. Through comprehensive paired evaluations of manipulation policies in real-world environments, we demonstrate a strong correlation between policy performance in WorldEval and real-world scenarios. Furthermore, our method significantly outperforms popular methods such as real-to-sim approach.", "source": "curated", "first_author": "Yaxuan Li", "url": "https://arxiv.org/abs/2505.19017", "fig": "figures/worldeval.webp", "figs": ["figures/worldeval-1.webp"]}, {"id": "pi0", "title": "$π_0$: A Vision-Language-Action Flow Model for General Robot Control", "tree": "il", "branch": "vla", "authors": ["Kevin Black", "Noah Brown", "Danny Driess", "Adnan Esmail", "Michael Equi", "Chelsea Finn", "Niccolo Fusai", "et al."], "corresponding": [], "institutions": ["Physical Intelligence"], "lab": "Physical Intelligence (π)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-10-31", "topics": ["flow matching VLA", "generalist control", "dexterity"], "paradigm": ["VLA", "flow matching"], "method": ["PaliGemma VLM + flow-matching action expert at 50 Hz", "trained on 10k+ hours"], "projects": ["iris"], "related": ["act", "audio-vla", "dexvla", "dp", "dvla", "groot-n1", "helix", "iRe-VLA", "knowledge-insulation", "libero", "manflow", "openvla", "openvla-oft", "pi05", "pi06", "rdt1b", "ript-vla", "rt1", "rt2", "vlm4vla"], "arxiv": "2410.24164", "doi": "", "code": "", "site": "", "note": "The model that made continuous high-frequency VLA action heads standard. Flow matching gives you 50 Hz dexterous output that discretized tokens cannot reach — laundry folding as the headline task.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Robot learning holds tremendous promise to unlock the full potential of flexible, general, and dexterous robot systems, as well as to address some of the deepest questions in artificial intelligence. However, bringing robot learning to the level of generality required for effective real-world systems faces major obstacles in terms of data, generalization, and robustness. In this paper, we discuss how generalist robot policies (i.e., robot foundation models) can address these challenges, and how we can design effective generalist robot policies for complex and highly dexterous tasks. We propose a novel flow matching architecture built on top of a pre-trained vision-language model (VLM) to inherit Internet-scale semantic knowledge. We then discuss how this model can be trained on a large and diverse dataset from multiple dexterous robot platforms, including single-arm robots, dual-arm robots, and mobile manipulators. We evaluate our model in terms of its ability to perform tasks in zero shot after pre-training, follow language instructions from people and from a high-level VLM policy, and its ability to acquire new skills via fine-tuning. Our results cover a wide variety of tasks, such as laundry folding, table cleaning, and assembling boxes.", "source": "curated", "first_author": "Kevin Black", "url": "https://doi.org/10.5281/zenodo.20109732", "arxiv_url": "https://arxiv.org/abs/2410.24164", "fig": "figures/pi0.webp", "figs": ["figures/pi0-1.webp", "figures/pi0-2.webp", "figures/pi0-3.webp"]}, {"id": "ace-teleop", "title": "ACE: A Cross-Platform Visual-Exoskeletons System for Low-Cost Dexterous Teleoperation", "tree": "systems", "branch": "interfaces", "authors": ["Shiqi Yang", "Minghuan Liu", "Yuzhe Qin", "Runyu Ding", "Jialong Li", "Xuxin Cheng", "Ruihan Yang", "Sha Yi", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-08-21", "topics": ["cross-platform exoskeleton", "hand-facing camera", "low cost"], "paradigm": ["teleop"], "method": [], "projects": ["exo", "geodex", "mabel"], "related": ["anyteleop", "dexumi", "exo-viha", "gello", "homie"], "arxiv": "2408.11805", "doi": "10.48550/arxiv.2408.11805", "code": "", "site": "", "note": "One rig that teleoperates humanoid hands, arm-hand, arm-gripper and quadruped-gripper systems — a hand-facing camera for finger pose plus a portable exoskeleton base for the wrist. The cross-platform claim is the valuable part for MABEL: build the interface once, retarget to whatever the robot becomes.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Learning from demonstrations has shown to be an effective approach to robotic manipulation, especially with the recently collected large-scale robot data with teleoperation systems. Building an efficient teleoperation system across diverse robot platforms has become more crucial than ever. However, there is a notable lack of cost-effective and user-friendly teleoperation systems for different end-effectors, e.g., anthropomorphic robot hands and grippers, that can operate across multiple platforms. To address this issue, we develop ACE, a cross-platform visual-exoskeleton system for low-cost dexterous teleoperation. Our system utilizes a hand-facing camera to capture 3D hand poses and an exoskeleton mounted on a portable base, enabling accurate real-time capture of both finger and wrist poses. Compared to previous systems, which often require hardware customization according to different robots, our single system can generalize to humanoid hands, arm-hands, arm-gripper, and quadruped-gripper systems with high-precision teleoperation. This enables imitation learning for complex manipulation tasks on diverse platforms.", "source": "curated", "first_author": "Shiqi Yang", "url": "https://arxiv.org/abs/2408.11805", "fig": "figures/ace-teleop.webp", "figs": ["figures/ace-teleop-1.webp", "figures/ace-teleop-2.webp"]}, {"id": "adaptigraph", "title": "AdaptiGraph: Material-Adaptive Graph-Based Neural Dynamics for Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Kaifeng Zhang", "Baoyu Li", "Kris Hauser", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-07-10", "topics": ["material-adaptive dynamics", "online physical property inference", "GNN"], "paradigm": ["model-based RL"], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": ["bab-nd", "causal-wm", "d3fields", "egophys", "gs-dynamics", "particle-grid-dynamics", "phystwin", "physworld", "robocook", "robocraft"], "arxiv": "2407.07889", "doi": "10.48550/arXiv.2407.07889", "code": "", "site": "", "note": "<b>AdaptiGraph</b> (RSS 2024) — the piece the pure-video world models do not have: a dynamics model with an explicit <b>material latent</b> that is inferred online from a short interaction, so one model covers ropes, cloth and granular piles and <em>adapts</em> when the physical properties are unknown. If you are serious about 'close the loop' in real2sim2real, this is the closest published answer to what closing it means: the sim's parameters are estimated from the robot's own poking, continuously, not fitted once.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Predictive models are a crucial component of many robotic systems. Yet, constructing accurate predictive models for a variety of deformable objects, especially those with unknown physical properties, remains a significant challenge. This paper introduces AdaptiGraph, a learning-based dynamics modeling approach that enables robots to predict, adapt to, and control a wide array of challenging deformable materials with unknown physical properties. AdaptiGraph leverages the highly flexible graph-based neural dynamics (GBND) framework, which represents material bits as particles and employs a graph neural network (GNN) to predict particle motion. Its key innovation is a unified physical property-conditioned GBND model capable of predicting the motions of diverse materials with varying physical properties without retraining. Upon encountering new materials during online deployment, AdaptiGraph utilizes a physical property optimization process for a few-shot adaptation of the model, enhancing its fit to the observed interaction data. The adapted models can precisely simulate the dynamics and predict the motion of various deformable materials, such as ropes, granular media, rigid boxes, and cloth, while adapting to different physical properties, including stiffness, granular size, and center of pressure. On prediction and manipulation tasks involving a diverse set of real-world deformable objects, our method exhibits superior prediction accuracy and task proficiency over non-material-conditioned and non-adaptive models. The project page is available at https://robopil.github.io/adaptigraph/ .", "source": "curated", "first_author": "Kaifeng Zhang", "url": "https://arxiv.org/abs/2407.07889", "fig": "figures/adaptigraph.webp", "figs": ["figures/adaptigraph-1.webp", "figures/adaptigraph-2.webp"]}, {"id": "anyskin", "title": "AnySkin: Plug-and-play Skin Sensing for Robotic Touch", "tree": "systems", "branch": "tactile-hw", "authors": ["Raunaq Bhirangi", "Venkatesh Pattabiraman", "Enes Erciyes", "Yifeng Cao", "Tess Hellebrekers", "Lerrel Pinto"], "corresponding": ["Raunaq Bhirangi"], "institutions": ["New York University", "Meta AI"], "lab": "NYU GRAIL (Lerrel Pinto)", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-09-12", "topics": ["magnetic skin", "replaceable", "cross-instance generalization"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["dexskin", "digit", "egoemg", "gelsight", "miccheck", "reskin", "robotacdex", "rotate-without-seeing", "sonicsense"], "arxiv": "2409.08276", "doi": "", "code": "", "site": "", "note": "Magnetic rather than optical, and — the actual contribution — *replaceable without recalibration*, so policies transfer across sensor instances. That is the property that makes tactile learning reproducible. NYU lab, relevant to you.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "While tactile sensing is widely accepted as an important and useful sensing modality, its use pales in comparison to other sensory modalities like vision and proprioception. AnySkin addresses the critical challenges that impede the use of tactile sensing -- versatility, replaceability, and data reusability. Building on the simplistic design of ReSkin, and decoupling the sensing electronics from the sensing interface, AnySkin simplifies integration making it as straightforward as putting on a phone case and connecting a charger. Furthermore, AnySkin is the first uncalibrated tactile-sensor with cross-instance generalizability of learned manipulation policies. To summarize, this work makes three key contributions: first, we introduce a streamlined fabrication process and a design tool for creating an adhesive-free, durable and easily replaceable magnetic tactile sensor; second, we characterize slip detection and policy learning with the AnySkin sensor; and third, we demonstrate zero-shot generalization of models trained on one instance of AnySkin to new instances, and compare it with popular existing tactile solutions like DIGIT and ReSkin. Videos of experiments, fabrication details and design files can be found on https://any-skin.github.io/", "source": "curated", "first_author": "Raunaq Bhirangi", "url": "https://arxiv.org/abs/2409.08276", "fig": "figures/anyskin.webp", "figs": ["figures/anyskin-1.webp", "figures/anyskin-2.webp"]}, {"id": "acdc", "title": "Automated Creation of Digital Cousins for Robust Policy Learning", "tree": "systems", "branch": "sim-bench", "authors": ["Tianyuan Dai", "Josiah Wong", "Yunfan Jiang", "Chen Wang", "Cem Gökmen", "Ruohan Zhang", "Jiajun Wu", "Feifei Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-10-09", "topics": ["digital cousins", "automated scene creation", "asset substitution"], "paradigm": ["simulator"], "method": [], "projects": ["r2s2r"], "related": ["egoengine", "embodied-sim-survey", "physgen3d", "phystwin", "physworld", "prism", "r2s-ego", "robo-exp", "robodream", "robosnap", "simfoundry", "voxposer"], "arxiv": "2410.07408", "doi": "10.48550/arxiv.2410.07408", "code": "", "site": "", "note": "ACDC introduces <b>digital cousins</b> — scenes that preserve a real reference's structure and affordances while substituting different asset instances, rather than trying to twin it exactly. That is the key reframing for agentic scene construction: an exact twin is expensive and brittle, a cousin is cheap and <em>better for transfer</em>, because training across cousins is domain randomization with semantics. Stanford.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Training robot policies in the real world can be unsafe, costly, and difficult to scale. Simulation serves as an inexpensive and potentially limitless source of training data, but suffers from the semantics and physics disparity between simulated and real-world environments. These discrepancies can be minimized by training in digital twins, which serve as virtual replicas of a real scene but are expensive to generate and cannot produce cross-domain generalization. To address these limitations, we propose the concept of digital cousins, a virtual asset or scene that, unlike a digital twin, does not explicitly model a real-world counterpart but still exhibits similar geometric and semantic affordances. As a result, digital cousins simultaneously reduce the cost of generating an analogous virtual environment while also facilitating better robustness during sim-to-real domain transfer by providing a distribution of similar training scenes. Leveraging digital cousins, we introduce a novel method for their automated creation, and propose a fully automated real-to-sim-to-real pipeline for generating fully interactive scenes and training robot policies that can be deployed zero-shot in the original scene. We find that digital cousin scenes that preserve geometric and semantic affordances can be produced automatically, and can be used to train policies that outperform policies trained on digital twins, achieving 90% vs. 25% success rates under zero-shot sim-to-real transfer. Additional details are available at https://digital-cousins.github.io/.", "source": "curated", "first_author": "Tianyuan Dai", "url": "https://arxiv.org/abs/2410.07408", "fig": "figures/acdc.webp", "figs": ["figures/acdc-1.webp"]}, {"id": "dino-wm", "title": "DINO-WM: World Models on Pre-trained Visual Features enable Zero-shot Planning", "tree": "rl", "branch": "model-based", "authors": ["Gaoyue Zhou", "Hengkai Pan", "Yann LeCun", "Lerrel Pinto"], "corresponding": [], "institutions": [], "lab": "NYU CILVR", "venue": "ICML", "venue_type": "conference", "year": 2024, "date": "2024-11-07", "topics": ["frozen DINOv2 features", "zero-shot planning", "no reward model"], "paradigm": ["model-based RL"], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": ["causal-wm", "dreamerv3", "navdreamer", "navigation-world-models", "pointworld", "td-mpc", "vjepa2", "vla-jepa"], "arxiv": "2411.04983", "doi": "10.48550/arXiv.2411.04983", "code": "", "site": "", "note": "<b>DINO-WM</b> — do not learn a representation at all. Freeze DINOv2 patch features, learn dynamics <em>on top of them</em>, and plan to a goal specified as a target feature map. No reconstruction, no reward, no task-specific training, and it still plans zero-shot. This is the single most practical foundational world model in this list for someone who wants a working baseline this month: the representation is already trained, and you only pay for the dynamics head.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The ability to predict future outcomes given control actions is fundamental for physical reasoning. However, such predictive models, often called world models, remains challenging to learn and are typically developed for task-specific solutions with online policy learning. To unlock world models' true potential, we argue that they should 1) be trainable on offline, pre-collected trajectories, 2) support test-time behavior optimization, and 3) facilitate task-agnostic reasoning. To this end, we present DINO World Model (DINO-WM), a new method to model visual dynamics without reconstructing the visual world. DINO-WM leverages spatial patch features pre-trained with DINOv2, enabling it to learn from offline behavioral trajectories by predicting future patch features. This allows DINO-WM to achieve observational goals through action sequence optimization, facilitating task-agnostic planning by treating goal features as prediction targets. We demonstrate that DINO-WM achieves zero-shot behavioral solutions at test time on six environments without expert demonstrations, reward modeling, or pre-learned inverse models, outperforming prior state-of-the-art work across diverse task families such as arbitrarily configured mazes, push manipulation with varied object shapes, and multi-particle scenarios.", "source": "curated", "first_author": "Gaoyue Zhou", "url": "https://arxiv.org/abs/2411.04983", "fig": "figures/dino-wm.webp", "figs": ["figures/dino-wm-1.webp"]}, {"id": "droid", "title": "DROID: A Large-Scale In-The-Wild Robot Manipulation Dataset", "tree": "il", "branch": "data-scaling", "authors": ["Alexander Khazatsky", "Karl Pertsch", "Suraj Nair", "Ashwin Balakrishna", "Sudeep Dasari", "et al."], "corresponding": [], "institutions": ["Stanford University", "UC Berkeley", "13 institutions"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-03-19", "topics": ["in-the-wild dataset", "Franka", "diversity"], "paradigm": ["dataset"], "method": [], "projects": [], "related": ["agibot-world", "bridgedata", "oxe", "roboworld", "umi"], "arxiv": "2403.12945", "doi": "", "code": "", "site": "", "note": "76k trajectories across 564 scenes, collected with one standardized hardware setup. Standardization is what makes it more useful per-episode than OXE.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The creation of large, diverse, high-quality robot manipulation datasets is an important stepping stone on the path toward more capable and robust robotic manipulation policies. However, creating such datasets is challenging: collecting robot manipulation data in diverse environments poses logistical and safety challenges and requires substantial investments in hardware and human labour. As a result, even the most general robot manipulation policies today are mostly trained on data collected in a small number of environments with limited scene and task diversity. In this work, we introduce DROID (Distributed Robot Interaction Dataset), a diverse robot manipulation dataset with 76k demonstration trajectories or 350 hours of interaction data, collected across 564 scenes and 84 tasks by 50 data collectors in North America, Asia, and Europe over the course of 12 months. We demonstrate that training with DROID leads to policies with higher performance and improved generalization ability. We open source the full dataset, policy learning code, and a detailed guide for reproducing our robot hardware setup.", "source": "curated", "first_author": "Alexander Khazatsky", "url": "https://arxiv.org/abs/2403.12945", "fig": "figures/droid.webp", "figs": ["figures/droid-1.webp", "figures/droid-2.webp"]}, {"id": "disney-bdx", "title": "Design and Control of a Bipedal Robotic Character", "tree": "systems", "branch": "humanoid-hw", "authors": ["Ruben Grandia", "Espen Knoop", "Michael A. Hopkins", "Georg Wiedebach", "Jared Bishop", "Steven Pickles", "David Müller", "Moritz Bächer"], "corresponding": ["Ruben Grandia"], "institutions": ["Disney Research"], "lab": "Disney Research Zurich", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "", "topics": ["robotic character", "expressive motion", "bipedal design"], "paradigm": ["hardware", "RL"], "method": [], "projects": ["iris", "mabel"], "related": ["amp", "berkeley-humanoid", "coros-codesign", "deepmimic"], "arxiv": "", "doi": "10.15607/rss.2024.xx.103", "code": "", "site": "", "note": "In your MABEL hardware and CRV_iris folders. The BD-X droid — the best existing example of combining stylized, expressive motion with real bipedal robustness. Relevant to MABEL's presence/character dimension, which most research platforms ignore entirely.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CRV_iris/disney_biped.pdf", "abstract": "Legged robots have achieved impressive feats in dynamic locomotion in challenging unstructured terrain. However, in entertainment applications, the design and control of these robots face additional challenges in appealing to human audiences. This work aims to unify expressive, artist-directed motions and robust dynamic mobility for legged robots. To this end, we introduce a new bipedal robot, designed with a focus on character-driven mechanical features. We present a reinforcement learning-based control architecture to robustly execute artistic motions conditioned on command signals. During runtime, these command signals are generated by an animation engine which composes and blends between multiple animation sources. Finally, an intuitive operator interface enables real-time show performances with the robot. The complete system results in a believable robotic character, and paves the way for enhanced human-robot engagement in various contexts, in entertainment robotics and beyond.", "source": "curated", "first_author": "Ruben Grandia", "url": "https://doi.org/10.15607/rss.2024.xx.103", "fig": "figures/disney-bdx.webp", "figs": ["figures/disney-bdx-1.webp", "figures/disney-bdx-2.webp", "figures/disney-bdx-3.webp"]}, {"id": "dexcap", "title": "DexCap: Scalable and Portable Mocap Data Collection System for Dexterous Manipulation", "tree": "systems", "branch": "interfaces", "authors": ["Chen Wang", "Haochen Shi", "Weizhuo Wang", "Ruohan Zhang", "Li Fei-Fei", "C. Karen Liu"], "corresponding": ["Chen Wang"], "institutions": ["Stanford University"], "lab": "Stanford Vision & Learning (SVL)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-03-12", "topics": ["portable mocap", "glove capture", "dexterous data"], "paradigm": ["hardware", "IL"], "method": ["Rokoko glove + LiDAR + SLAM chest rig", "human-in-the-loop correction during deployment"], "projects": ["geodex", "r2s2r", "wam-tactile"], "related": ["dexmv", "dexumi", "doglove", "egokit", "egomimic", "realdexumi", "ruka", "sunday-gelato", "umi", "videodex"], "arxiv": "2403.07788", "doi": "", "code": "", "site": "", "note": "In your library twice (Gloves/ and Dexterous Manipulation Sim-to-real/). The portable glove+SLAM rig is the most direct hardware precedent for 'world-action model from UMI gloves'. Its residual correction mechanism is also worth noting — human fixes at deployment time, folded back into training.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/Dexterous Manipulation Sim-to-real/DexCap.pdf", "abstract": "Imitation learning from human hand motion data presents a promising avenue for imbuing robots with human-like dexterity in real-world manipulation tasks. Despite this potential, substantial challenges persist, particularly with the portability of existing hand motion capture (mocap) systems and the complexity of translating mocap data into effective robotic policies. To tackle these issues, we introduce DexCap, a portable hand motion capture system, alongside DexIL, a novel imitation algorithm for training dexterous robot skills directly from human hand mocap data. DexCap offers precise, occlusion-resistant tracking of wrist and finger motions based on SLAM and electromagnetic field together with 3D observations of the environment. Utilizing this rich dataset, DexIL employs inverse kinematics and point cloud-based imitation learning to seamlessly replicate human actions with robot hands. Beyond direct learning from human motion, DexCap also offers an optional human-in-the-loop correction mechanism during policy rollouts to refine and further improve task performance. Through extensive evaluation across six challenging dexterous manipulation tasks, our approach not only demonstrates superior performance but also showcases the system's capability to effectively learn from in-the-wild mocap data, paving the way for future data collection methods in the pursuit of human-level robot dexterity. More details can be found at https://dex-cap.github.io", "source": "curated", "first_author": "Chen Wang", "url": "https://arxiv.org/abs/2403.07788", "fig": "figures/dexcap.webp", "figs": ["figures/dexcap-1.webp", "figures/dexcap-2.webp", "figures/dexcap-3.webp"]}, {"id": "eureka", "title": "Eureka: Human-Level Reward Design via Coding Large Language Models", "tree": "systems", "branch": "agentic", "authors": ["Yecheng Jason Ma", "William Liang", "Guanzhi Wang", "De-An Huang", "Osbert Bastani", "Dinesh Jayaraman", "Yuke Zhu", "Linxi Fan", "Anima Anandkumar"], "corresponding": ["Yecheng Jason Ma"], "institutions": ["NVIDIA", "University of Pennsylvania", "Caltech", "UT Austin"], "lab": "NVIDIA GEAR", "venue": "ICLR", "venue_type": "conference", "year": 2024, "date": "2023-10-19", "topics": ["LLM reward design", "evolutionary search", "pen spinning"], "paradigm": ["LLM", "RL"], "method": ["GPT-4 writes reward code", "evaluates against RL training statistics", "mutates"], "projects": ["geodex"], "related": ["ai-scientist", "dreureka", "harbor", "rf-agent", "robogrammar", "robomorph", "text2robot", "voyager"], "arxiv": "2310.12931", "doi": "", "code": "", "site": "", "note": "The clearest existing template for agentic auto-research in robotics: the LLM writes reward *code*, trains a policy, reads the training curves, and rewrites. Pen spinning on a Shadow Hand is the headline. This is the seed of your dexterous-auto-research idea.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Large Language Models (LLMs) have excelled as high-level semantic planners for sequential decision-making tasks. However, harnessing them to learn complex low-level manipulation tasks, such as dexterous pen spinning, remains an open problem. We bridge this fundamental gap and present Eureka, a human-level reward design algorithm powered by LLMs. Eureka exploits the remarkable zero-shot generation, code-writing, and in-context improvement capabilities of state-of-the-art LLMs, such as GPT-4, to perform evolutionary optimization over reward code. The resulting rewards can then be used to acquire complex skills via reinforcement learning. Without any task-specific prompting or pre-defined reward templates, Eureka generates reward functions that outperform expert human-engineered rewards. In a diverse suite of 29 open-source RL environments that include 10 distinct robot morphologies, Eureka outperforms human experts on 83% of the tasks, leading to an average normalized improvement of 52%. The generality of Eureka also enables a new gradient-free in-context learning approach to reinforcement learning from human feedback (RLHF), readily incorporating human inputs to improve the quality and the safety of the generated rewards without model updating. Finally, using Eureka rewards in a curriculum learning setting, we demonstrate for the first time, a simulated Shadow Hand capable of performing pen spinning tricks, adeptly manipulating a pen in circles at rapid speed.", "source": "curated", "first_author": "Yecheng Jason Ma", "url": "https://arxiv.org/abs/2310.12931", "fig": "figures/eureka.webp", "figs": ["figures/eureka-1.webp", "figures/eureka-2.webp"]}, {"id": "exbody2", "title": "ExBody2: Advanced Expressive Humanoid Whole-Body Control", "tree": "rl", "branch": "motion-imitation", "authors": ["Mazeyu Ji", "Peng", "Xuanbin", "Fangchen Liu", "Jialong Li", "Ge Yang", "Xuxin Cheng", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-12-17", "topics": ["expressive whole-body tracking", "velocity decoupling", "teacher filtering"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["adaptive-tracking", "amo", "cross-humanoid-wbc", "exbody", "gmr", "handoff", "hover", "hugwbc", "kungfu-bot", "resmimic", "stablemimic", "twist"], "arxiv": "2412.13196", "doi": "10.48550/arxiv.2412.13196", "code": "", "site": "", "note": "The successor to ExBody, and the current reference for general whole-body motion tracking. Two ideas carry: decouple whole-body velocity tracking from body-landmark tracking, and use a teacher policy to <b>filter out motions the robot physically cannot do</b> before training on them. That filtering step is why most tracking pipelines fail on raw mocap.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "This paper tackles the challenge of enabling real-world humanoid robots to perform expressive and dynamic whole-body motions while maintaining overall stability and robustness. We propose Advanced Expressive Whole-Body Control (Exbody2), a method for producing whole-body tracking controllers that are trained on both human motion capture and simulated data and then transferred to the real world. We introduce a technique for decoupling the velocity tracking of the entire body from tracking body landmarks. We use a teacher policy to produce intermediate data that better conforms to the robot's kinematics and to automatically filter away infeasible whole-body motions. This two-step approach enabled us to produce a student policy that can be deployed on the robot that can walk, crouch, and dance. We also provide insight into the trade-off between versatility and the tracking performance on specific motions. We observed significant improvement of tracking performance after fine-tuning on a small amount of data, at the expense of the others.", "source": "curated", "first_author": "Mazeyu Ji", "url": "https://arxiv.org/abs/2412.13196", "fig": "figures/exbody2.webp", "figs": ["figures/exbody2-1.webp"]}, {"id": "residual-assembly", "title": "From Imitation to Refinement -- Residual RL for Precise Assembly", "tree": "rl", "branch": "manipulation", "authors": ["Lars Ankile", "Anthony Simeonov", "Idan Shenfeld", "Marcel Torne", "Pulkit Agrawal"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-07-23", "topics": ["residual RL on top of imitation", "precise assembly"], "paradigm": ["IL", "RL"], "method": [], "projects": ["r2s2r"], "related": ["far-dex", "hil-serl", "pld", "video2sim2real"], "arxiv": "2407.16677", "doi": "10.48550/arxiv.2407.16677", "code": "", "site": "", "note": "'From Imitation to Refinement' — the clearest statement of the division of labour: imitation gets you the coarse trajectory, residual RL supplies the precision that demonstrations cannot, on assembly tasks where tolerances are sub-millimetre. Read it for the argument, not the task.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Recent advances in Behavior Cloning (BC) have made it easy to teach robots new tasks. However, we find that the ease of teaching comes at the cost of unreliable performance that saturates with increasing data for tasks requiring precision. The performance saturation can be attributed to two critical factors: (a) distribution shift resulting from the use of offline data and (b) the lack of closed-loop corrective control caused by action chucking (predicting a set of future actions executed open-loop) critical for BC performance. Our key insight is that by predicting action chunks, BC policies function more like trajectory \"planners\" than closed-loop controllers necessary for reliable execution. To address these challenges, we devise a simple yet effective method, ResiP (Residual for Precise Manipulation), that overcomes the reliability problem while retaining BC's ease of teaching and long-horizon capabilities. ResiP augments a frozen, chunked BC model with a fully closed-loop residual policy trained with reinforcement learning (RL) that addresses distribution shifts and introduces closed-loop corrections over open-loop execution of action chunks predicted by the BC trajectory planner. Videos, code, and data: https://residual-assembly.github.io.", "source": "curated", "first_author": "Lars Ankile", "url": "https://arxiv.org/abs/2407.16677", "fig": "figures/residual-assembly.webp", "figs": ["figures/residual-assembly-1.webp", "figures/residual-assembly-2.webp", "figures/residual-assembly-3.webp"]}, {"id": "gello", "title": "GELLO: A General, Low-Cost, and Intuitive Teleoperation Framework for Robot Manipulators", "tree": "systems", "branch": "bimanual", "authors": ["Philipp Wu", "Yide Shentu", "Zhongke Yi", "Xingyu Lin", "Pieter Abbeel"], "corresponding": ["Philipp Wu"], "institutions": ["UC Berkeley"], "lab": "BAIR", "venue": "IROS", "venue_type": "conference", "year": 2024, "date": "2023-09-24", "topics": ["kinematic-twin teleoperation", "low cost"], "paradigm": ["hardware", "teleop"], "method": ["3D-printed scaled kinematic replica with encoders as the leader device"], "projects": ["iris", "mabel"], "related": ["ace-teleop", "aloha", "modpack", "open-television", "so100", "telemoma", "umi"], "arxiv": "2309.13037", "doi": "", "code": "", "site": "", "note": "In your CRV_iris/imitation folder. A $300 printed kinematic twin beats VR and spacemouse on both success rate and operator preference. If you build a leader device for MABEL, start here.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/CRV_iris/imitation/2309.13037v2.pdf", "abstract": "Humans can teleoperate robots to accomplish complex manipulation tasks. Imitation learning has emerged as a powerful framework that leverages human teleoperated demonstrations to teach robots new skills. However, the performance of the learned policies is bottlenecked by the quality, scale, and variety of the demonstration data. In this paper, we aim to lower the barrier to collecting large and high-quality human demonstration data by proposing a GEneraL framework for building LOw-cost and intuitive teleoperation systems for robotic manipulation (GELLO). Given a target robot arm, we build a GELLO controller device that has the same kinematic structure as the target arm, leveraging 3D-printed parts and economical off-the-shelf motors. GELLO is easy to build and intuitive to use. Through an extensive user study, we show that GELLO enables more reliable and efficient demonstration collection compared to other cost efficient teleoperation devices commonly used in the imitation learning literature such as virtual reality controllers and 3D spacemouses. We further demonstrate the capabilities of GELLO for performing complex bi-manual and contact-rich manipulation tasks. To make GELLO accessible to everyone, we have designed and built GELLO systems for 3 commonly used robotic arms: Franka, UR5, and xArm. All software and hardware are open-sourced and can be found on our website: https://wuphilipp.github.io/gello/.", "source": "curated", "first_author": "Philipp Wu", "url": "https://arxiv.org/abs/2309.13037", "fig": "figures/gello.webp", "figs": ["figures/gello-1.webp", "figures/gello-2.webp", "figures/gello-3.webp"]}, {"id": "genie", "title": "Genie: Generative Interactive Environments", "tree": "il", "branch": "world-models", "authors": ["Jake Bruce", "Michael Dennis", "Ashley Edwards", "Jack Parker-Holder", "Yuge Shi", "et al."], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind", "venue": "ICML", "venue_type": "conference", "year": 2024, "date": "2024-02-23", "topics": ["generative environments", "latent actions", "unlabelled video"], "paradigm": ["world model"], "method": ["latent action model learned from unlabelled internet video"], "projects": ["r2s2r", "wam-tactile"], "related": ["adaworld", "dreamgen", "genie-envisioner", "genie3", "ivideogpt", "robocurate", "unipi", "vjepa2", "worldmodels"], "arxiv": "2402.15391", "doi": "", "code": "", "site": "", "note": "Learns a *latent action space* from unlabelled video with no action labels at all. That trick is the key enabler for building world-action models from human video — including UMI-glove capture.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce Genie, the first generative interactive environment trained in an unsupervised manner from unlabelled Internet videos. The model can be prompted to generate an endless variety of action-controllable virtual worlds described through text, synthetic images, photographs, and even sketches. At 11B parameters, Genie can be considered a foundation world model. It is comprised of a spatiotemporal video tokenizer, an autoregressive dynamics model, and a simple and scalable latent action model. Genie enables users to act in the generated environments on a frame-by-frame basis despite training without any ground-truth action labels or other domain-specific requirements typically found in the world model literature. Further the resulting learned latent action space facilitates training agents to imitate behaviors from unseen videos, opening the path for training generalist agents of the future.", "source": "curated", "first_author": "Jake Bruce", "url": "https://arxiv.org/abs/2402.15391", "fig": "figures/genie.webp", "figs": ["figures/genie-1.webp", "figures/genie-2.webp", "figures/genie-3.webp"]}, {"id": "hearing-touch", "title": "Hearing Touch: Audio-Visual Pretraining for Contact-Rich Manipulation", "tree": "il", "branch": "tactile-learn", "authors": ["Jared Mejia", "Victoria Dean", "Tess Hellebrekers", "Abhinav Gupta"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-05-14", "topics": ["audio-visual pretraining", "contact-rich manipulation"], "paradigm": ["representation learning", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["av-contact", "dj-mix-analysis", "maniwav", "r3m", "sonicsense", "vibecheck"], "arxiv": "2405.08576", "doi": "10.48550/arxiv.2405.08576", "code": "", "site": "", "note": "Audio-visual pretraining with piezoelectric contact microphones, and the result that matters: sound improves generalisation most in the <b>low-data</b> regime and under texture change. For a task defined by friction against a rotating surface, that is exactly the failure mode you need covered.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Although pre-training on a large amount of data is beneficial for robot learning, current paradigms only perform large-scale pretraining for visual representations, whereas representations for other modalities are trained from scratch. In contrast to the abundance of visual data, it is unclear what relevant internet-scale data may be used for pretraining other modalities such as tactile sensing. Such pretraining becomes increasingly crucial in the low-data regimes common in robotics applications. In this paper, we address this gap by using contact microphones as an alternative tactile sensor. Our key insight is that contact microphones capture inherently audio-based information, allowing us to leverage large-scale audio-visual pretraining to obtain representations that boost the performance of robotic manipulation. To the best of our knowledge, our method is the first approach leveraging large-scale multisensory pre-training for robotic manipulation. For supplementary information including videos of real robot experiments, please see https://sites.google.com/view/hearing-touch.", "source": "curated", "first_author": "Jared Mejia", "url": "https://arxiv.org/abs/2405.08576", "fig": "figures/hearing-touch.webp", "figs": ["figures/hearing-touch-1.webp", "figures/hearing-touch-2.webp", "figures/hearing-touch-3.webp"]}, {"id": "lapa", "title": "Latent Action Pretraining from Videos", "tree": "il", "branch": "human-video", "authors": ["Seonghyeon Ye", "Joel Jang", "Byeongguk Jeon", "Sejune Joo", "Jianwei Yang", "Baolin Peng", "Ajay Mandlekar", "Reuben Tan", "Yu-Wei Chao", "Bill Yuchen Lin", "Lars Liden", "Kimin Lee", "Jianfeng Gao", "Luke Zettlemoyer", "Dieter Fox", "Minjoon Seo"], "corresponding": [], "institutions": ["NVIDIA", "KAIST", "University of Washington", "Microsoft Research"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2024, "date": "2024-10-15", "topics": ["latent action quantization", "VQ-VAE over frame transitions", "action-free pretraining"], "paradigm": ["IL", "VLA"], "method": [], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["adaworld", "dreamdojo", "ego2robot", "lawam", "lawm", "lawm-3d"], "arxiv": "2410.11758", "doi": "10.48550/arXiv.2410.11758", "code": "", "site": "", "note": "<b>LAPA</b> — Latent Action Pretraining from Videos. Learn a discrete latent action space from frame <em>transitions</em> with a VQ-VAE-style objective, pretrain a VLA to predict those latent actions on action-free human video, then map latents to real actions with a small labelled set. This is the canonical answer to 'how do I use 44,000 hours of video that has no action labels', and the mechanism DreamDojo later scales up with continuous rather than discrete latents.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce Latent Action Pretraining for general Action models (LAPA), an unsupervised method for pretraining Vision-Language-Action (VLA) models without ground-truth robot action labels. Existing Vision-Language-Action models require action labels typically collected by human teleoperators during pretraining, which significantly limits possible data sources and scale. In this work, we propose a method to learn from internet-scale videos that do not have robot action labels. We first train an action quantization model leveraging VQ-VAE-based objective to learn discrete latent actions between image frames, then pretrain a latent VLA model to predict these latent actions from observations and task descriptions, and finally finetune the VLA on small-scale robot manipulation data to map from latent to robot actions. Experimental results demonstrate that our method significantly outperforms existing techniques that train robot manipulation policies from large-scale videos. Furthermore, it outperforms the state-of-the-art VLA model trained with robotic action labels on real-world manipulation tasks that require language conditioning, generalization to unseen objects, and semantic generalization to unseen instructions. Training only on human manipulation videos also shows positive transfer, opening up the potential for leveraging web-scale data for robotics foundation model.", "source": "curated", "first_author": "Seonghyeon Ye", "url": "https://arxiv.org/abs/2410.11758", "fig": "figures/lapa.webp", "figs": ["figures/lapa-1.webp"]}, {"id": "so100", "title": "LeRobot / SO-100 & SO-101: Standard Open Arm", "tree": "systems", "branch": "bimanual", "authors": ["Rémi Cadène", "Simon Alibert", "Alexander Soare", "Quentin Gallouédec", "Adil Zouitine", "Thomas Wolf", "The Robot Studio"], "corresponding": [], "institutions": ["Hugging Face", "The Robot Studio"], "lab": "Hugging Face LeRobot", "venue": "open hardware", "venue_type": "software", "year": 2024, "date": "", "topics": ["ultra-low-cost arm", "open source", "community"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "mabel"], "related": ["aloha", "berkeley-humanoid-lite", "gello", "koch", "openpyro"], "arxiv": "", "doi": "", "code": "https://github.com/huggingface/lerobot", "site": "", "note": "A ~$120 printable 5-DOF arm with a maintained learning stack behind it. The cost floor dropped by two orders of magnitude from ALOHA, which changes who can run experiments at all. SO-101 fixes the wiring and assembly pain of SO-100.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Rémi Cadène", "url": "https://github.com/huggingface/lerobot"}, {"id": "h2o", "title": "Learning Human-to-Humanoid Real-Time Whole-Body Teleoperation", "tree": "il", "branch": "retargeting", "authors": ["Tairan He", "Zhengyi Luo", "Wenli Xiao", "Chong Zhang", "Kris Kitani", "Changliu Liu", "Guanya Shi"], "corresponding": ["Tairan He"], "institutions": ["Carnegie Mellon University"], "lab": "CMU LeCAR", "venue": "IROS", "venue_type": "conference", "year": 2024, "date": "2024-03-07", "topics": ["whole-body teleoperation", "human-to-humanoid"], "paradigm": ["RL", "teleop"], "method": ["shape-fitted retargeting + privileged RL tracking policy", "RGB-only human input"], "projects": ["mabel"], "related": ["deepmimic", "exbody", "gmr", "hover", "human2humanoid", "omnih2o", "phc", "twist", "x-op"], "arxiv": "2403.04436", "doi": "", "code": "", "site": "", "note": "Real-time whole-body humanoid teleop from a single RGB camera. The retargeting half — fit the human shape to the robot's kinematics first, then track — is the part MABEL needs.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We present Human to Humanoid (H2O), a reinforcement learning (RL) based framework that enables real-time whole-body teleoperation of a full-sized humanoid robot with only an RGB camera. To create a large-scale retargeted motion dataset of human movements for humanoid robots, we propose a scalable \"sim-to-data\" process to filter and pick feasible motions using a privileged motion imitator. Afterwards, we train a robust real-time humanoid motion imitator in simulation using these refined motions and transfer it to the real humanoid robot in a zero-shot manner. We successfully achieve teleoperation of dynamic whole-body motions in real-world scenarios, including walking, back jumping, kicking, turning, waving, pushing, boxing, etc. To the best of our knowledge, this is the first demonstration to achieve learning-based real-time whole-body humanoid teleoperation.", "source": "curated", "first_author": "Tairan He", "url": "https://arxiv.org/abs/2403.04436", "fig": "figures/h2o.webp", "figs": ["figures/h2o-1.webp", "figures/h2o-2.webp", "figures/h2o-3.webp"]}, {"id": "maniwav", "title": "ManiWAV: Learning Robot Manipulation from In-the-Wild Audio-Visual Data", "tree": "il", "branch": "tactile-learn", "authors": ["Zeyi Liu", "Cheng Chi", "Eric Cousineau", "Naveen Kuppuswamy", "Benjamin Burchfiel", "Shuran Song"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-06-27", "topics": ["ear-in-hand", "in-the-wild audio-visual data", "contact sensing"], "paradigm": ["IL", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["audio-vla", "av-contact", "dexumi", "dj-mix-analysis", "hearing-touch", "see-hear-feel", "sonicsense", "umi", "vibecheck"], "arxiv": "2406.19464", "doi": "10.48550/arxiv.2406.19464", "code": "", "site": "", "note": "The key paper for an audio-driven system: an <b>ear-in-hand</b> device — a contact microphone on a UMI-style handheld gripper — that collects audio-visual demonstrations in the wild, and policies learned directly from them. It is the UMI argument with sound added, which makes it the obvious substrate for a turntable system.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Audio signals provide rich information for the robot interaction and object properties through contact. This information can surprisingly ease the learning of contact-rich robot manipulation skills, especially when the visual information alone is ambiguous or incomplete. However, the usage of audio data in robot manipulation has been constrained to teleoperated demonstrations collected by either attaching a microphone to the robot or object, which significantly limits its usage in robot learning pipelines. In this work, we introduce ManiWAV: an 'ear-in-hand' data collection device to collect in-the-wild human demonstrations with synchronous audio and visual feedback, and a corresponding policy interface to learn robot manipulation policy directly from the demonstrations. We demonstrate the capabilities of our system through four contact-rich manipulation tasks that require either passively sensing the contact events and modes, or actively sensing the object surface materials and states. In addition, we show that our system can generalize to unseen in-the-wild environments by learning from diverse in-the-wild human demonstrations.", "source": "curated", "first_author": "Zeyi Liu", "url": "https://arxiv.org/abs/2406.19464", "fig": "figures/maniwav.webp", "figs": ["figures/maniwav-1.webp", "figures/maniwav-2.webp", "figures/maniwav-3.webp"]}, {"id": "maskedmimic", "title": "MaskedMimic: Unified Physics-Based Character Control Through Masked Motion Inpainting", "tree": "rl", "branch": "motion-imitation", "authors": ["Chen Tessler", "Yunrong Guo", "Ofir Nabati", "Gal Chechik", "Xue Bin Peng"], "corresponding": ["Chen Tessler"], "institutions": ["NVIDIA", "Technion", "Simon Fraser University"], "lab": "NVIDIA", "venue": "SIGGRAPH Asia", "venue_type": "conference", "year": 2024, "date": "2024-09-22", "topics": ["masked modelling", "unified control", "motion inpainting"], "paradigm": ["RL"], "method": ["train with randomly masked goal specifications so one policy serves many interfaces"], "projects": ["mabel"], "related": ["ase", "beyond-mimic", "deepmimic", "handoff", "hover", "omnih2o", "phc"], "arxiv": "2409.14393", "doi": "", "code": "", "site": "", "note": "The BERT move applied to control: mask parts of the goal at training time and a single policy covers joystick, waypoint, VR-trio and full-body targets. HOVER is the same insight on real humanoid hardware.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/MaskedMimic.pdf", "abstract": "Crafting a single, versatile physics-based controller that can breathe life into interactive characters across a wide spectrum of scenarios represents an exciting frontier in character animation. An ideal controller should support diverse control modalities, such as sparse target keyframes, text instructions, and scene information. While previous works have proposed physically simulated, scene-aware control models, these systems have predominantly focused on developing controllers that each specializes in a narrow set of tasks and control modalities. This work presents MaskedMimic, a novel approach that formulates physics-based character control as a general motion inpainting problem. Our key insight is to train a single unified model to synthesize motions from partial (masked) motion descriptions, such as masked keyframes, objects, text descriptions, or any combination thereof. This is achieved by leveraging motion tracking data and designing a scalable training method that can effectively utilize diverse motion descriptions to produce coherent animations. Through this process, our approach learns a physics-based controller that provides an intuitive control interface without requiring tedious reward engineering for all behaviors of interest. The resulting controller supports a wide range of control modalities and enables seamless transitions between disparate tasks. By unifying character control through motion inpainting, MaskedMimic creates versatile virtual characters. These characters can dynamically adapt to complex scenes and compose diverse motions on demand, enabling more interactive and immersive experiences.", "source": "curated", "first_author": "Chen Tessler", "url": "https://arxiv.org/abs/2409.14393", "fig": "figures/maskedmimic.webp", "figs": ["figures/maskedmimic-1.webp", "figures/maskedmimic-2.webp"]}, {"id": "mobile-aloha", "title": "Mobile ALOHA: Learning Bimanual Mobile Manipulation with Low-Cost Whole-Body Teleoperation", "tree": "systems", "branch": "bimanual", "authors": ["Zipeng Fu", "Tony Z. Zhao", "Chelsea Finn"], "corresponding": ["Zipeng Fu"], "institutions": ["Stanford University"], "lab": "Stanford IRIS Lab", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-01-04", "topics": ["mobile bimanual manipulation", "whole-body teleoperation"], "paradigm": ["hardware", "IL"], "method": ["ALOHA arms on a wheeled base", "operator physically tethered for base control", "co-train with static data"], "projects": ["mabel"], "related": ["act", "aloha", "aloha2", "behavior-robot-suite", "homer", "telemoma", "tidybot2", "umi", "umi-on-legs"], "arxiv": "2401.02117", "doi": "", "code": "", "site": "https://mobile-aloha.github.io/", "note": "You named this one. Two results matter: the tether-style whole-body teleop rig, and the finding that co-training with static ALOHA data lifts mobile task success by up to 90%. The co-training result is the one that generalizes to MABEL.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Imitation learning from human demonstrations has shown impressive performance in robotics. However, most results focus on table-top manipulation, lacking the mobility and dexterity necessary for generally useful tasks. In this work, we develop a system for imitating mobile manipulation tasks that are bimanual and require whole-body control. We first present Mobile ALOHA, a low-cost and whole-body teleoperation system for data collection. It augments the ALOHA system with a mobile base, and a whole-body teleoperation interface. Using data collected with Mobile ALOHA, we then perform supervised behavior cloning and find that co-training with existing static ALOHA datasets boosts performance on mobile manipulation tasks. With 50 demonstrations for each task, co-training can increase success rates by up to 90%, allowing Mobile ALOHA to autonomously complete complex mobile manipulation tasks such as sauteing and serving a piece of shrimp, opening a two-door wall cabinet to store heavy cooking pots, calling and entering an elevator, and lightly rinsing a used pan using a kitchen faucet. Project website: https://mobile-aloha.github.io", "source": "curated", "first_author": "Zipeng Fu", "url": "https://arxiv.org/abs/2401.02117", "fig": "figures/mobile-aloha.webp", "figs": ["figures/mobile-aloha-1.webp"]}, {"id": "omnih2o", "title": "OmniH2O: Universal and Dexterous Human-to-Humanoid Whole-Body Teleoperation and Learning", "tree": "il", "branch": "retargeting", "authors": ["Tairan He", "Zhengyi Luo", "Xialin He", "Wenli Xiao", "Chong Zhang", "Weinan Zhang", "Kris Kitani", "Changliu Liu", "Guanya Shi"], "corresponding": ["Tairan He"], "institutions": ["Carnegie Mellon University", "Shanghai Jiao Tong University"], "lab": "CMU LeCAR", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-06-13", "topics": ["dexterous whole-body teleop", "multiple interfaces"], "paradigm": ["RL", "teleop"], "method": [], "projects": ["mabel"], "related": ["h2o", "homie", "hover", "maskedmimic", "phc", "twist"], "arxiv": "2406.08858", "doi": "", "code": "", "site": "", "note": "In your library. Extends H2O to dexterous hands and to many control interfaces (VR, RGB, language, autonomous), plus a demonstration dataset. The interface-agnostic framing leads straight to HOVER.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/omniH2O.pdf", "abstract": "We present OmniH2O (Omni Human-to-Humanoid), a learning-based system for whole-body humanoid teleoperation and autonomy. Using kinematic pose as a universal control interface, OmniH2O enables various ways for a human to control a full-sized humanoid with dexterous hands, including using real-time teleoperation through VR headset, verbal instruction, and RGB camera. OmniH2O also enables full autonomy by learning from teleoperated demonstrations or integrating with frontier models such as GPT-4. OmniH2O demonstrates versatility and dexterity in various real-world whole-body tasks through teleoperation or autonomy, such as playing multiple sports, moving and manipulating objects, and interacting with humans. We develop an RL-based sim-to-real pipeline, which involves large-scale retargeting and augmentation of human motion datasets, learning a real-world deployable policy with sparse sensor input by imitating a privileged teacher policy, and reward designs to enhance robustness and stability. We release the first humanoid whole-body control dataset, OmniH2O-6, containing six everyday tasks, and demonstrate humanoid whole-body skill learning from teleoperated datasets.", "source": "curated", "first_author": "Tairan He", "url": "https://arxiv.org/abs/2406.08858", "fig": "figures/omnih2o.webp", "figs": ["figures/omnih2o-1.webp", "figures/omnih2o-2.webp"]}, {"id": "openvla", "title": "OpenVLA: An Open-Source Vision-Language-Action Model", "tree": "il", "branch": "vla", "authors": ["Moo Jin Kim", "Karl Pertsch", "Siddharth Karamcheti", "Ted Xiao", "Ashwin Balakrishna", "et al."], "corresponding": ["Moo Jin Kim"], "institutions": ["Stanford University", "UC Berkeley", "Toyota Research Institute", "Google DeepMind"], "lab": "Stanford IRIS", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-06-13", "topics": ["open-source VLA", "7B", "LoRA fine-tuning"], "paradigm": ["VLA"], "method": ["Prismatic VLM backbone + discretized actions", "trained on 970k OXE episodes"], "projects": [], "related": ["audio-vla", "dexvla", "iRe-VLA", "libero", "octo", "openvla-oft", "oxe", "pi0", "rt2", "vlm4vla"], "arxiv": "2406.09246", "doi": "", "code": "", "site": "", "note": "The VLA everyone actually fine-tunes, because the weights and the training code are both open. OpenVLA-OFT later showed the discretized action head was the bottleneck, not the backbone.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Large policies pretrained on a combination of Internet-scale vision-language data and diverse robot demonstrations have the potential to change how we teach robots new skills: rather than training new behaviors from scratch, we can fine-tune such vision-language-action (VLA) models to obtain robust, generalizable policies for visuomotor control. Yet, widespread adoption of VLAs for robotics has been challenging as 1) existing VLAs are largely closed and inaccessible to the public, and 2) prior work fails to explore methods for efficiently fine-tuning VLAs for new tasks, a key component for adoption. Addressing these challenges, we introduce OpenVLA, a 7B-parameter open-source VLA trained on a diverse collection of 970k real-world robot demonstrations. OpenVLA builds on a Llama 2 language model combined with a visual encoder that fuses pretrained features from DINOv2 and SigLIP. As a product of the added data diversity and new model components, OpenVLA demonstrates strong results for generalist manipulation, outperforming closed models such as RT-2-X (55B) by 16.5% in absolute task success rate across 29 tasks and multiple robot embodiments, with 7x fewer parameters. We further show that we can effectively fine-tune OpenVLA for new settings, with especially strong generalization results in multi-task environments involving multiple objects and strong language grounding abilities, and outperform expressive from-scratch imitation learning methods such as Diffusion Policy by 20.4%. We also explore compute efficiency; as a separate contribution, we show that OpenVLA can be fine-tuned on consumer GPUs via modern low-rank adaptation methods and served efficiently via quantization without a hit to downstream success rate. Finally, we release model checkpoints, fine-tuning notebooks, and our PyTorch codebase with built-in support for training VLAs at scale on Open X-Embodiment datasets.", "source": "curated", "first_author": "Moo Jin Kim", "url": "https://arxiv.org/abs/2406.09246", "fig": "figures/openvla.webp", "figs": ["figures/openvla-1.webp"]}, {"id": "hil-serl", "title": "Precise and Dexterous Robotic Manipulation via Human-in-the-Loop Reinforcement Learning", "tree": "rl", "branch": "post-training", "authors": ["Jianlan Luo", "Charles Xu", "Jeffrey Wu", "Sergey Levine"], "corresponding": ["Jianlan Luo"], "institutions": ["UC Berkeley"], "lab": "BAIR (RAIL)", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-10-29", "topics": ["human-in-the-loop RL", "precise manipulation"], "paradigm": ["RL"], "method": ["human corrections folded into the replay buffer during on-robot RL"], "projects": ["iris"], "related": ["dagger", "expo-ft", "hand-in-loop", "omnitactune", "pi06", "pld", "residual-assembly", "ript-vla", "rlpd", "serl", "video2sim2real"], "arxiv": "2410.21845", "doi": "", "code": "", "site": "", "note": "Near-100% success on genuinely hard contact-rich tasks in 1-2.5 hours of real training. The strongest existing argument that RL post-training beats more imitation data once you are chasing the last 10%.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Reinforcement learning (RL) holds great promise for enabling autonomous acquisition of complex robotic manipulation skills, but realizing this potential in real-world settings has been challenging. We present a human-in-the-loop vision-based RL system that demonstrates impressive performance on a diverse set of dexterous manipulation tasks, including dynamic manipulation, precision assembly, and dual-arm coordination. Our approach integrates demonstrations and human corrections, efficient RL algorithms, and other system-level design choices to learn policies that achieve near-perfect success rates and fast cycle times within just 1 to 2.5 hours of training. We show that our method significantly outperforms imitation learning baselines and prior RL approaches, with an average 2x improvement in success rate and 1.8x faster execution. Through extensive experiments and analysis, we provide insights into the effectiveness of our approach, demonstrating how it learns robust, adaptive policies for both reactive and predictive control strategies. Our results suggest that RL can indeed learn a wide range of complex vision-based manipulation policies directly in the real world within practical training times. We hope this work will inspire a new generation of learned robotic manipulation techniques, benefiting both industrial applications and research advancements. Videos and code are available at our project website https://hil-serl.github.io/.", "source": "curated", "first_author": "Jianlan Luo", "url": "https://arxiv.org/abs/2410.21845", "fig": "figures/hil-serl.webp"}, {"id": "rekep", "title": "ReKep: Spatio-Temporal Reasoning of Relational Keypoint Constraints for Robotic Manipulation", "tree": "systems", "branch": "planning-control", "authors": ["Wenlong Huang", "Chen Wang", "Yunzhu Li", "Ruohan Zhang", "Li Fei-Fei"], "corresponding": [], "institutions": ["Stanford University", "Columbia University"], "lab": "Stanford Vision & Learning (SVL)", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-09-03", "topics": ["relational keypoint constraints", "constraints as Python functions", "hierarchical optimization"], "paradigm": ["LLM planning"], "method": [], "projects": ["geodex", "mabel"], "related": ["behavior-robot-suite", "d3fields", "robo-exp", "voxposer"], "arxiv": "2409.01652", "doi": "10.48550/arXiv.2409.01652", "code": "", "site": "", "note": "<b>ReKep</b> — VoxPoser's successor and a better formulation: express the task as <b>relational keypoint constraints</b>, Python functions mapping 3D keypoints to a cost, produced by a VLM and then solved hierarchically into robot actions. Constraints compose where value maps merely add, and they give you something a world model does not: a <em>checkable</em> statement of what the task requires, which is exactly what you need if an agent is going to propose tasks autonomously.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Representing robotic manipulation tasks as constraints that associate the robot and the environment is a promising way to encode desired robot behaviors. However, it remains unclear how to formulate the constraints such that they are 1) versatile to diverse tasks, 2) free of manual labeling, and 3) optimizable by off-the-shelf solvers to produce robot actions in real-time. In this work, we introduce Relational Keypoint Constraints (ReKep), a visually-grounded representation for constraints in robotic manipulation. Specifically, ReKep is expressed as Python functions mapping a set of 3D keypoints in the environment to a numerical cost. We demonstrate that by representing a manipulation task as a sequence of Relational Keypoint Constraints, we can employ a hierarchical optimization procedure to solve for robot actions (represented by a sequence of end-effector poses in SE(3)) with a perception-action loop at a real-time frequency. Furthermore, in order to circumvent the need for manual specification of ReKep for each new task, we devise an automated procedure that leverages large vision models and vision-language models to produce ReKep from free-form language instructions and RGB-D observations. We present system implementations on a wheeled single-arm platform and a stationary dual-arm platform that can perform a large variety of manipulation tasks, featuring multi-stage, in-the-wild, bimanual, and reactive behaviors, all without task-specific data or environment models. Website at https://rekep-robot.github.io/.", "source": "curated", "first_author": "Wenlong Huang", "url": "https://arxiv.org/abs/2409.01652", "fig": "figures/rekep.webp", "figs": ["figures/rekep-1.webp"]}, {"id": "radosavovic-humanoid", "title": "Real-World Humanoid Locomotion with Reinforcement Learning", "tree": "rl", "branch": "locomotion", "authors": ["Ilija Radosavovic", "Tete Xiao", "Bike Zhang", "Trevor Darrell", "Jitendra Malik", "Koushil Sreenath"], "corresponding": ["Ilija Radosavovic"], "institutions": ["UC Berkeley"], "lab": "BAIR / Hybrid Robotics", "venue": "Science Robotics", "venue_type": "journal", "year": 2024, "date": "2023-03-06", "topics": ["humanoid locomotion", "causal transformer", "in-context adaptation"], "paradigm": ["RL"], "method": ["causal transformer over observation-action history", "trained in sim", "zero-shot to Digit"], "projects": ["mabel"], "related": ["anymal-lee", "rma"], "arxiv": "2303.03381", "doi": "", "code": "", "site": "", "note": "Treats locomotion as next-token prediction over the observation-action stream, which gives in-context adaptation for free. Early evidence that sequence models beat MLPs once the environment is non-stationary.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid robots that can autonomously operate in diverse environments have the potential to help address labour shortages in factories, assist elderly at homes, and colonize new planets. While classical controllers for humanoid robots have shown impressive results in a number of settings, they are challenging to generalize and adapt to new environments. Here, we present a fully learning-based approach for real-world humanoid locomotion. Our controller is a causal transformer that takes the history of proprioceptive observations and actions as input and predicts the next action. We hypothesize that the observation-action history contains useful information about the world that a powerful transformer model can use to adapt its behavior in-context, without updating its weights. We train our model with large-scale model-free reinforcement learning on an ensemble of randomized environments in simulation and deploy it to the real world zero-shot. Our controller can walk over various outdoor terrains, is robust to external disturbances, and can adapt in context.", "source": "curated", "first_author": "Ilija Radosavovic", "url": "https://arxiv.org/abs/2303.03381", "fig": "figures/radosavovic-humanoid.webp"}, {"id": "serl", "title": "SERL: A Software Suite for Sample-Efficient Robotic Reinforcement Learning", "tree": "rl", "branch": "post-training", "authors": ["Jianlan Luo", "Zheyuan Hu", "Charles Xu", "You Liang Tan", "Jacob Berg", "Archit Sharma", "Stefan Schaal", "Chelsea Finn", "Abhishek Gupta", "Sergey Levine"], "corresponding": ["Jianlan Luo"], "institutions": ["UC Berkeley", "University of Washington", "Stanford University"], "lab": "BAIR (RAIL)", "venue": "ICRA", "venue_type": "conference", "year": 2024, "date": "2024-01-29", "topics": ["real-world RL", "sample efficiency", "software suite"], "paradigm": ["RL"], "method": ["RLPD-style off-policy RL with demos", "learned reward classifiers", "forward-backward resets"], "projects": ["iris"], "related": ["daydreamer", "harbor", "hil-serl", "rlinf-vla", "rlpd", "sac", "vip"], "arxiv": "2401.16013", "doi": "", "code": "", "site": "", "note": "Makes real-robot RL a two-hour proposition rather than a research project. The reset and reward infrastructure here is the concrete template for your real-world RL harness idea.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "In recent years, significant progress has been made in the field of robotic reinforcement learning (RL), enabling methods that handle complex image observations, train in the real world, and incorporate auxiliary data, such as demonstrations and prior experience. However, despite these advances, robotic RL remains hard to use. It is acknowledged among practitioners that the particular implementation details of these algorithms are often just as important (if not more so) for performance as the choice of algorithm. We posit that a significant challenge to widespread adoption of robotic RL, as well as further development of robotic RL methods, is the comparative inaccessibility of such methods. To address this challenge, we developed a carefully implemented library containing a sample efficient off-policy deep RL method, together with methods for computing rewards and resetting the environment, a high-quality controller for a widely-adopted robot, and a number of challenging example tasks. We provide this library as a resource for the community, describe its design choices, and present experimental results. Perhaps surprisingly, we find that our implementation can achieve very efficient learning, acquiring policies for PCB board assembly, cable routing, and object relocation between 25 to 50 minutes of training per policy on average, improving over state-of-the-art results reported for similar tasks in the literature. These policies achieve perfect or near-perfect success rates, extreme robustness even under perturbations, and exhibit emergent recovery and correction behaviors. We hope that these promising results and our high-quality open-source implementation will provide a tool for the robotics community to facilitate further developments in robotic RL. Our code, documentation, and videos can be found at https://serl-robot.github.io/", "source": "curated", "first_author": "Jianlan Luo", "url": "https://arxiv.org/abs/2401.16013", "fig": "figures/serl.webp", "figs": ["figures/serl-1.webp"]}, {"id": "sonicsense", "title": "SonicSense: Object Perception from In-Hand Acoustic Vibration", "tree": "systems", "branch": "tactile-hw", "authors": ["Jiaxun Liu", "Boyuan Chen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-06-25", "topics": ["in-hand acoustic vibration", "contact microphones per fingertip"], "paradigm": ["hardware", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "dexskin", "hearing-touch", "leap-hand", "maniwav", "miccheck", "vibecheck"], "arxiv": "2406.17932", "doi": "10.48550/arxiv.2406.17932", "code": "", "site": "", "note": "A four-fingered hand with a contact microphone in <b>every fingertip</b>, identifying materials, container contents and 3D shape from tapping and shaking. Duke (Boyuan Chen). This is the hardware pattern for a dexterous audio hand — and it is cheap, because a piezo disc costs a dollar.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce SonicSense, a holistic design of hardware and software to enable rich robot object perception through in-hand acoustic vibration sensing. While previous studies have shown promising results with acoustic sensing for object perception, current solutions are constrained to a handful of objects with simple geometries and homogeneous materials, single-finger sensing, and mixing training and testing on the same objects. SonicSense enables container inventory status differentiation, heterogeneous material prediction, 3D shape reconstruction, and object re-identification from a diverse set of 83 real-world objects. Our system employs a simple but effective heuristic exploration policy to interact with the objects as well as end-to-end learning-based algorithms to fuse vibration signals to infer object properties. Our framework underscores the significance of in-hand acoustic vibration sensing in advancing robot tactile perception.", "source": "curated", "first_author": "Jiaxun Liu", "url": "https://arxiv.org/abs/2406.17932", "fig": "figures/sonicsense.webp", "figs": ["figures/sonicsense-1.webp", "figures/sonicsense-2.webp"]}, {"id": "telemoma", "title": "TeleMoMa: A Modular and Versatile Teleoperation System for Mobile Manipulation", "tree": "systems", "branch": "interfaces", "authors": ["Shivin Dass", "Wensi Ai", "Yuqian Jiang", "Samik Singh", "Jiaheng Hu", "Ruohan Zhang", "Peter Stone", "Ben Abbatematteo", "Roberto Martín-Martín"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-03-12", "topics": ["modular teleoperation", "mobile manipulation", "multi-interface"], "paradigm": ["teleop"], "method": [], "projects": ["m2", "mabel"], "related": ["gello", "homie", "mobile-aloha", "modpack", "tidybot2"], "arxiv": "2403.07869", "doi": "10.48550/arxiv.2403.07869", "code": "", "site": "", "note": "A modular whole-body teleop interface for <b>mobile</b> manipulators that unifies RGB-D, VR controllers, keyboard and joystick — and mixes them. Demonstrated on Tiago++, HSR and Fetch. If MABEL needs one teleop stack that survives changing the input device, this is the architecture.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "A critical bottleneck limiting imitation learning in robotics is the lack of data. This problem is more severe in mobile manipulation, where collecting demonstrations is harder than in stationary manipulation due to the lack of available and easy-to-use teleoperation interfaces. In this work, we demonstrate TeleMoMa, a general and modular interface for whole-body teleoperation of mobile manipulators. TeleMoMa unifies multiple human interfaces including RGB and depth cameras, virtual reality controllers, keyboard, joysticks, etc., and any combination thereof. In its more accessible version, TeleMoMa works using simply vision (e.g., an RGB-D camera), lowering the entry bar for humans to provide mobile manipulation demonstrations. We demonstrate the versatility of TeleMoMa by teleoperating several existing mobile manipulators - PAL Tiago++, Toyota HSR, and Fetch - in simulation and the real world. We demonstrate the quality of the demonstrations collected with TeleMoMa by training imitation learning policies for mobile manipulation tasks involving synchronized whole-body motion. Finally, we also show that TeleMoMa's teleoperation channel enables teleoperation on site, looking at the robot, or remote, sending commands and observations through a computer network, and perform user studies to evaluate how easy it is for novice users to learn to collect demonstrations with different combinations of human interfaces enabled by our system. We hope TeleMoMa becomes a helpful tool for the community enabling researchers to collect whole-body mobile manipulation demonstrations. For more information and video results, https://robin-lab.cs.utexas.edu/telemoma-web.", "source": "curated", "first_author": "Shivin Dass", "url": "https://arxiv.org/abs/2403.07869", "fig": "figures/telemoma.webp", "figs": ["figures/telemoma-1.webp", "figures/telemoma-2.webp"]}, {"id": "umi-on-legs", "title": "UMI on Legs: Making Manipulation Policies Mobile with Manipulation-Centric Whole-body Controllers", "tree": "systems", "branch": "interfaces", "authors": ["Huy Ha", "Yihuai Gao", "Zipeng Fu", "Jie Tan", "Shuran Song"], "corresponding": ["Huy Ha"], "institutions": ["Stanford University", "Google DeepMind"], "lab": "Stanford REALab", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-07-15", "topics": ["mobile manipulation", "whole-body controller", "UMI"], "paradigm": ["hardware", "RL", "IL"], "method": ["task-frame end-effector trajectory as the interface between a UMI policy and an RL whole-body controller"], "projects": ["mabel", "wam-tactile"], "related": ["falcon", "handeye-delivery", "hmi-robotfree", "mobile-aloha", "realdexumi", "tidybot2", "umi", "wholebodyvla"], "arxiv": "2407.10353", "doi": "", "code": "", "site": "", "note": "The clean decomposition: the manipulation policy speaks end-effector trajectories in the task frame, and an RL whole-body controller figures out how to deliver them. That interface is exactly what a mobile dexterous UMI on MABEL would need.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce UMI-on-Legs, a new framework that combines real-world and simulation data for quadruped manipulation systems. We scale task-centric data collection in the real world using a hand-held gripper (UMI), providing a cheap way to demonstrate task-relevant manipulation skills without a robot. Simultaneously, we scale robot-centric data in simulation by training whole-body controller for task-tracking without task simulation setups. The interface between these two policies is end-effector trajectories in the task frame, inferred by the manipulation policy and passed to the whole-body controller for tracking. We evaluate UMI-on-Legs on prehensile, non-prehensile, and dynamic manipulation tasks, and report over 70% success rate on all tasks. Lastly, we demonstrate the zero-shot cross-embodiment deployment of a pre-trained manipulation policy checkpoint from prior work, originally intended for a fixed-base robot arm, on our quadruped system. We believe this framework provides a scalable path towards learning expressive manipulation skills on dynamic robot embodiments. Please checkout our website for robot videos, code, and data: https://umi-on-legs.github.io", "source": "curated", "first_author": "Huy Ha", "url": "https://arxiv.org/abs/2407.10353", "fig": "figures/umi-on-legs.webp", "figs": ["figures/umi-on-legs-1.webp", "figures/umi-on-legs-2.webp", "figures/umi-on-legs-3.webp"]}, {"id": "umi", "title": "Universal Manipulation Interface: In-The-Wild Robot Teaching Without In-The-Wild Robots", "tree": "systems", "branch": "interfaces", "authors": ["Cheng Chi", "Zhenjia Xu", "Chuer Pan", "Eric Cousineau", "Benjamin Burchfiel", "Siyuan Feng", "Russ Tedrake", "Shuran Song"], "corresponding": ["Cheng Chi", "Shuran Song"], "institutions": ["Stanford University", "Columbia University", "Toyota Research Institute"], "lab": "Stanford REALab (Shuran Song)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-02-15", "topics": ["handheld gripper", "in-the-wild data", "embodiment-agnostic capture"], "paradigm": ["hardware", "IL"], "method": ["handheld gripper + fisheye GoPro", "IMU-aware pose tracking", "latency-matched inference"], "projects": ["geodex", "mabel", "r2s2r", "wam-tactile"], "related": ["airexo", "dexcap", "dexumi", "dp", "droid", "egokit", "fast-umi", "gello", "hmi-robotfree", "maniwav", "mobile-aloha", "one-demo", "open-aoe", "open-television", "realdexumi", "sunday-gelato", "tidybot2", "ume", "umi-on-legs", "wt-umi"], "arxiv": "2402.10329", "doi": "", "code": "https://github.com/real-stanford/universal_manipulation_interface", "site": "https://umi-gripper.github.io/", "note": "The interface your UMI catalogue is about and the backbone of several of your research ideas. The core claim — that you can collect deployable demonstrations with no robot present — holds because of unglamorous engineering: side mirrors for implicit stereo, an IMU-aware SLAM pipeline, and explicit inference-latency matching. Read the hardware section, not just the method.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We present Universal Manipulation Interface (UMI) -- a data collection and policy learning framework that allows direct skill transfer from in-the-wild human demonstrations to deployable robot policies. UMI employs hand-held grippers coupled with careful interface design to enable portable, low-cost, and information-rich data collection for challenging bimanual and dynamic manipulation demonstrations. To facilitate deployable policy learning, UMI incorporates a carefully designed policy interface with inference-time latency matching and a relative-trajectory action representation. The resulting learned policies are hardware-agnostic and deployable across multiple robot platforms. Equipped with these features, UMI framework unlocks new robot manipulation capabilities, allowing zero-shot generalizable dynamic, bimanual, precise, and long-horizon behaviors, by only changing the training data for each task. We demonstrate UMI's versatility and efficacy with comprehensive real-world experiments, where policies learned via UMI zero-shot generalize to novel environments and objects when trained on diverse human demonstrations. UMI's hardware and software system is open-sourced at https://umi-gripper.github.io.", "source": "curated", "first_author": "Cheng Chi", "url": "https://arxiv.org/abs/2402.10329", "fig": "figures/umi.webp", "figs": ["figures/umi-1.webp", "figures/umi-2.webp", "figures/umi-3.webp"]}, {"id": "wheeled-posforce", "title": "Wheeled Humanoid Bilateral Teleoperation with Position-Force Control Modes for Dynamic Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Amartya Purushottam", "Jack Yan", "Christopher Xu", "Youngwoo Sim", "João Ramos"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-07-16", "topics": ["position-force control modes", "bilateral teleoperation", "wheeled humanoid"], "paradigm": ["control", "teleop"], "method": [], "projects": ["m2", "mabel"], "related": ["falcon", "ume", "wheeled-bilateral", "wheeled-multistage"], "arxiv": "2407.12189", "doi": "10.48550/arxiv.2407.12189", "code": "", "site": "", "note": "Already in your M2 library. Switching between position and force control modes during wheeled-humanoid teleoperation — which is exactly the decision MABEL has to make when it goes from reaching to pushing.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2024_2407.12189v1_Wheeled_Humanoid_Bilateral_Teleoperation_with_Position_Force.pdf", "abstract": "Remote-controlled humanoid robots can revolutionize manufacturing, construction, and healthcare industries by performing complex or dangerous manual tasks traditionally done by humans. We refer to these behaviors as Dynamic Loco-Manipulation (DLM). To successfully complete these tasks, humans control the position of their bodies and contact forces at their hands. To enable similar whole-body control in humanoids, we introduce loco-manipulation retargeting strategies with switched position and force control modes in a bilateral teleoperation framework. Our proposed locomotion mappings use the pitch and yaw of the operator's torso to control robot position or acceleration. The manipulation retargeting maps the operator's arm movements to the robot's arms for joint-position or impedance control of the end-effector. A Human-Machine Interface captures the teleoperator's motion and provides haptic feedback to their torso, enhancing their awareness of the robot's interactions with the environment. In this paper, we demonstrate two forms of DLM. First, we show the robot slotting heavy boxes (5-10.5 kg), weighing up to 83% of the robot's weight, into desired positions. Second, we show human-robot collaboration for carrying an object, where the robot and teleoperator take on leader and follower roles.", "source": "curated", "first_author": "Amartya Purushottam", "url": "https://arxiv.org/abs/2407.12189", "fig": "figures/wheeled-posforce.webp", "figs": ["figures/wheeled-posforce-1.webp", "figures/wheeled-posforce-2.webp", "figures/wheeled-posforce-3.webp"]}, {"id": "aloha", "title": "ALOHA: A Low-cost Open-source Hardware System for Bimanual Teleoperation", "tree": "systems", "branch": "bimanual", "authors": ["Tony Z. Zhao", "Vikash Kumar", "Sergey Levine", "Chelsea Finn"], "corresponding": ["Tony Z. Zhao"], "institutions": ["Stanford University"], "lab": "Stanford IRIS Lab", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-04-23", "topics": ["open hardware", "bimanual teleoperation", "low cost"], "paradigm": ["hardware"], "method": ["leader-follower joint-space teleoperation with ViperX/WidowX arms", "~$20k BOM"], "projects": ["iris", "mabel"], "related": ["act", "aloha2", "gello", "koch", "mobile-aloha", "openpyro", "so100"], "arxiv": "", "doi": "", "code": "", "site": "https://tonyzhaozh.github.io/aloha/", "note": "The hardware introduced in the ACT paper (arXiv 2304.13705) — no separate publication — and arguably the more consequential half of it. Direct joint-space leader-follower with no force feedback turned out to be enough for fine bimanual work, and the $20k price point put it in reach of most labs. Every bimanual platform below is responding to it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Tony Z. Zhao", "url": "https://tonyzhaozh.github.io/aloha/"}, {"id": "anyteleop", "title": "AnyTeleop: A General Vision-Based Dexterous Robot Arm-Hand Teleoperation System", "tree": "il", "branch": "retargeting", "authors": ["Yuzhe Qin", "Wei Yang", "Binghao Huang", "Karl Van Wyk", "Hao Su", "Xiaolong Wang", "Yu-Wei Chao", "Dieter Fox"], "corresponding": ["Yuzhe Qin"], "institutions": ["UC San Diego", "NVIDIA"], "lab": "UCSD Xiaolong Wang Lab", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-07-10", "topics": ["general teleoperation", "cross-hand retargeting"], "paradigm": ["teleop"], "method": [], "projects": ["geodex", "mabel"], "related": ["ace-teleop", "bunny-visionpro", "c2dex", "dex-retargeting", "dexpilot", "open-television", "retarget-objectives"], "arxiv": "2307.04577", "doi": "", "code": "", "site": "", "note": "One system, many hands and arms, camera-only. The open dex-retargeting library that came out of this is the practical starting point for MABEL/GeoDex teleop.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Vision-based teleoperation offers the possibility to endow robots with human-level intelligence to physically interact with the environment, while only requiring low-cost camera sensors. However, current vision-based teleoperation systems are designed and engineered towards a particular robot model and deploy environment, which scales poorly as the pool of the robot models expands and the variety of the operating environment increases. In this paper, we propose AnyTeleop, a unified and general teleoperation system to support multiple different arms, hands, realities, and camera configurations within a single system. Although being designed to provide great flexibility to the choice of simulators and real hardware, our system can still achieve great performance. For real-world experiments, AnyTeleop can outperform a previous system that was designed for a specific robot hardware with a higher success rate, using the same robot. For teleoperation in simulation, AnyTeleop leads to better imitation learning performance, compared with a previous system that is particularly designed for that simulator. Project page: https://yzqin.github.io/anyteleop/.", "source": "curated", "first_author": "Yuzhe Qin", "url": "https://arxiv.org/abs/2307.04577", "fig": "figures/anyteleop.webp", "figs": ["figures/anyteleop-1.webp", "figures/anyteleop-2.webp"]}, {"id": "d3fields", "title": "D$^3$Fields: Dynamic 3D Descriptor Fields for Zero-Shot Generalizable Rearrangement", "tree": "systems", "branch": "perception", "authors": ["Yixuan Wang", "Mingtong Zhang", "Zhuoran Li", "Tarik Kelestemur", "Katherine Driggs-Campbell", "Jiajun Wu", "Li Fei-Fei", "Yunzhu Li"], "corresponding": [], "institutions": ["Stanford University", "Columbia University", "University of Illinois Urbana-Champaign", "NUS"], "lab": "RoboPIL (Yunzhu Li)", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-09-28", "topics": ["dynamic 3D descriptor fields", "zero-shot generalization", "category-level correspondence"], "paradigm": ["representation"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["adaptigraph", "gs-dynamics", "pointworld", "rekep", "robo-exp", "robocraft", "voxposer"], "arxiv": "2309.16118", "doi": "10.48550/arxiv.2309.16118", "code": "", "site": "", "note": "<b>D³Fields</b> — fuse multi-view RGBD into a 3D field carrying both geometry and <em>semantic descriptors</em> from a foundation model, so a task can be specified once with a 2D goal image and then executed zero-shot on new instances of the category. This is the representation layer that makes the rest of the RoboPIL stack category-general, and it is directly usable in your real2sim2real pipeline as the thing that tells you which reconstructed object is which.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Scene representation is a crucial design choice in robotic manipulation systems. An ideal representation is expected to be 3D, dynamic, and semantic to meet the demands of diverse manipulation tasks. However, previous works often lack all three properties simultaneously. In this work, we introduce D$^3$Fields -- dynamic 3D descriptor fields. These fields are implicit 3D representations that take in 3D points and output semantic features and instance masks. They can also capture the dynamics of the underlying 3D environments. Specifically, we project arbitrary 3D points in the workspace onto multi-view 2D visual observations and interpolate features derived from visual foundational models. The resulting fused descriptor fields allow for flexible goal specifications using 2D images with varied contexts, styles, and instances. To evaluate the effectiveness of these descriptor fields, we apply our representation to rearrangement tasks in a zero-shot manner. Through extensive evaluation in real worlds and simulations, we demonstrate that D$^3$Fields are effective for zero-shot generalizable rearrangement tasks. We also compare D$^3$Fields with state-of-the-art implicit 3D representations and show significant improvements in effectiveness and efficiency.", "source": "curated", "first_author": "Yixuan Wang", "url": "https://arxiv.org/abs/2309.16118", "fig": "figures/d3fields.webp", "figs": ["figures/d3fields-1.webp", "figures/d3fields-2.webp"]}, {"id": "dp", "title": "Diffusion Policy: Visuomotor Policy Learning via Action Diffusion", "tree": "il", "branch": "visuomotor", "authors": ["Cheng Chi", "Siyuan Feng", "Yilun Du", "Zhenjia Xu", "Eric Cousineau", "Benjamin Burchfiel", "Shuran Song"], "corresponding": ["Cheng Chi", "Shuran Song"], "institutions": ["Columbia University", "Toyota Research Institute", "MIT"], "lab": "Columbia/Stanford REALab (Shuran Song)", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-03-07", "topics": ["diffusion policy", "multimodal actions", "visuomotor control"], "paradigm": ["IL", "diffusion"], "method": ["DDPM over action sequences", "receding-horizon execution", "FiLM-conditioned U-Net"], "projects": ["iris"], "related": ["act", "alohaunleashed", "beyond-mimic", "consistency-policy", "dp3", "implicit-bc", "manflow", "one-demo", "pi0", "umi"], "arxiv": "2303.04137", "doi": "", "code": "", "site": "", "note": "Sitting on your Desktop as 2303.04137v5-3.pdf. The action-decoder that became the default. It beats ACT mainly on multimodal tasks where averaging demonstrations is fatal; ACT is still faster and simpler when demonstrations are unimodal.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "This paper introduces Diffusion Policy, a new way of generating robot behavior by representing a robot's visuomotor policy as a conditional denoising diffusion process. We benchmark Diffusion Policy across 12 different tasks from 4 different robot manipulation benchmarks and find that it consistently outperforms existing state-of-the-art robot learning methods with an average improvement of 46.9%. Diffusion Policy learns the gradient of the action-distribution score function and iteratively optimizes with respect to this gradient field during inference via a series of stochastic Langevin dynamics steps. We find that the diffusion formulation yields powerful advantages when used for robot policies, including gracefully handling multimodal action distributions, being suitable for high-dimensional action spaces, and exhibiting impressive training stability. To fully unlock the potential of diffusion models for visuomotor policy learning on physical robots, this paper presents a set of key technical contributions including the incorporation of receding horizon control, visual conditioning, and the time-series diffusion transformer. We hope this work will help motivate a new generation of policy learning techniques that are able to leverage the powerful generative modeling capabilities of diffusion models. Code, data, and training details is publicly available diffusion-policy.cs.columbia.edu", "source": "curated", "first_author": "Cheng Chi", "url": "https://arxiv.org/abs/2303.04137", "fig": "figures/dp.webp", "figs": ["figures/dp-1.webp", "figures/dp-2.webp", "figures/dp-3.webp"]}, {"id": "wheeled-bilateral", "title": "Dynamic Mobile Manipulation via Whole-Body Bilateral Teleoperation of a Wheeled Humanoid", "tree": "rl", "branch": "loco-manip", "authors": ["Amartya Purushottam", "Yeongtae Jung", "Christopher Xu", "João Ramos"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-07-03", "topics": ["wheeled humanoid", "bilateral teleoperation", "dynamic loco-manipulation"], "paradigm": ["teleop", "control"], "method": [], "projects": ["mabel"], "related": ["deep-wbc", "falcon", "wheeled-multistage", "wheeled-payload", "wheeled-posforce"], "arxiv": "2307.01350", "doi": "10.48550/arxiv.2307.01350", "code": "", "site": "", "note": "A wheeled humanoid doing dynamic mobile manipulation under whole-body bilateral teleoperation. MABEL is a wheeled humanoid, and this line of work is the closest existing control literature to that morphology — the balance problem on wheels is genuinely different from legs and is under-served in the RL literature.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Humanoid robots have the potential to help human workers by realizing physically demanding manipulation tasks such as moving large boxes within warehouses. We define such tasks as Dynamic Mobile Manipulation (DMM). This paper presents a framework for DMM via whole-body teleoperation, built upon three key contributions: Firstly, a teleoperation framework employing a Human Machine Interface (HMI) and a bi-wheeled humanoid, SATYRR, is proposed. Secondly, the study introduces a dynamic locomotion mapping, utilizing human-robot reduced order models, and a kinematic retargeting strategy for manipulation tasks. Additionally, the paper discusses the role of whole-body haptic feedback for wheeled humanoid control. Finally, the system's effectiveness and mappings for DMM are validated through locomanipulation experiments and heavy box pushing tasks. Here we show two forms of DMM: grasping a target moving at an average speed of 0.4 m/s, and pushing boxes weighing up to 105\\% of the robot's weight. By simultaneously adjusting their pitch and using their arms, the pilot adjusts the robot pose to apply larger contact forces and move a heavy box at a constant velocity of 0.2 m/s.", "source": "curated", "first_author": "Amartya Purushottam", "url": "https://arxiv.org/abs/2307.01350", "fig": "figures/wheeled-bilateral.webp", "figs": ["figures/wheeled-bilateral-1.webp", "figures/wheeled-bilateral-2.webp", "figures/wheeled-bilateral-3.webp"]}, {"id": "rlpd", "title": "Efficient Online Reinforcement Learning with Offline Data", "tree": "rl", "branch": "post-training", "authors": ["Philip J. Ball", "Laura Smith", "Ilya Kostrikov", "Sergey Levine"], "corresponding": [], "institutions": ["University of Oxford", "UC Berkeley"], "lab": "BAIR", "venue": "ICML", "venue_type": "conference", "year": 2023, "date": "2023-02-06", "topics": ["offline-to-online RL", "sample efficiency"], "paradigm": ["RL"], "method": ["symmetric sampling from offline and online buffers", "layer norm", "ensembles"], "projects": [], "related": ["hil-serl", "sac", "serl"], "arxiv": "2302.02948", "doi": "", "code": "", "site": "", "note": "The algorithmic core underneath SERL/HIL-SERL. Three unglamorous tricks — 50/50 sampling, LayerNorm on critics, critic ensembles — turn offline data into a usable head start instead of a liability.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Sample efficiency and exploration remain major challenges in online reinforcement learning (RL). A powerful approach that can be applied to address these issues is the inclusion of offline data, such as prior trajectories from a human expert or a sub-optimal exploration policy. Previous methods have relied on extensive modifications and additional complexity to ensure the effective use of this data. Instead, we ask: can we simply apply existing off-policy methods to leverage offline data when learning online? In this work, we demonstrate that the answer is yes; however, a set of minimal but important changes to existing off-policy RL algorithms are required to achieve reliable performance. We extensively ablate these design choices, demonstrating the key factors that most affect performance, and arrive at a set of recommendations that practitioners can readily apply, whether their data comprise a small number of expert demonstrations or large volumes of sub-optimal trajectories. We see that correct application of these simple recommendations can provide a $\\mathbf{2.5\\times}$ improvement over existing approaches across a diverse set of competitive benchmarks, with no additional computational overhead. We have released our code at https://github.com/ikostrikov/rlpd.", "source": "curated", "first_author": "Philip J. Ball", "url": "https://arxiv.org/abs/2302.02948", "fig": "figures/rlpd.webp"}, {"id": "egoexo4d", "title": "Ego-Exo4D: Understanding Skilled Human Activity from First- and Third-Person Perspectives", "tree": "il", "branch": "human-video", "authors": ["Kristen Grauman", "Andrew Westbury", "Lorenzo Torresani", "Kris Kitani", "Jitendra Malik", "et al."], "corresponding": [], "institutions": ["Meta AI", "15 universities"], "lab": "FAIR", "venue": "CVPR", "venue_type": "conference", "year": 2023, "date": "2023-11-30", "topics": ["ego-exo paired video", "skilled activity"], "paradigm": ["dataset"], "method": [], "projects": ["r2s2r"], "related": ["ego4d", "humanoid-policy", "open-aoe"], "arxiv": "2311.18259", "doi": "", "code": "", "site": "", "note": "Time-synchronized first- and third-person video of the same skilled activity. This is the dataset that makes your third-person-vs-egocentric comparison an experiment rather than a debate.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We present Ego-Exo4D, a diverse, large-scale multimodal multiview video dataset and benchmark challenge. Ego-Exo4D centers around simultaneously-captured egocentric and exocentric video of skilled human activities (e.g., sports, music, dance, bike repair). 740 participants from 13 cities worldwide performed these activities in 123 different natural scene contexts, yielding long-form captures from 1 to 42 minutes each and 1,286 hours of video combined. The multimodal nature of the dataset is unprecedented: the video is accompanied by multichannel audio, eye gaze, 3D point clouds, camera poses, IMU, and multiple paired language descriptions -- including a novel \"expert commentary\" done by coaches and teachers and tailored to the skilled-activity domain. To push the frontier of first-person video understanding of skilled human activity, we also present a suite of benchmark tasks and their annotations, including fine-grained activity understanding, proficiency estimation, cross-view translation, and 3D hand/body pose. All resources are open sourced to fuel new research in the community. Project page: http://ego-exo4d-data.org/", "source": "curated", "first_author": "Kristen Grauman", "url": "https://research-information.bris.ac.uk/en/publications/52745b26-defc-438a-9da8-fc779020c594", "arxiv_url": "https://arxiv.org/abs/2311.18259", "fig": "figures/egoexo4d.webp", "figs": ["figures/egoexo4d-1.webp"]}, {"id": "leap-hand", "title": "LEAP Hand: Low-Cost, Efficient, and Anthropomorphic Hand for Robot Learning", "tree": "systems", "branch": "hands", "authors": ["Kenneth Shaw", "Ananye Agarwal", "Deepak Pathak"], "corresponding": ["Kenneth Shaw"], "institutions": ["Carnegie Mellon University"], "lab": "CMU Pathak Lab", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-09-12", "topics": ["low-cost hand", "open source", "anthropomorphic"], "paradigm": ["hardware"], "method": ["novel MCP joint kinematics giving universal abduction", "~$2000 printable BOM"], "projects": ["geodex"], "related": ["aero-hand", "allegro", "dexwrist", "dlr-hand", "orca-hand", "ruka", "ruka-v2", "shadow-hand", "sonicsense", "videodex"], "arxiv": "2309.06440", "doi": "", "code": "", "site": "https://leaphand.com/", "note": "The hand that made dexterity research affordable. The kinematic contribution is real — the MCP design keeps the fingers dexterous in every pose, which the cheap serial-chain alternatives do not. Start here for the 22-DOF open hand idea and note what LEAP deliberately gives up (no tendons, no wrist, limited fingertip force).", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Dexterous manipulation has been a long-standing challenge in robotics. While machine learning techniques have shown some promise, results have largely been currently limited to simulation. This can be mostly attributed to the lack of suitable hardware. In this paper, we present LEAP Hand, a low-cost dexterous and anthropomorphic hand for machine learning research. In contrast to previous hands, LEAP Hand has a novel kinematic structure that allows maximal dexterity regardless of finger pose. LEAP Hand is low-cost and can be assembled in 4 hours at a cost of 2000 USD from readily available parts. It is capable of consistently exerting large torques over long durations of time. We show that LEAP Hand can be used to perform several manipulation tasks in the real world -- from visual teleoperation to learning from passive video data and sim2real. LEAP Hand significantly outperforms its closest competitor Allegro Hand in all our experiments while being 1/8th of the cost. We release detailed assembly instructions, the Sim2Real pipeline and a development platform with useful APIs on our website at https://leap-hand.github.io/", "source": "curated", "first_author": "Kenneth Shaw", "url": "https://arxiv.org/abs/2309.06440", "fig": "figures/leap-hand.webp", "figs": ["figures/leap-hand-1.webp", "figures/leap-hand-2.webp", "figures/leap-hand-3.webp"]}, {"id": "act", "title": "Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware", "tree": "il", "branch": "visuomotor", "authors": ["Tony Z. Zhao", "Vikash Kumar", "Sergey Levine", "Chelsea Finn"], "corresponding": ["Tony Z. Zhao"], "institutions": ["Stanford University", "UC Berkeley", "Meta AI"], "lab": "Stanford IRIS Lab (Chelsea Finn)", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-04-23", "topics": ["action chunking", "bimanual manipulation", "low-cost hardware"], "paradigm": ["IL", "CVAE"], "method": ["action-chunking transformer with CVAE latent", "temporal ensembling"], "projects": ["iris"], "related": ["aloha", "aloha2", "alohaunleashed", "dp", "hrdt", "implicit-bc", "mobile-aloha", "openvla-oft", "pi0", "rdt1b"], "arxiv": "2304.13705", "doi": "", "code": "", "site": "", "note": "ACT + ALOHA, the paper your IRIS CVAE stack descends from. The two ideas that carried: predict a chunk of future actions rather than one step (kills compounding error), and ensemble overlapping chunks at inference (kills jitter). The CVAE latent matters less than either.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Fine manipulation tasks, such as threading cable ties or slotting a battery, are notoriously difficult for robots because they require precision, careful coordination of contact forces, and closed-loop visual feedback. Performing these tasks typically requires high-end robots, accurate sensors, or careful calibration, which can be expensive and difficult to set up. Can learning enable low-cost and imprecise hardware to perform these fine manipulation tasks? We present a low-cost system that performs end-to-end imitation learning directly from real demonstrations, collected with a custom teleoperation interface. Imitation learning, however, presents its own challenges, particularly in high-precision domains: errors in the policy can compound over time, and human demonstrations can be non-stationary. To address these challenges, we develop a simple yet novel algorithm, Action Chunking with Transformers (ACT), which learns a generative model over action sequences. ACT allows the robot to learn 6 difficult tasks in the real world, such as opening a translucent condiment cup and slotting a battery with 80-90% success, with only 10 minutes worth of demonstrations. Project website: https://tonyzhaozh.github.io/aloha/", "source": "curated", "first_author": "Tony Z. Zhao", "url": "https://arxiv.org/abs/2304.13705", "fig": "figures/act.webp", "figs": ["figures/act-1.webp", "figures/act-2.webp"]}, {"id": "oxe", "title": "Open X-Embodiment: Robotic Learning Datasets and RT-X Models", "tree": "il", "branch": "data-scaling", "authors": ["Open X-Embodiment Collaboration (Abby O'Neill", "Abdul Rehman", "Abhinav Gupta", "et al.)"], "corresponding": [], "institutions": ["34 labs worldwide"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2023, "date": "2023-10-13", "topics": ["cross-embodiment dataset", "RT-X"], "paradigm": ["dataset"], "method": [], "projects": [], "related": ["agibot-world", "bridgedata", "crossformer", "dlr-vla", "droid", "generalist-ai", "octo", "openvla", "rt1"], "arxiv": "2310.08864", "doi": "", "code": "", "site": "", "note": "1M+ trajectories, 22 embodiments, 34 labs. The ImageNet moment for robot data, and the training set underneath Octo, OpenVLA and most open generalist policies.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Large, high-capacity models trained on diverse datasets have shown remarkable successes on efficiently tackling downstream applications. In domains from NLP to Computer Vision, this has led to a consolidation of pretrained models, with general pretrained backbones serving as a starting point for many applications. Can such a consolidation happen in robotics? Conventionally, robotic learning methods train a separate model for every application, every robot, and even every environment. Can we instead train generalist X-robot policy that can be adapted efficiently to new robots, tasks, and environments? In this paper, we provide datasets in standardized data formats and models to make it possible to explore this possibility in the context of robotic manipulation, alongside experimental results that provide an example of effective X-robot policies. We assemble a dataset from 22 different robots collected through a collaboration between 21 institutions, demonstrating 527 skills (160266 tasks). We show that a high-capacity model trained on this data, which we call RT-X, exhibits positive transfer and improves the capabilities of multiple robots by leveraging experience from other platforms. More details can be found on the project website https://robotics-transformer-x.github.io.", "source": "curated", "first_author": "Open X-Embodiment Collaboration (Abby O'Neill", "url": "https://www.research.ed.ac.uk/en/publications/f5c31f8b-3f1e-4cde-bd8e-b2c12dde8a75", "arxiv_url": "https://arxiv.org/abs/2310.08864", "fig": "figures/oxe.webp", "figs": ["figures/oxe-1.webp", "figures/oxe-2.webp", "figures/oxe-3.webp"]}, {"id": "rt1", "title": "RT-1: Robotics Transformer for Real-World Control at Scale", "tree": "il", "branch": "visuomotor", "authors": ["Anthony Brohan", "Noah Brown", "Justice Carbajal", "Yevgen Chebotar", "Joseph Dabis", "et al."], "corresponding": [], "institutions": ["Google"], "lab": "Robotics at Google", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2022-12-13", "topics": ["robot transformer", "tokenized actions", "scale"], "paradigm": ["IL", "transformer"], "method": [], "projects": [], "related": ["icrt", "octo", "oxe", "pi0", "rt2"], "arxiv": "2212.06817", "doi": "", "code": "", "site": "", "note": "The first convincing demonstration that robot policies follow the data-scaling curve. Discretized action tokens, 130k episodes, 700 tasks.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "By transferring knowledge from large, diverse, task-agnostic datasets, modern machine learning models can solve specific downstream tasks either zero-shot or with small task-specific datasets to a high level of performance. While this capability has been demonstrated in other fields such as computer vision, natural language processing or speech recognition, it remains to be shown in robotics, where the generalization capabilities of the models are particularly critical due to the difficulty of collecting real-world robotic data. We argue that one of the keys to the success of such general robotic models lies with open-ended task-agnostic training, combined with high-capacity architectures that can absorb all of the diverse, robotic data. In this paper, we present a model class, dubbed Robotics Transformer, that exhibits promising scalable model properties. We verify our conclusions in a study of different model classes and their ability to generalize as a function of the data size, model size, and data diversity based on a large-scale data collection on real robots performing real-world tasks. The project's website and videos can be found at robotics-transformer1.github.io", "source": "curated", "first_author": "Anthony Brohan", "url": "https://arxiv.org/abs/2212.06817", "fig": "figures/rt1.webp", "figs": ["figures/rt1-1.webp"]}, {"id": "rt2", "title": "RT-2: Vision-Language-Action Models Transfer Web Knowledge to Robotic Control", "tree": "il", "branch": "vla", "authors": ["Anthony Brohan", "Noah Brown", "Justice Carbajal", "Yevgen Chebotar", "Xi Chen", "et al."], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind Robotics", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-07-28", "topics": ["vision-language-action", "web knowledge transfer"], "paradigm": ["VLA"], "method": ["co-fine-tune a VLM on web VQA and robot trajectories", "actions as text tokens"], "projects": [], "related": ["gemini-robotics", "openvla", "pi0", "pi05", "rt1"], "arxiv": "2307.15818", "doi": "", "code": "", "site": "", "note": "Defined the VLA category: actions as another language the VLM speaks, so web semantics leak into the policy. Everything from OpenVLA to π0.5 is arguing with this design.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We study how vision-language models trained on Internet-scale data can be incorporated directly into end-to-end robotic control to boost generalization and enable emergent semantic reasoning. Our goal is to enable a single end-to-end trained model to both learn to map robot observations to actions and enjoy the benefits of large-scale pretraining on language and vision-language data from the web. To this end, we propose to co-fine-tune state-of-the-art vision-language models on both robotic trajectory data and Internet-scale vision-language tasks, such as visual question answering. In contrast to other approaches, we propose a simple, general recipe to achieve this goal: in order to fit both natural language responses and robotic actions into the same format, we express the actions as text tokens and incorporate them directly into the training set of the model in the same way as natural language tokens. We refer to such category of models as vision-language-action models (VLA) and instantiate an example of such a model, which we call RT-2. Our extensive evaluation (6k evaluation trials) shows that our approach leads to performant robotic policies and enables RT-2 to obtain a range of emergent capabilities from Internet-scale training. This includes significantly improved generalization to novel objects, the ability to interpret commands not present in the robot training data (such as placing an object onto a particular number or icon), and the ability to perform rudimentary reasoning in response to user commands (such as picking up the smallest or largest object, or the one closest to another object). We further show that incorporating chain of thought reasoning allows RT-2 to perform multi-stage semantic reasoning, for example figuring out which object to pick up for use as an improvised hammer (a rock), or which type of drink is best suited for someone who is tired (an energy drink).", "source": "curated", "first_author": "Anthony Brohan", "url": "https://arxiv.org/abs/2307.15818", "fig": "figures/rt2.webp", "figs": ["figures/rt2-1.webp"]}, {"id": "roco", "title": "RoCo: Dialectic Multi-Robot Collaboration with Large Language Models", "tree": "rl", "branch": "marl", "authors": ["Zhao Mandi", "Shreeya Jain", "Shuran Song"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-07-10", "topics": ["LLM dialogue", "multi-arm planning", "benchmark"], "paradigm": ["LLM", "MARL"], "method": [], "projects": ["m2"], "related": ["chorus", "climrs", "embo-team", "interact", "llm-teams", "mappo", "voyager"], "arxiv": "2307.04738", "doi": "10.48550/arxiv.2307.04738", "code": "", "site": "", "note": "Robots that literally talk to each other: LLMs negotiate the task split, then a multi-arm motion planner executes the agreed waypoints. RoCoBench came out of it and is now the standard evaluation. The human-in-the-loop mode — a person as one of the agents — is the cleanest existing bridge between your multi-robot and HRI ideas.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We propose a novel approach to multi-robot collaboration that harnesses the power of pre-trained large language models (LLMs) for both high-level communication and low-level path planning. Robots are equipped with LLMs to discuss and collectively reason task strategies. They then generate sub-task plans and task space waypoint paths, which are used by a multi-arm motion planner to accelerate trajectory planning. We also provide feedback from the environment, such as collision checking, and prompt the LLM agents to improve their plan and waypoints in-context. For evaluation, we introduce RoCoBench, a 6-task benchmark covering a wide range of multi-robot collaboration scenarios, accompanied by a text-only dataset for agent representation and reasoning. We experimentally demonstrate the effectiveness of our approach -- it achieves high success rates across all tasks in RoCoBench and adapts to variations in task semantics. Our dialog setup offers high interpretability and flexibility -- in real world experiments, we show RoCo easily incorporates human-in-the-loop, where a user can communicate and collaborate with a robot agent to complete tasks together. See project website https://project-roco.github.io for videos and code.", "source": "curated", "first_author": "Zhao Mandi", "url": "https://arxiv.org/abs/2307.04738", "fig": "figures/roco.webp", "figs": ["figures/roco-1.webp", "figures/roco-2.webp"]}, {"id": "chen-visual-inhand", "title": "Visual Dexterity: In-Hand Reorientation of Novel and Complex Object Shapes", "tree": "rl", "branch": "manipulation", "authors": ["Tao Chen", "Megha Tippur", "Siyang Wu", "Vikash Kumar", "Edward Adelson", "Pulkit Agrawal"], "corresponding": ["Tao Chen", "Pulkit Agrawal"], "institutions": ["MIT", "Meta AI"], "lab": "MIT Improbable AI Lab", "venue": "Science Robotics", "venue_type": "journal", "year": 2023, "date": "2022-11-21", "topics": ["vision-based in-hand reorientation", "novel objects"], "paradigm": ["RL"], "method": ["depth-only policy", "teacher-student", "real-time reorientation of unseen shapes"], "projects": ["geodex"], "related": ["artic-tools", "chen-inhand", "clutterdexgrasp", "dextreme", "humanoid-dex-s2r", "qi-inhand"], "arxiv": "2211.11744", "doi": "", "code": "", "site": "", "note": "The vision-based version — single depth camera, novel and complex shapes, no object model. This is the benchmark your GeoDex work sits against.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "In-hand object reorientation is necessary for performing many dexterous manipulation tasks, such as tool use in less structured environments that remain beyond the reach of current robots. Prior works built reorientation systems assuming one or many of the following: reorienting only specific objects with simple shapes, limited range of reorientation, slow or quasistatic manipulation, simulation-only results, the need for specialized and costly sensor suites, and other constraints which make the system infeasible for real-world deployment. We present a general object reorientation controller that does not make these assumptions. It uses readings from a single commodity depth camera to dynamically reorient complex and new object shapes by any rotation in real-time, with the median reorientation time being close to seven seconds. The controller is trained using reinforcement learning in simulation and evaluated in the real world on new object shapes not used for training, including the most challenging scenario of reorienting objects held in the air by a downward-facing hand that must counteract gravity during reorientation. Our hardware platform only uses open-source components that cost less than five thousand dollars. Although we demonstrate the ability to overcome assumptions in prior work, there is ample scope for improving absolute performance. For instance, the challenging duck-shaped object not used for training was dropped in 56 percent of the trials. When it was not dropped, our controller reoriented the object within 0.4 radians (23 degrees) 75 percent of the time. Videos are available at: https://taochenshh.github.io/projects/visual-dexterity.", "source": "curated", "first_author": "Tao Chen", "url": "https://arxiv.org/abs/2211.11744", "fig": "figures/chen-visual-inhand.webp"}, {"id": "voxposer", "title": "VoxPoser: Composable 3D Value Maps for Robotic Manipulation with Language Models", "tree": "systems", "branch": "planning-control", "authors": ["Wenlong Huang", "Chen Wang", "Ruohan Zhang", "Yunzhu Li", "Jiajun Wu", "Li Fei-Fei"], "corresponding": [], "institutions": ["Stanford University", "University of Illinois Urbana-Champaign"], "lab": "Stanford Vision & Learning (SVL)", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-07-12", "topics": ["LLM-composed 3D value maps", "zero-shot manipulation", "no task training"], "paradigm": ["LLM planning"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["acdc", "aim-wam", "behavior-robot-suite", "d3fields", "rekep", "robo-exp"], "arxiv": "2307.05973", "doi": "10.48550/arXiv.2307.05973", "code": "", "site": "", "note": "<b>VoxPoser</b> (CoRL 2023) — an LLM writes code that composes <b>3D value maps</b> (affordance and constraint voxel grids) which a motion planner then optimizes through, giving zero-shot manipulation with no task-specific training data at all. The lasting idea is the interface: the language model is not asked to output actions, it is asked to output an <em>objective</em>, and a classical optimizer supplies the competence. That division of labour is what makes agentic robot systems work.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Large language models (LLMs) are shown to possess a wealth of actionable knowledge that can be extracted for robot manipulation in the form of reasoning and planning. Despite the progress, most still rely on pre-defined motion primitives to carry out the physical interactions with the environment, which remains a major bottleneck. In this work, we aim to synthesize robot trajectories, i.e., a dense sequence of 6-DoF end-effector waypoints, for a large variety of manipulation tasks given an open-set of instructions and an open-set of objects. We achieve this by first observing that LLMs excel at inferring affordances and constraints given a free-form language instruction. More importantly, by leveraging their code-writing capabilities, they can interact with a vision-language model (VLM) to compose 3D value maps to ground the knowledge into the observation space of the agent. The composed value maps are then used in a model-based planning framework to zero-shot synthesize closed-loop robot trajectories with robustness to dynamic perturbations. We further demonstrate how the proposed framework can benefit from online experiences by efficiently learning a dynamics model for scenes that involve contact-rich interactions. We present a large-scale study of the proposed method in both simulated and real-robot environments, showcasing the ability to perform a large variety of everyday manipulation tasks specified in free-form natural language. Videos and code at https://voxposer.github.io", "source": "curated", "first_author": "Wenlong Huang", "url": "https://arxiv.org/abs/2307.05973", "fig": "figures/voxposer.webp", "figs": ["figures/voxposer-1.webp", "figures/voxposer-2.webp"]}, {"id": "daydreamer", "title": "DayDreamer: World Models for Physical Robot Learning", "tree": "rl", "branch": "model-based", "authors": ["Philipp Wu", "Alejandro Escontrela", "Danijar Hafner", "Ken Goldberg", "Pieter Abbeel"], "corresponding": ["Philipp Wu"], "institutions": ["UC Berkeley"], "lab": "BAIR", "venue": "CoRL", "venue_type": "conference", "year": 2022, "date": "2022-06-28", "topics": ["real-world model-based RL", "sample efficiency"], "paradigm": ["model-based RL"], "method": ["Dreamer trained directly on physical robots", "no simulator"], "projects": ["wam-tactile"], "related": ["dreamer-v1", "dreamerv3", "dreamerv4", "sac", "serl", "tdmpc2"], "arxiv": "2206.14176", "doi": "", "code": "", "site": "", "note": "In your ~/Desktop/papers/RL. A quadruped learns to walk in one hour of real experience, no simulator. The single best argument that world models buy you real-world sample efficiency.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/daydreamer.pdf", "abstract": "To solve tasks in complex environments, robots need to learn from experience. Deep reinforcement learning is a common approach to robot learning but requires a large amount of trial and error to learn, limiting its deployment in the physical world. As a consequence, many advances in robot learning rely on simulators. On the other hand, learning inside of simulators fails to capture the complexity of the real world, is prone to simulator inaccuracies, and the resulting behaviors do not adapt to changes in the world. The Dreamer algorithm has recently shown great promise for learning from small amounts of interaction by planning within a learned world model, outperforming pure reinforcement learning in video games. Learning a world model to predict the outcomes of potential actions enables planning in imagination, reducing the amount of trial and error needed in the real environment. However, it is unknown whether Dreamer can facilitate faster learning on physical robots. In this paper, we apply Dreamer to 4 robots to learn online and directly in the real world, without simulators. Dreamer trains a quadruped robot to roll off its back, stand up, and walk from scratch and without resets in only 1 hour. We then push the robot and find that Dreamer adapts within 10 minutes to withstand perturbations or quickly roll over and stand back up. On two different robotic arms, Dreamer learns to pick and place multiple objects directly from camera images and sparse rewards, approaching human performance. On a wheeled robot, Dreamer learns to navigate to a goal position purely from camera images, automatically resolving ambiguity about the robot orientation. Using the same hyperparameters across all experiments, we find that Dreamer is capable of online learning in the real world, establishing a strong baseline. We release our infrastructure for future applications of world models to robot learning.", "source": "curated", "first_author": "Philipp Wu", "url": "https://arxiv.org/abs/2206.14176", "fig": "figures/daydreamer.webp", "figs": ["figures/daydreamer-1.webp", "figures/daydreamer-2.webp", "figures/daydreamer-3.webp"]}, {"id": "ego4d", "title": "Ego4D: Around the World in 3,000 Hours of Egocentric Video", "tree": "il", "branch": "human-video", "authors": ["Kristen Grauman", "Andrew Westbury", "Eugene Byrne", "Zachary Chavis", "Antonino Furnari", "et al."], "corresponding": ["Kristen Grauman"], "institutions": ["Meta AI", "UT Austin", "88 institutions"], "lab": "FAIR", "venue": "CVPR", "venue_type": "conference", "year": 2022, "date": "2021-10-13", "topics": ["egocentric video dataset", "benchmark"], "paradigm": ["dataset"], "method": [], "projects": ["r2s2r"], "related": ["egoexo4d", "egolive", "egomimic", "human-video-survey", "humanoid-policy", "open-aoe", "r3m", "videodex"], "arxiv": "2110.07058", "doi": "", "code": "", "site": "", "note": "The substrate for the whole egocentric-learning agenda. 3,670 hours, 74 locations, 9 countries. If you are building an egocentric dexterity pipeline this is the pretraining corpus.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We introduce Ego4D, a massive-scale egocentric video dataset and benchmark suite. It offers 3,670 hours of daily-life activity video spanning hundreds of scenarios (household, outdoor, workplace, leisure, etc.) captured by 931 unique camera wearers from 74 worldwide locations and 9 different countries. The approach to collection is designed to uphold rigorous privacy and ethics standards with consenting participants and robust de-identification procedures where relevant. Ego4D dramatically expands the volume of diverse egocentric video footage publicly available to the research community. Portions of the video are accompanied by audio, 3D meshes of the environment, eye gaze, stereo, and/or synchronized videos from multiple egocentric cameras at the same event. Furthermore, we present a host of new benchmark challenges centered around understanding the first-person visual experience in the past (querying an episodic memory), present (analyzing hand-object manipulation, audio-visual conversation, and social interactions), and future (forecasting activities). By publicly sharing this massive annotated dataset and benchmark suite, we aim to push the frontier of first-person perception. Project page: https://ego4d-data.org/", "source": "curated", "first_author": "Kristen Grauman", "url": "https://arxiv.org/abs/2110.07058", "fig": "figures/ego4d.webp", "figs": ["figures/ego4d-1.webp", "figures/ego4d-2.webp", "figures/ego4d-3.webp"]}, {"id": "miki2022", "title": "Learning robust perceptive locomotion for quadrupedal robots in the wild", "tree": "rl", "branch": "locomotion", "authors": ["Takahiro Miki", "Joonho Lee", "Jemin Hwangbo", "Lorenz Wellhausen", "Vladlen Koltun", "Marco Hutter"], "corresponding": ["Takahiro Miki"], "institutions": ["ETH Zürich"], "lab": "ETH RSL", "venue": "Science Robotics", "venue_type": "journal", "year": 2022, "date": "2022-01-19", "topics": ["perceptive locomotion", "exteroception", "belief state"], "paradigm": ["RL"], "method": ["attention-gated belief encoder fusing exteroception with proprioception"], "projects": [], "related": ["anymal-lee", "anymal-parkour", "attention-map-hutter", "extreme-parkour"], "arxiv": "2201.08117", "doi": "", "code": "", "site": "", "note": "Solves the real failure mode of perceptive locomotion: the elevation map is often wrong. The belief encoder learns *when to distrust vision* and fall back on proprioception.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Legged robots that can operate autonomously in remote and hazardous environments will greatly increase opportunities for exploration into under-explored areas. Exteroceptive perception is crucial for fast and energy-efficient locomotion: perceiving the terrain before making contact with it enables planning and adaptation of the gait ahead of time to maintain speed and stability. However, utilizing exteroceptive perception robustly for locomotion has remained a grand challenge in robotics. Snow, vegetation, and water visually appear as obstacles on which the robot cannot step~-- or are missing altogether due to high reflectance. Additionally, depth perception can degrade due to difficult lighting, dust, fog, reflective or transparent surfaces, sensor occlusion, and more. For this reason, the most robust and general solutions to legged locomotion to date rely solely on proprioception. This severely limits locomotion speed, because the robot has to physically feel out the terrain before adapting its gait accordingly. Here we present a robust and general solution to integrating exteroceptive and proprioceptive perception for legged locomotion. We leverage an attention-based recurrent encoder that integrates proprioceptive and exteroceptive input. The encoder is trained end-to-end and learns to seamlessly combine the different perception modalities without resorting to heuristics. The result is a legged locomotion controller with high robustness and speed. The controller was tested in a variety of challenging natural and urban environments over multiple seasons and completed an hour-long hike in the Alps in the time recommended for human hikers.", "source": "curated", "first_author": "Takahiro Miki", "url": "https://arxiv.org/abs/2201.08117", "fig": "figures/miki2022.webp", "figs": ["figures/miki2022-1.webp"]}, {"id": "r3m", "title": "R3M: A Universal Visual Representation for Robot Manipulation", "tree": "il", "branch": "human-video", "authors": ["Suraj Nair", "Aravind Rajeswaran", "Vikash Kumar", "Chelsea Finn", "Abhinav Gupta"], "corresponding": ["Suraj Nair"], "institutions": ["Stanford University", "Meta AI"], "lab": "Stanford IRIS; FAIR", "venue": "CoRL", "venue_type": "conference", "year": 2022, "date": "2022-03-23", "topics": ["visual representation", "Ego4D pretraining"], "paradigm": ["representation learning"], "method": ["time-contrastive + video-language alignment on Ego4D"], "projects": ["r2s2r"], "related": ["ego4d", "egoscale", "hearing-touch", "vip", "vjepa2"], "arxiv": "2203.12601", "doi": "", "code": "", "site": "", "note": "Pretrain a frozen visual encoder on human video, then do BC on top with 10x fewer demos. The first widely-used demonstration that human video transfers to robot control at all.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We study how visual representations pre-trained on diverse human video data can enable data-efficient learning of downstream robotic manipulation tasks. Concretely, we pre-train a visual representation using the Ego4D human video dataset using a combination of time-contrastive learning, video-language alignment, and an L1 penalty to encourage sparse and compact representations. The resulting representation, R3M, can be used as a frozen perception module for downstream policy learning. Across a suite of 12 simulated robot manipulation tasks, we find that R3M improves task success by over 20% compared to training from scratch and by over 10% compared to state-of-the-art visual representations like CLIP and MoCo. Furthermore, R3M enables a Franka Emika Panda arm to learn a range of manipulation tasks in a real, cluttered apartment given just 20 demonstrations. Code and pre-trained models are available at https://tinyurl.com/robotr3m.", "source": "curated", "first_author": "Suraj Nair", "url": "https://arxiv.org/abs/2203.12601", "fig": "figures/r3m.webp", "figs": ["figures/r3m-1.webp"]}, {"id": "robocraft", "title": "RoboCraft: Learning to See, Simulate, and Shape Elasto-Plastic Objects with Graph Networks", "tree": "il", "branch": "world-models", "authors": ["Haochen Shi", "Huazhe Xu", "Zhiao Huang", "Yunzhu Li", "Jiajun Wu"], "corresponding": [], "institutions": ["Stanford University", "MIT", "UC San Diego"], "lab": "RoboPIL (Yunzhu Li)", "venue": "RSS", "venue_type": "conference", "year": 2022, "date": "2022-05-05", "topics": ["graph neural dynamics", "elasto-plastic objects", "particle representation"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["adaptigraph", "d3fields", "particle-grid-dynamics", "phystwin", "robocook"], "arxiv": "2205.02909", "doi": "10.1177/02783649231219020", "code": "", "site": "", "note": "<b>RoboCraft</b> — the start of Yunzhu Li's structured-world-model line. Represent dough as <b>particles</b>, learn a GNN dynamics model over them from real interaction, and plan to shape it. The argument that runs through this whole group's work begins here: a world model with the right <em>structure</em> (particles, graphs, springs) needs orders of magnitude less data than a video model and extrapolates instead of interpolating.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Modeling and manipulating elasto-plastic objects are essential capabilities for robots to perform complex industrial and household interaction tasks (e.g., stuffing dumplings, rolling sushi, and making pottery). However, due to the high degree of freedom of elasto-plastic objects, significant challenges exist in virtually every aspect of the robotic manipulation pipeline, e.g., representing the states, modeling the dynamics, and synthesizing the control signals. We propose to tackle these challenges by employing a particle-based representation for elasto-plastic objects in a model-based planning framework. Our system, RoboCraft, only assumes access to raw RGBD visual observations. It transforms the sensing data into particles and learns a particle-based dynamics model using graph neural networks (GNNs) to capture the structure of the underlying system. The learned model can then be coupled with model-predictive control (MPC) algorithms to plan the robot's behavior. We show through experiments that with just 10 minutes of real-world robotic interaction data, our robot can learn a dynamics model that can be used to synthesize control signals to deform elasto-plastic objects into various target shapes, including shapes that the robot has never encountered before. We perform systematic evaluations in both simulation and the real world to demonstrate the robot's manipulation capabilities and ability to generalize to a more complex action space, different tool shapes, and a mixture of motion modes. We also conduct comparisons between RoboCraft and untrained human subjects controlling the gripper to manipulate deformable objects in both simulation and the real world. Our learned model-based planning framework is comparable to and sometimes better than human subjects on the tested tasks.", "source": "curated", "first_author": "Haochen Shi", "url": "https://arxiv.org/abs/2205.02909", "fig": "figures/robocraft.webp", "figs": ["figures/robocraft-1.webp", "figures/robocraft-2.webp", "figures/robocraft-3.webp"]}, {"id": "robosen", "title": "Robosen auto-converting Transformers (Optimus Prime, Grimlock)", "tree": "systems", "branch": "codesign", "authors": ["Robosen Robotics"], "corresponding": [], "institutions": ["Robosen Robotics", "Hasbro"], "lab": "Robosen", "venue": "commercial", "venue_type": "industry", "year": 2022, "date": "2022-01-01", "topics": ["auto-converting bipedal robot", "high servo count", "consumer scale"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["berkeley-humanoid-lite", "jdeite", "letrons", "toddlerbot"], "arxiv": "", "doi": "", "code": "", "site": "https://us.robosen.com/", "note": "The most under-rated engineering reference here: a consumer product that converts <b>automatically</b> between a walking biped and a vehicle, with 34 servos and no human in the loop. At ~0.5 m it is the scale a PhD can actually iterate on, and it proves the mechanism is tractable when you stop trying to build it at 4 m. Closed-source and scripted — but the packaging is a masterclass.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Robosen Robotics", "url": "https://us.robosen.com/"}, {"id": "td-mpc", "title": "Temporal Difference Learning for Model Predictive Control", "tree": "rl", "branch": "model-based", "authors": ["Nicklas Hansen", "Xiaolong Wang", "Hao Su"], "corresponding": [], "institutions": ["UC San Diego"], "lab": "", "venue": "ICML", "venue_type": "conference", "year": 2022, "date": "2022-03-09", "topics": ["task-oriented latent dynamics", "no reconstruction", "MPC with a learned value"], "paradigm": ["model-based RL"], "method": [], "projects": ["geodex", "m2", "wam-tactile"], "related": ["bab-nd", "dino-wm", "dreamer-v2", "planet", "pwm", "tdmpc2"], "arxiv": "2203.04955", "doi": "10.48550/arXiv.2203.04955", "code": "", "site": "", "note": "<b>TD-MPC</b> — the counter-argument to reconstruction. Do not model the pixels; model only what the <em>reward and value</em> depend on, and plan short-horizon MPC with a learned terminal value to cover the rest. It is the cheapest good world model in the literature, and for a contact-rich task where the visual scene is mostly irrelevant clutter, it is very often the right one. The direct ancestor of TD-MPC2.", "aka": "TD-MPC", "stars": 3, "curated": true, "local": "", "abstract": "Data-driven model predictive control has two key advantages over model-free methods: a potential for improved sample efficiency through model learning, and better performance as computational budget for planning increases. However, it is both costly to plan over long horizons and challenging to obtain an accurate model of the environment. In this work, we combine the strengths of model-free and model-based methods. We use a learned task-oriented latent dynamics model for local trajectory optimization over a short horizon, and use a learned terminal value function to estimate long-term return, both of which are learned jointly by temporal difference learning. Our method, TD-MPC, achieves superior sample efficiency and asymptotic performance over prior work on both state and image-based continuous control tasks from DMControl and Meta-World. Code and video results are available at https://nicklashansen.github.io/td-mpc.", "source": "curated", "first_author": "Nicklas Hansen", "url": "https://arxiv.org/abs/2203.04955", "fig": "figures/td-mpc.webp", "figs": ["figures/td-mpc-1.webp"], "arxiv_url": "https://arxiv.org/abs/2203.04955"}, {"id": "mappo", "title": "The Surprising Effectiveness of PPO in Cooperative, Multi-Agent Games", "tree": "rl", "branch": "marl", "authors": ["Chao Yu", "Akash Velu", "Eugene Vinitsky", "Jiaxuan Gao", "Yu Wang", "Alexandre Bayen", "Yi Wu"], "corresponding": ["Chao Yu"], "institutions": ["Tsinghua University", "UC Berkeley", "New York University"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2022, "date": "2021-03-02", "topics": ["cooperative MARL", "CTDE"], "paradigm": ["MARL"], "method": ["PPO with centralized value function", "decentralized execution"], "projects": ["m2"], "related": ["chorus", "climrs", "decentralized-aerial", "maddpg", "ppo", "qmix", "roco"], "arxiv": "2103.01955", "doi": "", "code": "", "site": "", "note": "The MARL baseline you should start M2 from. Centralized-critic PPO beats most specialized MARL algorithms once you tune it properly, and it is far simpler to debug.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Proximal Policy Optimization (PPO) is a ubiquitous on-policy reinforcement learning algorithm but is significantly less utilized than off-policy learning algorithms in multi-agent settings. This is often due to the belief that PPO is significantly less sample efficient than off-policy methods in multi-agent systems. In this work, we carefully study the performance of PPO in cooperative multi-agent settings. We show that PPO-based multi-agent algorithms achieve surprisingly strong performance in four popular multi-agent testbeds: the particle-world environments, the StarCraft multi-agent challenge, Google Research Football, and the Hanabi challenge, with minimal hyperparameter tuning and without any domain-specific algorithmic modifications or architectures. Importantly, compared to competitive off-policy methods, PPO often achieves competitive or superior results in both final returns and sample efficiency. Finally, through ablation studies, we analyze implementation and hyperparameter factors that are critical to PPO's empirical performance, and give concrete practical suggestions regarding these factors. Our results show that when using these practices, simple PPO-based methods can be a strong baseline in cooperative multi-agent reinforcement learning. Source code is released at \\url{https://github.com/marlbenchmark/on-policy}.", "source": "curated", "first_author": "Chao Yu", "url": "https://arxiv.org/abs/2103.01955", "fig": "figures/mappo.webp"}, {"id": "transform2act", "title": "Transform2Act: Learning a Transform-and-Control Policy for Efficient Agent Design", "tree": "systems", "branch": "codesign", "authors": ["Ye Yuan", "Yuda Song", "Zhengyi Luo", "Wen Sun", "Kris Kitani"], "corresponding": ["Ye Yuan"], "institutions": ["Carnegie Mellon University", "Cornell University"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2022, "date": "2021-10-07", "topics": ["design as action", "morphology optimization"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["codesign-rl", "derl", "dinev-codesign", "dyret", "gcnt", "hardware-as-policy", "meta-rl-legged", "neural-graph-evo", "robogrammar", "s-cheetah", "taskagnostic-morph"], "arxiv": "2110.03659", "doi": "", "code": "", "site": "", "note": "The cleanest formulation in the area: make the design choices *actions* in an extended MDP, so one RL algorithm optimizes body and behaviour with no separate outer loop. This is the formulation to start from for the quadruped co-design idea.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "An agent's functionality is largely determined by its design, i.e., skeletal structure and joint attributes (e.g., length, size, strength). However, finding the optimal agent design for a given function is extremely challenging since the problem is inherently combinatorial and the design space is prohibitively large. Additionally, it can be costly to evaluate each candidate design which requires solving for its optimal controller. To tackle these problems, our key idea is to incorporate the design procedure of an agent into its decision-making process. Specifically, we learn a conditional policy that, in an episode, first applies a sequence of transform actions to modify an agent's skeletal structure and joint attributes, and then applies control actions under the new design. To handle a variable number of joints across designs, we use a graph-based policy where each graph node represents a joint and uses message passing with its neighbors to output joint-specific actions. Using policy gradient methods, our approach enables joint optimization of agent design and control as well as experience sharing across different designs, which improves sample efficiency substantially. Experiments show that our approach, Transform2Act, outperforms prior methods significantly in terms of convergence speed and final performance. Notably, Transform2Act can automatically discover plausible designs similar to giraffes, squids, and spiders. Code and videos are available at https://sites.google.com/view/transform2act.", "source": "curated", "first_author": "Ye Yuan", "url": "https://arxiv.org/abs/2110.03659", "fig": "figures/transform2act.webp", "figs": ["figures/transform2act-1.webp"]}, {"id": "chen-inhand", "title": "A System for General In-Hand Object Re-Orientation", "tree": "rl", "branch": "manipulation", "authors": ["Tao Chen", "Jie Xu", "Pulkit Agrawal"], "corresponding": ["Tao Chen"], "institutions": ["MIT"], "lab": "MIT Improbable AI Lab", "venue": "CoRL", "venue_type": "conference", "year": 2021, "date": "2021-11-04", "topics": ["in-hand reorientation", "gravity-agnostic dexterity"], "paradigm": ["RL"], "method": ["teacher-student with privileged object state", "gravity curriculum"], "projects": ["geodex"], "related": ["chen-visual-inhand", "dactyl", "dexplore", "dextreme", "qi-inhand"], "arxiv": "2111.03043", "doi": "", "code": "", "site": "", "note": "Reorients 2000+ objects including hand-facing-down, which Dactyl could not do. Best-paper at CoRL 2021 and the point where in-hand RL stopped being one-object-one-paper.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "In-hand object reorientation has been a challenging problem in robotics due to high dimensional actuation space and the frequent change in contact state between the fingers and the objects. We present a simple model-free framework that can learn to reorient objects with both the hand facing upwards and downwards. We demonstrate the capability of reorienting over 2000 geometrically different objects in both cases. The learned policies show strong zero-shot transfer performance on new objects. We provide evidence that these policies are amenable to real-world operation by distilling them to use observations easily available in the real world. The videos of the learned policies are available at: https://taochenshh.github.io/projects/in-hand-reorientation.", "source": "curated", "first_author": "Tao Chen", "url": "https://hdl.handle.net/1721.1/143155", "arxiv_url": "https://arxiv.org/abs/2111.03043", "fig": "figures/chen-inhand.webp", "figs": ["figures/chen-inhand-1.webp", "figures/chen-inhand-2.webp"]}, {"id": "amp", "title": "AMP: Adversarial Motion Priors for Stylized Physics-Based Character Control", "tree": "rl", "branch": "motion-imitation", "authors": ["Xue Bin Peng", "Ze Ma", "Pieter Abbeel", "Sergey Levine", "Angjoo Kanazawa"], "corresponding": ["Xue Bin Peng"], "institutions": ["UC Berkeley"], "lab": "BAIR", "venue": "SIGGRAPH", "venue_type": "conference", "year": 2021, "date": "2021-04-05", "topics": ["adversarial imitation", "motion style", "unstructured mocap"], "paradigm": ["RL", "adversarial IL"], "method": ["discriminator-based style reward + task reward"], "projects": [], "related": ["add", "ase", "deepmimic", "disney-bdx", "gail"], "arxiv": "2104.02180", "doi": "", "code": "", "site": "", "note": "Replaces DeepMimic's per-frame tracking with a learned style discriminator, so you can train on an unstructured motion dataset and combine style with an arbitrary task reward. This is what let motion priors escape single-clip imitation.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Synthesizing graceful and life-like behaviors for physically simulated characters has been a fundamental challenge in computer animation. Data-driven methods that leverage motion tracking are a prominent class of techniques for producing high fidelity motions for a wide range of behaviors. However, the effectiveness of these tracking-based methods often hinges on carefully designed objective functions, and when applied to large and diverse motion datasets, these methods require significant additional machinery to select the appropriate motion for the character to track in a given scenario. In this work, we propose to obviate the need to manually design imitation objectives and mechanisms for motion selection by utilizing a fully automated approach based on adversarial imitation learning. High-level task objectives that the character should perform can be specified by relatively simple reward functions, while the low-level style of the character's behaviors can be specified by a dataset of unstructured motion clips, without any explicit clip selection or sequencing. These motion clips are used to train an adversarial motion prior, which specifies style-rewards for training the character through reinforcement learning (RL). The adversarial RL procedure automatically selects which motion to perform, dynamically interpolating and generalizing from the dataset. Our system produces high-quality motions that are comparable to those achieved by state-of-the-art tracking-based techniques, while also being able to easily accommodate large datasets of unstructured motion clips. Composition of disparate skills emerges automatically from the motion prior, without requiring a high-level motion planner or other task-specific annotations of the motion clips. We demonstrate the effectiveness of our framework on a diverse cast of complex simulated characters and a challenging suite of motor control tasks.", "source": "curated", "first_author": "Xue Bin Peng", "url": "https://escholarship.org/uc/item/9rh2t6tt", "arxiv_url": "https://arxiv.org/abs/2104.02180", "fig": "figures/amp.webp", "figs": ["figures/amp-1.webp"]}, {"id": "diff-contact-design", "title": "An End-to-End Differentiable Framework for Contact-Aware Robot Design", "tree": "systems", "branch": "codesign", "authors": ["Jie Xu", "Tao Chen", "Lara Zlokapa", "Michael Foshey", "Wojciech Matusik", "Shinjiro Sueda", "Pulkit Agrawal"], "corresponding": ["Jie Xu"], "institutions": ["MIT"], "lab": "MIT CSAIL; Improbable AI", "venue": "RSS", "venue_type": "conference", "year": 2021, "date": "2021-07-15", "topics": ["differentiable simulation", "contact-aware design", "hands"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["coros-codesign", "dinev-codesign", "fit2form", "hardware-as-policy", "task2morph"], "arxiv": "2107.07501", "doi": "", "code": "", "site": "", "note": "Differentiable simulation through contact, applied to designing manipulators and hands. The contact part is what makes it relevant — for dexterity the design objective *is* the contact behaviour.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The current dominant paradigm for robotic manipulation involves two separate stages: manipulator design and control. Because the robot's morphology and how it can be controlled are intimately linked, joint optimization of design and control can significantly improve performance. Existing methods for co-optimization are limited and fail to explore a rich space of designs. The primary reason is the trade-off between the complexity of designs that is necessary for contact-rich tasks against the practical constraints of manufacturing, optimization, contact handling, etc. We overcome several of these challenges by building an end-to-end differentiable framework for contact-aware robot design. The two key components of this framework are: a novel deformation-based parameterization that allows for the design of articulated rigid robots with arbitrary, complex geometry, and a differentiable rigid body simulator that can handle contact-rich scenarios and computes analytical gradients for a full spectrum of kinematic and dynamic parameters. On multiple manipulation tasks, our framework outperforms existing methods that either only optimize for control or for design using alternate representations or co-optimize using gradient-free methods.", "source": "curated", "first_author": "Jie Xu", "url": "https://arxiv.org/abs/2107.07501", "fig": "figures/diff-contact-design.webp", "figs": ["figures/diff-contact-design-1.webp", "figures/diff-contact-design-2.webp", "figures/diff-contact-design-3.webp"]}, {"id": "derl", "title": "Embodied Intelligence via Learning and Evolution", "tree": "systems", "branch": "codesign", "authors": ["Agrim Gupta", "Silvio Savarese", "Surya Ganguli", "Li Fei-Fei"], "corresponding": ["Agrim Gupta"], "institutions": ["Stanford University"], "lab": "Stanford Vision & Learning", "venue": "Nature Communications", "venue_type": "journal", "year": 2021, "date": "2021-02-03", "topics": ["evolution", "morphological intelligence", "Baldwin effect"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["codesign-rl", "evogym", "morph-hypernet", "neural-graph-evo", "sims1994", "taskagnostic-morph", "transform2act"], "arxiv": "2102.02202", "doi": "10.1038/s41467-021-25874-z", "code": "", "site": "", "note": "DERL. Shows morphology and learning speed co-evolve — better bodies are the ones that learn faster, not just the ones that score higher. The 'morphological Baldwin effect' is the result to cite when arguing co-design is more than hyperparameter search.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "The intertwined processes of learning and evolution in complex environmental niches have resulted in a remarkable diversity of morphological forms. Moreover, many aspects of animal intelligence are deeply embodied in these evolved morphologies. However, the principles governing relations between environmental complexity, evolved morphology, and the learnability of intelligent control, remain elusive, partially due to the substantial challenge of performing large-scale in silico experiments on evolution and learning. We introduce Deep Evolutionary Reinforcement Learning (DERL): a novel computational framework which can evolve diverse agent morphologies to learn challenging locomotion and manipulation tasks in complex environments using only low level egocentric sensory information. Leveraging DERL we demonstrate several relations between environmental complexity, morphological intelligence and the learnability of control. First, environmental complexity fosters the evolution of morphological intelligence as quantified by the ability of a morphology to facilitate the learning of novel tasks. Second, evolution rapidly selects morphologies that learn faster, thereby enabling behaviors learned late in the lifetime of early ancestors to be expressed early in the lifetime of their descendants. In agents that learn and evolve in complex environments, this result constitutes the first demonstration of a long-conjectured morphological Baldwin effect. Third, our experiments suggest a mechanistic basis for both the Baldwin effect and the emergence of morphological intelligence through the evolution of morphologies that are more physically stable and energy efficient, and can therefore facilitate learning and control.", "source": "curated", "first_author": "Agrim Gupta", "url": "https://doi.org/10.1038/s41467-021-25874-z", "arxiv_url": "https://arxiv.org/abs/2102.02202", "fig": "figures/derl.webp", "figs": ["figures/derl-1.webp", "figures/derl-2.webp", "figures/derl-3.webp"]}, {"id": "isaacgym", "title": "Isaac Gym: High Performance GPU-Based Physics Simulation For Robot Learning", "tree": "systems", "branch": "sim-bench", "authors": ["Viktor Makoviychuk", "Lukasz Wawrzyniak", "Yunrong Guo", "Michelle Lu", "Kier Storey", "et al."], "corresponding": [], "institutions": ["ETH Zurich", "Nvidia (United Kingdom)", "University of Toronto"], "lab": "NVIDIA", "venue": "NeurIPS Datasets & Benchmarks", "venue_type": "conference", "year": 2021, "date": "2021-08-24", "topics": ["GPU physics", "massively parallel RL"], "paradigm": ["simulator"], "method": [], "projects": [], "related": ["dextreme", "genesis", "isaaclab", "mujoco", "rudin2021"], "arxiv": "2108.10470", "doi": "", "code": "", "site": "", "note": "End-to-end GPU simulation — physics and policy in the same memory. This is what makes 4096-environment training normal. Superseded by Isaac Lab but the paper is the reference.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Isaac Gym offers a high performance learning platform to train policies for wide variety of robotics tasks directly on GPU. Both physics simulation and the neural network policy training reside on GPU and communicate by directly passing data from physics buffers to PyTorch tensors without ever going through any CPU bottlenecks. This leads to blazing fast training times for complex robotics tasks on a single GPU with 2-3 orders of magnitude improvements compared to conventional RL training that uses a CPU based simulator and GPU for neural networks. We host the results and videos at \\url{https://sites.google.com/view/isaacgym-nvidia} and isaac gym can be downloaded at \\url{https://developer.nvidia.com/isaac-gym}.", "source": "curated", "first_author": "Viktor Makoviychuk", "url": "https://arxiv.org/abs/2108.10470", "arxiv_url": "https://arxiv.org/abs/2108.10470", "fig": "figures/isaacgym.webp", "figs": ["figures/isaacgym-1.webp"]}, {"id": "rudin2021", "title": "Learning to Walk in Minutes Using Massively Parallel Deep Reinforcement Learning", "tree": "rl", "branch": "locomotion", "authors": ["Nikita Rudin", "David Hoeller", "Philipp Reist", "Marco Hutter"], "corresponding": ["Nikita Rudin"], "institutions": ["ETH Zürich", "NVIDIA"], "lab": "ETH RSL", "venue": "CoRL", "venue_type": "conference", "year": 2021, "date": "2021-09-24", "topics": ["massively parallel RL", "GPU simulation"], "paradigm": ["RL"], "method": ["thousands of parallel envs on one GPU", "game-inspired curriculum"], "projects": ["m2", "mabel"], "related": ["anymal-hwangbo", "anymal-lee", "booster-rl", "isaacgym", "isaaclab", "meta-rl-legged", "multi-loco", "ppo"], "arxiv": "2109.11978", "doi": "", "code": "", "site": "", "note": "The wall-clock unlock. Legged Gym / Isaac Gym made 4096 parallel environments normal and turned locomotion training from a cluster job into a coffee break. Almost every result below assumes it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "In this work, we present and study a training set-up that achieves fast policy generation for real-world robotic tasks by using massive parallelism on a single workstation GPU. We analyze and discuss the impact of different training algorithm components in the massively parallel regime on the final policy performance and training times. In addition, we present a novel game-inspired curriculum that is well suited for training with thousands of simulated robots in parallel. We evaluate the approach by training the quadrupedal robot ANYmal to walk on challenging terrain. The parallel approach allows training policies for flat terrain in under four minutes, and in twenty minutes for uneven terrain. This represents a speedup of multiple orders of magnitude compared to previous work. Finally, we transfer the policies to the real robot to validate the approach. We open-source our training code to help accelerate further research in the field of learned legged locomotion.", "source": "curated", "first_author": "Nikita Rudin", "url": "https://arxiv.org/abs/2109.11978", "fig": "figures/rudin2021.webp", "figs": ["figures/rudin2021-1.webp", "figures/rudin2021-2.webp"]}, {"id": "rma", "title": "RMA: Rapid Motor Adaptation for Legged Robots", "tree": "rl", "branch": "locomotion", "authors": ["Ashish Kumar", "Zipeng Fu", "Deepak Pathak", "Jitendra Malik"], "corresponding": ["Ashish Kumar"], "institutions": ["Berkeley College", "Carnegie Mellon University", "University of California, Berkeley"], "lab": "BAIR; CMU", "venue": "RSS", "venue_type": "conference", "year": 2021, "date": "2021-07-08", "topics": ["online adaptation", "sim-to-real", "latent dynamics"], "paradigm": ["RL"], "method": ["environment-factor encoder + adaptation module regressed from proprioceptive history"], "projects": ["mabel", "r2s2r"], "related": ["anymal-hwangbo", "anymal-lee", "asap", "dextreme", "domain-rand", "qi-inhand", "radosavovic-humanoid"], "arxiv": "2107.04034", "doi": "", "code": "", "site": "", "note": "Reframes sim-to-real as online system identification: a small adaptation module infers a latent environment vector from recent proprioception at 100 Hz. The trick generalizes far beyond legs — the same structure shows up in in-hand manipulation and loco-manipulation.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Successful real-world deployment of legged robots would require them to adapt in real-time to unseen scenarios like changing terrains, changing payloads, wear and tear. This paper presents Rapid Motor Adaptation (RMA) algorithm to solve this problem of real-time online adaptation in quadruped robots. RMA consists of two components: a base policy and an adaptation module. The combination of these components enables the robot to adapt to novel situations in fractions of a second. RMA is trained completely in simulation without using any domain knowledge like reference trajectories or predefined foot trajectory generators and is deployed on the A1 robot without any fine-tuning. We train RMA on a varied terrain generator using bioenergetics-inspired rewards and deploy it on a variety of difficult terrains including rocky, slippery, deformable surfaces in environments with grass, long vegetation, concrete, pebbles, stairs, sand, etc. RMA shows state-of-the-art performance across diverse real-world as well as simulation experiments. Video results at https://ashish-kmr.github.io/rma-legged-robots/", "source": "curated", "first_author": "Ashish Kumar", "url": "https://arxiv.org/abs/2107.04034", "arxiv_url": "https://arxiv.org/abs/2107.04034", "fig": "figures/rma.webp", "figs": ["figures/rma-1.webp"]}, {"id": "dyret", "title": "Real-world embodied AI through a morphologically adaptive quadruped robot", "tree": "systems", "branch": "codesign", "authors": ["Tønnes F. Nygaard", "Charles P. Martin", "Jim Torresen", "Kyrre Glette", "David Howard"], "corresponding": ["Tønnes F. Nygaard"], "institutions": ["University of Oslo", "CSIRO"], "lab": "", "venue": "Nature Machine Intelligence", "venue_type": "journal", "year": 2021, "date": "", "topics": ["morphologically adaptive quadruped", "real-world evolution"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["getup-morphologies", "meta-rl-legged", "mit-cheetah", "s-cheetah", "text2robot", "transform2act", "x2n"], "arxiv": "", "doi": "10.1038/s42256-021-00320-3", "code": "", "site": "", "note": "DyRET — a quadruped that physically changes its own leg lengths and evolves its morphology *in the real world*, not in simulation. The single most relevant prior work for the fast-running-dog co-design idea, and the one that shows how brutal the real-world version of this loop is.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Tønnes F. Nygaard", "url": "https://doi.org/10.1038/s42256-021-00320-3"}, {"id": "dj-mix-analysis", "title": "A Computational Analysis of Real-World DJ Mixes using Mix-To-Track Subsequence Alignment", "tree": "il", "branch": "tactile-learn", "authors": ["Tae‐Jun Kim", "Minsuk Choi", "Evan Sacks", "Yi‐Hsuan Yang", "Juhan Nam"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2020, "date": "2020-08-24", "topics": ["DJ mix analysis", "mix-to-track alignment", "cue points and transitions"], "paradigm": ["audio"], "method": [], "projects": [], "related": ["audio-vla", "hearing-touch", "maniwav"], "arxiv": "2008.10267", "doi": "10.48550/arxiv.2008.10267", "code": "", "site": "", "note": "ISMIR 2020. Aligns a real DJ mix back to its source tracks to recover cue points, transition length, mix segmentation and the musical changes the DJ made. This is where the <b>evaluation metrics</b> for a robot DJ come from — it is the only literature that quantifies what a DJ actually did, rather than whether a listener liked it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "A DJ mix is a sequence of music tracks concatenated seamlessly, typically rendered for audiences in a live setting by a DJ on stage. As a DJ mix is produced in a studio or the live version is recorded for music streaming services, computational methods to analyze DJ mixes, for example, extracting track information or understanding DJ techniques, have drawn research interests. Many of previous works are, however, limited to identifying individual tracks in a mix or segmenting it, and the sizes of the datasets are usually small. In this paper, we provide an in-depth analysis of DJ music by aligning a mix to its original music tracks. We set up the subsequence alignment such that the audio features are less sensitive to the tempo or key change of the original track in a mix. This approach provides temporally tight mix-to-track matching from which we can obtain cue-points, transition length, mix segmentation, and musical changes in DJ performance. Using 1,557 mixes from 1001Tracklists including 13,728 tracks and 20,765 transitions, we conduct the proposed analysis and show a wide range of statistics, which may elucidate the creative process of DJ music making.", "source": "curated", "first_author": "Tae‐Jun Kim", "url": "https://arxiv.org/abs/2008.10267", "fig": "figures/dj-mix-analysis.webp", "figs": ["figures/dj-mix-analysis-1.webp"]}, {"id": "hardware-as-policy", "title": "Hardware as Policy: Mechanical and Computational Co-Optimization using Deep Reinforcement Learning", "tree": "systems", "branch": "codesign", "authors": ["Tianjian Chen", "Zhanpeng He", "Matei Ciocarlie"], "corresponding": ["Tianjian Chen"], "institutions": ["Columbia University"], "lab": "Columbia ROAM Lab (Ciocarlie)", "venue": "CoRL", "venue_type": "conference", "year": 2020, "date": "2020-08-05", "topics": ["differentiable hardware", "co-optimization"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["diff-contact-design", "fit2form", "transform2act"], "arxiv": "2008.04460", "doi": "", "code": "", "site": "", "note": "Treats the mechanism itself as part of the policy network, so hardware parameters get gradients from the same backward pass. Demonstrated on underactuated hands — the most directly relevant formulation for dexterous-hand co-design.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Deep Reinforcement Learning (RL) has shown great success in learning complex control policies for a variety of applications in robotics. However, in most such cases, the hardware of the robot has been considered immutable, modeled as part of the environment. In this study, we explore the problem of learning hardware and control parameters together in a unified RL framework. To achieve this, we propose to model the robot body as a \"hardware policy\", analogous to and optimized jointly with its computational counterpart. We show that, by modeling such hardware policies as auto-differentiable computational graphs, the ensuing optimization problem can be solved efficiently by gradient-based algorithms from the Policy Optimization family. We present two such design examples: a toy mass-spring problem, and a real-world problem of designing an underactuated hand. We compare our method against traditional co-optimization approaches, and also demonstrate its effectiveness by building a physical prototype based on the learned hardware parameters. Videos and more details are available at https://roamlab.github.io/hwasp/ .", "source": "curated", "first_author": "Tianjian Chen", "url": "https://arxiv.org/abs/2008.04460", "arxiv_url": "https://arxiv.org/abs/2008.04460", "fig": "figures/hardware-as-policy.webp"}, {"id": "anymal-lee", "title": "Learning Quadrupedal Locomotion over Challenging Terrain", "tree": "rl", "branch": "locomotion", "authors": ["Joonho Lee", "Jemin Hwangbo", "Lorenz Wellhausen", "Vladlen Koltun", "Marco Hutter"], "corresponding": ["Joonho Lee"], "institutions": ["ETH Zürich", "Intel Labs"], "lab": "ETH RSL", "venue": "Science Robotics", "venue_type": "journal", "year": 2020, "date": "2020-10-21", "topics": ["blind locomotion", "rough terrain", "proprioception"], "paradigm": ["RL"], "method": ["teacher-student privileged distillation", "temporal convolution over proprioceptive history"], "projects": ["mabel"], "related": ["anymal-hwangbo", "anymal-parkour", "miki2022", "radosavovic-humanoid", "rma", "rudin2021"], "arxiv": "2010.11251", "doi": "", "code": "", "site": "", "note": "Teacher-student privileged distillation in its canonical form: train with terrain ground truth, distill into a policy that only sees joint history. Nearly every legged and humanoid pipeline since reuses this two-stage structure.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Some of the most challenging environments on our planet are accessible to quadrupedal animals but remain out of reach for autonomous machines. Legged locomotion can dramatically expand the operational domains of robotics. However, conventional controllers for legged locomotion are based on elaborate state machines that explicitly trigger the execution of motion primitives and reflexes. These designs have escalated in complexity while falling short of the generality and robustness of animal locomotion. Here we present a radically robust controller for legged locomotion in challenging natural environments. We present a novel solution to incorporating proprioceptive feedback in locomotion control and demonstrate remarkable zero-shot generalization from simulation to natural environments. The controller is trained by reinforcement learning in simulation. It is based on a neural network that acts on a stream of proprioceptive signals. The trained controller has taken two generations of quadrupedal ANYmal robots to a variety of natural environments that are beyond the reach of prior published work in legged locomotion. The controller retains its robustness under conditions that have never been encountered during training: deformable terrain such as mud and snow, dynamic footholds such as rubble, and overground impediments such as thick vegetation and gushing water. The presented work opens new frontiers for robotics and indicates that radical robustness in natural environments can be achieved by training in much simpler domains.", "source": "curated", "first_author": "Joonho Lee", "url": "https://arxiv.org/abs/2010.11251", "fig": "figures/anymal-lee.webp", "figs": ["figures/anymal-lee-1.webp", "figures/anymal-lee-2.webp"]}, {"id": "dreamer-v2", "title": "Mastering Atari with Discrete World Models", "tree": "rl", "branch": "model-based", "authors": ["Danijar Hafner", "Timothy Lillicrap", "Mohammad Norouzi", "Jimmy Ba"], "corresponding": [], "institutions": ["University of Toronto", "Google DeepMind", "Google"], "lab": "Google DeepMind", "venue": "ICLR", "venue_type": "conference", "year": 2020, "date": "2020-10-05", "topics": ["categorical latents", "straight-through gradients", "Atari"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["dreamer-v1", "dreamerv3", "iris", "planet", "storm", "td-mpc"], "arxiv": "2010.02193", "doi": "10.48550/arxiv.2010.02193", "code": "", "site": "", "note": "<b>DreamerV2</b> — the first world-model agent to beat top single-GPU model-free agents on Atari, and the paper that replaced Gaussian latents with <b>categorical</b> ones trained by straight-through gradients. The ablation is the reason to read it: categorical latents win not because discreteness is fashionable but because a multi-modal posterior stops the model from averaging two possible futures into one blurry impossible one. Every subsequent discrete-token world model inherits this argument.", "aka": "Dreamer V2", "stars": 3, "curated": true, "local": "", "abstract": "Intelligent agents need to generalize from past experience to achieve goals in complex environments. World models facilitate such generalization and allow learning behaviors from imagined outcomes to increase sample-efficiency. While learning world models from image inputs has recently become feasible for some tasks, modeling Atari games accurately enough to derive successful behaviors has remained an open challenge for many years. We introduce DreamerV2, a reinforcement learning agent that learns behaviors purely from predictions in the compact latent space of a powerful world model. The world model uses discrete representations and is trained separately from the policy. DreamerV2 constitutes the first agent that achieves human-level performance on the Atari benchmark of 55 tasks by learning behaviors inside a separately trained world model. With the same computational budget and wall-clock time, Dreamer V2 reaches 200M frames and surpasses the final performance of the top single-GPU agents IQN and Rainbow. DreamerV2 is also applicable to tasks with continuous actions, where it learns an accurate world model of a complex humanoid robot and solves stand-up and walking from only pixel inputs.", "source": "curated", "first_author": "Danijar Hafner", "url": "https://arxiv.org/abs/2010.02193", "fig": "figures/dreamer-v2.webp", "figs": ["figures/dreamer-v2-1.webp"], "arxiv_url": "https://arxiv.org/abs/2010.02193"}, {"id": "robogrammar", "title": "RoboGrammar: Graph Grammar for Terrain-Optimized Robot Design", "tree": "systems", "branch": "codesign", "authors": ["Allan Zhao", "Jie Xu", "Mina Konaković-Luković", "Josephine Hughes", "Andrew Spielberg", "Daniela Rus", "Wojciech Matusik"], "corresponding": ["Allan Zhao"], "institutions": ["MIT"], "lab": "MIT CSAIL", "venue": "SIGGRAPH Asia", "venue_type": "conference", "year": 2020, "date": "", "topics": ["morphology search", "graph grammar", "terrain adaptation"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["codesign-rl", "eureka", "lipson2000", "neural-graph-evo", "robomorph", "sims1994", "task2morph", "transform2act"], "arxiv": "", "doi": "10.1145/3414685.3417831", "code": "", "site": "", "note": "Constrains the morphology search space with a grammar so the search is tractable and the results are buildable. The right starting point for agentic co-design — the agent should propose within a grammar, not in free space.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Allan Zhao", "url": "https://doi.org/10.1145/3414685.3417831"}, {"id": "dreamer-v1", "title": "Dream to Control: Learning Behaviors by Latent Imagination", "tree": "rl", "branch": "model-based", "authors": ["Danijar Hafner", "Timothy Lillicrap", "Jimmy Ba", "Mohammad Norouzi"], "corresponding": [], "institutions": ["University of Toronto", "Google DeepMind", "Google"], "lab": "Google DeepMind", "venue": "ICLR", "venue_type": "conference", "year": 2019, "date": "2019-12-03", "topics": ["latent imagination", "analytic value gradients", "actor-critic in a world model"], "paradigm": ["model-based RL"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["daydreamer", "dreamer-v2", "dreamerv3", "planet", "pwm"], "arxiv": "1912.01603", "doi": "10.48550/arxiv.1912.01603", "code": "", "site": "", "note": "<b>Dreamer (v1)</b> — 'Dream to Control'. The step past PlaNet: instead of re-planning with CEM every step, learn an actor and a critic <em>entirely inside</em> the imagined rollouts, and backpropagate analytic value gradients through the learned dynamics. That is the move that makes world models competitive rather than merely elegant — gradients through a differentiable model carry far more information per sample than a score-function estimate. Hafner, Lillicrap, Ba, Norouzi; note <b>Jimmy Ba is UofT</b>, so this line runs through your own department.", "aka": "Dreamer V1", "stars": 3, "curated": true, "local": "", "abstract": "Learned world models summarize an agent's experience to facilitate learning complex behaviors. While learning world models from high-dimensional sensory inputs is becoming feasible through deep learning, there are many potential ways for deriving behaviors from them. We present Dreamer, a reinforcement learning agent that solves long-horizon tasks from images purely by latent imagination. We efficiently learn behaviors by propagating analytic gradients of learned state values back through trajectories imagined in the compact state space of a learned world model. On 20 challenging visual control tasks, Dreamer exceeds existing approaches in data-efficiency, computation time, and final performance.", "source": "curated", "first_author": "Danijar Hafner", "url": "https://arxiv.org/abs/1912.01603", "fig": "figures/dreamer-v1.webp", "figs": ["figures/dreamer-v1-1.webp", "figures/dreamer-v1-2.webp", "figures/dreamer-v1-3.webp"], "arxiv_url": "https://arxiv.org/abs/1912.01603"}, {"id": "anymal-hwangbo", "title": "Learning agile and dynamic motor skills for legged robots", "tree": "rl", "branch": "locomotion", "authors": ["Jemin Hwangbo", "Joonho Lee", "Alexey Dosovitskiy", "Dario Bellicoso", "Vassilios Tsounis", "Vladlen Koltun", "Marco Hutter"], "corresponding": ["Jemin Hwangbo", "Marco Hutter"], "institutions": ["ETH Zürich", "Intel Labs"], "lab": "ETH Robotic Systems Lab (RSL)", "venue": "Science Robotics", "venue_type": "journal", "year": 2019, "date": "2019-01-16", "topics": ["legged locomotion", "sim-to-real", "actuator modelling"], "paradigm": ["RL"], "method": ["learned actuator network", "domain randomization", "TRPO"], "projects": ["mabel"], "related": ["anymal-lee", "dexndm", "mit-cheetah", "ppo", "rma", "rudin2021"], "arxiv": "1901.08652", "doi": "", "code": "", "site": "", "note": "The origin point for RL locomotion that actually transfers. The key idea is not the policy but the learned actuator network: identify the series-elastic actuator's real torque response from data and put *that* in the simulator, instead of hoping domain randomization papers over it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Legged robots pose one of the greatest challenges in robotics. Dynamic and agile maneuvers of animals cannot be imitated by existing methods that are crafted by humans. A compelling alternative is reinforcement learning, which requires minimal craftsmanship and promotes the natural evolution of a control policy. However, so far, reinforcement learning research for legged robots is mainly limited to simulation, and only few and comparably simple examples have been deployed on real systems. The primary reason is that training with real robots, particularly with dynamically balancing systems, is complicated and expensive. In the present work, we introduce a method for training a neural network policy in simulation and transferring it to a state-of-the-art legged system, thereby leveraging fast, automated, and cost-effective data generation schemes. The approach is applied to the ANYmal robot, a sophisticated medium-dog-sized quadrupedal system. Using policies trained in simulation, the quadrupedal machine achieves locomotion skills that go beyond what had been achieved with prior methods: ANYmal is capable of precisely and energy-efficiently following high-level body velocity commands, running faster than before, and recovering from falling even in complex configurations.", "source": "curated", "first_author": "Jemin Hwangbo", "url": "https://arxiv.org/abs/1901.08652", "fig": "figures/anymal-hwangbo.webp", "figs": ["figures/anymal-hwangbo-1.webp"]}, {"id": "mit-cheetah", "title": "MIT Mini Cheetah: A Platform for Pushing the Limits of Dynamic Quadruped Control", "tree": "systems", "branch": "humanoid-hw", "authors": ["Benjamin Katz", "Jared Di Carlo", "Sangbae Kim"], "corresponding": ["Benjamin Katz"], "institutions": ["MIT"], "lab": "MIT Biomimetic Robotics Lab", "venue": "ICRA", "venue_type": "conference", "year": 2019, "date": "", "topics": ["proprioceptive actuation", "dynamic quadruped", "low cost"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["anymal-hwangbo", "dyret", "s-cheetah", "spine-phase", "swheg", "unitree-g1"], "arxiv": "", "doi": "10.1109/icra.2019.8793865", "code": "", "site": "", "note": "The proprioceptive-actuator argument — high torque density, low gear ratio, backdrivable — that made cheap dynamic legged robots possible. The single most important hardware reference for the co-design-a-fast-dog idea.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Benjamin Katz", "url": "https://doi.org/10.1109/icra.2019.8793865"}, {"id": "deepmimic", "title": "DeepMimic: Example-Guided Deep Reinforcement Learning of Physics-Based Character Skills", "tree": "rl", "branch": "motion-imitation", "authors": ["Xue Bin Peng", "Pieter Abbeel", "Sergey Levine", "Michiel van de Panne"], "corresponding": ["Xue Bin Peng"], "institutions": ["UC Berkeley", "University of British Columbia"], "lab": "BAIR; UBC Motion Lab", "venue": "SIGGRAPH", "venue_type": "conference", "year": 2018, "date": "2018-04-08", "topics": ["motion imitation", "physics-based characters", "reward shaping"], "paradigm": ["RL"], "method": ["pose-tracking reward", "reference state initialization", "early termination"], "projects": ["mabel"], "related": ["add", "amp", "asap", "ase", "beyond-mimic", "dexplore", "disney-bdx", "exbody", "h2o", "humanup", "maskedmimic", "phc", "stablemimic", "videomimic", "visualmimic"], "arxiv": "1804.02717", "doi": "", "code": "", "site": "", "note": "The single most load-bearing paper in humanoid whole-body control today. Two implementation details did most of the work — reference state initialization (start episodes anywhere in the clip) and early termination (kill the episode on loss of balance) — and both are still in every tracking policy you will train.", "aka": "", "stars": 3, "curated": true, "local": "~/Desktop/papers/RL/deepmimic.pdf", "abstract": "A longstanding goal in character animation is to combine data-driven specification of behavior with a system that can execute a similar behavior in a physical simulation, thus enabling realistic responses to perturbations and environmental variation. We show that well-known reinforcement learning (RL) methods can be adapted to learn robust control policies capable of imitating a broad range of example motion clips, while also learning complex recoveries, adapting to changes in morphology, and accomplishing user-specified goals. Our method handles keyframed motions, highly-dynamic actions such as motion-captured flips and spins, and retargeted motions. By combining a motion-imitation objective with a task objective, we can train characters that react intelligently in interactive settings, e.g., by walking in a desired direction or throwing a ball at a user-specified target. This approach thus combines the convenience and motion quality of using motion clips to define the desired style and appearance, with the flexibility and generality afforded by RL methods and physics-based animation. We further explore a number of methods for integrating multiple clips into the learning process to develop multi-skilled agents capable of performing a rich repertoire of diverse skills. We demonstrate results using multiple characters (human, Atlas robot, bipedal dinosaur, dragon) and a large variety of skills, including locomotion, acrobatics, and martial arts.", "source": "curated", "first_author": "Xue Bin Peng", "url": "https://arxiv.org/abs/1804.02717", "fig": "figures/deepmimic.webp", "figs": ["figures/deepmimic-1.webp"]}, {"id": "jdeite", "title": "J-deite RIDE: a 4 m rideable humanoid that transforms into a car", "tree": "systems", "branch": "codesign", "authors": ["Kenji Ishida (BRAVE ROBOTICS)", "Asratec", "Sansei Technologies"], "corresponding": [], "institutions": ["BRAVE ROBOTICS", "Asratec", "Sansei Technologies"], "lab": "BRAVE ROBOTICS", "venue": "industry", "venue_type": "industry", "year": 2018, "date": "2018-04-26", "topics": ["car-humanoid transformation", "topology change", "rideable"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["changing-morph-dyn", "letrons", "robosen", "x2n"], "arxiv": "", "doi": "", "code": "", "site": "https://braverobotics.com/en/2018/04/26/j-deite-ride-release/", "note": "The only real car ↔ humanoid robot at scale: 4 m tall, carries two people, drives at 60 kph on wheels, walks as a biped, and converts in about <b>one minute</b> using Asratec's V-Sido controller. Read it as the existence proof and as the benchmark to beat — the transformation is quasi-static and scripted, and the bipedal walking is slow and statically stable. Everything interesting about learned control is untouched.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Kenji Ishida (BRAVE ROBOTICS)", "url": "https://braverobotics.com/en/2018/04/26/j-deite-ride-release/"}, {"id": "dactyl", "title": "Learning Dexterous In-Hand Manipulation", "tree": "rl", "branch": "manipulation", "authors": ["OpenAI: Marcin Andrychowicz", "Bowen Baker", "Maciek Chociej", "Rafal Jozefowicz", "Bob McGrew", "et al."], "corresponding": ["OpenAI"], "institutions": ["OpenAI"], "lab": "OpenAI", "venue": "IJRR", "venue_type": "journal", "year": 2018, "date": "2018-08-01", "topics": ["in-hand manipulation", "dexterity", "domain randomization"], "paradigm": ["RL"], "method": ["massive domain randomization", "LSTM policy", "vision-based pose estimation"], "projects": ["geodex", "r2s2r"], "related": ["chen-inhand", "dextreme", "domain-rand", "qi-inhand", "rotate-without-seeing", "shadow-hand"], "arxiv": "1808.00177", "doi": "", "code": "", "site": "", "note": "The existence proof for RL dexterity, and the cautionary tale about its cost — ~100 years of simulated experience on a Shadow Hand for one task. Everything since has been about making this affordable.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We use reinforcement learning (RL) to learn dexterous in-hand manipulation policies which can perform vision-based object reorientation on a physical Shadow Dexterous Hand. The training is performed in a simulated environment in which we randomize many of the physical properties of the system like friction coefficients and an object's appearance. Our policies transfer to the physical robot despite being trained entirely in simulation. Our method does not rely on any human demonstrations, but many behaviors found in human manipulation emerge naturally, including finger gaiting, multi-finger coordination, and the controlled use of gravity. Our results were obtained using the same distributed RL system that was used to train OpenAI Five. We also include a video of our results: https://youtu.be/jwSbzNHGflM", "source": "curated", "first_author": "OpenAI: Marcin Andrychowicz", "url": "https://arxiv.org/abs/1808.00177", "arxiv_url": "https://arxiv.org/abs/1808.00177", "fig": "figures/dactyl.webp", "figs": ["figures/dactyl-1.webp", "figures/dactyl-2.webp", "figures/dactyl-3.webp"]}, {"id": "planet", "title": "Learning Latent Dynamics for Planning from Pixels", "tree": "rl", "branch": "model-based", "authors": ["Danijar Hafner", "Timothy Lillicrap", "Ian Fischer", "Ruben Villegas", "David Ha", "Honglak Lee", "James Davidson"], "corresponding": [], "institutions": ["University of Toronto", "Google DeepMind", "Google", "University of Michigan"], "lab": "Google DeepMind", "venue": "ICML", "venue_type": "conference", "year": 2018, "date": "2018-11-12", "topics": ["recurrent state-space model", "latent planning", "pixels"], "paradigm": ["model-based RL"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["dreamer-v1", "dreamer-v2", "dreamerv3", "td-mpc", "worldmodels"], "arxiv": "1811.04551", "doi": "10.48550/arxiv.1811.04551", "code": "", "site": "", "note": "<b>PlaNet</b> — the paper that made latent-space planning from pixels actually work, and the direct ancestor of every Dreamer. Its contribution is the <b>RSSM</b>: a recurrent state-space model with a deterministic path <em>and</em> a stochastic path, because a purely stochastic latent forgets and a purely deterministic one cannot represent uncertainty. Planning is CEM in latent space, so no policy is learned at all — Dreamer's insight one year later was that you should learn one. Read it for the RSSM ablation in §4: it is the cleanest evidence anywhere for why the two-path latent is not an arbitrary choice.", "aka": "PlaNet", "stars": 3, "curated": true, "local": "", "abstract": "Planning has been very successful for control tasks with known environment dynamics. To leverage planning in unknown environments, the agent needs to learn the dynamics from interactions with the world. However, learning dynamics models that are accurate enough for planning has been a long-standing challenge, especially in image-based domains. We propose the Deep Planning Network (PlaNet), a purely model-based agent that learns the environment dynamics from images and chooses actions through fast online planning in latent space. To achieve high performance, the dynamics model must accurately predict the rewards ahead for multiple time steps. We approach this using a latent dynamics model with both deterministic and stochastic transition components. Moreover, we propose a multi-step variational inference objective that we name latent overshooting. Using only pixel observations, our agent solves continuous control tasks with contact dynamics, partial observability, and sparse rewards, which exceed the difficulty of tasks that were previously solved by planning with learned models. PlaNet uses substantially fewer episodes and reaches final performance close to and sometimes higher than strong model-free algorithms.", "source": "curated", "first_author": "Danijar Hafner", "url": "https://arxiv.org/abs/1811.04551", "fig": "figures/planet.webp", "arxiv_url": "https://arxiv.org/abs/1811.04551"}, {"id": "sac", "title": "Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor", "tree": "rl", "branch": "foundations", "authors": ["Tuomas Haarnoja", "Aurick Zhou", "Pieter Abbeel", "Sergey Levine"], "corresponding": [], "institutions": ["UC Berkeley"], "lab": "BAIR", "venue": "ICML", "venue_type": "conference", "year": 2018, "date": "2018-01-04", "topics": ["off-policy RL", "maximum entropy"], "paradigm": ["RL"], "method": ["entropy-regularized actor-critic", "twin critics"], "projects": [], "related": ["daydreamer", "ppo", "rlpd", "serl", "td3"], "arxiv": "1801.01290", "doi": "", "code": "", "site": "", "note": "The default when samples are expensive — i.e. whenever you are learning on real hardware. Maximum-entropy formulation also gives you a temperature knob that behaves like an exploration schedule you do not have to tune by hand.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Model-free deep reinforcement learning (RL) algorithms have been demonstrated on a range of challenging decision making and control tasks. However, these methods typically suffer from two major challenges: very high sample complexity and brittle convergence properties, which necessitate meticulous hyperparameter tuning. Both of these challenges severely limit the applicability of such methods to complex, real-world domains. In this paper, we propose soft actor-critic, an off-policy actor-critic deep RL algorithm based on the maximum entropy reinforcement learning framework. In this framework, the actor aims to maximize expected reward while also maximizing entropy. That is, to succeed at the task while acting as randomly as possible. Prior deep RL methods based on this framework have been formulated as Q-learning methods. By combining off-policy updates with a stable stochastic actor-critic formulation, our method achieves state-of-the-art performance on a range of continuous control benchmark tasks, outperforming prior on-policy and off-policy methods. Furthermore, we demonstrate that, in contrast to other off-policy algorithms, our approach is very stable, achieving very similar performance across different random seeds.", "source": "curated", "first_author": "Tuomas Haarnoja", "url": "https://arxiv.org/abs/1801.01290", "fig": "figures/sac.webp"}, {"id": "worldmodels", "title": "World Models", "tree": "il", "branch": "world-models", "authors": ["David Ha", "Jürgen Schmidhuber"], "corresponding": [], "institutions": ["Google Brain", "IDSIA"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2018, "date": "2018-03-27", "topics": ["latent dynamics", "learning in imagination"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["dreamerv3", "genie", "planet", "unipi"], "arxiv": "1803.10122", "doi": "", "code": "", "site": "", "note": "The paper that named the idea. VAE + RNN + tiny controller, trained entirely inside the learned dream. Everything below is scale and architecture on top of this skeleton.", "aka": "Ha & Schmidhuber", "stars": 3, "curated": true, "local": "", "abstract": "We explore building generative neural network models of popular reinforcement learning environments. Our world model can be trained quickly in an unsupervised manner to learn a compressed spatial and temporal representation of the environment. By using features extracted from the world model as inputs to an agent, we can train a very compact and simple policy that can solve the required task. We can even train our agent entirely inside of its own hallucinated dream generated by its world model, and transfer this policy back into the actual environment. An interactive version of this paper is available at https://worldmodels.github.io/", "source": "curated", "first_author": "David Ha", "url": "https://doi.org/10.5281/zenodo.1207631", "arxiv_url": "https://arxiv.org/abs/1803.10122", "fig": "figures/worldmodels.webp", "figs": ["figures/worldmodels-1.webp", "figures/worldmodels-2.webp", "figures/worldmodels-3.webp"]}, {"id": "domain-rand", "title": "Domain Randomization for Transferring Deep Neural Networks from Simulation to the Real World", "tree": "rl", "branch": "sim2real", "authors": ["Josh Tobin", "Rachel Fong", "Alex Ray", "Jonas Schneider", "Wojciech Zaremba", "Pieter Abbeel"], "corresponding": [], "institutions": ["OpenAI (United States)", "University of California, Berkeley"], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2017, "date": "2017-03-20", "topics": ["domain randomization", "sim-to-real"], "paradigm": ["RL"], "method": [], "projects": ["r2s2r"], "related": ["asap", "dactyl", "dreureka", "rma"], "arxiv": "1703.06907", "doi": "", "code": "", "site": "", "note": "The original 'randomize until reality looks like just another sample' argument. Still the first thing to try, and still insufficient on its own for contact-rich or deformable tasks.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Bridging the 'reality gap' that separates simulated robotics from experiments on hardware could accelerate robotic research through improved data availability. This paper explores domain randomization, a simple technique for training models on simulated images that transfer to real images by randomizing rendering in the simulator. With enough variability in the simulator, the real world may appear to the model as just another variation. We focus on the task of object localization, which is a stepping stone to general robotic manipulation skills. We find that it is possible to train a real-world object detector that is accurate to $1.5$cm and robust to distractors and partial occlusions using only data from a simulator with non-realistic random textures. To demonstrate the capabilities of our detectors, we show they can be used to perform grasping in a cluttered environment. To our knowledge, this is the first successful transfer of a deep neural network trained only on simulated RGB images (without pre-training on real images) to the real world for the purpose of robotic control.", "source": "curated", "first_author": "Josh Tobin", "url": "https://arxiv.org/abs/1703.06907", "arxiv_url": "https://arxiv.org/abs/1703.06907", "fig": "figures/domain-rand.webp", "figs": ["figures/domain-rand-1.webp", "figures/domain-rand-2.webp"]}, {"id": "gelsight", "title": "GelSight: High-Resolution Robot Tactile Sensors for Estimating Geometry and Force", "tree": "systems", "branch": "tactile-hw", "authors": ["Wenzhen Yuan", "Siyuan Dong", "Edward H. Adelson"], "corresponding": ["Wenzhen Yuan"], "institutions": ["Massachusetts Institute of Technology"], "lab": "MIT CSAIL / Adelson Lab", "venue": "Sensors", "venue_type": "journal", "year": 2017, "date": "2017-11-29", "topics": ["vision-based tactile sensing", "geometry", "force"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "dexskin", "digit"], "arxiv": "", "doi": "10.3390/s17122762", "code": "", "site": "", "note": "The sensor that made tactile sensing a computer-vision problem — elastomer + camera + photometric stereo gives you sub-millimetre geometry. Every vision-based skin below descends from it.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Wenzhen Yuan", "url": "https://doi.org/10.3390/s17122762"}, {"id": "ppo", "title": "Proximal Policy Optimization Algorithms", "tree": "rl", "branch": "foundations", "authors": ["John Schulman", "Filip Wolski", "Prafulla Dhariwal", "Alec Radford", "Oleg Klimov"], "corresponding": [], "institutions": ["OpenAI"], "lab": "OpenAI", "venue": "arXiv", "venue_type": "preprint", "year": 2017, "date": "2017-07-20", "topics": ["policy gradient", "on-policy RL"], "paradigm": ["RL"], "method": ["clipped surrogate objective"], "projects": [], "related": ["anymal-hwangbo", "dqn", "mappo", "rudin2021", "sac", "trpo"], "arxiv": "1707.06347", "doi": "", "code": "", "site": "", "note": "Still the workhorse behind essentially every legged-locomotion and humanoid result in this library. Its dominance is less about optimality than about tolerance: it survives bad reward scales, huge parallel batch sizes, and non-stationary curricula without diverging.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "We propose a new family of policy gradient methods for reinforcement learning, which alternate between sampling data through interaction with the environment, and optimizing a \"surrogate\" objective function using stochastic gradient ascent. Whereas standard policy gradient methods perform one gradient update per data sample, we propose a novel objective function that enables multiple epochs of minibatch updates. The new methods, which we call proximal policy optimization (PPO), have some of the benefits of trust region policy optimization (TRPO), but they are much simpler to implement, more general, and have better sample complexity (empirically). Our experiments test PPO on a collection of benchmark tasks, including simulated robotic locomotion and Atari game playing, and we show that PPO outperforms other online policy gradient methods, and overall strikes a favorable balance between sample complexity, simplicity, and wall-time.", "source": "curated", "first_author": "John Schulman", "url": "https://arxiv.org/abs/1707.06347", "fig": "figures/ppo.webp"}, {"id": "mujoco", "title": "MuJoCo: A Physics Engine for Model-Based Control", "tree": "systems", "branch": "sim-bench", "authors": ["Emanuel Todorov", "Tom Erez", "Yuval Tassa"], "corresponding": ["Emanuel Todorov"], "institutions": ["University of Washington"], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2012, "date": "", "topics": ["physics engine", "contact dynamics"], "paradigm": ["simulator"], "method": [], "projects": [], "related": ["genesis", "isaacgym", "newton"], "arxiv": "", "doi": "10.1109/iros.2012.6386109", "code": "", "site": "", "note": "Soft-constraint contact model, fast and differentiable-ish. The default for anything where contact accuracy matters more than photorealism — which is most of this library.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Emanuel Todorov", "url": "https://doi.org/10.1109/iros.2012.6386109"}, {"id": "dagger", "title": "A Reduction of Imitation Learning and Structured Prediction to No-Regret Online Learning", "tree": "il", "branch": "foundations", "authors": ["Stéphane Ross", "Geoffrey J. Gordon", "J. Andrew Bagnell"], "corresponding": ["Stéphane Ross"], "institutions": ["Carnegie Mellon University"], "lab": "", "venue": "AISTATS", "venue_type": "conference", "year": 2010, "date": "2010-11-02", "topics": ["covariate shift", "interactive imitation"], "paradigm": ["BC"], "method": ["iteratively aggregate expert labels on the learner's own state distribution"], "projects": [], "related": ["alvinn", "hil-serl"], "arxiv": "1011.0686", "doi": "", "code": "", "site": "", "note": "The formal statement of why naive BC drifts, and the fix. Every 'human-in-the-loop correction' system since — including HIL-SERL — is DAgger with better ergonomics.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "Sequential prediction problems such as imitation learning, where future observations depend on previous predictions (actions), violate the common i.i.d. assumptions made in statistical learning. This leads to poor performance in theory and often in practice. Some recent approaches provide stronger guarantees in this setting, but remain somewhat unsatisfactory as they train either non-stationary or stochastic policies and require a large number of iterations. In this paper, we propose a new iterative algorithm, which trains a stationary deterministic policy, that can be seen as a no regret algorithm in an online learning setting. We show that any such no regret algorithm, combined with additional reduction assumptions, must find a policy with good performance under the distribution of observations it induces in such sequential settings. We demonstrate that this new approach outperforms previous approaches on two challenging imitation learning problems and a benchmark sequence labeling problem.", "source": "curated", "first_author": "Stéphane Ross", "url": "https://arxiv.org/abs/1011.0686", "arxiv_url": "https://arxiv.org/abs/1011.0686", "fig": "figures/dagger.webp"}, {"id": "sims1994", "title": "Evolving Virtual Creatures", "tree": "systems", "branch": "codesign", "authors": ["Karl Sims"], "corresponding": [], "institutions": ["Thinking Machines Corporation"], "lab": "", "venue": "SIGGRAPH", "venue_type": "conference", "year": 1994, "date": "", "topics": ["evolutionary co-design", "morphology and control"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["codesign-rl", "derl", "evogym", "lipson2000", "neural-graph-evo", "robogrammar"], "arxiv": "", "doi": "10.1145/3596711.3596785", "code": "", "site": "", "note": "The origin of the whole field: bodies and brains evolved together in simulated physics, in 1994. Every modern co-design paper is a re-run of this with better optimizers. Read it to see how little the framing has changed and how much the tooling has.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Karl Sims", "url": "https://doi.org/10.1145/3596711.3596785"}, {"id": "kungfu-bot", "title": "A Kung Fu Athlete Bot That Can Do It All Day: Highly Dynamic, Balance-Challenging Motion Dataset and Autonomous Fall-Resilient Tracking", "tree": "rl", "branch": "motion-imitation", "authors": ["Zhongxiang Lei", "Lulu Cao", "Xuyang Wang", "Tianyi Qian", "Jinyan Liu", "Xuesong Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-14", "topics": ["highly dynamic motion dataset", "fall-resilient tracking"], "paradigm": ["RL", "dataset"], "method": [], "projects": ["mabel"], "related": ["exbody2", "humanup", "stablemimic"], "arxiv": "2602.13656", "doi": "10.48550/arxiv.2602.13656", "code": "", "site": "", "note": "A balance-challenging dynamic motion dataset plus autonomous fall-resilient tracking — the robot keeps working all day because it recovers by itself. The dataset is the part to reuse.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Current humanoid motion tracking systems can execute routine and moderately dynamic behaviors, yet significant gaps remain near hardware performance limits and algorithmic robustness boundaries. Martial arts represent an extreme case of highly dynamic human motion, characterized by rapid center-of-mass shifts, complex coordination, and abrupt posture transitions. However, datasets tailored to such high-intensity scenarios remain scarce. To address this gap, we construct KungFuAthlete, a high-dynamic martial arts motion dataset derived from professional athletes' daily training videos. The dataset includes ground and jump subsets covering representative complex motion patterns. The jump subset exhibits substantially higher joint, linear, and angular velocities compared to commonly used datasets such as LAFAN1, PHUMA, and AMASS, indicating significantly increased motion intensity and complexity. Importantly, even professional athletes may fail during highly dynamic movements. Similarly, humanoid robots are prone to instability and falls under external disturbances or execution errors. Most prior work assumes motion execution remains within safe states and lacks a unified strategy for modeling unsafe states and enabling reliable autonomous recovery. We propose a novel training paradigm that enables a single policy to jointly learn high-dynamic motion tracking and fall recovery, unifying agile execution and stabilization within one framework. This framework expands robotic capability from pure motion tracking to recovery-enabled execution, promoting more robust and autonomous humanoid performance in real-world high-dynamic scenarios.", "source": "curated", "first_author": "Zhongxiang Lei", "url": "https://arxiv.org/abs/2602.13656", "fig": "figures/kungfu-bot.webp", "figs": ["figures/kungfu-bot-1.webp"]}, {"id": "ace-ego-0", "title": "ACE-Ego-0: Unifying Egocentric Human and Robotic Data for VLA Pretraining", "tree": "il", "branch": "human-video", "authors": ["Hao Li", "Ganlong Zhao", "Yufei Liu", "Haotian Hou", "Guoquan Ye", "Tongyan Fang", "Chunxiao Liu", "Sicong Huang", "Jianbo Liu", "Xiaogang Wang", "Hongsheng Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-15", "topics": ["unified pretraining", "robot + sim + human data mixing"], "paradigm": ["VLA"], "method": [], "projects": ["r2s2r"], "related": ["dreamgen", "egoscale", "humanoid-policy", "pld"], "arxiv": "2606.17200", "doi": "10.48550/arxiv.2606.17200", "code": "", "site": "", "note": "Mixes 4.53K hours of robot and simulation data with 1.48K hours of pseudo-action-labelled egocentric human data in one pretraining recipe. The mixing ratio between the three sources is the question every VLA team faces and almost nobody reports; this one does.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models benefit from large-scale and diverse embodied data, yet scaling robot trajectory collection is costly and labor-intensive. Recent advances show that large-scale egocentric human videos provide complementary real-world supervision in pretraining. However, joint training on human and robot data remains challenging due to divergences in action spaces, embodiment structures, temporal dynamics, and supervision quality. We introduce ACE-EGO-0, a unified VLA pretraining framework jointly leveraging heterogeneous data sources. To extract large-scale pretraining supervision from egocentric human videos, we build a scalable egocentric video-to-action pipeline that converts raw human videos into robot-format pseudo-action trajectories. To make these labels comparable with robot demonstrations, ACE-EGO-0 uses a unified action representation based on camera-space actions, morphology conditioning, and time-aligned action chunking. To robustly leverage noisy pseudo-action supervision from egocentric human videos, we formulate a reliability-aware training objective with a human auxiliary loss that concentrates supervision on reliable signals. We instantiate ACE-EGO-0 on 4.53K hours of robot and simulation data, together with 1.48K hours of pseudo-action-labeled egocentric human data. Experiments show that incorporating large-scale human supervision under reliability-aware weighting consistently improves both unified joint pretraining and supervised fine-tuning. ACE-EGO-0 achieves state-of-the-art performance on RoboCasa GR1 TableTop and RoboTwin 2.0, while demonstrating strong transfer to real-world bimanual manipulation.", "source": "curated", "first_author": "Hao Li", "url": "https://arxiv.org/abs/2606.17200", "fig": "figures/ace-ego-0.webp", "figs": ["figures/ace-ego-0-1.webp", "figures/ace-ego-0-2.webp"]}, {"id": "aim-wam", "title": "AIM: Intent-Aware Unified world action Modeling with Spatial Value Maps", "tree": "il", "branch": "world-models", "authors": ["Liaoyuan Fan", "Zetian Xu", "Chen Cao", "Wenyao Zhang", "Mingqi Yuan", "Jiayu Chen"], "corresponding": [], "institutions": ["Shanghai Jiao Tong University", "University of Hong Kong"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-13", "topics": ["intent-aware world action modeling", "spatial value maps"], "paradigm": ["world model"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dreamzero", "interact", "voxposer", "wla-model"], "arxiv": "2604.11135", "doi": "10.48550/arxiv.2604.11135", "code": "", "site": "", "note": "<b>AIM</b> — intent-aware world-action modelling with <b>spatial value maps</b>, which is a neat fusion of the VoxPoser representation with the WAM formulation. Interesting for the HRI idea: a world model that predicts what the <em>human</em> intends, not only what the scene will do.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Pretrained video generation models provide strong priors for robot control, but existing unified world action models still struggle to decode reliable actions without substantial robot-specific training. We attribute this limitation to a structural mismatch: while video models capture how scenes evolve, action generation requires explicit reasoning about where to interact and the underlying manipulation intent. We introduce AIM, an intent-aware unified world action model that bridges this gap via an explicit spatial interface. Instead of decoding actions directly from future visual representations, AIM predicts an aligned spatial value map that encodes task-relevant interaction structure, enabling a control-oriented abstraction of future dynamics. Built on a pretrained video generation model, AIM jointly models future observations and value maps within a shared mixture-of-transformers architecture. It employs intent-causal attention to route future information to the action branch exclusively through the value representation. We further propose a self-distillation reinforcement learning stage that freezes the video and value branches and optimizes only the action head using dense rewards derived from projected value-map responses together with sparse task-level signals. To support training and evaluation, we construct a simulation dataset of 30K manipulation trajectories with synchronized multi-view observations, actions, and value-map annotations. Experiments on RoboTwin 2.0 benchmark show that AIM achieves a 94.0% average success rate, significantly outperforming prior unified world action baselines. Notably, the improvement is more pronounced in long-horizon and contact-sensitive manipulation tasks, demonstrating the effectiveness of explicit spatial-intent modeling as a bridge between visual world modeling and robot control.", "source": "curated", "first_author": "Liaoyuan Fan", "url": "https://arxiv.org/abs/2604.11135", "fig": "figures/aim-wam.webp", "figs": ["figures/aim-wam-1.webp"]}, {"id": "aero-hand", "title": "Aero Hand Open: A Simulation-Ready Tendon-Driven Hand for Dexterous Manipulation Learning", "tree": "systems", "branch": "hands", "authors": ["Nan Wang", "Mohit Yadav", "Jonathan Wulff", "Aidan Rosenbaum", "Kezhou Chen", "Yuvan Sharma", "Xu Dong", "Yiwei Tao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-28", "topics": ["tendon-driven hand", "simulation-ready", "low cost"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["leap-hand", "mm-hand", "orca-hand", "ruka-v2"], "arxiv": "2608.28578", "doi": "", "code": "", "site": "", "note": "Aero Hand Open: five fingers, sixteen revolute joints, seven motors, 374 g, $314 of parts, and — the part that matters — shipped simulation-ready. Cheapest credible entry point if you want hardware in hand this month.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Tendon-driven hands are anthropomorphic, and moving the actuators off the joints is what makes a hand of this capability affordable to build. Two effects produce that saving. Routing force through a cable removes the requirement that a motor fit inside the joint it drives, so smaller and cheaper motors suffice, and one motor can drive several joints through a single cable, so fewer motors are needed. They are also harder to learn on than a direct-drive hand. The underactuated transmission that produces the saving is itself difficult to represent in a simulator, and the joints one cable drives are not independently commandable. We present Aero Hand Open, a tendon-driven anthropomorphic hand that is released simulation-ready. Three things ship with it. A simulation model reproduces the cable transmission itself. An identified actuation map connects that model to the motor commands in both directions, including the three-way coupling of the thumb. A reinforcement learning package trains policies for the hand. Together they let a policy be trained entirely in simulation and run on the hand with no fine-tuning and no state estimation. We release the mechanical design, the simulation model, the identified mapping, the training environment and the deployment stack.", "source": "curated", "first_author": "Nan Wang", "url": "https://arxiv.org/abs/2608.28578", "fig": "figures/aero-hand.webp", "figs": ["figures/aero-hand-1.webp"]}, {"id": "corgi", "title": "CORGI: Consistency-Aware 3D Dog Reconstruction from a Single Image in the Wild", "tree": "il", "branch": "human-video", "authors": ["Yuxiao Wu", "Weile Li", "B X Zhu", "Yumeng Liu", "Youcheng Cai", "Ligang Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-01", "topics": ["consistency-aware 3D dog reconstruction", "single image"], "paradigm": ["representation learning"], "method": [], "projects": [], "related": ["animal-avatars", "barc", "dogmo"], "arxiv": "2607.00321", "doi": "10.48550/arxiv.2607.00321", "code": "", "site": "", "note": "2026 single-image 3D dog reconstruction with explicit consistency handling. Together with DogMo it makes 'build the robot from dog footage' a data problem rather than an aspiration.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Reconstructing high-fidelity 3D models of highly articulated animals, such as dogs, from a single in-the-wild image remains a formidable challenge. In this paper, we introduce CORGI, a novel framework for consistency-aware 3D dog reconstruction from a single unconstrained image that completely eliminates the need for 3D supervision. To overcome generative inconsistencies and the lack of multi-view capture, our pipeline introduces three core components. First, we propose a Canonical-Driven Orbital Generation (CDOG) strategy, utilizing specialized Canonical and Orbit LoRAs to normalize arbitrary input poses and synthesize reliable 360-degree video observations. Second, we design a Consistency-aware Deformable 3DGS (CA-3DGS) module that anchors on a D-SMAL prior, explicitly modeling per-view generative errors through dedicated neural deformation fields to learn accurate vertex-level displacements. Finally, to eliminate structural distortions and recover high-frequency details, we introduce a self-supervised Deformation-Conditioned Generative Repair (DCGR) module. Extensive experiments demonstrate that CORGI achieves state-of-the-art performance, generalizing seamlessly across diverse dog breeds to produce geometrically accurate, visually coherent, and fully animatable 3D assets ready for downstream applications.", "source": "curated", "first_author": "Yuxiao Wu", "url": "https://arxiv.org/abs/2607.00321", "fig": "figures/corgi.webp"}, {"id": "causal-wm", "title": "Causal World Modeling for Robot Control", "tree": "rl", "branch": "model-based", "authors": ["Lin Li", "Qihang Zhang", "Yiming Luo", "Shuai Yang", "Ruilin Wang", "Fei Han", "Mingrui Yu", "Zelin Gao", "Nan Xue", "Xing Zhu", "Yujun Shen", "Yinghao Xu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-29", "topics": ["causal structure in world models", "robot control", "spurious correlation"], "paradigm": ["world model"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["adaptigraph", "beyond-vla-wm", "dino-wm", "dreamerv3"], "arxiv": "2601.21998", "doi": "", "code": "", "site": "", "note": "<b>Causal World Modeling for Robot Control</b> — the failure this addresses is real and under-discussed: a world model trained on correlational data will happily learn that the gripper closing <em>causes</em> the light to change, and plan accordingly. Relevant to anything trained on passively collected video, which is now most of the field.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "This work highlights that video world modeling, alongside vision-language pre-training, establishes a fresh and independent foundation for robot learning. Intuitively, video world models provide the ability to imagine the near future by understanding the causality between actions and visual dynamics. Inspired by this, we introduce LingBot-VA, an autoregressive diffusion framework that learns frame prediction and policy execution simultaneously. Our model features three carefully crafted designs: (1) a shared latent space, integrating vision and action tokens, driven by a Mixture-of-Transformers (MoT) architecture, (2) a closed-loop rollout mechanism, allowing for ongoing acquisition of environmental feedback with ground-truth observations, (3) an asynchronous inference pipeline, parallelizing action prediction and motor execution to support efficient control. We evaluate our model on both simulation benchmarks and real-world scenarios, where it shows significant promise in long-horizon manipulation, data efficiency in post-training, and strong generalizability to novel configurations. The code and model are made publicly available to facilitate the community.", "source": "curated", "first_author": "Lin Li", "url": "https://arxiv.org/abs/2601.21998", "fig": "figures/causal-wm.webp", "figs": ["figures/causal-wm-1.webp", "figures/causal-wm-2.webp"]}, {"id": "force-grasp-s2r", "title": "Closing the Reality Gap: Zero-Shot Sim-to-Real Deployment for Dexterous Force-Based Grasping and Manipulation", "tree": "rl", "branch": "manipulation", "authors": ["Zhe Zhao", "Haoyu Dong", "Zhengmao He", "Yang Li", "Xinyu Yi", "Zhibin Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-06", "topics": ["zero-shot sim-to-real", "force-based dexterous grasping"], "paradigm": ["RL", "sim2real"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["clutterdexgrasp", "dexndm", "dexsim2real"], "arxiv": "2601.02778", "doi": "", "code": "", "site": "", "note": "Force-based dexterous grasping transferred zero-shot. Read alongside DexNDM for the two competing answers to the hand reality gap.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Human-like dexterous hands with multiple fingers offer human-level manipulation capabilities, but training control policies that can directly deploy on real hardware remains difficult due to contact-rich physics and imperfect actuation. We close this gap with a practical sim-to-real reinforcement learning (RL) framework that utilizes dense tactile feedback combined with joint torque sensing to explicitly regulate physical interactions. To enable effective sim-to-real transfer, we introduce (i) a computationally fast tactile simulation that computes distances between dense virtual tactile units and the object via parallel forward kinematics, providing high-rate, high-resolution touch signals needed by RL; (ii) a current-to-torque calibration that eliminates the need for torque sensors on dexterous hands by mapping motor current to joint torque; and (iii) actuator dynamics modeling to bridge the actuation gaps with randomization of non-ideal effects such as backlash, torque-speed saturation. Using an asymmetric actor-critic PPO pipeline trained entirely in simulation, our policies deploy directly to a five-finger hand. The resulting policies demonstrated two essential skills: (1) command-based, controllable grasp force tracking, and (2) reorientation of objects in the hand, both of which were robustly executed without fine-tuning on the robot. By combining tactile and torque in the observation space with effective sensing/actuation modeling, our system provides a practical solution to achieve reliable dexterous manipulation. To our knowledge, this is the first demonstration of controllable grasping on a multi-finger dexterous hand trained entirely in simulation and transferred zero-shot on real hardware.", "source": "curated", "first_author": "Zhe Zhao", "url": "https://arxiv.org/abs/2601.02778", "fig": "figures/force-grasp-s2r.webp"}, {"id": "decowam", "title": "DECOWAM: Decoupled Whole-Body World-Action Model for Legged Mobile Manipulation", "tree": "il", "branch": "world-models", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai AI Laboratory"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-20", "topics": ["world-action model", "whole-body", "legged manipulation"], "paradigm": ["world model"], "method": [], "projects": ["m2", "mabel", "wam-tactile"], "related": ["falcon", "flare", "mobilewam", "motionwam", "oa-wam", "wholebodyvla"], "arxiv": "2608.20114", "doi": "", "code": "", "site": "", "note": "From your M2 library. Decoupling locomotion and manipulation inside the world model is the same structural choice FALCON makes in pure RL.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.20114v1_DECOWAM_Decoupled_Whole_Body_World_Action_Model_for_Legged_M.pdf", "abstract": "Mobile manipulation requires a robot to predict how locomotion and arm motion jointly alter future observations and control. Existing world-action models, developed largely for fixed-base platforms, do not explicitly distinguish camera ego-motion from base and arm actions. Here we introduce DECOWAM, a whole-body world-action model that separates these factors through dedicated conditional interfaces. DECOWAM freezes an adapted FastWAM backbone and trains residual adapters, an action-equivalent future bottleneck distilled from privileged observations, adversarially separated base and arm latents, and base-velocity conditioning for video prediction. We further introduce ARMDOG, a real-robot dataset that synchronizes video, whole-body state and action, and language. On a fixed replay protocol, DECOWAM improved both future-video and action prediction over FastWAM, reducing action MSE by 21.7% with 25.95M trainable adaptation parameters. Across 79 closed-loop trials per method, it achieved the highest observed whole-body coordination and base-displacement robustness among the compared systems, while task completion remained comparable to the strongest baseline. These results show that embodiment-aware factorization can support parameter-efficient joint visual prediction and whole-body control under moving viewpoints.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2608.20114", "fig": "figures/decowam.webp", "figs": ["figures/decowam-1.webp", "figures/decowam-2.webp"]}, {"id": "deform360", "title": "Deform360: A Massive Multi-view Visuotactile Dataset for Deformable World Models", "tree": "il", "branch": "tactile-learn", "authors": ["Hongyu Li", "Wanjia Fu", "Xiaoyan Cong", "Zekun Li", "Binghao Huang", "Hanxiao Jiang", "Xintong He", "Yiqing Liang", "Rao Fu", "Tao Lu", "Srinath Sridhar", "Kevin A. Smith", "George Konidaris", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "MIT", "Brown University"], "lab": "RoboPIL (Yunzhu Li)", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-06", "topics": ["multi-view visuotactile dataset", "deformable world models", "360-degree capture"], "paradigm": ["dataset"], "method": [], "projects": ["wam-tactile"], "related": ["dream-tac", "egophys", "omnivta", "phystwin", "tactile-wam"], "arxiv": "2607.05390", "doi": "", "code": "", "site": "", "note": "<b>Deform360</b> — a massive multi-view <em>visuotactile</em> dataset built specifically for deformable world models. The dataset that the tactile-WAM literature has been missing: visual world models of soft objects have had data for a while, visuotactile ones have not.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Predicting object dynamics (i.e., world modeling) is a fundamental challenge for robotic manipulation, and modeling deformable objects presents a particularly difficult case due to their high-dimensional state spaces and complex material properties. While current world models approach this through two distinct paradigms: learning the dynamics over the 2D pixel space or more explicit 3D geometric space. A systematic understanding of their relative strengths and limitations remains elusive due to the lack of diverse, large-scale real-world data. To address this, we present Deform360, a large-scale visuotactile dataset featuring 198 daily-life objects, 1,980 interaction sequences, and over 215 hours of observations from 41 surround-view cameras and bimanual tactile grippers to capture both global motion and contact-induced local deformations. Leveraging a novel markerless visuotactile 3D tracking pipeline to extract dense geometry and motion, we systematically evaluate current state-of-the-art world models, comparing 2D video models against 3D particle models. Finally, we provide a preliminary demonstration indicating the real-world applicability of our dataset by performing robot planning tasks on deformable objects. Our analysis reveals key insights into the trade-offs between structural priors and scalability, providing a solid benchmark for future research in generalizable deformable object-centric world modeling. Project website: https://deform360.lhy.xyz", "source": "curated", "first_author": "Hongyu Li", "url": "https://arxiv.org/abs/2607.05390", "fig": "figures/deform360.webp"}, {"id": "dexndm", "title": "DexNDM: Closing the Reality Gap for Dexterous In-Hand Rotation via Joint-Wise Neural Dynamics Model", "tree": "rl", "branch": "manipulation", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2026, "date": "2025-10-09", "topics": ["Dexterous Manipulation Sim-to-real", "dexterous manipulation", "in-hand manipulation", "neural dynamics", "reality gap"], "paradigm": ["RL", "model-based"], "method": [], "projects": ["geodex", "r2s2r", "wam-tactile"], "related": ["anymal-hwangbo", "clutterdexgrasp", "dexsim2real", "force-grasp-s2r", "humanoid-dex-s2r", "wheeled-payload"], "arxiv": "2510.08556", "doi": "", "code": "", "site": "", "note": "In ~/Desktop/papers/Dexterous Manipulation Sim-to-real. Joint-wise learned dynamics is the hand analogue of Hwangbo's actuator network — identify the real actuator, not the whole world.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/Dexterous Manipulation Sim-to-real/DEXNDM-dexterous-.pdf", "abstract": "Achieving generalized in-hand object rotation remains a significant challenge in robotics, largely due to the difficulty of transferring policies from simulation to the real world. The complex, contact-rich dynamics of dexterous manipulation create a \"reality gap\" that has limited prior work to constrained scenarios involving simple geometries, limited object sizes and aspect ratios, constrained wrist poses, or customized hands. We address this sim-to-real challenge with a novel framework that enables a single policy, trained in simulation, to generalize to a wide variety of objects and conditions in the real world. The core of our method is a joint-wise dynamics model that learns to bridge the reality gap by effectively fitting limited amount of real-world collected data and then adapting the sim policy's actions accordingly. The model is highly data-efficient and generalizable across different whole-hand interaction distributions by factorizing dynamics across joints, compressing system-wide influences into low-dimensional variables, and learning each joint's evolution from its own dynamic profile, implicitly capturing these net effects. We pair this with a fully autonomous data collection strategy that gathers diverse, real-world interaction data with minimal human intervention. Our complete pipeline demonstrates unprecedented generality: a single policy successfully rotates challenging objects with complex shapes (e.g., animals), high aspect ratios (up to 5.33), and small sizes, all while handling diverse wrist orientations and rotation axes. Comprehensive real-world evaluations and a teleoperation application for complex tasks validate the effectiveness and robustness of our approach. Website: https://meowuu7.github.io/DexNDM/", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2510.08556", "arxiv_url": "https://arxiv.org/abs/2510.08556", "fig": "figures/x-dexndmclosingtherealitygapfordexterousin.webp"}, {"id": "dexsim2real", "title": "DexSim2Real: Foundation Model-Guided Sim-to-Real Transfer for Generalizable Dexterous Manipulation", "tree": "rl", "branch": "sim2real", "authors": ["Zijian Zeng", "Fei Ding", "Huiming Yang", "Xianwei Li", "Yuhao Liao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-03", "topics": ["foundation-model-guided sim-to-real", "dexterous transfer"], "paradigm": ["sim2real", "RL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["dexndm", "dreureka", "force-grasp-s2r", "simweaver"], "arxiv": "2605.05241", "doi": "", "code": "", "site": "", "note": "Uses a foundation model to guide the sim-to-real transfer rather than hand-tuning randomization ranges — the same move DrEureka makes, applied to dexterity.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Sim-to-real transfer remains a critical bottleneck for deploying dexterous manipulation policies learned in simulation to real-world robots. Existing approaches rely on manually designed domain randomization or task-specific adaptation, limiting their generalizability across diverse manipulation scenarios. We present DexSim2Real, an integrated framework that leverages vision-language foundation models to bridge the sim-to-real gap for dexterous manipulation. Our system combines three components: (1) Foundation Model-Guided Domain Randomization (FM-DR), which uses a vision-language model as a visual realism critic to optimize simulation parameters via closed-loop CMA-ES, complementing text-based approaches like DrEureka with direct visual feedback; (2) a Tactile-Visual Cross-Attention Policy (TVCAP) that adapts cross-attention visuo-tactile fusion to zero-shot sim-to-real RL; and (3) a Progressive Skill Curriculum (PSC) that builds on LLM-based task decomposition with a difficulty scheduler tailored to contact-rich dexterous tasks. Extensive experiments on six challenging manipulation tasks with blinded evaluation demonstrate that DexSim2Real achieves a 78.2% average real-world success rate, outperforming DrEureka and DeXtreme while reducing the sim-to-real performance gap to only 8.3%.", "source": "curated", "first_author": "Zijian Zeng", "url": "https://arxiv.org/abs/2605.05241", "fig": "figures/dexsim2real.webp"}, {"id": "do-as-i-do", "title": "Do as I Do: Dexterous Manipulation Data from Everyday Human Videos", "tree": "il", "branch": "human-video", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["UC Berkeley"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-19", "topics": ["everyday human video", "dexterous data"], "paradigm": ["IL"], "method": [], "projects": ["m2", "r2s2r"], "related": ["egosteer", "human-video-survey", "phantom", "simdex"], "arxiv": "2606.19333", "doi": "", "code": "", "site": "", "note": "From your M2 library. Same thesis as your idea, recent enough that it should be a primary comparison point rather than background.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2606.19333v1_Do_as_I_Do_Dexterous_Manipulation_Data_from_Everyday_Human_V.pdf", "abstract": "How can we scalably generate data for robotic manipulation, especially on human-like platforms such as dexterous multi-fingered hands? Learning from human videos has recently emerged as a likely answer to this question. However, difficulties in estimating hand-object interaction and crossing the human-to-robot embodiment gap have hindered the adoption of abundant monocular RGB-only human videos as the primary source of robot manipulation data. In this work, we present DO AS I DO, an algorithm to reconstruct and retarget monocular RGB human videos to multi-fingered dexterous robotic hands. DO AS I DO reconstructs hand-object interactions from various egocentric and exocentric in-the-wild video sources. The algorithm then retargets these hand-object interaction estimates into a sequence of actions executable in the real world, yielding robot-complete manipulation data from disparate human videos. Overall, DO AS I DO outperforms previous state of the art in estimating hand-object interactions and extracting dexterous manipulation trajectories from RGB videos, as we show in experiments on datasets with ground truths and on a dataset of video clips collected online. Our experiments enable us to propose an efficacy playbook for practitioners collecting human data for manipulation.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2606.19333", "fig": "figures/do-as-i-do.webp", "figs": ["figures/do-as-i-do-1.webp", "figures/do-as-i-do-2.webp"]}, {"id": "dyna2", "title": "Dyna Robotics DYNA-2: scaling human video for commercial dexterity", "tree": "systems", "branch": "agentic", "authors": ["Dyna Robotics"], "corresponding": [], "institutions": ["Dyna Robotics"], "lab": "Dyna Robotics", "venue": "tech report", "venue_type": "industry", "year": 2026, "date": "", "topics": ["robot foundation model", "human video scaling", "commercial deployment"], "paradigm": ["VLA"], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": ["dyna", "figure-fleet", "generalist-ai", "pi06", "sunday-gelato"], "arxiv": "", "doi": "", "code": "", "site": "https://www.dyna.co/", "note": "Reports average task success roughly doubling from 20% to 53% as training data scaled from 1,000 to 1,000,000 hours of human video. Treat the numbers as unaudited, but the <em>shape</em> — a clear, unsaturated scaling curve on human rather than robot data — is the claim that matters for your UMI-glove thesis.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Dyna Robotics", "url": "https://www.dyna.co/"}, {"id": "ev-wm", "title": "EV-WM: Event-Verified World Models for Long-Horizon Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Kailin Wang", "Haoxiang Jie", "Yaoyuan Yan", "Jiacheng Zhou", "Zhiyou Heng"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-11", "topics": ["event-verified world models", "long-horizon manipulation", "verification signal"], "paradigm": ["world model"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["dream-tac", "himem-wam", "robogaze", "tacforesight", "worldeval"], "arxiv": "2606.13053", "doi": "10.48550/arxiv.2606.13053", "code": "", "site": "", "note": "<b>EV-WM</b> — verify the world model's rollout against discrete <em>events</em> (contact made, object lifted) rather than trusting pixel-level prediction over a long horizon. A cheap, sound idea, and a natural fit with tactile sensing, where contact events are exactly what you can measure reliably.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Pretrained-feature world models provide a useful substrate for robot imagination, but visual or latent prediction alone does not determine whether an imagined future satisfies task-relevant predicates. Long-horizon manipulation requires progress signals that are relational, predicate-level, and physically grounded: whether an object has moved, whether a drawer or contact state has changed, whether a placement predicate is satisfied, and whether a candidate future is reliable enough for execution. We introduce \\textbf{EV-WM}, a predicate-grounded verification framework for world-model planning. EV-WM rolls out candidate futures in pretrained visual-feature space, decodes them into structured event states, and scores them using task-progress, semantic-consistency, physical-feasibility, and uncertainty terms. The verifier guides sampling-based planning, gates candidate actions, and, in the contact-sensitive LIBERO wine-rack setting, selects among PPO-generated proposals. Across navigation, deformable-object, wall-constrained, and language-described manipulation studies, EV-WM shows that predicate-grounded verification can make feature-space world-model planning more interpretable and better aligned with task progress.", "source": "curated", "first_author": "Kailin Wang", "url": "https://arxiv.org/abs/2606.13053", "fig": "figures/ev-wm.webp"}, {"id": "expo-ft", "title": "EXPO-FT: Sample-Efficient Reinforcement Learning Finetuning for Vision-Language-Action Models", "tree": "rl", "branch": "post-training", "authors": ["Perry Dong", "Kuo-Han Hung", "Tian Gao", "Dorsa Sadigh", "Chelsea Finn"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-25", "topics": ["sample-efficient RL fine-tuning", "VLA"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["hil-serl", "pld", "ript-vla", "vla-jumpstart"], "arxiv": "2605.25477", "doi": "", "code": "", "site": "", "note": "Sample efficiency is the binding constraint for on-robot VLA fine-tuning. Compare its budget against HIL-SERL's 1-2.5 hours before choosing an approach.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The ability to efficiently and reliably learn new tasks has been a foundational challenge in robotics. Vision-Language-Action (VLA) models have demonstrated strong generalization across diverse manipulation tasks, yet pretrained policies consistently fall short of the reliability required for real-world deployment. Reinforcement learning (RL) fine-tuning offers a promising path to bridge this gap, but existing approaches either train from scratch without fully leveraging pretrained priors, or fine-tune VLAs without achieving the sample efficiency and success rates that practical deployment demands. We present EXPO-FT, a system for stable, sample-efficient RL finetuning of pretrained VLA policies that closes this gap. Our system solves a suite of challenging manipulation tasks, including routing string lights and inserting the plug to light it up, striking a pool ball into a pocket, and inserting a flower into a wine bottle, each requiring combinations of high precision, dynamic actions, and robustness to varied initial states. Our system achieves perfect task performance (30/30 successes) across all evaluated tasks within an average of 19.1 minutes of online robot data, outperforming both prior RL-from-scratch and VLA finetuning approaches. We release an open-source codebase with the aim of facilitating broader adoption of RL finetuning of VLA models in robotics.", "source": "curated", "first_author": "Perry Dong", "url": "https://arxiv.org/abs/2605.25477", "fig": "figures/expo-ft.webp"}, {"id": "egoemg", "title": "EgoEMG: A Multimodal Egocentric Dataset with Bilateral EMG and Vision for Hand Pose Estimation", "tree": "il", "branch": "tactile-learn", "authors": ["Ziheng Xi", "Jiayi Yu", "Yitao Wang", "Yanbo Duan", "Jianjiang Feng", "Jie Zhou"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-07", "topics": ["EMG", "egocentric", "hand pose", "intent"], "paradigm": ["dataset"], "method": [], "projects": ["geodex", "r2s2r", "wam-tactile"], "related": ["anyskin", "dexskin", "doglove", "dream-tac"], "arxiv": "2605.05712", "doi": "10.48550/arxiv.2605.05712", "code": "", "site": "", "note": "Bilateral 8-channel wrist EMG at 2 kHz paired with egocentric video, RGB-D and hand mocap, over 41 participants and 60 gestures. EMG carries force and intent <em>before</em> motion is visible, which is precisely the blind spot of a vision-only world model. Under-explored and cheap to instrument.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Surface electromyography (sEMG) records muscle activity during hand movement and can be decoded to recover detailed hand articulation. EMG and egocentric vision are complementary for hand sensing: EMG captures fine-grained finger articulation even under occlusion and poor lighting, while vision provides global hand configuration. However, no existing dataset synchronizes both modalities. We present EgoEMG, a multimodal egocentric dataset for bimanual hand pose estimation. EgoEMG includes bilateral wristband EMG with 16 total channels (8 per wrist) sampled at 2 kHz, 120 Hz IMU, egocentric wide-angle RGB video, external RGB-D video, and mocap-derived hand motion with wrist articulation angles. The dataset covers 41 participants performing 60 gesture classes, including 30 single-hand gestures and 30 bimanual gestures, totaling more than 10 hours of recording. We also introduce a benchmark with three tasks -- EMG-to-pose, vision-to-pose, and EMG+vision fusion -- under a shared joint-angle prediction target and common generalization split axes (cross-gesture, cross-user, and combined). As baselines, we evaluate EMGFormer for EMG-to-pose and generic ResNet/ViT backbones for vision-to-pose. We further study a residual fusion architecture that improves over matched lightweight vision-only baselines. Together, EgoEMG and its benchmark establish a foundation for future research on multimodal hand pose estimation with EMG and vision.", "source": "curated", "first_author": "Ziheng Xi", "url": "https://arxiv.org/abs/2605.05712", "fig": "figures/egoemg.webp", "figs": ["figures/egoemg-1.webp", "figures/egoemg-2.webp"]}, {"id": "egokit", "title": "EgoKit: Towards Unified Low-Cost Egocentric Data Collection with Heterogeneous Devices", "tree": "systems", "branch": "interfaces", "authors": ["Liuchuan Yu", "Erdem Murat", "Beichen Wang", "Yan Zeng", "Tingting Luo", "Huizhen Zhou", "Shanghao Li", "Huining Feng", "Zhigen Zhao", "Ning Yang", "Ke Jing", "Yunhao Liu", "Ruoya Sheng"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-16", "topics": ["low-cost egocentric capture", "heterogeneous devices", "open toolkit"], "paradigm": ["hardware"], "method": [], "projects": ["r2s2r"], "related": ["dexcap", "egomimic", "open-aoe", "umi"], "arxiv": "2605.16797", "doi": "10.48550/arxiv.2605.16797", "code": "", "site": "", "note": "A unified low-cost toolkit for egocentric collection across phones, action cameras and XR headsets, logging head pose and OpenXR 26-joint hand tracking alongside video. If you run your own capture rather than consuming someone else's corpus, start here.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Egocentric video is increasingly used as a data source for robot learning, activity understanding, and embodied AI research, but collecting it at scale remains fragmented in practice: each candidate host device, such as an Android phone, iPhone, iPad, smart glasses, or extended reality (XR) headset, exposes a different SDK, a different policy on raw camera access, and different limitations on external USB cameras and on-device tracking. Synchronized ego-view and wrist-view capture is therefore typically obtained by either committing to a single proprietary platform or building one-off rigs that do not transfer across devices. To address this gap, we present EgoKit, a toolkit that exposes the same egocentric recording workflow across six heterogeneous host devices. Across all supported devices, EgoKit presents the same recording interaction and produces locally stored video with a uniform log format; on XR headsets, it additionally logs head pose and OpenXR-standard 26-joint hand tracking aligned to the video streams. The companion accessories, including two wrist cameras with mounts, a head strap, and a USB-C hub, add wrist-view capture to any supported host without custom hardware fabrication. EgoKit is available at \\url{https://egokit.chuange.org/}.", "source": "curated", "first_author": "Liuchuan Yu", "url": "https://arxiv.org/abs/2605.16797", "fig": "figures/egokit.webp", "figs": ["figures/egokit-1.webp", "figures/egokit-2.webp", "figures/egokit-3.webp"]}, {"id": "egolive", "title": "EgoLive: A Large-Scale Egocentric Dataset from Real-World Human Tasks", "tree": "il", "branch": "human-video", "authors": ["Yihang Li", "Xuelong Wei", "Jingzhou Luo", "Yingjing Xiao", "Yibo Bai", "Guangyuan Zhou", "Teng Zou", "Chenguang Gui", "Jiajun Wen", "He Zhang", "Kangliang Chen", "Xing Pan", "Shuaiyan Liu", "Daming Wang", "Tao An", "Jiayi Li", "Shibo Jin", "Wanwan Zhang", "Tianyu Wang", "Boren Wei", "Zhixuan Huang", "Fangsheng Liu", "Ruodai Li", "Hui Zhang", "Anson Li", "Yicheng Gong", "Peng Cao", "Jiaming Liang", "Liang Lin"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-26", "topics": ["egocentric dataset", "semantic coverage", "long tail"], "paradigm": ["dataset"], "method": [], "projects": ["r2s2r"], "related": ["ego4d", "egoscale"], "arxiv": "2604.23570", "doi": "10.48550/arxiv.2604.23570", "code": "", "site": "", "note": "Argues that the long tail of semantic coverage matters more than hour count, and benchmarks itself against EgoDex and Xperience-10M on exactly that axis. The useful corrective to a table sorted by hours.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The advancement of robot learning is currently hindered by the scarcity of large-scale, high-quality datasets. While established data collection methods such as teleoperation and universal manipulation interfaces dominate current datasets, they suffer from inherent limitations in scalability and real-world deployability. Human egocentric video collection, by contrast, has emerged as a promising approach to enable scalable, natural and in-the-wild data collection. As such, we present EgoLive, a large-scale, high-quality egocentric dataset designed explicitly for robot manipulation learning. EgoLive establishes three distinctive technical advantages over existing egocentric datasets: first, it represents the largest open-source annotated egocentric dataset focused on real-world task-oriented human routines to date; second, it delivers leading data quality via a customized head-mounted capture device and comprehensive high-precision multi-modal annotations; third, all data is collected exclusively in unconstrained real-world scenarios and encompasses vertical field human working data, including home service, retail, and other practical work scenarios, providing superior diversity and ecological validity. With the introduction of EgoLive, we aim to provide the research community with a scalable, high-quality dataset that accelerates breakthroughs in generalizable robotic models and facilitates the real-world deployment of robot systems.", "source": "curated", "first_author": "Yihang Li", "url": "https://arxiv.org/abs/2604.23570", "fig": "figures/egolive.webp", "figs": ["figures/egolive-1.webp"]}, {"id": "egosteer", "title": "EgoSteer: A Full-Stack System Towards Steerable Dexterous Manipulation from Egocentric Videos", "tree": "systems", "branch": "interfaces", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["University of Pennsylvania", "Generalist AI", "Peking University", "AgiBot"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-09", "topics": ["egocentric steering", "dexterous manipulation system"], "paradigm": ["system"], "method": [], "projects": ["geodex", "m2", "r2s2r"], "related": ["dexumi", "do-as-i-do", "egomimic"], "arxiv": "2607.09701", "doi": "", "code": "", "site": "", "note": "From your M2 humanoid_platforms folder. Full-stack egocentric dexterity — a close neighbour of your ego-dexterity idea.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2607.09701v1_EgoSteer_A_Full_Stack_System_Towards_Steerable_Dexterous_Man.pdf", "abstract": "Steerability is a defining capability of generalist robot policies, yet remains largely absent in dexterous-hand systems for lack of large-scale, language-aligned, and action-accurate demonstration data. To address this bottleneck, we present a full-stack system that scales dexterous VLA pre-training from egocentric human videos and enables data-efficient real-robot post-training. It integrates EgoSmith, a data pipeline that curates in-the-wild egocentric videos into 9.6K hours of high-quality pre-training data with 9x higher throughput and better accuracy than prior SOTA; a unified robot stack for teleoperation and human-in-the-loop correction; and EgoSteer, a world-model-enhanced VLA trained on optimized infrastructure. Human-data pre-training equips EgoSteer with language-guided manipulation priors, which are grounded through robot post-training and improved by DAgger refinement. Empirically, EgoSteer robustly executes free-form instructions across 40+ diverse tasks, demonstrating failure recovery, dexterity, and generalization. The pre-trained model also few-shot adapts to complex long-horizon tasks, including box folding, on two embodiments with 75+% success. We open-source the system, data, and model at https://egosteer.github.io/.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2607.09701", "fig": "figures/egosteer.webp", "figs": ["figures/egosteer-1.webp", "figures/egosteer-2.webp"]}, {"id": "embo-team", "title": "EmboTeam: Grounding LLM Reasoning into Reactive Behavior Trees via PDDL for Embodied Multi-Robot Collaboration", "tree": "systems", "branch": "agentic", "authors": ["Haishan Zeng", "Mengna Wang", "Peng Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-16", "topics": ["behaviour trees", "PDDL grounding", "multi-robot planning"], "paradigm": ["LLM"], "method": [], "projects": ["m2"], "related": ["harbor", "llm-teams", "roco", "voyager"], "arxiv": "2601.11063", "doi": "10.48550/arxiv.2601.11063", "code": "", "site": "", "note": "Grounds LLM reasoning into reactive behaviour trees via PDDL, so the plan is executable and re-plannable rather than a paragraph of hope. The architecture to copy if you want an LLM in M2's planning loop without giving up reactivity.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In embodied artificial intelligence, enabling heterogeneous robot teams to execute long-horizon tasks from high-level instructions remains a critical challenge. While large language models (LLMs) show promise in instruction parsing and preliminary planning, they exhibit limitations in long-term reasoning and dynamic multi-robot coordination. We propose EmboTeam, a novel embodied multi-robot task planning framework that addresses these issues through a three-stage cascaded architecture: 1) It leverages an LLM to parse instructions and generate Planning Domain Definition Language (PDDL) problem descriptions, thereby transforming commands into formal planning problems; 2) It combines the semantic reasoning of LLMs with the search capabilities of a classical planner to produce optimized action sequences; 3) It compiles the resulting plan into behavior trees for reactive control. The framework supports dynamically sized heterogeneous robot teams via a shared blackboard mechanism for communication and state synchronization. To validate our approach, we introduce the MACE-THOR benchmark dataset, comprising 42 complex tasks across 8 distinct household layouts. Experiments show EmboTeam improves the task success rate from 12% to 55% and goal condition recall from 32% to 72% over the LaMMA-P baseline.", "source": "curated", "first_author": "Haishan Zeng", "url": "https://arxiv.org/abs/2601.11063", "fig": "figures/embo-team.webp", "figs": ["figures/embo-team-1.webp"]}, {"id": "geniworld", "title": "GeniWorld: A Generalizable Interactive World Model for Robotic Manipulation via Visual Actions", "tree": "il", "branch": "world-models", "authors": ["Chenghao Gu", "Hanyang Yu", "Jingbo Zhang", "Haitao Lin", "Wenyao Zhang", "Jinghe Wang", "Hanglei Jin", "Shuzhao Xie", "Jingyan Jiang", "Zhi Wang"], "corresponding": [], "institutions": ["Tsinghua University", "HKUST", "Tencent"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-06", "topics": ["generalizable interactive world model", "visual actions", "unseen scenarios"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["dreamdojo", "dreamzero", "genie-envisioner", "ivideogpt"], "arxiv": "2608.06332", "doi": "10.48550/arxiv.2608.06332", "code": "", "site": "", "note": "<b>GeniWorld</b> — an interactive world model conditioned on <em>visual actions</em>, aimed squarely at generalization to unseen scenes. The most recent entry in the interactive-WAM line at the time of writing; useful mainly as a generalization benchmark to position against.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Generalist robot policies exhibit strong capabilities, but their robustness in complex and unseen environments remains limited. Scaling robot learning and evaluation in diverse real-world environments remains costly and challenging. Action-conditioned world models offer a promising alternative, but they often suffer from limited action controllability and poor generalization to out-of-distribution (OOD) scenarios. To this end, we present GeniWorld, an interactive world model for robots that generalizes robustly across unseen scenarios. Building on pretrained video generative models, we use URDF-based rendering to transform numerical actions into visual action representations, enabling spatially grounded action control. By explicitly decoupling embodiment kinematics from environmental dynamics, our model mitigates scene overfitting and facilitates modeling of robot-environment interactions. To achieve closed-loop control, we construct an autoregressive video prediction model integrated with high-frequency robot kinematic control, enabling interaction with both robot policies and human teleoperators. In our experiments, even when trained solely on limited fixed-scene data, our model achieves superior in-domain performance and robust zero-shot generalization to highly randomized, unseen environments. For downstream applications, GeniWorld serves as a scalable policy evaluator that remains reliable under environmental perturbations. Furthermore, even with limited real-world demonstrations, GeniWorld generates diverse manipulation trajectories within the world model, improving downstream policy performance and robustness in complex environments.", "source": "curated", "first_author": "Chenghao Gu", "url": "https://arxiv.org/abs/2608.06332", "fig": "figures/geniworld.webp", "figs": ["figures/geniworld-1.webp", "figures/geniworld-2.webp", "figures/geniworld-3.webp"]}, {"id": "handoff", "title": "HANDOFF: Humanoid Agentic Task-Space Whole-Body Control via Distilled Complementary Teachers", "tree": "rl", "branch": "loco-manip", "authors": ["Lizhi Yang", "Junheng Li", "Nehar Poddar", "Yiling Hou", "Gio Huh", "Robert Griffin", "Georgia Gkioxari", "Aaron Ames"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-04", "topics": ["task-space whole-body control", "distilled teachers", "agentic"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["amo", "exbody2", "hover", "maskedmimic", "spatial-brain"], "arxiv": "2606.06493", "doi": "10.48550/arxiv.2606.06493", "code": "", "site": "", "note": "Agentic task-space whole-body control distilled from complementary teacher policies. The distillation of several specialist teachers into one deployable policy is the same move [[hover]] makes for command modes.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "For a humanoid robot to be deployed in the real world, the choice of command space (i.e., the interface between task planning and whole-body control) is crucial. Existing whole-body controllers typically demand dense kinematic or spatial references that planners struggle to synthesize from task semantics. We instead propose a compact, explicit interface that is intuitive, general, modular, and expressive enough for diverse loco-manipulation skills. To this end, we introduce HANDOFF, a single humanoid whole-body controller that follows this interface and is distilled via multi-teacher KL distillation under a context-conditioned gating scheme into a mixture-of-experts student from three complementary specialists: whole-body motion tracking with safety-filtered data, locomotion, and fall-recovery. On the Unitree G1, HANDOFF matches state-of-the-art velocity tracking and offers one of the largest robust manipulation workspaces. We further demonstrate hardware feasibility through multiple natural-language-driven task roll-outs, powered by a VLM-driven agentic planner with no task-specific data or controller fine-tuning.", "source": "curated", "first_author": "Lizhi Yang", "url": "https://arxiv.org/abs/2606.06493", "fig": "figures/handoff.webp", "figs": ["figures/handoff-1.webp"]}, {"id": "hand-in-loop", "title": "Hand-in-the-Loop: Improving VLA Policies for Dexterous Manipulation via Seamless Hand-Arm Intervention", "tree": "rl", "branch": "post-training", "authors": ["Zhuohang Li", "Liqun Huang", "Wei Xu", "Zhengming Zhu", "Nie Lin", "Xiao Ma", "Xinjun Sheng", "Ruoshi Wen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-14", "topics": ["hand-arm intervention", "VLA improvement", "dexterous manipulation"], "paradigm": ["RL", "VLA"], "method": [], "projects": ["geodex", "mabel"], "related": ["hil-serl", "pi06", "pld", "ript-vla"], "arxiv": "2605.15157", "doi": "10.48550/arxiv.2605.15157", "code": "", "site": "", "note": "Seamless hand-arm human intervention to improve a dexterous VLA — HIL-SERL's idea carried into multi-finger manipulation, where taking over mid-episode is much harder than with a gripper.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models are prone to compounding errors in dexterous manipulation, where high-dimensional action spaces and contact-rich dynamics amplify small policy deviations over long horizons. While Interactive Imitation Learning (IIL) can refine policies through human correction data, applying it to high-degree-of-freedom (DoF) robotic hands remains challenging due to a command mismatch between human teleoperation and policy execution at the intervention moment, which causes abrupt robot-hand configuration changes, or \"gesture jumps\". We present Hand-in-the-Loop (HandITL), a seamless human-in-the-loop intervention method that blends human corrective intent with autonomous policy execution to avoid gesture jumps during bimanual dexterous manipulation. Compared with taking over control using direct teleoperation, HandITL reduces intervention jitter by 99.8% and preserves robust post-intervention manipulation, reducing grasp failures by 87.5% and mean completion time by 19.1%. We validate HandITL on tasks requiring bimanual coordination, tool use, and fine-grained long-horizon manipulation. When used to collect correction data for policy refinement, HandITL yields policies that outperform those trained with standard teleoperation data by 19% on average across three long-horizon dexterous tasks.", "source": "curated", "first_author": "Zhuohang Li", "url": "https://arxiv.org/abs/2605.15157", "fig": "figures/hand-in-loop.webp", "figs": ["figures/hand-in-loop-1.webp"]}, {"id": "himem-wam", "title": "HiMem-WAM: Hierarchical Memory-Gated World Action Models for Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Xiaoquan Sun", "Ruijian Zhang", "Chen Cao", "Yihan Sun", "Jiahui Chen", "Zetian Xu", "Bo Chen", "Haijier Chen", "Zhen Yang", "Jiarun Zhu", "Yijun Hong", "JingZhe Xu", "Jingrui Pang", "Mingqi Yuan", "Jiayu Chen"], "corresponding": [], "institutions": ["Tsinghua University", "University of Hong Kong"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-09", "topics": ["hierarchical memory gating", "long-horizon consistency"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["dreamdojo", "ev-wm", "storm", "wla-model"], "arxiv": "2606.10363", "doi": "10.48550/arxiv.2606.10363", "code": "", "site": "", "note": "<b>HiMem-WAM</b> — hierarchical memory gating for world-action models, targeting the failure that limits every autoregressive WAM in practice: context drift over a long rollout. Relevant if your tactile WAM has to stay coherent across a multi-minute contact-rich task.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World Action Models (WAMs) have emerged as a new powerful paradigm for embodied intelligence, learning action-relevant visual dynamics that significantly enhance generalization and robustness. However, existing WAMs still struggle with task-relevant memory in long-horizon robotic manipulation. To address this, we present HiMem-WAM, a Hierarchical Memory-Gated WAM that integrates motion-centric latent actions, high-level skill latents, and boundary-triggered memory updates. Specifically, we develop a hierarchical latent action framework that jointly learns low-level motion and high-level skill latents, providing structured temporal abstraction. Meanwhile, a boundary-aware memory gate writes compact task states at predicted skill transitions, enabling causal inference without test-time generation of future video or optical flow estimation. Evaluated on LIBERO, LIBERO-PLUS, RMBench and real-world tasks, HiMem-WAM shows that hierarchical latents improve robustness under deployment perturbations, and the memory module substantially benefits memory-dependent long-horizon manipulation.", "source": "curated", "first_author": "Xiaoquan Sun", "url": "https://arxiv.org/abs/2606.10363", "fig": "figures/himem-wam.webp", "figs": ["figures/himem-wam-1.webp"]}, {"id": "humannet", "title": "HumanNet: Scaling Human-centric Video Learning to One Million Hours", "tree": "il", "branch": "human-video", "authors": ["Yufan Deng", "Daquan Zhou"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-07", "topics": ["human-centric video learning", "million-hour scale"], "paradigm": ["IL", "VLA"], "method": [], "projects": ["r2s2r"], "related": ["dreamgen", "ego2robot", "egoscale"], "arxiv": "2605.06747", "doi": "10.48550/arxiv.2605.06747", "code": "", "site": "", "note": "Aimed explicitly at the million-hour regime, with controlled VLA ablations isolating what an extra order of magnitude of human video actually buys. The ablation is the useful part — very few papers separate the value of scale from the value of everything else they changed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Progress in embodied intelligence increasingly depends on scalable data infrastructure. While vision and language have scaled with internet corpora, learning physical interaction remains constrained by the lack of large, diverse, and richly annotated human activity data. We present HumanNet, a one-million-hour human-centric video corpus that captures how humans interact with the physical world at scale. HumanNet spans both first-person and third-person perspectives and covers fine-grained activities, human-object interactions, tool use, and long-horizon behaviors across diverse real-world environments. Beyond raw video, the dataset provides interaction-centric annotations, including captions, motion descriptions, and hand and body-related signals, enabling motion-aware and interaction-aware learning. Beyond scale, HumanNet introduces a systematic data curation paradigm for embodied learning, where human-centric filtering, temporal structuring, viewpoint diversity, and annotation enrichment are treated as first-class design principles. This design transforms unstructured internet video into a scalable substrate for representation learning, activity understanding, motion generation, and human-to-robot transfer. We conduct a first-step validation on the value of this design through controlled vision-language-action ablation: under a fixed set of validation data, continued training from the Qwen VLM model with 1000 hours of egocentric video drawn from HumanNet surpasses the continued training with 100 hours of real-robot data from Magic Cobot, indicating that egocentric human video could be a scalable and cost-effective substitute for robot data. By building this project, we aim to explore the opportunity to scale embodied foundation models using human-centric videos, rather than relying solely on robot-specific data.", "source": "curated", "first_author": "Yufan Deng", "url": "https://arxiv.org/abs/2605.06747", "fig": "figures/humannet.webp", "figs": ["figures/humannet-1.webp", "figures/humannet-2.webp"]}, {"id": "spatial-brain", "title": "Humanoid Whole-Body Manipulation via Active Spatial Brain and Generalizable Action Cerebellum", "tree": "rl", "branch": "loco-manip", "authors": ["Zhizhao Liang", "Yi-Lin Wei", "Xuhang Chen", "Mu Lin", "Yi-Xiang He", "Zhexi Luo", "Jun-Hui Liu", "Kun-Yu Lin", "Wei-Shi Zheng"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-20", "topics": ["whole-body manipulation", "spatial reasoning", "generalizable action"], "paradigm": ["VLA", "RL"], "method": [], "projects": ["mabel"], "related": ["cosmos-reason", "groot-n1", "handoff", "helix"], "arxiv": "2605.21133", "doi": "10.48550/arxiv.2605.21133", "code": "", "site": "", "note": "Splits humanoid whole-body manipulation into an 'active spatial brain' and a 'generalizable action cerebellum' — the dual-system split again, this time with spatial reasoning as the slow half.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In this paper, we explore spatial-aware humanoid whole-body manipulation task. Compared with tabletop settings, this task poses two key challenges: 1) Spatial understanding is challenging in complex 3D environments with diverse spatial relations. 2) Action generation is difficult to generalize, as limited and costly real-robot data restricts data-driven models generalization. To address these challenges, we propose a generalizable humanoid loco-manipulation framework that leverages the spatial perception and action generation capabilities of multi-agent large models. Specifically, our framework includes two components: Active Spatial Brain for active spatial perception and decision-making, and Generalizable Action Cerebellum for executable robot action generation. The first component actively perceives the spatial scene and makes decisions on task planning and subtask decomposition. The second component generate executable robot actions based on the decisions made by the first module without needs of task-specific real robot data. To benchmark our framework, we design a set of spatial manipulation tasks from two perspectives: evaluating spatial perception and understanding, and assessing real-robot task performance. The results demonstrate strong performance on both aspects across diverse tasks and environments.", "source": "curated", "first_author": "Zhizhao Liang", "url": "https://arxiv.org/abs/2605.21133", "fig": "figures/spatial-brain.webp", "figs": ["figures/spatial-brain-1.webp"]}, {"id": "lawm-3d", "title": "LAWM-3D: Learning 3D-Aware Latent Actions from Human Videos for Generalizable Robot World Models", "tree": "il", "branch": "world-models", "authors": ["Jiarui Yang", "Jiale Zhange", "Jiawei Li", "Hang Guo", "Wen Huang", "Jinpeng Wang", "Peidong Liu", "Shu-Tao Xia"], "corresponding": [], "institutions": ["Tsinghua University", "EPFL"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-06", "topics": ["3D-aware latent actions", "multi-view + single-view joint training", "human video"], "paradigm": ["world model", "IL"], "method": [], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["ego2robot", "hrdt", "lapa", "lawam", "lawm", "pointworld"], "arxiv": "2608.05706", "doi": "10.48550/arxiv.2608.05706", "code": "", "site": "", "note": "<b>LAWM-3D</b> — latent actions that are <em>3D-aware</em>, trained jointly on multi-view and single-view data so that a latent learned from a monocular human video still means something metric. The obvious failure mode of LAPA-style latents is that they encode 2D optical flow; this addresses it directly, and matters for any pipeline where the human video and the robot see different viewpoints.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models enable agents to perform forward rollout and planning without real-world interaction. However, their application in open-world embodied intelligence remains limited by the high cost of action annotations and the heterogeneity of action spaces across platforms. Recently, latent action models (LAMs) have alleviated this bottleneck by learning action representations directly from unlabeled human videos in a self-supervised manner. Nevertheless, most existing LAMs rely on single-view inputs and operate primarily in 2D pixel space, raising a fundamental question: can simply incorporating multi-view videos into LAM training endow the learned latent actions with 3D-aware perception? Our study shows that the answer is negative. The primary reasons lie in future-frame appearance leakage as well as inter-camera appearance discrepancies and viewpoint variations. To address these issues, we propose LAWM-3D, which introduces three tightly coupled key designs: (1) a multi-view invariant unified action tokenization scheme for learning 3D-aware latent actions; (2) a geometric alignment constraint that anchors intermediate encoder features to a pretrained 3D foundation model, thereby explicitly providing cross-view geometric correspondences; and (3) a non-injective RGB-D joint reconstruction objective that prevents shortcut learning from future-frame appearance information, forcing the LAM to focus supervision on motion cues with geometric significance. Importantly, these components are not simply stacked but are tightly coupled through a unified motivation. Built upon a two-stage paradigm of large-scale human video pretraining followed by robot fine-tuning, extensive experiments demonstrate that the proposed 3D-aware latent actions significantly improve world model performance, achieving SOTA results in generation quality, physical consistency, and generalization ability.", "source": "curated", "first_author": "Jiarui Yang", "url": "https://arxiv.org/abs/2608.05706", "fig": "figures/lawm-3d.webp", "figs": ["figures/lawm-3d-1.webp", "figures/lawm-3d-2.webp", "figures/lawm-3d-3.webp"]}, {"id": "intention-priors", "title": "Learning Human-Intention Priors from Large-Scale Human Demonstrations for Robotic Manipulation", "tree": "il", "branch": "human-video", "authors": ["Yifan Xie", "Yuan Wang", "Guangyu Chen", "Jinkun Liu", "Yu Sun", "Wenbo Ding"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-27", "topics": ["human-intention priors", "large-scale demonstrations"], "paradigm": ["IL"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["ego2robot", "egoscale", "humanoid-policy", "interact"], "arxiv": "2604.24681", "doi": "10.48550/arxiv.2604.24681", "code": "", "site": "", "note": "Learns intention priors from large-scale human demonstrations rather than action labels. A different answer to the embodiment gap: transfer the <em>intent</em>, and let the robot solve the kinematics itself.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Human videos contain rich manipulation priors, but using them for robot learning remains difficult because raw observations entangle scene understanding, human motion, and embodiment-specific action. We introduce MoT-HRA, a hierarchical vision-language-action framework that learns human-intention priors from large-scale human demonstrations. We first curate HA-2.2M, a 2.2M-episode action-language dataset reconstructed from heterogeneous human videos through hand-centric filtering, spatial reconstruction, temporal segmentation, and language alignment. On top of this dataset, MoT-HRA factorizes manipulation into three coupled experts: a vision-language expert predicts an embodiment-agnostic 3D trajectory, an intention expert models MANO-style hand motion as a latent human-motion prior, and a fine expert maps the intention-aware representation to robot action chunks. A shared-attention trunk and read-only key-value transfer allow downstream control to use human priors while limiting interference with upstream representations. Experiments on hand motion generation, simulated manipulation, and real-world robot tasks show that MoT-HRA improves motion plausibility and robust control under distribution shift.", "source": "curated", "first_author": "Yifan Xie", "url": "https://arxiv.org/abs/2604.24681", "fig": "figures/intention-priors.webp", "figs": ["figures/intention-priors-1.webp"]}, {"id": "lucid-xr", "title": "Lucid-XR: An Extended-Reality Data Engine for Robotic Manipulation", "tree": "systems", "branch": "interfaces", "authors": ["Yajvan Ravan", "Adam Rashid", "Alan Yu", "Kai McClennen", "Gio Huh", "Kevin Yang", "Zhutian Yang", "Qinxi Yu", "Xiaolong Wang", "Phillip Isola", "Ge Yang"], "corresponding": [], "institutions": ["MIT", "UC Berkeley"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2026, "date": "2026-04-30", "topics": ["XR data engine", "synthetic demonstrations"], "paradigm": ["hardware", "IL"], "method": [], "projects": [], "related": ["bunny-visionpro", "open-television"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. XR as a demonstration *generator* rather than just a controller.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/858_Lucid_XR_An_Extended_Reali.pdf", "abstract": "", "source": "curated", "first_author": "Yajvan Ravan", "url": "https://arxiv.org/abs/2605.00244", "fig": "figures/lucid-xr.webp", "figs": ["figures/lucid-xr-1.webp", "figures/lucid-xr-2.webp", "figures/lucid-xr-3.webp"]}, {"id": "midas-hand", "title": "MIDAS Hand: Modular low-Impedance Direct-drive Anthropomorphic Sensing Hand", "tree": "systems", "branch": "hands", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-14", "topics": ["direct-drive hand", "low impedance", "modular"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "m2"], "related": ["dexwrist", "dlr-hand", "orca-hand", "pds-joint", "ruka"], "arxiv": "2607.14487", "doi": "", "code": "", "site": "", "note": "From your M2 library. Direct-drive is the opposite bet from tendons — worse packing, far better force transparency. The relevant tradeoff study for your hand design.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2607.14487v1_MIDAS_Hand_Modular_low_Impedance_Direct_drive_Anthropomorphi.pdf", "abstract": "Dexterous manipulation is limited not only by algorithms but by a shortage of accessible hand hardware that combines human-scale morphology, ease of manufacturing or maintenance, tactile sensing, and practical cost. Existing dexterous hands tend to optimize some of these properties at the expense of others. We present MIDAS Hand, a low-cost, open-source, human-scale dexterous hand with integrated tactile sensing for manipulation research. MIDAS Hand provides 16 total degrees of freedom (DoF) with 13 active DoF, directly driven actuation with measurably low backdrive torque, and 283 three-axis tactile taxels in a compact 700 g package with a bill of materials under 3,000 USD. Built from 3D-printed components, it assembles in under three hours while providing the strength, repeatability, and maintainability needed for repeated real-world experiments. Alongside the hardware, we release a full stack: design files, build documentation, control and tactile Python APIs, simulation models, and retargeting and teleoperation pipelines. We characterize MIDAS Hand through workspace and grasp-taxonomy analysis, payload and reliability tests, backdrivability measurements, and teleoperation demonstrations with tactile sensing, showing that it offers a balanced, reproducible platform for tactile dexterous manipulation and human-to-robot data collection. Project page: https://midas-hand.com", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2607.14487", "fig": "figures/midas-hand.webp", "figs": ["figures/midas-hand-1.webp", "figures/midas-hand-2.webp", "figures/midas-hand-3.webp"]}, {"id": "mobilewam", "title": "MobileWAM: Bridging World Action Models to Mobile Manipulation with Chain-of-Foresight", "tree": "il", "branch": "world-models", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Jiao Tong University", "Fudan University", "HKUST"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-04", "topics": ["world-action model", "mobile manipulation"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["m2", "mabel", "wam-tactile"], "related": ["decowam", "flare", "motionwam", "oa-wam", "vjepa2"], "arxiv": "2608.04657", "doi": "", "code": "", "site": "", "note": "From your M2 whole_body_loco_manipulation folder. Directly relevant to the mobile-dexterous-UMI and WAM threads — read alongside DECOWAM.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.04657v2_MobileWAM_Bridging_World_Action_Models_to_Mobile_Manipulatio.pdf", "abstract": "World action models (WAMs) built on video generation backbones are a rising recipe for robot learning, yet remain confined to tabletop manipulation. Mobile manipulation demands simultaneous locomotion and whole-body manipulation amid scene-scale dynamics, yet is still dominated by dynamics-blind visual encoders with hand-crafted coordination. We bridge this gap with MobileWAM, a mixture-of-transformers architecture that fuses a pretrained video diffusion transformer with a lightweight action expert through layerwise joint attention, translating internet-scale motion priors into whole-body control. To reconcile the heterogeneous dynamics of moving and manipulating, each feed-forward layer of the action expert becomes a three-expert mixture of shared, locomotion, and manipulation experts, softly routed by the motion intent in the action tokens. To densify supervision, we further propose Chain-of-Foresight (CoF): intermediate representations sequentially predict a chain of future latent chunks, each step conditioned on its predecessor. CoF pairs naturally with our decoupled video--action denoising scheme. At deployment, the WAM serves as a pure current-frame encoder; foresight acts only through gradients, so at inference the foresight chain and video generation are discarded, leaving only policy-level cost. MobileWAM surpasses state-of-the-art mobile manipulation policies on ManiSkill-HAB and fine-tunes to a real ARX Lift2 mobile manipulator across diverse tasks with strong generalization. Code will be released soon.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2608.04657", "fig": "figures/mobilewam.webp", "figs": ["figures/mobilewam-1.webp"]}, {"id": "groot-n16", "title": "NVIDIA Isaac GR00T N1.6", "tree": "systems", "branch": "vla", "authors": ["NVIDIA GEAR Lab"], "corresponding": [], "institutions": ["NVIDIA"], "lab": "NVIDIA GEAR Lab", "venue": "tech report", "venue_type": "industry", "year": 2026, "date": "2026-01-01", "topics": ["humanoid foundation model", "Cosmos Reason integration"], "paradigm": ["VLA"], "method": [], "projects": ["mabel"], "related": ["cosmos-reason", "figure-fleet", "groot-n1", "groot-n15"], "arxiv": "", "doi": "", "code": "", "site": "https://research.nvidia.com/labs/gear/gr00t-n1_6/", "note": "Integrates Cosmos Reason as the deep-thinking System 2, and reports beating N1.5 on simulated benchmarks and on real bimanual YAM, AgiBot Genie-1 and Unitree G1. Relevant to MABEL because it defines what an off-the-shelf humanoid policy is expected to do in 2026.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "NVIDIA GEAR Lab", "url": "https://research.nvidia.com/labs/gear/gr00t-n1_6/"}, {"id": "navdreamer", "title": "NavDreamer: Video Models as Zero-Shot 3D Navigators", "tree": "rl", "branch": "model-based", "authors": ["Xijie Huang", "Weiqi Gai", "Tianyue Wu", "Congyu Wang", "Zhiyang Liu", "Xin Zhou", "Yuze Wu", "Fei Gao"], "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2026, "date": "2026-02-10", "topics": ["video model as a 3D navigator", "zero-shot navigation", "no map"], "paradigm": ["world model"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dino-wm", "gaia-2", "navigation-world-models", "vista-wm"], "arxiv": "2602.09765", "doi": "10.1109/LRA.2026.3713727", "code": "", "site": "", "note": "<b>NavDreamer</b> — video models as zero-shot 3D navigators. The mobile-base counterpart to the manipulation WAMs, and relevant to MABEL for exactly that reason: if the wheeled base can be driven by imagining forward, the navigation stack stops being a separate subsystem with its own map.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Previous Vision-Language-Action models face critical limitations in navigation: scarce, diverse data from labor-intensive collection and static representations that fail to capture temporal dynamics and physical laws. We propose NavDreamer, a video-based framework for 3D navigation that leverages generative video models as a universal interface between language instructions and navigation trajectories. Our main hypothesis is that video's ability to encode spatiotemporal information and physical dynamics, combined with internet-scale availability, enables strong zero-shot generalization in navigation. To mitigate the stochasticity of generative predictions, we introduce a sampling-based optimization method that utilizes a VLM for trajectory scoring and selection. An inverse dynamics model is employed to decode executable waypoints from generated video plans for navigation. To systematically evaluate this paradigm in several video model backbones, we introduce a comprehensive benchmark covering object navigation, precise navigation, spatial grounding, language control, and scene reasoning. Extensive experiments demonstrate robust generalization across novel objects and unseen environments, with ablation studies revealing that navigation's high-level decision-making nature makes it particularly suited for video-based planning.", "source": "curated", "first_author": "Xijie Huang", "url": "https://arxiv.org/abs/2602.09765", "fig": "figures/navdreamer.webp", "figs": ["figures/navdreamer-1.webp", "figures/navdreamer-2.webp"]}, {"id": "oa-wam", "title": "OA-WAM: Object-Addressable World Action Model for Robust Robot Manipulation", "tree": "il", "branch": "world-models", "authors": ["Yushan Liu", "Peibo Sun", "Shoujie Li", "Yifan Xie", "Lingfeng Zhang", "Xintao Chao", "Shiyuan Dong", "Fang Chen", "Xiao-Ping Zhang", "Wenbo Ding"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-07", "topics": ["object-addressable world-action model", "robustness"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["decowam", "dream-tac", "dreamzero", "mobilewam", "vjepa2"], "arxiv": "2605.06481", "doi": "", "code": "", "site": "", "note": "OA-WAM makes the world-action model address objects explicitly rather than predicting an undifferentiated scene. Object-centric structure is the most promising answer to WAM rollouts degrading over long horizons.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World Action Models (WAMs) enhance Vision-Language-Action policies by jointly predicting scene evolution and robot actions, but existing methods usually represent the predicted world as holistic images, video tokens, or global latents. These representations are difficult for an action decoder to address when an instruction refers to a particular object, especially under scene shifts where object identity is entangled with context. We propose OA-WAM, an Object-Addressable World Action Model for robust robot manipulation. OA-WAM decomposes each frame into N+1 slot states, with one robot slot and N object slots. Each slot contains a persistent address vector and a time-varying content vector, and is fused with text, image, proprioception, and past-action tokens in a block-causal sequence. A world head predicts next-frame slot states, while a flow-matching action head decodes a 16-step continuous action chunk in the same forward pass. Addressability is enforced by routing cross-slot attention through address-only keys and resetting the address slice at every transformer layer, separating which object to act on from what that object currently is without adding extra tokens. OA-WAM matches strong VLA and WAM baselines on LIBERO (97.8%) and SimplerEnv (79.3%), reaches state-of-the-art performance on the most relevant LIBERO-Plus geometric axes, and remains competitive on the seven-axis aggregate. A causal slot-intervention test yields a swap-binding cosine of 0.87, versus at most 0.09 for holistic baselines. These results suggest that addressable object states provide an effective interface for robust world-action modeling under scene perturbations.", "source": "curated", "first_author": "Yushan Liu", "url": "https://arxiv.org/abs/2605.06481", "fig": "figures/oa-wam.webp", "figs": ["figures/oa-wam-1.webp", "figures/oa-wam-2.webp"]}, {"id": "omnitactune", "title": "OmniTacTune: Policy-Agnostic Real-World RL for Tactile Residual Adaptation of Visual Policies", "tree": "systems", "branch": "tactile-hw", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Georgia Tech", "University of Maryland"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-03", "topics": ["tactile residual RL", "real-world fine-tuning"], "paradigm": ["RL"], "method": [], "projects": ["geodex", "m2", "wam-tactile"], "related": ["dexskin", "far-dex", "hil-serl"], "arxiv": "2607.03723", "doi": "", "code": "", "site": "", "note": "From your M2 library. Tactile residual RL on top of an existing policy — the tactile version of your residual-world-model idea.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2607.03723v1_OmniTacTune_Policy_Agnostic_Real_World_RL_for_Tactile_Residu.pdf", "abstract": "Visual policies learned from human videos, teleoperation, and robot demonstrations offer scalable motion priors, but often fail in contact-rich manipulation, where success significantly depends on local force and contact geometry. Tactile sensing provides these complementary signals, yet tactile data remain costly to collect and hard to generalize across sensors, robots, and tasks. We introduce OmniTacTune, a policy-agnostic real-world RL pipeline that adapts tactile feedback to pretrained visual policies through residual correction. OmniTacTune uses a two-stage design: it first bootstraps tactile-aware learning from autonomous base-policy rollouts, then learns a lightweight tactile residual policy through online interaction. Extensive experiments show that OmniTacTune generalizes across diverse contact-rich tasks, visual base policies, and tactile representations. Across four real-world contact-rich tasks, it improves visual base policies from 5-40% success to 85-100% within 40-80 minutes, demonstrating an efficient path for adapting tactile feedback to scalable visual robot policies. Project page: https://colinyu1.github.io/omnitactune-site/", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2607.03723", "fig": "figures/omnitactune.webp", "figs": ["figures/omnitactune-1.webp"]}, {"id": "omnivta", "title": "OmniVTA: Visuo-Tactile World Modeling for Contact-Rich Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Yuhang Zheng", "Songen Gu", "Yupeng Zheng", "Weize Li", "Yujie Zang", "Shuai Tian", "Xiang Li", "Ce Hao", "Chen Gao", "Si Liu", "Haoran Li", "Yilun Chen", "Shuicheng Yan", "Wenchao Ding"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-19", "topics": ["visuo-tactile world modelling"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["deform360", "diamond", "dream-tac", "tacforesight", "tactile-wam", "vt-wam"], "arxiv": "2603.19201", "doi": "", "code": "", "site": "", "note": "The earliest of this 2026 cluster. Useful as the baseline the later tactile-WAM papers compare against.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Contact-rich manipulation tasks, such as wiping and assembly, require accurate perception of contact forces, friction changes, and state transitions that cannot be reliably inferred from vision alone. Despite growing interest in visuo-tactile manipulation, progress is constrained by two persistent limitations: existing datasets are small in scale and narrow in task coverage, and current methods treat tactile signals as passive observations rather than using them to model contact dynamics or enable closed-loop control explicitly. In this paper, we present \\textbf{OmniViTac}, a large-scale visuo-tactile-action dataset comprising $21{,}000+$ trajectories across $86$ tasks and $100+$ objects, organized into six physics-grounded interaction patterns. Building on this dataset, we propose \\textbf{OmniVTA}, a world-model-based visuo-tactile manipulation framework that integrates four tightly coupled modules: a self-supervised tactile encoder, a two-stream visuo-tactile world model for predicting short-horizon contact evolution, a contact-aware fusion policy for action generation, and a 60Hz reflexive controller that corrects deviations between predicted and observed tactile signals in a closed loop. Real-robot experiments across all six interaction categories show that OmniVTA outperforms existing methods and generalizes well to unseen objects and geometric configurations, confirming the value of combining predictive contact modeling with high-frequency tactile feedback for contact-rich manipulation. All data, models, and code will be made publicly available on the project website at https://mrsecant.github.io/OmniVTA.", "source": "curated", "first_author": "Yuhang Zheng", "url": "https://arxiv.org/abs/2603.19201", "fig": "figures/omnivta.webp", "figs": ["figures/omnivta-1.webp", "figures/omnivta-2.webp"]}, {"id": "one-demo", "title": "One Demo is Worth a Thousand Trajectories: Action-View Augmentation for Visuomotor Policies", "tree": "il", "branch": "in-context", "authors": ["Chuer Pan", "Litian Liang", "Dominik Bauer", "Eric Cousineau", "Benjamin Burchfiel", "Siyuan Feng", "Shuran Song"], "corresponding": [], "institutions": ["Stanford University", "Columbia University"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2026, "date": "2026-06-17", "topics": ["data augmentation", "single demonstration"], "paradigm": ["IL"], "method": [], "projects": [], "related": ["demohlm", "dp", "egodemogen", "humanego", "icrt", "umi"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. Augmenting the viewpoint and action of one demo to synthesize many — the cheap alternative to collecting more.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/130_One_Demo_is_Worth_a_Thousa.pdf", "abstract": "", "source": "curated", "first_author": "Chuer Pan", "url": "https://arxiv.org/abs/2606.19586", "fig": "figures/one-demo.webp", "figs": ["figures/one-demo-1.webp", "figures/one-demo-2.webp", "figures/one-demo-3.webp"]}, {"id": "open-aoe", "title": "Open-AoE: An Open Egocentric Manipulation Dataset and Toolchain for Embodied Learning", "tree": "il", "branch": "data-scaling", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-14", "topics": ["egocentric dataset", "open toolchain"], "paradigm": ["dataset"], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": ["ego4d", "egoexo4d", "egokit", "human-video-survey", "umi"], "arxiv": "2607.14183", "doi": "", "code": "", "site": "", "note": "From your M2 library. An open egocentric manipulation dataset *with* the capture toolchain — directly usable for the ego-dexterity and UMI-WAM ideas.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2607.14183v2_Open_AoE_An_Open_Egocentric_Manipulation_Dataset_and_Toolcha.pdf", "abstract": "Egocentric videos of human manipulation provide scalable supervision for embodied intelligence, yet existing resources rarely combine low-cost continuous capture, manipulation-level structured annotations, and reusable tools for robot learning. We present Open-AoE, an open, community-oriented egocentric manipulation dataset and toolchain spanning the full pipeline from smartphone capture to model training. Its first release contains approximately 2,000 hours of manipulation video collected in natural environments by 500+ contributors using 400+ smartphones. The dataset provides text annotations, MANO-based hand poses, camera trajectories, and temporally localized atomic actions. Open-AoE further includes a data processing pipeline that transforms raw recordings into structured samples through temporal action segmentation, semantic annotation, hand reconstruction, and camera trajectory reconstruction. Meanwhile, we provide a separate downstream toolchain supports visualization, cross-embodiment retargeting, model-specific data conversion, and training recipes for VLA policies, WAMs, and World Models. By integrating scalable capture, structured processing, and downstream adaptation, Open-AoE reduces the barriers to both data contribution and reuse, providing practical open infrastructure for embodied model training, human-to-robot transfer, and world modeling.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2607.14183", "fig": "figures/open-aoe.webp", "figs": ["figures/open-aoe-1.webp", "figures/open-aoe-2.webp"]}, {"id": "openpyro", "title": "OpenPyRo-A1: An Open Python-Based Low-Cost Bimanual Robot for Embodied AI", "tree": "systems", "branch": "bimanual", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2026, "date": "2026-01-01", "topics": ["open bimanual hardware", "Python stack"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["aloha", "so100"], "arxiv": "", "doi": "10.1109/lra.2025.3634886", "code": "", "site": "", "note": "In your MABEL hardware folder. Recent RA-L open bimanual platform — useful as a design and BOM comparison point for MABEL.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "(see PDF)", "url": "https://doi.org/10.1109/lra.2025.3634886"}, {"id": "prism", "title": "PRISM: Personalized Robotic Dataset Generation via Image-based Scene and Motion Synthesis", "tree": "systems", "branch": "sim-bench", "authors": ["Dogyu Ko", "H. Y. Kim", "Chanyoung Yeo", "Dowoon Lee", "Taeho Park", "Hyoseok Hwang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-06", "topics": ["personalized dataset generation", "scene and motion synthesis"], "paradigm": ["simulator", "IL"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "dreamgen", "egodemogen", "gigaworld-0", "learn-any-images", "physworld", "rigvid", "robodream", "robosnap", "simfoundry"], "arxiv": "2607.04880", "doi": "10.48550/arxiv.2607.04880", "code": "", "site": "", "note": "Generates robot datasets by synthesising both the scene and the motion from images. The natural pairing with cousin generation: once the agent can vary the scene, it should vary the demonstration too.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent advances in large-scale pretrained vision-language-action models have improved robot policy learning, but directly deploying such policies in user-specific environments remains challenging due to limited generalization, which inevitably requires collecting a dataset tailored to the target environment. Teleoperation yields well-aligned data but is costly and difficult to scale, whereas simulation scales easily but struggles to resemble the target environment and generate task-specific trajectories. To meet both simultaneously, we propose PRISM, an end-to-end pipeline that generates personalized robotic datasets from a single image and a natural-language instruction. PRISM constructs digital cousin scenes that are semantically and geometrically aligned with the user environment yet diverse at the instance level, and synthesizes executable demonstrations without human teleoperation. Extensive experiments show that policies trained on PRISM-generated datasets outperform those trained on baseline-generated datasets on LIBERO and LIBERO-Plus, achieve up to 100\\% success rate on three real-world manipulation tasks, and maintain stronger performance when evaluated in environments that differ from those seen during training.", "source": "curated", "first_author": "Dogyu Ko", "url": "https://arxiv.org/abs/2607.04880", "fig": "figures/prism.webp", "figs": ["figures/prism-1.webp"]}, {"id": "pointworld", "title": "PointWorld: Scaling 3D World Models for In-The-Wild Robotic Manipulation", "tree": "il", "branch": "world-models", "authors": ["Wenlong Huang", "Yu-Wei Chao", "Arsalan Mousavian", "Ming-Yu Liu", "Dieter Fox", "Kaichun Mo", "Li Fei-Fei"], "corresponding": [], "institutions": ["NVIDIA", "Stanford University"], "lab": "Stanford Vision & Learning (SVL)", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-07", "topics": ["3D point-based world model", "in-the-wild manipulation", "scaling"], "paradigm": ["world model"], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": ["d3fields", "dino-wm", "lawm-3d", "particle-grid-dynamics"], "arxiv": "2601.03782", "doi": "10.48550/arxiv.2601.03782", "code": "", "site": "", "note": "<b>PointWorld</b> — scaling <b>3D</b> world models rather than 2D video ones for in-the-wild manipulation. The structural counter-current to the video-diffusion WAMs, and closer in spirit to the RoboPIL line: predict in a geometric representation, because that is where manipulation actually happens.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humans anticipate, from a glance and a contemplated action of their bodies, how the 3D world will respond, a capability that is equally vital for robotic manipulation. We introduce PointWorld, a large pre-trained 3D world model that unifies state and action in a shared 3D space as 3D point flows: given one or few RGB-D images and a sequence of low-level robot action commands, PointWorld forecasts per-pixel displacements in 3D that respond to the given actions. By representing actions as 3D point flows instead of embodiment-specific action spaces (e.g., joint positions), this formulation directly conditions on physical geometries of robots while seamlessly integrating learning across embodiments. To train our 3D world model, we curate a large-scale dataset spanning real and simulated robotic manipulation in open-world environments, enabled by recent advances in 3D vision and simulated environments, totaling about 2M trajectories and 500 hours across a single-arm Franka and a bimanual humanoid. Through rigorous, large-scale empirical studies of backbones, action representations, learning objectives, partial observability, data mixtures, domain transfers, and scaling, we distill design principles for large-scale 3D world modeling. With a real-time (0.1s) inference speed, PointWorld can be efficiently integrated in the model-predictive control (MPC) framework for manipulation. We demonstrate that a single pre-trained checkpoint enables a real-world Franka robot to perform rigid-body pushing, deformable and articulated object manipulation, and tool use, without requiring any demonstrations or post-training and all from a single image captured in-the-wild. Project website at https://point-world.github.io/.", "source": "curated", "first_author": "Wenlong Huang", "url": "https://arxiv.org/abs/2601.03782", "fig": "figures/pointworld.webp", "figs": ["figures/pointworld-1.webp", "figures/pointworld-2.webp", "figures/pointworld-3.webp"]}, {"id": "r2s-ego", "title": "R2S-EGO: Dual-Proxy Refinement for Sparse-Capture Real-to-Sim", "tree": "rl", "branch": "sim2real", "authors": ["Shuai Fang", "Xin Deng", "Yuchen Kang", "Zhenjiang Li", "Jie Chen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-07", "topics": ["sparse-capture real-to-sim", "reconstruction refinement"], "paradigm": ["real2sim"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "egoengine", "egophys", "polaris", "simfoundry", "video2sim2real"], "arxiv": "2608.06827", "doi": "", "code": "", "site": "", "note": "R2S-EGO attacks the practical failure of ego-capture real-to-sim: you never get enough views. Dual-proxy refinement for sparse capture. This is the engineering problem that will actually eat your time.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Real-to-sim (R2S) depends on scene representations that render observations along robot ego trajectories, yet dense multi-view capture limits per-environment real-image capture-count efficiency, and sparse human capture can leave behavior-scoped robot views under-supported. Camera-controlled synthesis can fill missing views, but its use in R2S requires behavior-admissible queries and capture-anchored structural conditioning. We present R2S-EGO, which couples a simulator-derived robot proxy that represents the behavior-scoped executable query domain with a capture-anchored geometry proxy that supplies scene-specific structural conditions. Within this domain, fixed- budget selection targets current support deficits for which geometry support is available. The generated observations are assimilated as pseudo-observations to refine the visual asset, while real captures remain anchors. The fused geometry proxy also supplies the scene collision surface, which is refreshed between rounds. Together, these updates refine the existing simulation scene while its robot dynamics and control stack stay fixed. Across 48 frozen Unitree G1 ego views in three Replica scenes, six-view R2S-EGO reaches 19.062 dB PSNR, compared with 14.226 dB for the strongest reported R2S baseline. Across five paired policy-training seeds, R2S-EGO achieves 82.5% +/- 6.8% real-G1 sitting success, compared with 10.0% +/- 10.5% for GaussGym.", "source": "curated", "first_author": "Shuai Fang", "url": "https://arxiv.org/abs/2608.06827", "fig": "figures/r2s-ego.webp", "figs": ["figures/r2s-ego-1.webp"]}, {"id": "rf-agent", "title": "RF-Agent: Automated Reward Function Design via Language Agent Tree Search", "tree": "systems", "branch": "agentic", "authors": ["Ning Gao", "Xiuhui Zhang", "Xingyu Jiang", "Mukang You", "Mohan Zhang", "Yue Deng"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-27", "topics": ["automated reward design", "language agent tree search"], "paradigm": ["LLM", "RL"], "method": [], "projects": ["geodex"], "related": ["dreureka", "eureka", "harbor", "robomorph"], "arxiv": "2602.23876", "doi": "", "code": "", "site": "", "note": "RF-Agent replaces Eureka's evolutionary loop with language agent tree search over reward functions. The successor you should benchmark against if you go the automated-reward route.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Designing efficient reward functions for low-level control tasks is a challenging problem. Recent research aims to reduce reliance on expert experience by using Large Language Models (LLMs) with task information to generate dense reward functions. These methods typically rely on training results as feedback, iteratively generating new reward functions with greedy or evolutionary algorithms. However, they suffer from poor utilization of historical feedback and inefficient search, resulting in limited improvements in complex control tasks. To address this challenge, we propose RF-Agent, a framework that treats LLMs as language agents and frames reward function design as a sequential decision-making process, enhancing optimization through better contextual reasoning. RF-Agent integrates Monte Carlo Tree Search (MCTS) to manage the reward design and optimization process, leveraging the multi-stage contextual reasoning ability of LLMs. This approach better utilizes historical information and improves search efficiency to identify promising reward functions. Outstanding experimental results in 17 diverse low-level control tasks demonstrate the effectiveness of our method. The source code is available at https://github.com/deng-ai-lab/RF-Agent.", "source": "curated", "first_author": "Ning Gao", "url": "https://arxiv.org/abs/2602.23876", "fig": "figures/rf-agent.webp", "figs": ["figures/rf-agent-1.webp"]}, {"id": "robocurate", "title": "RoboCurate: Harnessing Diversity with Action-Verified Neural Trajectory for Robot Learning", "tree": "il", "branch": "world-models", "authors": ["Seungku Kim", "Suhyeok Jang", "Byungjun Yoon", "Dongyoung Kim", "John Won", "Jinwoo Shin"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-21", "topics": ["action-verified neural trajectories", "data curation"], "paradigm": ["world model", "IL"], "method": [], "projects": ["wam-tactile"], "related": ["dreamgen", "genie", "pld"], "arxiv": "2602.18742", "doi": "", "code": "", "site": "", "note": "RoboCurate is the necessary follow-up to DreamGen: generated trajectories are only useful if they are physically executable, so verify the actions before training on them. If you build a world-model data engine, this is the filtering step you will need.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Synthetic data generated by video generative models has shown promise for robot learning as a scalable pipeline, but it often suffers from inconsistent action quality due to imperfectly generated videos. Recently, vision-language models (VLMs) have been leveraged to validate video quality, but they have limitations in distinguishing physically accurate videos and, even then, cannot directly evaluate the generated actions themselves. To tackle this issue, we introduce RoboCurate, a novel synthetic robot data generation framework that evaluates and filters the quality of annotated actions by comparing them with simulation replay. Specifically, RoboCurate replays the predicted actions in a simulator and assesses action quality by measuring the consistency of motion between the simulator rollout and the generated video. In addition, we unlock observation diversity beyond the available dataset via image-to-image editing and apply action-preserving video-to-video transfer to further augment appearance. We observe RoboCurate's generated data yield substantial relative improvements in success rates compared to using real data only, achieving +70.1% on GR-1 Tabletop (300 demos), +16.1% on DexMimicGen in the pre-training setup, and +179.9% in the challenging real-world ALLEX humanoid dexterous manipulation setting.", "source": "curated", "first_author": "Seungku Kim", "url": "https://arxiv.org/abs/2602.18742", "fig": "figures/robocurate.webp", "figs": ["figures/robocurate-1.webp", "figures/robocurate-2.webp", "figures/robocurate-3.webp"]}, {"id": "robodream", "title": "RoboDream: Compositional World Models for Scalable Robot Data Synthesis", "tree": "il", "branch": "world-models", "authors": ["Junjie Ye", "Rong Xue", "Basile Van Hoorick", "Runhao Li", "Harshitha Rajaprakash", "Pavel Tokmakov", "Muhammad Zubair Irshad", "Vitor Guizilini", "Yue Wang"], "corresponding": [], "institutions": ["Toyota Research Institute"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-01", "topics": ["compositional world models", "scalable data synthesis", "scene and skill recombination"], "paradigm": ["world model"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "dreamgen", "gigaworld-0", "prism", "rigvid", "simfoundry"], "arxiv": "2606.02577", "doi": "10.48550/arxiv.2606.02577", "code": "", "site": "", "note": "<b>RoboDream</b> — compositional world models used as a <em>data engine</em>: rather than generating one long rollout, compose scenes and skills so the synthetic distribution covers combinations no demonstration contained. This is the mechanism your agentic scene-construction idea needs, stated in world-model terms rather than simulator terms.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Scaling robot learning requires large-scale, diverse demonstrations, yet real-world data collection via teleoperation remains prohibitively expensive and time-consuming. While video diffusion models offer a promising avenue for data scaling, existing generative approaches are often limited to superficial visual augmentation, or suffer from embodiment hallucinations that yield physically infeasible motions. We present a generalizable embodiment-centric world model that achieves scalable data generation by synthesizing photorealistic demonstrations with novel objects, in novel scenes, and from novel viewpoints. Our approach anchors generation to rendered robot motion while conditioning on explicit scene and object priors, effectively decoupling trajectory execution from environment synthesis. This formulation has the potential to unlock two powerful data scaling capabilities: (1) retrieval and rebirth, which repurposes existing trajectories into entirely new contexts without new motion data; and (2) prop-free teleoperation, where operators manipulate empty air and the model hallucinates the target objects and scene afterwards, eliminating reset time. We demonstrate with real-world experiments that our generated data consistently improves downstream policy performance and significantly reduces real-world data requirements across diverse manipulation tasks.", "source": "curated", "first_author": "Junjie Ye", "url": "https://arxiv.org/abs/2606.02577", "fig": "figures/robodream.webp", "figs": ["figures/robodream-1.webp", "figures/robodream-2.webp", "figures/robodream-3.webp"]}, {"id": "robogaze", "title": "RoboGaze: Evaluating Robot World Models via Structured Vision-Language Analysis", "tree": "systems", "branch": "sim-bench", "authors": ["Minh-Loi Nguyen", "Nghiem Tuong Diep", "Hung Khang Nguyen", "Minh Le", "Doanh Le Thien", "Hoang H. Tran", "Dung D. Le", "Vu N. Duong", "Daniel Sonntag", "An Thai Le", "Duy Minh Ho Nguyen", "Vien Anh Ngo", "Tran Van Nhiem"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-22", "topics": ["evaluating world models via structured vision-language analysis"], "paradigm": ["benchmark"], "method": [], "projects": ["wam-tactile"], "related": ["ev-wm", "roboworld", "worldarena2", "worldeval"], "arxiv": "2606.28385", "doi": "10.48550/arxiv.2606.28385", "code": "", "site": "", "note": "<b>RoboGaze</b> — evaluate a robot world model by having a VLM interrogate its rollouts in a structured way, instead of scoring pixels with FVD. The right instinct: a world model should be judged on whether the <em>events</em> it predicts are right, not on whether the texture is.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent advances in robot world models enable synthetic video generation for embodied prediction and planning. However, evaluating these videos is challenging: visually realistic outputs often violate physical laws, temporal consistency, or task logic, while conventional metrics and monolithic Vision-Language Model (VLM) judges fail to generalize or provide precise diagnostic value. We present RoboGaze, a training-free, multi-agent VLM framework that provides structured, interpretable evaluation for generated robot-manipulation videos. Given a task instruction and video, RoboGaze operates via a three-stage pipeline: task-scene grounding, dimension-specific specialist routing, and critic-based verification. It outputs temporally localized glitch reports categorized under a novel 6-dimension, 30-type robotics-specific taxonomy. To benchmark RoboGaze, we introduce a human-validated dataset of 382 clips spanning simulated and real-world multi-view manipulation. Evaluating eight open-source and proprietary VLM backbones, RoboGaze dramatically outperforms zero-shot baselines, improving description-F1 by up to +43 points and temporal alignment (F1 x IoU) by up to +37 points, closing approximately 85% of the gap to the human ceiling. Furthermore, its critic verifier mitigates the \"cry-wolf\" false-positive flaw of standard VLMs, lifting clean-clip accuracy from under 25% to over 80%. RoboGaze offers a scalable, highly interpretable diagnostic tool for the rigorous evaluation of robot world models.", "source": "curated", "first_author": "Minh-Loi Nguyen", "url": "https://arxiv.org/abs/2606.28385", "fig": "figures/robogaze.webp", "figs": ["figures/robogaze-1.webp"]}, {"id": "robosnap", "title": "RoboSnap: One-Shot Real-to-Sim Scene Generation for Generalizable Robot Learning and Evaluation", "tree": "systems", "branch": "sim-bench", "authors": ["Shujie Zhang", "Jingkun Yi", "Weipeng Zhong", "Zirui Zhou", "Yangkun Zhu", "Hanqing Wang", "X X Xu", "Weinan Zhang", "Chunhua Shen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-07", "topics": ["one-shot real-to-sim from a single image"], "paradigm": ["simulator"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "learn-any-images", "physgen3d", "prism", "simfoundry"], "arxiv": "2607.06699", "doi": "10.48550/arxiv.2607.06699", "code": "", "site": "", "note": "A single RGB image to a simulation-ready scene. The cheapest possible entry to the reconstruction leg — worth trying before committing to multi-view egocentric capture, if only to see how far one photo goes.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recovering real-world scenes as interactive simulation environments can enable generalizable robot learning and reproducible policy evaluation. However, constructing scenes that are both physically stable and visually faithful remains slow and expensive. In this work, we present RoboSnap, a real-to-sim framework that turns a single RGB image into a simulation-ready scene. The key idea is a layered design that separates the physics-critical interaction area from the surrounding visual context: collision-aware foreground assets are refined for stable robot interaction, while a 3D Gaussian splatting visual layer preserves faithful background appearance under novel views. Experiments on DROID scenes and real-robot tasks show that RoboSnap achieves reliable trajectory replay in the recovered scenes, supports task-specific synthetic data generation for policy training, and yields meaningful sim-real correlation for policy evaluation. To further support real-to-sim research, we introduce DROID-Sim, a real-to-sim companion dataset constructed from 564 real-world scenes in DROID. Extensive experiments suggest that the value of real-to-sim methods lies not only in high-fidelity visual reconstruction, but in turning real environments into reusable infrastructure for robot learning and evaluation.", "source": "curated", "first_author": "Shujie Zhang", "url": "https://arxiv.org/abs/2607.06699", "fig": "figures/robosnap.webp", "figs": ["figures/robosnap-1.webp"]}, {"id": "robotacdex", "title": "RoboTacDex: A Dexterous Visual-Tactile-Action Dataset for Humanoid Manipulation", "tree": "systems", "branch": "tactile-hw", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Unitree Robotics"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-31", "topics": ["visual-tactile dataset", "humanoid manipulation"], "paradigm": ["dataset"], "method": [], "projects": ["geodex", "m2", "wam-tactile"], "related": ["anyskin", "dexskin"], "arxiv": "2606.31836", "doi": "", "code": "", "site": "", "note": "From your M2 library. A paired visual-tactile dataset is precisely what a tactile world model needs for pretraining.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2606.31836v1_RoboTacDex_A_Dexterous_Visual_Tactile_Action_Dataset_for_Hum.pdf", "abstract": "In the field of robot learning, large-scale and diverse demonstration trajectories provide the fundamental basis for enhancing robotic manipulation ability. We introduce RoboTacDex, a large, multi-modal, and diverse dataset of dexterous manipulation behaviors performed with a humanoid robot. Built on the publicly accessible humanoid robot Unitree G1, RoboTacDex consists of 6k trajectories covering 19 tasks, 23 skills, and interactions with 22 objects. RoboTacDex provides comprehensive records including multi-view RGB and depth information, tactile feedback, and detailed semantic annotations. Furthermore, the dataset features a variety of relatively challenging tasks that can only be completed by dual arms and dexterous hands, aiming to mimic human-like operational logic and simulate real-world manipulation complexity. To ensure data collection quality, we develop an improved multi-camera synchronization system to enable millisecond data synchronization and recording of modalities. In our experiments, we evaluate three representative imitation learning models on our dataset, analyzing their performance as well as their respective strengths and limitations across different task categories. Successful trial results and a moderate level of generalization capabilities across a suite of tasks indicate the effectiveness and diversity of the collected dataset. Our dataset will be open-sourced soon.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2606.31836", "fig": "figures/robotacdex.webp", "figs": ["figures/robotacdex-1.webp", "figures/robotacdex-2.webp", "figures/robotacdex-3.webp"]}, {"id": "beyond-vla-wm", "title": "Robots Need More than VLA and World Models", "tree": "il", "branch": "world-models", "authors": ["Elis Karcini", "Faisal Mehrban", "Quang Nguyen", "Mac Schwager", "Arash Ajoudani", "Cesar Cadena", "Jan Peters", "Marco Hutter", "Haitham Bou-Ammar"], "corresponding": [], "institutions": ["Stanford University", "ETH Zurich"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-04", "topics": ["position paper", "limits of VLA and world-model paradigms"], "paradigm": ["position"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["causal-wm", "wam-tutorial", "wm-survey-robot"], "arxiv": "2606.06556", "doi": "10.48550/arxiv.2606.06556", "code": "", "site": "", "note": "<b>Robots Need More than VLA and World Models</b> — the dissenting position, and worth keeping in the library precisely because everything else in this section agrees with itself. Read it as a checklist of the objections your own WAM introduction will have to answer.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Generalist robot intelligence is often framed as a policy-scaling problem: collect more robot demonstrations, train larger Vision-Language-Action (VLA) models, and expect broader generalisation. In this position paper, we argue that this framing is incomplete. The central bottleneck is not only policy learning, but the absence of mechanisms that convert the world's abundant unstructured behavioural data into grounded robot supervision. Human motion, internet video, simulation rollouts, and interactive demonstrations contain rich information about tasks, goals, contacts, failures, and physical constraints, yet most of this information is not directly usable by robot policies because it lacks embodiment-specific action labels, task semantics, and reward structure. We identify four missing components for the next generation of robotics: data interfaces for autolabelling unstructured behaviour, embodiment interfaces for retargeting human motion to robot actions, world-model interfaces for physics-grounded 3D reasoning, and reward interfaces for inferring task progress and success from video and language. We survey recent progress in robot foundation models, cross-embodiment datasets, learning from video, world models, and reward modelling, and propose a research agenda for building robotics systems that can learn not only from robot demonstrations, but from the broader physical world.", "source": "curated", "first_author": "Elis Karcini", "url": "https://arxiv.org/abs/2606.06556", "fig": "figures/beyond-vla-wm.webp"}, {"id": "cross-humanoid-wbc", "title": "Scalable and General Whole-Body Control for Cross-Humanoid Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Yufei Xue", "Yunfeng Lin", "Wentao Dong", "Yang Tang", "Jingbo Wang", "Jiangmiao Pang", "Ming Zhou", "Minghuan Liu", "Weinan Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-05", "topics": ["cross-embodiment whole-body control", "scalability"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["chorus", "crossformer", "exbody2", "gcnt", "getup-morphologies", "hugwbc"], "arxiv": "2602.05791", "doi": "10.48550/arxiv.2602.05791", "code": "", "site": "", "note": "Whole-body control that scales across different humanoid bodies rather than being retrained per robot. Relevant if MABEL's morphology keeps changing — which, during a hardware programme, it will.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Learning-based whole-body controllers have become a key driver for humanoid robots, yet most existing approaches require robot-specific training. In this paper, we study the problem of cross-embodiment humanoid control and show that a single policy can robustly generalize across a wide range of humanoid robot designs with one-time training. We introduce XHugWBC, a novel cross-embodiment training framework that enables generalist humanoid control through: (1) physics-consistent morphological randomization, (2) semantically aligned observation and action spaces across diverse humanoid robots, and (3) effective policy architectures modeling morphological and dynamical properties. XHugWBC is not tied to any specific robot. Instead, it internalizes a broad distribution of morphological and dynamical characteristics during training. By learning motion priors from diverse randomized embodiments, the policy acquires a strong structural bias that supports zero-shot transfer to previously unseen robots. Experiments on twelve simulated humanoids and seven real-world robots demonstrate the strong generalization and robustness of the resulting universal controller.", "source": "curated", "first_author": "Yufei Xue", "url": "https://arxiv.org/abs/2602.05791", "fig": "figures/cross-humanoid-wbc.webp", "figs": ["figures/cross-humanoid-wbc-1.webp"]}, {"id": "simdex", "title": "SiMDex: Mining Similar Egocentric Videos for Cross-Embodiment Dexterous Manipulation", "tree": "il", "branch": "human-video", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Generalist AI", "Tsinghua University", "Shanghai Jiao Tong University", "HKU", "University of Tokyo", "ByteDance Seed"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-04", "topics": ["egocentric retrieval", "cross-embodiment dexterity"], "paradigm": ["IL"], "method": [], "projects": ["m2", "r2s2r"], "related": ["do-as-i-do", "ego2robot", "egomimic", "human-video-survey"], "arxiv": "2608.04196", "doi": "", "code": "", "site": "", "note": "From your M2 library. Retrieval over egocentric video as a way to get task-relevant data instead of training on everything — a cheap lever for the ego-dexterity idea.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.04196v1_SiMDex_Mining_Similar_Egocentric_Videos_for_Cross_Embodiment.pdf", "abstract": "Recent years have witnessed an explosive trend of scaling ego-centric human videos for robot manipulation, yet it remains unclear which data actually benefits dexterous manipulation. We present SiMDex, a similarity-based data mining framework that casts human data selection for VLA post-training in dexterous manipulation as a recommendation problem. For each robot demonstration, SiMDex employs a three-layer recall-ranking-re-ranking pipeline to extract task-relevant subsets from a pool of ~32M egocentric human samples, operating in a morphology-agnostic action space that requires no changes to VLA architecture or training. Against a strong baseline trained with an equal amount of randomly sampled human data, SiMDex uses only ~1.49M mined samples (<5% of the pool) yet improves the overall success rate from 47.7% to 61.1%, showing that selective curation outperforms indiscriminate data mixing.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2608.04196", "fig": "figures/simdex.webp", "figs": ["figures/simdex-1.webp"]}, {"id": "simweaver", "title": "SimWeaver: Zero-Shot RGB Sim-to-Real for Deformable Manipulation", "tree": "rl", "branch": "sim2real", "authors": ["Wenkang Hu", "Haoran Wang", "Yitong Li", "Liu Liu", "Mengao Zhao", "Lai Jiang", "Xincheng Tang", "Junhang Wei", "Zhengjie Shu", "Zhendong Wang", "Zhizhong Su", "Huamin Wang", "Ruigang Yang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-13", "topics": ["deformable manipulation", "zero-shot RGB transfer"], "paradigm": ["sim2real", "RL"], "method": [], "projects": [], "related": ["dexsim2real", "dist-r2s2r", "genesis", "real2sim", "soft-gentle"], "arxiv": "2606.15338", "doi": "", "code": "", "site": "", "note": "SimWeaver claims zero-shot RGB sim-to-real for deformables, which is the harder half of the problem (rendering gap) rather than the physics half. Note the honest framing in this literature: pixel-based deformable policy learning has been stuck precisely on this gap.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "RGB sim-to-real for deformable manipulation has remained largely unsolved without real-world fine-tuning. We present SimWeaver, which trains zero-shot RGB VLA policies on 200 simulated demonstrations per task, reaching above 80% per-task and 91% average real-world success across 5 diverse deformable tasks including plastic-bag manipulation, without teleoperation or per-task calibration. SimWeaver combines a reliable measurement-backed simulator (SimWeaver-Sim) with an extensible asset framework supporting single-image generation(SimWeaver-Asset), a deterministic topology-aware trajectory synthesizer (SimWeaver-Syn), and a sim-to-real protocol with ISP-aware photometric augmentation (SimWeaver-Real). On silk grasping, the sim-trained policy reaches 100% under visual distribution shifts where real-data baselines drop to 9-70%, at two orders of magnitude lower per-trajectory cost. We will release SimWeaver and a representative asset subset. Project page: https://simweaver.github.io/", "source": "curated", "first_author": "Wenkang Hu", "url": "https://arxiv.org/abs/2606.15338", "fig": "figures/simweaver.webp", "figs": ["figures/simweaver-1.webp"]}, {"id": "skild-scale", "title": "Skild AI: scaling to 100,000 hours and one-shot video imitation", "tree": "systems", "branch": "agentic", "authors": ["Skild AI"], "corresponding": [], "institutions": ["Skild AI"], "lab": "Skild AI (CMU spin-off)", "venue": "company blog", "venue_type": "industry", "year": 2026, "date": "", "topics": ["cross-embodiment scaling", "one-shot imitation from video"], "paradigm": ["VLA"], "method": [], "projects": ["r2s2r"], "related": ["generalist-ai", "icrt", "skild", "vid2robot"], "arxiv": "", "doi": "", "code": "", "site": "https://www.skild.ai/", "note": "Reports that scaling from 1,000 to 100,000 hours produced a model that can watch a single video of an unseen task and attempt it, with ~66% success on unseen tasks. Unaudited, but the one-shot video-conditioned framing is exactly the in-context-learning direction you are interested in.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Skild AI", "url": "https://www.skild.ai/"}, {"id": "stablemimic", "title": "StableMimic: Smooth Human-Like Recovery for Humanoid Motion Tracking - Learning Beyond the Tracking Distribution for Structured Post-Fall Behavior", "tree": "rl", "branch": "motion-imitation", "authors": ["Weihao Wu", "Ming Huang", "Ruofei Liu", "Jinglei Nie", "Shuxiang Guo", "Chunying Li"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-03", "topics": ["post-fall behaviour", "recovery beyond the tracking distribution"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["deepmimic", "exbody2", "fall-safety", "humanup", "kungfu-bot"], "arxiv": "2608.02385", "doi": "10.48550/arxiv.2608.02385", "code": "", "site": "", "note": "Structured post-fall behaviour for motion-tracking policies — what happens when the robot leaves the distribution the tracker was trained on. The honest complement to every DeepMimic-descended controller in this library.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humanoid motion trackers perform reliably within learned tracking distributions, but falls can move the robot into low-height, contact-rich states from which an advancing command is temporarily unreachable. Tracking-only policies may chase infeasible references, producing rapid, large-amplitude limb corrections that increase risk to the robot and its surroundings. We present StableMimic, a unified tracker trained beyond the nominal tracking distribution. Perturbed resets around multiple human get-up references expose prone, supine, off-balance, and intermediate ground-contact states, shaping structured recovery that returns the robot to the trackable region. Because tracking and recovery occupy markedly different state--action distributions, StableMimic uses dedicated experts for each regime and a proprioceptive gate that continuously blends their actions. A hidden successor-state objective teaches human-reference-shaped recovery without exposing reference identity or phase to the deployed Actor; deployment requires no get-up reference, recovery command, trajectory retrieval, or external policy switch. On the complete retargeted LAFAN1 dance subset, StableMimic achieves the lowest errors on all four tracking metrics among five methods. Across 100 matched push-to-fall trials per method, it recovers in 100/100 and attains the lowest values on six of seven post-fall motion and load measures, supporting improved interaction safety under this protocol. Real Unitree G1 dance and standing-reference deployments qualitatively demonstrate bounded limb motion, autonomous recovery, and command resumption.", "source": "curated", "first_author": "Weihao Wu", "url": "https://arxiv.org/abs/2608.02385", "fig": "figures/stablemimic.webp", "figs": ["figures/stablemimic-1.webp", "figures/stablemimic-2.webp", "figures/stablemimic-3.webp"]}, {"id": "tacforesight", "title": "TacForeSight: Force-Guided Tactile World Model for Contact-Rich Manipulation", "tree": "il", "branch": "world-models", "authors": ["Yujie Zang", "Yuhang Zheng", "Xian Nie", "Yupeng Zheng", "Shuai Tian", "Songen Gu", "Chen Gao", "Zining Wang", "Shuicheng Yan", "Wenchao Ding"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-09", "topics": ["force-guided prediction", "tactile world model"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["diamond", "dream-tac", "ev-wm", "omnivta", "rotate-without-seeing"], "arxiv": "2606.11184", "doi": "", "code": "", "site": "", "note": "Force-guided rather than image-guided: the prediction target is the contact wrench. Closest to the 'predict slip before it is visible' framing you want.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Contact-rich manipulation requires robots to continuously perceive and regulate evolving physical interactions under dynamic contact transitions or complex surface geometries. Recent imitation learning methods improve contact-aware control by incorporating tactile or force feedback, but they rarely model the asymmetric spatiotemporal roles of global force and local tactile sensing. To address this, we propose TacForeSight, a lightweight force-conditioned tactile foresight framework for real-time manipulation. The core component is TacForceWM, a tactile world model that predicts short-horizon tactile latent dynamics from dual-finger tactile observations conditioned on high-frequency wrist force and torque signals. Another key component, the Predictive Tactile-Conditioned Policy, leverages the predicted latents as anticipatory contact priors, models the current-to-future tactile evolution via cross-attention, and adaptively fuses visuo-tactile features through a tactile-guided gating module. By forecasting purely within a compact latent space, TacForeSight enables proactive contact reasoning with efficient real-time inference suitable for high-frequency manipulation control. Real-robot experiments on five representative tasks and three in-process perturbation settings show that TacForeSight consistently outperforms existing baselines, particularly under dynamic contact disturbances. All models and datasets will be made publicly available on the project website at https://tacforesight.github.io/ProjectPage.", "source": "curated", "first_author": "Yujie Zang", "url": "https://arxiv.org/abs/2606.11184", "fig": "figures/tacforesight.webp", "figs": ["figures/tacforesight-1.webp", "figures/tacforesight-2.webp"]}, {"id": "tactile-wam", "title": "Tactile-WAM: Touch-Aware World Action Model with Tactile Asymmetric Attention", "tree": "il", "branch": "world-models", "authors": ["Siyu Wu", "Linjing You", "Junjie Zhu", "Yaozu Liu", "Huang Kaixiang", "Chen Yonghang", "Jituo Li", "Changhao Zhang", "Jian Liu", "Hengshuo Chu", "Qi Li", "Hengshuang Zhao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-25", "topics": ["tactile attention", "world-action model"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["deform360", "dream-tac", "dreamdojo", "omnivta", "vt-wam"], "arxiv": "2606.26663", "doi": "", "code": "", "site": "", "note": "Asymmetric attention between the visual and tactile streams — the architectural question of how to fuse two modalities with very different rates and dimensionalities, which is the first thing you will hit.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World Action Models (WAMs) jointly predict future visual observations and actions, but visual futures alone often miss slip, jamming, contact-direction changes, and subtle misalign- ment in contact-rich manipulation. Tactile signals reveal these hidden physical states, yet naive tactile-token injection can disrupt visual dynamics modeling due to the limited scale of tactile data, a phenomenon we term tactile pollution. We in- troduce Tactile-WAM, which uses asymmetric attention to block video queries from tactile keys while preserving tac- tile access for action queries. A contact-change-aware bias further strengthens action attention to touch. Because tactile pixel changes do not reliably reflect contact changes, we derive Observed proxy changes drive the attention bias, while future- proxy supervision preserves action-relevant contact dynamics in predicted tactile representations. On ManiFeel, visual-path isolation reduces deviation from the RGB-only trajectory by 21.8% in MSE at the step-matched 20K checkpoint without a statistically detectable change in ground-truth video qual- ity. The full model improves average success from 15.6% to 32.7%, with VideoClean providing the largest gain. On five real-robot tasks, Tactile-WAM achieves 49.2% success.", "source": "curated", "first_author": "Siyu Wu", "url": "https://arxiv.org/abs/2606.26663", "fig": "figures/tactile-wam.webp", "figs": ["figures/tactile-wam-1.webp", "figures/tactile-wam-2.webp"]}, {"id": "unirelo", "title": "UniReLo: Learning a Unified Humanoid Policy from Fall Recovery to Locomotion across Diverse Terrains", "tree": "rl", "branch": "locomotion", "authors": ["Xiaoyu Xu", "Zhiming Chen", "Yuenan Zhao", "Ran Song", "Wei Zhang (405)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-08", "topics": ["unified fall recovery and locomotion", "diverse terrain"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["fall-safety", "getup-morphologies", "hover", "hugwbc", "humanup"], "arxiv": "2606.08922", "doi": "10.48550/arxiv.2606.08922", "code": "", "site": "", "note": "One policy spanning fall recovery <em>and</em> locomotion across terrains, rather than a recovery policy bolted onto a walking policy. The unification argument matters when you are counting how many policies fit on the robot.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Reliable fall recovery, which commonly aims at attaining a nominal upright posture, is essential for the autonomous operation of humanoid robots in unstructured field environments. Although existing posture-centered methods can synthesize coordinated whole-body recovery motions from diverse fallen configurations, they may result in a dynamically fragile support state, leading to secondary loss of balance or unstable resumption of commanded locomotion, particularly under terrain-dependent contact conditions. We propose to learn a unified humanoid policy from fall recovery to locomotion (UniReLo) across heterogeneous field terrains. UniReLo leverages continuously gated multi-scale motion priors to modulate frame-, sequence-, and gait-level adversarial supervision according to recovery progress, preserving the distinct temporal structures of recovery and locomotion without requiring fixed-threshold switching. In addition, terrain-conditioned recovery guidance evaluates the evolving support state using a terrain-relative support representation and support-feasibility assessment. Simulation and outdoor real-world experiments demonstrate that UniReLo can deliver stable and continuous recovery-to-locomotion behaviors for humanoids across diverse field terrains. The supplementary video is available at https://vsislab.github.io/UniReLo/.", "source": "curated", "first_author": "Xiaoyu Xu", "url": "https://arxiv.org/abs/2606.08922", "fig": "figures/unirelo.webp", "figs": ["figures/unirelo-1.webp", "figures/unirelo-2.webp"]}, {"id": "vla-jepa", "title": "VLA-JEPA: Enhancing Vision-Language-Action Model with Latent World Model", "tree": "il", "branch": "world-models", "authors": ["Jingwen Sun", "Wenyao Zhang", "Zekun Qi", "Shaojie Ren", "Zezhi Liu", "Hanxin Zhu", "Guangzhong Sun", "Xin Jin", "Zhibo Chen"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Jiao Tong University", "USTC"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-10", "topics": ["JEPA-style latent world model inside a VLA", "predictive representation"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["dino-wm", "dreamzero", "flare", "lawam", "vjepa2"], "arxiv": "2602.10098", "doi": "", "code": "", "site": "", "note": "<b>VLA-JEPA</b> — bolt a joint-embedding predictive world model onto a VLA so the policy is trained against latent future prediction rather than pixel reconstruction. Read it beside FLARE and LaWAM: three independent 2026 papers converging on 'predict in latent space, never decode', which is now close to settled practice.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Pretraining Vision-Language-Action (VLA) policies on internet-scale video is appealing, yet current latent-action objectives often learn the wrong thing: they remain anchored to pixel variation rather than action-relevant state transitions, making them vulnerable to appearance bias, nuisance motion, and information leakage. We introduce VLA-JEPA, a JEPA-style pretraining framework that sidesteps these pitfalls by design. The key idea is leakage-free state prediction: a target encoder produces latent representations from future frames, while the student pathway sees only the current observation -- future information is used solely as supervision targets, never as input. By predicting in latent space rather than pixel space, VLA-JEPA learns dynamics abstractions that are robust to camera motion and irrelevant background changes. This yields a simple two-stage recipe -- JEPA pretraining followed by action-head fine-tuning -- without the multi-stage complexity of prior latent-action pipelines. Experiments on LIBERO, LIBERO-Plus, SimplerEnv and real-world manipulation tasks show that VLA-JEPA achieves consistent gains in generalization and robustness over existing methods.", "source": "curated", "first_author": "Jingwen Sun", "url": "https://arxiv.org/abs/2602.10098", "fig": "figures/vla-jepa.webp", "figs": ["figures/vla-jepa-1.webp"]}, {"id": "vlm4vla", "title": "VLM4VLA: Revisiting Vision-Language-Models in Vision-Language-Action Models", "tree": "il", "branch": "vla", "authors": ["Jianke Zhang", "Xiaoyu Chen", "Qiuyue Wang", "Mingsheng Li", "Yanjiang Guo", "Yucheng Hu", "Jiajun Zhang", "Shuai Bai", "Junyang Lin", "Jianyu Chen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-01-06", "topics": ["VLM backbone ablation", "VLA design"], "paradigm": ["VLA"], "method": [], "projects": [], "related": ["groot-n1", "openvla", "openvla-oft", "pi0"], "arxiv": "2601.03309", "doi": "", "code": "", "site": "", "note": "VLM4VLA asks the question nobody in the VLA literature answers honestly: how much does the VLM backbone actually matter? Useful as a sanity check before you spend money on a bigger backbone.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models, which integrate pretrained large Vision-Language Models (VLM) into their policy backbone, are gaining significant attention for their promising generalization capabilities. This paper revisits a fundamental yet seldom systematically studied question: how VLM choice and competence translate to downstream VLA policies performance? We introduce VLM4VLA, a minimal adaptation pipeline that converts general-purpose VLMs into VLA policies using only a small set of new learnable parameters for fair and efficient comparison. Despite its simplicity, VLM4VLA proves surprisingly competitive with more sophisticated network designs. Through extensive empirical studies on various downstream tasks across three benchmarks, we find that while VLM initialization offers a consistent benefit over training from scratch, a VLM's general capabilities are poor predictors of its downstream task performance. This challenges common assumptions, indicating that standard VLM competence is necessary but insufficient for effective embodied control. We further investigate the impact of specific embodied capabilities by fine-tuning VLMs on seven auxiliary embodied tasks (e.g., embodied QA, visual pointing, depth estimation). Contrary to intuition, improving a VLM's performance on specific embodied skills does not guarantee better downstream control performance. Finally, modality-level ablations identify the visual module in VLM, rather than the language component, as the primary performance bottleneck. We demonstrate that injecting control-relevant supervision into the vision encoder of the VLM yields consistent gains, even when the encoder remains frozen during downstream fine-tuning. This isolates a persistent domain gap between current VLM pretraining objectives and the requirements of embodied action-planning.", "source": "curated", "first_author": "Jianke Zhang", "url": "https://arxiv.org/abs/2601.03309", "fig": "figures/vlm4vla.webp", "figs": ["figures/vlm4vla-1.webp"]}, {"id": "vt-wam", "title": "VT-WAM: Visual-Tactile World Action Model for Contact-Rich Manipulation", "tree": "il", "branch": "world-models", "authors": ["Shuai Tian", "Yupeng Zheng", "Yuhang Zheng", "Songen Gu", "Yujie Zang", "Yuxing Qin", "Weize Li", "Haoran Li", "Wenchao Ding", "Dongbin Zhao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-02", "topics": ["visual-tactile", "world-action model"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["dream-tac", "omnivta", "tactile-wam"], "arxiv": "2607.02503", "doi": "", "code": "", "site": "", "note": "Visual-tactile WAM for contact-rich manipulation. Same team lineage as OmniVTA — read the two together to see what actually changed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Contact-rich manipulation requires policies to react to local deformation, pressure, slip, and friction, yet these cues are temporally sparse and often invisible in visual observations. Existing visual-tactile policies usually feed tactile observations directly into action prediction, but rarely model tactile deformation dynamics during action generation. In this paper, we introduce VT-WAM, a Visual-Tactile World Action Model that jointly learns future visual prediction, tactile deformation prediction, and action prediction within a unified flow matching framework. In particular, VT-WAM introduces (1) Asymmetric Mixture-of-Transformers (MoT) attention to bridge a first-frame visual anchor with temporal tactile dynamics, and (2) contact-gated Action-Visual-Tactile Attention Guidance (AVTAG) to encourage action queries to rely on tactile evidence during contact phases. Across six real-world contact-rich manipulation tasks, VT-WAM achieves a 71.67% average success rate, outperforming Fast-WAM by 26.67% and OmniVTLA by 35.84%. Ablations demonstrate that modeling tactile deformation dynamics and guiding contact-phase tactile attention are both important for contact-rich tasks. Project website: https://vt-wam.github.io/.", "source": "curated", "first_author": "Shuai Tian", "url": "https://arxiv.org/abs/2607.02503", "fig": "figures/vt-wam.webp", "figs": ["figures/vt-wam-1.webp", "figures/vt-wam-2.webp", "figures/vt-wam-3.webp"]}, {"id": "wt-umi", "title": "WT-UMI: Tactile-based Whole-Body Manipulation via Force-Supervised Contact-Aware Planning", "tree": "systems", "branch": "interfaces", "authors": ["Jaehwi Jang", "Zhaoyuan Gu", "Alfred Cueva", "Zimeng Chai", "Junjie Sheng", "Thong Nguyen", "Himank Galundia", "Yifan Wu", "Huishu Xue", "Isaac Legene", "Ojas Mediratta", "Davin Doan", "Andrew Collins", "Sarah Sadegh", "KyoungMok Kim", "Rishita Dhalbisoi", "Zun Chen", "Ye Zhao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-11", "topics": ["tactile UMI", "whole-body manipulation", "force supervision"], "paradigm": ["hardware", "teleop"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dexumi", "realdexumi", "ume", "umi"], "arxiv": "2606.13232", "doi": "", "code": "", "site": "", "note": "A tactile UMI variant driving whole-body manipulation with force-supervised contact-aware planning. The hardware half of the tactile-WAM project, and directly relevant to a MABEL dexterous UMI.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Whole-body humanoid manipulation of bulky, deformable, and shared-load objects requires distributed contact sensing and explicit force regulation, yet most imitation policies treat contact force only implicitly. On the other hand, different demonstration sources provide complementary modalities with inherent trade-offs: human demonstrations capture natural contact forces but not robot-executable actions, while teleoperation directly records robot actions but with less natural force regulation. This paper presents \\textbf{WT-UMI}, a wearable whole-body tactile interface worn by human operators or mounted on humanoids, providing accurate observations of tactile images, contact forces, and end-effector poses across both human demonstration and humanoid teleoperation modes. We introduce a force-conditioned target-pose correction module that converts measured human poses into contact-aware robot targets by learning corrections from teleoperation data. To leverage the natural force interaction in human data, we propose a force-supervised planner that predicts end-effector pose chunks and contact-force trajectories. The predicted contact force serves as the reference for a tactile-based admittance controller. Across five contact-rich tasks spanning deformable objects, bulky rigid objects, and human--humanoid collaboration, WT-UMI improves success rate and reduces contact-position tracking error over four policy baselines. Our project page is available at https://wt-umi.github.io/WTUMI/.", "source": "curated", "first_author": "Jaehwi Jang", "url": "https://arxiv.org/abs/2606.13232", "fig": "figures/wt-umi.webp", "figs": ["figures/wt-umi-1.webp", "figures/wt-umi-2.webp"]}, {"id": "wm-survey-manip", "title": "World Models for Robotic Manipulation: A Survey", "tree": "il", "branch": "world-models", "authors": ["Fangyuan Wang", "Ziyuan Wang", "Guorui Pei", "Mengshi Zhang", "Canxi Liang", "Jun Hu", "Zhongxuan Li", "Jinsong Wu", "Ning Han", "Zeqing Zhang", "Jiaming Qi", "Hongmin Wu", "Shiyao Zhang", "Pai Zheng", "Jia Pan", "David Navarro-Alarcon", "Sichao Liu", "Peng Zhou"], "corresponding": [], "institutions": ["University of Hong Kong", "KTH", "NTU Singapore"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-27", "topics": ["survey", "world models specifically for manipulation"], "paradigm": ["survey"], "method": [], "projects": ["wam-tactile"], "related": ["wam-tutorial", "wm-survey-robot", "worldarena2"], "arxiv": "2606.00113", "doi": "", "code": "", "site": "", "note": "<b>World Models for Robotic Manipulation: A Survey</b> — narrower than the general survey and therefore more useful for a manipulation project; the two overlap but disagree on taxonomy in instructive ways.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Robotic manipulation depends on the ability to anticipate how actions reshape objects, contacts, and scene geometry before execution. Learned world models provide this capability by predicting task-relevant future evolution under robot intervention, yet the term now spans latent dynamics models, action-conditioned video generators, three- and four-dimensional scene predictors, physics-informed simulators, and predictive modules inside vision-language-action systems. This breadth has fragmented the literature and obscured the design choices that matter for manipulation. We survey world models for robotic manipulation through three questions: what future representation is predicted, how prediction is connected to action, and when prediction is used in the robot-learning pipeline. We operationally define a world model as an action-conditioned predictive system and distinguish it from perception modules, inverse models, policies, rewards, and value functions. We then organize existing work into five representation families, develop a functional taxonomy that separates integrated prediction-action models from explicit predictive planners, and characterize infrastructure roles including synthetic experience generation, candidate filtering, search-based evaluation, learned environments, and outcome verification. We further map these roles across pretraining, post-training, and inference adaptation, review 34 manipulation datasets, and synthesize evaluation protocols for predictive fidelity, task performance, and simulator reliability. This survey shows that world models are evolving from task-specific dynamics predictors into predictive infrastructure for robot learning, while exposing open challenges in contact modeling, hallucination control, action alignment, and benchmarking under closed-loop use.", "source": "curated", "first_author": "Fangyuan Wang", "url": "https://arxiv.org/abs/2606.00113", "fig": "figures/wm-survey-manip.webp", "figs": ["figures/wm-survey-manip-1.webp"]}, {"id": "wla-model", "title": "World-Language-Action Model for Unified World Modeling, Language Reasoning, and Action Synthesis", "tree": "il", "branch": "world-models", "authors": ["Yi Yang", "Zhihong Liu", "Siqi Kou", "Yiyang Chen", "Yanzhe Hu", "Jianbo Zhou", "Boyuan Zhao", "Zhijie Wei", "Xiao Xia", "Xueqi Li", "Pengfei Liu", "Zhijie Deng"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-04", "topics": ["unified world modeling + language reasoning + action synthesis"], "paradigm": ["world model", "VLA"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["aim-wam", "cosmos-policy", "dreamzero", "himem-wam"], "arxiv": "2606.05979", "doi": "10.48550/arxiv.2606.05979", "code": "", "site": "", "note": "<b>World-Language-Action model</b> — the three-way unification: predict the world, reason in language, emit actions, in one model. Read it for the ablation on whether the language-reasoning branch actually helps the action branch, which is the question every unified model should be asked and few are.", "aka": "WLA", "stars": 2, "curated": true, "local": "", "abstract": "We propose world-language-action (WLA) models as a new class of embodied foundation models. WLA takes textual instructions, images, and robot states as inputs to jointly predict textual subtasks, subgoal images, and robot actions, conjoining the \\emph{world modeling interface} to learn from extensive egocentric videos as in the world-action model (WAM) and the \\emph{language reasoning} capacities to solve complex long-horizon tasks as in vision-language-action (VLA) models. At the core of WLA lies an \\emph{autoregressive (AR)} Transformer backbone, instead of a bidirectional diffusion Transformer as in WAMs, to predict the \\emph{next state}, comprising the \\emph{semantic-level} textual intention and complementary \\emph{fine-grained} physical dynamics. The physical dynamics are supervised by the world modeling objective based on a dedicated World Expert, and are leveraged to ease the characterization of the state-action correlation for the Action Expert. WLA leverages meta-queries to make the world prediction \\emph{implicitly} impact the action generation so that the former can be disabled during inference. The world prediction can also be activated to enable test-time scaling for improved robot control. Our WLA-0 prototype, with 2B active parameters, achieves 40 ms per inference on an NVIDIA RTX 5090. Evaluations across simulated and real-world environments demonstrate that WLA-0 achieves state-of-the-art multi-task and long-horizon learning abilities, e.g., 92.94\\% success rate on RoboTwin2.0 Clean and 56.5\\% success rate on RMBench. WLA-0 also holds the promise to learn novel tasks directly from \\emph{cross-embodiment robot videos} without action annotations.", "source": "curated", "first_author": "Yi Yang", "url": "https://arxiv.org/abs/2606.05979", "fig": "figures/wla-model.webp", "figs": ["figures/wla-model-1.webp", "figures/wla-model-2.webp"], "arxiv_url": "https://arxiv.org/abs/2606.05979"}, {"id": "worldarena2", "title": "WorldArena 2.0: Extending Embodied World Model Benchmarking on Modality, Functionality and Platform", "tree": "systems", "branch": "sim-bench", "authors": ["Yu Shang", "Yinzhou Tang", "Yiding Ma", "Zhuohang Li", "Lei Jin", "Weikang Su", "Xin Jin", "Zhaolu Wang", "Ziyou Wang", "Xin Zhang", "Haisheng Su", "Weizhen He", "Wei Wu", "Haoyi Duan", "Gordon Wetzstein", "Xihui Liu", "Dhruv Shah", "Zhaoxiang Zhang", "Zhibo Chen", "Jun Zhu", "Yonghong Tian", "Tat-Seng Chua", "Wenwu Zhu", "Chen Gao", "Yong Li"], "corresponding": [], "institutions": ["Stanford University", "Tsinghua University", "Peking University", "Shanghai Jiao Tong University", "University of Hong Kong", "Zhejiang University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-18", "topics": ["embodied world model benchmark", "modality/functionality/platform axes"], "paradigm": ["benchmark"], "method": [], "projects": ["wam-tactile"], "related": ["robogaze", "roboworld", "wm-survey-manip", "wm-survey-robot", "worldeval"], "arxiv": "2605.17912", "doi": "10.48550/arxiv.2605.17912", "code": "", "site": "", "note": "<b>WorldArena 2.0</b> — the benchmark that says what a world model is supposed to be good at, broken down by modality, functionality and platform. Use it to pick your evaluation axes rather than inventing them, especially since the tactile modality is where the axis list is thinnest.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models have emerged as a central paradigm for embodied intelligence, enabling agents to predict action-conditioned future and reason about environmental dynamics. However, existing embodied world model benchmarks are still largely confined to vision-only prediction, offline embodied applications, and simulator-based evaluation, making them insufficient for assessing increasingly comprehensive world models. In this work, we introduce WorldArena 2.0, an expanded benchmark that systematically broadens embodied world model evaluation along three dimensions: modality, functionality, and platform. Along the modality dimension, WorldArena 2.0 extends evaluation from vision-only to visuotactile modalities, enabling assessment of multimodal perception and prediction. Along the functionality dimension, it extends beyond policy evaluation and planning to assess world models as interactive RL environments for policy optimization. Along the platform dimension, it moves beyond simulator-only evaluation to a diverse suite of simulated and real-world robotic settings across multiple embodiments. Under a standardized protocol, WorldArena 2.0 comprehensively evaluates perceptual quality, interactive utility, and cross-platform performance, providing a comprehensive testbed for tracking progress toward embodied world models. The benchmark is available at: https://world-arena.ai.", "source": "curated", "first_author": "Yu Shang", "url": "https://arxiv.org/abs/2605.17912", "fig": "figures/worldarena2.webp", "figs": ["figures/worldarena2-1.webp"]}, {"id": "x-op", "title": "X-OP: Cross-Morphology Whole-Body Teleoperation via MPC Retargeting", "tree": "il", "branch": "retargeting", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["UC Berkeley", "Amazon", "Unitree Robotics"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-07", "topics": ["cross-morphology retargeting", "MPC"], "paradigm": ["teleop"], "method": [], "projects": ["exo", "m2", "mabel"], "related": ["c2dex", "h2o", "hover", "human2humanoid"], "arxiv": "2606.07934", "doi": "", "code": "", "site": "", "note": "In your MABEL teleop_and_retargeting folder. MPC-based retargeting across morphologies — the optimization-based counterpoint to learned retargeting.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2606.07934v1_X_OP_Cross_Morphology_Whole_Body_Teleoperation_via_MPC_Retar.pdf", "abstract": "Whole-body teleoperation is essential for scalable robot data collection in loco-manipulation tasks, yet existing approaches relying on exoskeleton suits or multi-camera setups impose prohibitive cost, complexity, and environmental constraints. Recent methods using a single extended reality (XR) device with end-to-end reinforcement learning policies partially address these limitations but require robot-specific retraining, suffer from out-of-distribution failures, and rely on motion retargeting that neglects dynamic feasibility. We propose a hierarchical whole-body teleoperation framework driven by a single XR device that generalizes across diverse robot morphologies without retraining robot-specific policies. A Model Predictive Control (MPC)-based motion retargeter jointly optimizes alignment with the operator's intent and the robot's dynamic feasibility, generating optimal commands for existing low-level controllers. To ensure robust online execution, we introduce a state synchronization method that resets the simulator state at each MPC step to handle noisy real-world measurements and contact sensitivity, and integrate SLAM-based global pose feedback to mitigate long-term drift. Simulation results show higher success rates on whole-body control tasks for both a humanoid (over 30% lower completion time and 20% lower power consumption) and a mobile manipulator (zero collisions) compared to baselines. Real-world experiments further validate the effectiveness and flexibility of our method, demonstrating the successful deployment of the proposed retargeter on both platforms for whole-body control tasks and the ease of allowing users to adjust teleoperation behavior based on their preferences. This plug-and-play framework offers a scalable, morphology-agnostic solution for whole-body robot teleoperation, enabling real-time behavioral customization and broad applicability across platforms.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2606.07934", "fig": "figures/x-op.webp", "figs": ["figures/x-op-1.webp", "figures/x-op-2.webp", "figures/x-op-3.webp"]}, {"id": "yor", "title": "YOR: Your Own Mobile Manipulator for Generalizable Robotics", "tree": "systems", "branch": "mobile-manip", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["UC Berkeley", "New York University", "Dexterity Inc."], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-11", "topics": ["open mobile manipulator", "reproducibility"], "paradigm": ["hardware"], "method": [], "projects": ["m2", "mabel"], "related": ["behavior-robot-suite", "tidybot2"], "arxiv": "2602.11150", "doi": "", "code": "", "site": "", "note": "From your M2 humanoid_platforms folder (also in your pdf2web data). Recent open mobile manipulator — compare its BOM against TidyBot++ before committing MABEL's base design.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2602.11150v1_YOR_Your_Own_Mobile_Manipulator_for_Generalizable_Robotics.pdf", "abstract": "Recent advances in robot learning have generated significant interest in capable platforms that may eventually approach human-level competence. This interest, combined with the commoditization of actuators, has propelled growth in low-cost robotic platforms. However, the optimal form factor for mobile manipulation, especially on a budget, remains an open question. We introduce YOR, an open-source, low-cost mobile manipulator that integrates an omnidirectional base, a telescopic vertical lift, and two arms with grippers to achieve whole-body mobility and manipulation. Our design emphasizes modularity, ease of assembly using off-the-shelf components, and affordability, with a bill-of-materials cost under 10,000 USD. We demonstrate YOR's capability by completing tasks that require coordinated whole-body control, bimanual manipulation, and autonomous navigation. Overall, YOR offers competitive functionality for mobile manipulation research at a fraction of the cost of existing platforms. Project website: https://www.yourownrobot.ai/", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2602.11150", "fig": "figures/yor.webp", "figs": ["figures/yor-1.webp", "figures/yor-2.webp"]}, {"id": "1x-hand", "title": "1X Redwood Hand / NEO", "tree": "systems", "branch": "hands", "authors": ["1X Technologies"], "corresponding": [], "institutions": ["1X Technologies"], "lab": "1X", "venue": "industry", "venue_type": "industry", "year": 2025, "date": "", "topics": ["tendon-driven humanoid hand", "high DOF", "home robot"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["dlr-hand", "orca-hand", "ruka"], "arxiv": "", "doi": "", "code": "", "site": "https://www.1x.tech/", "note": "You named this as the model for your 22-DOF hand. Industry, so specs are announcements rather than reviewed results — but the tendon-driven, high-DOF, soft-covered direction is the one to copy. Verify DOF counts against a current source before citing.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "1X Technologies", "url": "https://www.1x.tech/neo"}, {"id": "embodied-sim-survey", "title": "A Survey: Learning Embodied Intelligence from Physical Simulators and World Models", "tree": "systems", "branch": "sim-bench", "authors": ["Xiaoxiao Long", "Qingrui Zhao", "Kaiwen Zhang", "Zihao Zhang", "Dingrui Wang", "Yumeng Liu", "Zhengjie Shu", "Yi Lu", "Shouzheng Wang", "Xinzhe Wei", "Wei Li", "Wei Yin", "Yao Yao", "Jia Pan", "Qiu Shen", "Ruigang Yang", "Xun Cao", "Qionghai Dai"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Jiao Tong University", "University of Hong Kong"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-07-01", "topics": ["survey", "physical simulators vs world models for embodied intelligence"], "paradigm": ["survey"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "phystwin", "physworld", "wm-survey-robot"], "arxiv": "2507.00917", "doi": "10.48550/arxiv.2507.00917", "code": "", "site": "", "note": "<b>Learning Embodied Intelligence from Physical Simulators and World Models</b> — the survey that treats the analytic simulator and the learned world model as competing answers to the same question, rather than as separate fields. That comparison is precisely the framing your Real2Sim2Real project sits inside, so it is worth the read even though surveys rarely are.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The pursuit of artificial general intelligence (AGI) has placed embodied intelligence at the forefront of robotics research. Embodied intelligence focuses on agents capable of perceiving, reasoning, and acting within the physical world. Achieving robust embodied intelligence requires not only advanced perception and control, but also the ability to ground abstract cognition in real-world interactions. Two foundational technologies, physical simulators and world models, have emerged as critical enablers in this quest. Physical simulators provide controlled, high-fidelity environments for training and evaluating robotic agents, allowing safe and efficient development of complex behaviors. In contrast, world models empower robots with internal representations of their surroundings, enabling predictive planning and adaptive decision-making beyond direct sensory input. This survey systematically reviews recent advances in learning embodied AI through the integration of physical simulators and world models. We analyze their complementary roles in enhancing autonomy, adaptability, and generalization in intelligent robots, and discuss the interplay between external simulation and internal modeling in bridging the gap between simulated training and real-world deployment. By synthesizing current progress and identifying open challenges, this survey aims to provide a comprehensive perspective on the path toward more capable and generalizable embodied AI systems. We also maintain an active repository that contains up-to-date literature and open-source projects at https://github.com/NJU3DV-LoongGroup/Embodied-World-Models-Survey.", "source": "curated", "first_author": "Xiaoxiao Long", "url": "https://arxiv.org/abs/2507.00917", "fig": "figures/embodied-sim-survey.webp", "figs": ["figures/embodied-sim-survey-1.webp", "figures/embodied-sim-survey-2.webp"]}, {"id": "adaworld", "title": "AdaWorld: Learning Adaptable World Models with Latent Actions", "tree": "il", "branch": "world-models", "authors": ["Shenyuan Gao", "Siyuan Zhou", "Yilun Du", "Jun Zhang", "Chuang Gan"], "corresponding": [], "institutions": ["MIT", "HKUST"], "lab": "", "venue": "ICML", "venue_type": "conference", "year": 2025, "date": "2025-03-24", "topics": ["adaptable world models", "latent actions", "fast adaptation to new environments"], "paradigm": ["world model"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["dreamdojo", "genie", "ivideogpt", "lapa", "lawm"], "arxiv": "2503.18938", "doi": "10.48550/arXiv.2503.18938", "code": "", "site": "", "note": "<b>AdaWorld</b> — condition the world model on latent actions extracted from video so that adapting to a new environment is a small fine-tune rather than a retrain. The adaptation axis of the latent-action idea, as opposed to LAPA's pretraining axis.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models aim to learn action-controlled future prediction and have proven essential for the development of intelligent agents. However, most existing world models rely heavily on substantial action-labeled data and costly training, making it challenging to adapt to novel environments with heterogeneous actions through limited interactions. This limitation can hinder their applicability across broader domains. To overcome this limitation, we propose AdaWorld, an innovative world model learning approach that enables efficient adaptation. The key idea is to incorporate action information during the pretraining of world models. This is achieved by extracting latent actions from videos in a self-supervised manner, capturing the most critical transitions between frames. We then develop an autoregressive world model that conditions on these latent actions. This learning paradigm enables highly adaptable world models, facilitating efficient transfer and learning of new actions even with limited interactions and finetuning. Our comprehensive experiments across multiple environments demonstrate that AdaWorld achieves superior performance in both simulation quality and visual planning.", "source": "curated", "first_author": "Shenyuan Gao", "url": "https://arxiv.org/abs/2503.18938", "fig": "figures/adaworld.webp", "figs": ["figures/adaworld-1.webp", "figures/adaworld-2.webp", "figures/adaworld-3.webp"]}, {"id": "agent-laboratory", "title": "Agent Laboratory: Using LLM Agents as Research Assistants", "tree": "systems", "branch": "agentic", "authors": ["Samuel Schmidgall", "Yusheng Su", "Ze Wang", "Ximeng Sun", "Jialian Wu", "Xiaodong Yu", "Jiang Liu", "Michael Moor", "Zicheng Liu", "Emad Barsoum"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-01-08", "topics": ["LLM research assistants", "automated experimentation"], "paradigm": ["LLM"], "method": [], "projects": [], "related": ["ai-scientist", "harbor", "real-deep-research", "voyager"], "arxiv": "2501.04227", "doi": "", "code": "", "site": "", "note": "Agent Laboratory — LLM agents running the literature-review / experiment / report loop. Non-robotics, but the honest reference point for what auto-research can and cannot do right now.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Historically, scientific discovery has been a lengthy and costly process, demanding substantial time and resources from initial conception to final results. To accelerate scientific discovery, reduce research costs, and improve research quality, we introduce Agent Laboratory, an autonomous LLM-based framework capable of completing the entire research process. This framework accepts a human-provided research idea and progresses through three stages--literature review, experimentation, and report writing to produce comprehensive research outputs, including a code repository and a research report, while enabling users to provide feedback and guidance at each stage. We deploy Agent Laboratory with various state-of-the-art LLMs and invite multiple researchers to assess its quality by participating in a survey, providing human feedback to guide the research process, and then evaluate the final paper. We found that: (1) Agent Laboratory driven by o1-preview generates the best research outcomes; (2) The generated machine learning code is able to achieve state-of-the-art performance compared to existing methods; (3) Human involvement, providing feedback at each stage, significantly improves the overall quality of research; (4) Agent Laboratory significantly reduces research expenses, achieving an 84% decrease compared to previous autonomous research methods. We hope Agent Laboratory enables researchers to allocate more effort toward creative ideation rather than low-level coding and writing, ultimately accelerating scientific discovery.", "source": "curated", "first_author": "Samuel Schmidgall", "url": "https://arxiv.org/abs/2501.04227", "fig": "figures/agent-laboratory.webp", "figs": ["figures/agent-laboratory-1.webp"]}, {"id": "agibot-world", "title": "AgiBot World Colosseo: A Large-scale Manipulation Platform for Scalable and Intelligent Embodied Systems", "tree": "il", "branch": "data-scaling", "authors": ["AgiBot-World-Contributors"], "corresponding": [], "institutions": ["AgiBot", "Shanghai AI Laboratory"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-09", "topics": ["large-scale dataset", "standardized platform"], "paradigm": ["dataset"], "method": [], "projects": [], "related": ["droid", "oxe", "pi05"], "arxiv": "2503.06669", "doi": "", "code": "", "site": "", "note": "1M+ trajectories from a standardized fleet. The industrial answer to DROID, with hardware uniformity as the design principle.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We explore how scalable robot data can address real-world challenges for generalized robotic manipulation. Introducing AgiBot World, a large-scale platform comprising over 1 million trajectories across 217 tasks in five deployment scenarios, we achieve an order-of-magnitude increase in data scale compared to existing datasets. Accelerated by a standardized collection pipeline with human-in-the-loop verification, AgiBot World guarantees high-quality and diverse data distribution. It is extensible from grippers to dexterous hands and visuo-tactile sensors for fine-grained skill acquisition. Building on top of data, we introduce Genie Operator-1 (GO-1), a novel generalist policy that leverages latent action representations to maximize data utilization, demonstrating predictable performance scaling with increased data volume. Policies pre-trained on our dataset achieve an average performance improvement of 30% over those trained on Open X-Embodiment, both in in-domain and out-of-distribution scenarios. GO-1 exhibits exceptional capability in real-world dexterous and long-horizon tasks, achieving over 60% success rate on complex tasks and outperforming prior RDT approach by 32%. By open-sourcing the dataset, tools, and models, we aim to democratize access to large-scale, high-quality robot data, advancing the pursuit of scalable and general-purpose intelligence.", "source": "curated", "first_author": "AgiBot-World-Contributors", "url": "https://arxiv.org/abs/2503.06669", "fig": "figures/agibot-world.webp", "figs": ["figures/agibot-world-1.webp", "figures/agibot-world-2.webp"]}, {"id": "airexo", "title": "AirExo-2: Scaling up Generalizable Robotic Imitation Learning with Low-Cost Exoskeletons", "tree": "systems", "branch": "interfaces", "authors": ["Hongjie Fang", "Chenxi Wang", "Yiming Wang", "Jingjing Chen", "Shangning Xia", "Jun Lv", "Hao-Shu Fang", "Cewu Lu"], "corresponding": [], "institutions": ["Shanghai Jiao Tong University"], "lab": "SJTU MVIG (Cewu Lu)", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-03-05", "topics": ["low-cost exoskeleton", "in-the-wild demonstrations"], "paradigm": ["hardware", "IL"], "method": [], "projects": ["exo", "mabel"], "related": ["dexumi", "ume", "umi"], "arxiv": "2503.03081", "doi": "", "code": "", "site": "", "note": "Exoskeleton capture that transfers without robot data. The main competitor framing for UME.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Scaling up robotic imitation learning for real-world applications requires efficient and scalable demonstration collection methods. While teleoperation is effective, it depends on costly and inflexible robot platforms. In-the-wild demonstrations offer a promising alternative, but existing collection devices have key limitations: handheld setups offer limited observational coverage, and whole-body systems often require fine-tuning with robot data due to domain gaps. To address these challenges, we present AirExo-2, a low-cost exoskeleton system for large-scale in-the-wild data collection, along with several adaptors that transform collected data into pseudo-robot demonstrations suitable for policy learning. We further introduce RISE-2, a generalizable imitation learning policy that fuses 3D spatial and 2D semantic perception for robust manipulations. Experiments show that RISE-2 outperforms prior state-of-the-art methods on both in-domain and generalization evaluations. Trained solely on adapted in-the-wild data produced by AirExo-2, the RISE-2 policy achieves comparable performance to the policy trained with teleoperated data, highlighting the effectiveness and potential of AirExo-2 for scalable and generalizable imitation learning.", "source": "curated", "first_author": "Hongjie Fang", "url": "https://arxiv.org/abs/2503.03081", "fig": "figures/airexo.webp", "figs": ["figures/airexo-1.webp", "figures/airexo-2.webp"]}, {"id": "attention-map-hutter", "title": "Attention-Based Map Encoding for Learning Generalized Legged Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Junzhe He", "Chong Zhang", "Fabian Jenelten", "Ruben Grandia", "Moritz Bächer", "Marco Hutter"], "corresponding": [], "institutions": ["ETH Zürich", "Disney Research"], "lab": "ETH RSL", "venue": "Science Robotics", "venue_type": "journal", "year": 2025, "date": "2025-06-11", "topics": ["terrain encoding", "attention", "generalization"], "paradigm": ["RL"], "method": ["attention over local height-map tokens"], "projects": [], "related": ["anymal-parkour", "miki2022"], "arxiv": "2506.09588", "doi": "", "code": "", "site": "", "note": "In your library. The modern successor to Miki's belief encoder — attention over map tokens instead of a fixed CNN.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/RL/Attention-Based Map_2025_hutter_sr.pdf", "abstract": "Dynamic locomotion of legged robots is a critical yet challenging topic in expanding the operational range of mobile robots. It requires precise planning when possible footholds are sparse, robustness against uncertainties and disturbances, and generalizability across diverse terrains. While traditional model-based controllers excel at planning on complex terrains, they struggle with real-world uncertainties. Learning-based controllers offer robustness to such uncertainties but often lack precision on terrains with sparse steppable areas. Hybrid methods achieve enhanced robustness on sparse terrains by combining both methods but are computationally demanding and constrained by the inherent limitations of model-based planners. To achieve generalized legged locomotion on diverse terrains while preserving the robustness of learning-based controllers, this paper proposes to learn an attention-based map encoding conditioned on robot proprioception, which is trained as part of the end-to-end controller using reinforcement learning. We show that the network learns to focus on steppable areas for future footholds when the robot dynamically navigates diverse and challenging terrains. We synthesize behaviors that exhibit robustness against uncertainties while enabling precise and agile traversal of sparse terrains. Additionally, our method offers a way to interpret the topographical perception of a neural network. We have trained two controllers for a 12-DoF quadrupedal robot and a 23-DoF humanoid robot respectively and tested the resulting controllers in the real world under various challenging indoor and outdoor scenarios, including ones unseen during training.", "source": "curated", "first_author": "Junzhe He", "url": "https://arxiv.org/abs/2506.09588", "fig": "figures/attention-map-hutter.webp", "figs": ["figures/attention-map-hutter-1.webp", "figures/attention-map-hutter-2.webp"]}, {"id": "audio-vla", "title": "Audio-VLA: Adding Contact Audio Perception to Vision-Language-Action Model for Robotic Manipulation", "tree": "il", "branch": "vla", "authors": ["Wei", "Xiangyi", "Haotian Zhang", "Xinyi Cao", "Siyu Xie", "Weifeng Ge", "Yang Li", "Changbo Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-13", "topics": ["contact audio in a VLA", "multimodal action model"], "paradigm": ["VLA", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["dj-mix-analysis", "dream-tac", "maniwav", "openvla", "pi0"], "arxiv": "2511.09958", "doi": "10.48550/arxiv.2511.09958", "code": "", "site": "", "note": "Adds contact-audio perception to a VLA. The natural baseline once you have an audio-equipped hand: does sound survive being fed into a large pretrained action model, or does it get ignored next to vision?", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The Vision-Language-Action models (VLA) have achieved significant advances in robotic manipulation recently. However, vision-only VLA models create fundamental limitations, particularly in perceiving interactive and manipulation dynamic processes. This paper proposes Audio-VLA, a multimodal manipulation policy that leverages contact audio to perceive contact events and dynamic process feedback. Audio-VLA overcomes the vision-only constraints of VLA models. Additionally, this paper introduces the Task Completion Rate (TCR) metric to systematically evaluate dynamic operational processes. Audio-VLA employs pre-trained DINOv2 and SigLIP as visual encoders, AudioCLIP as the audio encoder, and Llama2 as the large language model backbone. We apply LoRA fine-tuning to these pre-trained modules to achieve robust cross-modal understanding of both visual and acoustic inputs. A multimodal projection layer aligns features from different modalities into the same feature space. Moreover RLBench and LIBERO simulation environments are enhanced by adding collision-based audio generation to provide realistic sound feedback during object interactions. Since current robotic manipulation evaluations focus on final outcomes rather than providing systematic assessment of dynamic operational processes, the proposed TCR metric measures how well robots perceive dynamic processes during manipulation, creating a more comprehensive evaluation metric. Extensive experiments on LIBERO, RLBench, and two real-world tasks demonstrate Audio-VLA's superior performance over vision-only comparative methods, while the TCR metric effectively quantifies dynamic process perception capabilities.", "source": "curated", "first_author": "Wei", "url": "https://arxiv.org/abs/2511.09958", "fig": "figures/audio-vla.webp", "figs": ["figures/audio-vla-1.webp", "figures/audio-vla-2.webp"]}, {"id": "beyond-mimic", "title": "BeyondMimic: From Motion Tracking to Versatile Humanoid Control via Guided Diffusion", "tree": "rl", "branch": "motion-imitation", "authors": ["Hongwei Yu", "Qiayuan Liao", "Wenli Xiao", "Zi Wang", "Guanya Shi", "Koushil Sreenath"], "corresponding": [], "institutions": ["UC Berkeley", "Carnegie Mellon University"], "lab": "Hybrid Robotics (Sreenath); LeCAR (Shi)", "venue": "Science Robotics", "venue_type": "preprint", "year": 2025, "date": "2025-08-11", "topics": ["motion tracking", "guided diffusion", "versatile control"], "paradigm": ["RL", "diffusion"], "method": ["unified tracking policy + diffusion-based test-time steering"], "projects": ["mabel"], "related": ["asap", "deepmimic", "dp", "maskedmimic"], "arxiv": "2508.08241", "doi": "10.1126/scirobotics.adx8924", "code": "", "site": "", "note": "You named this one. The pitch is that a single well-trained tracking policy plus diffusion guidance at test time replaces a library of task-specific controllers — no retraining to get a new behaviour.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The human-like form of humanoid robots positions them uniquely to achieve the agility and versatility in motor skills that humans possess. Learning from human demonstrations offers a scalable approach to acquiring these capabilities. However, prior works either produce unnatural motions or rely on motion-specific tuning to achieve satisfactory naturalness. Furthermore, these methods are often motion- or goal-specific, lacking the versatility to compose diverse skills, especially when solving unseen tasks. We present BeyondMimic, a framework that scales to diverse motions and carries the versatility to compose them seamlessly in tackling unseen downstream tasks. At heart, a compact motion-tracking formulation enables mastering a wide range of radically agile behaviors, including aerial cartwheels, spin-kicks, flip-kicks, and sprinting, with a single setup and shared hyperparameters, all while achieving state-of-the-art human-like performance. Moving beyond the mere imitation of existing motions, we propose a unified latent diffusion model that empowers versatile goal specification, seamless task switching, and dynamic composition of these agile behaviors. Leveraging classifier guidance, a diffusion-specific technique for test-time optimization toward novel objectives, our model extends its capability to solve downstream tasks never encountered during training, including motion inpainting, joystick teleoperation, and obstacle avoidance, and transfers these skills zero-shot to real hardware. This work opens new frontiers for humanoid robots by pushing the limits of scalable human-like motor skill acquisition from human motion and advancing seamless motion synthesis that achieves generalization and versatility beyond training setups.", "source": "curated", "first_author": "Hongwei Yu", "url": "https://doi.org/10.1126/scirobotics.adx8924", "arxiv_url": "https://arxiv.org/abs/2508.08241", "fig": "figures/beyond-mimic.webp"}, {"id": "catch-it", "title": "Catch It! Learning to Catch in Flight with Mobile Dexterous Hands", "tree": "rl", "branch": "manipulation", "authors": ["Yuanhang Zhang", "Tianhai Liang", "Zhenyang Chen", "Yanjie Ze", "Huazhe Xu"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Qi Zhi Institute", "Stanford University"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2024-09-17", "topics": ["dynamic manipulation", "catching", "mobile dexterity"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["falcon", "visualmimic"], "arxiv": "2409.10319", "doi": "", "code": "", "site": "", "note": "In your library. Dynamic, whole-body, and dexterous at once — a good stress test for the mobile-dexterous-manipulation idea.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Catching objects in flight (i.e., thrown objects) is a common daily skill for humans, yet it presents a significant challenge for robots. This task requires a robot with agile and accurate motion, a large spatial workspace, and the ability to interact with diverse objects. In this paper, we build a mobile manipulator composed of a mobile base, a 6-DoF arm, and a 12-DoF dexterous hand to tackle such a challenging task. We propose a two-stage reinforcement learning framework to efficiently train a whole-body-control catching policy for this high-DoF system in simulation. The objects' throwing configurations, shapes, and sizes are randomized during training to enhance policy adaptivity to various trajectories and object characteristics in flight. The results show that our trained policy catches diverse objects with randomly thrown trajectories, at a high success rate of about 80\\% in simulation, with a significant improvement over the baselines. The policy trained in simulation can be directly deployed in the real world with onboard sensing and computation, which achieves catching sandbags in various shapes, randomly thrown by humans. Our project page is available at https://mobile-dex-catch.github.io/.", "source": "curated", "first_author": "Yuanhang Zhang", "url": "https://arxiv.org/abs/2409.10319", "fig": "figures/catch-it.webp", "figs": ["figures/catch-it-1.webp", "figures/catch-it-2.webp", "figures/catch-it-3.webp"]}, {"id": "clutterdexgrasp", "title": "ClutterDexGrasp: A Sim-to-Real System for General Dexterous Grasping in Cluttered Scenes", "tree": "rl", "branch": "manipulation", "authors": ["Zeyuan Chen", "Qiyang Yan", "Yuanpei Chen", "Tianhao Wu", "Jiyao Zhang", "Zihan Ding", "Jinzhou Li", "Yaodong Yang", "Hao Dong"], "corresponding": [], "institutions": ["Peking University"], "lab": "PKU-Agibot Lab / Hao Dong Lab", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-06-17", "topics": ["dexterous grasping", "clutter", "sim-to-real"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["artic-tools", "chen-visual-inhand", "dexndm", "force-grasp-s2r"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. Clutter is the setting where analytic grasp planners fall over and RL earns its keep.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/1153_ClutterDexGrasp_A_Sim_to_.pdf", "abstract": "", "source": "curated", "first_author": "Zeyuan Chen", "url": "https://arxiv.org/abs/2506.14317", "fig": "figures/clutterdexgrasp.webp", "figs": ["figures/clutterdexgrasp-1.webp"]}, {"id": "conrft", "title": "ConRFT: A Reinforced Fine-tuning Method for VLA Models via Consistency Policy", "tree": "rl", "branch": "post-training", "authors": ["Yuhui Chen", "Shuai Tian", "Shugao Liu", "Yingting Zhou", "Haoran Li", "Dongbin Zhao"], "corresponding": [], "institutions": ["Chinese Academy of Sciences"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2025, "date": "2025-02-08", "topics": ["VLA fine-tuning", "consistency policy"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["consistency-policy", "iRe-VLA", "manflow", "ript-vla"], "arxiv": "2502.05450", "doi": "", "code": "", "site": "", "note": "Consistency-policy action head makes RL fine-tuning of a diffusion-style VLA tractable by cutting the denoising chain to a few steps.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models have shown substantial potential in real-world robotic manipulation. However, fine-tuning these models through supervised learning struggles to achieve robust performance due to limited, inconsistent demonstrations, especially in contact-rich environments. In this paper, we propose a reinforced fine-tuning approach for VLA models, named ConRFT, which consists of offline and online fine-tuning with a unified consistency-based training objective, to address these challenges. In the offline stage, our method integrates behavior cloning and Q-learning to effectively extract policy from a small set of demonstrations and stabilize value estimating. In the online stage, the VLA model is further fine-tuned via consistency policy, with human interventions to ensure safe exploration and high sample efficiency. We evaluate our approach on eight diverse real-world manipulation tasks. It achieves an average success rate of 96.3% within 45-90 minutes of online fine-tuning, outperforming prior supervised methods with a 144% improvement in success rate and 1.9x shorter episode length. This work highlights the potential of integrating reinforcement learning to enhance the performance of VLA models for real-world robotic applications. Videos and code are available at our project website https://cccedric.github.io/conrft/.", "source": "curated", "first_author": "Yuhui Chen", "url": "https://arxiv.org/abs/2502.05450", "fig": "figures/conrft.webp", "figs": ["figures/conrft-1.webp"]}, {"id": "cosmos", "title": "Cosmos World Foundation Model Platform for Physical AI", "tree": "il", "branch": "world-models", "authors": ["NVIDIA"], "corresponding": [], "institutions": ["NVIDIA"], "lab": "NVIDIA", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-01-07", "topics": ["world foundation model", "physical AI", "synthetic data"], "paradigm": ["world model"], "method": [], "projects": [], "related": ["cosmos-policy", "cosmos-reason", "dreamerv4", "dreamgen", "gaia-2", "genie3", "gigaworld-0", "vjepa2"], "arxiv": "2501.03575", "doi": "", "code": "", "site": "", "note": "Open world-foundation-model weights plus tokenizers and a data pipeline. Practical if you want a pretrained video prior rather than training one.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Physical AI needs to be trained digitally first. It needs a digital twin of itself, the policy model, and a digital twin of the world, the world model. In this paper, we present the Cosmos World Foundation Model Platform to help developers build customized world models for their Physical AI setups. We position a world foundation model as a general-purpose world model that can be fine-tuned into customized world models for downstream applications. Our platform covers a video curation pipeline, pre-trained world foundation models, examples of post-training of pre-trained world foundation models, and video tokenizers. To help Physical AI builders solve the most critical problems of our society, we make Cosmos open-source and our models open-weight with permissive licenses available via https://github.com/nvidia-cosmos/cosmos-predict1.", "source": "curated", "first_author": "NVIDIA", "url": "https://arxiv.org/abs/2501.03575", "fig": "figures/cosmos.webp", "figs": ["figures/cosmos-1.webp", "figures/cosmos-2.webp"]}, {"id": "decentralized-aerial", "title": "Decentralized Aerial Manipulation of a Cable-Suspended Load using Multi-Agent Reinforcement Learning", "tree": "rl", "branch": "marl", "authors": ["Jack Zeng", "Andreu Matoses Gimenez", "Eugene Vinitsky", "Javier Alonso-Mora", "Sihao Sun"], "corresponding": [], "institutions": [], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-08-02", "topics": ["aerial manipulation", "cooperative transport", "decentralized control", "proceedings 2025"], "paradigm": ["MARL"], "method": [], "projects": ["m2"], "related": ["chorus", "h2compact", "mappo"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. The cleanest recent demonstration that decentralized MARL can hold a shared rigid constraint — exactly M2's collaborative-transport problem in a different medium.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/1086_Decentralized_Aerial_Mani.pdf", "abstract": "", "source": "curated", "first_author": "Jack Zeng", "url": "https://arxiv.org/abs/2508.01522", "fig": "figures/x-decentralizedaerialmanipulationofacables.webp"}, {"id": "demohlm", "title": "DemoHLM: From One Demonstration to Generalizable Humanoid Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Fu", "Yuhui", "Xie", "Feiyang", "Xu", "Chaoyi", "Xiong", "Jing", "Yuan", "Haoqi", "Lu", "Zongqing"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-13", "topics": ["one demonstration", "generalizable loco-manipulation"], "paradigm": ["RL", "IL"], "method": [], "projects": ["mabel"], "related": ["falcon", "humanego", "one-demo", "resmimic"], "arxiv": "2510.11258", "doi": "10.48550/arxiv.2510.11258", "code": "", "site": "", "note": "One demonstration to generalizable humanoid loco-manipulation. The sample-efficiency claim is the interesting one — compare against [[humanego]]'s thirty-minutes result before assuming you need scale.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Loco-manipulation is a fundamental challenge for humanoid robots to achieve versatile interactions in human environments. Although recent studies have made significant progress in humanoid whole-body control, loco-manipulation remains underexplored and often relies on hard-coded task definitions or costly real-world data collection, which limits autonomy and generalization. We present DemoHLM, a framework for humanoid loco-manipulation that enables generalizable loco-manipulation on a real humanoid robot from a single demonstration in simulation. DemoHLM adopts a hierarchy that integrates a low-level universal whole-body controller with high-level manipulation policies for multiple tasks. The whole-body controller maps whole-body motion commands to joint torques and provides omnidirectional mobility for the humanoid robot. The manipulation policies, learned in simulation via our data generation and imitation learning pipeline, command the whole-body controller with closed-loop visual feedback to execute challenging loco-manipulation tasks. Experiments show a positive correlation between the amount of synthetic data and policy performance, underscoring the effectiveness of our data generation pipeline and the data efficiency of our approach. Real-world experiments on a Unitree G1 robot equipped with an RGB-D camera validate the sim-to-real transferability of DemoHLM, demonstrating robust performance under spatial variations across ten loco-manipulation tasks.", "source": "curated", "first_author": "Fu", "url": "https://arxiv.org/abs/2510.11258", "fig": "figures/demohlm.webp"}, {"id": "berkeley-humanoid-lite", "title": "Demonstrating Berkeley Humanoid Lite: An Open-source, Accessible, and Customizable 3D-printed Humanoid Robot", "tree": "systems", "branch": "humanoid-hw", "authors": ["Yufeng Chi", "Qiayuan Liao", "Junfeng Long", "Xiaoyu Huang", "Zhongyu Li", "Koushil Sreenath"], "corresponding": [], "institutions": ["UC Berkeley"], "lab": "Hybrid Robotics Lab", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-04-24", "topics": ["3D-printed humanoid", "open source", "accessibility"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["berkeley-humanoid", "robosen", "so100", "toddlerbot", "x2n"], "arxiv": "2504.17249", "doi": "", "code": "", "site": "", "note": "In your MABEL hardware folder. Fully printable, ~$5k. The modular 3D-printed cycloidal gearbox is the interesting mechanical contribution.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/MABEL/literatures/hardware/2504.17249v1.pdf", "abstract": "Despite significant interest and advancements in humanoid robotics, most existing commercially available hardware remains high-cost, closed-source, and non-transparent within the robotics community. This lack of accessibility and customization hinders the growth of the field and the broader development of humanoid technologies. To address these challenges and promote democratization in humanoid robotics, we demonstrate Berkeley Humanoid Lite, an open-source humanoid robot designed to be accessible, customizable, and beneficial for the entire community. The core of this design is a modular 3D-printed gearbox for the actuators and robot body. All components can be sourced from widely available e-commerce platforms and fabricated using standard desktop 3D printers, keeping the total hardware cost under $5,000 (based on U.S. market prices). The design emphasizes modularity and ease of fabrication. To address the inherent limitations of 3D-printed gearboxes, such as reduced strength and durability compared to metal alternatives, we adopted a cycloidal gear design, which provides an optimal form factor in this context. Extensive testing was conducted on the 3D-printed actuators to validate their durability and alleviate concerns about the reliability of plastic components. To demonstrate the capabilities of Berkeley Humanoid Lite, we conducted a series of experiments, including the development of a locomotion controller using reinforcement learning. These experiments successfully showcased zero-shot policy transfer from simulation to hardware, highlighting the platform's suitability for research validation. By fully open-sourcing the hardware design, embedded code, and training and deployment frameworks, we aim for Berkeley Humanoid Lite to serve as a pivotal step toward democratizing the development of humanoid robotics. All resources are available at https://lite.berkeley-humanoid.org.", "source": "curated", "first_author": "Yufeng Chi", "url": "https://arxiv.org/abs/2504.17249", "fig": "figures/berkeley-humanoid-lite.webp", "figs": ["figures/berkeley-humanoid-lite-1.webp", "figures/berkeley-humanoid-lite-2.webp", "figures/berkeley-humanoid-lite-3.webp"]}, {"id": "dexvla", "title": "DexVLA: Vision-Language Model with Plug-In Diffusion Expert for General Robot Control", "tree": "il", "branch": "vla", "authors": ["Junjie Wen", "Yichen Zhu", "Jinming Li", "Zhibin Tang", "Chaomin Shen", "Feifei Feng"], "corresponding": [], "institutions": ["Midea Group", "East China Normal University"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-02-09", "topics": ["plug-in diffusion expert", "dexterous VLA"], "paradigm": ["VLA", "diffusion"], "method": [], "projects": [], "related": ["dvla", "groot-n1", "openvla", "pi0"], "arxiv": "2502.05855", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. A billion-parameter diffusion 'expert' that plugs into a frozen VLM — a modular alternative to co-training everything.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/517_DexVLA_Vision_Language_Mod.pdf", "abstract": "Enabling robots to perform diverse tasks across varied environments is a central challenge in robot learning. While vision-language-action (VLA) models have shown promise for generalizable robot skills, realizing their full potential requires addressing limitations in action representation and efficient training. Current VLA models often focus on scaling the vision-language model (VLM) component, while the action space representation remains a critical bottleneck. This paper introduces DexVLA, a novel framework designed to enhance the efficiency and generalization capabilities of VLAs for complex, long-horizon tasks across diverse robot embodiments. DexVLA features a novel diffusion-based action expert, scaled to one billion parameters, designed for cross-embodiment learning. A novel embodiment curriculum learning strategy facilitates efficient training: (1) pre-training the diffusion expert that is separable from the VLA on cross-embodiment data, (2) aligning the VLA model to specific embodiments, and (3) post-training for rapid adaptation to new tasks. We conduct comprehensive experiments across multiple embodiments, including single-arm, bimanual, and dexterous hand, demonstrating DexVLA's adaptability to challenging tasks without task-specific adaptation, its ability to learn dexterous skills on novel embodiments with limited data, and its capacity to complete complex, long-horizon tasks using only direct language prompting, such as laundry folding. In all settings, our method demonstrates superior performance compared to state-of-the-art models like Octo, OpenVLA, and Diffusion Policy.", "source": "curated", "first_author": "Junjie Wen", "url": "https://arxiv.org/abs/2502.05855", "fig": "figures/dexvla.webp", "figs": ["figures/dexvla-1.webp", "figures/dexvla-2.webp", "figures/dexvla-3.webp"]}, {"id": "dexwrist", "title": "DexWrist: A Robotic Wrist for Constrained and Dynamic Manipulation", "tree": "systems", "branch": "hands", "authors": ["Martin Peticco", "Gabriella Ulloa", "John Marangola", "Nitish Dashora", "Pulkit Agrawal"], "corresponding": [], "institutions": ["MIT"], "lab": "MIT Improbable AI Lab", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-07-01", "topics": ["robotic wrist", "dynamic manipulation", "backdrivability"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["leap-hand", "midas-hand", "orca-hand"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "In your Bionic Hand Motor/ folder. The wrist is the joint everyone under-designs; this one is built for dynamic, constrained motion. Worth reading alongside the 22-DOF hand plan.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/Bionic Hand Motor/Martin Peticco_dexwrist 2025.pdf", "abstract": "", "source": "curated", "first_author": "Martin Peticco", "url": "https://arxiv.org/abs/2507.01008", "fig": "figures/dexwrist.webp", "figs": ["figures/dexwrist-1.webp", "figures/dexwrist-2.webp", "figures/dexwrist-3.webp"]}, {"id": "dexplore", "title": "Dexplore: Scalable Neural Control for Dexterous Manipulation from Reference-Scoped Exploration", "tree": "rl", "branch": "manipulation", "authors": ["Sirui Xu", "Yu-Wei Chao", "Liuyu Bian", "Arsalan Mousavian", "Yu-Xiong Wang", "Liang-Yan Gui", "Wei Yang"], "corresponding": [], "institutions": ["UIUC", "NVIDIA"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-09-11", "topics": ["dexterous manipulation", "reference-guided exploration", "scalable control"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["chen-inhand", "deepmimic"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 proceedings folder. Uses reference motion as a soft exploration scope rather than a hard tracking target — a middle ground between DeepMimic-style tracking and free RL.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/787_Dexplore_Scalable_Neural_C.pdf", "abstract": "", "source": "curated", "first_author": "Sirui Xu", "url": "https://arxiv.org/abs/2509.09671", "fig": "figures/dexplore.webp", "figs": ["figures/dexplore-1.webp", "figures/dexplore-2.webp"]}, {"id": "dlr-vla", "title": "Discover, Learn, and Reinforce: Scaling Vision-Language-Action Pretraining with Diverse RL-Generated Trajectories", "tree": "rl", "branch": "post-training", "authors": ["Rushuai Yang", "Zhiyuan Feng", "Tianxiang Zhang", "Kaixin Wang", "Chuheng Zhang", "Li Zhao", "Xiu Su", "Yi Chen", "Jiang Bian"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-24", "topics": ["RL-generated trajectories", "VLA pretraining"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["oxe", "pld", "robocat"], "arxiv": "2511.19528", "doi": "", "code": "", "site": "", "note": "Inverts the usual order: use RL to *generate* diverse trajectories, then pretrain the VLA on them. Interesting because it makes RL a data engine rather than a fine-tuner.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Scaling vision-language-action (VLA) model pre-training requires large volumes of diverse, high-quality manipulation trajectories. Most current data is obtained via human teleoperation, which is expensive and difficult to scale. Reinforcement learning (RL) methods learn useful skills through autonomous exploration, making them a viable approach for generating data. However, standard RL training collapses to a narrow execution pattern, limiting its utility for large-scale pre-training. We propose Discover, Lea rn and Reinforce (DLR), an information-theoretic pattern discovery framework that generates multiple distinct, high-success behavioral patterns for VLA pretraining. Empirically, DLR generates a markedly more diverse trajectory corpus on LIBERO. Specifically, it learns multiple distinct, high-success strategies for the same task where standard RL discovers only one, and hence it covers substantially broader regions of the state-action space. When adapted to unseen downstream task suites, VLA models pretrained on our diverse RL data surpass counterparts trained on equal-sized standard RL datasets. Moreover, DLR exhibits positive data-scaling behavior that single-pattern RL lacks. These results position multi-pattern RL as a practical, scalable data engine for embodied foundation models.", "source": "curated", "first_author": "Rushuai Yang", "url": "https://arxiv.org/abs/2511.19528", "fig": "figures/dlr-vla.webp", "figs": ["figures/dlr-vla-1.webp"]}, {"id": "egodemogen", "title": "EgoDemoGen: Egocentric Demonstration Generation for Viewpoint Generalization in Robotic Manipulation", "tree": "il", "branch": "human-video", "authors": ["Yuan Xu", "Jiabing Yang", "Xiaofeng Wang", "Yixiang Chen", "Zheng Zhu", "Bowen Fang", "Guan Huang", "Xinze Chen", "Yun Ye", "Qiang Zhang", "Peiyan Li", "Xiangnan Wu", "Kai Wang", "Bing Zhan", "Shuo Lu", "Jing Liu", "Nianfeng Liu", "Yan Huang", "Liang Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-26", "topics": ["egocentric demonstration generation", "viewpoint generalization"], "paradigm": ["IL"], "method": [], "projects": ["r2s2r"], "related": ["egoengine", "egomimic", "one-demo", "prism"], "arxiv": "2509.22578", "doi": "", "code": "", "site": "", "note": "Synthesises novel egocentric viewpoints so the policy stops overfitting to where the camera was. Cheap augmentation that any ego-video pipeline should include.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Imitation learning based visuomotor policies have achieved strong performance in robotic manipulation, yet they often remain sensitive to egocentric viewpoint shifts. Unlike third-person viewpoint changes that only move the camera, egocentric shifts simultaneously alter both the camera pose and the robot action coordinate frame, making it necessary to jointly transfer action trajectories and synthesize corresponding observations under novel egocentric viewpoints. To address this challenge, we present EgoDemoGen, a framework that generates paired observation--action demonstrations under novel egocentric viewpoints through two key components: 1{)} EgoTrajTransfer, which transfers robot trajectories to the novel egocentric coordinate frame through motion-skill segmentation, geometry-aware transformation, and inverse kinematics filtering; and 2{)} EgoViewTransfer, a conditional video generation model that fuses a novel-viewpoint reprojected scene video and a robot motion video rendered from the transferred trajectory to synthesize photorealistic observations, trained with a self-supervised double reprojection strategy without requiring multi-viewpoint data. Experiments in simulation and real-world settings show that EgoDemoGen consistently improves policy success rates under both standard and novel egocentric viewpoints, with absolute gains of +24.6\\% and +16.9\\% in simulation and +16.0\\% and +23.0\\% on the real robot. Moreover, EgoViewTransfer achieves superior video generation quality for novel egocentric observations.", "source": "curated", "first_author": "Yuan Xu", "url": "https://arxiv.org/abs/2509.22578", "fig": "figures/egodemogen.webp", "figs": ["figures/egodemogen-1.webp", "figures/egodemogen-2.webp", "figures/egodemogen-3.webp"]}, {"id": "exo-viha", "title": "Exo-ViHa: A Cross-Platform Exoskeleton System with Visual and Haptic Feedback for Efficient Dexterous Skill Learning", "tree": "systems", "branch": "interfaces", "authors": ["Chao", "Xintao", "Shilong Mu", "Yushan Liu", "Shoujie Li", "Chuqiao Lyu", "Xiao–Ping Zhang", "Wenbo Ding"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-03", "topics": ["exoskeleton with visual and haptic feedback", "skill learning"], "paradigm": ["teleop"], "method": [], "projects": ["exo", "geodex", "mabel"], "related": ["ace-teleop", "doglove", "homie", "ume"], "arxiv": "2503.01543", "doi": "10.48550/arxiv.2503.01543", "code": "", "site": "", "note": "Cross-platform exoskeleton with both visual and haptic feedback. Haptics is the channel almost every low-cost rig drops, and it is what lets the operator feel a failed grasp before the camera shows it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Imitation learning has emerged as a powerful paradigm for robot skills learning. However, traditional data collection systems for dexterous manipulation face challenges, including a lack of balance between acquisition efficiency, consistency, and accuracy. To address these issues, we introduce Exo-ViHa, an innovative 3D-printed exoskeleton system that enables users to collect data from a first-person perspective while providing real-time haptic feedback. This system combines a 3D-printed modular structure with a slam camera, a motion capture glove, and a wrist-mounted camera. Various dexterous hands can be installed at the end, enabling it to simultaneously collect the posture of the end effector, hand movements, and visual data. By leveraging the first-person perspective and direct interaction, the exoskeleton enhances the task realism and haptic feedback, improving the consistency between demonstrations and actual robot deployments. In addition, it has cross-platform compatibility with various robotic arms and dexterous hands. Experiments show that the system can significantly improve the success rate and efficiency of data collection for dexterous manipulation tasks.", "source": "curated", "first_author": "Chao", "url": "https://arxiv.org/abs/2503.01543", "fig": "figures/exo-viha.webp", "figs": ["figures/exo-viha-1.webp", "figures/exo-viha-2.webp", "figures/exo-viha-3.webp"]}, {"id": "falcon", "title": "FALCON: Learning Force-Adaptive Humanoid Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Yuanhang Zhang", "Yifu Yuan", "Prajwal Gurunath", "Tairan He", "Shayegan Omidshafiei", "Ali-akbar Agha-mohammadi", "Marcello Vendittelli", "Guanya Shi"], "corresponding": [], "institutions": ["Carnegie Mellon University"], "lab": "CMU LeCAR Lab (Guanya Shi)", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-10", "topics": ["force-adaptive control", "loco-manipulation", "dual-agent RL"], "paradigm": ["RL"], "method": ["decoupled upper/lower-body agents with force curriculum"], "projects": ["m2", "mabel"], "related": ["amo", "catch-it", "decowam", "deep-wbc", "demohlm", "fetchman", "h2compact", "intention-tracking", "kinematics-multipolicy", "phhi-survey", "resmimic", "umi-on-legs", "visualmimic", "wheeled-bilateral", "wheeled-multistage", "wheeled-posforce"], "arxiv": "2505.06776", "doi": "", "code": "", "site": "", "note": "In your library. The decoupled upper/lower-body formulation with an explicit external-force curriculum is directly applicable to MABEL carrying tasks.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/RL/falcon.pdf", "abstract": "Humanoid loco-manipulation holds transformative potential for daily service and industrial tasks, yet achieving precise, robust whole-body control with 3D end-effector force interaction remains a major challenge. Prior approaches are often limited to lightweight tasks or quadrupedal/wheeled platforms. To overcome these limitations, we propose FALCON, a dual-agent reinforcement-learning-based framework for robust force-adaptive humanoid loco-manipulation. FALCON decomposes whole-body control into two specialized agents: (1) a lower-body agent ensuring stable locomotion under external force disturbances, and (2) an upper-body agent precisely tracking end-effector positions with implicit adaptive force compensation. These two agents are jointly trained in simulation with a force curriculum that progressively escalates the magnitude of external force exerted on the end effector while respecting torque limits. Experiments demonstrate that, compared to the baselines, FALCON achieves 2x more accurate upper-body joint tracking, while maintaining robust locomotion under force disturbances and achieving faster training convergence. Moreover, FALCON enables policy training without embodiment-specific reward or curriculum tuning. Using the same training setup, we obtain policies that are deployed across multiple humanoids, enabling forceful loco-manipulation tasks such as transporting payloads (0-20N force), cart-pulling (0-100N), and door-opening (0-40N) in the real world.", "source": "curated", "first_author": "Yuanhang Zhang", "url": "https://arxiv.org/abs/2505.06776", "fig": "figures/falcon.webp", "figs": ["figures/falcon-1.webp"]}, {"id": "flare", "title": "FLARE: Robot Learning with Implicit World Modeling", "tree": "il", "branch": "world-models", "authors": ["Ruijie Zheng", "Jing Wang", "Scott Reed", "Johan Bjorck", "Yu Fang", "Fengyuan Hu", "et al."], "corresponding": [], "institutions": ["NVIDIA", "University of Maryland"], "lab": "NVIDIA GEAR", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-21", "topics": ["implicit world modelling", "latent future alignment"], "paradigm": ["world model", "IL"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["decowam", "gr2", "lawam", "mobilewam", "vjepa2", "vla-jepa"], "arxiv": "2505.15659", "doi": "", "code": "", "site": "", "note": "In your M2 learning_from_human_video folder. Aligns policy latents with future observation embeddings — world modelling as an auxiliary loss rather than a separate model.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/learning_from_human_video/2025_2505.15659v1_FLARE_Robot_Learning_with_Implicit_World_Modeling.pdf", "abstract": "We introduce $\\textbf{F}$uture $\\textbf{LA}$tent $\\textbf{RE}$presentation Alignment ($\\textbf{FLARE}$), a novel framework that integrates predictive latent world modeling into robot policy learning. By aligning features from a diffusion transformer with latent embeddings of future observations, $\\textbf{FLARE}$ enables a diffusion transformer policy to anticipate latent representations of future observations, allowing it to reason about long-term consequences while generating actions. Remarkably lightweight, $\\textbf{FLARE}$ requires only minimal architectural modifications -- adding a few tokens to standard vision-language-action (VLA) models -- yet delivers substantial performance gains. Across two challenging multitask simulation imitation learning benchmarks spanning single-arm and humanoid tabletop manipulation, $\\textbf{FLARE}$ achieves state-of-the-art performance, outperforming prior policy learning baselines by up to 26%. Moreover, $\\textbf{FLARE}$ unlocks the ability to co-train with human egocentric video demonstrations without action labels, significantly boosting policy generalization to a novel object with unseen geometry with as few as a single robot demonstration. Our results establish $\\textbf{FLARE}$ as a general and scalable approach for combining implicit world modeling with high-frequency robotic control.", "source": "curated", "first_author": "Ruijie Zheng", "url": "https://arxiv.org/abs/2505.15659", "fig": "figures/flare.webp", "figs": ["figures/flare-1.webp", "figures/flare-2.webp"]}, {"id": "openvla-oft", "title": "Fine-Tuning Vision-Language-Action Models: Optimizing Speed and Success", "tree": "il", "branch": "vla", "authors": ["Moo Jin Kim", "Chelsea Finn", "Percy Liang"], "corresponding": [], "institutions": ["Stanford University"], "lab": "Stanford IRIS", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-27", "topics": ["VLA fine-tuning", "action chunking", "parallel decoding"], "paradigm": ["VLA"], "method": ["continuous actions + parallel decoding + action chunking instead of autoregressive tokens"], "projects": [], "related": ["act", "knowledge-insulation", "openvla", "pi0", "vlm4vla"], "arxiv": "2502.19645", "doi": "", "code": "", "site": "", "note": "26x faster inference and higher success by dropping autoregressive action tokens. A useful corrective to the assumption that VLA quality lives in the language model.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent vision-language-action models (VLAs) build upon pretrained vision-language models and leverage diverse robot datasets to demonstrate strong task execution, language following ability, and semantic generalization. Despite these successes, VLAs struggle with novel robot setups and require fine-tuning to achieve good performance, yet how to most effectively fine-tune them is unclear given many possible strategies. In this work, we study key VLA adaptation design choices such as different action decoding schemes, action representations, and learning objectives for fine-tuning, using OpenVLA as our representative base model. Our empirical analysis informs an Optimized Fine-Tuning (OFT) recipe that integrates parallel decoding, action chunking, a continuous action representation, and a simple L1 regression-based learning objective to altogether improve inference efficiency, policy performance, and flexibility in the model's input-output specifications. We propose OpenVLA-OFT, an instantiation of this recipe, which sets a new state of the art on the LIBERO simulation benchmark, significantly boosting OpenVLA's average success rate across four task suites from 76.5% to 97.1% while increasing action generation throughput by 26$\\times$. In real-world evaluations, our fine-tuning recipe enables OpenVLA to successfully execute dexterous, high-frequency control tasks on a bimanual ALOHA robot and outperform other VLAs ($π_0$ and RDT-1B) fine-tuned using their default recipes, as well as strong imitation learning policies trained from scratch (Diffusion Policy and ACT) by up to 15% (absolute) in average success rate. We release code for OFT and pretrained model checkpoints at https://openvla-oft.github.io/.", "source": "curated", "first_author": "Moo Jin Kim", "url": "https://arxiv.org/abs/2502.19645", "fig": "figures/openvla-oft.webp", "figs": ["figures/openvla-oft-1.webp", "figures/openvla-oft-2.webp"]}, {"id": "gaia-2", "title": "GAIA-2: A Controllable Multi-View Generative World Model for Autonomous Driving", "tree": "rl", "branch": "model-based", "authors": ["Lloyd Russell", "Anthony Hu", "Lorenzo Bertoni", "George Fedoseev", "Jamie Shotton", "Elahe Arani", "Gianluca Corrado"], "corresponding": [], "institutions": ["Wayve"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-26", "topics": ["latent diffusion", "multi-view", "controllable generation", "ego-dynamics conditioning"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["cosmos", "genie3", "navdreamer", "navigation-world-models", "vista-wm"], "arxiv": "2503.20523", "doi": "10.48550/arxiv.2503.20523", "code": "", "site": "", "note": "<b>GAIA-2</b> (Wayve) — a controllable multi-view latent-diffusion world model conditioned on ego dynamics, other agents, environment and road semantics. The industrial reference for <em>controllability</em>: it is the clearest published account of how many conditioning channels a usable world model needs, which is more than most robotics WAMs currently expose.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Generative models offer a scalable and flexible paradigm for simulating complex environments, yet current approaches fall short in addressing the domain-specific requirements of autonomous driving - such as multi-agent interactions, fine-grained control, and multi-camera consistency. We introduce GAIA-2, Generative AI for Autonomy, a latent diffusion world model that unifies these capabilities within a single generative framework. GAIA-2 supports controllable video generation conditioned on a rich set of structured inputs: ego-vehicle dynamics, agent configurations, environmental factors, and road semantics. It generates high-resolution, spatiotemporally consistent multi-camera videos across geographically diverse driving environments (UK, US, Germany). The model integrates both structured conditioning and external latent embeddings (e.g., from a proprietary driving model) to facilitate flexible and semantically grounded scene synthesis. Through this integration, GAIA-2 enables scalable simulation of both common and rare driving scenarios, advancing the use of generative world models as a core tool in the development of autonomous systems. Videos are available at https://wayve.ai/thinking/gaia-2.", "source": "curated", "first_author": "Lloyd Russell", "url": "https://arxiv.org/abs/2503.20523", "fig": "figures/gaia-2.webp", "figs": ["figures/gaia-2-1.webp", "figures/gaia-2-2.webp"]}, {"id": "gcnt", "title": "GCNT: Graph-Based Transformer Policies for Morphology-Agnostic Reinforcement Learning", "tree": "rl", "branch": "foundations", "authors": ["Yingbo Luo", "Meibao Yao", "Xueming Xiao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-21", "topics": ["morphology-agnostic policy", "graph transformer", "zero-shot to unseen bodies"], "paradigm": ["RL"], "method": [], "projects": [], "related": ["changing-morph-dyn", "cross-humanoid-wbc", "getup-morphologies", "morph-hypernet", "transform2act"], "arxiv": "2505.15211", "doi": "10.48550/arxiv.2505.15211", "code": "", "site": "", "note": "A GCN-plus-transformer policy that accepts an arbitrary number of modules and generalises zero-shot to morphologies it never saw. The architecture answer to 'one policy, changing body' — which is exactly what a transforming robot needs, since its body is a different graph before and after.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Training a universal controller for robots with different morphologies is a promising research trend, since it can significantly enhance the robustness and resilience of the robotic system. However, diverse morphologies can yield different dimensions of state space and action space, making it difficult to comply with traditional policy networks. Existing methods address this issue by modularizing the robot configuration, while do not adequately extract and utilize the overall morphological information, which has been proven crucial for training a universal controller. To this end, we propose GCNT, a morphology-agnostic policy network based on improved Graph Convolutional Network (GCN) and Transformer. It exploits the fact that GCN and Transformer can handle arbitrary number of modules to achieve compatibility with diverse morphologies. Our key insight is that the GCN is able to efficiently extract morphology information of robots, while Transformer ensures that it is fully utilized by allowing each node of the robot to communicate this information directly. Experimental results show that our method can generate resilient locomotion behaviors for robots with different configurations, including zero-shot generalization to robot morphologies not seen during training. In particular, GCNT achieved the best performance on 8 tasks in the 2 standard benchmarks.", "source": "curated", "first_author": "Yingbo Luo", "url": "https://arxiv.org/abs/2505.15211", "fig": "figures/gcnt.webp", "figs": ["figures/gcnt-1.webp", "figures/gcnt-2.webp", "figures/gcnt-3.webp"]}, {"id": "generalist-ai", "title": "Generalist AI: GEN-0 and Harmonic Reasoning", "tree": "systems", "branch": "agentic", "authors": ["Generalist AI"], "corresponding": [], "institutions": ["Generalist AI"], "lab": "Generalist", "venue": "company blog", "venue_type": "industry", "year": 2025, "date": "2025-10-01", "topics": ["robot foundation model", "scaling laws", "dexterity"], "paradigm": ["VLA"], "method": [], "projects": [], "related": ["dyna", "dyna2", "groot-n1", "oxe", "pi05", "skild", "skild-scale", "sunday-gelato"], "arxiv": "", "doi": "", "code": "", "site": "https://generalistai.com/", "note": "Argues for robot-data scaling laws from a 270k-hour proprietary corpus and a 'harmonic reasoning' architecture that fuses thinking and acting at high rate. The scaling-law claim is the interesting one and the least verifiable. Unreviewed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Generalist AI", "url": "https://generalistai.com/"}, {"id": "genie3", "title": "Genie 3: A New Frontier for World Models", "tree": "il", "branch": "world-models", "authors": ["Google DeepMind"], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind", "venue": "blog", "venue_type": "industry", "year": 2025, "date": "2025-08-05", "topics": ["interactive world model", "real-time generation"], "paradigm": ["world model"], "method": [], "projects": [], "related": ["cosmos", "dreamerv4", "gaia-2", "genie"], "arxiv": "", "doi": "", "code": "", "site": "https://deepmind.google/discover/blog/genie-3-a-new-frontier-for-world-models/", "note": "Real-time interactive 720p world generation with minute-scale consistency. Industry announcement, not a reviewed paper — but it sets the bar for what 'train the agent inside the model' now means.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Google DeepMind", "url": "https://deepmind.google/discover/blog/genie-3-a-new-frontier-for-world-models/"}, {"id": "gigaworld-0", "title": "GigaWorld-0: World Models as Data Engine to Empower Embodied AI", "tree": "il", "branch": "world-models", "authors": ["GigaWorld Team", "Angen Ye", "Boyuan Wang", "Chaojun Ni", "Guan Huang", "Guosheng Zhao", "Haoyun Li", "Jiagang Zhu", "Kerui Li", "Mengyuan Xu", "Qiuping Deng", "Siting Wang", "Wenkang Qin", "Xinze Chen", "Xiaofeng Wang", "Yankai Wang", "Yu Cao", "Yifan Chang", "Yuan Xu", "Yun Ye", "Yang Wang", "Yukun Zhou", "Zhengyuan Zhang", "Zhehao Dong", "Zheng Zhu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-25", "topics": ["world models as a data engine", "large-scale synthetic embodied data"], "paradigm": ["world model"], "method": [], "projects": ["r2s2r"], "related": ["cosmos", "dreamgen", "prism", "robodream"], "arxiv": "2511.19861", "doi": "10.48550/arxiv.2511.19861", "code": "", "site": "", "note": "<b>GigaWorld-0</b> — world models explicitly as a <em>data engine</em> for embodied AI, at scale. The industrial-scale version of the DreamGen argument; cite it for the claim that synthetic rollouts are now a primary rather than supplementary data source.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models are emerging as a foundational paradigm for scalable, data-efficient embodied AI. In this work, we present GigaWorld-0, a unified world model framework designed explicitly as a data engine for Vision-Language-Action (VLA) learning. GigaWorld-0 integrates two synergistic components: GigaWorld-0-Video, which leverages large-scale video generation to produce diverse, texture-rich, and temporally coherent embodied sequences under fine-grained control of appearance, camera viewpoint, and action semantics; and GigaWorld-0-3D, which combines 3D generative modeling, 3D Gaussian Splatting reconstruction, physically differentiable system identification, and executable motion planning to ensure geometric consistency and physical realism. Their joint optimization enables the scalable synthesis of embodied interaction data that is visually compelling, spatially coherent, physically plausible, and instruction-aligned. Training at scale is made feasible through our efficient GigaTrain framework, which exploits FP8-precision and sparse attention to drastically reduce memory and compute requirements. We conduct comprehensive evaluations showing that GigaWorld-0 generates high-quality, diverse, and controllable data across multiple dimensions. Critically, VLA model (e.g., GigaBrain-0) trained on GigaWorld-0-generated data achieve strong real-world performance, significantly improving generalization and task success on physical robots without any real-world interaction during training.", "source": "curated", "first_author": "GigaWorld Team", "url": "https://arxiv.org/abs/2511.19861", "fig": "figures/gigaworld-0.webp", "figs": ["figures/gigaworld-0-1.webp", "figures/gigaworld-0-2.webp"]}, {"id": "hrdt", "title": "H-RDT: Human Manipulation Enhanced Bimanual Robotic Manipulation", "tree": "il", "branch": "human-video", "authors": ["Hongzhe Bi", "Lingxuan Wu", "Tianwei Lin", "Hengkai Tan", "Zhizhong Su", "Hang Su", "Jun Zhu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-07-31", "topics": ["human video pretraining", "bimanual manipulation"], "paradigm": ["IL", "diffusion"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["act", "egoscale", "lawm-3d", "rdt1b"], "arxiv": "2507.23523", "doi": "10.48550/arxiv.2507.23523", "code": "", "site": "", "note": "Egocentric human video with paired 3D hand poses as a behavioural prior for bimanual robot manipulation. Relevant to MABEL specifically — bimanual is where human priors should transfer best and where robot data is scarcest.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Imitation learning for robotic manipulation faces a fundamental challenge: the scarcity of large-scale, high-quality robot demonstration data. Recent robotic foundation models often pre-train on cross-embodiment robot datasets to increase data scale, while they face significant limitations as the diverse morphologies and action spaces across different robot embodiments make unified training challenging. In this paper, we present H-RDT (Human to Robotics Diffusion Transformer), a novel approach that leverages human manipulation data to enhance robot manipulation capabilities. Our key insight is that large-scale egocentric human manipulation videos with paired 3D hand pose annotations provide rich behavioral priors that capture natural manipulation strategies and can benefit robotic policy learning. We introduce a two-stage training paradigm: (1) pre-training on large-scale egocentric human manipulation data, and (2) cross-embodiment fine-tuning on robot-specific data with modular action encoders and decoders. Built on a diffusion transformer architecture with 2B parameters, H-RDT uses flow matching to model complex action distributions. Extensive evaluations encompassing both simulation and real-world experiments, single-task and multitask scenarios, as well as few-shot learning and robustness assessments, demonstrate that H-RDT outperforms training from scratch and existing state-of-the-art methods, including Pi0 and RDT, achieving significant improvements of 13.9% and 40.5% over training from scratch in simulation and real-world experiments, respectively. The results validate our core hypothesis that human manipulation data can serve as a powerful foundation for learning bimanual robotic manipulation policies.", "source": "curated", "first_author": "Hongzhe Bi", "url": "https://arxiv.org/abs/2507.23523", "fig": "figures/hrdt.webp", "figs": ["figures/hrdt-1.webp"]}, {"id": "h2compact", "title": "H2-COMPACT: Human-Humanoid Co-Manipulation via Adaptive Contact Trajectory Policies", "tree": "rl", "branch": "marl", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["New York University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-23", "topics": ["human-robot co-manipulation", "contact trajectories"], "paradigm": ["RL", "IL"], "method": [], "projects": ["m2"], "related": ["chorus", "decentralized-aerial", "falcon", "intention-tracking", "interact", "kinematics-multipolicy", "phhi-survey", "workspace-opt"], "arxiv": "2505.17627", "doi": "", "code": "", "site": "", "note": "From your M2 collaborative_transport folder. Human-in-the-loop co-manipulation — the HRI half of M2.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/collaborative_transport/2025_2505.17627v1_H2_COMPACT_Human_Humanoid_Co_Manipulation_via_Adaptive_Conta.pdf", "abstract": "We present a hierarchical policy-learning framework that enables a legged humanoid to cooperatively carry extended loads with a human partner using only haptic cues for intent inference. At the upper tier, a lightweight behavior-cloning network consumes six-axis force/torque streams from dual wrist-mounted sensors and outputs whole-body planar velocity commands that capture the leader's applied forces. At the lower tier, a deep-reinforcement-learning policy, trained under randomized payloads (0-3 kg) and friction conditions in Isaac Gym and validated in MuJoCo and on a real Unitree G1, maps these high-level twists to stable, under-load joint trajectories. By decoupling intent interpretation (force -> velocity) from legged locomotion (velocity -> joints), our method combines intuitive responsiveness to human inputs with robust, load-adaptive walking. We collect training data without motion-capture or markers, only synchronized RGB video and F/T readings, employing SAM2 and WHAM to extract 3D human pose and velocity. In real-world trials, our humanoid achieves cooperative carry-and-move performance (completion time, trajectory deviation, velocity synchrony, and follower-force) on par with a blindfolded human-follower baseline. This work is the first to demonstrate learned haptic guidance fused with full-body legged control for fluid human-humanoid co-manipulation. Code and videos are available on the H2-COMPACT website.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2505.17627", "fig": "figures/h2compact.webp", "figs": ["figures/h2compact-1.webp"]}, {"id": "handeye-delivery", "title": "Hand-Eye Autonomous Delivery: Learning Humanoid Navigation, Locomotion and Reaching", "tree": "rl", "branch": "loco-manip", "authors": ["Sirui Chen", "Yufei Ye", "Ziang Cao", "Jennifer Lew", "Xu Pei", "C. Karen Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-08-05", "topics": ["navigation", "locomotion and reaching in one policy"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["fetchman", "homer", "umi-on-legs", "visualmimic"], "arxiv": "2508.03068", "doi": "10.48550/arxiv.2508.03068", "code": "", "site": "", "note": "Learns navigation, locomotion and reaching together for an autonomous delivery task. The end-to-end framing — no separate nav stack — is the opposite bet from MABEL's current modular design and worth knowing as the alternative.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We propose Hand-Eye Autonomous Delivery (HEAD), a framework that learns navigation, locomotion, and reaching skills for humanoids, directly from human motion and vision perception data. We take a modular approach where the high-level planner commands the target position and orientation of the hands and eyes of the humanoid, delivered by the low-level policy that controls the whole-body movements. Specifically, the low-level whole-body controller learns to track the three points (eyes, left hand, and right hand) from existing large-scale human motion capture data while high-level policy learns from human data collected by Aria glasses. Our modular approach decouples the ego-centric vision perception from physical actions, promoting efficient learning and scalability to novel scenes. We evaluate our method both in simulation and in the real-world, demonstrating humanoid's capabilities to navigate and reach in complex environments designed for humans.", "source": "curated", "first_author": "Sirui Chen", "url": "https://arxiv.org/abs/2508.03068", "fig": "figures/handeye-delivery.webp", "figs": ["figures/handeye-delivery-1.webp", "figures/handeye-delivery-2.webp"]}, {"id": "helix", "title": "Helix: A Vision-Language-Action Model for Generalist Humanoid Control", "tree": "il", "branch": "vla", "authors": ["Figure AI"], "corresponding": [], "institutions": ["Figure AI"], "lab": "Figure AI", "venue": "blog", "venue_type": "industry", "year": 2025, "date": "2025-02-20", "topics": ["humanoid VLA", "dual-system", "upper-body control"], "paradigm": ["VLA"], "method": [], "projects": ["mabel"], "related": ["figure-fleet", "groot-n1", "pi0", "spatial-brain", "sunday-gelato"], "arxiv": "", "doi": "", "code": "", "site": "https://www.figure.ai/news/helix", "note": "Industry blog, not a paper — included because the S1/S2 split (7-9 Hz VLM, 200 Hz visuomotor) became a reference architecture and because you asked for startup blogs. Treat claims as unreviewed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Figure AI", "url": "https://www.figure.ai/news/helix"}, {"id": "homer", "title": "HoMeR: Learning In-the-Wild Mobile Manipulation via Hybrid Imitation and Whole-Body Control", "tree": "systems", "branch": "mobile-manip", "authors": ["Priya Sundaresan", "Rhea Malhotra", "Phoebe Miller", "Jiaming Hu", "Jeannette Bohg", "Dorsa Sadigh"], "corresponding": [], "institutions": ["Stanford University"], "lab": "Stanford IRIS / IPRL", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-06-02", "topics": ["hybrid imitation", "whole-body control", "in-the-wild"], "paradigm": ["IL"], "method": [], "projects": ["mabel"], "related": ["behavior-robot-suite", "handeye-delivery", "mobile-aloha", "stretch", "tidybot", "tidybot2"], "arxiv": "2506.01185", "doi": "", "code": "", "site": "", "note": "Hybrid action space — keypose for the coarse motion, dense for the contact-rich part. A sensible answer to 'the base and the arm want different action representations'.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce HoMeR, an imitation learning framework for mobile manipulation that combines whole-body control with hybrid action modes that handle both long-range and fine-grained motion, enabling effective performance on realistic in-the-wild tasks. At its core is a fast, kinematics-based whole-body controller that maps desired end-effector poses to coordinated motion across the mobile base and arm. Within this reduced end-effector action space, HoMeR learns to switch between absolute pose predictions for long-range movement and relative pose predictions for fine-grained manipulation, offloading low-level coordination to the controller and focusing learning on task-level decisions. We deploy HoMeR on a holonomic mobile manipulator with a 7-DoF arm in a real home. We compare HoMeR to baselines without hybrid actions or whole-body control across 3 simulated and 3 real household tasks such as opening cabinets, sweeping trash, and rearranging pillows. Across tasks, HoMeR achieves an overall success rate of 79.17% using just 20 demonstrations per task, outperforming the next best baseline by 29.17 on average. HoMeR is also compatible with vision-language models and can leverage their internet-scale priors to better generalize to novel object appearances, layouts, and cluttered scenes. In summary, HoMeR moves beyond tabletop settings and demonstrates a scalable path toward sample-efficient, generalizable manipulation in everyday indoor spaces. Code, videos, and supplementary material are available at: http://homer-manip.github.io", "source": "curated", "first_author": "Priya Sundaresan", "url": "https://arxiv.org/abs/2506.01185", "fig": "figures/homer.webp", "figs": ["figures/homer-1.webp", "figures/homer-2.webp"]}, {"id": "iRe-VLA", "title": "Improving Vision-Language-Action Model with Online Reinforcement Learning", "tree": "rl", "branch": "post-training", "authors": ["Yanjiang Guo", "Jianke Zhang", "Xiaoyu Chen", "Xiang Ji", "Jianyu Chen"], "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Qi Zhi Institute"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2025-01-28", "topics": ["VLA post-training", "online RL"], "paradigm": ["RL", "VLA"], "method": ["alternating RL on the action head and supervised fine-tuning of the whole VLA"], "projects": [], "related": ["conrft", "knowledge-insulation", "openvla", "pi0", "pi06", "pld", "ript-vla"], "arxiv": "2501.16664", "doi": "", "code": "", "site": "", "note": "Directly on your RL-post-training idea. The alternation matters: pure RL on a full VLA destabilizes the language backbone, so they interleave it with supervised phases.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent studies have successfully integrated large vision-language models (VLMs) into low-level robotic control by supervised fine-tuning (SFT) with expert robotic datasets, resulting in what we term vision-language-action (VLA) models. Although the VLA models are powerful, how to improve these large models during interaction with environments remains an open question. In this paper, we explore how to further improve these VLA models via Reinforcement Learning (RL), a commonly used fine-tuning technique for large models. However, we find that directly applying online RL to large VLA models presents significant challenges, including training instability that severely impacts the performance of large models, and computing burdens that exceed the capabilities of most local machines. To address these challenges, we propose iRe-VLA framework, which iterates between Reinforcement Learning and Supervised Learning to effectively improve VLA models, leveraging the exploratory benefits of RL while maintaining the stability of supervised learning. Experiments in two simulated benchmarks and a real-world manipulation suite validate the effectiveness of our method.", "source": "curated", "first_author": "Yanjiang Guo", "url": "https://arxiv.org/abs/2501.16664", "fig": "figures/iRe-VLA.webp", "figs": ["figures/iRe-VLA-1.webp", "figures/iRe-VLA-2.webp", "figures/iRe-VLA-3.webp"]}, {"id": "ript-vla", "title": "Interactive Post-Training for Vision-Language-Action Models", "tree": "rl", "branch": "post-training", "authors": ["Shuhan Tan", "Kairan Dou", "Yue Zhao", "Philipp Krähenbühl"], "corresponding": [], "institutions": ["UT Austin"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-05-22", "topics": ["VLA post-training", "sparse reward", "interactive RL"], "paradigm": ["RL", "VLA"], "method": ["leave-one-out advantage over sampled rollouts", "binary success reward only"], "projects": [], "related": ["conrft", "expo-ft", "hand-in-loop", "hil-serl", "iRe-VLA", "pi0", "pi06", "pld", "rlinf-vla", "stare-vla"], "arxiv": "2505.17016", "doi": "", "code": "", "site": "", "note": "Post-trains a VLA with nothing but binary task success — no reward model, no value function. Practical because success is the one signal you can always instrument.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce RIPT-VLA, a simple and scalable reinforcement-learning-based interactive post-training paradigm that fine-tunes pretrained Vision-Language-Action (VLA) models using only sparse binary success rewards. Existing VLA training pipelines rely heavily on offline expert demonstration data and supervised imitation, limiting their ability to adapt to new tasks and environments under low-data regimes. RIPT-VLA addresses this by enabling interactive post-training with a stable policy optimization algorithm based on dynamic rollout sampling and leave-one-out advantage estimation. RIPT-VLA has the following characteristics. First, it applies to various VLA models, resulting in an improvement on the lightweight QueST model by 21.2%, and the 7B OpenVLA-OFT model to an unprecedented 97.5% success rate. Second, it is computationally efficient and data-efficient: with only one demonstration, RIPT-VLA enables an unworkable SFT model (4%) to succeed with a 97% success rate within 15 iterations. Furthermore, we demonstrate that the policy learned by RIPT-VLA generalizes across different tasks and scenarios and is robust to the initial state context. These results highlight RIPT-VLA as a practical and effective paradigm for post-training VLA models through minimal supervision.", "source": "curated", "first_author": "Shuhan Tan", "url": "https://arxiv.org/abs/2505.17016", "fig": "figures/ript-vla.webp"}, {"id": "kinematics-multipolicy", "title": "Kinematics-Aware Multi-Policy Reinforcement Learning for Force-Capable Humanoid Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Kaiyan Xiao", "Zihan Xu", "Zhe Cheng", "Chengju Liu", "Qijun Chen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-26", "topics": ["force-capable loco-manipulation", "multi-policy RL"], "paradigm": ["RL"], "method": [], "projects": ["m2", "mabel"], "related": ["amo", "falcon", "h2compact", "resmimic"], "arxiv": "2511.21169", "doi": "10.48550/arxiv.2511.21169", "code": "", "site": "", "note": "Kinematics-aware multi-policy RL for force-capable loco-manipulation. Read alongside [[falcon]] — both are attacking the same problem (a humanoid that can actually push and pull) from different structural choices.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humanoid robots, with their human-like morphology, hold great potential for industrial applications. However, existing loco-manipulation methods primarily focus on dexterous manipulation, falling short of the combined requirements for dexterity and proactive force interaction in high-load industrial scenarios. To bridge this gap, we propose a reinforcement learning-based framework with a decoupled three-stage training pipeline, consisting of an upper-body policy, a lower-body policy, and a delta-command policy. To accelerate upper-body training, a heuristic reward function is designed. By implicitly embedding forward kinematics priors, it enables the policy to converge faster and achieve superior performance. For the lower body, a force-based curriculum learning strategy is developed, enabling the robot to actively exert and regulate interaction forces with the environment.", "source": "curated", "first_author": "Kaiyan Xiao", "url": "https://arxiv.org/abs/2511.21169", "fig": "figures/kinematics-multipolicy.webp"}, {"id": "lawm", "title": "Latent Action Pretraining Through World Modeling", "tree": "il", "branch": "world-models", "authors": ["Bahey Tharwat", "Yara Nasser", "Ali Abouzeid", "Ian Reid"], "corresponding": [], "institutions": ["MBZUAI"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-22", "topics": ["latent action pretraining through world modeling", "self-supervised", "unlabelled data"], "paradigm": ["world model", "IL"], "method": [], "projects": ["mabel", "r2s2r", "wam-tactile"], "related": ["adaworld", "dreamdojo", "lapa", "lawam", "lawm-3d"], "arxiv": "2509.18428", "doi": "10.48550/arxiv.2509.18428", "code": "", "site": "", "note": "<b>LAWM</b> — latent-action pretraining where the supervisory signal <em>is</em> the world model: learn to predict the future, and the latent that best explains the transition becomes the action. The cleaner formulation of LAPA's idea, and the one to cite if you want to argue that world modelling and action representation are the same problem.", "aka": "LAWM", "stars": 2, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models have gained popularity for learning robotic manipulation tasks that follow language instructions. State-of-the-art VLAs, such as OpenVLA and $π_{0}$, were trained on large-scale, manually labeled action datasets collected through teleoperation. More recent approaches, including LAPA and villa-X, introduce latent action representations that enable unsupervised pretraining on unlabeled datasets by modeling abstract visual changes between frames. Although these methods have shown strong results, their large model sizes make deployment in real-world settings challenging. In this work, we propose LAWM, a model-agnostic framework to pretrain imitation learning models in a self-supervised way, by learning latent action representations from unlabeled video data through world modeling. These videos can be sourced from robot recordings or videos of humans performing actions with everyday objects. Our framework is able to transfer learned knowledge across tasks, environments, and embodiments. It outperforms models pretrained with ground-truth robot actions and other similar pretraining methods on the LIBERO benchmark and real-world setup, while being efficient and practical for real-world settings.", "source": "curated", "first_author": "Bahey Tharwat", "url": "https://arxiv.org/abs/2509.18428", "fig": "figures/lawm.webp", "figs": ["figures/lawm-1.webp", "figures/lawm-2.webp"], "arxiv_url": "https://arxiv.org/abs/2509.18428"}, {"id": "climrs", "title": "Leveraging Adaptive Group Negotiation for Heterogeneous Multi-Robot Collaboration with Large Language Models", "tree": "rl", "branch": "marl", "authors": ["Siqi Song", "叶悬冰", "Ru Li", "Yuqiang Li", "Shijie Wang", "Biqing Qi"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-29", "topics": ["group negotiation", "heterogeneous teams", "LLM coordination"], "paradigm": ["LLM", "MARL"], "method": [], "projects": ["m2"], "related": ["chorus", "mappo", "roco"], "arxiv": "2602.06967", "doi": "10.48550/arxiv.2602.06967", "code": "", "site": "", "note": "Adaptive group negotiation for heterogeneous teams, reporting >40% higher efficiency on complex tasks without losing success on simple ones. The efficiency-vs-robustness tradeoff is the one M2 will actually have to manage.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Multi-robot collaboration tasks often require heterogeneous robots to work together over long horizons under spatial constraints and environmental uncertainties. Although Large Language Models (LLMs) excel at reasoning and planning, their potential for coordinated control has not been fully explored. Inspired by human teamwork, we present CLiMRS (Cooperative Large-Language-Model-Driven Heterogeneous Multi-Robot System), an adaptive group negotiation framework among LLMs for multi-robot collaboration. This framework pairs each robot with an LLM agent and dynamically forms subgroups through a general proposal planner. Within each subgroup, a subgroup manager leads perception-driven multi-LLM discussions to get commands for actions. Feedback is provided by both robot execution outcomes and environment changes. This grouping-planning-execution-feedback loop enables efficient planning and robust execution. To evaluate these capabilities, we introduce CLiMBench, a heterogeneous multi-robot benchmark of challenging assembly tasks. Our experiments show that CLiMRS surpasses the best baseline, achieving over 40% higher efficiency on complex tasks without sacrificing success on simpler ones. Overall, our results demonstrate that leveraging human-inspired group formation and negotiation principles significantly enhances the efficiency of heterogeneous multi-robot collaboration. Our code is available here: https://github.com/song-siqi/CLiMRS.", "source": "curated", "first_author": "Siqi Song", "url": "https://arxiv.org/abs/2602.06967", "fig": "figures/climrs.webp", "figs": ["figures/climrs-1.webp", "figures/climrs-2.webp"]}, {"id": "manflow", "title": "ManiFlow: A General Robot Manipulation Policy via Consistency Flow Training", "tree": "il", "branch": "visuomotor", "authors": ["Ge Yan", "Jiyue Zhu", "Yuquan Deng", "Shiqi Yang", "Ri-Zhao Qiu", "Xuxin Cheng", "Marius Memmel", "Ranjay Krishna", "Ankit Goyal", "Xiaolong Wang", "Dieter Fox"], "corresponding": [], "institutions": ["UC San Diego", "University of Washington", "NVIDIA"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-09-01", "topics": ["flow matching", "general manipulation policy", "proceedings 2025"], "paradigm": ["IL", "flow matching"], "method": [], "projects": [], "related": ["conrft", "consistency-policy", "dp", "dp3", "pi0"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "From your CoRL 2025 folder. Flow matching is quietly displacing DDPM as the action decoder — same multimodality, far fewer inference steps.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/166_ManiFlow_A_General_Robot_M.pdf", "abstract": "", "source": "curated", "first_author": "Ge Yan", "url": "https://arxiv.org/abs/2509.01819", "fig": "figures/x-maniflowageneralrobotmanipulationpolicy.webp"}, {"id": "maniskill", "title": "ManiSkill3: GPU Parallelized Robotics Simulation and Rendering for Generalizable Embodied AI", "tree": "systems", "branch": "sim-bench", "authors": ["Stone Tao", "Fanbo Xiang", "Arth Shukla", "Yuzhe Qin", "Xander Hinrichsen", "et al."], "corresponding": [], "institutions": ["UC San Diego"], "lab": "UCSD Hao Su Lab", "venue": "RSS", "venue_type": "conference", "year": 2025, "date": "2024-10-01", "topics": ["GPU simulation", "benchmark", "generalizable manipulation"], "paradigm": ["simulator", "benchmark"], "method": [], "projects": [], "related": ["genesis", "isaaclab", "robocasa"], "arxiv": "2410.00425", "doi": "", "code": "", "site": "", "note": "Parallel rendering as well as parallel physics, which matters when your policy is vision-based.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Simulation has enabled unprecedented compute-scalable approaches to robot learning. However, many existing simulation frameworks typically support a narrow range of scenes/tasks and lack features critical for scaling generalizable robotics and sim2real. We introduce and open source ManiSkill3, the fastest state-visual GPU parallelized robotics simulator with contact-rich physics targeting generalizable manipulation. ManiSkill3 supports GPU parallelization of many aspects including simulation+rendering, heterogeneous simulation, pointclouds/voxels visual input, and more. Simulation with rendering on ManiSkill3 can run 10-1000x faster with 2-3x less GPU memory usage than other platforms, achieving up to 30,000+ FPS in benchmarked environments due to minimal python/pytorch overhead in the system, simulation on the GPU, and the use of the SAPIEN parallel rendering system. Tasks that used to take hours to train can now take minutes. We further provide the most comprehensive range of GPU parallelized environments/tasks spanning 12 distinct domains including but not limited to mobile manipulation for tasks such as drawing, humanoids, and dextrous manipulation in realistic scenes designed by artists or real-world digital twins. In addition, millions of demonstration frames are provided from motion planning, RL, and teleoperation. ManiSkill3 also provides a comprehensive set of baselines that span popular RL and learning-from-demonstrations algorithms.", "source": "curated", "first_author": "Stone Tao", "url": "https://arxiv.org/abs/2410.00425", "fig": "figures/maniskill.webp", "figs": ["figures/maniskill-1.webp", "figures/maniskill-2.webp", "figures/maniskill-3.webp"]}, {"id": "miccheck", "title": "MicCheck: Repurposing Off-the-Shelf Pin Microphones for Easy and Low-Cost Contact Sensing", "tree": "systems", "branch": "tactile-hw", "authors": ["Oh", "Steven", "Inui", "Tai", "Kuan", "Magdeline", "Lin", "Jia-Yeu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-23", "topics": ["pin microphones", "low-cost contact sensing"], "paradigm": ["hardware", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "sonicsense", "vibecheck"], "arxiv": "2511.18299", "doi": "10.48550/arxiv.2511.18299", "code": "", "site": "", "note": "Off-the-shelf pin microphones repurposed as contact sensors. The cheapest possible entry point to audio sensing — worth building in an afternoon before designing anything custom.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Robotic manipulation tasks are contact-rich, yet most imitation learning (IL) approaches rely primarily on vision, which struggles to capture stiffness, roughness, slip, and other fine interaction cues. Tactile signals can address this gap, but existing sensors often require expensive, delicate, or integration-heavy hardware. In this work, we introduce MicCheck, a plug-and-play acoustic sensing approach that repurposes an off-the-shelf Bluetooth pin microphone as a low-cost contact sensor. The microphone clips into a 3D-printed gripper insert and streams audio via a standard USB receiver, requiring no custom electronics or drivers. Despite its simplicity, the microphone provides signals informative enough for both perception and control. In material classification, it achieves 92.9% accuracy on a 10-class benchmark across four interaction types (tap, knock, slow press, drag). For manipulation, integrating pin microphone into an IL pipeline with open source hardware improves the success rate on picking and pouring task from 0.40 to 0.80 and enables reliable execution of contact-rich skills such as unplugging and sound-based sorting. Compared with high-resolution tactile sensors, pin microphones trade spatial detail for cost and ease of integration, offering a practical pathway for deploying acoustic contact sensing in low-cost robot setups.", "source": "curated", "first_author": "Oh", "url": "https://arxiv.org/abs/2511.18299", "fig": "figures/miccheck.webp", "figs": ["figures/miccheck-1.webp", "figures/miccheck-2.webp", "figures/miccheck-3.webp"]}, {"id": "groot-n15", "title": "NVIDIA Isaac GR00T N1.5: Improved Open Foundation Model for Humanoid Robots", "tree": "systems", "branch": "vla", "authors": ["NVIDIA GEAR Lab"], "corresponding": [], "institutions": ["NVIDIA"], "lab": "NVIDIA GEAR Lab (Jim Fan, Yuke Zhu)", "venue": "tech report", "venue_type": "industry", "year": 2025, "date": "2025-05-01", "topics": ["humanoid foundation model", "synthetic data co-training"], "paradigm": ["VLA", "diffusion"], "method": [], "projects": ["mabel"], "related": ["cosmos-reason", "dreamgen", "groot-n1", "groot-n16"], "arxiv": "", "doi": "", "code": "", "site": "https://research.nvidia.com/labs/gear/gr00t-n1_5/", "note": "N1 with a refined VLM and, more importantly, a training mix of real, simulated and DreamGen-generated synthetic data. The interesting claim is that neural trajectories carry enough signal to improve a foundation model — which is the DreamGen thesis, productised.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "NVIDIA GEAR Lab", "url": "https://research.nvidia.com/labs/gear/gr00t-n1_5/"}, {"id": "particle-grid-dynamics", "title": "Particle-Grid Neural Dynamics for Learning Deformable Object Models from RGB-D Videos", "tree": "il", "branch": "world-models", "authors": ["Kaifeng Zhang", "Baoyu Li", "Kris Hauser", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-06-18", "topics": ["particle-grid hybrid", "RGB-D supervision", "learned digital twin"], "paradigm": ["model-based RL"], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": ["adaptigraph", "bab-nd", "gs-dynamics", "phystwin", "pointworld", "robocraft"], "arxiv": "2506.15680", "doi": "10.48550/arxiv.2506.15680", "code": "", "site": "", "note": "<b>Particle-Grid Neural Dynamics</b> — particles carry the object, a spatial grid discretizes the space between them, and Gaussian splats render the result, all learned from RGB-D video. The hybrid exists because pure particle methods scale badly in neighbourhood queries and pure grid methods blur boundaries; this is the standard numerical trade-off borrowed into a learned model.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Modeling the dynamics of deformable objects is challenging due to their diverse physical properties and the difficulty of estimating states from limited visual information. We address these challenges with a neural dynamics framework that combines object particles and spatial grids in a hybrid representation. Our particle-grid model captures global shape and motion information while predicting dense particle movements, enabling the modeling of objects with varied shapes and materials. Particles represent object shapes, while the spatial grid discretizes the 3D space to ensure spatial continuity and enhance learning efficiency. Coupled with Gaussian Splattings for visual rendering, our framework achieves a fully learning-based digital twin of deformable objects and generates 3D action-conditioned videos. Through experiments, we demonstrate that our model learns the dynamics of diverse objects -- such as ropes, cloths, stuffed animals, and paper bags -- from sparse-view RGB-D recordings of robot-object interactions, while also generalizing at the category level to unseen instances. Our approach outperforms state-of-the-art learning-based and physics-based simulators, particularly in scenarios with limited camera views. Furthermore, we showcase the utility of our learned models in model-based planning, enabling goal-conditioned object manipulation across a range of tasks. The project page is available at https://kywind.github.io/pgnd .", "source": "curated", "first_author": "Kaifeng Zhang", "url": "https://arxiv.org/abs/2506.15680", "fig": "figures/particle-grid-dynamics.webp", "figs": ["figures/particle-grid-dynamics-1.webp"]}, {"id": "physgen3d", "title": "PhysGen3D: Crafting a Miniature Interactive World from a Single Image", "tree": "systems", "branch": "sim-bench", "authors": ["Boyuan Chen", "Hanxiao Jiang", "Shaowei Liu", "Saurabh Gupta", "Yunzhu Li", "Hao Zhao", "Shenlong Wang"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign", "Tsinghua University"], "lab": "RoboPIL (Yunzhu Li)", "venue": "CVPR", "venue_type": "conference", "year": 2025, "date": "2025-03-26", "topics": ["single image to interactive 3D scene", "amodal completion", "physics"], "paradigm": ["simulator"], "method": [], "projects": ["r2s2r"], "related": ["acdc", "phystwin", "physworld", "robosnap"], "arxiv": "2503.20746", "doi": "10.1109/CVPR52734.2025.00579", "code": "", "site": "", "note": "<b>PhysGen3D</b> (CVPR 2025) — one image to an amodal, camera-centric, physically interactive 3D scene. The extreme end of the reconstruction-cost spectrum; useful as the cheap baseline you must beat before justifying multi-view egocentric capture.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Envisioning physically plausible outcomes from a single image requires a deep understanding of the world's dynamics. To address this, we introduce PhysGen3D, a novel framework that transforms a single image into an amodal, camera-centric, interactive 3D scene. By combining advanced image-based geometric and semantic understanding with physics-based simulation, PhysGen3D creates an interactive 3D world from a static image, enabling us to \"imagine\" and simulate future scenarios based on user input. At its core, PhysGen3D estimates 3D shapes, poses, physical and lighting properties of objects, thereby capturing essential physical attributes that drive realistic object interactions. This framework allows users to specify precise initial conditions, such as object speed or material properties, for enhanced control over generated video outcomes. We evaluate PhysGen3D's performance against closed-source state-of-the-art (SOTA) image-to-video models, including Pika, Kling, and Gen-3, showing PhysGen3D's capacity to generate videos with realistic physics while offering greater flexibility and fine-grained control. Our results show that PhysGen3D achieves a unique balance of photorealism, physical plausibility, and user-driven interactivity, opening new possibilities for generating dynamic, physics-grounded video from an image.", "source": "curated", "first_author": "Boyuan Chen", "url": "https://arxiv.org/abs/2503.20746", "fig": "figures/physgen3d.webp", "figs": ["figures/physgen3d-1.webp", "figures/physgen3d-2.webp"]}, {"id": "add", "title": "Physics-Based Motion Imitation with Adversarial Differential Discriminators", "tree": "rl", "branch": "motion-imitation", "authors": ["Ziyu Zhang", "Sergey Bashkirov", "Dun Yang", "Michael Taylor", "Xue Bin Peng"], "corresponding": [], "institutions": ["Simon Fraser University", "Sony"], "lab": "", "venue": "SIGGRAPH Asia", "venue_type": "conference", "year": 2025, "date": "2025-05-08", "topics": ["adversarial imitation", "reward design"], "paradigm": ["RL", "adversarial IL"], "method": ["differential discriminator on state transitions", "removes manual reward terms"], "projects": [], "related": ["amp", "deepmimic", "gail"], "arxiv": "2505.04961", "doi": "", "code": "", "site": "", "note": "In your ~/Desktop/papers/RL. Gets rid of the hand-tuned multi-term tracking reward — the part of DeepMimic-style pipelines that costs the most engineer-hours.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Multi-objective optimization problems, which require the simultaneous optimization of multiple objectives, are prevalent across numerous applications. Existing multi-objective optimization methods often rely on manually-tuned aggregation functions to formulate a joint optimization objective. The performance of such hand-tuned methods is heavily dependent on careful weight selection, a time-consuming and laborious process. These limitations also arise in the setting of reinforcement-learning-based motion tracking methods for physically simulated characters, where intricately crafted reward functions are typically used to achieve high-fidelity results. Such solutions not only require domain expertise and significant manual tuning, but also limit the applicability of the resulting reward function across diverse skills. To bridge this gap, we present a novel adversarial multi-objective optimization technique that is broadly applicable to a range of multi-objective reinforcement-learning tasks, including motion tracking. Our proposed Adversarial Differential Discriminator (ADD) receives a single positive sample, yet is still effective at guiding the optimization process. We demonstrate that our technique can enable characters to closely replicate a variety of acrobatic and agile behaviors, achieving comparable quality to state-of-the-art motion-tracking methods, without relying on manually-designed reward functions. Code and results are available at https://add-moo.github.io/.", "source": "curated", "first_author": "Ziyu Zhang", "url": "https://arxiv.org/abs/2505.04961", "fig": "figures/add.webp", "figs": ["figures/add-1.webp", "figures/add-2.webp"]}, {"id": "polaris", "title": "PolaRiS: Scalable Real-to-Sim Evaluations for Generalist Robot Policies", "tree": "systems", "branch": "sim-bench", "authors": ["Arhan Jain", "Mingtong Zhang", "Kanav Arora", "William Chen", "Marcel Torne", "Muhammad Zubair Irshad", "Sergey Zakharov", "Yue Wang", "Sergey Levine", "Chelsea Finn", "Wei-Chiu Ma", "Dhruv Shah", "Abhishek Gupta", "Karl Pertsch"], "corresponding": [], "institutions": [], "lab": "", "venue": "Robotics: Science and Systems XXII", "venue_type": "preprint", "year": 2025, "date": "2025-12-18", "topics": ["real-to-sim evaluation", "generalist policy benchmarking"], "paradigm": ["simulator", "benchmark"], "method": [], "projects": ["r2s2r"], "related": ["egoengine", "harbor", "r2s-ego"], "arxiv": "2512.16881", "doi": "10.15607/rss.2026.xxii.062", "code": "", "site": "", "note": "PolaRiS uses real-to-sim as an *evaluation* substrate rather than a training one — build the twin so you can benchmark policies cheaply. Worth adopting as your evaluation harness even if your training story differs.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "A significant challenge for robot learning research is our ability to accurately measure and compare the performance of robot policies. Benchmarking in robotics is historically challenging due to the stochasticity, reproducibility, and time-consuming nature of real-world rollouts. This challenge is exacerbated for recent generalist policies, which has to be evaluated across a wide variety of scenes and tasks. Evaluation in simulation offers a scalable complement to real world evaluations, but the visual and physical domain gap between existing simulation benchmarks and the real world has made them an unreliable signal for policy improvement. Furthermore, building realistic and diverse simulated environments has traditionally required significant human effort and expertise. To bridge the gap, we introduce Policy Evaluation and Environment Reconstruction in Simulation (PolaRiS), a scalable real-to-sim framework for high-fidelity simulated robot evaluation. PolaRiS utilizes neural reconstruction methods to turn short video scans of real-world scenes into interactive simulation environments. Additionally, we develop a simple simulation data co-training recipe that bridges remaining real-to-sim gaps and enables zero-shot evaluation in unseen simulation environments. Through extensive paired evaluations between simulation and the real world, we demonstrate that PolaRiS evaluations provide a much stronger correlation to real world generalist policy performance than existing simulated benchmarks. Its simplicity also enables rapid creation of diverse simulated environments. As such, this work takes a step towards distributed and democratized evaluation for the next generation of robotic foundation models.", "source": "curated", "first_author": "Arhan Jain", "url": "https://doi.org/10.15607/rss.2026.xxii.062", "arxiv_url": "https://arxiv.org/abs/2512.16881", "fig": "figures/polaris.webp", "figs": ["figures/polaris-1.webp"]}, {"id": "rdt1b", "title": "RDT-1B: a Diffusion Foundation Model for Bimanual Manipulation", "tree": "il", "branch": "vla", "authors": ["Songming Liu", "Lingxuan Wu", "Bangguo Yu", "Hengkai Tan", "Huayu Chen", "Zhengyi Wang", "Chengyang Xu", "Hang Su", "Jun Zhu"], "corresponding": ["Jun Zhu"], "institutions": ["Tsinghua University"], "lab": "Tsinghua TSAIL", "venue": "ICLR", "venue_type": "conference", "year": 2025, "date": "2024-10-10", "topics": ["bimanual foundation model", "diffusion transformer"], "paradigm": ["VLA", "diffusion"], "method": [], "projects": ["mabel"], "related": ["act", "groot-n1", "hrdt", "pi0"], "arxiv": "2410.07864", "doi": "", "code": "", "site": "", "note": "Largest open bimanual diffusion foundation model at release, with a physically-interpretable unified action space across embodiments.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Bimanual manipulation is essential in robotics, yet developing foundation models is extremely challenging due to the inherent complexity of coordinating two robot arms (leading to multi-modal action distributions) and the scarcity of training data. In this paper, we present the Robotics Diffusion Transformer (RDT), a pioneering diffusion foundation model for bimanual manipulation. RDT builds on diffusion models to effectively represent multi-modality, with innovative designs of a scalable Transformer to deal with the heterogeneity of multi-modal inputs and to capture the nonlinearity and high frequency of robotic data. To address data scarcity, we further introduce a Physically Interpretable Unified Action Space, which can unify the action representations of various robots while preserving the physical meanings of original actions, facilitating learning transferrable physical knowledge. With these designs, we managed to pre-train RDT on the largest collection of multi-robot datasets to date and scaled it up to 1.2B parameters, which is the largest diffusion-based foundation model for robotic manipulation. We finally fine-tuned RDT on a self-created multi-task bimanual dataset with over 6K+ episodes to refine its manipulation capabilities. Experiments on real robots demonstrate that RDT significantly outperforms existing methods. It exhibits zero-shot generalization to unseen objects and scenes, understands and follows language instructions, learns new skills with just 1~5 demonstrations, and effectively handles complex, dexterous tasks. We refer to https://rdt-robotics.github.io/rdt-robotics/ for the code and videos.", "source": "curated", "first_author": "Songming Liu", "url": "https://arxiv.org/abs/2410.07864", "fig": "figures/rdt1b.webp", "figs": ["figures/rdt1b-1.webp", "figures/rdt1b-2.webp"]}, {"id": "rlinf-vla", "title": "RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models", "tree": "rl", "branch": "post-training", "authors": ["Hongzhi Zang", "Mingjie Wei", "Si Xu", "Yongji Wu", "Zhen Guo", "Yuanqing Wang", "Hao Lin", "Peihong Wang", "Liangzhi Shi", "Yuqing Xie", "Zhexuan Xu", "Zhihao Liu", "Kang Chen", "Wenhao Tang", "Quanlu Zhang", "Weinan Zhang", "Chao Yu", "Yu Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2025, "date": "2025-10-08", "topics": ["RL infrastructure for VLA", "unified framework"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["pld", "ript-vla", "serl"], "arxiv": "2510.06710", "doi": "", "code": "", "site": "", "note": "RLinf-VLA is the plumbing — a unified, efficient framework for RL on VLAs, accepted to RSS 2026. If you are going to do this, start from someone else's infrastructure rather than writing your own rollout loop.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent studies have demonstrated the potential of reinforcement learning (RL) to improve the task performance of vision-language-action (VLA) models through interaction. However, current efforts remain fragmented, lacking a unified platform for fair comparison across architectures and algorithms, as well as an efficient system design for scalable training. Therefore, we present RLinf-VLA, a unified and efficient framework for scalable RL training of VLA models. RLinf-VLA standardizes the integration of diverse VLA architectures, RL algorithms, and heterogeneous simulators through a unified interface, enabling extensibility and reproducibility. To improve efficiency, the framework adopts a flexible resource allocation architecture for rendering, inference, and training in RL pipelines. In particular, RLinf-VLA introduces a hybrid fine-grained pipeline allocation strategy that achieves a 1.61$\\times$-1.88$\\times$ training speedup on ManiSkill. Using this framework, RL-trained models achieve strong performance across embodied benchmarks, including 98.11% success on 130 LIBERO tasks, 97.66% success on 25 ManiSkill tasks, and 84.63% average success across 6 RoboTwin tasks. In addition, RLinf-VLA distills a set of effective practices for RL-based VLA training. We envision RLinf-VLA as a foundational framework for efficient, unified, and reproducible research in embodied intelligence.", "source": "curated", "first_author": "Hongzhi Zang", "url": "https://arxiv.org/abs/2510.06710", "fig": "figures/rlinf-vla.webp"}, {"id": "real-deep-research", "title": "Real Deep Research for AI, Robotics and Beyond", "tree": "systems", "branch": "agentic", "authors": ["Xueyan Zou", "Jianglong Ye", "Hao Zhang", "Xiaoyu Xiang", "Mingyu Ding", "Zhaojing Yang", "Yong Jae Lee", "Zhuowen Tu", "Sifei Liu", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-23", "topics": ["automated literature analysis", "research mapping"], "paradigm": ["LLM"], "method": [], "projects": [], "related": ["agent-laboratory", "ai-scientist"], "arxiv": "2510.20809", "doi": "", "code": "", "site": "", "note": "Real Deep Research: agentic analysis of the AI/robotics literature at scale. Amusingly, it is the automated version of what this Atlas does by hand — worth reading for the taxonomy method.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "With the rapid growth of research in AI and robotics now producing over 10,000 papers annually it has become increasingly difficult for researchers to stay up to date. Fast evolving trends, the rise of interdisciplinary work, and the need to explore domains beyond one's expertise all contribute to this challenge. To address these issues, we propose a generalizable pipeline capable of systematically analyzing any research area: identifying emerging trends, uncovering cross domain opportunities, and offering concrete starting points for new inquiry. In this work, we present Real Deep Research (RDR) a comprehensive framework applied to the domains of AI and robotics, with a particular focus on foundation models and robotics advancements. We also briefly extend our analysis to other areas of science. The main paper details the construction of the RDR pipeline, while the appendix provides extensive results across each analyzed topic. We hope this work sheds light for researchers working in the field of AI and beyond.", "source": "curated", "first_author": "Xueyan Zou", "url": "https://arxiv.org/abs/2510.20809", "fig": "figures/real-deep-research.webp", "figs": ["figures/real-deep-research-1.webp", "figures/real-deep-research-2.webp"]}, {"id": "learn-any-images", "title": "Robot Learning from Any Images", "tree": "il", "branch": "visuomotor", "authors": ["Zhao", "Siheng", "Mao", "Jiageng", "Chow", "Wei", "Shangguan", "Zeyu", "Shi", "Tianheng", "Xue", "Rong", "Zheng", "Yuxi", "Weng", "Yijia", "You", "Yang", "Seita", "Daniel", "Guibas", "Leonidas", "Zakharov", "Sergey", "Guizilini", "Vitor", "Wang", "Yue"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-26", "topics": ["policy learning from arbitrary images", "scene synthesis"], "paradigm": ["IL"], "method": [], "projects": ["r2s2r"], "related": ["phantom", "prism", "robosnap"], "arxiv": "2509.22970", "doi": "10.48550/arxiv.2509.22970", "code": "", "site": "", "note": "Turns ordinary images into training signal for manipulation. Relevant as the loosest end of the spectrum — if this works, the bar for how good your reconstruction must be drops considerably.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce RoLA, a framework that transforms any in-the-wild image into an interactive, physics-enabled robotic environment. Unlike previous methods, RoLA operates directly on a single image without requiring additional hardware or digital assets. Our framework democratizes robotic data generation by producing massive visuomotor robotic demonstrations within minutes from a wide range of image sources, including camera captures, robotic datasets, and Internet images. At its core, our approach combines a novel method for single-view physical scene recovery with an efficient visual blending strategy for photorealistic data collection. We demonstrate RoLA's versatility across applications like scalable robotic data generation and augmentation, robot learning from Internet images, and single-image real-to-sim-to-real systems for manipulators and humanoids. Video results are available at https://sihengz02.github.io/RoLA .", "source": "curated", "first_author": "Zhao", "url": "https://arxiv.org/abs/2509.22970", "fig": "figures/learn-any-images.webp", "figs": ["figures/learn-any-images-1.webp"]}, {"id": "stare-vla", "title": "STARE-VLA: Progressive Stage-Aware Reinforcement for Fine-Tuning Vision-Language-Action Models", "tree": "rl", "branch": "post-training", "authors": ["Feng Xu", "Guangyao Zhai", "Xin Kong", "Tingzhong Fu", "Daniel F. N. Gordon", "Xueli An", "Benjamin Busam"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-04", "topics": ["stage-aware reinforcement", "progressive fine-tuning"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["pld", "ript-vla"], "arxiv": "2512.05107", "doi": "", "code": "", "site": "", "note": "Stage-aware credit assignment for long-horizon VLA tasks — the sparse-reward problem that RIPT-VLA's binary success signal leaves open.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent advances in Vision-Language-Action (VLA) models, powered by large language models and reinforcement learning-based fine-tuning, have shown remarkable progress in robotic manipulation. Existing methods often treat long-horizon actions as linguistic sequences and apply trajectory-level optimization methods such as Trajectory-wise Preference Optimization (TPO) or Proximal Policy Optimization (PPO), leading to coarse credit assignment and unstable training. However, unlike language, where a unified semantic meaning is preserved despite flexible sentence order, action trajectories progress through causally chained stages with different learning difficulties. This motivates progressive stage optimization. Thereby, we present Stage-Aware Reinforcement (STARE), a module that decomposes a long-horizon action trajectory into semantically meaningful stages and provides dense, interpretable, and stage-aligned reinforcement signals. Integrating STARE into TPO and PPO, we yield Stage-Aware TPO (STA-TPO) and Stage-Aware PPO (STA-PPO) for offline stage-wise preference and online intra-stage interaction, respectively. Further building on supervised fine-tuning as initialization, we propose the Imitation -> Preference -> Interaction (IPI), a serial fine-tuning pipeline for improving action accuracy in VLA models. Experiments on SimplerEnv and ManiSkill3 demonstrate substantial gains, achieving state-of-the-art success rates of 98.0 percent on SimplerEnv and 96.4 percent on ManiSkill3 tasks.", "source": "curated", "first_author": "Feng Xu", "url": "https://arxiv.org/abs/2512.05107", "fig": "figures/stare-vla.webp", "figs": ["figures/stare-vla-1.webp"]}, {"id": "soft-gentle", "title": "Sim-to-Real Gentle Manipulation of Deformable and Fragile Objects with Stress-Guided Reinforcement Learning", "tree": "rl", "branch": "manipulation", "authors": ["Kei Ikemura", "Yifei Dong", "David Blanco-Mulero", "Alberta Longhini", "Li Chen", "Florian T. Pokorny"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-29", "topics": ["deformable and fragile objects", "stress-guided RL"], "paradigm": ["RL", "sim2real"], "method": [], "projects": [], "related": ["newton", "simweaver"], "arxiv": "2510.25405", "doi": "", "code": "", "site": "", "note": "Stress-guided reward for gentle manipulation. The useful contribution is making 'do not break it' a differentiable objective rather than a constraint you hope holds.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Robotic manipulation of deformable and fragile objects presents significant challenges, as excessive stress can lead to irreversible damage to the object. While existing solutions rely on accurate object models or specialized sensors and grippers, this adds complexity and often lacks generalization. To address this problem, we present a vision-based reinforcement learning approach that incorporates a stress-penalized reward to discourage damage to the object explicitly. In addition, to bootstrap learning, we incorporate offline demonstrations as well as a designed curriculum progressing from rigid proxies to deformables. We evaluate the proposed method in both simulated and real-world scenarios, showing that the policy learned in simulation can be transferred to the real world in a zero-shot manner, performing tasks such as picking up and pushing tofu. Our results show that the learned policies exhibit a damage-aware, gentle manipulation behavior, demonstrating their effectiveness by decreasing the stress applied to fragile objects by 36.5% while achieving the task goals, compared to vanilla RL policies.", "source": "curated", "first_author": "Kei Ikemura", "url": "https://arxiv.org/abs/2510.25405", "fig": "figures/soft-gentle.webp", "figs": ["figures/soft-gentle-1.webp", "figures/soft-gentle-2.webp"]}, {"id": "sunday-gelato", "title": "Sunday Robotics: Gelato Foundation Model & Skill Capture", "tree": "systems", "branch": "agentic", "authors": ["Sunday Robotics"], "corresponding": [], "institutions": ["Sunday Robotics"], "lab": "Sunday Robotics", "venue": "company blog", "venue_type": "industry", "year": 2025, "date": "2025-11-01", "topics": ["home robot", "skill capture glove", "foundation model"], "paradigm": ["VLA"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dexcap", "dexumi", "dyna2", "generalist-ai", "helix", "mimic-robotics", "pi05", "umi"], "arxiv": "", "doi": "", "code": "", "site": "https://www.humanoidsdaily.com/news/sunday-unveils-memo-a-wheeled-domestic-robot-that-learns-from-200-gloves", "note": "Came out of stealth late 2025 with 'Memo' and a handheld Skill Capture device — a commercial UMI-glove lineage, which is directly your world-action-model-from-gloves thesis with a company behind it. Company blog: unreviewed, no benchmarks. Track it as competitive intelligence, not evidence.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Sunday Robotics", "url": "https://www.humanoidsdaily.com/news/sunday-unveils-memo-a-wheeled-domestic-robot-that-learns-from-200-gloves"}, {"id": "toddlerbot", "title": "ToddlerBot: Open-Source ML-Compatible Humanoid Platform for Loco-Manipulation", "tree": "systems", "branch": "humanoid-hw", "authors": ["Haochen Shi", "Weizhuo Wang", "Shuran Song", "C. Karen Liu"], "corresponding": ["Haochen Shi"], "institutions": ["Stanford University"], "lab": "Stanford SVL / REALab", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-02-02", "topics": ["small humanoid", "open source", "loco-manipulation"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["berkeley-humanoid", "berkeley-humanoid-lite", "robosen"], "arxiv": "2502.00893", "doi": "", "code": "", "site": "", "note": "Small, cheap, repairable, and explicitly built so that the sim model matches the hardware. The sim-fidelity-first design argument is worth reading for MABEL.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Learning-based robotics research driven by data demands a new approach to robot hardware design-one that serves as both a platform for policy execution and a tool for embodied data collection to train policies. We introduce ToddlerBot, a low-cost, open-source humanoid robot platform designed for scalable policy learning and research in robotics and AI. ToddlerBot enables seamless acquisition of high-quality simulation and real-world data. The plug-and-play zero-point calibration and transferable motor system identification ensure a high-fidelity digital twin, enabling zero-shot policy transfer from simulation to the real world. A user-friendly teleoperation interface facilitates streamlined real-world data collection for learning motor skills from human demonstrations. Utilizing its data collection ability and anthropomorphic design, ToddlerBot is an ideal platform to perform whole-body loco-manipulation. Additionally, ToddlerBot's compact size (0.56m, 3.4kg) ensures safe operation in real-world environments. Reproducibility is achieved with an entirely 3D-printed, open-source design and commercially available components, keeping the total cost under 6,000 USD. Comprehensive documentation allows assembly and maintenance with basic technical expertise, as validated by a successful independent replication of the system. We demonstrate ToddlerBot's capabilities through arm span, payload, endurance tests, loco-manipulation tasks, and a collaborative long-horizon scenario where two robots tidy a toy session together. By advancing ML-compatibility, capability, and reproducibility, ToddlerBot provides a robust platform for scalable learning and dynamic policy execution in robotics research.", "source": "curated", "first_author": "Haochen Shi", "url": "https://arxiv.org/abs/2502.00893", "fig": "figures/toddlerbot.webp", "figs": ["figures/toddlerbot-1.webp"]}, {"id": "adaptive-tracking", "title": "Towards Adaptable Humanoid Control via Adaptive Motion Tracking", "tree": "rl", "branch": "motion-imitation", "authors": ["Huang", "Tao", "Wang", "Huayi", "Ren", "Junli", "Yin", "Kangning", "Wang", "Zirui", "Chen", "Xiao", "Jia", "Feiyu", "Zhang", "Wentao", "Long", "Junfeng", "Wang", "Jingbo", "Pang", "Jiangmiao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-16", "topics": ["adaptive motion tracking", "controllability"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["exbody2", "hover", "phc", "resmimic"], "arxiv": "2510.14454", "doi": "10.48550/arxiv.2510.14454", "code": "", "site": "", "note": "Adaptive motion tracking for humanoid control — the tracking policy adjusts rather than assuming the reference is feasible. Complementary to ExBody2's filter-before-training approach.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humanoid robots are envisioned to adapt demonstrated motions to diverse real-world conditions while accurately preserving motion patterns. Existing motion prior approaches enable well adaptability with a few motions but often sacrifice imitation accuracy, whereas motion-tracking methods achieve accurate imitation yet require many training motions and a test-time target motion to adapt. To combine their strengths, we introduce AdaMimic, a novel motion tracking algorithm that enables adaptable humanoid control from a single reference motion. To reduce data dependence while ensuring adaptability, our method first creates an augmented dataset by sparsifying the single reference motion into keyframes and applying light editing with minimal physical assumptions. A policy is then initialized by tracking these sparse keyframes to generate dense intermediate motions, and adapters are subsequently trained to adjust tracking speed and refine low-level actions based on the adjustment, enabling flexible time warping that further improves imitation accuracy and adaptability. We validate these significant improvements in our approach in both simulation and the real-world Unitree G1 humanoid robot in multiple tasks across a wide range of adaptation conditions. Videos and code are available at https://taohuang13.github.io/adamimic.github.io/.", "source": "curated", "first_author": "Huang", "url": "https://arxiv.org/abs/2510.14454", "fig": "figures/adaptive-tracking.webp", "figs": ["figures/adaptive-tracking-1.webp", "figures/adaptive-tracking-2.webp", "figures/adaptive-tracking-3.webp"]}, {"id": "manip-survey", "title": "Towards a Unified Understanding of Robot Manipulation: A Comprehensive Survey", "tree": "il", "branch": "foundations", "authors": ["Shuanghao Bai", "Wenxuan Song", "Jiayi Chen", "Yuheng Ji", "Zhide Zhong", "Jin Yang", "Han Zhao", "Wanqi Zhou", "Wei Zhao", "Zhe Li", "Pengxiang Ding", "Cheng Chi", "Haoang Li", "Chang Xu", "Xiaolong Zheng", "Donglin Wang", "Shanghang Zhang", "Badong Chen"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-13", "topics": ["survey", "robot manipulation", "unified taxonomy"], "paradigm": ["survey"], "method": [], "projects": [], "related": ["human-video-survey"], "arxiv": "2510.10903", "doi": "", "code": "", "site": "", "note": "A broad 2025 manipulation survey. Useful as a cross-check on this Atlas's taxonomy and as a source of citations you have missed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Embodied intelligence has witnessed remarkable progress in recent years, driven by advances in computer vision, natural language processing, and the rise of large-scale multimodal models. Among its core challenges, robot manipulation stands out as a fundamental yet intricate problem, requiring the seamless integration of perception, planning, and control to enable interaction within diverse and unstructured environments. This survey presents a comprehensive overview of robotic manipulation, encompassing foundational background, task-organized benchmarks and datasets, and a unified taxonomy of existing methods. We extend the classical division between high-level planning and low-level control by broadening high-level planning to include language, code, motion, affordance, and 3D representations, while introducing a new taxonomy of low-level learning-based control grounded in training paradigms such as input modeling, latent learning, and policy learning. Furthermore, we provide the first dedicated taxonomy of key bottlenecks, focusing on data collection, utilization, and generalization, and conclude with an extensive review of real-world applications. Compared with prior surveys, our work offers both a broader scope and deeper insight, serving as an accessible roadmap for newcomers and a structured reference for experienced researchers. All related resources, including research papers, open-source datasets, and projects, are curated for the community at https://github.com/BaiShuanghao/Awesome-Robotics-Manipulation.", "source": "curated", "first_author": "Shuanghao Bai", "url": "https://arxiv.org/abs/2510.10903", "fig": "figures/manip-survey.webp"}, {"id": "fall-safety", "title": "Unified Humanoid Fall-Safety Policy from a Few Demonstrations", "tree": "rl", "branch": "locomotion", "authors": ["Xu", "Zhengjie", "Li", "Ye", "Lin", "Kwan-yee", "Yu", "Stella X."], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-10", "topics": ["fall-safety", "few demonstrations", "protective behaviour"], "paradigm": ["RL", "IL"], "method": [], "projects": ["mabel"], "related": ["humanup", "stablemimic", "unirelo"], "arxiv": "2511.07407", "doi": "10.48550/arxiv.2511.07407", "code": "", "site": "", "note": "Falling <em>well</em> — protective behaviour learned from a few demonstrations — rather than only getting up afterwards. Cheaper than replacing a humanoid, and almost nobody works on it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Falling is an inherent risk of humanoid mobility. Maintaining stability is thus a primary safety focus in robot control and learning, yet no existing approach fully averts loss of balance. When instability does occur, prior work addresses only isolated aspects of falling: avoiding falls, choreographing a controlled descent, or standing up afterward. Consequently, humanoid robots lack integrated strategies for impact mitigation and prompt recovery when real falls defy these scripts. We aim to go beyond keeping balance to make the entire fall-and-recovery process safe and autonomous: prevent falls when possible, reduce impact when unavoidable, and stand up when fallen. By fusing sparse human demonstrations with reinforcement learning and an adaptive diffusion-based memory of safe reactions, we learn adaptive whole-body behaviors that unify fall prevention, impact mitigation, and rapid recovery in one policy. Experiments in simulation and on a Unitree G1 demonstrate robust sim-to-real transfer, lower impact forces, and consistently fast recovery across diverse disturbances, pointing towards safer, more resilient humanoids in real environments. Videos are available at https://firm2025.github.io/.", "source": "curated", "first_author": "Xu", "url": "https://arxiv.org/abs/2511.07407", "fig": "figures/fall-safety.webp", "figs": ["figures/fall-safety-1.webp"]}, {"id": "vibecheck", "title": "VibeCheck: Using Active Acoustic Tactile Sensing for Contact-Rich Manipulation", "tree": "systems", "branch": "tactile-hw", "authors": ["Kaidi Zhang", "Do-Gon Kim", "Eric T. Chang", "Hsinying Liang", "Zhanpeng He", "Kathryn Lampo", "Peng Wu", "Ioannis Kymissis", "Matei Ciocarlie"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-04-22", "topics": ["active acoustic sensing", "contact-rich manipulation"], "paradigm": ["hardware", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["hearing-touch", "maniwav", "miccheck", "sonicsense"], "arxiv": "2504.15535", "doi": "10.48550/arxiv.2504.15535", "code": "", "site": "", "note": "<b>Active</b> acoustic sensing — inject a vibration and listen to the response, rather than waiting for the task to make a noise. For a turntable, the platter is already vibrating, so you get an active probe for free.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The acoustic response of an object can reveal a lot about its global state, for example its material properties or the extrinsic contacts it is making with the world. In this work, we build an active acoustic sensing gripper equipped with two piezoelectric fingers: one for generating signals, the other for receiving them. By sending an acoustic vibration from one finger to the other through an object, we gain insight into an object's acoustic properties and contact state. We use this system to classify objects, estimate grasping position, estimate poses of internal structures, and classify the types of extrinsic contacts an object is making with the environment. Using our contact type classification model, we tackle a standard long-horizon manipulation problem: peg insertion. We use a simple simulated transition model based on the performance of our sensor to train an imitation learning policy that is robust to imperfect predictions from the classifier. We finally demonstrate the policy on a UR5 robot with active acoustic sensing as the only feedback. Videos can be found at https://roamlab.github.io/vibecheck .", "source": "curated", "first_author": "Kaidi Zhang", "url": "https://arxiv.org/abs/2504.15535", "fig": "figures/vibecheck.webp", "figs": ["figures/vibecheck-1.webp", "figures/vibecheck-2.webp", "figures/vibecheck-3.webp"]}, {"id": "visualmimic", "title": "VisualMimic: Visual Humanoid Loco-Manipulation via Motion Tracking and Generation", "tree": "rl", "branch": "motion-imitation", "authors": ["Shaofeng Yin", "Yanjie Ze", "Hong-Xing Yu", "C. Karen Liu", "Jiajun Wu"], "corresponding": [], "institutions": ["Stanford University", "Tsinghua University"], "lab": "Stanford Vision & Learning (SVL)", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-25", "topics": ["visual loco-manipulation", "hierarchical control"], "paradigm": ["RL"], "method": ["low-level tracking policy + high-level visual policy generating motion targets"], "projects": ["mabel"], "related": ["catch-it", "deep-wbc", "deepmimic", "falcon", "fetchman", "handeye-delivery", "humanoidbench"], "arxiv": "2509.20322", "doi": "", "code": "", "site": "", "note": "In your library. The hierarchical split — vision picks the reference motion, a tracker executes it — is currently the most reliable recipe for humanoid loco-manipulation.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/RL/visualmimic.pdf", "abstract": "Humanoid loco-manipulation in unstructured environments demands tight integration of egocentric perception and whole-body control. However, existing approaches either depend on external motion capture systems or fail to generalize across diverse tasks. We introduce VisualMimic, a visual sim-to-real framework that unifies egocentric vision with hierarchical whole-body control for humanoid robots. VisualMimic combines a task-agnostic low-level keypoint tracker -- trained from human motion data via a teacher-student scheme -- with a task-specific high-level policy that generates keypoint commands from visual and proprioceptive input. To ensure stable training, we inject noise into the low-level policy and clip high-level actions using human motion statistics. VisualMimic enables zero-shot transfer of visuomotor policies trained in simulation to real humanoid robots, accomplishing a wide range of loco-manipulation tasks such as box lifting, pushing, football dribbling, and kicking. Beyond controlled laboratory settings, our policies also generalize robustly to outdoor environments. Videos are available at: https://visualmimic.github.io .", "source": "curated", "first_author": "Shaofeng Yin", "url": "https://arxiv.org/abs/2509.20322", "fig": "figures/visualmimic.webp", "figs": ["figures/visualmimic-1.webp", "figures/visualmimic-2.webp", "figures/visualmimic-3.webp"]}, {"id": "wheeled-multistage", "title": "Whole-Body Bilateral Teleoperation with Multi-Stage Object Parameter Estimation for Wheeled Humanoid Locomanipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Baek", "Donghoon", "Purushottam", "Amartya", "Choi", "Jason J.", "Ramos", "Joao"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-08-13", "topics": ["object parameter estimation", "bilateral teleoperation", "wheeled loco-manipulation"], "paradigm": ["control"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["asap", "falcon", "wheeled-bilateral", "wheeled-payload", "wheeled-posforce"], "arxiv": "2508.09846", "doi": "10.48550/arxiv.2508.09846", "code": "", "site": "", "note": "Multi-stage estimation of the manipulated object's parameters during wheeled-humanoid teleoperation. On a balancing base, an unknown payload is not a nuisance — it changes the plant. Estimating it online is the difference between carrying a box and falling over.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2025_2508.09846v1_Whole_Body_Bilateral_Teleoperation_with_Multi_Stage_Object_P.pdf", "abstract": "This paper presents an object-aware whole-body bilateral teleoperation framework for wheeled humanoid loco-manipulation. This framework combines whole-body bilateral teleoperation with an online multi-stage object inertial parameter estimation module, which is the core technical contribution of this work. The multi-stage process sequentially integrates a vision-based object size estimator, an initial parameter guess generated by a large vision-language model (VLM), and a decoupled hierarchical sampling strategy. The visual size estimate and VLM prior offer a strong initial guess of the object's inertial parameters, significantly reducing the search space for sampling-based refinement and improving the overall estimation speed. A hierarchical strategy first estimates mass and center of mass, then infers inertia from object size to ensure physically feasible parameters, while a decoupled multi-hypothesis scheme enhances robustness to VLM prior errors. Our estimator operates in parallel with high-fidelity simulation and hardware, enabling real-time online updates. The estimated parameters are then used to update the wheeled humanoid's equilibrium point, allowing the operator to focus more on locomotion and manipulation. This integration improves the haptic force feedback for dynamic synchronization, enabling more dynamic whole-body teleoperation. By compensating for object dynamics using the estimated parameters, the framework also improves manipulation tracking while preserving compliant behavior. We validate the system on a customized wheeled humanoid with a robotic gripper and human-machine interface, demonstrating real-time execution of lifting, delivering, and releasing tasks with a payload weighing approximately one-third of the robot's body weight.", "source": "curated", "first_author": "Baek", "url": "https://arxiv.org/abs/2508.09846", "fig": "figures/wheeled-multistage.webp", "figs": ["figures/wheeled-multistage-1.webp"]}, {"id": "mimic-robotics", "title": "mimic robotics: Dexterous Manipulation from Human Demonstration", "tree": "systems", "branch": "agentic", "authors": ["mimic robotics (ETH Zürich spin-off)"], "corresponding": [], "institutions": ["mimic robotics"], "lab": "mimic robotics", "venue": "company blog", "venue_type": "industry", "year": 2025, "date": "", "topics": ["dexterous hand", "human demonstration", "industrial deployment"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["dexumi", "faive-ball", "orca-hand", "ruka", "sunday-gelato"], "arxiv": "", "doi": "", "code": "", "site": "https://www.mimicrobotics.com/", "note": "ETH spin-off (Faive Robotics lineage) building a tendon-driven human-like hand and learning from human demonstration for industrial pick-and-place. The closest commercial analogue to the GeoDex hand + ego-video plan. Unreviewed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "mimic robotics (ETH Zürich spin-off)", "url": "https://www.mimicrobotics.com/"}, {"id": "dp3", "title": "3D Diffusion Policy: Generalizable Visuomotor Policy Learning via Simple 3D Representations", "tree": "il", "branch": "visuomotor", "authors": ["Yanjie Ze", "Gu Zhang", "Kangning Zhang", "Chenyuan Hu", "Muhan Wang", "Huazhe Xu"], "corresponding": ["Yanjie Ze"], "institutions": ["Shanghai Qi Zhi Institute", "Tsinghua University", "Stanford University"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-03-06", "topics": ["3D representations", "point clouds", "sample efficiency"], "paradigm": ["IL", "diffusion"], "method": ["sparse point-cloud encoder + diffusion action head"], "projects": ["iris"], "related": ["dp", "manflow"], "arxiv": "2403.03954", "doi": "", "code": "", "site": "", "note": "Swapping RGB for a compact point cloud buys a large jump in sample efficiency and view robustness. Relevant to IRIS if you ever move off pure RGB.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Imitation learning provides an efficient way to teach robots dexterous skills; however, learning complex skills robustly and generalizablely usually consumes large amounts of human demonstrations. To tackle this challenging problem, we present 3D Diffusion Policy (DP3), a novel visual imitation learning approach that incorporates the power of 3D visual representations into diffusion policies, a class of conditional action generative models. The core design of DP3 is the utilization of a compact 3D visual representation, extracted from sparse point clouds with an efficient point encoder. In our experiments involving 72 simulation tasks, DP3 successfully handles most tasks with just 10 demonstrations and surpasses baselines with a 24.2% relative improvement. In 4 real robot tasks, DP3 demonstrates precise control with a high success rate of 85%, given only 40 demonstrations of each task, and shows excellent generalization abilities in diverse aspects, including space, viewpoint, appearance, and instance. Interestingly, in real robot experiments, DP3 rarely violates safety requirements, in contrast to baseline methods which frequently do, necessitating human intervention. Our extensive evaluation highlights the critical importance of 3D representations in real-world robot learning. Videos, code, and data are available on https://3d-diffusion-policy.github.io .", "source": "curated", "first_author": "Yanjie Ze", "url": "https://arxiv.org/abs/2403.03954", "fig": "figures/dp3.webp", "figs": ["figures/dp3-1.webp", "figures/dp3-2.webp", "figures/dp3-3.webp"]}, {"id": "twisting-waist", "title": "A Robust Quadruped Robot with Twisting Waist for Flexible Motions", "tree": "systems", "branch": "codesign", "authors": ["Quancheng Qian", "Xiaoyi Wei", "Zonghao Zhang", "Jiaxin Tu", "Yueqi Zhang", "Taixian Hou", "Xiaofei Gao", "Peng Zhai", "Lihua Zhang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-10-08", "topics": ["twisting waist", "quadruped flexibility"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["s-cheetah", "spine-phase"], "arxiv": "2410.05884", "doi": "10.48550/arxiv.2410.05884", "code": "", "site": "", "note": "A single twisting waist joint rather than a full spine — the minimal version of the same idea, and a useful data point on how much of the benefit one DOF buys.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The waist plays a crucial role in the agile movement of many animals in nature. It provides the torso with additional degrees of freedom and flexibility, inspiring researchers to incorporate this biological feature into robotic structures to enhance robot locomotion. This paper presents a cost-effective and low-complexity waist mechanism integrated into the structure of the open-source robot solo8, adding a new degree of freedom (DOF) to its torso. We refer to this novel robot as solo9. Additionally, we propose a full-body control method for the waist-equipped quadruped robot based on generative adversarial imitation learning (GAIL). During training, the discriminator is used as input for iterative optimization of the policy and dataset, enabling solo9 to achieve flexible steering maneuvers across various gaits. Extensive tests of solo9's steering capabilities, terrain adaptability, and robustness are conducted in both simulation and real-world scenarios, with detailed comparisons to solo8 and solo12, demonstrating the effectiveness of the control algorithm and the advantages of the waist mechanism.", "source": "curated", "first_author": "Quancheng Qian", "url": "https://arxiv.org/abs/2410.05884", "fig": "figures/twisting-waist.webp", "figs": ["figures/twisting-waist-1.webp", "figures/twisting-waist-2.webp", "figures/twisting-waist-3.webp"]}, {"id": "aloha2", "title": "ALOHA 2: An Enhanced Low-Cost Hardware for Bimanual Teleoperation", "tree": "systems", "branch": "bimanual", "authors": ["ALOHA 2 Team", "Jorge Aldaco", "Travis Armstrong", "Robert Baruch", "Jeff Bingham", "et al."], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind Robotics", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-05-03", "topics": ["open hardware", "improved ergonomics", "MuJoCo model"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["act", "aloha", "alohaunleashed", "mobile-aloha"], "arxiv": "2405.02292", "doi": "", "code": "", "site": "", "note": "Better grippers, gravity compensation, a validated MuJoCo model. The MuJoCo model is the underrated part — it makes the platform simulatable rather than just buildable.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Diverse demonstration datasets have powered significant advances in robot learning, but the dexterity and scale of such data can be limited by the hardware cost, the hardware robustness, and the ease of teleoperation. We introduce ALOHA 2, an enhanced version of ALOHA that has greater performance, ergonomics, and robustness compared to the original design. To accelerate research in large-scale bimanual manipulation, we open source all hardware designs of ALOHA 2 with a detailed tutorial, together with a MuJoCo model of ALOHA 2 with system identification. See the project website at aloha-2.github.io.", "source": "curated", "first_author": "ALOHA 2 Team", "url": "https://arxiv.org/abs/2405.02292", "fig": "figures/aloha2.webp", "figs": ["figures/aloha2-1.webp", "figures/aloha2-2.webp", "figures/aloha2-3.webp"]}, {"id": "alohaunleashed", "title": "ALOHA Unleashed: A Simple Recipe for Robot Dexterity", "tree": "il", "branch": "visuomotor", "authors": ["Tony Z. Zhao", "Jonathan Tompson", "Danny Driess", "Pete Florence", "Kamyar Ghasemipour", "Chelsea Finn", "Ayzaan Wahid"], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind Robotics", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-10-17", "topics": ["bimanual dexterity", "diffusion", "scale"], "paradigm": ["IL", "diffusion"], "method": [], "projects": ["iris"], "related": ["act", "aloha2", "dp"], "arxiv": "2410.13126", "doi": "", "code": "", "site": "", "note": "In your CRV_iris/imitation folder. The 'just scale demonstrations + a transformer diffusion head' recipe, on genuinely hard bimanual tasks (shoelaces, shirt hanging).", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/CRV_iris/imitation/2410.13126v1-3.pdf", "abstract": "Recent work has shown promising results for learning end-to-end robot policies using imitation learning. In this work we address the question of how far can we push imitation learning for challenging dexterous manipulation tasks. We show that a simple recipe of large scale data collection on the ALOHA 2 platform, combined with expressive models such as Diffusion Policies, can be effective in learning challenging bimanual manipulation tasks involving deformable objects and complex contact rich dynamics. We demonstrate our recipe on 5 challenging real-world and 3 simulated tasks and demonstrate improved performance over state-of-the-art baselines. The project website and videos can be found at aloha-unleashed.github.io.", "source": "curated", "first_author": "Tony Z. Zhao", "url": "https://arxiv.org/abs/2410.13126", "fig": "figures/alohaunleashed.webp", "figs": ["figures/alohaunleashed-1.webp", "figures/alohaunleashed-2.webp"]}, {"id": "anymal-parkour", "title": "ANYmal Parkour: Learning Agile Navigation for Quadrupedal Robots", "tree": "rl", "branch": "locomotion", "authors": ["David Hoeller", "Nikita Rudin", "Dhionis Sako", "Marco Hutter"], "corresponding": ["David Hoeller"], "institutions": ["ETH Zürich", "NVIDIA"], "lab": "ETH RSL", "venue": "Science Robotics", "venue_type": "journal", "year": 2024, "date": "2023-06-26", "topics": ["parkour", "skill library", "navigation"], "paradigm": ["RL"], "method": ["specialized skill policies + high-level navigation policy selecting among them"], "projects": [], "related": ["anymal-lee", "attention-map-hutter", "extreme-parkour", "miki2022"], "arxiv": "2306.14874", "doi": "", "code": "", "site": "", "note": "The hierarchical counterpoint to end-to-end parkour: train separate jump/climb/crouch specialists, then learn a navigator that picks between them. More engineering, more reliability.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Performing agile navigation with four-legged robots is a challenging task due to the highly dynamic motions, contacts with various parts of the robot, and the limited field of view of the perception sensors. In this paper, we propose a fully-learned approach to train such robots and conquer scenarios that are reminiscent of parkour challenges. The method involves training advanced locomotion skills for several types of obstacles, such as walking, jumping, climbing, and crouching, and then using a high-level policy to select and control those skills across the terrain. Thanks to our hierarchical formulation, the navigation policy is aware of the capabilities of each skill, and it will adapt its behavior depending on the scenario at hand. Additionally, a perception module is trained to reconstruct obstacles from highly occluded and noisy sensory data and endows the pipeline with scene understanding. Compared to previous attempts, our method can plan a path for challenging scenarios without expert demonstration, offline computation, a priori knowledge of the environment, or taking contacts explicitly into account. While these modules are trained from simulated data only, our real-world experiments demonstrate successful transfer on hardware, where the robot navigates and crosses consecutive challenging obstacles with speeds of up to two meters per second. The supplementary video can be found on the project website: https://sites.google.com/leggedrobotics.com/agile-navigation", "source": "curated", "first_author": "David Hoeller", "url": "https://arxiv.org/abs/2306.14874", "fig": "figures/anymal-parkour.webp", "figs": ["figures/anymal-parkour-1.webp"]}, {"id": "animal-avatars", "title": "Animal Avatars: Reconstructing Animatable 3D Animals from Casual Videos", "tree": "il", "branch": "human-video", "authors": ["Remy Sabathier", "Niloy J. Mitra", "David R. Novotny"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-03-25", "topics": ["animatable 3D animals from casual video"], "paradigm": ["representation learning"], "method": [], "projects": ["r2s2r"], "related": ["barc", "corgi", "dogmo", "hsmal", "videomimic"], "arxiv": "2403.17103", "doi": "10.48550/arxiv.2403.17103", "code": "", "site": "", "note": "Animatable 3D animals reconstructed from casual video — the pipeline that turns YouTube footage of a running dog into something with joints you can measure.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We present a method to build animatable dog avatars from monocular videos. This is challenging as animals display a range of (unpredictable) non-rigid movements and have a variety of appearance details (e.g., fur, spots, tails). We develop an approach that links the video frames via a 4D solution that jointly solves for animal's pose variation, and its appearance (in a canonical pose). To this end, we significantly improve the quality of template-based shape fitting by endowing the SMAL parametric model with Continuous Surface Embeddings, which brings image-to-mesh reprojection constaints that are denser, and thus stronger, than the previously used sparse semantic keypoint correspondences. To model appearance, we propose an implicit duplex-mesh texture that is defined in the canonical pose, but can be deformed using SMAL pose coefficients and later rendered to enforce a photometric compatibility with the input video frames. On the challenging CoP3D and APTv2 datasets, we demonstrate superior results (both in terms of pose estimates and predicted appearance) to existing template-free (RAC) and template-based approaches (BARC, BITE).", "source": "curated", "first_author": "Remy Sabathier", "url": "https://arxiv.org/abs/2403.17103", "fig": "figures/animal-avatars.webp", "figs": ["figures/animal-avatars-1.webp"]}, {"id": "bab-nd", "title": "BaB-ND: Long-Horizon Motion Planning with Branch-and-Bound and Neural Dynamics", "tree": "systems", "branch": "planning-control", "authors": ["Keyi Shen", "Jiangwei Yu", "Jose Barreiros", "Huan Zhang", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign", "Toyota Research Institute"], "lab": "RoboPIL (Yunzhu Li)", "venue": "ICLR", "venue_type": "conference", "year": 2024, "date": "2024-12-12", "topics": ["branch-and-bound", "long-horizon planning with neural dynamics", "global optimality"], "paradigm": ["model-based RL"], "method": [], "projects": ["m2", "wam-tactile"], "related": ["adaptigraph", "particle-grid-dynamics", "robocook", "td-mpc"], "arxiv": "2412.09584", "doi": "10.48550/arXiv.2412.09584", "code": "", "site": "", "note": "<b>BaB-ND</b> — the honest observation that planning inside a learned dynamics model is a <em>non-convex</em> problem and that sampling-based planners quietly settle for local optima. Uses branch-and-bound with neural-network verification bounds to get global structure on contact-rich tasks — planar pushing with obstacles, sorting, rope routing. Read it if your MPC ever looks like it is failing for reasons the model cannot explain.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Neural-network-based dynamics models learned from observational data have shown strong predictive capabilities for scene dynamics in robotic manipulation tasks. However, their inherent non-linearity presents significant challenges for effective planning. Current planning methods, often dependent on extensive sampling or local gradient descent, struggle with long-horizon motion planning tasks involving complex contact events. In this paper, we present a GPU-accelerated branch-and-bound (BaB) framework for motion planning in manipulation tasks that require trajectory optimization over neural dynamics models. Our approach employs a specialized branching heuristics to divide the search space into subdomains, and applies a modified bound propagation method, inspired by the state-of-the-art neural network verifier alpha-beta-CROWN, to efficiently estimate objective bounds within these subdomains. The branching process guides planning effectively, while the bounding process strategically reduces the search space. Our framework achieves superior planning performance, generating high-quality state-action trajectories and surpassing existing methods in challenging, contact-rich manipulation tasks such as non-prehensile planar pushing with obstacles, object sorting, and rope routing in both simulated and real-world settings. Furthermore, our framework supports various neural network architectures, ranging from simple multilayer perceptrons to advanced graph neural dynamics models, and scales efficiently with different model sizes.", "source": "curated", "first_author": "Keyi Shen", "url": "https://arxiv.org/abs/2412.09584", "fig": "figures/bab-nd.webp", "figs": ["figures/bab-nd-1.webp", "figures/bab-nd-2.webp"]}, {"id": "berkeley-humanoid", "title": "Berkeley Humanoid: A Research Platform for Learning-based Control", "tree": "systems", "branch": "humanoid-hw", "authors": ["Qiayuan Liao", "Bike Zhang", "Xuanyu Huang", "Xiaoyu Huang", "Zhongyu Li", "Koushil Sreenath"], "corresponding": ["Qiayuan Liao"], "institutions": ["UC Berkeley"], "lab": "Hybrid Robotics Lab (Sreenath)", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-07-31", "topics": ["research humanoid", "learning-based control", "open design"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["berkeley-humanoid-lite", "disney-bdx", "toddlerbot", "unitree-g1"], "arxiv": "2407.21781", "doi": "", "code": "", "site": "", "note": "In your MABEL hardware folder. A mid-scale humanoid designed around the requirements of RL rather than around classical control — low reflected inertia, good sim fidelity.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/MABEL/literatures/hardware/2407.21781v1.pdf", "abstract": "We introduce Berkeley Humanoid, a reliable and low-cost mid-scale humanoid research platform for learning-based control. Our lightweight, in-house-built robot is designed specifically for learning algorithms with low simulation complexity, anthropomorphic motion, and high reliability against falls. The robot's narrow sim-to-real gap enables agile and robust locomotion across various terrains in outdoor environments, achieved with a simple reinforcement learning controller using light domain randomization. Furthermore, we demonstrate the robot traversing for hundreds of meters, walking on a steep unpaved trail, and hopping with single and double legs as a testimony to its high performance in dynamical walking. Capable of omnidirectional locomotion and withstanding large perturbations with a compact setup, our system aims for scalable, sim-to-real deployment of learning-based humanoid systems. Please check http://berkeley-humanoid.com for more details.", "source": "curated", "first_author": "Qiayuan Liao", "url": "https://arxiv.org/abs/2407.21781", "fig": "figures/berkeley-humanoid.webp", "figs": ["figures/berkeley-humanoid-1.webp"]}, {"id": "bunny-visionpro", "title": "Bunny-VisionPro: Real-Time Bimanual Dexterous Teleoperation for Imitation Learning", "tree": "systems", "branch": "interfaces", "authors": ["Runyu Ding", "Yuzhe Qin", "Jiyue Zhu", "Chengzhe Jia", "Shiqi Yang", "Ruihan Yang", "Xiaojuan Qi", "Xiaolong Wang"], "corresponding": [], "institutions": ["HKU", "UC San Diego"], "lab": "UCSD Xiaolong Wang Lab", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-07-04", "topics": ["Vision Pro teleoperation", "haptic feedback"], "paradigm": ["teleop"], "method": [], "projects": ["mabel"], "related": ["anyteleop", "doglove", "lucid-xr", "open-television"], "arxiv": "2407.03162", "doi": "", "code": "", "site": "", "note": "Apple Vision Pro as a bimanual dexterous teleop device with haptic cues. Relevant given your visionOS app work in the MABEL repo.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Teleoperation is a crucial tool for collecting human demonstrations, but controlling robots with bimanual dexterous hands remains a challenge. Existing teleoperation systems struggle to handle the complexity of coordinating two hands for intricate manipulations. We introduce Bunny-VisionPro, a real-time bimanual dexterous teleoperation system that leverages a VR headset. Unlike previous vision-based teleoperation systems, we design novel low-cost devices to provide haptic feedback to the operator, enhancing immersion. Our system prioritizes safety by incorporating collision and singularity avoidance while maintaining real-time performance through innovative designs. Bunny-VisionPro outperforms prior systems on a standard task suite, achieving higher success rates and reduced task completion times. Moreover, the high-quality teleoperation demonstrations improve downstream imitation learning performance, leading to better generalizability. Notably, Bunny-VisionPro enables imitation learning with challenging multi-stage, long-horizon dexterous manipulation tasks, which have rarely been addressed in previous work. Our system's ability to handle bimanual manipulations while prioritizing safety and real-time performance makes it a powerful tool for advancing dexterous manipulation and imitation learning.", "source": "curated", "first_author": "Runyu Ding", "url": "https://arxiv.org/abs/2407.03162", "fig": "figures/bunny-visionpro.webp", "figs": ["figures/bunny-visionpro-1.webp", "figures/bunny-visionpro-2.webp"]}, {"id": "consistency-policy", "title": "Consistency Policy: Accelerated Visuomotor Policies via Consistency Distillation", "tree": "il", "branch": "visuomotor", "authors": ["Aaditya Prasad", "Kevin Lin", "Jimmy Wu", "Linqi Zhou", "Jeannette Bohg"], "corresponding": [], "institutions": ["Stanford University"], "lab": "Stanford IPRL (Bohg)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-05-13", "topics": ["policy distillation", "inference speed"], "paradigm": ["IL", "diffusion"], "method": [], "projects": [], "related": ["conrft", "dp", "manflow"], "arxiv": "2405.07503", "doi": "", "code": "", "site": "", "note": "Diffusion policy at usable control rates via consistency distillation. Matters the moment you want DP on a 200 Hz arm.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Many robotic systems, such as mobile manipulators or quadrotors, cannot be equipped with high-end GPUs due to space, weight, and power constraints. These constraints prevent these systems from leveraging recent developments in visuomotor policy architectures that require high-end GPUs to achieve fast policy inference. In this paper, we propose Consistency Policy, a faster and similarly powerful alternative to Diffusion Policy for learning visuomotor robot control. By virtue of its fast inference speed, Consistency Policy can enable low latency decision making in resource-constrained robotic setups. A Consistency Policy is distilled from a pretrained Diffusion Policy by enforcing self-consistency along the Diffusion Policy's learned trajectories. We compare Consistency Policy with Diffusion Policy and other related speed-up methods across 6 simulation tasks as well as three real-world tasks where we demonstrate inference on a laptop GPU. For all these tasks, Consistency Policy speeds up inference by an order of magnitude compared to the fastest alternative method and maintains competitive success rates. We also show that the Conistency Policy training procedure is robust to the pretrained Diffusion Policy's quality, a useful result that helps practioners avoid extensive testing of the pretrained model. Key design decisions that enabled this performance are the choice of consistency objective, reduced initial sample variance, and the choice of preset chaining steps.", "source": "curated", "first_author": "Aaditya Prasad", "url": "https://arxiv.org/abs/2405.07503", "fig": "figures/consistency-policy.webp", "figs": ["figures/consistency-policy-1.webp"]}, {"id": "diamond", "title": "Diffusion for World Modeling: Visual Details Matter in Atari", "tree": "rl", "branch": "model-based", "authors": ["Eloi Alonso", "Adam Jelley", "Vincent Micheli", "Anssi Kanervisto", "Amos Storkey", "Tim Pearce", "François Fleuret"], "corresponding": [], "institutions": ["University of Geneva", "University of Edinburgh", "Microsoft Research"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2024, "date": "2024-05-20", "topics": ["diffusion world model", "pixel-space imagination", "visual detail"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["dream-tac", "iris", "omnivta", "storm", "tacforesight"], "arxiv": "2405.12399", "doi": "10.48550/arXiv.2405.12399", "code": "", "site": "", "note": "<b>DIAMOND</b> — 'Visual Details Matter'. Runs the world model as a <em>diffusion</em> process in pixel space rather than a latent autoencoder, and then shows, concretely, which Atari failures were caused by the autoencoder throwing away small but decision-relevant detail. Directly relevant to your tactile WAM: a tactile signal <em>is</em> a small decision-relevant detail, and a latent trained on a reconstruction loss will happily discard it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models constitute a promising approach for training reinforcement learning agents in a safe and sample-efficient manner. Recent world models predominantly operate on sequences of discrete latent variables to model environment dynamics. However, this compression into a compact discrete representation may ignore visual details that are important for reinforcement learning. Concurrently, diffusion models have become a dominant approach for image generation, challenging well-established methods modeling discrete latents. Motivated by this paradigm shift, we introduce DIAMOND (DIffusion As a Model Of eNvironment Dreams), a reinforcement learning agent trained in a diffusion world model. We analyze the key design choices that are required to make diffusion suitable for world modeling, and demonstrate how improved visual details can lead to improved agent performance. DIAMOND achieves a mean human normalized score of 1.46 on the competitive Atari 100k benchmark; a new best for agents trained entirely within a world model. We further demonstrate that DIAMOND's diffusion world model can stand alone as an interactive neural game engine by training on static Counter-Strike: Global Offensive gameplay. To foster future research on diffusion for world modeling, we release our code, agents, videos and playable world models at https://diamond-wm.github.io.", "source": "curated", "first_author": "Eloi Alonso", "url": "https://arxiv.org/abs/2405.12399", "fig": "figures/diamond.webp", "figs": ["figures/diamond-1.webp"]}, {"id": "morph-hypernet", "title": "Distilling Morphology-Conditioned Hypernetworks for Efficient Universal Morphology Control", "tree": "rl", "branch": "foundations", "authors": ["Xiong Zheng", "Risto Vuorio", "Jacob Beck", "Matthieu Zimmer", "Shao Kun", "Shimon Whiteson"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-02-09", "topics": ["morphology-conditioned hypernetwork", "universal morphology control"], "paradigm": ["RL"], "method": [], "projects": [], "related": ["changing-morph-dyn", "derl", "gcnt", "getup-morphologies"], "arxiv": "2402.06570", "doi": "10.48550/arxiv.2402.06570", "code": "", "site": "", "note": "Distils morphology-conditioned hypernetworks so one network emits the weights for whatever body it is given. Cheaper at inference than a graph transformer, which matters when the policy has to run on the robot.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Learning a universal policy across different robot morphologies can significantly improve learning efficiency and enable zero-shot generalization to unseen morphologies. However, learning a highly performant universal policy requires sophisticated architectures like transformers (TF) that have larger memory and computational cost than simpler multi-layer perceptrons (MLP). To achieve both good performance like TF and high efficiency like MLP at inference time, we propose HyperDistill, which consists of: (1) A morphology-conditioned hypernetwork (HN) that generates robot-wise MLP policies, and (2) A policy distillation approach that is essential for successful training. We show that on UNIMAL, a benchmark with hundreds of diverse morphologies, HyperDistill performs as well as a universal TF teacher policy on both training and unseen test robots, but reduces model size by 6-14 times, and computational cost by 67-160 times in different environments. Our analysis attributes the efficiency advantage of HyperDistill at inference time to knowledge decoupling, i.e., the ability to decouple inter-task and intra-task knowledge, a general principle that could also be applied to improve inference efficiency in other domains.", "source": "curated", "first_author": "Xiong Zheng", "url": "https://arxiv.org/abs/2402.06570", "fig": "figures/morph-hypernet.webp"}, {"id": "dreureka", "title": "DrEureka: Language Model Guided Sim-To-Real Transfer", "tree": "systems", "branch": "agentic", "authors": ["Yecheng Jason Ma", "William Liang", "Hungju Wang", "Sam Wang", "Yuke Zhu", "Linxi Fan", "Osbert Bastani", "Dinesh Jayaraman"], "corresponding": [], "institutions": ["University of Pennsylvania", "NVIDIA", "UT Austin"], "lab": "NVIDIA GEAR", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-06-04", "topics": ["LLM-guided domain randomization", "sim-to-real"], "paradigm": ["LLM", "RL"], "method": [], "projects": ["r2s2r"], "related": ["asap", "dexsim2real", "domain-rand", "eureka", "rf-agent"], "arxiv": "2406.01967", "doi": "", "code": "", "site": "", "note": "Eureka extended to writing the domain-randomization config too — the LLM tunes the sim-to-real knobs a grad student normally tunes. Yoga-ball balancing as the demo.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Transferring policies learned in simulation to the real world is a promising strategy for acquiring robot skills at scale. However, sim-to-real approaches typically rely on manual design and tuning of the task reward function as well as the simulation physics parameters, rendering the process slow and human-labor intensive. In this paper, we investigate using Large Language Models (LLMs) to automate and accelerate sim-to-real design. Our LLM-guided sim-to-real approach, DrEureka, requires only the physics simulation for the target task and automatically constructs suitable reward functions and domain randomization distributions to support real-world transfer. We first demonstrate that our approach can discover sim-to-real configurations that are competitive with existing human-designed ones on quadruped locomotion and dexterous manipulation tasks. Then, we showcase that our approach is capable of solving novel robot tasks, such as quadruped balancing and walking atop a yoga ball, without iterative manual design.", "source": "curated", "first_author": "Yecheng Jason Ma", "url": "https://arxiv.org/abs/2406.01967", "fig": "figures/dreureka.webp", "figs": ["figures/dreureka-1.webp", "figures/dreureka-2.webp"]}, {"id": "gs-dynamics", "title": "Dynamic 3D Gaussian Tracking for Graph-Based Neural Dynamics Modeling", "tree": "il", "branch": "world-models", "authors": ["Mingtong Zhang", "Kaifeng Zhang", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-10-24", "topics": ["dynamic 3D Gaussian tracking", "graph dynamics from monocular video", "rendering-in-the-loop"], "paradigm": ["model-based RL"], "method": [], "projects": ["r2s2r"], "related": ["adaptigraph", "d3fields", "particle-grid-dynamics", "phystwin"], "arxiv": "2410.18912", "doi": "10.48550/arXiv.2410.18912", "code": "", "site": "", "note": "<b>GS-Dynamics</b> — learn graph-based neural dynamics by tracking dynamic 3D Gaussians, so the dynamics model is supervised through a <em>renderer</em> from ordinary RGB video rather than from particle ground truth nobody has. The bridge between 'structured dynamics models are better' and 'but we can only collect video'.", "aka": "GS-Dynamics", "stars": 2, "curated": true, "local": "", "abstract": "Videos of robots interacting with objects encode rich information about the objects' dynamics. However, existing video prediction approaches typically do not explicitly account for the 3D information from videos, such as robot actions and objects' 3D states, limiting their use in real-world robotic applications. In this work, we introduce a framework to learn object dynamics directly from multi-view RGB videos by explicitly considering the robot's action trajectories and their effects on scene dynamics. We utilize the 3D Gaussian representation of 3D Gaussian Splatting (3DGS) to train a particle-based dynamics model using Graph Neural Networks. This model operates on sparse control particles downsampled from the densely tracked 3D Gaussian reconstructions. By learning the neural dynamics model on offline robot interaction data, our method can predict object motions under varying initial configurations and unseen robot actions. The 3D transformations of Gaussians can be interpolated from the motions of control particles, enabling the rendering of predicted future object states and achieving action-conditioned video prediction. The dynamics model can also be applied to model-based planning frameworks for object manipulation tasks. We conduct experiments on various kinds of deformable materials, including ropes, clothes, and stuffed animals, demonstrating our framework's ability to model complex shapes and dynamics. Our project page is available at https://gs-dynamics.github.io.", "source": "curated", "first_author": "Mingtong Zhang", "url": "https://arxiv.org/abs/2410.18912", "fig": "figures/gs-dynamics.webp", "figs": ["figures/gs-dynamics-1.webp"], "arxiv_url": "https://arxiv.org/abs/2410.18912"}, {"id": "llm-teams", "title": "Embodied LLM Agents Learn to Cooperate in Organized Teams", "tree": "systems", "branch": "agentic", "authors": ["Xudong Guo", "Kaixuan Huang", "Jiale Liu", "Wenhui Fan", "Natalia Vélez", "Qingyun Wu", "Huazheng Wang", "Thomas L. Griffiths", "Mengdi Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-03-19", "topics": ["organised teams", "emergent cooperation", "LLM agents"], "paradigm": ["LLM"], "method": [], "projects": [], "related": ["embo-team", "roco", "voyager"], "arxiv": "2403.12482", "doi": "10.48550/arxiv.2403.12482", "code": "", "site": "", "note": "Non-robotic, but the finding transfers: giving LLM agents an explicit organisational structure changes how well they cooperate. Relevant if M2's agents ever need roles rather than symmetry.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Large Language Models (LLMs) have emerged as integral tools for reasoning, planning, and decision-making, drawing upon their extensive world knowledge and proficiency in language-related tasks. LLMs thus hold tremendous potential for natural language interaction within multi-agent systems to foster cooperation. However, LLM agents tend to over-report and comply with any instruction, which may result in information redundancy and confusion in multi-agent cooperation. Inspired by human organizations, this paper introduces a framework that imposes prompt-based organization structures on LLM agents to mitigate these problems. Through a series of experiments with embodied LLM agents and human-agent collaboration, our results highlight the impact of designated leadership on team efficiency, shedding light on the leadership qualities displayed by LLM agents and their spontaneous cooperative behaviors. Further, we harness the potential of LLMs to propose enhanced organizational prompts, via a Criticize-Reflect process, resulting in novel organization structures that reduce communication costs and enhance team efficiency.", "source": "curated", "first_author": "Xudong Guo", "url": "https://arxiv.org/abs/2403.12482", "fig": "figures/llm-teams.webp", "figs": ["figures/llm-teams-1.webp", "figures/llm-teams-2.webp"]}, {"id": "exbody", "title": "Expressive Whole-Body Control for Humanoid Robots", "tree": "il", "branch": "retargeting", "authors": ["Xuxin Cheng", "Yandong Ji", "Junming Chen", "Ruihan Yang", "Ge Yang", "Xiaolong Wang"], "corresponding": ["Xuxin Cheng"], "institutions": ["UC San Diego", "MIT"], "lab": "UCSD Xiaolong Wang Lab", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-02-26", "topics": ["expressive motion", "upper-body tracking"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": ["deepmimic", "exbody2", "h2o", "hover"], "arxiv": "2402.16796", "doi": "", "code": "", "site": "", "note": "Track the upper body expressively, keep the lower body robust. The asymmetry is the insight — you do not need the legs to imitate, you need them to not fall.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Can we enable humanoid robots to generate rich, diverse, and expressive motions in the real world? We propose to learn a whole-body control policy on a human-sized robot to mimic human motions as realistic as possible. To train such a policy, we leverage the large-scale human motion capture data from the graphics community in a Reinforcement Learning framework. However, directly performing imitation learning with the motion capture dataset would not work on the real humanoid robot, given the large gap in degrees of freedom and physical capabilities. Our method Expressive Whole-Body Control (Exbody) tackles this problem by encouraging the upper humanoid body to imitate a reference motion, while relaxing the imitation constraint on its two legs and only requiring them to follow a given velocity robustly. With training in simulation and Sim2Real transfer, our policy can control a humanoid robot to walk in different styles, shake hands with humans, and even dance with a human in the real world. We conduct extensive studies and comparisons on diverse motions in both simulation and the real world to show the effectiveness of our approach.", "source": "curated", "first_author": "Xuxin Cheng", "url": "https://arxiv.org/abs/2402.16796", "fig": "figures/exbody.webp", "figs": ["figures/exbody-1.webp", "figures/exbody-2.webp", "figures/exbody-3.webp"]}, {"id": "extreme-parkour", "title": "Extreme Parkour with Legged Robots", "tree": "rl", "branch": "locomotion", "authors": ["Xuxin Cheng", "Kexin Shi", "Ananye Agarwal", "Deepak Pathak"], "corresponding": ["Xuxin Cheng"], "institutions": ["Carnegie Mellon University"], "lab": "CMU Robotics Institute (Pathak Lab)", "venue": "ICRA", "venue_type": "conference", "year": 2024, "date": "2023-09-25", "topics": ["parkour", "agile locomotion", "depth vision"], "paradigm": ["RL"], "method": ["inner-product reward for direction", "single depth camera", "two-stage distillation"], "projects": [], "related": ["anymal-parkour", "miki2022"], "arxiv": "2309.14341", "doi": "", "code": "", "site": "", "note": "Shows how far a single front-facing depth camera plus a well-shaped direction reward can go. Contrast with ANYmal parkour, which leans on a planner and more sensing.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humans can perform parkour by traversing obstacles in a highly dynamic fashion requiring precise eye-muscle coordination and movement. Getting robots to do the same task requires overcoming similar challenges. Classically, this is done by independently engineering perception, actuation, and control systems to very low tolerances. This restricts them to tightly controlled settings such as a predetermined obstacle course in labs. In contrast, humans are able to learn parkour through practice without significantly changing their underlying biology. In this paper, we take a similar approach to developing robot parkour on a small low-cost robot with imprecise actuation and a single front-facing depth camera for perception which is low-frequency, jittery, and prone to artifacts. We show how a single neural net policy operating directly from a camera image, trained in simulation with large-scale RL, can overcome imprecise sensing and actuation to output highly precise control behavior end-to-end. We show our robot can perform a high jump on obstacles 2x its height, long jump across gaps 2x its length, do a handstand and run across tilted ramps, and generalize to novel obstacle courses with different physical properties. Parkour videos at https://extreme-parkour.github.io/", "source": "curated", "first_author": "Xuxin Cheng", "url": "https://arxiv.org/abs/2309.14341", "fig": "figures/extreme-parkour.webp", "figs": ["figures/extreme-parkour-1.webp"]}, {"id": "fast-umi", "title": "FastUMI: A Scalable and Hardware-Independent Universal Manipulation Interface with Dataset", "tree": "systems", "branch": "interfaces", "authors": ["Ziniu Wu", "Tianyu Wang", "Zhaxizhuoma", "Chuyue Guan", "Zhongjie Jia", "Shuai Liang", "et al."], "corresponding": [], "institutions": ["Shanghai AI Laboratory"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-09-29", "topics": ["UMI variant", "hardware independence", "tracking"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["dexumi", "ume", "umi"], "arxiv": "2409.19499", "doi": "", "code": "", "site": "", "note": "Replaces UMI's SLAM pipeline with an off-the-shelf tracker to cut calibration pain. Relevant if you want capture that a non-expert can run.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Real-world manipulation data involving robotic arms is crucial for developing generalist action policies, yet such data remains scarce since existing data collection methods are hindered by high costs, hardware dependencies, and complex setup requirements. In this work, we introduce FastUMI, a substantial redesign of the Universal Manipulation Interface (UMI) system that addresses these challenges by enabling rapid deployment, simplifying hardware-software integration, and delivering robust performance in real-world data acquisition. Compared with UMI, FastUMI has several advantages: 1) It adopts a decoupled hardware design and incorporates extensive mechanical modifications, removing dependencies on specialized robotic components while preserving consistent observation perspectives. 2) It also refines the algorithmic pipeline by replacing complex Visual-Inertial Odometry (VIO) implementations with an off-the-shelf tracking module, significantly reducing deployment complexity while maintaining accuracy. 3) FastUMI includes an ecosystem for data collection, verification, and integration with both established and newly developed imitation learning algorithms, accelerating policy learning advancement. Additionally, we have open-sourced a high-quality dataset of over 10,000 real-world demonstration trajectories spanning 22 everyday tasks, forming one of the most diverse UMI-like datasets to date. Experimental results confirm that FastUMI facilitates rapid deployment, reduces operational costs and labor demands, and maintains robust performance across diverse manipulation scenarios, thereby advancing scalable data-driven robotic learning.", "source": "curated", "first_author": "Ziniu Wu", "url": "https://arxiv.org/abs/2409.19499", "fig": "figures/fast-umi.webp", "figs": ["figures/fast-umi-1.webp", "figures/fast-umi-2.webp", "figures/fast-umi-3.webp"]}, {"id": "gr2", "title": "GR-2: A Generative Video-Language-Action Model with Web-Scale Knowledge for Robot Manipulation", "tree": "il", "branch": "world-models", "authors": ["Chi-Lam Cheang", "Guangzeng Chen", "Ya Jing", "Tao Kong", "Hanbo Zhang", "Hang Li", "et al."], "corresponding": [], "institutions": ["ByteDance Research"], "lab": "ByteDance Seed", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-10-08", "topics": ["video-language-action", "web-scale pretraining"], "paradigm": ["world model", "VLA"], "method": [], "projects": [], "related": ["dreamgen", "flare", "gr1", "unipi", "vjepa2"], "arxiv": "2410.06158", "doi": "", "code": "", "site": "", "note": "Video pretraining at 38M clips, then joint video+action prediction. The 'predict the future you are about to cause' formulation in its most scaled form.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We present GR-2, a state-of-the-art generalist robot agent for versatile and generalizable robot manipulation. GR-2 is first pre-trained on a vast number of Internet videos to capture the dynamics of the world. This large-scale pre-training, involving 38 million video clips and over 50 billion tokens, equips GR-2 with the ability to generalize across a wide range of robotic tasks and environments during subsequent policy learning. Following this, GR-2 is fine-tuned for both video generation and action prediction using robot trajectories. It exhibits impressive multi-task learning capabilities, achieving an average success rate of 97.7% across more than 100 tasks. Moreover, GR-2 demonstrates exceptional generalization to new, previously unseen scenarios, including novel backgrounds, environments, objects, and tasks. Notably, GR-2 scales effectively with model size, underscoring its potential for continued growth and application. Project page: \\url{https://gr2-manipulation.github.io}.", "source": "curated", "first_author": "Chi-Lam Cheang", "url": "https://arxiv.org/abs/2410.06158", "fig": "figures/gr2.webp", "figs": ["figures/gr2-1.webp", "figures/gr2-2.webp", "figures/gr2-3.webp"]}, {"id": "genesis", "title": "Genesis: A Universal and Generative Physics Engine for Robotics and Beyond", "tree": "systems", "branch": "sim-bench", "authors": ["Genesis Authors (Zhou Xian", "et al.)"], "corresponding": [], "institutions": ["CMU", "Multi-institution consortium"], "lab": "", "venue": "open source", "venue_type": "software", "year": 2024, "date": "", "topics": ["universal physics engine", "generative simulation", "speed"], "paradigm": ["simulator"], "method": [], "projects": [], "related": ["isaacgym", "isaaclab", "maniskill", "mujoco", "newton", "simweaver"], "arxiv": "", "doi": "", "code": "https://github.com/Genesis-Embodied-AI/Genesis", "site": "", "note": "Claims very large speedups and unified multi-physics (rigid, MPM, SPH, cloth). The multi-material support is the reason to care for soft-object manipulation. Benchmark it yourself before believing the headline numbers.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Genesis Authors (Zhou Xian", "url": "https://github.com/Genesis-Embodied-AI/Genesis"}, {"id": "humanoidbench", "title": "HumanoidBench: Simulated Humanoid Benchmark for Whole-Body Locomotion and Manipulation", "tree": "systems", "branch": "sim-bench", "authors": ["Carmelo Sferrazza", "Dun-Ming Huang", "Xingyu Lin", "Youngwoon Lee", "Pieter Abbeel"], "corresponding": ["Carmelo Sferrazza"], "institutions": ["UC Berkeley"], "lab": "BAIR", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-03-15", "topics": ["humanoid benchmark", "whole-body tasks"], "paradigm": ["benchmark"], "method": [], "projects": ["mabel"], "related": ["libero", "robocasa", "visualmimic"], "arxiv": "2403.10506", "doi": "", "code": "", "site": "", "note": "In your Archieve/Humaniod Benchmark folder. 27 whole-body tasks; the honest finding is that current RL solves locomotion and fails at the manipulation half.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/HumanoidBench_Carmelo Sferrazza 2024.pdf", "abstract": "Humanoid robots hold great promise in assisting humans in diverse environments and tasks, due to their flexibility and adaptability leveraging human-like morphology. However, research in humanoid robots is often bottlenecked by the costly and fragile hardware setups. To accelerate algorithmic research in humanoid robots, we present a high-dimensional, simulated robot learning benchmark, HumanoidBench, featuring a humanoid robot equipped with dexterous hands and a variety of challenging whole-body manipulation and locomotion tasks. Our findings reveal that state-of-the-art reinforcement learning algorithms struggle with most tasks, whereas a hierarchical learning approach achieves superior performance when supported by robust low-level policies, such as walking or reaching. With HumanoidBench, we provide the robotics community with a platform to identify the challenges arising when solving diverse tasks with humanoid robots, facilitating prompt verification of algorithms and ideas. The open-source code is available at https://humanoid-bench.github.io.", "source": "curated", "first_author": "Carmelo Sferrazza", "url": "https://arxiv.org/abs/2403.10506", "fig": "figures/humanoidbench.webp", "figs": ["figures/humanoidbench-1.webp", "figures/humanoidbench-2.webp", "figures/humanoidbench-3.webp"]}, {"id": "navigation-world-models", "title": "Navigation World Models", "tree": "rl", "branch": "model-based", "authors": ["Amir Bar", "Gaoyue Zhou", "Danny Tran", "Trevor Darrell", "Yann LeCun"], "corresponding": [], "institutions": ["New York University", "Meta AI"], "lab": "Meta FAIR", "venue": "CVPR", "venue_type": "conference", "year": 2024, "date": "2024-12-04", "topics": ["conditional diffusion transformer", "navigation as trajectory imagination", "unfamiliar environments"], "paradigm": ["world model"], "method": [], "projects": ["mabel", "wam-tactile"], "related": ["dino-wm", "gaia-2", "navdreamer", "vista-wm", "vjepa2"], "arxiv": "2412.03572", "doi": "10.1109/CVPR52734.2025.01472", "code": "", "site": "", "note": "<b>Navigation World Models</b> (Bar, LeCun et al.) — plan navigation by imagining future observations under candidate trajectories, with a conditional diffusion transformer, including in environments never seen before. The clearest demonstration that a world model can do planning work in a domain where classical mapping already works well — which is the bar it has to clear.", "aka": "NWM", "stars": 2, "curated": true, "local": "", "abstract": "Navigation is a fundamental skill of agents with visual-motor capabilities. We introduce a Navigation World Model (NWM), a controllable video generation model that predicts future visual observations based on past observations and navigation actions. To capture complex environment dynamics, NWM employs a Conditional Diffusion Transformer (CDiT), trained on a diverse collection of egocentric videos of both human and robotic agents, and scaled up to 1 billion parameters. In familiar environments, NWM can plan navigation trajectories by simulating them and evaluating whether they achieve the desired goal. Unlike supervised navigation policies with fixed behavior, NWM can dynamically incorporate constraints during planning. Experiments demonstrate its effectiveness in planning trajectories from scratch or by ranking trajectories sampled from an external policy. Furthermore, NWM leverages its learned visual priors to imagine trajectories in unfamiliar environments from a single input image, making it a flexible and powerful tool for next-generation navigation systems.", "source": "curated", "first_author": "Amir Bar", "url": "https://arxiv.org/abs/2412.03572", "fig": "figures/navigation-world-models.webp", "figs": ["figures/navigation-world-models-1.webp"], "arxiv_url": "https://arxiv.org/abs/2412.03572"}, {"id": "octo", "title": "Octo: An Open-Source Generalist Robot Policy", "tree": "il", "branch": "visuomotor", "authors": ["Octo Model Team: Dibya Ghosh", "Homer Walke", "Karl Pertsch", "Kevin Black", "Oier Mees", "et al."], "corresponding": [], "institutions": ["UC Berkeley", "Stanford University", "Google DeepMind"], "lab": "BAIR (RAIL)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-05-20", "topics": ["generalist policy", "open weights", "cross-embodiment"], "paradigm": ["IL", "transformer"], "method": ["transformer with diffusion action head", "trained on Open X-Embodiment"], "projects": [], "related": ["bridgedata", "crossformer", "openvla", "oxe", "rt1"], "arxiv": "2405.12213", "doi": "", "code": "", "site": "", "note": "The open baseline before OpenVLA. Still the easiest generalist policy to actually fine-tune.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Large policies pretrained on diverse robot datasets have the potential to transform robotic learning: instead of training new policies from scratch, such generalist robot policies may be finetuned with only a little in-domain data, yet generalize broadly. However, to be widely applicable across a range of robotic learning scenarios, environments, and tasks, such policies need to handle diverse sensors and action spaces, accommodate a variety of commonly used robotic platforms, and finetune readily and efficiently to new domains. In this work, we aim to lay the groundwork for developing open-source, widely applicable, generalist policies for robotic manipulation. As a first step, we introduce Octo, a large transformer-based policy trained on 800k trajectories from the Open X-Embodiment dataset, the largest robot manipulation dataset to date. It can be instructed via language commands or goal images and can be effectively finetuned to robot setups with new sensory inputs and action spaces within a few hours on standard consumer GPUs. In experiments across 9 robotic platforms, we demonstrate that Octo serves as a versatile policy initialization that can be effectively finetuned to new observation and action spaces. We also perform detailed ablations of design decisions for the Octo model, from architecture to training data, to guide future research on building generalist robot models.", "source": "curated", "first_author": "Octo Model Team: Dibya Ghosh", "url": "https://arxiv.org/abs/2405.12213", "fig": "figures/octo.webp", "figs": ["figures/octo-1.webp", "figures/octo-2.webp"]}, {"id": "open-television", "title": "Open-TeleVision: Teleoperation with Immersive Active Visual Feedback", "tree": "il", "branch": "retargeting", "authors": ["Xuxin Cheng", "Jialong Li", "Shiqi Yang", "Ge Yang", "Xiaolong Wang"], "corresponding": ["Xuxin Cheng"], "institutions": ["UC San Diego", "MIT"], "lab": "UCSD Xiaolong Wang Lab", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-07-01", "topics": ["VR teleoperation", "active vision", "open hardware"], "paradigm": ["teleop"], "method": [], "projects": ["mabel"], "related": ["anyteleop", "bunny-visionpro", "gello", "lucid-xr", "modpack", "umi"], "arxiv": "2407.01512", "doi": "", "code": "", "site": "", "note": "Open-source stereo VR teleop with active head tracking. Cheap way to get high-quality bimanual demonstrations — a MABEL-relevant build.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Teleoperation serves as a powerful method for collecting on-robot data essential for robot learning from demonstrations. The intuitiveness and ease of use of the teleoperation system are crucial for ensuring high-quality, diverse, and scalable data. To achieve this, we propose an immersive teleoperation system Open-TeleVision that allows operators to actively perceive the robot's surroundings in a stereoscopic manner. Additionally, the system mirrors the operator's arm and hand movements on the robot, creating an immersive experience as if the operator's mind is transmitted to a robot embodiment. We validate the effectiveness of our system by collecting data and training imitation learning policies on four long-horizon, precise tasks (Can Sorting, Can Insertion, Folding, and Unloading) for 2 different humanoid robots and deploy them in the real world. The system is open-sourced at: https://robot-tv.github.io/", "source": "curated", "first_author": "Xuxin Cheng", "url": "https://arxiv.org/abs/2407.01512", "fig": "figures/open-television.webp", "figs": ["figures/open-television-1.webp", "figures/open-television-2.webp", "figures/open-television-3.webp"]}, {"id": "robocasa", "title": "RoboCasa: Large-Scale Simulation of Everyday Tasks for Generalist Robots", "tree": "systems", "branch": "sim-bench", "authors": ["Soroush Nasiriany", "Abhiram Maddukuri", "Lance Zhang", "Adeet Parikh", "Aaron Lo", "Abhishek Joshi", "Ajay Mandlekar", "Yuke Zhu"], "corresponding": [], "institutions": ["UT Austin", "NVIDIA"], "lab": "UT Austin RobIn Lab (Yuke Zhu)", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-06-04", "topics": ["household simulation", "generative scenes", "benchmark"], "paradigm": ["simulator", "benchmark"], "method": [], "projects": [], "related": ["behavior-robot-suite", "humanoidbench", "libero", "maniskill"], "arxiv": "2406.02523", "doi": "", "code": "", "site": "", "note": "120 kitchen scenes, 2500+ objects, generative task variation. The scaling story for simulated household data.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent advancements in Artificial Intelligence (AI) have largely been propelled by scaling. In Robotics, scaling is hindered by the lack of access to massive robot datasets. We advocate using realistic physical simulation as a means to scale environments, tasks, and datasets for robot learning methods. We present RoboCasa, a large-scale simulation framework for training generalist robots in everyday environments. RoboCasa features realistic and diverse scenes focusing on kitchen environments. We provide thousands of 3D assets across over 150 object categories and dozens of interactable furniture and appliances. We enrich the realism and diversity of our simulation with generative AI tools, such as object assets from text-to-3D models and environment textures from text-to-image models. We design a set of 100 tasks for systematic evaluation, including composite tasks generated by the guidance of large language models. To facilitate learning, we provide high-quality human demonstrations and integrate automated trajectory generation methods to substantially enlarge our datasets with minimal human burden. Our experiments show a clear scaling trend in using synthetically generated robot data for large-scale imitation learning and show great promise in harnessing simulation data in real-world tasks. Videos and open-source code are available at https://robocasa.ai/", "source": "curated", "first_author": "Soroush Nasiriany", "url": "https://arxiv.org/abs/2406.02523", "fig": "figures/robocasa.webp", "figs": ["figures/robocasa-1.webp", "figures/robocasa-2.webp", "figures/robocasa-3.webp"]}, {"id": "robo-exp", "title": "RoboEXP: Action-Conditioned Scene Graph via Interactive Exploration for Robotic Manipulation", "tree": "systems", "branch": "perception", "authors": ["Hanxiao Jiang", "Binghao Huang", "Ruihai Wu", "Zhuoran Li", "Shubham Garg", "Hooshang Nayyeri", "Shenlong Wang", "Yunzhu Li"], "corresponding": [], "institutions": ["Columbia University", "University of Illinois Urbana-Champaign", "Peking University", "NUS"], "lab": "RoboPIL (Yunzhu Li)", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-02-23", "topics": ["action-conditioned scene graph", "interactive exploration", "verify by acting"], "paradigm": ["representation"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["acdc", "d3fields", "rekep", "robocook", "voxposer"], "arxiv": "2402.15487", "doi": "10.48550/arXiv.2402.15487", "code": "", "site": "", "note": "<b>RoboEXP</b> (CoRL 2024) — build the scene graph by <em>interacting</em>: open the drawer to discover what is in it, rather than inferring it from a single view. The action-conditioned scene graph is the right data structure for an agentic system that has to construct its own understanding of a scene, and it is the piece your agentic-physical idea is currently missing.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce the novel task of interactive scene exploration, wherein robots autonomously explore environments and produce an action-conditioned scene graph (ACSG) that captures the structure of the underlying environment. The ACSG accounts for both low-level information (geometry and semantics) and high-level information (action-conditioned relationships between different entities) in the scene. To this end, we present the Robotic Exploration (RoboEXP) system, which incorporates the Large Multimodal Model (LMM) and an explicit memory design to enhance our system's capabilities. The robot reasons about what and how to explore an object, accumulating new information through the interaction process and incrementally constructing the ACSG. Leveraging the constructed ACSG, we illustrate the effectiveness and efficiency of our RoboEXP system in facilitating a wide range of real-world manipulation tasks involving rigid, articulated objects, nested objects, and deformable objects.", "source": "curated", "first_author": "Hanxiao Jiang", "url": "https://arxiv.org/abs/2402.15487", "fig": "figures/robo-exp.webp", "figs": ["figures/robo-exp-1.webp", "figures/robo-exp-2.webp"]}, {"id": "robomorph", "title": "RoboMorph: Evolving Robot Morphology using Large Language Models", "tree": "systems", "branch": "codesign", "authors": ["Kevin Qiu", "Krzysztof Ciebiera", "Paweł Fijałkowski", "Marek Cygan", "Łukasz Kuciński"], "corresponding": [], "institutions": ["University of Warsaw", "IDEAS NCBR"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-07-11", "topics": ["LLM-driven morphology evolution"], "paradigm": ["co-design", "LLM"], "method": [], "projects": [], "related": ["eureka", "rf-agent", "robogrammar", "text2robot"], "arxiv": "2407.08626", "doi": "", "code": "", "site": "", "note": "An LLM as the mutation operator in a morphology search loop, with automatic reward design alongside. Directly the mechanism your agentic co-design idea proposes — read it to see what is already claimed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce RoboMorph, an automated approach for generating and optimizing modular robot designs using large language models (LLMs) and evolutionary algorithms. Each robot design is represented by a structured grammar, and we use LLMs to efficiently explore this design space. Traditionally, such exploration is time-consuming and computationally intensive. Using a best-shot prompting strategy combined with reinforcement learning (RL)-based control evaluation, RoboMorph iteratively refines robot designs within an evolutionary feedback loop. Across four terrain types, RoboMorph discovers diverse, terrain-specialized morphologies, including wheeled quadrupeds and hexapods, that match or outperform designs produced by Robogrammar's graph-search method. These results demonstrate that LLMs, when coupled with evolutionary selection, can serve as effective generative operators for automated robot design. Our project page and code are available at https://robomorph.github.io.", "source": "curated", "first_author": "Kevin Qiu", "url": "https://arxiv.org/abs/2407.08626", "fig": "figures/robomorph.webp", "figs": ["figures/robomorph-1.webp", "figures/robomorph-2.webp", "figures/robomorph-3.webp"]}, {"id": "crossformer", "title": "Scaling Cross-Embodied Learning: One Policy for Manipulation, Navigation, Locomotion and Aviation", "tree": "il", "branch": "visuomotor", "authors": ["Ria Doshi", "Homer Walke", "Oier Mees", "Sudeep Dasari", "Sergey Levine"], "corresponding": ["Ria Doshi"], "institutions": ["UC Berkeley"], "lab": "BAIR (RAIL)", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-08-21", "topics": ["cross-embodiment", "unified policy"], "paradigm": ["IL", "transformer"], "method": [], "projects": ["m2"], "related": ["chorus", "cross-humanoid-wbc", "multi-loco", "octo", "oxe", "skild"], "arxiv": "2408.11812", "doi": "", "code": "", "site": "", "note": "One transformer across arms, quadrupeds, drones and navigation, with no shared action space. Relevant to M2 if you want one policy across heterogeneous robots.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Modern machine learning systems rely on large datasets to attain broad generalization, and this often poses a challenge in robot learning, where each robotic platform and task might have only a small dataset. By training a single policy across many different kinds of robots, a robot learning method can leverage much broader and more diverse datasets, which in turn can lead to better generalization and robustness. However, training a single policy on multi-robot data is challenging because robots can have widely varying sensors, actuators, and control frequencies. We propose CrossFormer, a scalable and flexible transformer-based policy that can consume data from any embodiment. We train CrossFormer on the largest and most diverse dataset to date, 900K trajectories across 20 different robot embodiments. We demonstrate that the same network weights can control vastly different robots, including single and dual arm manipulation systems, wheeled robots, quadcopters, and quadrupeds. Unlike prior work, our model does not require manual alignment of the observation or action spaces. Extensive experiments in the real world show that our method matches the performance of specialist policies tailored for each embodiment, while also significantly outperforming the prior state of the art in cross-embodiment learning.", "source": "curated", "first_author": "Ria Doshi", "url": "https://arxiv.org/abs/2408.11812", "fig": "figures/crossformer.webp", "figs": ["figures/crossformer-1.webp"]}, {"id": "tdmpc2", "title": "TD-MPC2: Scalable, Robust World Models for Continuous Control", "tree": "rl", "branch": "model-based", "authors": ["Nicklas Hansen", "Hao Su", "Xiaolong Wang"], "corresponding": ["Nicklas Hansen"], "institutions": ["UC San Diego"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2024, "date": "2023-10-25", "topics": ["model predictive control", "latent dynamics", "multi-task"], "paradigm": ["model-based RL"], "method": ["decoder-free latent world model + MPPI planning"], "projects": [], "related": ["daydreamer", "dreamerv3", "muzero", "pwm", "td-mpc"], "arxiv": "2310.16828", "doi": "", "code": "", "site": "", "note": "Decoder-free: never reconstructs pixels, only predicts reward and value. Much cheaper than Dreamer and often stronger on continuous control.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "TD-MPC is a model-based reinforcement learning (RL) algorithm that performs local trajectory optimization in the latent space of a learned implicit (decoder-free) world model. In this work, we present TD-MPC2: a series of improvements upon the TD-MPC algorithm. We demonstrate that TD-MPC2 improves significantly over baselines across 104 online RL tasks spanning 4 diverse task domains, achieving consistently strong results with a single set of hyperparameters. We further show that agent capabilities increase with model and data size, and successfully train a single 317M parameter agent to perform 80 tasks across multiple task domains, embodiments, and action spaces. We conclude with an account of lessons, opportunities, and risks associated with large TD-MPC2 agents. Explore videos, models, data, code, and more at https://tdmpc2.com", "source": "curated", "first_author": "Nicklas Hansen", "url": "https://arxiv.org/abs/2310.16828", "fig": "figures/tdmpc2.webp", "figs": ["figures/tdmpc2-1.webp", "figures/tdmpc2-2.webp", "figures/tdmpc2-3.webp"]}, {"id": "ai-scientist", "title": "The AI Scientist: Towards Fully Automated Open-Ended Scientific Discovery", "tree": "systems", "branch": "agentic", "authors": ["Chris Lu", "Cong Lu", "Robert Tjarko Lange", "Jakob Foerster", "Jeff Clune", "David Ha"], "corresponding": [], "institutions": ["Sakana AI", "University of Oxford", "UBC"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-08-12", "topics": ["automated research", "hypothesis generation", "paper writing"], "paradigm": ["LLM"], "method": [], "projects": [], "related": ["agent-laboratory", "eureka", "real-deep-research", "voyager"], "arxiv": "2408.06292", "doi": "", "code": "", "site": "", "note": "Non-robotics, but the reference point for what 'auto-research' claims and where it breaks down (idea novelty and evaluation, not code generation). Read the criticism alongside it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "One of the grand challenges of artificial general intelligence is developing agents capable of conducting scientific research and discovering new knowledge. While frontier models have already been used as aides to human scientists, e.g. for brainstorming ideas, writing code, or prediction tasks, they still conduct only a small part of the scientific process. This paper presents the first comprehensive framework for fully automatic scientific discovery, enabling frontier large language models to perform research independently and communicate their findings. We introduce The AI Scientist, which generates novel research ideas, writes code, executes experiments, visualizes results, describes its findings by writing a full scientific paper, and then runs a simulated review process for evaluation. In principle, this process can be repeated to iteratively develop ideas in an open-ended fashion, acting like the human scientific community. We demonstrate its versatility by applying it to three distinct subfields of machine learning: diffusion modeling, transformer-based language modeling, and learning dynamics. Each idea is implemented and developed into a full paper at a cost of less than $15 per paper. To evaluate the generated papers, we design and validate an automated reviewer, which we show achieves near-human performance in evaluating paper scores. The AI Scientist can produce papers that exceed the acceptance threshold at a top machine learning conference as judged by our automated reviewer. This approach signifies the beginning of a new era in scientific discovery in machine learning: bringing the transformative benefits of AI agents to the entire research process of AI itself, and taking us closer to a world where endless affordable creativity and innovation can be unleashed on the world's most challenging problems. Our code is open-sourced at https://github.com/SakanaAI/AI-Scientist", "source": "curated", "first_author": "Chris Lu", "url": "https://arxiv.org/abs/2408.06292", "fig": "figures/ai-scientist.webp"}, {"id": "wheeled-payload", "title": "Toward Control of Wheeled Humanoid Robots with Unknown Payloads: Equilibrium Point Estimation via Real-to-Sim Adaptation", "tree": "rl", "branch": "sim2real", "authors": ["Donghoon Baek", "Youngwoo Sim", "Amartya Purushottam", "Saurabh Gupta", "João Ramos"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-03-16", "topics": ["unknown payloads", "equilibrium point estimation", "real-to-sim adaptation"], "paradigm": ["control", "real2sim"], "method": [], "projects": ["mabel", "r2s2r"], "related": ["asap", "dexndm", "real2sim", "wheeled-bilateral", "wheeled-multistage"], "arxiv": "2403.10948", "doi": "10.48550/arxiv.2403.10948", "code": "", "site": "", "note": "Equilibrium-point estimation for wheeled humanoids carrying unknown payloads, via real-to-sim adaptation. The real-to-sim framing connects it straight to your Real2Sim2Real project, on the exact morphology MABEL has.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Model-based controllers using a linearized model around the system's equilibrium point is a common approach in the control of a wheeled humanoid due to their less computational load and ease of stability analysis. However, controlling a wheeled humanoid robot while it lifts an unknown object presents significant challenges, primarily due to the lack of knowledge in object dynamics. This paper presents a framework designed for predicting the new equilibrium point explicitly to control a wheeled-legged robot with unknown dynamics. We estimated the total mass and center of mass of the system from its response to initially unknown dynamics, then calculated the new equilibrium point accordingly. To avoid using additional sensors (e.g., force torque sensor) and reduce the effort of obtaining expensive real data, a data-driven approach is utilized with a novel real-to-sim adaptation. A more accurate nonlinear dynamics model, offering a closer representation of real-world physics, is injected into a rigid-body simulation for real-to-sim adaptation. The nonlinear dynamics model parameters were optimized using Particle Swarm Optimization. The efficacy of this framework was validated on a physical wheeled inverted pendulum, a simplified model of a wheeled-legged robot. The experimental results indicate that employing a more precise analytical model with optimized parameters significantly reduces the gap between simulation and reality, thus improving the efficiency of a model-based controller in controlling a wheeled robot with unknown dynamics", "source": "curated", "first_author": "Donghoon Baek", "url": "https://arxiv.org/abs/2403.10948", "fig": "figures/wheeled-payload.webp", "figs": ["figures/wheeled-payload-1.webp", "figures/wheeled-payload-2.webp", "figures/wheeled-payload-3.webp"]}, {"id": "unitree-g1", "title": "Unitree G1 / H1 Humanoid Platforms", "tree": "systems", "branch": "humanoid-hw", "authors": ["Unitree Robotics"], "corresponding": [], "institutions": ["Unitree Robotics"], "lab": "", "venue": "commercial", "venue_type": "industry", "year": 2024, "date": "", "topics": ["commercial humanoid", "low cost", "research platform"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["asap", "berkeley-humanoid", "groot-n1", "hover", "mit-cheetah"], "arxiv": "", "doi": "", "code": "", "site": "https://www.unitree.com/g1", "note": "The platform most 2024-26 humanoid learning papers in this library actually run on. Worth tracking because it sets the de-facto hardware assumptions of the field.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Unitree Robotics", "url": "https://www.unitree.com/g1"}, {"id": "gr1", "title": "Unleashing Large-Scale Video Generative Pre-training for Visual Robot Manipulation", "tree": "il", "branch": "world-models", "authors": ["Hongtao Wu", "Ya Jing", "Chilam Cheang", "Guangzeng Chen", "Jiafeng Xu", "Xinghang Li", "Minghuan Liu", "Hang Li", "Tao Kong"], "corresponding": ["Tao Kong"], "institutions": ["ByteDance Research"], "lab": "ByteDance Seed / Robotics", "venue": "ICLR", "venue_type": "conference", "year": 2024, "date": "2023-12-20", "topics": ["video pretraining", "manipulation", "GPT-style"], "paradigm": ["world model", "IL"], "method": [], "projects": [], "related": ["gr2", "unipi"], "arxiv": "2312.13139", "doi": "", "code": "", "site": "", "note": "Pretrain on video prediction, fine-tune to predict actions and future frames jointly. The direct ancestor of the world-action-model family.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Generative pre-trained models have demonstrated remarkable effectiveness in language and vision domains by learning useful representations. In this paper, we extend the scope of this effectiveness by showing that visual robot manipulation can significantly benefit from large-scale video generative pre-training. We introduce GR-1, a straightforward GPT-style model designed for multi-task language-conditioned visual robot manipulation. GR-1 takes as inputs a language instruction, a sequence of observation images, and a sequence of robot states. It predicts robot actions as well as future images in an end-to-end manner. Thanks to a flexible design, GR-1 can be seamlessly finetuned on robot data after pre-trained on a large-scale video dataset. We perform extensive experiments on the challenging CALVIN benchmark and a real robot. On CALVIN benchmark, our method outperforms state-of-the-art baseline methods and improves the success rate from 88.9% to 94.9%. In the setting of zero-shot unseen scene generalization, GR-1 improves the success rate from 53.3% to 85.4%. In real robot experiments, GR-1 also outperforms baseline methods and shows strong potentials in generalization to unseen scenes and objects. We provide inaugural evidence that a unified GPT-style transformer, augmented with large-scale video generative pre-training, exhibits remarkable generalization to multi-task visual robot manipulation. Project page: https://GR1-Manipulation.github.io", "source": "curated", "first_author": "Hongtao Wu", "url": "https://arxiv.org/abs/2312.13139", "fig": "figures/gr1.webp", "figs": ["figures/gr1-1.webp"]}, {"id": "vid2robot", "title": "Vid2Robot: End-to-end Video-conditioned Policy Learning with Cross-Attention Transformers", "tree": "il", "branch": "in-context", "authors": ["Vidhi Jain", "Maria Attarian", "Nikhil J Joshi", "Ayzaan Wahid", "Danny Driess", "Quan Vuong", "et al."], "corresponding": [], "institutions": ["Carnegie Mellon University", "Google DeepMind"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2024, "date": "2024-03-19", "topics": ["video-conditioned policy", "human prompt video"], "paradigm": ["IL"], "method": [], "projects": ["r2s2r"], "related": ["human-video-survey", "icrt", "mimicplay", "skild-scale"], "arxiv": "2403.12943", "doi": "", "code": "", "site": "", "note": "Condition the policy on a human video of the task instead of a language string. A bridge between in-context learning and learning from human video.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Large-scale multi-task robotic manipulation systems often rely on text to specify the task. In this work, we explore whether a robot can learn by observing humans. To do so, the robot must understand a person's intent and perform the inferred task despite differences in the embodiments and environments. We introduce Vid2Robot, an end-to-end video-conditioned policy that takes human videos demonstrating manipulation tasks as input and produces robot actions. Our model is trained with a large dataset of prompt video-robot trajectory pairs to learn unified representations of human and robot actions from videos. Vid2Robot uses cross-attention transformer layers between video features and the current robot state to produce the actions and perform the same task as shown in the video. We use auxiliary contrastive losses to align the prompt and robot video representations for better policies. We evaluate Vid2Robot on real-world robots and observe over 20% improvement over BC-Z when using human prompt videos. Further, we also show cross-object motion transfer ability that enables video-conditioned policies to transfer a motion observed on one object in the prompt video to another object in the robot's own environment. Videos available at https://vid2robot.github.io", "source": "curated", "first_author": "Vidhi Jain", "url": "https://arxiv.org/abs/2403.12943", "fig": "figures/vid2robot.webp", "figs": ["figures/vid2robot-1.webp", "figures/vid2robot-2.webp", "figures/vid2robot-3.webp"]}, {"id": "vista-wm", "title": "Vista: A Generalizable Driving World Model with High Fidelity and Versatile Controllability", "tree": "rl", "branch": "model-based", "authors": ["Shenyuan Gao", "Jiazhi Yang", "Li Chen", "Kashyap Chitta", "Yihang Qiu", "Andreas Geiger", "Jun Zhang", "Hongyang Li"], "corresponding": [], "institutions": ["Shanghai AI Laboratory", "HKUST"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2024, "date": "2024-05-27", "topics": ["high-fidelity driving world model", "long-horizon generation", "versatile control"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["dreamdojo", "gaia-2", "navdreamer", "navigation-world-models"], "arxiv": "2405.17398", "doi": "10.48550/arXiv.2405.17398", "code": "", "site": "", "note": "<b>Vista</b> — a generalizable driving world model with structural and motion losses added specifically to stop long-horizon rollouts from degenerating. Included because driving is where video world models were forced to confront long-horizon drift first, and the fixes transfer.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models can foresee the outcomes of different actions, which is of paramount importance for autonomous driving. Nevertheless, existing driving world models still have limitations in generalization to unseen environments, prediction fidelity of critical details, and action controllability for flexible application. In this paper, we present Vista, a generalizable driving world model with high fidelity and versatile controllability. Based on a systematic diagnosis of existing methods, we introduce several key ingredients to address these limitations. To accurately predict real-world dynamics at high resolution, we propose two novel losses to promote the learning of moving instances and structural information. We also devise an effective latent replacement approach to inject historical frames as priors for coherent long-horizon rollouts. For action controllability, we incorporate a versatile set of controls from high-level intentions (command, goal point) to low-level maneuvers (trajectory, angle, and speed) through an efficient learning strategy. After large-scale training, the capabilities of Vista can seamlessly generalize to different scenarios. Extensive experiments on multiple datasets show that Vista outperforms the most advanced general-purpose video generator in over 70% of comparisons and surpasses the best-performing driving world model by 55% in FID and 27% in FVD. Moreover, for the first time, we utilize the capacity of Vista itself to establish a generalizable reward for real-world action evaluation without accessing the ground truth actions.", "source": "curated", "first_author": "Shenyuan Gao", "url": "https://arxiv.org/abs/2405.17398", "fig": "figures/vista-wm.webp", "figs": ["figures/vista-wm-1.webp", "figures/vista-wm-2.webp", "figures/vista-wm-3.webp"]}, {"id": "av-contact", "title": "Visual-auditory Extrinsic Contact Estimation", "tree": "il", "branch": "tactile-learn", "authors": ["Xili Yi", "Jayjun Lee", "Nima Fazeli"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-09-22", "topics": ["extrinsic contact estimation", "visual-auditory fusion"], "paradigm": ["audio"], "method": [], "projects": ["wam-tactile"], "related": ["dexskin", "hearing-touch", "maniwav"], "arxiv": "2409.14608", "doi": "10.48550/arxiv.2409.14608", "code": "", "site": "", "note": "Estimates <em>extrinsic</em> contact — where the held object touches the world, not where the hand touches the object — from vision and sound together. For a stylus on a record, extrinsic contact is the whole task.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Robust manipulation often hinges on a robot's ability to perceive extrinsic contacts-contacts between a grasped object and its surrounding environment. However, these contacts are difficult to observe through vision alone due to occlusions, limited resolution, and ambiguous near-contact states. In this paper, we propose a visual-auditory method for extrinsic contact estimation that integrates global scene information from vision with local contact cues obtained through active audio sensing. Our approach equips a robotic gripper with contact microphones and conduction speakers, enabling the system to emit and receive acoustic signals through the grasped object to detect external contacts. We train our perception pipeline entirely in simulation and zero-shot transfer to the real world. To bridge the sim-to-real gap, we introduce a real-to-sim audio hallucination technique, injecting real-world audio samples into simulated scenes with ground-truth contact labels. The resulting multimodal model accurately estimates both the location and size of extrinsic contacts across a range of cluttered and occluded scenarios. We further demonstrate that explicit contact prediction significantly improves policy learning for downstream contact-rich manipulation tasks.", "source": "curated", "first_author": "Xili Yi", "url": "https://arxiv.org/abs/2409.14608", "fig": "figures/av-contact.webp", "figs": ["figures/av-contact-1.webp", "figures/av-contact-2.webp"]}, {"id": "workspace-opt", "title": "Workspace Optimization Techniques to Improve Prediction of Human Motion During Human-Robot Collaboration", "tree": "rl", "branch": "marl", "authors": ["Tung", "Yi-Shiuan", "Luebbers", "Matthew B.", "Roncone", "Alessandro", "Hayes", "Bradley"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-01-23", "topics": ["workspace arrangement", "human motion prediction", "legibility"], "paradigm": ["HRI"], "method": [], "projects": ["m2"], "related": ["h2compact", "intention-tracking", "interact", "phhi-survey"], "arxiv": "2401.12965", "doi": "10.48550/arxiv.2401.12965", "code": "", "site": "", "note": "A genuinely different idea: instead of predicting human motion better, <em>arrange the workspace</em> so the human moves more predictably, including projecting virtual obstacles in AR. Cheap, and it attacks the variance rather than the model.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Understanding human intentions is critical for safe and effective human-robot collaboration. While state of the art methods for human goal prediction utilize learned models to account for the uncertainty of human motion data, that data is inherently stochastic and high variance, hindering those models' utility for interactions requiring coordination, including safety-critical or close-proximity tasks. Our key insight is that robot teammates can deliberately configure shared workspaces prior to interaction in order to reduce the variance in human motion, realizing classifier-agnostic improvements in goal prediction. In this work, we present an algorithmic approach for a robot to arrange physical objects and project \"virtual obstacles\" using augmented reality in shared human-robot workspaces, optimizing for human legibility over a given set of tasks. We compare our approach against other workspace arrangement strategies using two human-subjects studies, one in a virtual 2D navigation domain and the other in a live tabletop manipulation domain involving a robotic manipulator arm. We evaluate the accuracy of human motion prediction models learned from each condition, demonstrating that our workspace optimization technique with virtual obstacles leads to higher robot prediction accuracy using less training data.", "source": "curated", "first_author": "Tung", "url": "https://arxiv.org/abs/2401.12965", "fig": "figures/workspace-opt.webp", "figs": ["figures/workspace-opt-1.webp", "figures/workspace-opt-2.webp"]}, {"id": "ivideogpt", "title": "iVideoGPT: Interactive VideoGPTs are Scalable World Models", "tree": "il", "branch": "world-models", "authors": ["Jialong Wu", "Shaofeng Yin", "Ningya Feng", "Xu He", "Dong Li", "Jianye Hao", "Mingsheng Long"], "corresponding": [], "institutions": ["Tsinghua University", "Huawei"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2024, "date": "2024-05-24", "topics": ["interactive video GPT", "scalable world model", "compressive tokenization"], "paradigm": ["world model"], "method": [], "projects": ["wam-tactile"], "related": ["adaworld", "dreamdojo", "genie", "genie-envisioner", "geniworld", "iris"], "arxiv": "2405.15223", "doi": "10.48550/arXiv.2405.15223", "code": "", "site": "", "note": "<b>iVideoGPT</b> — makes a video GPT <em>interactive</em> (action-conditioned, observation-fed) with a compressive tokenizer, which is what separates a world model from a video generator. The scalability argument that Genie and the WAM line both build on.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "World models empower model-based agents to interactively explore, reason, and plan within imagined environments for real-world decision-making. However, the high demand for interactivity poses challenges in harnessing recent advancements in video generative models for developing world models at scale. This work introduces Interactive VideoGPT (iVideoGPT), a scalable autoregressive transformer framework that integrates multimodal signals--visual observations, actions, and rewards--into a sequence of tokens, facilitating an interactive experience of agents via next-token prediction. iVideoGPT features a novel compressive tokenization technique that efficiently discretizes high-dimensional visual observations. Leveraging its scalable architecture, we are able to pre-train iVideoGPT on millions of human and robotic manipulation trajectories, establishing a versatile foundation that is adaptable to serve as interactive world models for a wide range of downstream tasks. These include action-conditioned video prediction, visual planning, and model-based reinforcement learning, where iVideoGPT achieves competitive performance compared with state-of-the-art methods. Our work advances the development of interactive general world models, bridging the gap between generative video models and practical model-based reinforcement learning applications. Code and pre-trained models are available at https://thuml.github.io/iVideoGPT.", "source": "curated", "first_author": "Jialong Wu", "url": "https://arxiv.org/abs/2405.15223", "fig": "figures/ivideogpt.webp", "figs": ["figures/ivideogpt-1.webp", "figures/ivideogpt-2.webp"]}, {"id": "bridgedata", "title": "BridgeData V2: A Dataset for Robot Learning at Scale", "tree": "il", "branch": "data-scaling", "authors": ["Homer Walke", "Kevin Black", "Abraham Lee", "Moo Jin Kim", "Max Du", "Chongyi Zheng", "et al."], "corresponding": [], "institutions": ["UC Berkeley", "Stanford University", "Google DeepMind"], "lab": "BAIR (RAIL)", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-08-24", "topics": ["dataset", "generalization"], "paradigm": ["dataset"], "method": [], "projects": [], "related": ["droid", "octo", "oxe"], "arxiv": "2308.12952", "doi": "", "code": "", "site": "", "note": "60k trajectories on WidowX. The standard low-cost-arm generalization benchmark before DROID.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce BridgeData V2, a large and diverse dataset of robotic manipulation behaviors designed to facilitate research on scalable robot learning. BridgeData V2 contains 60,096 trajectories collected across 24 environments on a publicly available low-cost robot. BridgeData V2 provides extensive task and environment variability, leading to skills that can generalize across environments, domains, and institutions, making the dataset a useful resource for a broad range of researchers. Additionally, the dataset is compatible with a wide variety of open-vocabulary, multi-task learning methods conditioned on goal images or natural language instructions. In our experiments, we train 6 state-of-the-art imitation learning and offline reinforcement learning methods on our dataset, and find that they succeed on a suite of tasks requiring varying amounts of generalization. We also demonstrate that the performance of these methods improves with more data and higher capacity models, and that training on a greater variety of skills leads to improved generalization. By publicly sharing BridgeData V2 and our pre-trained models, we aim to accelerate research in scalable robot learning methods. Project page at https://rail-berkeley.github.io/bridgedata", "source": "curated", "first_author": "Homer Walke", "url": "https://arxiv.org/abs/2308.12952", "fig": "figures/bridgedata.webp", "figs": ["figures/bridgedata-1.webp", "figures/bridgedata-2.webp", "figures/bridgedata-3.webp"]}, {"id": "dextreme", "title": "DeXtreme: Transfer of Agile In-hand Manipulation from Simulation to Reality", "tree": "rl", "branch": "manipulation", "authors": ["Ankur Handa", "Arthur Allshire", "Viktor Makoviychuk", "Aleksei Petrenko", "Ritvik Singh", "et al."], "corresponding": ["Ankur Handa"], "institutions": ["NVIDIA"], "lab": "NVIDIA Seattle Robotics Lab", "venue": "ICRA", "venue_type": "conference", "year": 2023, "date": "2022-10-25", "topics": ["in-hand manipulation", "sim-to-real", "GPU RL"], "paradigm": ["RL"], "method": ["Isaac Gym domain randomization", "ADR", "pose estimation from vision"], "projects": ["geodex"], "related": ["chen-inhand", "chen-visual-inhand", "dactyl", "isaacgym", "rma"], "arxiv": "2210.13702", "doi": "", "code": "", "site": "", "note": "Reproduces Dactyl-class dexterity on commodity hardware and a fraction of the compute. The gap between this and Dactyl is almost entirely simulator throughput.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recent work has demonstrated the ability of deep reinforcement learning (RL) algorithms to learn complex robotic behaviours in simulation, including in the domain of multi-fingered manipulation. However, such models can be challenging to transfer to the real world due to the gap between simulation and reality. In this paper, we present our techniques to train a) a policy that can perform robust dexterous manipulation on an anthropomorphic robot hand and b) a robust pose estimator suitable for providing reliable real-time information on the state of the object being manipulated. Our policies are trained to adapt to a wide range of conditions in simulation. Consequently, our vision-based policies significantly outperform the best vision policies in the literature on the same reorientation task and are competitive with policies that are given privileged state information via motion capture systems. Our work reaffirms the possibilities of sim-to-real transfer for dexterous manipulation in diverse kinds of hardware and simulator setups, and in our case, with the Allegro Hand and Isaac Gym GPU-based simulation. Furthermore, it opens up possibilities for researchers to achieve such results with commonly-available, affordable robot hands and cameras. Videos of the resulting policy and supplementary information, including experiments and demos, can be found at https://dextreme.org/", "source": "curated", "first_author": "Ankur Handa", "url": "https://arxiv.org/abs/2210.13702", "fig": "figures/dextreme.webp", "figs": ["figures/dextreme-1.webp", "figures/dextreme-2.webp"]}, {"id": "dex-retargeting", "title": "Dex-Retargeting: A Unified Motion Retargeting Library for Dexterous Hands", "tree": "il", "branch": "retargeting", "authors": ["Yuzhe Qin", "et al."], "corresponding": [], "institutions": ["UC San Diego"], "lab": "", "venue": "software", "venue_type": "software", "year": 2023, "date": "", "topics": ["retargeting library", "open source"], "paradigm": ["teleop"], "method": [], "projects": ["geodex"], "related": ["anyteleop", "c2dex", "dexpilot"], "arxiv": "", "doi": "", "code": "https://github.com/dexsuite/dex-retargeting", "site": "", "note": "The library, not a paper. Vector/position/DexPilot retargeting for most common hands — use it as the baseline your learned retargeting has to beat.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Yuzhe Qin", "url": "https://github.com/dexsuite/dex-retargeting"}, {"id": "diffusebot", "title": "DiffuseBot: Breeding Soft Robots With Physics-Augmented Generative Diffusion Models", "tree": "systems", "branch": "codesign", "authors": ["Tsun-Hsuan Wang", "Juntian Zheng", "Pingchuan Ma", "Yilun Du", "Byungchul Kim", "Andrew Spielberg", "Joshua Tenenbaum", "Chuang Gan", "Daniela Rus"], "corresponding": ["Tsun-Hsuan Wang"], "institutions": ["MIT"], "lab": "MIT CSAIL", "venue": "NeurIPS", "venue_type": "conference", "year": 2023, "date": "2023-11-28", "topics": ["generative design", "diffusion over morphology", "soft robots"], "paradigm": ["co-design", "diffusion"], "method": [], "projects": [], "related": ["evogym", "fit2form", "text2robot"], "arxiv": "2311.17053", "doi": "", "code": "", "site": "", "note": "A diffusion model over robot morphologies, with differentiable physics steering the sampling. The generative-prior half of agentic co-design — the LLM proposes, something like this actually draws it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Nature evolves creatures with a high complexity of morphological and behavioral intelligence, meanwhile computational methods lag in approaching that diversity and efficacy. Co-optimization of artificial creatures' morphology and control in silico shows promise for applications in physical soft robotics and virtual character creation; such approaches, however, require developing new learning algorithms that can reason about function atop pure structure. In this paper, we present DiffuseBot, a physics-augmented diffusion model that generates soft robot morphologies capable of excelling in a wide spectrum of tasks. DiffuseBot bridges the gap between virtually generated content and physical utility by (i) augmenting the diffusion process with a physical dynamical simulation which provides a certificate of performance, and (ii) introducing a co-design procedure that jointly optimizes physical design and control by leveraging information about physical sensitivities from differentiable simulation. We showcase a range of simulated and fabricated robots along with their capabilities. Check our website at https://diffusebot.github.io/", "source": "curated", "first_author": "Tsun-Hsuan Wang", "url": "https://arxiv.org/abs/2311.17053", "fig": "figures/diffusebot.webp", "figs": ["figures/diffusebot-1.webp"]}, {"id": "faive-ball", "title": "Getting the Ball Rolling: Learning a Dexterous Policy for a Biomimetic Tendon-Driven Hand with Rolling Contact Joints", "tree": "systems", "branch": "hands", "authors": ["Yasunori Toshimitsu", "Benedek Forrai", "Barnabas Gavin Cangan", "Ulrich Steger", "Manuel Knecht", "Stefan Weirich", "Robert K. Katzschmann"], "corresponding": [], "institutions": [], "lab": "", "venue": "2023 IEEE-RAS 22nd International Confere", "venue_type": "journal", "year": 2023, "date": "2023-08-04", "topics": ["biomimetic tendon hand", "rolling contact joints", "RL policy"], "paradigm": ["hardware", "RL"], "method": [], "projects": ["geodex"], "related": ["mimic-robotics", "orca-hand", "ruka-v2"], "arxiv": "2308.02453", "doi": "10.1109/Humanoids57100.2023.10375231", "code": "", "site": "", "note": "The ETH Faive hand (the lineage behind mimic robotics), with rolling-contact joints and a learned dexterous policy. Read for the joint mechanism, which is the part tendon hands usually get wrong.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Biomimetic, dexterous robotic hands have the potential to replicate much of the tasks that a human can do, and to achieve status as a general manipulation platform. Recent advances in reinforcement learning (RL) frameworks have achieved remarkable performance in quadrupedal locomotion and dexterous manipulation tasks. Combined with GPU-based highly parallelized simulations capable of simulating thousands of robots in parallel, RL-based controllers have become more scalable and approachable. However, in order to bring RL-trained policies to the real world, we require training frameworks that output policies that can work with physical actuators and sensors as well as a hardware platform that can be manufactured with accessible materials yet is robust enough to run interactive policies. This work introduces the biomimetic tendon-driven Faive Hand and its system architecture, which uses tendon-driven rolling contact joints to achieve a 3D printable, robust high-DoF hand design. We model each element of the hand and integrate it into a GPU simulation environment to train a policy with RL, and achieve zero-shot transfer of a dexterous in-hand sphere rotation skill to the physical robot hand.", "source": "curated", "first_author": "Yasunori Toshimitsu", "url": "https://arxiv.org/abs/2308.02453", "fig": "figures/faive-ball.webp", "figs": ["figures/faive-ball-1.webp", "figures/faive-ball-2.webp", "figures/faive-ball-3.webp"]}, {"id": "interact", "title": "InteRACT: Transformer Models for Human Intent Prediction Conditioned on Robot Actions", "tree": "rl", "branch": "marl", "authors": ["Kushal Kedia", "Atiksh Bhardwaj", "Prithwish Dan", "Sanjiban Choudhury"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-11-21", "topics": ["human intent prediction conditioned on robot action", "collaborative manipulation"], "paradigm": ["HRI", "transformer"], "method": [], "projects": ["m2"], "related": ["aim-wam", "h2compact", "intention-priors", "intention-tracking", "phhi-survey", "roco", "workspace-opt"], "arxiv": "2311.12943", "doi": "10.48550/arxiv.2311.12943", "code": "", "site": "", "note": "Predicts human intent <em>conditioned on what the robot is doing</em>. Almost every intent dataset treats the human as if the robot were not there, which is exactly the assumption that breaks in collaboration. The teleoperation setup and the collected human-robot data are open-sourced.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In collaborative human-robot manipulation, a robot must predict human intents and adapt its actions accordingly to smoothly execute tasks. However, the human's intent in turn depends on actions the robot takes, creating a chicken-or-egg problem. Prior methods ignore such inter-dependency and instead train marginal intent prediction models independent of robot actions. This is because training conditional models is hard given a lack of paired human-robot interaction datasets. Can we instead leverage large-scale human-human interaction data that is more easily accessible? Our key insight is to exploit a correspondence between human and robot actions that enables transfer learning from human-human to human-robot data. We propose a novel architecture, InteRACT, that pre-trains a conditional intent prediction model on large human-human datasets and fine-tunes on a small human-robot dataset. We evaluate on a set of real-world collaborative human-robot manipulation tasks and show that our conditional model improves over various marginal baselines. We also introduce new techniques to tele-operate a 7-DoF robot arm and collect a diverse range of human-robot collaborative manipulation data, which we open-source.", "source": "curated", "first_author": "Kushal Kedia", "url": "https://arxiv.org/abs/2311.12943", "fig": "figures/interact.webp", "figs": ["figures/interact-1.webp", "figures/interact-2.webp"]}, {"id": "libero", "title": "LIBERO: Benchmarking Knowledge Transfer for Lifelong Robot Learning", "tree": "systems", "branch": "sim-bench", "authors": ["Bo Liu", "Yifeng Zhu", "Chongkai Gao", "Yihao Feng", "Qiang Liu", "Yuke Zhu", "Peter Stone"], "corresponding": [], "institutions": ["UT Austin"], "lab": "UT Austin", "venue": "NeurIPS Datasets & Benchmarks", "venue_type": "conference", "year": 2023, "date": "2023-06-05", "topics": ["lifelong learning benchmark", "knowledge transfer"], "paradigm": ["benchmark"], "method": [], "projects": [], "related": ["humanoidbench", "openvla", "pi0", "robocasa"], "arxiv": "2306.03310", "doi": "", "code": "", "site": "", "note": "The standard VLA evaluation suite. Useful mostly because everyone reports on it — treat saturation on LIBERO as weak evidence of anything real.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Lifelong learning offers a promising paradigm of building a generalist agent that learns and adapts over its lifespan. Unlike traditional lifelong learning problems in image and text domains, which primarily involve the transfer of declarative knowledge of entities and concepts, lifelong learning in decision-making (LLDM) also necessitates the transfer of procedural knowledge, such as actions and behaviors. To advance research in LLDM, we introduce LIBERO, a novel benchmark of lifelong learning for robot manipulation. Specifically, LIBERO highlights five key research topics in LLDM: 1) how to efficiently transfer declarative knowledge, procedural knowledge, or the mixture of both; 2) how to design effective policy architectures and 3) effective algorithms for LLDM; 4) the robustness of a lifelong learner with respect to task ordering; and 5) the effect of model pretraining for LLDM. We develop an extendible procedural generation pipeline that can in principle generate infinitely many tasks. For benchmarking purpose, we create four task suites (130 tasks in total) that we use to investigate the above-mentioned research topics. To support sample-efficient learning, we provide high-quality human-teleoperated demonstration data for all tasks. Our extensive experiments present several insightful or even unexpected discoveries: sequential finetuning outperforms existing lifelong learning methods in forward transfer, no single visual encoder architecture excels at all types of knowledge transfer, and naive supervised pretraining can hinder agents' performance in the subsequent LLDM. Check the website at https://libero-project.github.io for the code and the datasets.", "source": "curated", "first_author": "Bo Liu", "url": "https://arxiv.org/abs/2306.03310", "fig": "figures/libero.webp", "figs": ["figures/libero-1.webp"]}, {"id": "unipi", "title": "Learning Universal Policies via Text-Guided Video Generation", "tree": "il", "branch": "world-models", "authors": ["Yilun Du", "Sherry Yang", "Bo Dai", "Hanjun Dai", "Ofir Nachum", "Joshua B. Tenenbaum", "Dale Schuurmans", "Pieter Abbeel"], "corresponding": [], "institutions": ["MIT", "UC Berkeley", "Google DeepMind"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2023, "date": "2023-02-01", "topics": ["video generation as planning", "universal policy"], "paradigm": ["world model", "diffusion"], "method": ["generate a video plan", "then infer actions with an inverse dynamics model"], "projects": [], "related": ["genie", "gr1", "gr2", "rigvid", "worldmodels"], "arxiv": "2302.00111", "doi": "", "code": "", "site": "", "note": "Plan in pixel space, act with inverse dynamics. The clean statement of 'video prediction is a policy if you can invert it'.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "A goal of artificial intelligence is to construct an agent that can solve a wide variety of tasks. Recent progress in text-guided image synthesis has yielded models with an impressive ability to generate complex novel images, exhibiting combinatorial generalization across domains. Motivated by this success, we investigate whether such tools can be used to construct more general-purpose agents. Specifically, we cast the sequential decision making problem as a text-conditioned video generation problem, where, given a text-encoded specification of a desired goal, a planner synthesizes a set of future frames depicting its planned actions in the future, after which control actions are extracted from the generated video. By leveraging text as the underlying goal specification, we are able to naturally and combinatorially generalize to novel goals. The proposed policy-as-video formulation can further represent environments with different state and action spaces in a unified space of images, which, for example, enables learning and generalization across a variety of robot manipulation tasks. Finally, by leveraging pretrained language embeddings and widely available videos from the internet, the approach enables knowledge transfer through predicting highly realistic video plans for real robots.", "source": "curated", "first_author": "Yilun Du", "url": "https://arxiv.org/abs/2302.00111", "fig": "figures/unipi.webp", "figs": ["figures/unipi-1.webp"]}, {"id": "mimicplay", "title": "MimicPlay: Long-Horizon Imitation Learning by Watching Human Play", "tree": "il", "branch": "human-video", "authors": ["Chen Wang", "Linxi Fan", "Jiankai Sun", "Ruohan Zhang", "Li Fei-Fei", "Danfei Xu", "Anima Anandkumar", "Yuke Zhu"], "corresponding": [], "institutions": ["Stanford University", "NVIDIA", "Georgia Tech", "UT Austin"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-02-24", "topics": ["human play data", "hierarchical planning"], "paradigm": ["IL"], "method": [], "projects": ["r2s2r"], "related": ["egomimic", "phantom", "vid2robot"], "arxiv": "2302.12422", "doi": "", "code": "", "site": "", "note": "Human play video trains the high-level planner; a small amount of teleop trains the low-level controller. A good division of labour for expensive robot time.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Imitation learning from human demonstrations is a promising paradigm for teaching robots manipulation skills in the real world. However, learning complex long-horizon tasks often requires an unattainable amount of demonstrations. To reduce the high data requirement, we resort to human play data - video sequences of people freely interacting with the environment using their hands. Even with different morphologies, we hypothesize that human play data contain rich and salient information about physical interactions that can readily facilitate robot policy learning. Motivated by this, we introduce a hierarchical learning framework named MimicPlay that learns latent plans from human play data to guide low-level visuomotor control trained on a small number of teleoperated demonstrations. With systematic evaluations of 14 long-horizon manipulation tasks in the real world, we show that MimicPlay outperforms state-of-the-art imitation learning methods in task success rate, generalization ability, and robustness to disturbances. Code and videos are available at https://mimic-play.github.io", "source": "curated", "first_author": "Chen Wang", "url": "https://arxiv.org/abs/2302.12422", "fig": "figures/mimicplay.webp", "figs": ["figures/mimicplay-1.webp", "figures/mimicplay-2.webp"]}, {"id": "isaaclab", "title": "Orbit: A Unified Simulation Framework for Interactive Robot Learning Environments", "tree": "systems", "branch": "sim-bench", "authors": ["Mayank Mittal", "Calvin Yu", "Qinxi Yu", "Jingzhou Liu", "Nikita Rudin", "David Hoeller", "et al."], "corresponding": [], "institutions": ["NVIDIA", "ETH Zürich"], "lab": "NVIDIA; ETH RSL", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2023, "date": "2023-01-10", "topics": ["simulation framework", "robot learning"], "paradigm": ["simulator"], "method": [], "projects": ["mabel"], "related": ["booster-rl", "genesis", "isaacgym", "maniskill", "newton", "rudin2021"], "arxiv": "2301.04195", "doi": "", "code": "", "site": "", "note": "The maintained successor to Isaac Gym. What you would actually build MABEL's RL stack on today.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We present Orbit, a unified and modular framework for robot learning powered by NVIDIA Isaac Sim. It offers a modular design to easily and efficiently create robotic environments with photo-realistic scenes and high-fidelity rigid and deformable body simulation. With Orbit, we provide a suite of benchmark tasks of varying difficulty -- from single-stage cabinet opening and cloth folding to multi-stage tasks such as room reorganization. To support working with diverse observations and action spaces, we include fixed-arm and mobile manipulators with different physically-based sensors and motion generators. Orbit allows training reinforcement learning policies and collecting large demonstration datasets from hand-crafted or expert solutions in a matter of minutes by leveraging GPU-based parallelization. In summary, we offer an open-sourced framework that readily comes with 16 robotic platforms, 4 sensor modalities, 10 motion generators, more than 20 benchmark tasks, and wrappers to 4 learning libraries. With this framework, we aim to support various research areas, including representation learning, reinforcement learning, imitation learning, and task and motion planning. We hope it helps establish interdisciplinary collaborations in these communities, and its modularity makes it easily extensible for more tasks and applications in the future.", "source": "curated", "first_author": "Mayank Mittal", "url": "https://arxiv.org/abs/2301.04195", "fig": "figures/isaaclab.webp", "figs": ["figures/isaaclab-1.webp", "figures/isaaclab-2.webp", "figures/isaaclab-3.webp"]}, {"id": "phc", "title": "Perpetual Humanoid Control for Real-time Simulated Avatars", "tree": "rl", "branch": "motion-imitation", "authors": ["Zhengyi Luo", "Jinkun Cao", "Alexander Winkler", "Kris Kitani", "Weipeng Xu"], "corresponding": ["Zhengyi Luo"], "institutions": ["Carnegie Mellon University", "Meta Reality Labs"], "lab": "", "venue": "ICCV", "venue_type": "conference", "year": 2023, "date": "2023-05-10", "topics": ["motion tracking", "failure recovery", "avatars"], "paradigm": ["RL"], "method": ["progressive multiplicative control policy", "fail-state recovery"], "projects": [], "related": ["adaptive-tracking", "deepmimic", "h2o", "maskedmimic", "omnih2o"], "arxiv": "2305.06456", "doi": "", "code": "", "site": "", "note": "Tracks the whole AMASS corpus with one policy and recovers from failure instead of resetting. PHC checkpoints are the de-facto motion-tracking backbone reused by H2O/OmniH2O and friends.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We present a physics-based humanoid controller that achieves high-fidelity motion imitation and fault-tolerant behavior in the presence of noisy input (e.g. pose estimates from video or generated from language) and unexpected falls. Our controller scales up to learning ten thousand motion clips without using any external stabilizing forces and learns to naturally recover from fail-state. Given reference motion, our controller can perpetually control simulated avatars without requiring resets. At its core, we propose the progressive multiplicative control policy (PMCP), which dynamically allocates new network capacity to learn harder and harder motion sequences. PMCP allows efficient scaling for learning from large-scale motion databases and adding new tasks, such as fail-state recovery, without catastrophic forgetting. We demonstrate the effectiveness of our controller by using it to imitate noisy poses from video-based pose estimators and language-based motion generators in a live and real-time multi-person avatar use case.", "source": "curated", "first_author": "Zhengyi Luo", "url": "https://arxiv.org/abs/2305.06456", "fig": "figures/phc.webp", "figs": ["figures/phc-1.webp"]}, {"id": "robocat", "title": "RoboCat: A Self-Improving Generalist Agent for Robotic Manipulation", "tree": "il", "branch": "in-context", "authors": ["Konstantinos Bousmalis", "Giulia Vezzani", "Dushyant Rao", "Coline Devin", "Alex X. Lee", "et al."], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind Robotics", "venue": "TMLR", "venue_type": "journal", "year": 2023, "date": "2023-06-20", "topics": ["self-improvement", "few-shot adaptation", "multi-embodiment"], "paradigm": ["IL"], "method": ["fine-tune on 100-1000 demos", "then self-generate data for the next round"], "projects": [], "related": ["dlr-vla", "icrt", "voyager"], "arxiv": "2306.11706", "doi": "", "code": "", "site": "", "note": "The self-improvement loop: adapt, generate your own data, fold it back in. A concrete template for the auto-research idea, minus the hypothesis generation.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The ability to leverage heterogeneous robotic experience from different robots and tasks to quickly master novel skills and embodiments has the potential to transform robot learning. Inspired by recent advances in foundation models for vision and language, we propose a multi-embodiment, multi-task generalist agent for robotic manipulation. This agent, named RoboCat, is a visual goal-conditioned decision transformer capable of consuming action-labelled visual experience. This data spans a large repertoire of motor control skills from simulated and real robotic arms with varying sets of observations and actions. With RoboCat, we demonstrate the ability to generalise to new tasks and robots, both zero-shot as well as through adaptation using only 100-1000 examples for the target task. We also show how a trained model itself can be used to generate data for subsequent training iterations, thus providing a basic building block for an autonomous improvement loop. We investigate the agent's capabilities, with large-scale evaluations both in simulation and on three different real robot embodiments. We find that as we grow and diversify its training data, RoboCat not only shows signs of cross-task transfer, but also becomes more efficient at adapting to new tasks.", "source": "curated", "first_author": "Konstantinos Bousmalis", "url": "https://arxiv.org/abs/2306.11706", "fig": "figures/robocat.webp"}, {"id": "robocook", "title": "RoboCook: Long-Horizon Elasto-Plastic Object Manipulation with Diverse Tools", "tree": "il", "branch": "world-models", "authors": ["Haochen Shi", "Huazhe Xu", "Samuel Clarke", "Yunzhu Li", "Jiajun Wu"], "corresponding": [], "institutions": ["Stanford University", "University of Illinois Urbana-Champaign"], "lab": "RoboPIL (Yunzhu Li)", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-06-26", "topics": ["long-horizon deformable manipulation", "tool selection", "GNN dynamics"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["adaptigraph", "bab-nd", "robo-exp", "robocraft"], "arxiv": "2306.14447", "doi": "10.48550/arXiv.2306.14447", "code": "", "site": "", "note": "<b>RoboCook</b> — RoboCraft extended to long horizons and a <em>toolbox</em>: the system picks which tool to use as well as how to use it, dumpling-making end to end. The interesting part for you is the hierarchy — a learned dynamics model at the bottom, discrete tool selection on top — which is the same shape as choosing a grasp before refining it.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humans excel in complex long-horizon soft body manipulation tasks via flexible tool use: bread baking requires a knife to slice the dough and a rolling pin to flatten it. Often regarded as a hallmark of human cognition, tool use in autonomous robots remains limited due to challenges in understanding tool-object interactions. Here we develop an intelligent robotic system, RoboCook, which perceives, models, and manipulates elasto-plastic objects with various tools. RoboCook uses point cloud scene representations, models tool-object interactions with Graph Neural Networks (GNNs), and combines tool classification with self-supervised policy learning to devise manipulation plans. We demonstrate that from just 20 minutes of real-world interaction data per tool, a general-purpose robot arm can learn complex long-horizon soft object manipulation tasks, such as making dumplings and alphabet letter cookies. Extensive evaluations show that RoboCook substantially outperforms state-of-the-art approaches, exhibits robustness against severe external disturbances, and demonstrates adaptability to different materials.", "source": "curated", "first_author": "Haochen Shi", "url": "https://arxiv.org/abs/2306.14447", "fig": "figures/robocook.webp", "figs": ["figures/robocook-1.webp", "figures/robocook-2.webp"]}, {"id": "robot-synesthesia", "title": "Robot Synesthesia: In-Hand Manipulation with Visuotactile Sensing", "tree": "rl", "branch": "manipulation", "authors": ["Ying Yuan", "Haichuan Che", "Yuzhe Qin", "Binghao Huang", "Zhao-Heng Yin", "Kang-Won Lee", "Yi Wu", "Soo‐Chul Lim", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-12-04", "topics": ["visuotactile fusion", "in-hand manipulation"], "paradigm": ["RL"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["dexskin", "qi-inhand", "see-hear-feel"], "arxiv": "2312.01853", "doi": "10.48550/arxiv.2312.01853", "code": "", "site": "", "note": "Fuses vision and touch into a shared point-cloud representation for in-hand manipulation. The representation trick — put both modalities in the same 3D space instead of concatenating features — generalises to audio.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Executing contact-rich manipulation tasks necessitates the fusion of tactile and visual feedback. However, the distinct nature of these modalities poses significant challenges. In this paper, we introduce a system that leverages visual and tactile sensory inputs to enable dexterous in-hand manipulation. Specifically, we propose Robot Synesthesia, a novel point cloud-based tactile representation inspired by human tactile-visual synesthesia. This approach allows for the simultaneous and seamless integration of both sensory inputs, offering richer spatial information and facilitating better reasoning about robot actions. The method, trained in a simulated environment and then deployed to a real robot, is applicable to various in-hand object rotation tasks. Comprehensive ablations are performed on how the integration of vision and touch can improve reinforcement learning and Sim2Real performance. Our project page is available at https://yingyuan0414.github.io/visuotactile/ .", "source": "curated", "first_author": "Ying Yuan", "url": "https://arxiv.org/abs/2312.01853", "fig": "figures/robot-synesthesia.webp", "figs": ["figures/robot-synesthesia-1.webp", "figures/robot-synesthesia-2.webp", "figures/robot-synesthesia-3.webp"]}, {"id": "rotate-without-seeing", "title": "Rotating without Seeing: Towards In-hand Dexterity through Touch", "tree": "rl", "branch": "manipulation", "authors": ["Zhao-Heng Yin", "Binghao Huang", "Yuzhe Qin", "Qifeng Chen", "Xiaolong Wang"], "corresponding": ["Zhao-Heng Yin"], "institutions": ["HKUST", "UC San Diego"], "lab": "UCSD Xiaolong Wang Lab", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-03-20", "topics": ["touch-only dexterity", "binary tactile sensing"], "paradigm": ["RL"], "method": ["binary contact sensors", "teacher-student", "sim-to-real with tactile randomization"], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "dactyl", "dexskin", "qi-inhand", "tacforesight"], "arxiv": "2303.10880", "doi": "", "code": "", "site": "", "note": "Touch-only in-hand rotation. Direct evidence for your tactile-world-model idea: contact signals carry enough state for dexterity even with vision removed entirely.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Tactile information plays a critical role in human dexterity. It reveals useful contact information that may not be inferred directly from vision. In fact, humans can even perform in-hand dexterous manipulation without using vision. Can we enable the same ability for the multi-finger robot hand? In this paper, we present Touch Dexterity, a new system that can perform in-hand object rotation using only touching without seeing the object. Instead of relying on precise tactile sensing in a small region, we introduce a new system design using dense binary force sensors (touch or no touch) overlaying one side of the whole robot hand (palm, finger links, fingertips). Such a design is low-cost, giving a larger coverage of the object, and minimizing the Sim2Real gap at the same time. We train an in-hand rotation policy using Reinforcement Learning on diverse objects in simulation. Relying on touch-only sensing, we can directly deploy the policy in a real robot hand and rotate novel objects that are not presented in training. Extensive ablations are performed on how tactile information help in-hand manipulation.Our project is available at https://touchdexterity.github.io.", "source": "curated", "first_author": "Zhao-Heng Yin", "url": "https://arxiv.org/abs/2303.10880", "fig": "figures/rotate-without-seeing.webp", "figs": ["figures/rotate-without-seeing-1.webp", "figures/rotate-without-seeing-2.webp", "figures/rotate-without-seeing-3.webp"]}, {"id": "storm", "title": "STORM: Efficient Stochastic Transformer based World Models for Reinforcement Learning", "tree": "rl", "branch": "model-based", "authors": ["Weipu Zhang", "Gang Wang", "Jian Sun", "Yetian Yuan", "Gao Huang"], "corresponding": [], "institutions": ["Tsinghua University", "Beijing Institute of Technology"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2023, "date": "2023-10-14", "topics": ["stochastic transformer world model", "efficient imagination"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["diamond", "dreamer-v2", "dreamerv3", "himem-wam", "iris"], "arxiv": "2310.09615", "doi": "10.48550/arXiv.2310.09615", "code": "", "site": "", "note": "<b>STORM</b> — a stochastic transformer world model that gets DreamerV3-level Atari performance at a fraction of the training cost. Worth reading beside IRIS as the efficiency-tuned version of the same idea; together they are the argument that the transformer, not the recurrence, is now the default dynamics backbone.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Recently, model-based reinforcement learning algorithms have demonstrated remarkable efficacy in visual input environments. These approaches begin by constructing a parameterized simulation world model of the real environment through self-supervised learning. By leveraging the imagination of the world model, the agent's policy is enhanced without the constraints of sampling from the real environment. The performance of these algorithms heavily relies on the sequence modeling and generation capabilities of the world model. However, constructing a perfectly accurate model of a complex unknown environment is nearly impossible. Discrepancies between the model and reality may cause the agent to pursue virtual goals, resulting in subpar performance in the real environment. Introducing random noise into model-based reinforcement learning has been proven beneficial. In this work, we introduce Stochastic Transformer-based wORld Model (STORM), an efficient world model architecture that combines the strong sequence modeling and generation capabilities of Transformers with the stochastic nature of variational autoencoders. STORM achieves a mean human performance of $126.7\\%$ on the Atari $100$k benchmark, setting a new record among state-of-the-art methods that do not employ lookahead search techniques. Moreover, training an agent with $1.85$ hours of real-time interaction experience on a single NVIDIA GeForce RTX 3090 graphics card requires only $4.3$ hours, showcasing improved efficiency compared to previous methodologies.", "source": "curated", "first_author": "Weipu Zhang", "url": "https://arxiv.org/abs/2310.09615", "fig": "figures/storm.webp", "figs": ["figures/storm-1.webp"]}, {"id": "tidybot", "title": "TidyBot: Personalized Robot Assistance with Large Language Models", "tree": "systems", "branch": "mobile-manip", "authors": ["Jimmy Wu", "Rika Antonova", "Adam Kan", "Marion Lepert", "Andy Zeng", "Shuran Song", "Jeannette Bohg", "Szymon Rusinkiewicz", "Thomas Funkhouser"], "corresponding": ["Jimmy Wu"], "institutions": ["Princeton University", "Stanford University", "Columbia University", "Google"], "lab": "Princeton Vision & Robotics", "venue": "IROS", "venue_type": "conference", "year": 2023, "date": "2023-05-09", "topics": ["personalization", "LLM summarization", "household tidying"], "paradigm": ["LLM", "system"], "method": [], "projects": [], "related": ["behavior-robot-suite", "homer", "tidybot2"], "arxiv": "2305.05658", "doi": "", "code": "", "site": "", "note": "LLMs used for the narrow, correct thing: summarizing a handful of user examples into a generalizable preference rule ('light clothes in the drawer'). Not a manipulation advance; a personalization one.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "For a robot to personalize physical assistance effectively, it must learn user preferences that can be generally reapplied to future scenarios. In this work, we investigate personalization of household cleanup with robots that can tidy up rooms by picking up objects and putting them away. A key challenge is determining the proper place to put each object, as people's preferences can vary greatly depending on personal taste or cultural background. For instance, one person may prefer storing shirts in the drawer, while another may prefer them on the shelf. We aim to build systems that can learn such preferences from just a handful of examples via prior interactions with a particular person. We show that robots can combine language-based planning and perception with the few-shot summarization capabilities of large language models (LLMs) to infer generalized user preferences that are broadly applicable to future interactions. This approach enables fast adaptation and achieves 91.2% accuracy on unseen objects in our benchmark dataset. We also demonstrate our approach on a real-world mobile manipulator called TidyBot, which successfully puts away 85.0% of objects in real-world test scenarios.", "source": "curated", "first_author": "Jimmy Wu", "url": "https://arxiv.org/abs/2305.05658", "fig": "figures/tidybot.webp"}, {"id": "vip", "title": "VIP: Towards Universal Visual Reward and Representation via Value-Implicit Pre-Training", "tree": "il", "branch": "human-video", "authors": ["Yecheng Jason Ma", "Shagun Sodhani", "Dinesh Jayaraman", "Osbert Bastani", "Vikash Kumar", "Amy Zhang"], "corresponding": [], "institutions": ["University of Pennsylvania", "Meta AI"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2023, "date": "2022-09-30", "topics": ["visual reward", "goal-conditioned value", "human video"], "paradigm": ["representation learning"], "method": [], "projects": [], "related": ["r3m", "serl"], "arxiv": "2210.00030", "doi": "", "code": "", "site": "", "note": "Gets a dense reward function out of human video via implicit value learning. Useful if you want to reward real-world RL without instrumenting the task.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Reward and representation learning are two long-standing challenges for learning an expanding set of robot manipulation skills from sensory observations. Given the inherent cost and scarcity of in-domain, task-specific robot data, learning from large, diverse, offline human videos has emerged as a promising path towards acquiring a generally useful visual representation for control; however, how these human videos can be used for general-purpose reward learning remains an open question. We introduce $\\textbf{V}$alue-$\\textbf{I}$mplicit $\\textbf{P}$re-training (VIP), a self-supervised pre-trained visual representation capable of generating dense and smooth reward functions for unseen robotic tasks. VIP casts representation learning from human videos as an offline goal-conditioned reinforcement learning problem and derives a self-supervised dual goal-conditioned value-function objective that does not depend on actions, enabling pre-training on unlabeled human videos. Theoretically, VIP can be understood as a novel implicit time contrastive objective that generates a temporally smooth embedding, enabling the value function to be implicitly defined via the embedding distance, which can then be used to construct the reward for any goal-image specified downstream task. Trained on large-scale Ego4D human videos and without any fine-tuning on in-domain, task-specific data, VIP's frozen representation can provide dense visual reward for an extensive set of simulated and $\\textbf{real-robot}$ tasks, enabling diverse reward-based visual control methods and significantly outperforming all prior pre-trained representations. Notably, VIP can enable simple, $\\textbf{few-shot}$ offline RL on a suite of real-world robot tasks with as few as 20 trajectories.", "source": "curated", "first_author": "Yecheng Jason Ma", "url": "https://arxiv.org/abs/2210.00030", "fig": "figures/vip.webp", "figs": ["figures/vip-1.webp"]}, {"id": "voyager", "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models", "tree": "systems", "branch": "agentic", "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang", "Ajay Mandlekar", "Chaowei Xiao", "Yuke Zhu", "Linxi Fan", "Anima Anandkumar"], "corresponding": [], "institutions": ["NVIDIA", "Caltech", "UT Austin", "Stanford University"], "lab": "NVIDIA GEAR", "venue": "TMLR", "venue_type": "journal", "year": 2023, "date": "2023-05-25", "topics": ["open-ended agent", "skill library", "self-directed curriculum"], "paradigm": ["LLM"], "method": [], "projects": [], "related": ["agent-laboratory", "ai-scientist", "cosmos-reason", "embo-team", "eureka", "harbor", "llm-teams", "robocat", "roco"], "arxiv": "2305.16291", "doi": "", "code": "", "site": "", "note": "Minecraft, not robots — but the growing executable skill library plus self-proposed curriculum is the architecture your 'agentic physical robot' idea needs.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We introduce Voyager, the first LLM-powered embodied lifelong learning agent in Minecraft that continuously explores the world, acquires diverse skills, and makes novel discoveries without human intervention. Voyager consists of three key components: 1) an automatic curriculum that maximizes exploration, 2) an ever-growing skill library of executable code for storing and retrieving complex behaviors, and 3) a new iterative prompting mechanism that incorporates environment feedback, execution errors, and self-verification for program improvement. Voyager interacts with GPT-4 via blackbox queries, which bypasses the need for model parameter fine-tuning. The skills developed by Voyager are temporally extended, interpretable, and compositional, which compounds the agent's abilities rapidly and alleviates catastrophic forgetting. Empirically, Voyager shows strong in-context lifelong learning capability and exhibits exceptional proficiency in playing Minecraft. It obtains 3.3x more unique items, travels 2.3x longer distances, and unlocks key tech tree milestones up to 15.3x faster than prior SOTA. Voyager is able to utilize the learned skill library in a new Minecraft world to solve novel tasks from scratch, while other techniques struggle to generalize. We open-source our full codebase and prompts at https://voyager.minedojo.org/.", "source": "curated", "first_author": "Guanzhi Wang", "url": "https://arxiv.org/abs/2305.16291", "fig": "figures/voyager.webp", "figs": ["figures/voyager-1.webp", "figures/voyager-2.webp", "figures/voyager-3.webp"]}, {"id": "dinev-codesign", "title": "A Versatile Co-Design Approach For Dynamic Legged Robots", "tree": "systems", "branch": "codesign", "authors": ["Traiko Dinev", "Carlos Mastalli", "Vladimir Ivan", "Steve Tonneau", "Sethu Vijayakumar"], "corresponding": [], "institutions": ["University of Edinburgh"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2022, "date": "2021-03-08", "topics": ["differentiating through optimal control", "co-design"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["coros-codesign", "diff-contact-design", "opt-legged-control", "transform2act"], "arxiv": "2103.04660", "doi": "", "code": "", "site": "", "note": "Differentiate through the whole motion solver to get design gradients. Complementary to Transform2Act: gradients where you have a model, RL where you do not.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We present a versatile framework for the computational co-design of legged robots and dynamic maneuvers. Current state-of-the-art approaches are typically based on random sampling or concurrent optimization. We propose a novel bilevel optimization approach that exploits the derivatives of the motion planning sub-problem (i.e., the lower level). These motion-planning derivatives allow us to incorporate arbitrary design constraints and costs in an general-purpose nonlinear program (i.e., the upper level). Our approach allows for the use of any differentiable motion planner in the lower level and also allows for an upper level that captures arbitrary design constraints and costs. It efficiently optimizes the robot's morphology, payload distribution and actuator parameters while considering its full dynamics, joint limits and physical constraints such as friction cones. We demonstrate these capabilities by designing quadruped robots that jump and trot. We show that our method is able to design a more energy-efficient Solo robot for these tasks.", "source": "curated", "first_author": "Traiko Dinev", "url": "https://arxiv.org/abs/2103.04660", "fig": "figures/dinev-codesign.webp", "figs": ["figures/dinev-codesign-1.webp"]}, {"id": "ase", "title": "ASE: Large-Scale Reusable Adversarial Skill Embeddings for Physically Simulated Characters", "tree": "rl", "branch": "motion-imitation", "authors": ["Xue Bin Peng", "Yunrong Guo", "Lina Halper", "Sergey Levine", "Sanja Fidler"], "corresponding": [], "institutions": ["UC Berkeley", "NVIDIA", "University of Toronto"], "lab": "BAIR; NVIDIA Toronto AI Lab", "venue": "SIGGRAPH", "venue_type": "conference", "year": 2022, "date": "2022-05-04", "topics": ["skill embeddings", "latent skill space", "reusable priors"], "paradigm": ["RL", "adversarial IL"], "method": ["adversarial skill discovery into a latent space", "then task RL over latents"], "projects": [], "related": ["amp", "deepmimic", "maskedmimic"], "arxiv": "2205.01906", "doi": "", "code": "", "site": "", "note": "Pretrain a latent skill space once, then solve downstream tasks by searching in it. The ancestor of every 'latent skill' loco-manipulation paper in this library.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The incredible feats of athleticism demonstrated by humans are made possible in part by a vast repertoire of general-purpose motor skills, acquired through years of practice and experience. These skills not only enable humans to perform complex tasks, but also provide powerful priors for guiding their behaviors when learning new tasks. This is in stark contrast to what is common practice in physics-based character animation, where control policies are most typically trained from scratch for each task. In this work, we present a large-scale data-driven framework for learning versatile and reusable skill embeddings for physically simulated characters. Our approach combines techniques from adversarial imitation learning and unsupervised reinforcement learning to develop skill embeddings that produce life-like behaviors, while also providing an easy to control representation for use on new downstream tasks. Our models can be trained using large datasets of unstructured motion clips, without requiring any task-specific annotation or segmentation of the motion data. By leveraging a massively parallel GPU-based simulator, we are able to train skill embeddings using over a decade of simulated experiences, enabling our model to learn a rich and versatile repertoire of skills. We show that a single pre-trained model can be effectively applied to perform a diverse set of new tasks. Our system also allows users to specify tasks through simple reward functions, and the skill embedding then enables the character to automatically synthesize complex and naturalistic strategies in order to achieve the task objectives.", "source": "curated", "first_author": "Xue Bin Peng", "url": "https://arxiv.org/abs/2205.01906", "fig": "figures/ase.webp", "figs": ["figures/ase-1.webp"]}, {"id": "barc", "title": "BARC: Learning to Regress 3D Dog Shape from Images by Exploiting Breed Information", "tree": "il", "branch": "human-video", "authors": ["Nadine Rueegg", "Silvia Zuffi", "Konrad Schindler", "Michael J. Black"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-03-29", "topics": ["3D dog shape regression", "breed priors"], "paradigm": ["representation learning"], "method": [], "projects": [], "related": ["animal-avatars", "corgi", "dogmo", "hsmal"], "arxiv": "2203.15536", "doi": "10.48550/arxiv.2203.15536", "code": "", "site": "", "note": "Regresses 3D dog shape from ordinary images using breed information as a prior. The shape half of 'measure the animal' — you need body proportions before you can scale a kinematic chain to them.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Our goal is to recover the 3D shape and pose of dogs from a single image. This is a challenging task because dogs exhibit a wide range of shapes and appearances, and are highly articulated. Recent work has proposed to directly regress the SMAL animal model, with additional limb scale parameters, from images. Our method, called BARC (Breed-Augmented Regression using Classification), goes beyond prior work in several important ways. First, we modify the SMAL shape space to be more appropriate for representing dog shape. But, even with a better shape model, the problem of regressing dog shape from an image is still challenging because we lack paired images with 3D ground truth. To compensate for the lack of paired data, we formulate novel losses that exploit information about dog breeds. In particular, we exploit the fact that dogs of the same breed have similar body shapes. We formulate a novel breed similarity loss consisting of two parts: One term encourages the shape of dogs from the same breed to be more similar than dogs of different breeds. The second one, a breed classification loss, helps to produce recognizable breed-specific shapes. Through ablation studies, we find that our breed losses significantly improve shape accuracy over a baseline without them. We also compare BARC qualitatively to WLDO with a perceptual study and find that our approach produces dogs that are significantly more realistic. This work shows that a-priori information about genetic similarity can help to compensate for the lack of 3D training data. This concept may be applicable to other animal species or groups of species. Our code is publicly available for research purposes at https://barc.is.tue.mpg.de/.", "source": "curated", "first_author": "Nadine Rueegg", "url": "https://arxiv.org/abs/2203.15536", "fig": "figures/barc.webp", "figs": ["figures/barc-1.webp", "figures/barc-2.webp", "figures/barc-3.webp"]}, {"id": "deep-wbc", "title": "Deep Whole-Body Control: Learning a Unified Policy for Manipulation and Locomotion", "tree": "rl", "branch": "loco-manip", "authors": ["Zipeng Fu", "Xuxin Cheng", "Deepak Pathak"], "corresponding": ["Zipeng Fu"], "institutions": ["Carnegie Mellon University", "Stanford University"], "lab": "CMU Pathak Lab", "venue": "CoRL", "venue_type": "conference", "year": 2022, "date": "2022-10-18", "topics": ["whole-body control", "legged manipulation"], "paradigm": ["RL"], "method": ["single policy over arm+legs with regularized online adaptation"], "projects": ["mabel"], "related": ["falcon", "fetchman", "visualmimic", "wheeled-bilateral"], "arxiv": "2210.10044", "doi": "", "code": "", "site": "", "note": "Made the case that arm and legs should share one policy rather than be composed from two controllers.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "An attached arm can significantly increase the applicability of legged robots to several mobile manipulation tasks that are not possible for the wheeled or tracked counterparts. The standard hierarchical control pipeline for such legged manipulators is to decouple the controller into that of manipulation and locomotion. However, this is ineffective. It requires immense engineering to support coordination between the arm and legs, and error can propagate across modules causing non-smooth unnatural motions. It is also biological implausible given evidence for strong motor synergies across limbs. In this work, we propose to learn a unified policy for whole-body control of a legged manipulator using reinforcement learning. We propose Regularized Online Adaptation to bridge the Sim2Real gap for high-DoF control, and Advantage Mixing exploiting the causal dependency in the action space to overcome local minima during training the whole-body system. We also present a simple design for a low-cost legged manipulator, and find that our unified policy can demonstrate dynamic and agile behaviors across several task setups. Videos are at https://maniploco.github.io", "source": "curated", "first_author": "Zipeng Fu", "url": "https://arxiv.org/abs/2210.10044", "fig": "figures/deep-wbc.webp", "figs": ["figures/deep-wbc-1.webp", "figures/deep-wbc-2.webp"]}, {"id": "dexmv", "title": "DexMV: Imitation Learning for Dexterous Manipulation from Human Videos", "tree": "il", "branch": "human-video", "authors": ["Yuzhe Qin", "Yueh-Hua Wu", "Shaowei Liu", "Hanwen Jiang", "Ruihan Yang", "Yang Fu", "Xiaolong Wang"], "corresponding": ["Yuzhe Qin"], "institutions": ["UC San Diego"], "lab": "UCSD Xiaolong Wang Lab", "venue": "ECCV", "venue_type": "conference", "year": 2022, "date": "2021-08-12", "topics": ["dexterous manipulation from video", "hand pose"], "paradigm": ["IL", "RL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["c2dex", "dexcap", "videodex"], "arxiv": "2108.05877", "doi": "", "code": "", "site": "", "note": "Early and still-useful pipeline: 3D hand-object pose from video -> retargeted demonstrations -> RL/IL on a multi-finger hand.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "While significant progress has been made on understanding hand-object interactions in computer vision, it is still very challenging for robots to perform complex dexterous manipulation. In this paper, we propose a new platform and pipeline DexMV (Dexterous Manipulation from Videos) for imitation learning. We design a platform with: (i) a simulation system for complex dexterous manipulation tasks with a multi-finger robot hand and (ii) a computer vision system to record large-scale demonstrations of a human hand conducting the same tasks. In our novel pipeline, we extract 3D hand and object poses from videos, and propose a novel demonstration translation method to convert human motion to robot demonstrations. We then apply and benchmark multiple imitation learning algorithms with the demonstrations. We show that the demonstrations can indeed improve robot learning by a large margin and solve the complex tasks which reinforcement learning alone cannot solve. More details can be found in the project page: https://yzqin.github.io/dexmv", "source": "curated", "first_author": "Yuzhe Qin", "url": "https://arxiv.org/abs/2108.05877", "fig": "figures/dexmv.webp", "figs": ["figures/dexmv-1.webp"]}, {"id": "evogym", "title": "Evolution Gym: A Large-Scale Benchmark for Evolving Soft Robots", "tree": "systems", "branch": "codesign", "authors": ["Jagdeep Singh Bhatia", "Holly Jackson", "Yunsheng Tian", "Jie Xu", "Wojciech Matusik"], "corresponding": [], "institutions": ["Massachusetts Institute of Technology"], "lab": "MIT CSAIL", "venue": "NeurIPS Datasets & Benchmarks", "venue_type": "conference", "year": 2022, "date": "2022-01-24", "topics": ["co-design benchmark", "soft robots"], "paradigm": ["co-design", "benchmark"], "method": [], "projects": [], "related": ["derl", "diffusebot", "sims1994"], "arxiv": "2201.09863", "doi": "", "code": "", "site": "", "note": "The benchmark the area needed. Useful for sanity-checking a co-design algorithm before spending GPU-months on a quadruped.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Both the design and control of a robot play equally important roles in its task performance. However, while optimal control is well studied in the machine learning and robotics community, less attention is placed on finding the optimal robot design. This is mainly because co-optimizing design and control in robotics is characterized as a challenging problem, and more importantly, a comprehensive evaluation benchmark for co-optimization does not exist. In this paper, we propose Evolution Gym, the first large-scale benchmark for co-optimizing the design and control of soft robots. In our benchmark, each robot is composed of different types of voxels (e.g., soft, rigid, actuators), resulting in a modular and expressive robot design space. Our benchmark environments span a wide range of tasks, including locomotion on various types of terrains and manipulation. Furthermore, we develop several robot co-evolution algorithms by combining state-of-the-art design optimization methods and deep reinforcement learning techniques. Evaluating the algorithms on our benchmark platform, we observe robots exhibiting increasingly complex behaviors as evolution progresses, with the best evolved designs solving many of our proposed tasks. Additionally, even though robot designs are evolved autonomously from scratch without prior knowledge, they often grow to resemble existing natural creatures while outperforming hand-designed robots. Nevertheless, all tested algorithms fail to find robots that succeed in our hardest environments. This suggests that more advanced algorithms are required to explore the high-dimensional design space and evolve increasingly intelligent robots -- an area of research in which we hope Evolution Gym will accelerate progress. Our website with code, environments, documentation, and tutorials is available at http://evogym.csail.mit.edu.", "source": "curated", "first_author": "Jagdeep Singh Bhatia", "url": "https://proceedings.neurips.cc/paper/2021/file/118921efba23fc329e6560b27861f0c2-Paper.pdf", "arxiv_url": "https://arxiv.org/abs/2201.09863", "fig": "figures/evogym.webp", "figs": ["figures/evogym-1.webp"]}, {"id": "intention-tracking", "title": "Hierarchical Intention Tracking for Robust Human-Robot Collaboration in  Industrial Assembly Tasks", "tree": "rl", "branch": "marl", "authors": ["Zhe Huang", "Ye-Ji Mun", "Xiang Li", "Yiqing Xie", "Ninghan Zhong", "Weihang Liang", "Junyi Geng", "Tan Chen", "Katherine Driggs-Campbell"], "corresponding": [], "institutions": ["University of Illinois Urbana-Champaign"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-03-16", "topics": ["intention tracking", "industrial assembly", "human-robot collaboration"], "paradigm": ["HRI"], "method": [], "projects": ["m2"], "related": ["falcon", "h2compact", "interact", "phhi-survey", "workspace-opt"], "arxiv": "2203.09063", "doi": "10.48550/arxiv.2203.09063", "code": "", "site": "", "note": "Hierarchical intention tracking so the robot can both avoid interrupting the human and step in when the human is failing. That second behaviour — assistive intervention — is what separates a collaborator from a compliant obstacle.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Collaborative robots require effective human intention estimation to safely and smoothly work with humans in less structured tasks such as industrial assembly, where human intention continuously changes. We propose the concept of intention tracking and introduce a collaborative robot system that concurrently tracks intentions at hierarchical levels. The high-level intention is tracked to estimate human's interaction pattern and enable robot to (1) avoid collision with human to minimize interruption and (2) assist human to correct failure. The low-level intention estimate provides robot with task-related information. We implement the system on a UR5e robot and demonstrate robust, seamless and ergonomic human-robot collaboration in an ablative pilot study of an assembly use case. Our robot demonstrations and videos are available at \\\\url{https://sites.google.com/view/hierarchicalintentiontracking}. ", "source": "curated", "first_author": "Zhe Huang", "url": "https://arxiv.org/abs/2203.09063", "fig": "figures/intention-tracking.webp", "figs": ["figures/intention-tracking-1.webp", "figures/intention-tracking-2.webp", "figures/intention-tracking-3.webp"]}, {"id": "qi-inhand", "title": "In-Hand Object Rotation via Rapid Motor Adaptation", "tree": "rl", "branch": "manipulation", "authors": ["Haozhi Qi", "Ashish Kumar", "Roberto Calandra", "Yi Ma", "Jitendra Malik"], "corresponding": ["Haozhi Qi"], "institutions": ["UC Berkeley", "Meta AI"], "lab": "BAIR", "venue": "CoRL", "venue_type": "conference", "year": 2022, "date": "2022-10-10", "topics": ["in-hand rotation", "online adaptation", "proprioception"], "paradigm": ["RL"], "method": ["RMA-style adaptation module on an Allegro hand", "proprioception only"], "projects": ["geodex"], "related": ["allegro", "chen-inhand", "chen-visual-inhand", "dactyl", "rma", "robot-synesthesia", "rotate-without-seeing"], "arxiv": "2210.04887", "doi": "", "code": "", "site": "", "note": "RMA ported from legs to fingers, on a cheap Allegro hand with no vision. The proprioception-only constraint is what makes it deployable.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Generalized in-hand manipulation has long been an unsolved challenge of robotics. As a small step towards this grand goal, we demonstrate how to design and learn a simple adaptive controller to achieve in-hand object rotation using only fingertips. The controller is trained entirely in simulation on only cylindrical objects, which then - without any fine-tuning - can be directly deployed to a real robot hand to rotate dozens of objects with diverse sizes, shapes, and weights over the z-axis. This is achieved via rapid online adaptation of the controller to the object properties using only proprioception history. Furthermore, natural and stable finger gaits automatically emerge from training the control policy via reinforcement learning. Code and more videos are available at https://haozhi.io/hora", "source": "curated", "first_author": "Haozhi Qi", "url": "https://arxiv.org/abs/2210.04887", "fig": "figures/qi-inhand.webp", "figs": ["figures/qi-inhand-1.webp"]}, {"id": "meta-rl-legged", "title": "Meta Reinforcement Learning for Optimal Design of Legged Robots", "tree": "systems", "branch": "codesign", "authors": ["Álvaro Belmonte-Baeza", "Joonho Lee", "Giorgio Valsecchi", "Marco Hutter"], "corresponding": [], "institutions": ["ETH Zürich"], "lab": "ETH RSL", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2022, "date": "2022-10-06", "topics": ["meta-RL", "legged design optimization"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["dyret", "rudin2021", "transform2act"], "arxiv": "2210.02750", "doi": "", "code": "", "site": "", "note": "Trains a design-conditioned policy once, then evaluates candidate morphologies cheaply without retraining. The practical trick that makes legged co-design affordable — read it before committing compute to the dog.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The process of robot design is a complex task and the majority of design decisions are still based on human intuition or tedious manual tuning. A more informed way of facing this task is computational design methods where design parameters are concurrently optimized with corresponding controllers. Existing approaches, however, are strongly influenced by predefined control rules or motion templates and cannot provide end-to-end solutions. In this paper, we present a design optimization framework using model-free meta reinforcement learning, and its application to the optimizing kinematics and actuator parameters of quadrupedal robots. We use meta reinforcement learning to train a locomotion policy that can quickly adapt to different designs. This policy is used to evaluate each design instance during the design optimization. We demonstrate that the policy can control robots of different designs to track random velocity commands over various rough terrains. With controlled experiments, we show that the meta policy achieves close-to-optimal performance for each design instance after adaptation. Lastly, we compare our results against a model-based baseline and show that our approach allows higher performance while not being constrained by predefined motions or gait patterns.", "source": "curated", "first_author": "Álvaro Belmonte-Baeza", "url": "https://arxiv.org/abs/2210.02750", "fig": "figures/meta-rl-legged.webp", "figs": ["figures/meta-rl-legged-1.webp", "figures/meta-rl-legged-2.webp"]}, {"id": "opt-legged-control", "title": "Optimization-Based Control for Dynamic Legged Robots", "tree": "rl", "branch": "locomotion", "authors": ["Patrick M. Wensing", "Michael Posa", "Yue Hu", "Adrien Escande", "Nicolas Mansard", "Andrea Del Prete"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-11-21", "topics": ["optimization-based control", "dynamic legged robots", "survey"], "paradigm": ["control"], "method": [], "projects": ["mabel"], "related": ["changing-morph-dyn", "coros-codesign", "dinev-codesign"], "arxiv": "2211.11644", "doi": "10.48550/arxiv.2211.11644", "code": "", "site": "", "note": "Survey of optimization-based control for dynamic legged robots. The model-based half you will need for the transformation manoeuvre, where you have a good model and only a couple of seconds of trajectory to plan.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In a world designed for legs, quadrupeds, bipeds, and humanoids have the opportunity to impact emerging robotics applications from logistics, to agriculture, to home assistance. The goal of this survey is to cover the recent progress toward these applications that has been driven by model-based optimization for the real-time generation and control of movement. The majority of the research community has converged on the idea of generating locomotion control laws by solving an optimal control problem (OCP) in either a model-based or data-driven manner. However, solving the most general of these problems online remains intractable due to complexities from intermittent unidirectional contacts with the environment, and from the many degrees of freedom of legged robots. This survey covers methods that have been pursued to make these OCPs computationally tractable, with specific focus on how environmental contacts are treated, how the model can be simplified, and how these choices affect the numerical solution methods employed. The survey focuses on model-based optimization, covering its recent use in a stand alone fashion, and suggesting avenues for combination with learning-based formulations to further accelerate progress in this growing field.", "source": "curated", "first_author": "Patrick M. Wensing", "url": "https://arxiv.org/abs/2211.11644", "fig": "figures/opt-legged-control.webp", "figs": ["figures/opt-legged-control-1.webp", "figures/opt-legged-control-2.webp"]}, {"id": "real2sim", "title": "Planar Robot Casting with Real2Sim2Real Self-Supervised Learning", "tree": "rl", "branch": "sim2real", "authors": ["Vincent Lim", "Huang Huang", "Lawrence Yunliang Chen", "Jonathan Wang", "Jeffrey Ichnowski", "Daniel Seita", "Michael Laskey", "Ken Goldberg"], "corresponding": [], "institutions": ["UC Berkeley"], "lab": "AUTOLAB", "venue": "ICRA", "venue_type": "conference", "year": 2022, "date": "2021-11-08", "topics": ["real2sim2real", "dynamics identification", "deformables"], "paradigm": ["RL", "sim2real"], "method": [], "projects": ["r2s2r"], "related": ["asap", "dist-r2s2r", "simweaver", "video2sim2real", "videomimic", "wheeled-payload"], "arxiv": "2111.04814", "doi": "", "code": "", "site": "", "note": "An explicit real2sim2real loop on a deformable (a cable). Useful precedent for both the real2sim2real idea and the soft-object sim-to-real gap.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "This paper introduces the task of {\\em Planar Robot Casting (PRC)}: where one planar motion of a robot arm holding one end of a cable causes the other end to slide across the plane toward a desired target. PRC allows the cable to reach points beyond the robot workspace and has applications for cable management in homes, warehouses, and factories. To efficiently learn a PRC policy for a given cable, we propose Real2Sim2Real, a self-supervised framework that automatically collects physical trajectory examples to tune parameters of a dynamics simulator using Differential Evolution, generates many simulated examples, and then learns a policy using a weighted combination of simulated and physical data. We evaluate Real2Sim2Real with three simulators, Isaac Gym-segmented, Isaac Gym-hybrid, and PyBullet, two function approximators, Gaussian Processes and Neural Networks (NNs), and three cables with differing stiffness, torsion, and friction. Results with 240 physical trials suggest that the PRC policies can attain median error distance (as % of cable length) ranging from 8% to 14%, outperforming baselines and policies trained on only real or only simulated examples. Code, data, and videos are available at https://tinyurl.com/robotcast.", "source": "curated", "first_author": "Vincent Lim", "url": "https://arxiv.org/abs/2111.04814", "fig": "figures/real2sim.webp", "figs": ["figures/real2sim-1.webp", "figures/real2sim-2.webp", "figures/real2sim-3.webp"]}, {"id": "swheg", "title": "SWheg: A Wheel-Leg Transformable Robot With Minimalist Actuator Realization", "tree": "systems", "branch": "codesign", "authors": ["Cunxi Dai", "Xiaohan Liu", "Jianxiang Zhou", "Zhengtao Liu", "Zhenzhong Jia"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-10-27", "topics": ["wheel-leg transformable", "minimalist actuation"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["mit-cheetah", "reconfig-wheelleg", "var-topology-truss", "x2n"], "arxiv": "2210.15126", "doi": "10.48550/arxiv.2210.15126", "code": "", "site": "", "note": "A wheel-leg transformable robot that reconfigures with a <b>single</b> extra actuator per wheel. The mechanical lesson for any transformer: the transformation should cost you one actuator, not a second robot.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "This article presents the design, implementation, and performance evaluation of SWheg, a novel modular wheel-leg transformable robot family with minimalist actuator realization. SWheg takes advantage of both wheeled and legged locomotion by seamlessly integrating them on a single platform. In contrast to other designs that use multiple actuators, SWheg uses only one actuator to drive the transformation of all the wheel-leg modules in sync. This means an N-legged SWheg robot requires only N+1 actuators, which can significantly reduce the cost and malfunction rate of the platform. The tendon-driven wheel-leg transformation mechanism based on a four-bar linkage can perform fast morphology transitions between wheels and legs. We validated the design principle with two SWheg robots with four and six wheel-leg modules separately, namely Quadrupedal SWheg and Hexapod SWheg. The design process, mechatronics infrastructure, and the gait behavioral development of both platforms were discussed. The performance of the robot was evaluated in various scenarios, including driving and turning in wheeled mode, step crossing, irregular terrain passing, and stair climbing in legged mode. The comparison between these two platforms was also discussed.", "source": "curated", "first_author": "Cunxi Dai", "url": "https://arxiv.org/abs/2210.15126", "fig": "figures/swheg.webp", "figs": ["figures/swheg-1.webp", "figures/swheg-2.webp", "figures/swheg-3.webp"]}, {"id": "see-hear-feel", "title": "See, Hear, and Feel: Smart Sensory Fusion for Robotic Manipulation", "tree": "il", "branch": "tactile-learn", "authors": ["Hao Li", "Yizhi Zhang", "Junzhe Zhu", "Shaoxiong Wang", "Michelle A. Lee", "Huazhe Xu", "Edward H. Adelson", "Feifei Li", "Ruohan Gao", "Jiajun Wu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-12-07", "topics": ["multisensory fusion", "vision touch audio"], "paradigm": ["IL", "audio"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["maniwav", "robot-synesthesia"], "arxiv": "2212.03858", "doi": "10.48550/arxiv.2212.03858", "code": "", "site": "", "note": "The early systematic study of fusing vision, touch and audio for manipulation, and still the reference for <em>when</em> each modality actually helps. Read it before designing the fusion architecture.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Humans use all of their senses to accomplish different tasks in everyday activities. In contrast, existing work on robotic manipulation mostly relies on one, or occasionally two modalities, such as vision and touch. In this work, we systematically study how visual, auditory, and tactile perception can jointly help robots to solve complex manipulation tasks. We build a robot system that can see with a camera, hear with a contact microphone, and feel with a vision-based tactile sensor, with all three sensory modalities fused with a self-attention model. Results on two challenging tasks, dense packing and pouring, demonstrate the necessity and power of multisensory perception for robotic manipulation: vision displays the global status of the robot but can often suffer from occlusion, audio provides immediate feedback of key moments that are even invisible, and touch offers precise local geometry for decision making. Leveraging all three modalities, our robotic system significantly outperforms prior methods.", "source": "curated", "first_author": "Hao Li", "url": "https://arxiv.org/abs/2212.03858", "fig": "figures/see-hear-feel.webp", "figs": ["figures/see-hear-feel-1.webp", "figures/see-hear-feel-2.webp"]}, {"id": "stretch", "title": "The Design of Stretch: A Compact, Lightweight Mobile Manipulator for Indoor Human Environments", "tree": "systems", "branch": "mobile-manip", "authors": ["Charles C. Kemp", "Aaron Edsinger", "Henry M. Clever", "Blaine Matulevich"], "corresponding": [], "institutions": ["Hello Robot"], "lab": "Hello Robot", "venue": "ICRA", "venue_type": "conference", "year": 2022, "date": "2021-09-21", "topics": ["commercial mobile manipulator", "home robotics"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": ["homer", "tidybot2"], "arxiv": "2109.10892", "doi": "", "code": "", "site": "", "note": "In your MABEL hardware folder. The lightweight telescoping-arm design point — very different tradeoff from a bimanual humanoid, and the one most home-robotics research actually runs on.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/MABEL/literatures/hardware/2109.10892v3.pdf", "abstract": "Mobile manipulators for indoor human environments can serve as versatile devices that perform a variety of tasks, yet adoption of this technology has been limited. Reducing size, weight, and cost could facilitate adoption, but risks restricting capabilities. We present a novel design that reduces size, weight, and cost, while supporting a variety of tasks. The core design consists of a two-wheeled differential-drive mobile base, a lift, and a telescoping arm configured to achieve Cartesian motion at the end of the arm. Design extensions include a 1 degree-of-freedom (DOF) wrist to stow a tool, a 2-DOF dexterous wrist to pitch and roll a tool, and a compliant gripper. We justify our design with anthropometry and mathematical models of static stability. We also provide empirical support from teleoperating and autonomously controlling a commercial robot based on our design (the Stretch RE1 from Hello Robot Inc.) to perform tasks in real homes.", "source": "curated", "first_author": "Charles C. Kemp", "url": "https://arxiv.org/abs/2109.10892", "fig": "figures/stretch.webp", "figs": ["figures/stretch-1.webp", "figures/stretch-2.webp", "figures/stretch-3.webp"]}, {"id": "iris", "title": "Transformers are Sample-Efficient World Models", "tree": "rl", "branch": "model-based", "authors": ["Vincent Micheli", "Eloi Alonso", "François Fleuret"], "corresponding": [], "institutions": ["University of Geneva"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2022, "date": "2022-09-01", "topics": ["discrete autoencoder", "transformer world model", "sample efficiency"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["diamond", "dreamer-v2", "genie-envisioner", "ivideogpt", "storm"], "arxiv": "2209.00588", "doi": "10.48550/arXiv.2209.00588", "code": "", "site": "", "note": "<b>IRIS</b> — 'Transformers are Sample-Efficient World Models'. Tokenize frames with a discrete autoencoder, then let an autoregressive transformer be the dynamics model, so imagination becomes next-token prediction. This is the architectural bridge between the Dreamer line and everything that later got called a video world model: once the world model is a sequence model over tokens, all the scaling machinery from language transfers.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Deep reinforcement learning agents are notoriously sample inefficient, which considerably limits their application to real-world problems. Recently, many model-based methods have been designed to address this issue, with learning in the imagination of a world model being one of the most prominent approaches. However, while virtually unlimited interaction with a simulated environment sounds appealing, the world model has to be accurate over extended periods of time. Motivated by the success of Transformers in sequence modeling tasks, we introduce IRIS, a data-efficient agent that learns in a world model composed of a discrete autoencoder and an autoregressive Transformer. With the equivalent of only two hours of gameplay in the Atari 100k benchmark, IRIS achieves a mean human normalized score of 1.046, and outperforms humans on 10 out of 26 games, setting a new state of the art for methods without lookahead search. To foster future research on Transformers and world models for sample-efficient reinforcement learning, we release our code and models at https://github.com/eloialonso/iris.", "source": "curated", "first_author": "Vincent Micheli", "url": "https://arxiv.org/abs/2209.00588", "fig": "figures/iris.webp", "figs": ["figures/iris-1.webp", "figures/iris-2.webp", "figures/iris-3.webp"]}, {"id": "videodex", "title": "VideoDex: Learning Dexterity from Internet Videos", "tree": "il", "branch": "human-video", "authors": ["Kenneth Shaw", "Shikhar Bahl", "Deepak Pathak"], "corresponding": ["Kenneth Shaw"], "institutions": ["Carnegie Mellon University"], "lab": "CMU Pathak Lab", "venue": "CoRL", "venue_type": "conference", "year": 2022, "date": "2022-12-08", "topics": ["internet video", "dexterity priors"], "paradigm": ["IL"], "method": [], "projects": ["geodex", "r2s2r"], "related": ["dexcap", "dexmv", "ego4d", "leap-hand"], "arxiv": "2212.04498", "doi": "", "code": "", "site": "", "note": "Internet video as a prior over *how hands move*, transferred to a LEAP-style hand. Same lab as LEAP Hand — hardware and learning co-designed.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "To build general robotic agents that can operate in many environments, it is often imperative for the robot to collect experience in the real world. However, this is often not feasible due to safety, time, and hardware restrictions. We thus propose leveraging the next best thing as real-world experience: internet videos of humans using their hands. Visual priors, such as visual features, are often learned from videos, but we believe that more information from videos can be utilized as a stronger prior. We build a learning algorithm, VideoDex, that leverages visual, action, and physical priors from human video datasets to guide robot behavior. These actions and physical priors in the neural network dictate the typical human behavior for a particular robot task. We test our approach on a robot arm and dexterous hand-based system and show strong results on various manipulation tasks, outperforming various state-of-the-art methods. Videos at https://video-dex.github.io", "source": "curated", "first_author": "Kenneth Shaw", "url": "https://arxiv.org/abs/2212.04498", "fig": "figures/videodex.webp", "figs": ["figures/videodex-1.webp", "figures/videodex-2.webp", "figures/videodex-3.webp"]}, {"id": "implicit-bc", "title": "Implicit Behavioral Cloning", "tree": "il", "branch": "foundations", "authors": ["Pete Florence", "Corey Lynch", "Andy Zeng", "Oscar Ramirez", "Ayzaan Wahid", "Laura Downs", "Adrian Wong", "Johnny Lee", "Igor Mordatch", "Jonathan Tompson"], "corresponding": [], "institutions": ["Google"], "lab": "Robotics at Google", "venue": "CoRL", "venue_type": "conference", "year": 2021, "date": "2021-09-01", "topics": ["energy-based policies", "multimodality"], "paradigm": ["BC"], "method": ["EBM over (observation", "action) instead of explicit regression"], "projects": [], "related": ["act", "dp"], "arxiv": "2109.00137", "doi": "", "code": "", "site": "", "note": "Identified the problem diffusion policy later solved better: regressing to the mean of a multimodal demonstration set produces an action that satisfies nobody.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We find that across a wide range of robot policy learning scenarios, treating supervised policy learning with an implicit model generally performs better, on average, than commonly used explicit models. We present extensive experiments on this finding, and we provide both intuitive insight and theoretical arguments distinguishing the properties of implicit models compared to their explicit counterparts, particularly with respect to approximating complex, potentially discontinuous and multi-valued (set-valued) functions. On robotic policy learning tasks we show that implicit behavioral cloning policies with energy-based models (EBM) often outperform common explicit (Mean Square Error, or Mixture Density) behavioral cloning policies, including on tasks with high-dimensional action spaces and visual image inputs. We find these policies provide competitive results or outperform state-of-the-art offline reinforcement learning methods on the challenging human-expert tasks from the D4RL benchmark suite, despite using no reward information. In the real world, robots with implicit policies can learn complex and remarkably subtle behaviors on contact-rich tasks from human demonstrations, including tasks with high combinatorial complexity and tasks requiring 1mm precision.", "source": "curated", "first_author": "Pete Florence", "url": "https://arxiv.org/abs/2109.00137", "fig": "figures/implicit-bc.webp", "figs": ["figures/implicit-bc-1.webp", "figures/implicit-bc-2.webp", "figures/implicit-bc-3.webp"]}, {"id": "var-topology-truss", "title": "Motion Planning for Variable Topology Trusses: Reconfiguration and Locomotion", "tree": "systems", "branch": "codesign", "authors": ["Chao Liu", "Sencheng Yu", "Mark Yim"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2021, "date": "2021-07-31", "topics": ["variable topology truss", "reconfiguration and locomotion planning"], "paradigm": ["planning"], "method": [], "projects": [], "related": ["changing-morph-dyn", "swheg", "x2n"], "arxiv": "2108.00309", "doi": "10.48550/arxiv.2108.00309", "code": "", "site": "", "note": "Motion planning for trusses that change their <b>connection topology</b>, not just their joint angles. The closest existing formal treatment of reconfiguration where members change role — which is the abstract version of a body panel becoming a limb.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Truss robots are highly redundant parallel robotic systems that can be applied in a variety of scenarios. The variable topology truss (VTT) is a class of modular truss robots. As self-reconfigurable modular robots, a VTT is composed of many edge modules that can be rearranged into various structures depending on the task. These robots change their shape by not only controlling joint positions as with fixed morphology robots, but also reconfiguring the connectivity between truss members in order to change their topology. The motion planning problem for VTT robots is difficult due to their varying morphology, high dimensionality, the high likelihood for self-collision, and complex motion constraints. In this paper, a new motion planning framework to dramatically alter the structure of a VTT is presented. It can also be used to solve locomotion tasks that are much more efficient compared with previous work. Several test scenarios are used to show its effectiveness. Supplementary materials are available at https://www.modlabupenn.org/vtt-motion-planning/.", "source": "curated", "first_author": "Chao Liu", "url": "https://arxiv.org/abs/2108.00309", "fig": "figures/var-topology-truss.webp"}, {"id": "reskin", "title": "ReSkin: versatile, replaceable, lasting tactile skins", "tree": "systems", "branch": "tactile-hw", "authors": ["Raunaq Bhirangi", "Tess Hellebrekers", "Carmel Majidi", "Abhinav Gupta"], "corresponding": [], "institutions": ["Carnegie Mellon University", "Meta AI"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2021, "date": "2021-11-01", "topics": ["magnetic skin", "durability"], "paradigm": ["hardware"], "method": [], "projects": ["wam-tactile"], "related": ["anyskin", "digit"], "arxiv": "2111.00071", "doi": "", "code": "", "site": "", "note": "AnySkin's predecessor — elastomer with embedded magnetic particles.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Soft sensors have continued growing interest in robotics, due to their ability to enable both passive conformal contact from the material properties and active contact data from the sensor properties. However, the same properties of conformal contact result in faster deterioration of soft sensors and larger variations in their response characteristics over time and across samples, inhibiting their ability to be long-lasting and replaceable. ReSkin is a tactile soft sensor that leverages machine learning and magnetic sensing to offer a low-cost, diverse and compact solution for long-term use. Magnetic sensing separates the electronic circuitry from the passive interface, making it easier to replace interfaces as they wear out while allowing for a wide variety of form factors. Machine learning allows us to learn sensor response models that are robust to variations across fabrication and time, and our self-supervised learning algorithm enables finer performance enhancement with small, inexpensive data collection procedures. We believe that ReSkin opens the door to more versatile, scalable and inexpensive tactile sensation modules than existing alternatives.", "source": "curated", "first_author": "Raunaq Bhirangi", "url": "https://arxiv.org/abs/2111.00071", "fig": "figures/reskin.webp", "figs": ["figures/reskin-1.webp", "figures/reskin-2.webp"]}, {"id": "taskagnostic-morph", "title": "Task-Agnostic Morphology Evolution", "tree": "systems", "branch": "codesign", "authors": ["Donald J. Hejna III", "Pieter Abbeel", "Lerrel Pinto"], "corresponding": [], "institutions": ["Shandong University of Political Science and Law", "Supélec", "University of Applied Sciences and Arts of Southern Switzerland", "University of California, Berkeley"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2021, "date": "2021-02-25", "topics": ["task-agnostic morphology", "information-theoretic objective"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["derl", "transform2act"], "arxiv": "2102.13100", "doi": "", "code": "", "site": "", "note": "Evolve morphologies without committing to a task, by maximizing the diversity of behaviour the body can express. Relevant if you want a hand that is good at dexterity in general rather than at one benchmark.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Deep reinforcement learning primarily focuses on learning behavior, usually overlooking the fact that an agent's function is largely determined by form. So, how should one go about finding a morphology fit for solving tasks in a given environment? Current approaches that co-adapt morphology and behavior use a specific task's reward as a signal for morphology optimization. However, this often requires expensive policy optimization and results in task-dependent morphologies that are not built to generalize. In this work, we propose a new approach, Task-Agnostic Morphology Evolution (TAME), to alleviate both of these issues. Without any task or reward specification, TAME evolves morphologies by only applying randomly sampled action primitives on a population of agents. This is accomplished using an information-theoretic objective that efficiently ranks agents by their ability to reach diverse states in the environment and the causality of their actions. Finally, we empirically demonstrate that across 2D, 3D, and manipulation environments TAME can evolve morphologies that match the multi-task performance of those learned with task supervised algorithms. Our code and videos can be found at https://sites.google.com/view/task-agnostic-evolution.", "source": "curated", "first_author": "Donald J. Hejna III", "url": "https://arxiv.org/abs/2102.13100", "arxiv_url": "https://arxiv.org/abs/2102.13100", "fig": "figures/taskagnostic-morph.webp", "figs": ["figures/taskagnostic-morph-1.webp"]}, {"id": "digit", "title": "DIGIT: A Novel Design for a Low-Cost Compact High-Resolution Tactile Sensor with Application to In-Hand Manipulation", "tree": "systems", "branch": "tactile-hw", "authors": ["Mike Lambeta", "Po-Wei Chou", "Stephen Tian", "Brian Yang", "Benjamin Maloon", "Victoria Rose Most", "et al."], "corresponding": ["Mike Lambeta"], "institutions": ["Meta AI"], "lab": "FAIR", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2020, "date": "2020-05-24", "topics": ["compact tactile sensor", "open hardware"], "paradigm": ["hardware"], "method": [], "projects": ["geodex", "wam-tactile"], "related": ["anyskin", "dexskin", "gelsight", "reskin"], "arxiv": "2005.14679", "doi": "", "code": "", "site": "", "note": "GelSight shrunk to fingertip scale and open-sourced. The default fingertip sensor for multi-finger hands.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Despite decades of research, general purpose in-hand manipulation remains one of the unsolved challenges of robotics. One of the contributing factors that limit current robotic manipulation systems is the difficulty of precisely sensing contact forces -- sensing and reasoning about contact forces are crucial to accurately control interactions with the environment. As a step towards enabling better robotic manipulation, we introduce DIGIT, an inexpensive, compact, and high-resolution tactile sensor geared towards in-hand manipulation. DIGIT improves upon past vision-based tactile sensors by miniaturizing the form factor to be mountable on multi-fingered hands, and by providing several design improvements that result in an easier, more repeatable manufacturing process, and enhanced reliability. We demonstrate the capabilities of the DIGIT sensor by training deep neural network model-based controllers to manipulate glass marbles in-hand with a multi-finger robotic hand. To provide the robotic community access to reliable and low-cost tactile sensors, we open-source the DIGIT design at https://digit.ml/.", "source": "curated", "first_author": "Mike Lambeta", "url": "https://arxiv.org/abs/2005.14679", "fig": "figures/digit.webp", "figs": ["figures/digit-1.webp", "figures/digit-2.webp", "figures/digit-3.webp"]}, {"id": "fit2form", "title": "Fit2Form: 3D Generative Model for Robot Gripper Form Design", "tree": "systems", "branch": "codesign", "authors": ["Huy Ha", "Shubham Agrawal", "Shuran Song"], "corresponding": ["Huy Ha"], "institutions": ["Columbia University"], "lab": "Columbia REALab", "venue": "CoRL", "venue_type": "conference", "year": 2020, "date": "2020-11-12", "topics": ["generative gripper design", "task-driven form"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["diff-contact-design", "diffusebot", "hardware-as-policy"], "arxiv": "2011.06498", "doi": "", "code": "", "site": "", "note": "Generate the gripper geometry from the task rather than picking from a catalogue. The natural ancestor of 'let a generative model propose the hand'.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The 3D shape of a robot's end-effector plays a critical role in determining it's functionality and overall performance. Many industrial applications rely on task-specific gripper designs to ensure the system's robustness and accuracy. However, the process of manual hardware design is both costly and time-consuming, and the quality of the resulting design is dependent on the engineer's experience and domain expertise, which can easily be out-dated or inaccurate. The goal of this work is to use machine learning algorithms to automate the design of task-specific gripper fingers. We propose Fit2Form, a 3D generative design framework that generates pairs of finger shapes to maximize design objectives (i.e., grasp success, stability, and robustness) for target grasp objects. We model the design objectives by training a Fitness network to predict their values for pairs of gripper fingers and their corresponding grasp objects. This Fitness network then provides supervision to a 3D Generative network that produces a pair of 3D finger geometries for the target grasp object. Our experiments demonstrate that the proposed 3D generative design framework generates parallel jaw gripper finger shapes that achieve more stable and robust grasps compared to other general-purpose and task-specific gripper design algorithms. Video can be found at https://youtu.be/utKHP3qb1bg.", "source": "curated", "first_author": "Huy Ha", "url": "https://arxiv.org/abs/2011.06498", "fig": "figures/fit2form.webp", "figs": ["figures/fit2form-1.webp", "figures/fit2form-2.webp"]}, {"id": "muzero", "title": "Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model", "tree": "rl", "branch": "model-based", "authors": ["Julian Schrittwieser", "Ioannis Antonoglou", "Thomas Hubert", "Karen Simonyan", "Laurent Sifre", "et al."], "corresponding": [], "institutions": ["Google DeepMind"], "lab": "DeepMind", "venue": "Nature", "venue_type": "journal", "year": 2020, "date": "2019-11-19", "topics": ["learned model planning", "MCTS"], "paradigm": ["model-based RL"], "method": [], "projects": [], "related": ["dqn", "dreamerv3", "tdmpc2"], "arxiv": "1911.08265", "doi": "", "code": "", "site": "", "note": "Value-equivalent models: learn only what planning needs, not the pixels. Conceptual ancestor of TD-MPC's decoder-free design.", "aka": "MuZero", "stars": 2, "curated": true, "local": "", "abstract": "Constructing agents with planning capabilities has long been one of the main challenges in the pursuit of artificial intelligence. Tree-based planning methods have enjoyed huge success in challenging domains, such as chess and Go, where a perfect simulator is available. However, in real-world problems the dynamics governing the environment are often complex and unknown. In this work we present the MuZero algorithm which, by combining a tree-based search with a learned model, achieves superhuman performance in a range of challenging and visually complex domains, without any knowledge of their underlying dynamics. MuZero learns a model that, when applied iteratively, predicts the quantities most directly relevant to planning: the reward, the action-selection policy, and the value function. When evaluated on 57 different Atari games - the canonical video game environment for testing AI techniques, in which model-based planning approaches have historically struggled - our new algorithm achieved a new state of the art. When evaluated on Go, chess and shogi, without any knowledge of the game rules, MuZero matched the superhuman performance of the AlphaZero algorithm that was supplied with the game rules.", "source": "curated", "first_author": "Julian Schrittwieser", "url": "https://arxiv.org/abs/1911.08265", "fig": "figures/muzero.webp", "figs": ["figures/muzero-1.webp"], "arxiv_url": "https://arxiv.org/abs/1911.08265"}, {"id": "dexpilot", "title": "DexPilot: Vision Based Teleoperation of Dexterous Robotic Hand-Arm System", "tree": "il", "branch": "retargeting", "authors": ["Ankur Handa", "Karl Van Wyk", "Wei Yang", "Jacky Liang", "Yu-Wei Chao", "Qian Wan", "Stan Birchfield", "Nathan Ratliff", "Dieter Fox"], "corresponding": [], "institutions": ["Nvidia (United States)"], "lab": "NVIDIA Seattle Robotics Lab", "venue": "ICRA", "venue_type": "conference", "year": 2019, "date": "2019-10-07", "topics": ["vision-based teleoperation", "hand retargeting"], "paradigm": ["teleop"], "method": [], "projects": ["geodex"], "related": ["anyteleop", "c2dex", "dex-retargeting", "retarget-objectives"], "arxiv": "1910.03135", "doi": "", "code": "", "site": "", "note": "Markerless hand teleop of a 23-DOF hand-arm. The retargeting cost function here (fingertip distances, not joint angles) is the ancestor of most later work.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Teleoperation offers the possibility of imparting robotic systems with sophisticated reasoning skills, intuition, and creativity to perform tasks. However, current teleoperation solutions for high degree-of-actuation (DoA), multi-fingered robots are generally cost-prohibitive, while low-cost offerings usually provide reduced degrees of control. Herein, a low-cost, vision based teleoperation system, DexPilot, was developed that allows for complete control over the full 23 DoA robotic system by merely observing the bare human hand. DexPilot enables operators to carry out a variety of complex manipulation tasks that go beyond simple pick-and-place operations. This allows for collection of high dimensional, multi-modality, state-action data that can be leveraged in the future to learn sensorimotor policies for challenging manipulation tasks. The system performance was measured through speed and reliability metrics across two human demonstrators on a variety of tasks. The videos of the experiments can be found at https://sites.google.com/view/dex-pilot.", "source": "curated", "first_author": "Ankur Handa", "url": "https://arxiv.org/abs/1910.03135", "arxiv_url": "https://arxiv.org/abs/1910.03135", "fig": "figures/dexpilot.webp", "figs": ["figures/dexpilot-1.webp", "figures/dexpilot-2.webp", "figures/dexpilot-3.webp"]}, {"id": "codesign-rl", "title": "Jointly Learning to Construct and Control Agents using Deep Reinforcement Learning", "tree": "systems", "branch": "codesign", "authors": ["Charles Schaff", "David Yunis", "Ayan Chakrabarti", "Matthew R. Walter"], "corresponding": ["Charles Schaff"], "institutions": ["Toyota Technological Institute at Chicago"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2019, "date": "2018-01-04", "topics": ["morphology-control co-optimization"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["derl", "robogrammar", "sims1994", "task2morph", "transform2act"], "arxiv": "1801.01432", "doi": "", "code": "", "site": "", "note": "The canonical formulation of design and control as one RL problem — maintain a distribution over designs and update it with the policy.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "The physical design of a robot and the policy that controls its motion are inherently coupled, and should be determined according to the task and environment. In an increasing number of applications, data-driven and learning-based approaches, such as deep reinforcement learning, have proven effective at designing control policies. For most tasks, the only way to evaluate a physical design with respect to such control policies is empirical--i.e., by picking a design and training a control policy for it. Since training these policies is time-consuming, it is computationally infeasible to train separate policies for all possible designs as a means to identify the best one. In this work, we address this limitation by introducing a method that performs simultaneous joint optimization of the physical design and control network. Our approach maintains a distribution over designs and uses reinforcement learning to optimize a control policy to maximize expected reward over the design distribution. We give the controller access to design parameters to allow it to tailor its policy to each design in the distribution. Throughout training, we shift the distribution towards higher-performing designs, eventually converging to a design and control policy that are jointly optimal. We evaluate our approach in the context of legged locomotion, and demonstrate that it discovers novel designs and walking gaits, outperforming baselines in both performance and efficiency.", "source": "curated", "first_author": "Charles Schaff", "url": "https://arxiv.org/abs/1801.01432", "fig": "figures/codesign-rl.webp", "figs": ["figures/codesign-rl-1.webp"]}, {"id": "neural-graph-evo", "title": "Neural Graph Evolution: Towards Efficient Automatic Robot Design", "tree": "systems", "branch": "codesign", "authors": ["Tingwu Wang", "Yuhao Zhou", "Sanja Fidler", "Jimmy Ba"], "corresponding": [], "institutions": ["University of Toronto", "NVIDIA"], "lab": "Vector Institute", "venue": "ICLR", "venue_type": "conference", "year": 2019, "date": "2019-06-12", "topics": ["graph neural networks", "morphology search"], "paradigm": ["co-design", "RL"], "method": [], "projects": [], "related": ["derl", "robogrammar", "sims1994", "transform2act"], "arxiv": "1906.05370", "doi": "", "code": "", "site": "", "note": "Represents the robot as a graph and mutates it, with a GNN policy that transfers across designs. University of Toronto — worth knowing locally.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Despite the recent successes in robotic locomotion control, the design of robot relies heavily on human engineering. Automatic robot design has been a long studied subject, but the recent progress has been slowed due to the large combinatorial search space and the difficulty in evaluating the found candidates. To address the two challenges, we formulate automatic robot design as a graph search problem and perform evolution search in graph space. We propose Neural Graph Evolution (NGE), which performs selection on current candidates and evolves new ones iteratively. Different from previous approaches, NGE uses graph neural networks to parameterize the control policies, which reduces evaluation cost on new candidates with the help of skill transfer from previously evaluated designs. In addition, NGE applies Graph Mutation with Uncertainty (GM-UC) by incorporating model uncertainty, which reduces the search space by balancing exploration and exploitation. We show that NGE significantly outperforms previous methods by an order of magnitude. As shown in experiments, NGE is the first algorithm that can automatically discover kinematically preferred robotic graph structures, such as a fish with two symmetrical flat side-fins and a tail, or a cheetah with athletic front and back legs. Instead of using thousands of cores for weeks, NGE efficiently solves searching problem within a day on a single 64 CPU-core Amazon EC2 machine.", "source": "curated", "first_author": "Tingwu Wang", "url": "https://arxiv.org/abs/1906.05370", "fig": "figures/neural-graph-evo.webp"}, {"id": "alvinn", "title": "ALVINN: An Autonomous Land Vehicle in a Neural Network", "tree": "il", "branch": "foundations", "authors": ["Dean A. Pomerleau"], "corresponding": ["Dean Pomerleau"], "institutions": ["Carnegie Mellon University"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2018, "date": "2018-06-30", "topics": ["behaviour cloning", "end-to-end driving"], "paradigm": ["BC"], "method": [], "projects": [], "related": ["dagger"], "arxiv": "", "doi": "10.1184/r1/6603146.v1", "code": "", "site": "", "note": "Behaviour cloning, 1989. Worth keeping in the tree as the reminder that the architecture changed and the failure mode (compounding error off-distribution) did not.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Dean A. Pomerleau", "url": "https://doi.org/10.1184/r1/6603146.v1"}, {"id": "td3", "title": "Addressing Function Approximation Error in Actor-Critic Methods", "tree": "rl", "branch": "foundations", "authors": ["Scott Fujimoto", "Herke van Hoof", "David Meger"], "corresponding": [], "institutions": ["McGill University", "University of Amsterdam"], "lab": "", "venue": "ICML", "venue_type": "conference", "year": 2018, "date": "2018-02-26", "topics": ["off-policy RL", "overestimation bias"], "paradigm": ["RL"], "method": ["twin critics", "delayed policy update", "target smoothing"], "projects": [], "related": ["sac"], "arxiv": "1802.09477", "doi": "", "code": "", "site": "", "note": "Where the twin-critic trick that SAC inherited comes from.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In value-based reinforcement learning methods such as deep Q-learning, function approximation errors are known to lead to overestimated value estimates and suboptimal policies. We show that this problem persists in an actor-critic setting and propose novel mechanisms to minimize its effects on both the actor and the critic. Our algorithm builds on Double Q-learning, by taking the minimum value between a pair of critics to limit overestimation. We draw the connection between target networks and overestimation bias, and suggest delaying policy updates to reduce per-update error and further improve performance. We evaluate our method on the suite of OpenAI gym tasks, outperforming the state of the art in every environment tested.", "source": "curated", "first_author": "Scott Fujimoto", "url": "https://arxiv.org/abs/1802.09477", "fig": "figures/td3.webp", "figs": ["figures/td3-1.webp", "figures/td3-2.webp"]}, {"id": "coros-codesign", "title": "Computational co-optimization of design parameters and motion trajectories for robotic systems", "tree": "systems", "branch": "codesign", "authors": ["Sehoon Ha", "Stelian Coros", "Alexander Alspach", "Joohyung Kim", "Katsu Yamane"], "corresponding": [], "institutions": ["Disney Research", "ETH Zürich"], "lab": "", "venue": "IJRR", "venue_type": "journal", "year": 2018, "date": "", "topics": ["trajectory-design co-optimization", "sensitivity analysis"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["diff-contact-design", "dinev-codesign", "disney-bdx", "opt-legged-control", "spine-phase"], "arxiv": "", "doi": "10.1177/0278364918771172", "code": "", "site": "", "note": "In your library's orbit via the Disney line. The implicit-function/sensitivity approach — differentiate the optimal trajectory with respect to the design parameters — is the model-based alternative to RL-in-the-loop, and it is far more sample-efficient when your model is good.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Sehoon Ha", "url": "https://doi.org/10.1177/0278364918771172"}, {"id": "qmix", "title": "QMIX: Monotonic Value Function Factorisation for Deep Multi-Agent Reinforcement Learning", "tree": "rl", "branch": "marl", "authors": ["Tabish Rashid", "Mikayel Samvelyan", "Christian Schroeder de Witt", "Gregory Farquhar", "Jakob Foerster", "Shimon Whiteson"], "corresponding": [], "institutions": ["Meta (Israel)", "University of Oxford"], "lab": "Whiteson Research Lab", "venue": "ICML", "venue_type": "conference", "year": 2018, "date": "2018-03-30", "topics": ["value factorization", "CTDE"], "paradigm": ["MARL"], "method": ["monotonic mixing network over per-agent Q-values"], "projects": ["m2"], "related": ["maddpg", "mappo"], "arxiv": "1803.11485", "doi": "", "code": "", "site": "", "note": "The value-decomposition line. Matters for M2 mainly as the alternative framing to MAPPO: credit assignment by factorization instead of by a centralized critic.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "In many real-world settings, a team of agents must coordinate their behaviour while acting in a decentralised way. At the same time, it is often possible to train the agents in a centralised fashion in a simulated or laboratory setting, where global state information is available and communication constraints are lifted. Learning joint action-values conditioned on extra state information is an attractive way to exploit centralised learning, but the best strategy for then extracting decentralised policies is unclear. Our solution is QMIX, a novel value-based method that can train decentralised policies in a centralised end-to-end fashion. QMIX employs a network that estimates joint action-values as a complex non-linear combination of per-agent values that condition only on local observations. We structurally enforce that the joint-action value is monotonic in the per-agent values, which allows tractable maximisation of the joint action-value in off-policy learning, and guarantees consistency between the centralised and decentralised policies. We evaluate QMIX on a challenging set of StarCraft II micromanagement tasks, and show that QMIX significantly outperforms existing value-based multi-agent reinforcement learning methods.", "source": "curated", "first_author": "Tabish Rashid", "url": "https://arxiv.org/abs/1803.11485", "arxiv_url": "https://arxiv.org/abs/1803.11485", "fig": "figures/qmix.webp"}, {"id": "maddpg", "title": "Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments", "tree": "rl", "branch": "marl", "authors": ["Ryan Lowe", "Yi Wu", "Aviv Tamar", "Jean Harb", "Pieter Abbeel", "Igor Mordatch"], "corresponding": [], "institutions": ["OpenAI", "UC Berkeley", "McGill University"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2017, "date": "2017-06-07", "topics": ["mixed cooperative-competitive", "centralized critic"], "paradigm": ["MARL"], "method": [], "projects": ["m2"], "related": ["mappo", "qmix"], "arxiv": "1706.02275", "doi": "10.48550/arxiv.1706.02275", "code": "", "site": "", "note": "The original centralized-critic/decentralized-actor template.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We explore deep reinforcement learning methods for multi-agent domains. We begin by analyzing the difficulty of traditional algorithms in the multi-agent case: Q-learning is challenged by an inherent non-stationarity of the environment, while policy gradient suffers from a variance that increases as the number of agents grows. We then present an adaptation of actor-critic methods that considers action policies of other agents and is able to successfully learn policies that require complex multi-agent coordination. Additionally, we introduce a training regimen utilizing an ensemble of policies for each agent that leads to more robust multi-agent policies. We show the strength of our approach compared to existing methods in cooperative as well as competitive scenarios, where agent populations are able to discover various physical and informational coordination strategies.", "source": "curated", "first_author": "Ryan Lowe", "url": "https://doi.org/10.57702/apbqmp7u", "arxiv_url": "https://arxiv.org/abs/1706.02275", "fig": "figures/maddpg.webp"}, {"id": "gail", "title": "Generative Adversarial Imitation Learning", "tree": "il", "branch": "foundations", "authors": ["Jonathan Ho", "Stefano Ermon"], "corresponding": [], "institutions": ["OpenAI (United States)", "Stanford University"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2016, "date": "2016-06-10", "topics": ["adversarial imitation", "occupancy matching"], "paradigm": ["adversarial IL"], "method": [], "projects": [], "related": ["add", "amp"], "arxiv": "1606.03476", "doi": "10.48550/arxiv.1606.03476", "code": "", "site": "", "note": "Imitation as distribution matching rather than action regression. AMP is this idea moved into physics-based character control.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "Consider learning a policy from example expert behavior, without interaction with the expert or access to reinforcement signal. One approach is to recover the expert's cost function with inverse reinforcement learning, then extract a policy from that cost function with reinforcement learning. This approach is indirect and can be slow. We propose a new general framework for directly extracting a policy from data, as if it were obtained by reinforcement learning following inverse reinforcement learning. We show that a certain instantiation of our framework draws an analogy between imitation learning and generative adversarial networks, from which we derive a model-free imitation learning algorithm that obtains significant performance gains over existing model-free methods in imitating complex behaviors in large, high-dimensional environments.", "source": "curated", "first_author": "Jonathan Ho", "url": "https://arxiv.org/abs/1606.03476", "arxiv_url": "https://arxiv.org/abs/1606.03476", "fig": "figures/gail.webp"}, {"id": "dqn", "title": "Human-level control through deep reinforcement learning", "tree": "rl", "branch": "foundations", "authors": ["Volodymyr Mnih", "Koray Kavukcuoglu", "David Silver", "Andrei A. Rusu", "Joel Veness", "et al."], "corresponding": ["Koray Kavukcuoglu", "Demis Hassabis"], "institutions": ["Google (United Kingdom)", "Google DeepMind (United Kingdom)"], "lab": "DeepMind", "venue": "Nature", "venue_type": "journal", "year": 2015, "date": "2015-02-26", "topics": ["value-based RL", "Atari"], "paradigm": ["RL"], "method": ["experience replay", "target network"], "projects": [], "related": ["muzero", "ppo"], "arxiv": "", "doi": "10.1038/nature14236", "code": "", "site": "", "note": "The paper that started the modern era. Historical anchor for the RL tree.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Volodymyr Mnih", "url": "https://doi.org/10.1038/nature14236"}, {"id": "trpo", "title": "Trust Region Policy Optimization", "tree": "rl", "branch": "foundations", "authors": ["John Schulman", "Sergey Levine", "Philipp Moritz", "Michael I. Jordan", "Pieter Abbeel"], "corresponding": [], "institutions": ["University of California, Berkeley"], "lab": "", "venue": "ICML", "venue_type": "conference", "year": 2015, "date": "2015-02-19", "topics": ["policy gradient", "trust region"], "paradigm": ["RL"], "method": ["KL-constrained natural gradient"], "projects": [], "related": ["ppo"], "arxiv": "1502.05477", "doi": "10.48550/arxiv.1502.05477", "code": "", "site": "", "note": "The monotonic-improvement argument PPO approximates away. Worth reading to know what PPO's clip is standing in for.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "We describe an iterative procedure for optimizing policies, with guaranteed monotonic improvement. By making several approximations to the theoretically-justified procedure, we develop a practical algorithm, called Trust Region Policy Optimization (TRPO). This algorithm is similar to natural policy gradient methods and is effective for optimizing large nonlinear policies such as neural networks. Our experiments demonstrate its robust performance on a wide variety of tasks: learning simulated robotic swimming, hopping, and walking gaits; and playing Atari games using images of the screen as input. Despite its approximations that deviate from the theory, TRPO tends to give monotonic improvement, with little tuning of hyperparameters.", "source": "curated", "first_author": "John Schulman", "url": "http://proceedings.mlr.press/v37/schulman15.pdf", "arxiv_url": "https://arxiv.org/abs/1502.05477", "fig": "figures/trpo.webp"}, {"id": "allegro", "title": "Allegro Hand", "tree": "systems", "branch": "hands", "authors": ["Wonik Robotics"], "corresponding": [], "institutions": ["Wonik Robotics"], "lab": "", "venue": "commercial", "venue_type": "industry", "year": 2012, "date": "", "topics": ["four-finger hand", "direct drive", "research platform"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["leap-hand", "qi-inhand", "shadow-hand"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "The 16-DOF workhorse of RL dexterity papers (Qi, Chen, DeXtreme). Robust, backdrivable enough, and cheap relative to Shadow.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Wonik Robotics", "url": "https://www.wonikrobotics.com/robot-hand"}, {"id": "dlr-hand", "title": "The DLR Hand Arm System", "tree": "systems", "branch": "hands", "authors": ["Markus Grebenstein", "Alin Albu-Schäffer", "Thomas Bahls", "Maxime Chalon", "Oliver Eiberger", "Werner Friedl", "et al."], "corresponding": ["Markus Grebenstein"], "institutions": ["Deutsches Zentrum für Luft- und Raumfahrt e. V. (DLR)"], "lab": "DLR Institute of Robotics and Mechatronics", "venue": "ICRA", "venue_type": "conference", "year": 2011, "date": "2011-05-01", "topics": ["variable stiffness", "robust hand-arm", "biomimetic"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["1x-hand", "leap-hand", "midas-hand", "orca-hand", "shadow-hand"], "arxiv": "", "doi": "10.1109/icra.2011.5980371", "code": "", "site": "", "note": "In your Robot Hands/ folder. The variable-stiffness, impact-tolerant design philosophy — a different axis from DOF count, and the one cheap hands most obviously lack.", "aka": "", "stars": 2, "curated": true, "local": "~/Desktop/papers/Robot Hands/DLR.pdf", "abstract": "", "source": "curated", "first_author": "Markus Grebenstein", "url": "https://doi.org/10.1109/icra.2011.5980371", "fig": "figures/dlr-hand.webp", "figs": ["figures/dlr-hand-1.webp", "figures/dlr-hand-2.webp", "figures/dlr-hand-3.webp"]}, {"id": "shadow-hand", "title": "The Shadow Dexterous Hand", "tree": "systems", "branch": "hands", "authors": ["Shadow Robot Company"], "corresponding": [], "institutions": ["Shadow Robot Company"], "lab": "", "venue": "commercial", "venue_type": "industry", "year": 2005, "date": "", "topics": ["anthropomorphic hand", "tendon driven", "24 DOF"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["allegro", "dactyl", "dlr-hand", "leap-hand"], "arxiv": "", "doi": "", "code": "", "site": "", "note": "The 20-actuator, 24-DOF reference hand that Dactyl and a decade of dexterity research ran on. Historically important, practically unaffordable — which is why everything below exists.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Shadow Robot Company", "url": "https://www.shadowrobot.com/dexterous-hand-series/"}, {"id": "lipson2000", "title": "Automatic design and manufacture of robotic lifeforms", "tree": "systems", "branch": "codesign", "authors": ["Hod Lipson", "Jordan B. Pollack"], "corresponding": [], "institutions": ["Brandeis University"], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2000, "date": "", "topics": ["evolutionary design", "automatic fabrication"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["robogrammar", "sims1994", "text2robot"], "arxiv": "", "doi": "10.1038/35023115", "code": "", "site": "", "note": "Evolved morphologies actually 3D-printed and run. The first closing of the design–fabricate–test loop without a human in it, which is exactly what an agentic co-design system has to reproduce.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Hod Lipson", "url": "https://doi.org/10.1038/35023115"}, {"id": "x-0alatentpredictiveworldactionmodelforcon", "title": "$ω$-0: A Latent Predictive World Action Model for Concurrent Humanoid Loco-Manipulation", "tree": "il", "branch": "world-models", "authors": ["Zhe Li", "Zhenzhe Zhang", "Yangyang Wei", "Wenjie Zhang", "Xichen Yuan", "Peiyuan Zhi", "Gen Li", "Xinying Guo", "Fengjie Gao", "Jianfei Yang", "Shanghang Zhang"], "first_author": "Zhe Li", "corresponding": [], "institutions": ["Peking University", "HKUST"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-06", "topics": ["world model", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": [], "arxiv": "2608.06375", "doi": "", "url": "https://arxiv.org/abs/2608.06375", "code": "", "site": "", "abstract": "Humanoid household tasks often require concurrent loco-manipulation, where the robot must move, adjust posture, maintain balance, and manipulate objects as a single coordinated behavior. Yet existing humanoid policies typically decompose locomotion and manipulation, while recent world-action models remain either arm-centric or video-centered. We present $ω$-0, a latent predictive whole-body world-action model for real-world humanoid concurrent loco-manipulation. Given a language instruction, current visual observation, and robot proprioceptive state, $ω$-0 directly predicts controller-compatible whole-body action latents for real-robot execution. Rather than reconstructing future videos, $ω$-0 learns compact future observation embeddings as a lightweight predictive objective, coupling latent visual foresight with diffusion-based whole-body action generation. The model supports egocentric RGB, exocentric RGB, and exocentric depth inputs, and leverages controller-based simulation replay to ground human/public visual-motion priors into robot-executable action latents. We further collect $ω$-HOME, a 40+ hour real-world household humanoid dataset with synchronized multi-view observations, whole-body SMPL motions, robot states, and action latents. Real-world experiments on 11 household tasks demonstrate that a single $ω$-0 model can produce smooth manipulate-while-moving behaviors and consistently outperform representative imitation learning, VLA, humanoid, and WAM baselines.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.06375v2_0_A_Latent_Predictive_World_Action_Model_for_Concurrent_Hum.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.06375", "fig": "figures/x-0alatentpredictiveworldactionmodelforcon.webp"}, {"id": "x-acompacttoploadingrobotforendovascularin", "title": "A Compact Top-Loading Robot for Endovascular Interventions: Design, Control and Evaluation", "tree": "il", "branch": "visuomotor", "authors": ["Jonas Fischer", "Lennart Karstensen", "Franziska Mathis-Ullrich"], "first_author": "Jonas Fischer", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-13", "topics": ["multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.11779", "doi": "", "url": "https://arxiv.org/abs/2607.11779", "code": "", "site": "", "abstract": "Robot-assisted endovascular intervention can potentially reduce radiation exposure, improve surgeon ergonomics, enable telesurgery, support active assistance and autonomy, and enhance procedural precision. However, existing systems often suffer from limited procedural coverage because constrained patient-side setups, restricted flexibility, and complex instrument exchange hinder clinical workflow integration. This work presents a compact robotic system for endovascular interventions that enables continuous translational and rotational manipulation of standard endovascular instruments. The system consists of two alternating carts with pneumatically actuated membrane grippers integrated into rotating gripper gears. Its top-loading design allows rapid exchange of instruments such as guidewires and catheters without changing the robotic setup. A leader-follower control strategy enables continuous motion despite the finite stroke of each cart. The system was evaluated in motion-tracking experiments with guidewires and catheters and in an in vitro vascular phantom. The motion-tracking experiments showed generally smooth translational and rotational motion profiles. Across all tested guidewire and catheter experiments, the mean relative tracking errors were 3.6% for translational motion and 4.1% for rotational motion. In the vascular phantom, robot-assisted navigation reached the target in most trials, demonstrating the feasibility of the proposed manipulation concept under in vitro conditions. The presented robotic system demonstrates technical feasibility for continuous manipulation of standard endovascular instruments in bench-top and in vitro experiments. The compact top-loading design may ease instrument exchange and clinical workflow integration. Future work will focus on improving gripping performance, actuation speed, force feedback, and evaluation in more clinically realistic settings.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2607.11779v1_A_Compact_Top_Loading_Robot_for_Endovascular_Interventions_D.pdf", "source": "library", "fig": "figures/x-acompacttoploadingrobotforendovascularin.webp"}, {"id": "x-aprogressawareleaderfollowermidairdockin", "title": "A Progress-Aware Leader-Follower Midair Docking System for Dual-Drone Aerial Manipulation", "tree": "systems", "branch": "sim-bench", "authors": ["Yifan Cai", "Jan Ming Kevin Tan", "Xiangqi Li", "Chenzhe Jin", "Narsimlu Kemsaram", "Valerio Modugno"], "first_author": "Yifan Cai", "corresponding": [], "institutions": ["The London College", "University College London"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-28", "topics": ["simulation & benchmarks", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.29410", "doi": "", "url": "https://arxiv.org/abs/2605.29410", "code": "", "site": "", "abstract": "Reliable midair docking between small unmanned aerial vehicles (UAVs) is essential for modular aerial cooperation and manipulation, but it requires precise relative-pose control and repeatable platform under tight thrust and payload constraints. We present a dual-drone docking platform where two quadrotors operate in a leader-follower formation and dock using a lightweight modular frame with passive magnetic latching. A progress-aware mission supervisor manages phase transitions: approach, alignment, capture, and settle. This platform integrates a complete hardware-software stack (ROS 2 with Crazyflie/PX4 interfaces) and synchronized logging for benchmark evaluation. We evaluate the platform in simulation and real-world experiments using quantitative metrics such as formation error, baseline and yaw consistency, docking success rate, time-to-dock, and failure-mode statistics. The platform enables statistically grounded comparison of docking supervision and synchronization strategies and provides a practical testbed for modular aerial cooperation and repeatable midair aerial manipulation.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2605.29410v1_A_Progress_Aware_Leader_Follower_Midair_Docking_System_for_D.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.29410", "fig": "figures/x-aprogressawareleaderfollowermidairdockin.webp"}, {"id": "x-asurveyonsensorbasedplanningandcontrolfo", "title": "A Survey on Sensor-based Planning and Control for Unmanned Underwater Vehicles", "tree": "systems", "branch": "planning-control", "authors": ["Shivam Vishwakarma", "Tejal Bedmutha", "Dharmendra Kumar Patel", "Vijay Bhaskar Semwal", "Leena Vachhani"], "first_author": "Shivam Vishwakarma", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-06", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2604.05003", "doi": "", "url": "https://arxiv.org/abs/2604.05003", "code": "", "site": "", "abstract": "This survey examines recent sensor-based planning and control methods for Unmanned Underwater Vehicles (UUVs). In complex, uncertain underwater environments, UUVs require advanced planning and control strategies for effective navigation. These vehicles face significant challenges including drifting and noisy sensor measurements, absence of Global Navigation Satellite System (GNSS) signals, and low-bandwidth, high-latency underwater acoustic communications. The focus is on reactive local planning layers that adapt to real-time sensor inputs such as SONAR and Inertial Measurement Units (IMU) to improve localization accuracy and autonomy in dynamic ocean conditions, enabling dynamic obstacle avoidance and on-the-fly re-planning. The survey categorizes the existing literature into decoupled and coupled architectures for sensor-based planning and control. The decoupled architecture sequentially addresses planning and control stages, whereas coupled architectures offer tighter feedback loops for more immediate responsiveness. A comparative analysis of coupled planning and control methods reveals that while PID controllers are simple, they lack predictive capability for complex maneuvers. Model Predictive Control (MPC) offers superior path optimization but can be computationally intensive, and invariant-set controllers provide strong safety guarantees at the potential cost of agility in confined environments. Key contributions include a taxonomy of architectures combining planning and control, a focus on adaptive local planning, and an analysis of controller roles in integrated planning frameworks for autonomous navigation of UUVs.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2604.05003v1_A_Survey_on_Sensor_based_Planning_and_Control_for_Unmanned_U.pdf", "source": "library", "fig": "figures/x-asurveyonsensorbasedplanningandcontrolfo.webp"}, {"id": "x-aceegohandrepurposingvideodiffusionmodel", "title": "ACE-Ego-Hand: Repurposing Video Diffusion Models for Occlusion-Robust Egocentric 3D Hand Motion Recovery", "tree": "il", "branch": "human-video", "authors": ["Yufei Liu", "Xixi Wang", "Hao Li", "Ganlong Zhao", "Kaitong Cai", "Chengkai Jin", "Chunxiao Liu", "Jianbo Liu", "Siyuan Huang", "Xingang Pan", "Hongsheng Li"], "first_author": "Yufei Liu", "corresponding": [], "institutions": ["TU Munich", "Shanghai Jiao Tong University", "CUHK", "HKU"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-20", "topics": ["human video", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.20308", "doi": "10.48550/arxiv.2608.20308", "url": "https://arxiv.org/abs/2608.20308", "code": "", "site": "", "abstract": "Egocentric video offers scalable manipulation data for embodied AI, yet recovering metric 3D hand trajectories remains challenging due to severe object occlusion and frequent out-of-sight gaps. Existing single-frame and windowed temporal regressors fail when a hand shortly leaves the frame, while recent video diffusion models (VDMs) rely on heavy, stochastic multi-step sampling as pixel-space renderers. We instead repurpose VDM into a deterministic geometry encoder. A single forward pass over the clean latent exposes scene content beyond current observations, including occluded and out-of-sight hands. We introduce ACE-Ego-Hand, an offline clip-level framework that extracts features via a Deterministic Clean-Latent Encoder and decodes them with a Bidirectional Spatiotemporal Decoder. ACE-Ego-Hand recovers continuous bimanual trajectories with metric placement and no external detector, while a Ray-Based Camera Solver supports a second configuration that requires no test-time camera intrinsics. Across five egocentric benchmarks, ACE-Ego-Hand sets a new state of the art, cutting MPJPE-p by 30% on occlusion-heavy ARCTIC and 40% on HOT3D. These gains reach 46%-61% once out-of-sight hands are included in the evaluation, offering a scalable path from everyday human video to robot manipulation data.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.20308v1_DreamHand_Repurposing_Video_Diffusion_Models_for_Occlusion_R.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.20308", "fig": "figures/x-aceegohandrepurposingvideodiffusionmodel.webp"}, {"id": "x-aclmadmmbaseddistributedmodelpredictivec", "title": "ACLM: ADMM-Based Distributed Model Predictive Control for Collaborative Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Ziyi Zhou", "Pengyuan Shu", "Ruize Cao", "Yuntian Zhao", "Ye Zhao"], "first_author": "Ziyi Zhou", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-07", "topics": ["collaborative transport", "distributed mpc", "loco-manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2603.07095", "doi": "10.48550/arxiv.2603.07095", "url": "https://arxiv.org/abs/2603.07095", "code": "", "site": "", "abstract": "Collaborative transportation of heavy payloads via loco-manipulation is a challenging yet essential capability for legged robots operating in complex, unstructured environments. Centralized planning methods, e.g., holistic trajectory optimization, capture dynamic coupling among robots and payloads but scale poorly with system size, limiting real-time applicability. In contrast, hierarchical and fully decentralized approaches often neglect force and dynamic interactions, leading to conservative behavior. This study proposes an Alternating Direction Method of Multipliers (ADMM)-based distributed model predictive control framework for collaborative loco-manipulation with a team of quadruped robots with manipulators. By exploiting the payload-induced coupling structure, the global optimal control problem is decomposed into parallel individual-robot-level subproblems with consensus constraints. The distributed planner operates in a receding-horizon fashion and achieves fast convergence, requiring only a few ADMM iterations per planning cycle. A wrench-aware whole-body controller executes the planned trajectories, tracking both motion and interaction wrenches. Extensive simulations with up to four robots demonstrate scalability, real-time performance, and robustness to model uncertainty.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2603.07095v1_ACLM_ADMM_Based_Distributed_Model_Predictive_Control_for_Col.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2603.07095", "fig": "figures/x-aclmadmmbaseddistributedmodelpredictivec.webp"}, {"id": "x-admmbasedsafetycriticaldistributednmpcfo", "title": "ADMM-Based Safety-Critical Distributed NMPC for Cooperative Transportation by Quadrupedal Robots", "tree": "rl", "branch": "marl", "authors": ["Ruturaj Sambhus", "Kapi Ketan Mehta", "Yicheng Zeng", "Kaveh Akbari Hamed"], "first_author": "Ruturaj Sambhus", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-18", "topics": ["multi-agent", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.17007", "doi": "", "url": "https://arxiv.org/abs/2607.17007", "code": "", "site": "", "abstract": "This paper presents a safety-critical distributed nonlinear model predictive control (DNMPC) framework for cooperative payload transportation by teams of quadrupedal robots. The proposed approach models the robotic team and the shared payload as a dynamically coupled networked system with rigid holonomic coupling constraints arising from cooperative transportation. To enable distributed real-time optimization, the centralized finite-horizon optimal control problem is decomposed into parallel local NMPC subproblems coordinated through the alternating direction method of multipliers (ADMM). The resulting distributed framework enforces consensus over both payload-state and interaction-wrench trajectories while explicitly incorporating acceleration-level holonomic coupling constraints within the distributed predictive control formulation. Safety-critical obstacle avoidance constraints for both the robotic agents and payload are enforced using higher-order control barrier functions (HOCBFs). The framework is validated through numerical simulations with teams of two, three, and four quadrupedal robots transporting shared payloads in cluttered environments. Real-time experiments on two- and three-robot teams demonstrate safe and robust transportation under payload uncertainty and external disturbances. Compared with centralized NMPC, the proposed framework achieves up to 23% reduction in average NLP solve time while maintaining comparable closed-loop performance. Ablation studies further demonstrate robustness to communication delays and show that explicit payload-state consensus and holonomic constraints substantially improve payload tracking and distributed coordination over existing wrench-only consensus formulations.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2607.17007v1_ADMM_Based_Safety_Critical_Distributed_NMPC_for_Cooperative.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.17007", "fig": "figures/x-admmbasedsafetycriticaldistributednmpcfo.webp"}, {"id": "x-amrposeanactiveledmarkerbasedrelativepos", "title": "AMR-Pose: An Active LED Marker-Based Relative Pose Estimation Framework With Probabilistic Switching PnP for Cooperative AUVs", "tree": "rl", "branch": "marl", "authors": ["Zeyu Sha", "Xiaorui Wang", "Mingyang Yang", "Feitian Zhang"], "first_author": "Zeyu Sha", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-13", "topics": ["multi-agent", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.12866", "doi": "10.48550/arxiv.2608.12866", "url": "https://arxiv.org/abs/2608.12866", "code": "", "site": "", "abstract": "Reliable relative pose estimation between autonomous underwater vehicles (AUVs) is critical for cooperative ocean exploration, sampling, and multi-robot coordination. However, achieving robust vision-based relative localization in underwater environments remains challenging due to severe optical degradation, including turbidity, illumination variations, reflections, and intermittent feature occlusions. This paper presents AMR-Pose, an active LED marker-based relative pose estimation framework for cooperative AUVs. A compact marker module consisting of one red central LED and three blue peripheral LEDs is developed and integrated onto the leader AUV to provide distinctive visual features under complex underwater conditions. Building upon the detected marker observations, a probabilistic switching Perspective-n-Point estimator (PSwPnP) is developed by combining Lie-group pose propagation on $SE(3)$, probabilistic marker association, and visibility-adaptive measurement fusion for robust six-degree-of-freedom relative pose estimation. The proposed framework dynamically adapts the estimation process according to marker visibility, maintaining geometric consistency and temporal stability during partial observations and visibility transitions. Extensive water-tank experiments with motion-capture ground truth validate that AMR-Pose achieves accurate, smooth, and robust relative pose estimation under challenging underwater conditions. Closed-loop leader-follower experiments further demonstrate its feasibility for real-time relative pose feedback in cooperative underwater robotics.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2608.12866v1_AMR_Pose_An_Active_LED_Marker_Based_Relative_Pose_Estimation.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.12866", "fig": "figures/x-amrposeanactiveledmarkerbasedrelativepos.webp"}, {"id": "x-agenticfocusobjectpreservingmixedreality", "title": "AgenticFocus: Object-Preserving Mixed Reality Synthesis from Human FPV Video for Dexterous Humanoid Learning", "tree": "il", "branch": "human-video", "authors": ["Iaroslav Kolomiets", "Miguel Altamirano Cabrera", "Artem Lykov", "Jeffrin Sam", "Dmitrii Iarchuk", "Yara Mahmoud", "Daniia Zinniatullina", "Mikhail Konenkov", "Dzmitry Tsetserukou"], "first_author": "Iaroslav Kolomiets", "corresponding": [], "institutions": ["Presidential Executive Office", "Skolkovo Institute of Science and Technology"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-07-09", "topics": ["human video", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2607.08857", "doi": "", "url": "https://arxiv.org/abs/2607.08857", "code": "", "site": "", "abstract": "Human egocentric video is a scalable supervision source for humanoid policy learning, but current pipelines struggle with hand-object occlusion, oversimplified motion, or specialized capture hardware. We introduce AgenticFocus, a Mixed Reality synthesis pipeline that converts ordinary first-person-view human videos into robot-trainable demonstrations by restoring occluded object geometry, reconstructing full-hand motion, and retargeting it to a humanoid embodiment through camera-relative alignment and layered compositing. The resulting dataset pairs focused visual observations with synchronized robot actions and states. AgenticFocus achieves lower trajectory error and smoother wrist motion than cross-embodiment baselines, with SPARC scores of -5.18 versus -5.56 and -6.05.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2607.08857v2_AgenticFocus_Object_Preserving_Mixed_Reality_Synthesis_from.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.08857", "fig": "figures/x-agenticfocusobjectpreservingmixedreality.webp"}, {"id": "x-anoranassistedmarlapproachfordynamicside", "title": "An O-RAN-Assisted MARL Approach for Dynamic Sidelink and Infrastructure Selection in V2X Communications", "tree": "rl", "branch": "marl", "authors": ["Maria Barbosa", "Kelvin Dias"], "first_author": "Maria Barbosa", "corresponding": [], "institutions": ["Universidade Federal de Pernambuco"], "lab": "", "venue": "IEEE Transactions on Vehicular Technology", "venue_type": "preprint", "year": 2026, "date": "2026-08-17", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.17210", "doi": "10.1109/tvt.2026.3721784", "url": "https://doi.org/10.1109/tvt.2026.3721784", "code": "", "site": "", "abstract": "Future applications in the 6G-based Internet of Vehicles will leverage sidelink (SL) transmissions in Vehicle-to-Everything (V2X) scenarios. However, SL-based direct communication can significantly increase interference among vehicles and between vehicles and other entities of the Intelligent Transportation System. Thus, both Vehicle-to-Vehicle communications and Vulnerable Road Users (VRUs) uplink resources may be degraded or subject to starvation. Existing solutions primarily focus on improving resource allocation and pair selection. Nonetheless, they lack a comprehensive approach to tackle the communication modes and the entire network. To address these challenges, this paper leverages Open RAN to manage V2X communication and proposes a multi-agent reinforcement learning (MARL) resource-aware system. Open RAN provides control loops through a global view of the network and also an open interface-based framework for machine learning models applied to resource decision-making. Meanwhile, the MARL model aims to mitigate interference, optimize resource usage, and enhance quality of service by optimally selecting between sidelink and network transmissions. To reduce system complexity, this work employs a clustering strategy. Each agent manages a group of pairs, rather than assigning one agent to each pair. The solution supports this design by adopting a centralized training with decentralized execution approach, empowered by Open RAN. The strategy uses offline training and an off-policy approach, in which each agent stores experience for fine-tuning. Results indicate that the MARL approach reduces average loss by 21% and latency by 19% in Vehicle-only scenarios. In coexistence VRU scenarios, loss and latency drop by 18% and 30%, respectively, compared to the single-agent approach.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.17210v1_An_O_RAN_Assisted_MARL_Approach_for_Dynamic_Sidelink_and_Inf.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.17210", "fig": "figures/x-anoranassistedmarlapproachfordynamicside.webp"}, {"id": "x-bowconnectparallelbayesianoptimizationov", "title": "BOWConnect: Parallel Bayesian Optimization over Windows with Learned Local Cost Maps for Sample-Efficient Kinodynamic Motion Planning", "tree": "systems", "branch": "planning-control", "authors": ["Sourav Raxit", "Abdullah Al Redwan Newaz", "Jose Fuentes", "Leonardo Bobadilla"], "first_author": "Sourav Raxit", "corresponding": [], "institutions": ["Florida International University", "LSU New Orleans", "Louisiana State University Health Sciences Center New Orleans"], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2026, "date": "2026-06-25", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.27292", "doi": "", "url": "https://arxiv.org/abs/2606.27292", "code": "", "site": "", "abstract": "This paper presents BOWConnect, a bidirectional parallel kinodynamic motion planner that addresses three fundamental limitations of existing sampling-based methods: sample inefficiency in high-dimensional state spaces, unreliable cost heuristics under dynamic constraints, and poor performance in narrow passage environments. Unlike classical planners that rely on random control sampling and geometric distance heuristics, BOWConnect integrates Bayesian Optimization over Windows (BOW) as a learning-based steering function within a parallel tree-based exploration framework, enabling each worker to learn local cost maps and constraints to guide sampling toward dynamically feasible and collision-free controls. A bidirectional architecture simultaneously grows forward and backward trees from the start and goal regions in parallel threads, with a spatial hashing mechanism enabling fast connection queries and a boundary value problem solver generating kinodynamically consistent bridge trajectories. Extensive evaluations across ten benchmark environments demonstrate that BOWConnect achieves 100\\% success while delivering the fastest or near-fastest planning time in complex scenarios, including narrow passages and non-convex spaces where state-of-the-art planners fail or degrade substantially. Real-world deployment on a ground vehicle and a quadrotor confirms real-time planning with no collisions. Videos of real-world and simulated experiments, high-resolution versions of the figures, and the open-source code are available at https://bow-connect.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2606.27292v1_BOWConnect_Parallel_Bayesian_Optimization_over_Windows_with.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.27292", "fig": "figures/x-bowconnectparallelbayesianoptimizationov.webp"}, {"id": "x-beyondimplicitforceevaluatingexplicitfor", "title": "Beyond Implicit Force: Evaluating Explicit Force-Torque Proxies in Action Chunking with Transformers", "tree": "rl", "branch": "manipulation", "authors": ["King Hang Wong", "Lingqiao Liu", "Feras Dayoub"], "first_author": "King Hang Wong", "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2026, "date": "2026-07-16", "topics": ["dexterous manipulation", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.14578", "doi": "", "url": "https://arxiv.org/abs/2607.14578", "code": "", "site": "", "abstract": "Contact-rich manipulation requires policies to infer interaction state from signals that are often weakly observable through vision and kinematics alone. Action Chunking with Transformers (ACT) has shown strong performance in fine-grained manipulation, but many deployments collect demonstrations through leader-follower teleoperation, where tracking error between commanded leader motion and executed follower motion implicitly encodes contact, resistance, and constraint violation. This paper examines whether ACT's apparent force-awareness depends on this hidden interaction cue. We introduce an observation-centric ACT variant that predicts future follower joint states instead of leader commands, thereby removing the teleoperation-induced discrepancy signal while preserving the rest of the learning pipeline. We then evaluate whether simple joint-torque proxies, derived from onboard motor current or joint effort, can recover contact-aware behavior without external force/torque sensors. Across four real-world tasks spanning surface following, insertion, stiffness discrimination, and force-based stopping, removing the implicit cue leads to severe failures in force-critical phases. In contrast, torque-augmented policies recover robust contact behavior and improve the base ACT policy. These results demonstrate that, on real hardware, the implicit teleoperation cue is a recoverable source of force-awareness, where torque signals are available, a simple proxy matches, surpasses, or further enhances it.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2607.14578v1_Beyond_Implicit_Force_Evaluating_Explicit_Force_Torque_Proxi.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.14578", "fig": "figures/x-beyondimplicitforceevaluatingexplicitfor.webp"}, {"id": "x-colregscompliantcollisionavoidanceandgro", "title": "COLREGs Compliant Collision Avoidance and Grounding Prevention for Autonomous Marine Navigation", "tree": "systems", "branch": "planning-control", "authors": ["Mayur S. Patil", "Nataraj Sudharsan", "Veneela Ammula", "Jude Tomdio", "Jin Wang", "Michael Kei", "Sivakumar Rathinam", "Prabhakar R. Pagilla"], "first_author": "Mayur S. Patil", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-03", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2603.02484", "doi": "", "url": "https://arxiv.org/abs/2603.02484", "code": "", "site": "", "abstract": "Maritime Autonomous Surface Ships (MASS) are increasingly regarded as a promising solution to address crew shortages, improve navigational safety, and improve operational efficiency in the maritime industry. Nevertheless, the reliable deployment of MASS in real-world environments remains a significant challenge, particularly in congested waters where the majority of maritime accidents occur. This emphasizes the need for safe and regulation-aware motion planning strategies for MASS that are capable of operating under dynamic maritime conditions. This paper presents a unified motion planning method for MASS that achieves real time collision avoidance, compliance with International Regulations for Preventing Collisions at Sea (COLREGs), and grounding prevention. The proposed work introduces a convex optimization method that integrates velocity obstacle-based (VO) collision constraints, COLREGs-based directional constraints, and bathymetry-based grounding constraints to generate computationally efficient, rule-compliant optimal velocity selection. To enhance robustness, the classical VO method is extended to consider uncertainty in the position and velocity estimates of the target vessel. Unnavigable shallow water regions obtained from bathymetric data, which are inherently nonconvex, are approximated via convex geometries using a integer linear programming (ILP), allowing grounding constraints to be incorporated into the motion planning. The resulting optimization generates optimal and dynamically feasible input velocities that meet collision avoidance, regulatory compliance, kinodynamic limits, and grounding prevention requirements. Simulation results involving multi-vessel encounters demonstrate the effectiveness of the proposed method in producing safe and regulation-compliant maneuvers, highlighting the suitability of the proposed approach for real time autonomous maritime navigation.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2603.02484v1_COLREGs_Compliant_Collision_Avoidance_and_Grounding_Preventi.pdf", "source": "library", "fig": "figures/x-colregscompliantcollisionavoidanceandgro.webp"}, {"id": "x-cudampcagpunativesolverformodelpredictiv", "title": "CUDA MPC: A GPU-Native Solver for Model Predictive Control", "tree": "rl", "branch": "marl", "authors": ["Babak Akbari", "Melissa Greeff"], "first_author": "Babak Akbari", "corresponding": [], "institutions": ["NVIDIA"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-04", "topics": ["multi-agent", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.03051", "doi": "", "url": "https://arxiv.org/abs/2608.03051", "code": "", "site": "", "abstract": "Model Predictive Control (MPC) delivers constraint-aware control, but its reliance on online optimization limits its use on systems with fast dynamics, high-dimensional models, or long horizons. Existing GPU implementations typically treat the device as a linear-algebra accelerator, leaving the optimization loop dependent on repeated kernel launches and high-latency memory transfers. This paper introduces CUDA MPC, a GPU-native MPC framework that co-designs the optimization algorithm, execution model, and memory architecture for CUDA hardware. CUDA MPC pairs a parallel-in-horizon alternating direction method of multipliers (ADMM) splitting with a fused CUDA kernel that runs the entire iterative solve on the device. Intermediate optimization variables stay in low-latency, on-chip shared memory, and a localized atomic-flag protocol synchronizes only adjacent horizon blocks, minimizing host intervention, kernel-dispatch overhead, and global-memory traffic. Across six nonlinear robotics benchmarks spanning increasing state dimension and constraint density, CUDA MPC sustains real-time rates at horizons one to two orders of magnitude longer than CPU solvers: it solves an optimization-based collision-avoidance parking problem with 100 s of lookahead within a 0.1 s sampling interval, and is the only solver evaluated that achieves both real-time execution and collision-free coordination for a centralized 10-agent swarm, where acados and CasADi return no feasible solution and require 3.5 s and 4.5 s per solve. Against tensor-framework implementations of the same ADMM splitting, the fused kernel is up to $965\\times$ faster.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2608.03051v1_CUDA_MPC_A_GPU_Native_Solver_for_Model_Predictive_Control.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.03051", "fig": "figures/x-cudampcagpunativesolverformodelpredictiv.webp"}, {"id": "x-communicationreductionviasemanticbaseden", "title": "Communication Reduction via Semantic-Based Encoding in DMPC Using LSTMs", "tree": "il", "branch": "visuomotor", "authors": ["Torben Schiz", "Pedro H. J. Nardelli", "Henrik Ebel"], "first_author": "Torben Schiz", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-18", "topics": ["distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.17592", "doi": "10.48550/arxiv.2608.17592", "url": "https://arxiv.org/abs/2608.17592", "code": "", "site": "", "abstract": "The communication demands of distributed model prediction control (DMPC) can overwhelm even advanced wireless communication technologies as agents must exchange a significant amount of information at least once per time step. To semantically reduce communication demands, this work employs encoder-decoder networks built around long-short term memory (LSTM) cells in a distributed optimization algorithm. Agents publish a reduced representation of a message and receivers reconstruct the original message upon reception. In tests with reduced communication using formations of mobile robots, trained networks retain satisfactory performance and work reliably under conditions overwhelming full communication. As the results show, the usage of LSTMs either allows unprecedented reconstruction accuracy or the usage of different prediction-horizon lengths without the necessity to retrain.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2608.17592v1_Communication_Reduction_via_Semantic_Based_Encoding_in_DMPC.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.17592", "fig": "figures/x-communicationreductionviasemanticbaseden.webp"}, {"id": "x-compactlatentcoordinationforautonomousve", "title": "Compact Latent Coordination for Autonomous Vehicles at Unsignalized Intersections", "tree": "rl", "branch": "marl", "authors": ["Gil Lifshits", "Igal Bilik", "Gilad Katz"], "first_author": "Gil Lifshits", "corresponding": [], "institutions": ["Ben-Gurion University of the Negev"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-23", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.21488", "doi": "", "url": "https://arxiv.org/abs/2607.21488", "code": "", "site": "", "abstract": "Coordinating autonomous vehicles at unsignalized intersections remains a critical challenge for multi-agent reinforcement learning (MARL) systems, which typically struggle with combinatorial action spaces, reliance on privileged information, or rigid agent designs. We propose Master-Agent Proto-plan System (MAPS), a hierarchical deep reinforcement learning (DRL) architecture in which a centralized Master agent generates a compact, continuous embedding, denoted as proto-plan, that encodes a global coordination strategy. Decentralized Worker agents integrate this embedding with local observations to execute vehicle-specific control, decoupling strategic intent from tactical execution and enabling independent optimization of each module. As a proof-of-concept evaluation of this coordination mechanism, we test MAPS across 72 intersection configurations in HighwayEnv. MAPS achieves collision-free navigation while significantly reducing average travel time, outperforming state-of-the-art baselines. The learned proto-plans further exhibit robust generalization: a system trained with three agents achieves a 94% success rate when deployed zero-shot to five-agent scenarios, confirming that proto-plan-based hierarchical learning provides a promising framework for multi-vehicle coordination.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2607.21488v1_Compact_Latent_Coordination_for_Autonomous_Vehicles_at_Unsig.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.21488", "fig": "figures/x-compactlatentcoordinationforautonomousve.webp"}, {"id": "x-constrainedmpcbasedmotionplanningformorp", "title": "Constrained MPC-Based Motion Planning for Morphing Quadrotors in Ultra-Narrow Passages under Limited Perception", "tree": "systems", "branch": "planning-control", "authors": ["Harsh Modi", "Xiao Liang", "Minghui Zheng"], "first_author": "Harsh Modi", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-15", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.15999", "doi": "10.48550/arxiv.2605.15999", "url": "https://arxiv.org/abs/2605.15999", "code": "", "site": "", "abstract": "This paper introduces a motion planning framework to plan morphology and trajectory for morphing quadrotors under extremely constrained environments. We develop a novel obstacle avoidance cost function for nonlinear model predictive control (MPC) that enables navigation through extremely narrow gaps under limited perception from a 2D LiDAR. Classical artificial potential field-based costs typically have a high cost in narrow passages, artificially blocking the navigable path. In contrast, we propose a smooth exponential obstacle cost that preserves low traversal cost within narrow gaps while maintaining strong collision avoidance behavior. The formulation avoids hard activation thresholds and introduces a cost reduction factor to reduce the cost within narrow passages. Direct use of 2D LiDAR measurements in MPC allows navigation around arbitrarily shaped obstacles. The method is embedded within an acados-based nonlinear MPC framework. Simulation and experimental results demonstrate successful traversal of narrow corridors where typical repulsive cost functions would fail. The approach provides a computationally efficient and practical solution for navigating through tight spaces while maintaining safety from the obstacles. While we are implementing the framework on the morphing quadrotors, the cost function formulation is general-purpose for any mobile robot application, and is not limited to the morphing quadrotors. The implementation code is available at \\href{https://github.com/harshjmodi1996/morphocopter_mpc}{Github Repo} and a short video is available at \\href{https://zh.engr.tamu.edu/wp-content/uploads/sites/310/2026/03/MPC_MorphoCopter_video.mp4}{Video Link}.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2605.15999v1_Constrained_MPC_Based_Motion_Planning_for_Morphing_Quadrotor.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.15999", "fig": "figures/x-constrainedmpcbasedmotionplanningformorp.webp"}, {"id": "x-continualquadrupedrobotscoordinationvias", "title": "Continual Quadruped Robots Coordination via Semantic Skill Discovery", "tree": "rl", "branch": "marl", "authors": ["Daoqing Wang", "Yuchen Xiao", "Weixuan Huang", "Zhilong Zhang", "Shenghua Wan", "Meng Li (79487)", "Lei Yuan", "Yang Yu"], "first_author": "Daoqing Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-06", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.08102", "doi": "", "url": "https://arxiv.org/abs/2606.08102", "code": "", "site": "", "abstract": "Multi-quadruped coordination has attracted increasing attention due to its enhanced payload capacity, broader contact coverage, and improved adaptability to challenging tasks. Existing methods for multi-quadruped manipulation typically focus on predefined or closed task families, often relying on multi-agent reinforcement learning (MARL) to train task-specific coordination policies. However, such methods struggle in open-ended continual learning settings, where tasks arrive sequentially and robots are expected to acquire new coordination skills while reusing previously learned ones without catastrophic forgetting. To address this challenge, we propose Conquer, a semantic skill-library framework that formulates continual multi-quadruped coordination as a retrieve-adapt-update process. First, to accommodate varying team sizes across tasks, we design a team-structured Self-Allies-Goal (SAG) backbone that supports variable-cardinality robot teams by explicitly modeling each robot's own state, teammate context, and task goal. For each incoming task, Conquer constructs a task-level semantic descriptor from pre-execution information and retrieves a relevant skill from the library for adaptation. After successful execution, Conquer updates the skill library by extracting trajectory-level semantic descriptors and organizing them according to semantic distance, thereby enabling continual skill accumulation and cross-task knowledge transfer. Simulation experiments show that Conquer achieves a final average success rate of 95.6%, demonstrating strong forward transfer and negligible catastrophic forgetting. Real-world rollouts on Unitree Go2 teams further validate the deployment feasibility of Conquer for practical multi-quadruped coordination. Simulation and real-robot demonstration videos are available at: https://conquer-project.pages.dev/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2606.08102v2_Continual_Quadruped_Robots_Coordination_via_Semantic_Skill_D.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.08102", "fig": "figures/x-continualquadrupedrobotscoordinationvias.webp"}, {"id": "x-continuoustimegaussianbelieftreesformoti", "title": "Continuous-Time Gaussian Belief Trees for Motion Planning", "tree": "systems", "branch": "planning-control", "authors": ["Rayan Mazouz", "Qi Heng Ho", "Zachary N. Sunberg", "Morteza Lahijanian"], "first_author": "Rayan Mazouz", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-03", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.02884", "doi": "", "url": "https://arxiv.org/abs/2607.02884", "code": "", "site": "", "abstract": "We address sampling-based motion planning for continuous-time stochastic systems under process and measurement uncertainty, with probabilistic guarantees on safety and performance. The robot dynamics are modeled as a continuous-time linear stochastic differential equation, while sensor measurements arrive at discrete time instants. We derive an offline hybrid belief propagation model in which the belief evolves according to continuous-time ODEs between measurements and undergoes discrete Kalman filter update jumps at measurement times. To ensure safety, we introduce a belief-barrier-function-based safety checker for segment-level probabilistic verification. This enables the planner to certify safety over entire continuous trajectory segments and detect inter-sample chance-constraint violations that are missed by conventional node-based checks. Together, these components provide a principled framework for sampling-based belief planning that accounts for both continuous-time uncertainty propagation and continuous-time safety requirements. We integrate the method with RRT and SST planners and evaluate it across multiple benchmark environments. The results show that the proposed method achieves high success rates and robust enforcement of chance constraints, including in narrow-passage scenarios where discrete-time counterparts fail due to missed inter-sample unsafe behavior.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2607.02884v1_Continuous_Time_Gaussian_Belief_Trees_for_Motion_Planning.pdf", "source": "library", "fig": "figures/x-continuoustimegaussianbelieftreesformoti.webp"}, {"id": "x-cooperativelongropeskippingviamultiagent", "title": "Cooperative Long Rope Skipping via Multi-Agent Reinforcement Learning", "tree": "rl", "branch": "locomotion", "authors": ["Zihao Wang", "Shijie Peng", "Kerui Wu", "Yu Huang", "Ruiqi Xue", "Dong Liu", "Tian Xu", "Lei Yuan", "Yang Yu"], "first_author": "Zihao Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2026, "date": "2026-06-06", "topics": ["locomotion", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.08064", "doi": "", "url": "https://arxiv.org/abs/2606.08064", "code": "", "site": "", "abstract": "Humans exhibit remarkable motor agility, enabling a wide range of dynamic skills such as running and jumping, which highlights the great potential of humanoid robots for athletic locomotion. Among athletic sports, long rope skipping requires two rope turners to cooperatively swing the rope while adapting to a player under different jumping rhythms, making it a meaningful yet challenging task for humanoid robots. Although existing methods for humanoid sports have achieved success in single-agent and interaction-free settings, such as running, dancing, and parkour, task scenarios that require precise coordination among multiple participants remain largely unexplored. To this end, we propose Marope, a multi-agent reinforcement learning (MARL) framework for cooperative long rope skipping with multiple humanoid robots. Specifically, Marope adopts a hierarchical reinforcement learning framework for policy training. At the lower level, it learns decentralized rope manipulation policies through MARL, while at the upper level, a centralized scheduling policy is trained to coordinate the execution of the lower-level policies. To improve generalization across different player behavioral styles, Marope further incorporates diverse jumping policies into cooperative game training. We evaluate our approach on Unitree G1 humanoid robots in both simulation and real-world settings. Experimental results demonstrate that Marope outperforms various baselines, achieving more efficient and stable rope manipulation as well as more robust and adaptable cooperation with varied players.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2606.08064v1_Cooperative_Long_Rope_Skipping_via_Multi_Agent_Reinforcement.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.08064", "fig": "figures/x-cooperativelongropeskippingviamultiagent.webp"}, {"id": "x-cooptimizingsafetyandperformanceusingsaf", "title": "Cooptimizing Safety and Performance Using Safety Value-Constrained Model Predictive Control", "tree": "systems", "branch": "planning-control", "authors": ["Hao Wang", "Nam Nguyen", "Armand Jordana", "Ludovic Righetti", "Somil Bansal"], "first_author": "Hao Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-26", "topics": ["planning & control", "whole body control"], "paradigm": [], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2604.23863", "doi": "10.48550/arxiv.2604.23863", "url": "https://arxiv.org/abs/2604.23863", "code": "", "site": "", "abstract": "Autonomous systems are increasingly deployed in real-world environments, where they must achieve high performance while maintaining safety under state and input constraints. Although Model Predictive Control (MPC) provides a principled framework for constrained optimal control, guaranteeing safety beyond its finite planning horizon remains a fundamental challenge. In this work, we augment MPC with a safety value function-based terminal constraint that enforces membership in a control-invariant safe set at the end of each planning horizon. This formulation enables real-time synthesis of trajectories that are both high-performing and provably safe. We show that, under an exact safety value function and a feasible initialization, the proposed MPC scheme is recursively feasible, thereby ensuring persistent safety. In contrast to traditional terminal set constructions that rely on local linearizations or conservative approximations, our approach incorporates a reachability-based safety value function for terminal constraints, yielding less conservative and more expressive safety guarantees. We validate the proposed framework through simulation and hardware experiments on a Flexiv Rizon 10s manipulator. Results demonstrate improved constraint satisfaction and robustness compared to standard state-constrained MPC and reactive safety filtering, while maintaining competitive task performance. The full implementation and experiments are available on the project website.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/MABEL/literatures/whole_body_control/2604.23863v1.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2604.23863", "fig": "figures/x-cooptimizingsafetyandperformanceusingsaf.webp"}, {"id": "x-coveragepathplanningclassicalfoundations", "title": "Coverage Path Planning: Classical Foundations, Recent Advances, and Future Directions", "tree": "rl", "branch": "marl", "authors": ["Zongyuan Shen", "Shalabh Gupta", "Shancheng Zhao", "Dehua Zhou", "Gao Wang", "Zhongqiang Ren", "Yaming Ou", "Yikui Zhai", "C. L. Philip Chen"], "first_author": "Zongyuan Shen", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-12", "topics": ["multi-agent", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.10649", "doi": "", "url": "https://arxiv.org/abs/2607.10649", "code": "", "site": "", "abstract": "Coverage path planning (CPP) is a fundamental problem in robot motion planning, whose aim is to produce robot trajectories that provide complete coverage of target workspaces while minimizing task-specific objectives such as path length, overlap, number of turns, and energy consumption. CPP has widespread applications in cleaning, inspection, mapping, agriculture, manufacturing, surveillance, demining, and environmental monitoring. Although classical CPP has been extensively studied, recent advances have extended CPP beyond single-robot settings to multi-robot systems, complex 3D environments, constrained platforms, learning-based coverage planning, and visual coverage tasks. This paper presents a comprehensive survey of 125 representative works published primarily between 2015 and 2026, while presenting the evolution of recent developments in light of the classical CPP methods published before 2015. The CPP methods are organized into six main categories: single-robot CPP, multi-robot CPP, 3D CPP, constrained CPP, learning-based CPP, and visual CPP. For each category, the review summarizes the main planning formulations, representative algorithms, strengths, and limitations. In addition, the review analyzes how environmental knowledge, workspace geometry, robot constraints, sensing objectives, and coordination requirements shape the CPP problem. The survey further discusses open challenges in scalable online planning, multi-robot coordination, 3D and visual coverage, unified platform-constrained and resource-aware coverage, and learning-enhanced coverage. Thus, the survey provides a structured overview of recent CPP developments and future research directions.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2607.10649v1_Coverage_Path_Planning_Classical_Foundations_Recent_Advances.pdf", "source": "library", "fig": "figures/x-coveragepathplanningclassicalfoundations.webp"}, {"id": "x-credibilityawarelearningandcontrolforsaf", "title": "Credibility-Aware Learning and Control for Safe USV Navigation under Perception Uncertainty", "tree": "systems", "branch": "planning-control", "authors": ["Yuhang Zhang", "Shuqi Chai", "Yukang Zhang", "Liusha Yang", "Mingchuan Zhang", "Wei Wang", "Qingjiang Shi", "Quanbo Ge"], "first_author": "Yuhang Zhang", "corresponding": [], "institutions": ["Henan University of Science and Technology", "Shenzhen Research Institute of Big Data", "Shenzhen Technology University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-26", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.26974", "doi": "", "url": "https://arxiv.org/abs/2605.26974", "code": "", "site": "", "abstract": "Safe navigation for Unmanned Surface Vehicles (USVs) under the International Regulations for Preventing Collisions at Sea (COLREGs) remains challenging in dynamic maritime environments, especially when perception uncertainty is miscalibrated. Errors in state estimation can produce unreliable belief states that mislead value learning, while logic based on discrete traffic rules can cause abrupt action corrections. To address these challenges, we integrate Credibility-Weighted Value Learning (CWVL) with Covariance- and Recovery-Aware Control Barrier Function Quadratic Programming (CoReCBF-QP). CWVL derives a dynamic trust factor from the discrepancy between the covariance estimated by the filter and empirical error statistics. This factor modulates the critic's heteroscedastic loss and limits overfitting to miscalibrated observations. CoReCBF expands the collision geometry according to uncertainty and incorporates terms for braking and turning recovery. The resulting hyperbolic safety boundary preserves feasible avoidance velocities and supplies the QP safety constraint. A continuous COLREGs-aware reference in the objective promotes starboard maneuvers in Rule 14 head-on and Rule 15 give-way crossing encounters. Simulations show improved robustness in collision avoidance and COLREGs event compliance, achieving an 82.0\\% success rate with ten target ships beyond the training range.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2605.26974v3_Trust_Geometry_and_Rules_A_Credibility_Aware_Reinforcement_L.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.26974", "fig": "figures/x-credibilityawarelearningandcontrolforsaf.webp"}, {"id": "x-decentralizedllmdrivencoordinationofacou", "title": "Decentralized LLM-Driven Coordination of Acoustic Robots for Contactless Object Manipulation", "tree": "rl", "branch": "marl", "authors": ["Yingying Wang", "Narsimlu Kemsaram", "Sriram Subramanian"], "first_author": "Yingying Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-28", "topics": ["multi-agent", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.29378", "doi": "", "url": "https://arxiv.org/abs/2605.29378", "code": "", "site": "", "abstract": "Natural language interfaces can simplify interaction with multi-robot systems, especially when non-expert users need to issue high-level commands. Acoustic manipulation using ultrasonic phased arrays also enables contactless object handling for applications such as healthcare, laboratory automation, and precision transport. However, combining large language models (LLMs) with distributed acoustic mobile robots remains underexplored. This paper presents a decentralized framework for natural language-driven coordination of acoustic robots for contactless object manipulation. The system converts spoken instructions into executable multi-robot task plans using Whisper-based speech recognition, LLM-based semantic parsing, structured JSON task representation, and distributed scheduling. The JSON schema encodes robot assignments, temporal dependencies, spatial constraints, and synchronization requirements for sequential, parallel, and synchronized execution. The system is implemented on two TurtleBot3-based acoustic robots, each equipped with an ultrasonic phased array for contactless object transport. Experiments were conducted in three scenarios: sequential execution, parallel multi-robot transport, and synchronized cooperative manipulation. The system achieved task success rates of 96 percent for sequential tasks, 86 percent for parallel execution, and 70 percent for synchronized collaborative transport. These results show that natural language commands can be transformed into distributed robot actions for contactless manipulation, highlighting the potential of LLM-driven automation for human-robot interaction in distributed robotic systems.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2605.29378v1_Decentralized_LLM_Driven_Coordination_of_Acoustic_Robots_for.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.29378", "fig": "figures/x-decentralizedllmdrivencoordinationofacou.webp"}, {"id": "x-differentiablebeliefbasedopponentshaping", "title": "Differentiable Belief-based Opponent Shaping", "tree": "rl", "branch": "marl", "authors": ["Aarav G Sane", "Karthik Sivachandran", "Rohan Paleja"], "first_author": "Aarav G Sane", "corresponding": [], "institutions": ["Purdue University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-27", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.29042", "doi": "", "url": "https://arxiv.org/abs/2605.29042", "code": "", "site": "", "abstract": "Human coordination often relies on the ability to influence the beliefs of others through strategic action. In multi-agent reinforcement learning, opponent shaping attempts to replicate this influence, though existing methods typically operate within an opponent's parameter, policy, or value space. Meanwhile, belief-manipulation techniques in hidden-role games often rely on hard-coded objectives, such as deception or belief saturation. We propose Differentiable Belief-based Opponent Shaping (D-BOS), a first-order method that treats each observer's belief as the shaped opponent state and differentiates through $k$-step softmax-Bayes belief dynamics. Rather than explicitly rewarding deceptive or cooperative behavior, our method treats the belief state as the target for shaping. This allows the optimal strategy to emerge naturally from the environment's reward structure. This belief-space formulation provides an opponent-shaping signal by differentiating through opponent belief updates, and naturally extends to multiple observers by aggregating gradients over their individual inferred belief trajectories. Empirically, D-BOS outperforms PPO and BBM in hidden-role games, with the largest gains in mixed-motive settings.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2605.29042v1_Differentiable_Belief_based_Opponent_Shaping.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.29042", "fig": "figures/x-differentiablebeliefbasedopponentshaping.webp"}, {"id": "x-distributedmodelpredictivecontrolwithada", "title": "Distributed Model Predictive Control with Adaptive Safety Zones for Multi-Fleet Drone Operations", "tree": "rl", "branch": "marl", "authors": ["Linda Mümken", "Diyar Altinses", "Michael Schwung", "Stefan Lier", "Andreas Schwung"], "first_author": "Linda Mümken", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-08", "topics": ["multi-agent", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.20651", "doi": "", "url": "https://arxiv.org/abs/2606.20651", "code": "", "site": "", "abstract": "Autonomous drone swarms in space-constrained environments such as warehouses, inspection corridors, and urban delivery routes must share limited airspace safely at high vehicle density. Existing approaches rely on fixed safety zones sized for worst-case velocity, which wastes airspace in congested scenarios. We replace the fixed radius with an adaptive, speed-dependent safety sphere whose size scales with braking distance: tight at low speeds, expanded at high speeds. We develop both a centralized model predictive control (MPC) formulation and a distributed MPC (DMPC) in which each drone optimizes locally from detected neighbors, accommodating mixed fleets with non-cooperative agents. We prove feasibility up to the geometric packing limit evaluated at the minimum radius, establish Lyapunov stability under sufficient conditions on the adaptation parameter, drone density, and prediction horizon, and extend these guarantees to the distributed setting via a contraction condition that preserves the centralized stability margins. We further derive modified sphere-packing capacity bounds and a throughput-optimal crossing speed for narrow passages. Simulations confirm that the adaptive framework remains feasible where fixed-radius methods fail: it roughly doubles the admissible drone count, reduces traversal time through constrained passages by about 25 percent, and enables passage through openings impassable to static safety zones. The centralized variant realizes a larger fraction of the theoretical capacity, while the distributed variant offers a more realistic deployment model for mixed-fleet operations under the same safety guarantees.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2606.20651v1_Distributed_Model_Predictive_Control_with_Adaptive_Safety_Zo.pdf", "source": "library", "fig": "figures/x-distributedmodelpredictivecontrolwithada.webp"}, {"id": "x-distributedmotionplanningwithsafetyguara", "title": "Distributed Motion Planning with Safety Guarantees for Self-Reconfiguring Robotic Boats", "tree": "systems", "branch": "planning-control", "authors": ["Alejandro González-García", "Wei Wang", "Wei Xiao", "Wilm Decré", "Jan Swevers", "Carlo Ratti", "Daniela Rus"], "first_author": "Alejandro González-García", "corresponding": [], "institutions": ["MIT", "ByteDance Seed"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-22", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.20352", "doi": "", "url": "https://arxiv.org/abs/2607.20352", "code": "", "site": "", "abstract": "Aquatic self-reconfigurable robots must assemble into desired shapes while ensuring safe interactions among multiple agents. This paper proposes a hybrid framework that combines distributed Model Predictive Control (MPC) with Control Barrier Functions (CBFs) for multi-agent shape formation and reconfiguration. Given a desired shape and target assignment, a distributed MPC scheme, solved via the Alternating Direction Method of Multipliers (ADMM), computes coordinated trajectories through local optimization and information exchange. To ensure safety in real time, distributed CBF-based filters are applied to enforce inter-agent collision avoidance. The proposed approach leverages the predictive capabilities of MPC to mitigate local minima, while CBFs provide formal safety guarantees despite the nonconvexity of the underlying optimization problem. Simulation results with up to 25 agents and experimental validation with four physical robots demonstrate the effectiveness and scalability of the framework.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2607.20352v1_Distributed_Motion_Planning_with_Safety_Guarantees_for_Self.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.20352", "fig": "figures/x-distributedmotionplanningwithsafetyguara.webp"}, {"id": "x-duetdualrobotunderstandingviaefficientte", "title": "Duet: Dual-Robot Understanding via Efficient Teaching", "tree": "il", "branch": "retargeting", "authors": ["Yiqi Zhao", "Ruohai Ge", "Celina Shiyu Wang", "J P Ye", "Muchen Xu", "Minhao Li", "Sergey Zakharov", "Basile Van Hoorick", "Vitor Campagnolo Guizilini", "Leonidas Guibas", "Gaurav S. Sukhatme", "Jyotirmoy V. Deshmukh", "Yue Wang"], "first_author": "Yiqi Zhao", "corresponding": [], "institutions": ["Stanford University", "USC", "Toyota Research Institute"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-18", "topics": ["teleoperation", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.20990", "doi": "10.48550/arxiv.2606.20990", "url": "https://arxiv.org/abs/2606.20990", "code": "", "site": "", "abstract": "Dual-robot collaboration enables tasks that exceed the reach and payload of a single robot, such as collaboratively transporting objects across environments and executing coordinated handovers. Data acquisition is the primary bottleneck for training these systems. To this end, we introduce DUET, a dual-robot learning framework for mobile manipulation. For efficient data collection, we create a unified dual-embodiment synchronized VR-based teleoperation system for in-domain heterogeneous robot data collection. We further develop a complementary tracking pipeline that records human-human coordination and collaborative mobile manipulation priors. To allow efficient learning, we introduce an Action Chunking Transformer based architecture that first pretrains collaborative policies on efficient human-human demonstrations, before finetuning them on a minimal set of real-robot teleoperation trajectories. We develop a benchmark of four collaborative tasks to evaluate our framework using a Unitree G1 humanoid and a Dexmate Vega1 mobile manipulator. The results demonstrate that harnessing human priors not only yields superior task performance compared to baselines trained only on robot data, but also reduces the total human effort required for data collection. Our human data collection pipeline achieves 5.4x acceleration on average from teleoperation, but we perform equally or better than robot-only data trained policies across all tasks. Our project page is available at https://zhaoy37.github.io/Duet/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2606.20990v1_Duet_Dual_Robot_Understanding_via_Efficient_Teaching.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.20990", "fig": "figures/x-duetdualrobotunderstandingviaefficientte.webp"}, {"id": "x-dynamomainstantaneouspredictionofgrasppo", "title": "DynaMOMA: Instantaneous Prediction of Grasp Poses for Mobile Manipulation of Dynamic Objects", "tree": "rl", "branch": "loco-manip", "authors": ["Zhinan Yu", "Junyan Xu", "Jiazhao Zhang", "Zheng Qin", "Yijie Tang", "Y Huang", "Yihan Cao", "Zhiyuan Yu", "Yongjun Wang", "Renjiao Yi", "Chenyang Zhu", "Kai Xu"], "first_author": "Zhinan Yu", "corresponding": [], "institutions": ["Peking University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-24", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.25295", "doi": "", "url": "https://arxiv.org/abs/2606.25295", "code": "", "site": "", "abstract": "Mobile manipulation is a fundamental robotics task and has advanced rapidly in recent years, enabling robots to navigate, reach, and interact with objects in complex environments. However, mobile manipulation of dynamic objects remains highly challenging, as robots must coordinate the mobile base and arm while adapting to continuously evolving target poses. A key challenge lies in predicting temporally consistent short-horizon grasp trajectories from dynamic observations. In this work, we propose \\ours{}, a dynamic mobile manipulation framework that couples instantaneous grasp trajectory prediction with whole-body control policy. Our predictor uses an anchor-based diffusion model to generate temporally consistent short-horizon grasp trajectories conditioned on historical observations. The predicted trajectories are then encoded as compact features and fed to a whole-body reinforcement learning policy, which controls the mobile manipulator for dynamic grasping. We further introduce a anticipation-guided reward that equips the policy with an anticipatory grasping horizon by adaptively shifting the target from the current grasp observation to the instantaneously predicted grasp trajectory. Through extensive experiments in Isaac Gym simulation, we show that our method achieves strong performance in mobile manipulation of dynamic objects across diverse settings and grasping metrics. Furthermore, our predictor and policy demonstrate strong generalizability in real-world experiments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2606.25295v1_DynaMOMA_Instantaneous_Prediction_of_Grasp_Poses_for_Mobile.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.25295", "fig": "figures/x-dynamomainstantaneouspredictionofgrasppo.webp"}, {"id": "x-efluxelasticmultirobotformationnavigatio", "title": "EFLUX: Elastic Multi-Robot Formation Navigation and Adaptation with Agentic LLMs", "tree": "rl", "branch": "marl", "authors": ["Jinyuan Zhang", "Yuwei Wu", "Guangyao Shi", "Jonathan Diller", "Gaurav S. Sukhatme", "Vijay Kumar"], "first_author": "Jinyuan Zhang", "corresponding": [], "institutions": ["University of Pennsylvania", "University of Southern California"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-13", "topics": ["multi-agent", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.12050", "doi": "", "url": "https://arxiv.org/abs/2607.12050", "code": "", "site": "", "abstract": "Multi-robot teams operating in confined or cluttered environments must adapt both their formation geometry and group topology to navigate through complex obstacles. This adaptation requires two complementary behaviors: deformation, where the team continuously reshapes its geometry while remaining connected, and reconfiguration, where robots split into subgroups or merge back into a single formation. Existing methods often model these behaviors independently, connect them through handcrafted rules, or lack explicit geometric criteria for determining when each behavior should be invoked. However, challenging environments may require online changes in formation shape, connectivity, and effective team composition, making decoupled or rule-based approaches prone to suboptimal trajectories and deadlock. We propose EFLUX, a geometry-grounded LLM agentic framework for automatic and elastic multi-robot formation navigation. EFLUX extracts a structured scene representation and uses an LLM to reason jointly over both deformation actions, such as scaling and shearing, and reconfiguration actions, such as splitting and merging. These strategies are then translated into executable per-robot waypoints through a closed-loop generation, verification, and correction pipeline. Simulation and hardware experiments show that EFLUX enables safe, continuous, and elastic formation navigation in constrained environments, reducing deadlock and navigation failures compared with baselines while maintaining coherent multi-robot coordination.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2607.12050v1_EFLUX_Elastic_Multi_Robot_Formation_Navigation_and_Adaptatio.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.12050", "fig": "figures/x-efluxelasticmultirobotformationnavigatio.webp"}, {"id": "x-efficientrealworldonlinereinforcementlea", "title": "Efficient Real-World Online Reinforcement Learning for Robot Manipulation via Centralized Training and Critic Decomposition", "tree": "systems", "branch": "sim-bench", "authors": ["Changhao Li", "Yifang Zhang", "Heng Zhang", "Davide Torielli", "Damiano Gasperini", "Arturo Laurenzi", "Luca Muratore", "Arash Ajoudani", "Nikos Tsagarakis"], "first_author": "Changhao Li", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-10", "topics": ["simulation & benchmarks", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.09762", "doi": "", "url": "https://arxiv.org/abs/2608.09762", "code": "", "site": "", "abstract": "Real-world online reinforcement learning (RL) provides a promising approach for training robotic manipulation policies directly in the physical world, avoiding the sim-to-real gap and enabling continuous policy refinement through human-in-the-loop interaction. Recent methods have demonstrated sample-efficient learning through human intervention but remain limited to small randomization ranges and encounter challenges with the non-stationarity induced by concurrently training multiple agents. To address these limitations, we introduce a unified framework that combines centralized training with decentralized execution (CTDE) and a Hybrid Reward Architecture (HRA). This enables multiple actors to share a centralized multi-head critic. The critic is decomposed into task and grasp heads, corresponding to the sparse task reward and a potential-based grasping reward, respectively. We accordingly reformulate the critic and actor objectives to exploit the decomposed Q-values while explicitly accounting for the categorical action distribution of the discrete gripper policy. Experimental results demonstrate that the proposed framework substantially improves both sample efficiency and policy performance. We validate our approach on two robotic arms and a simulated humanoid robot across tennis ball and banana pick-and-place, pot reset, and simulated block relocation tasks under dimension-wise domain randomization, approximately 5-25x larger than those considered in prior work. Compared with a state-of-the-art baseline, our method improves the success rate from 60% to 80% on tennis ball pick-and-place, from 60% to 90% on banana pick-and-place, and from 25% to 95% on simulated block relocation, while also successfully accomplishing a task where the baseline consistently fails. Videos and more details are available at our project website: https://hil-harc.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.09762v1_Efficient_Real_World_Online_Reinforcement_Learning_for_Robot.pdf", "source": "library", "fig": "figures/x-efficientrealworldonlinereinforcementlea.webp"}, {"id": "x-exposingthelongtailinembodiedurbannaviga", "title": "Exposing the Long-tail in Embodied Urban Navigation via Scalable Learning from In-the-Wild Videos", "tree": "il", "branch": "vla", "authors": ["Bingyi Xia", "Han Bao", "Zhewei Chen", "Hanjing Ye", "Jingwen Yu", "Yuhan Pang", "Wenjun Xu", "Jiankun Wang"], "first_author": "Bingyi Xia", "corresponding": [], "institutions": ["HKUST"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-17", "topics": ["VLA", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.16476", "doi": "10.48550/arxiv.2608.16476", "url": "https://arxiv.org/abs/2608.16476", "code": "", "site": "", "abstract": "Learning embodied urban navigation policies from real-world data is constrained by the cost of task-specific data collection and the limited coverage of rare yet safety-critical scenarios. To address these challenges, we present a scalable framework for learning point-goal urban navigation from web-scale in-the-wild egocentric videos while systematically exposing its long tail. The framework automatically annotates uncurated web videos with metric trajectories and structured navigation semantics, which are then used to train a vision-language-action policy for interpretable navigation planning. We characterize the long tail based on model performance and the distribution of perception-motion patterns, and employ reflection-based analysis to diagnose recurring failure modes. Experiments on web-video data and real-world urban navigation tasks demonstrate effective knowledge transfer from unconstrained videos and reveal coherent long-tail structures beyond aggregate navigation performance.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.16476v1_Exposing_the_Long_tail_in_Embodied_Urban_Navigation_via_Scal.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.16476", "fig": "figures/x-exposingthelongtailinembodiedurbannaviga.webp"}, {"id": "x-ftwbclearningfaulttolerantwholebodycontr", "title": "FT-WBC: Learning Fault-Tolerant Whole-Body Control for Legged Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Yudong Zhong", "Pengfei Mai", "Sikai Guo", "Jiahang Cao", "Zhihai Bi", "Qiuyue Liu", "Ziyan Feng", "Jinni Zhou", "J Y"], "first_author": "Yudong Zhong", "corresponding": [], "institutions": ["HKU", "HKUST"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-23", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.24466", "doi": "", "url": "https://arxiv.org/abs/2606.24466", "code": "", "site": "", "abstract": "Legged manipulators combine the mobility of legged platforms with the manipulation capability of robotic arms. However, arm-induced Center-of-Mass shifts and dynamic disturbances make the system more prone to instability under actuator failures, potentially leading to falls, task failures, or safety risks. Existing fault-tolerant control methods mainly focus on locomotion alone, leaving the coupled problem of whole-body stability and arm reachability in fault-tolerant loco-manipulation largely unaddressed. To bridge this gap, we propose FT-WBC, a fault-tolerant loco-manipulation framework for robust whole-body control of legged manipulators under actuator failures. FT-WBC adopts a decoupled upper- and lower-body policy architecture and introduces two key modules: a Fault Estimator (FE) and a Posture Adaptation Module (PAM). The FE predicts faulty joints from lower-body proprioceptive histories, while the PAM uses this fault information to adapt the base posture plan generated by the arm policy, converting potentially unstable posture requests into safe and executable base posture commands. Through this fault-aware posture adaptation mechanism, FT-WBC synthesizes compensatory gaits under actuator failures and preserves as much arm workspace as possible while maintaining whole-body stability. Simulation and real-world experiments show that FT-WBC significantly improves survival rate and workspace under weakening or locked failures, and transfers zero-shot to a real legged manipulator in the real world.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2606.24466v2_FT_WBC_Learning_Fault_Tolerant_Whole_Body_Control_for_Legged.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.24466", "fig": "figures/x-ftwbclearningfaulttolerantwholebodycontr.webp"}, {"id": "fetchman", "title": "FetchMan: Learning Visual Humanoid Loco-Manipulation Policies from Simulated Experiences", "tree": "rl", "branch": "loco-manip", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["University of Washington", "UCLA", "Generalist AI", "Allen Institute for AI", "Unitree Robotics"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-17", "topics": ["visual loco-manipulation", "humanoid fetching"], "paradigm": ["RL"], "method": [], "projects": ["m2", "mabel"], "related": ["deep-wbc", "falcon", "handeye-delivery", "visualmimic"], "arxiv": "2608.17027", "doi": "", "code": "", "site": "", "note": "From your M2 literature. Recent visual loco-manipulation on humanoids.", "aka": "", "stars": 1, "curated": true, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.17027v1_FetchMan_Learning_Visual_Humanoid_Loco_Manipulation_Policies.pdf", "abstract": "Visual loco-manipulation policies that can generalize to novel scenes and objects have long been a goal of robotics research. However, today's data-hungry algorithms make collecting sufficient demonstrations a struggle for tabletop manipulation, and even more so for humanoids that must also walk and balance. Learning from simulated data and transferring that behavior to the real world, as is commonly done in locomotion, sidesteps this struggle, so we replicate that recipe for loco-manipulation. In doing so, we find that cloning synthetic demonstrations results in a low performance ceiling no matter the amount of training data. Reinforcement learning breaks through it, and refining the cloned policy with Flow-GRPO on a single sparse reward yields performance that synthetic behavior cloning cannot match. Together, these stages form our end-to-end sim-to-real pipeline spanning more than 150,000 scenes, which we use to train FetchMan. We evaluate it on FetchMan-Bench, a simulation benchmark we release, and deploy it zero-shot on a real Unitree G1, where our single-object reach-and-pick policy walks to and grasps a target across unseen scenes at 73.3% success. Finally, we extend this recipe to multi-object training, a first step toward loco-manipulation generalist policies at this data scale.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2608.17027", "arxiv_url": "https://arxiv.org/abs/2608.17027", "fig": "figures/fetchman.webp", "figs": ["figures/fetchman-1.webp", "figures/fetchman-2.webp"]}, {"id": "figure-fleet", "title": "Figure AI: Helix in the home and at BMW Spartanburg", "tree": "systems", "branch": "agentic", "authors": ["Figure AI"], "corresponding": [], "institutions": ["Figure AI"], "lab": "Figure AI", "venue": "company blog", "venue_type": "industry", "year": 2026, "date": "", "topics": ["humanoid deployment", "long-horizon autonomy", "fleet operation"], "paradigm": ["VLA"], "method": [], "projects": ["mabel"], "related": ["dyna2", "groot-n16", "helix"], "arxiv": "", "doi": "", "code": "", "site": "https://www.figure.ai/news", "note": "Figure 03 doing long-horizon kitchen work, fine manipulation and fleet operation at BMW. Useful only as a marker of the current industrial reliability bar; no methodology is disclosed.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Figure AI", "url": "https://www.figure.ai/news"}, {"id": "x-fromfoundationtoapplicationimprovingvlam", "title": "From Foundation to Application: Improving VLA Models in Practice", "tree": "il", "branch": "vla", "authors": ["Wei Wu", "Fangjing Wang", "Fan Lu", "He Sun", "Shi Liu", "Yunnan Wang", "Yibin Yan", "Yong Wang", "Shuailei Ma", "Xinyang Wang", "Yibin Liu", "Shuai Yang", "Tianxiang Zhou", "Kejia Zhang", "Lei Zhou", "Cheng Su", "Nan Xue", "Bin Tan", "Han Zhang", "Youchao Zhang", "Fei Liao", "Xing Zhu", "Yujun Shen", "Kecheng Zheng"], "first_author": "Wei Wu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-07", "topics": ["VLA", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2607.06403", "doi": "", "url": "https://arxiv.org/abs/2607.06403", "code": "", "site": "", "abstract": "Despite recent progress of VLA foundation models, the disparity between laboratory conditions and real-world applications continues to impede their practical implementation. To bridge this gap, we present LingBot-VLA 2.0, which advances LingBot-VLA through improvements in three functional domains. (1) Generalization across tasks and embodiments. Compared to the previous version, we revamp the data processing pipeline and curate around 60,000 hours of data for pretraining, including 50,000 hours of robot trajectories spanning 20 robot configurations and 10,000 hours of egocentric human videos. (2) Expanded action space in addition to dual-arm hardware platforms. In particular, our system accommodates degrees of freedom for the heads, waists, mobile bases, and dexterous hands, thereby empowering the robots to tackle more complex tasks in practical scenarios. (3) Predictive dynamics modeling for improved temporal reasoning. Specifically, we formulate future prediction as a proxy task, facilitated by a video representation model for semantic priors and a depth estimation model for geometric cues. Evaluations on the GM-100 benchmark, conducted in a generalist setting, validate the beneficial impact of these proposed modifications. Furthermore, benefiting from the expanded pretraining data that covers whole-body degrees of freedom, LingBot-VLA-2.0 demonstrates strong cross-embodiment long-horizon mobile manipulation capability across the two robotic platforms.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2026_2607.06403v1_From_Foundation_to_Application_Improving_VLA_Models_in_Pract.pdf", "source": "library", "fig": "figures/x-fromfoundationtoapplicationimprovingvlam.webp"}, {"id": "x-fromrawdatatoshared3dsemanticstaskorient", "title": "From Raw Data to Shared 3D Semantics: Task-Oriented Communication for Multi-Robot Collaboration", "tree": "rl", "branch": "marl", "authors": ["Ruibo Xue", "Jiedan Tan", "Fang Liu", "Jingwen Tong", "Taotao Wang", "Shuoyao Wang"], "first_author": "Ruibo Xue", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-09", "topics": ["multi-agent", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2602.08624", "doi": "", "url": "https://arxiv.org/abs/2602.08624", "code": "", "site": "", "abstract": "Multi-robot systems (MRS) rely on exchanging raw sensory data to cooperate in complex three-dimensional (3D) environments. However, this strategy often leads to severe communication congestion and high transmission latency, significantly degrading collaboration efficiency. This paper proposes a decentralized task-oriented semantic communication framework for multi-robot collaboration in unknown 3D environments. Each robot locally extracts compact, task-relevant semantics using a lightweight Pixel Difference Network (PiDiNet) with geometric processing. It shares only these semantic updates to build a task-sufficient 3D scene representation that supports cooperative perception, navigation, and object transport. Our numerical results show that the proposed method exhibits a dramatic reduction in communication overhead from $858.6$ Mb to $4.0$ Mb (over $200\\times$ compression gain) while improving collaboration efficiency by shortening task completion from $1,054$ to $281$ steps.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2602.08624v1_From_Raw_Data_to_Shared_3D_Semantics_Task_Oriented_Communica.pdf", "source": "library", "fig": "figures/x-fromrawdatatoshared3dsemanticstaskorient.webp"}, {"id": "x-geneticfuzzysystembasedmultirobotcoordin", "title": "Genetic Fuzzy System-Based Multi-Robot Coordination for Planetary Missions", "tree": "rl", "branch": "locomotion", "authors": ["Daegyun Choi", "Donghoon Kim"], "first_author": "Daegyun Choi", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-13", "topics": ["locomotion", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.12755", "doi": "10.48550/arxiv.2608.12755", "url": "https://arxiv.org/abs/2608.12755", "code": "", "site": "", "abstract": "This paper proposes a decentralized approach for a multi-robot system (MRS) using a genetic fuzzy system to perform a collaborative object transportation task that minimizes the total path length of the MRS in unstructured environment while avoiding obstacles. For an environment given by an elevation map, terrain traversability analysis with respect to the slope is performed to reduce the dimension and identify non-traversable areas that can be considered as obstacles, and the given map is converted into a traversability map in two dimensional space. In the training process, proposed fuzzy inference systems (FISs) to generate the MRS's velocity for transporting an object to a target position are optimized by a genetic algorithm with several scenarios, such as a local minima, a target that is close to an obstacle, and a cluttered environment. The trained FIS models are applied to the testing environment, which is the converted traversability map, and validated using multiple scenarios.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2608.12755v1_Genetic_Fuzzy_System_Based_Multi_Robot_Coordination_for_Plan.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.12755", "fig": "figures/x-geneticfuzzysystembasedmultirobotcoordin.webp"}, {"id": "x-hafadaptinggeneralistvlastohumanoidwhole", "title": "HAF: Adapting Generalist VLAs to Humanoid Whole-Body Loco-manipulation via Hierarchical Action Flow and Spectral Latent RL", "tree": "il", "branch": "vla", "authors": ["Langzhe Gu", "Chengkai Hou", "Meng Li", "Xinhua Wang", "Jiaming Liu", "Xinyuan Lv", "Bowei Zhang", "Shuanghao Bai", "Guangrun Li", "Jingyang He", "Gaole Dai", "Ziluo Ding", "Zhiyuan Xu", "Kuan Cheng", "Jian Tang", "Zhengping Che", "Shanghang Zhang"], "first_author": "Langzhe Gu", "corresponding": [], "institutions": ["Peking University"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-17", "topics": ["VLA", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.16837", "doi": "10.48550/arxiv.2608.16837", "url": "https://arxiv.org/abs/2608.16837", "code": "", "site": "", "abstract": "Humanoid robots hold great promise as general-purpose agents in human-centered environments, yet generalist vision-language-action (VLA) foundation models are not readily applicable to humanoid whole-body loco-manipulation. The high dimensionality and interdependence of humanoid motions make it challenging for conventional single-stage VLA architectures to coordinate locomotion, waist posture, and dual-arm manipulation effectively. Moreover, policies trained through offline behavior cloning can remain suboptimal during real-world deployment. Although online reinforcement learning can refine policies through real-world interaction, directly tuning large VLA backbones demands excessive computation and may introduce safety risks during real-robot exploration. To address these bottlenecks, we introduce HAF (Humanoid Adaptation Framework), a two-part framework consisting of HAF-VLA and HAF-Steer that transfers off-the-shelf generalist VLA foundation models to humanoid whole-body loco-manipulation. HAF-VLA is a hierarchical action-flow generator built on a pretrained flow-matching VLA. It splits full-body action denoising into three sequential stages with stage embeddings and cross-stage KV caches that retain kinematic dependencies, avoiding incoherent whole-body actions from one-shot generation. On top of the frozen HAF-VLA, HAF-Steer is a latent offline-to-online RL pipeline that leverages flow-matching invertibility and DCT-based dimensionality reduction to restrict RL optimization to a compact noise subspace and train a regularized SAC policy. This avoids updating the large VLA backbone and enables efficient real-world policy refinement. Evaluated on seven real-world humanoid loco-manipulation tasks, HAF surpasses vanilla single-stage VLA baselines and improves whole-body coordination and task performance. Project website: https://grange007.github.io/HAF .", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.16837v1_HAF_Adapting_Generalist_VLAs_to_Humanoid_Whole_Body_Loco_man.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.16837", "fig": "figures/x-hafadaptinggeneralistvlastohumanoidwhole.webp"}, {"id": "x-handcentrichumantorobottrajectorytransfe", "title": "Hand-centric Human-to-Robot Trajectory Transfer from Video Demonstrations via Open-World Contact Localization", "tree": "rl", "branch": "motion-imitation", "authors": ["Yitian Shi", "Di Wen", "Zhengqi Han", "Zicheng Guo", "Yu Hu", "Edgar Welte", "Kunyu Peng", "Rainer Stiefelhagen", "Rania Rayyes"], "first_author": "Yitian Shi", "corresponding": [], "institutions": ["KIT"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-09", "topics": ["motion imitation", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2606.10743", "doi": "", "url": "https://arxiv.org/abs/2606.10743", "code": "", "site": "", "abstract": "Learning from human video demonstrations remains challenging due to noisy hand-object interactions, unseen objects with partial observation, and cross-embodiment discrepancy. To address these challenges, we present \\textit{HOWTransfer} (\\emph{H}and-\\emph{O}bject \\emph{O}pen-\\emph{W}orld Transfer), a hand-centric framework that distills human demonstrations into contact-aware, taxonomy-informed, and diverse robotic trajectories. Instead of relying on object-specific descriptions, vision-language queries, or explicit object-state tracking, \\emph{HOWTransfer} recovers temporally consistent 3D hand motion and localizes temporal contact intervals by reasoning over observed hand-object interaction cues. The localized contact onsets are then used to retarget human grasp intent into multi-modal parallel-jaw grasp hypotheses, which are propagated along the recovered wrist trajectory to generate robot-executable motions. Finally, a trajectory editing stage refines contact alignment and produces diverse executable variants from a single demonstration. Experiments across diverse manipulation tasks show that \\emph{HOWTransfer} enables accurate contact localization and high-quality robot motion retargeting with $86\\%$ success, which is preferred over teleoperated trajectories in a blinded preference study.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2606.10743v1_Hand_centric_Human_to_Robot_Trajectory_Transfer_from_Video_D.pdf", "source": "library", "fig": "figures/x-handcentrichumantorobottrajectorytransfe.webp"}, {"id": "x-handeditaunifiedbenchmarkforegocentrichu", "title": "HandEdit: A Unified Benchmark for Egocentric Human-to-Robot Dexterous Hand Image Editing", "tree": "il", "branch": "human-video", "authors": ["Zhenjie Yang", "Xingyu Jiao", "Guopeng Zhong", "Shuzhe Yang", "Shi Che", "Chao Wu", "Chenyu Jiang", "Dongjie Zhang", "Yideng Zhang", "Zheng Zhang", "Muyun Jiang", "Haisheng Su", "Shuang Jin", "Donghang Zhang", "Chao Yang", "Li Chen", "Hongyang Li", "Zuxuan Wu", "Yu-Gang Jiang", "Xiaosong Jia", "Junchi Yan"], "first_author": "Zhenjie Yang", "corresponding": [], "institutions": ["OpenAI", "Shanghai Jiao Tong University", "Shanghai AI Laboratory", "Fudan University", "HKU", "ByteDance Seed"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-12", "topics": ["human video", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.12122", "doi": "10.48550/arxiv.2608.12122", "url": "https://arxiv.org/abs/2608.12122", "code": "", "site": "", "abstract": "Robotic manipulation with dexterous hands is a cornerstone of Embodied AI, yet its progress is stifled by the high cost of collecting embodiment-aware teleoperation data. While abundant egocentric videos of human hands offer a scalable alternative, the profound discrepancies in appearance, articulation, and camera viewpoints between human and robotic data raise significant challenges for co-training. Though existing general image-editing models demonstrate strong capabilities, they lack necessary embodiment-specific priors to fully bridge this gap. In this work, we present HandEdit, a unified large-scale embodiment-aware image-editing dataset and benchmark specifically designed to transform human hands and arms into various dexterous robotic embodiments within egocentric frames. HandEdit comprises over 200M editing instances derived from five diverse source datasets, covering 26 distinct URDFs, including 13 hand-only and 13 hand-arm configurations. Alongside the dataset, we establish a unified benchmark protocol with two tracks: Hand-only and Hand-Arm, supporting URDF-conditioned evaluation. We conduct extensive evaluations of 11 representative image-editing baselines using a multi-dimensional metric suite, including generic similarity metrics, VLM-based judgment, and embodiment-aware metrics. HandEdit serves as a critical resource at the intersection of image editing and robotics: it advances embodiment-aware editing models while enabling scalable dexterous robotic learning from abundant human video data, paving the way for more generalizable Embodied AI.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.12122v1_HandEdit_A_Unified_Benchmark_for_Egocentric_Human_to_Robot_D.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.12122", "fig": "figures/x-handeditaunifiedbenchmarkforegocentrichu.webp"}, {"id": "x-humanguidedcomanipulationofcarbonfiberpl", "title": "Human-Guided Co-Manipulation of Carbon Fiber Plies", "tree": "rl", "branch": "marl", "authors": ["Rami Ojanen", "James Fant-Male", "Roel Pieters"], "first_author": "Rami Ojanen", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-10", "topics": ["multi-agent", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.11818", "doi": "10.48550/arxiv.2606.11818", "url": "https://arxiv.org/abs/2606.11818", "code": "", "site": "", "abstract": "The handling of flexible materials is a difficult task to fully automate due to the challenges caused by the deformability of these types of objects. Meanwhile, a fully manual process can be ergonomically challenging, tedious and inefficient. Thus, human-robot collaboration (HRC) and cooperative manipulation (co-manipulation) have received increasing interest in this field as they enable human involvement when needed while also improving productivity. To enable efficient co-manipulation and interaction between the human operator and the robot, different modalities and control methods are required. In this paper, we present and examine different control methods for co-manipulation of carbon fiber plies, evaluating the pros and cons of each method in a controlled setting. We propose that a multimodal combination of speech commands, wrist-tracking through vision, and force with compliant control would provide the best solution for complete and intuitive control of the task.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2606.11818v1_Human_Guided_Co_Manipulation_of_Carbon_Fiber_Plies.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.11818", "fig": "figures/x-humanguidedcomanipulationofcarbonfiberpl.webp"}, {"id": "x-humanoidvlnaphysicsgroundedsimulatorandb", "title": "HumanoidVLN: A Physics-Grounded Simulator and Benchmark for Vision-Language Navigation Across Diverse Humanoid Embodiments", "tree": "rl", "branch": "locomotion", "authors": ["Quan-Dung Pham", "Anh Dao", "The-Anh Nguyen", "Minh Nguyen-Dinh", "Phuong Nam Dang", "Tri Pham", "Hung Tran", "Bach Dao", "Tuyen P. Le", "Truong Nguyen", "Quan Nguyen"], "first_author": "Quan-Dung Pham", "corresponding": [], "institutions": ["NVIDIA", "USC", "Toyota Research Institute"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-13", "topics": ["locomotion", "humanoid platforms"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.12860", "doi": "10.48550/arxiv.2608.12860", "url": "https://arxiv.org/abs/2608.12860", "code": "", "site": "", "abstract": "Vision-Language Navigation (VLN) for humanoid robots poses challenges existing benchmarks fail to address: bipedal locomotion imposes physical constraints absent from wheeled agents, humanoid morphologies vary across platforms, and egocentric observations are distorted by locomotion-induced camera dynamics. We present HumanoidVLN, a physics-grounded simulator and benchmark for VLN across diverse humanoid embodiments. Built on NVIDIA Isaac Sim, our platform supports an extensible set of humanoid configurations, demonstrated on four robots (Unitree G1, Unitree H1, Internal-A, Internal-B) spanning 10-12 lower-body DoF and heights from 1.17m to 1.80m, via a hierarchical control stack combining a reinforcement learning locomotion policy with interchangeable PD or MPC path trackers. New robots and VLN models integrate with minimal effort; we demonstrate compatibility with NaVILA, DualVLN, StreamVLN, and JanusVLN. Environments are drawn from artist-designed scenes and 3D Gaussian Splatting reconstructions, filtered for navigable areas exceeding 100 square meters. Instructions are generated by a dual generator-reviewer plus paraphraser multi-agent pipeline with human-in-the-loop verification, yielding 933 collision-aware reference episodes, each paired with one fine-grained instruction and three coarse-grained stylistic variants (formal, natural, casual). Across four models and four embodiments, JanusVLN achieves the highest mean success rate of 43.55% and nDTW of 48.38. In a 20-episode sim-to-real pilot with DualVLN and the Unitree G1, navigation errors correlate strongly (r=0.935), with a mean absolute difference of 0.68m and mean trajectory similarity of 0.782 (+/-0.188) nDTW. These results highlight the interaction between VLN models, controllers, and humanoid embodiments under physical execution. Code, benchmark, and data will be released upon acceptance at https://humanoid-vln.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2608.12860v1_HumanoidVLN_A_Physics_Grounded_Simulator_and_Benchmark_for_V.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.12860", "fig": "figures/x-humanoidvlnaphysicsgroundedsimulatorandb.webp"}, {"id": "x-hybridfeedbacksamplingforsampleefficient", "title": "Hybrid Feedback Sampling for Sample-Efficient Model Predictive Control", "tree": "rl", "branch": "loco-manip", "authors": ["Chaoyi Pan", "Zeji Yi", "John Zhang", "Zachary Manchester", "Guannan Qu", "Guanya Shi"], "first_author": "Chaoyi Pan", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-19", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.19443", "doi": "", "url": "https://arxiv.org/abs/2608.19443", "code": "", "site": "", "abstract": "Thanks to its parallelizability and flexibility, sampling-based Model Predictive Control (MPC) has become widely popular for controlling real-world robotic systems. However, for high-dimensional and open-loop unstable dynamical systems, the required number of samples to improve the control sequence will grow exponentially with the horizon, leading to poor sample efficiency and numerical instability. This paper investigates the instability of shooting methods in sampling-based MPC and shows that the optimal sampling proposal distribution can be realized by sampling with an optimized feedback policy. We refer to this algorithm as Feedback Sampling MPC (FS-MPC). FS-MPC involves a hybrid sampling design which balances local and global search based on the system stability and the available computation budget. Our theoretical analysis shows that our hybrid sampling approach achieves faster convergence than standard MPPI and better optimality than standard feedback sampling. Empirically, in diverse contact-rich control tasks like humanoid loco-manipulation and dexterous manipulation, we show that FS-MPC successfully tackles dynamically unstable tasks where standard sample-based approaches struggle, and strictly outperforms feedback policies alone. Finally, we validate our method on humanoid robot locomotion and manipulation tasks in the real world.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.19443v1_Hybrid_Feedback_Sampling_for_Sample_Efficient_Model_Predicti.pdf", "source": "library", "fig": "figures/x-hybridfeedbacksamplingforsampleefficient.webp"}, {"id": "x-joyaira05scalingrobotmanipulationlearnin", "title": "JoyAI-RA 0.5: Scaling Robot Manipulation Learning via Dual Action Alignment", "tree": "il", "branch": "world-models", "authors": ["JoyAI-RA Team"], "first_author": "JoyAI-RA Team", "corresponding": ["RA Team"], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-06", "topics": ["world model", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": [], "arxiv": "2608.05674", "doi": "", "url": "https://arxiv.org/abs/2608.05674", "code": "", "site": "", "abstract": "Robot data is scarce, so generalist policies need to learn from heterogeneous sources, including human egocentric video, simulation, and real robots, which differ in supervision and embodiment, with action labels missing or mutually incompatible. Human egocentric data scale best but sit farthest from robot data, and naive pooling causes negative transfer rather than knowledge sharing. We propose JoyAI-RA 0.5, a generalist Vision-Language-World-Action (VLWA) framework that couples physical world-dynamics priors with visual semantics and scales manipulation learning across such data via dual action alignment. Implicit action alignment infers latent actions from visual transitions, enabling action-free human, simulation, and robot data to guide a latent-action-conditioned world model in learning physical dynamics. Explicit alignment grounds reliable human and robot trajectories in a unified physical action space through a canonical action representation and camera-frame chunk-relative end-effector actions. An inner-outer-loop reinforcement stage then pairs efficient task adaptation with foundation-policy improvement. On a real-world AgiBot benchmark, JoyAI-RA performs strongly on both seen tasks and unseen variations. The task score improves consistently as the volume of human egocentric pretraining data increases and shows no sign of plateauing at our largest scale. This suggests that abundant but weakly labeled human experience can be converted into a transferable training signal, making human video not merely a weak auxiliary source but a primary axis along which manipulation capability can be scaled. Project page can be found at https://joyai-ra-05.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2608.05674v1_JoyAI_RA_0_5_Scaling_Robot_Manipulation_Learning_via_Dual_Ac.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.05674", "fig": "figures/x-joyaira05scalingrobotmanipulationlearnin.webp"}, {"id": "x-kilvokinematicinertiallidarvisualodometr", "title": "KILVO: Kinematic-Inertial-LiDAR-Visual Odometry with Robust Multimodal Adaptation for Humanoid Robots", "tree": "rl", "branch": "locomotion", "authors": ["Jixin Gao", "Fucheng Liu", "Teng Zhang", "Fusheng Zha"], "first_author": "Jixin Gao", "corresponding": [], "institutions": ["Harbin Institute of Technology"], "lab": "", "venue": "IEEE/ASME T-Mech", "venue_type": "journal", "year": 2026, "date": "2026-08-06", "topics": ["locomotion", "humanoid platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.05647", "doi": "10.1109/TMECH.2026.3721778", "url": "https://doi.org/10.1109/tmech.2026.3721778", "code": "", "site": "", "abstract": "This article presents a kinematic-inertial-LiDAR-visual odometry for humanoid robots, called KILVO. Tailored to the platform features, requirements, and real-world complexity, it fully utilizes the sensors commonly equipped on humanoid robots, including joint encoders, IMU, LiDAR, and camera, within an asynchronous-sequential hybrid error-state iterated Kalman filter (ESIKF). Specifically, inertial data are used for prediction, leg kinematics are processed asynchronously at a high rate and provide proprioceptive constraints, while exteroception is updated sequentially, first by registering LiDAR points for geometric priors and then by updating the visual component via photometric errors. Moreover, the framework is elaborately designed with multimodal adaptation for resilience to sensor failures. A compact contact estimation module is also developed, sharing information with state estimation without additional sensors. Extensive experiments on public datasets and in the real world across multiple humanoid robots, gait patterns, and scenarios demonstrate that KILVO achieves highly competitive accuracy, efficiency, and output rates, with strong robustness against sensor degradation and failures, making it more suitable for humanoid robots than state-of-the-art fusion methods. Our code and datasets are released on GitHub.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2608.05647v1_KILVO_Kinematic_Inertial_LiDAR_Visual_Odometry_with_Robust_M.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.05647", "fig": "figures/x-kilvokinematicinertiallidarvisualodometr.webp"}, {"id": "x-kinderaphysicalreasoningbenchmarkforrobo", "title": "KinDER: A Physical Reasoning Benchmark for Robot Learning and Planning", "tree": "systems", "branch": "bimanual", "authors": ["Yixuan Huang", "Bowen Li", "Vaibhav Saxena", "Yichao Liang", "Utkarsh Aashu Mishra", "Liang Ji", "Lihan Zha", "Jimmy Wu", "Nishanth Kumar", "Sebastian Scherer", "Danfei Xu", "Tom Silver"], "first_author": "Yixuan Huang", "corresponding": [], "institutions": ["Carnegie Mellon University", "MIT", "NVIDIA", "Georgia Tech", "Princeton University", "University of Cambridge"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2026, "date": "2026-04-28", "topics": ["open hardware", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2604.25788", "doi": "10.15607/rss.2026.xxii.184", "url": "https://doi.org/10.15607/rss.2026.xxii.184", "code": "", "site": "", "abstract": "Robotic systems that interact with the physical world must reason about kinematic and dynamic constraints imposed by their own embodiment, their environment, and the task at hand. We introduce KinDER, a benchmark for Kinematic and Dynamic Embodied Reasoning that targets physical reasoning challenges arising in robot learning and planning. KinDER comprises 25 procedurally generated environments, a Gymnasium-compatible Python library with parameterized skills and demonstrations, and a standardized evaluation suite with 13 implemented baselines spanning task and motion planning, imitation learning, reinforcement learning, and foundation-model-based approaches. The environments are designed to isolate five core physical reasoning challenges: basic spatial relations, nonprehensile multi-object manipulation, tool use, combinatorial geometric constraints, and dynamic constraints, disentangled from perception, language understanding, and application-specific complexity. Empirical evaluation shows that existing methods struggle to solve many of the environments, indicating substantial gaps in current approaches to physical reasoning. We additionally include real-to-sim-to-real experiments on a mobile manipulator to assess the correspondence between simulation and real-world physical interaction. KinDER is fully open-sourced and intended to enable systematic comparison across diverse paradigms for advancing physical reasoning in robotics. Website and code: https://prpl-group.com/kinder-site/", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2604.25788v2_KinDER_A_Physical_Reasoning_Benchmark_for_Robot_Learning_and.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2604.25788", "fig": "figures/x-kinderaphysicalreasoningbenchmarkforrobo.webp"}, {"id": "x-lucidlatentskillunifiedcontrolviaimagine", "title": "LUCID: Latent-Skill Unified Control via Imagined Dynamics for Long-Horizon Humanoid Loco-Manipulation", "tree": "il", "branch": "world-models", "authors": ["Cheng Guo", "Mingzhe Ni", "Angelo Cangelosi", "Arash Ajoudani"], "first_author": "Cheng Guo", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-07", "topics": ["world model", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2", "wam-tactile"], "related": [], "arxiv": "2608.07746", "doi": "10.48550/arxiv.2608.07746", "url": "https://arxiv.org/abs/2608.07746", "code": "", "site": "", "abstract": "Long-horizon humanoid loco-manipulation requires composing versatile whole-body skills and reliable high-level decision making. Existing methods often coordinate pretrained skills with scripted planners, finite-state machines or task-specific model-free policies, restricting their ability to handle complex task sequences. To address this limitation, we propose \\textbf{LUCID}, a hierarchical model-based reinforcement learning framework that plans over reusable skills through imagined rollouts of a learned dynamics model. LUCID first trains a structured latent-conditioned low-level policy via adversarial imitation and then freezes it while jointly learning a high-level policy and macro-dynamics world model. The world model predicts the temporally extended state transitions induced by latent decisions, enabling high-level policy optimization through imagined rollouts. We evaluate our framework across various simulated multi-object rearrangement scenarios. Experimental results show that LUCID improves the full-task success and partial-completion rates compared to prior baseline methods, demonstrating its effectiveness in complex sequential loco-manipulation tasks.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.07746v1_LUCID_Latent_Skill_Unified_Control_via_Imagined_Dynamics_for.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.07746", "fig": "figures/x-lucidlatentskillunifiedcontrolviaimagine.webp"}, {"id": "x-latentsemanticstateestimationforreliable", "title": "Latent Semantic State Estimation for Reliable Swarming of UAVs under Intermittent Connectivity", "tree": "rl", "branch": "marl", "authors": ["Paris A. Karakasis", "Walid Saad"], "first_author": "Paris A. Karakasis", "corresponding": [], "institutions": ["Virginia Tech"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-09", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.08895", "doi": "10.48550/arxiv.2608.08895", "url": "https://arxiv.org/abs/2608.08895", "code": "", "site": "", "abstract": "Cooperative multi-unmanned aerial vehicle (UAV) reconnaissance is often hindered by intermittent air-to-air communications where link dropouts lead to uncoordinated exploration and redundant mapping. Existing approaches rely on explicit exchange of high-dimensional spatial data or raw observations, incurring significant overhead, and often revert to reactive individual exploration during outages. This paper proposes a memory-augmented framework in which each UAV maintains a structured latent state decomposed into map, task, and memory components. During dropout, a generative predictor conditioned on the memory state infers substitute peer messages in the latent space, making the estimation task more tractable and directly aligned with the cooperative objective. The framework is trained end-to-end under the centralized training with decentralized execution paradigm. Simulation results demonstrate that the proposed framework closely matches the performance of a fully connected swarm, while remaining robust across a wide range of link failure conditions.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.08895v1_Latent_Semantic_State_Estimation_for_Reliable_Swarming_of_UA.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.08895", "fig": "figures/x-latentsemanticstateestimationforreliable.webp"}, {"id": "x-learningagilenavigationincrowdedenvironm", "title": "Learning Agile Navigation in Crowded Environments for Quadruped Robots", "tree": "rl", "branch": "locomotion", "authors": ["Shuyu Wu", "Zeyu Liu", "Tianbao Zhang", "Fanxing Li", "Fangyu Sun", "Mingkang Xiong", "Wei Xi", "W K Yu", "Danping Zou"], "first_author": "Shuyu Wu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-16", "topics": ["locomotion", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.15036", "doi": "10.48550/arxiv.2607.15036", "url": "https://arxiv.org/abs/2607.15036", "code": "", "site": "", "abstract": "Navigating dynamic and crowded environments presents significant challenges for quadruped robots due to severe sensor occlusion and unpredictable human motion. Existing approaches face a trade-off: model-based methods, such as Velocity Obstacles (VO), theoretically guarantee safety but rely on accurate obstacle motion estimates that often fail in dense crowds, while end-to-end learning methods offer robustness but lack motion prediction capability of obstacles, leading to collisions or conservative behaviors. To solve this, we propose VOP-Nav, a novel navigation system that combines the geometric safety of VO with the agile adaptability of end-to-end learning. Using only local onboard observations, our system avoids explicit obstacle detection and tracking pipelines. The VOP-Net processes multi-frame LiDAR data to implicitly encode dynamic constraints and predict a safe velocity region derived from Velocity Obstacle theory. Importantly, the VO predictions serve a dual role: they are used as input to the navigation policy during inference and as a reward signal during training to encourage safe motion. Evaluations in Isaac Gym demonstrate that VOP-Nav achieves higher success rates than all baselines while balancing locomotion speed and collision avoidance. Real-world deployment on a Unitree Go2 quadruped robot further validates the system's robustness and efficiency in complex indoor and outdoor dynamic environments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2607.15036v1_Learning_Agile_Navigation_in_Crowded_Environments_for_Quadru.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.15036", "fig": "figures/x-learningagilenavigationincrowdedenvironm.webp"}, {"id": "x-learningdynamicpickandplaceforaleggedman", "title": "Learning Dynamic Pick-and-Place for a Legged Manipulator", "tree": "rl", "branch": "loco-manip", "authors": ["Moonkyu Jung", "Jiseong Lee", "Zhengmao He", "Donghoon Youm", "Juhyeok Mun", "HyeongJun Kim", "Hyunsik Oh", "D CHOI", "Jungwoo Hur", "Jie Song", "Jemin Hwangbo"], "first_author": "Moonkyu Jung", "corresponding": [], "institutions": ["Korea Advanced Institute of Science and Technology"], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2026, "date": "2026-05-15", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.15713", "doi": "10.1109/LRA.2026.3688092", "url": "https://doi.org/10.1109/lra.2026.3688092", "code": "", "site": "", "abstract": "Legged manipulators extend robotic capabilities beyond static manipulation by integrating agile locomotion with versatile arm control. However, achieving precise manipulation while maintaining coordinated locomotion remains a major challenge. This work presents a hierarchical reinforcement learning framework for dynamic pick-and-place tasks using a quadruped equipped with a 6-DOF robotic arm. The framework incorporates an explicit mass estimation module enabling adaptive whole-body control for objects with varying weights. In simulation, the system achieves an 86.05% success rate with payloads up to 2.3 kg. The approach is further validated through real-world experiments across six representative scenarios with controlled variations in object physical properties (size and mass) and task heights. Specifically, within a wide vertical workspace ranging from ground level to 1.1~m-high tabletops, the system demonstrates an average success rate of 73.3% for payloads up to 1.3 kg, with an average execution time of 4.06 s. Unlike prior works that handle lightweight objects and execute pick-and-place motions with slow, piecewise motions, the proposed framework exploits concurrent locomotion and manipulation for dynamic, continuous execution. These results demonstrate the potential of quadrupedal mobile manipulators for adaptive, whole-body pick-and-place with heavier payloads and extended workspaces.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2605.15713v1_Learning_Dynamic_Pick_and_Place_for_a_Legged_Manipulator.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.15713", "fig": "figures/x-learningdynamicpickandplaceforaleggedman.webp"}, {"id": "x-learninglocomanipulationfromsmpcdemonstr", "title": "Learning Loco-Manipulation From SMPC Demonstrations With Sparse Offline-to-Online RL", "tree": "rl", "branch": "loco-manip", "authors": ["Martin Schuck", "Maks Sorokin", "Simone Manni", "Duy Ta", "Angela P. Schoellig", "Marco Hutter", "Simon Le Cleac'H", "Jan Brüdigam"], "first_author": "Martin Schuck", "corresponding": [], "institutions": ["ETH Zürich", "TU Munich"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-12", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.12063", "doi": "10.48550/arxiv.2608.12063", "url": "https://arxiv.org/abs/2608.12063", "code": "", "site": "", "abstract": "Integrating locomotion and manipulation is essential for robot autonomy, but scaling standard Reinforcement Learning (RL) to complex tasks is severely bottlenecked by the slow, manual process of dense reward shaping. To bypass this limitation, we leverage Sample-based Model Predictive Control (SMPC) entirely in simulation as an automated, rapidly tunable expert to generate massive offline datasets. Because this data solves the fundamental exploration problem, we can train an off-policy RL agent using purely sparse task rewards, drastically reducing the time required to learn new skills and eliminating the need for manual tuning. Integrating this high-level agent with a low-level dynamic stability controller yields more optimal behaviors that strictly align with true task objectives, ultimately allowing the learned policies to surpass the original optimal control teacher. We validate the robustness of this sim-to-real framework by successfully deploying complex loco-manipulation skills across different morphologies, including an arm-equipped Spot quadruped and a G1 humanoid.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.12063v1_Learning_Loco_Manipulation_From_SMPC_Demonstrations_With_Spa.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.12063", "fig": "figures/x-learninglocomanipulationfromsmpcdemonstr.webp"}, {"id": "x-learningpanoramaawarevlaformobilemanipul", "title": "Learning Panorama-Aware VLA for Mobile Manipulation with Whole-Body Teleoperation", "tree": "il", "branch": "vla", "authors": ["Donglin Yang", "Haoran Chen", "Xingyu Chen", "Lixing Liu", "Manyi Li", "Changhe Tu", "Ke Xu", "Xiaojian Ma", "Si Liu"], "first_author": "Donglin Yang", "corresponding": [], "institutions": ["Beihang University", "Johns Hopkins University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-03", "topics": ["VLA", "humanoid platforms", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.02257", "doi": "", "url": "https://arxiv.org/abs/2608.02257", "code": "", "site": "", "abstract": "Mobile manipulation is a key capability for embodied intelligence, enabling robots to accomplish complex multi-stage tasks in open-world environments. However, mobile manipulation poses two key challenges for vision-language-action (VLA) policies: At the data level, the efficient collection of high-quality whole-body demonstrations demands the coordinated control of both the mobile base and the robotic arms; at the model level, existing VLA models predominantly rely on local camera observations, whose limited field of view hinders global spatial understanding. To address these challenges, we develop a whole-body teleoperation system and a panoramic-aware VLA policy. The system enables coordinated control of a wheeled bimanual robot through a single VR interface and supports the acquisition of a real-world mobile manipulation dataset comprising 5.5 hours of multimodal demonstrations. Building upon this dataset, we propose PanoVLA, a panorama-aware vision-language-action policy for mobile bimanual manipulation. Built upon a Mixture-of-Transformers architecture, PanoVLA introduces global spatial context through dedicated panorama encoding and fusion modules, enabling effective integration of panoramic observations with language instructions and robot states for action generation. Evaluation on four real-world mobile manipulation tasks demonstrates that PanoVLA achieves an average stage completion rate of 91.3\\% and an end-to-end success rate of 73.4\\%, substantially outperforming local-view baselines. These results demonstrate that incorporating panoramic spatial context improves spatial understanding and closed-loop manipulation performance in mobile robots.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2608.02257v1_Learning_Panorama_Aware_VLA_for_Mobile_Manipulation_with_Who.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.02257", "fig": "figures/x-learningpanoramaawarevlaformobilemanipul.webp"}, {"id": "x-learningterrainawarewholebodycontrolforp", "title": "Learning Terrain-Aware Whole-Body Control for Perceptive Legged Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Sikai Guo", "Yudong Zhong", "Guoyang Zhao", "Botao Dang", "Zhihai Bi", "Jun Ma"], "first_author": "Sikai Guo", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2026, "date": "2026-05-29", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.31343", "doi": "", "url": "https://arxiv.org/abs/2605.31343", "code": "", "site": "", "abstract": "Legged manipulators integrate exceptional terrain adaptability along with mobile manipulation capabilities, which make them highly promising for deployment in human-centric environments. By coordinating the control of both legs and arms, a whole-body controller can significantly expand the operational workspace of legged manipulators. However, many existing whole-body controllers primarily depend on proprioception and do not incorporate the critical exteroception required for effective terrain topology perception. This limitation can hinder their ability to adapt to varying environmental conditions and navigate complex terrains effectively. In this paper, we introduce TA-WBC, a terrain-aware whole-body control framework for legged manipulators, which features a novel RL-based unified policy tailored to whole-body loco-manipulation tasks in various terrains. Specifically, we employ a \\rev{hierarchical exteroceptive encoder} to extract terrain features, providing an essential basis for the robot to proactively adapt posture and footholds. Furthermore, to facilitate stable cross-terrain loco-manipulation, we propose a novel end-effector sampling method based on the foot contact plane, \\rev{decoupling the manipulation target from base height, roll, and pitch variations}. Moreover, a dual-policy distillation module is introduced to integrate expansive whole-body motion with terrain adaptability without catastrophic forgetting. The simulation and real-world experiments validate the robustness of our proposed controller, which leads to a larger reachable space, less tracking error, and reduced unexpected stumbles. This unified policy highlights the promising capabilities of legged manipulators in performing loco-manipulation tasks across complex terrains.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2605.31343v1_Learning_Terrain_Aware_Whole_Body_Control_for_Perceptive_Leg.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.31343", "fig": "figures/x-learningterrainawarewholebodycontrolforp.webp"}, {"id": "x-mindcavsmultiintelligencenegotiationandd", "title": "MIND-CAVs: Multi-Intelligence Negotiation and Decision System for CAVs based on Intent-Driven Autonomy", "tree": "rl", "branch": "marl", "authors": ["Mainak Mondal", "Yihang Feng", "Yangchao Luo", "Han Song"], "first_author": "Mainak Mondal", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-16", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.14688", "doi": "", "url": "https://arxiv.org/abs/2607.14688", "code": "", "site": "", "abstract": "Modern autonomous vehicles largely operate as isolated agents: they rely on on-board perception and decision modules and broadcast Basic Safety Messages (BSMs) that expose only low-level kinematic state. While existing cooperative driving frameworks enable limited sensor sharing, they rarely communicate high-level maneuver intentions, and edge computing is primarily used for content delivery rather than decision arbitration. As a result, current connected autonomy lacks a principled mechanism for making globally consistent, intent-aware coordination decisions across vehicles. To address this gap, we propose MIND-CAVs, a Multi-Intelligence Negotiation and Decision framework for connected autonomous vehicles (CAVs) based on intent-driven autonomy. Each vehicle abstracts raw sensor observations into structured intent representations, exchanges them over V2X links, and receives globally consistent coordination plans from roadside edge servers. Edge agents combine learned and rule-based arbitration mechanisms to negotiate conflicting intents among vehicles, while a cloud platform records decisions for auditing and continual retraining. We implement MIND-CAVs in a CARLA-based AI-in-the-loop platform and evaluate it in multi-lane highway scenarios involving conflicting maneuvers and route-constrained exits. Experimental results show improved maneuver completion time and reduced unsafe proximity and unnecessary braking compared with isolated autonomy, first-come-first-served arbitration, and multi-agent reinforcement learning baselines.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2607.14688v2_MIND_CAVs_Multi_Intelligence_Negotiation_and_Decision_System.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.14688", "fig": "figures/x-mindcavsmultiintelligencenegotiationandd.webp"}, {"id": "modpack", "title": "ModPack: An Extensible Teleoperation Interface for Bimanual Mobile Manipulation", "tree": "systems", "branch": "interfaces", "authors": ["(see PDF)"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-19", "topics": ["modular teleop interface", "bimanual"], "paradigm": ["teleop"], "method": [], "projects": ["m2", "mabel"], "related": ["gello", "open-television", "telemoma"], "arxiv": "2607.19479", "doi": "10.48550/arxiv.2607.19479", "code": "", "site": "", "note": "From your M2 library. Modular teleop hardware — a build reference for MABEL.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Existing teleoperation systems are often tailored to specific robot hardware and task domains, limiting their scalability and adaptability. We present ModPack, a modular and extensible teleoperation system designed to support diverse robot embodiments and task requirements within a unified framework. At the core of ModPack is a self-contained wearable \"backpack\" that integrates onboard computation, power, communication, and data storage. Built on top of this shared interface, the system supports plug-and-play capability modules including joint-level teleoperation with haptic feedback, mobile manipulation, and active perception. Experiments across two distinct robot platforms and real-world mobile manipulation tasks demonstrate that ModPack provides a flexible and reusable framework for data collection and policy learning. To support future research, we open-source the complete hardware design and software stack. Project website: https://modpack-robotics.github.io/", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2607.19479", "arxiv_url": "https://arxiv.org/abs/2607.19479", "fig": "figures/modpack.webp", "figs": ["figures/modpack-1.webp"]}, {"id": "x-motionplanningindynamicenvironmentsasurv", "title": "Motion Planning in Dynamic Environments: A Survey from Classical to Modern Methods", "tree": "systems", "branch": "planning-control", "authors": ["Zongyuan Shen", "Yaming Ou", "Shalabh Gupta", "Shancheng Zhao", "Dehua Zhou", "Gao Wang", "Zhongqiang Ren", "Junfeng Fan", "Long Cheng"], "first_author": "Zongyuan Shen", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-01", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.02677", "doi": "", "url": "https://arxiv.org/abs/2606.02677", "code": "", "site": "", "abstract": "Motion planning in dynamic environments requires robots to continuously adapt their paths in response to environmental changes for safe and uninterrupted navigation. While many surveys have reviewed planning in static settings, systematic reviews focused on dynamic environments remain limited. This paper presents a comprehensive survey of 138 works, primarily published between 2015 and 2025, spanning both classical and learning-based approaches. The motion planning methods are grouped into five categories based on the concepts of sampling, graph search, model predictive control, learning, and additional classical local planning approaches, including velocity obstacles, potential fields and dynamic windows. The learning techniques include supervised learning and reinforcement learning. We also discuss the role of dynamic perception in motion planning, covering techniques for detecting and modeling moving obstacles using cameras, LiDAR, and event-based sensors. The survey analyzes the principles, strengths, and limitations of each method, with particular attention to challenges unique to dynamic environments, such as prediction uncertainty, human-robot interaction, and the freezing robot problem. The survey provides researchers with a structured understanding of motion planning methods in dynamic environments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2606.02677v1_Motion_Planning_in_Dynamic_Environments_A_Survey_from_Classi.pdf", "source": "library", "fig": "figures/x-motionplanningindynamicenvironmentsasurv.webp"}, {"id": "x-multirobotboxtransportoverdifferentsurfa", "title": "Multi-Robot Box Transport over Different Surfaces with Decentralized Role-based Proportional Control", "tree": "rl", "branch": "locomotion", "authors": ["Aditya Bhatt", "Himavarshini Yarragangu", "Urvish Shah", "Venkata Sai Yaswanth Mohan Thota", "Souma Chowdhury"], "first_author": "Aditya Bhatt", "corresponding": [], "institutions": ["NVIDIA"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-26", "topics": ["locomotion", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.26430", "doi": "", "url": "https://arxiv.org/abs/2605.26430", "code": "", "site": "", "abstract": "Collaborative transport of objects via pushing by multiple robots has many applications, ranging from construction and warehouse environments to post disaster debris clean-up. Achieving collaborative transport over surfaces with different inclination and friction properties however poses unique challenges. To address these challenges, this paper presents an asynchronous decentralized task and motion planning approach for transporting rectangular boxes of varying mass over flat, uphill and downhill terrain. Such a decentralized approach alleviates communication, synchronization and consensus needs and mitigates single point of failure issues. Our approach, called R2P2 or Roles with Rules and Proportional-control Primitive, assigns roles (e.g., push, support and prevent) to robots based on rules cognizant of the mode of manipulation needed (box rotation vs translation); this is followed by either rule-based control or proportional control of robot velocity based on the roles. Each robot is assumed to observe the location and heading of self and the box in executing the role and controls. R2P2 is evaluated with a six-robot team deployed in a simulator built using NVIDIA IsaacSim -- demonstrating generalizability across different surface friction/inclination and box mass scenarios, and better success rate compared to a standard virtual-leader-follower method. R2P2 is also successfully validated with a physical experiment, where it is executed onboard four turtlebots tasked with moving a 1.2 kg box.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2605.26430v2_Multi_Robot_Box_Transport_over_Different_Surfaces_with_Decen.pdf", "source": "library", "fig": "figures/x-multirobotboxtransportoverdifferentsurfa.webp"}, {"id": "x-orcaaplatformforopensourcedexterityresea", "title": "ORCA: A Platform for Open-Source Dexterity Research", "tree": "rl", "branch": "manipulation", "authors": ["Francesco Capuano", "Maximilian Eberlein", "Fabrice Bourquin", "Clemens Claudio Christoph"], "first_author": "Francesco Capuano", "corresponding": [], "institutions": ["ETH Zürich", "University of Oxford"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-12", "topics": ["dexterous manipulation", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2606.14561", "doi": "", "url": "https://arxiv.org/abs/2606.14561", "code": "", "site": "", "abstract": "Robotics manipulation research increasingly focuses on two-finger parallel grippers for their effectiveness, affordability, and ease of teleoperation. Grippers are nonetheless limited by their form factor, often requiring bimanual setups even for simple reorientation tasks. Anthropomorphic hands are a more natural platform for dexterous robot learning -- closer to the human hand, and capable of learning from human video -- yet they remain hard to use in learning research: even where open and accessible hand hardware exists, the software for control, simulation, teleoperation, and retargeting is scattered in one-off code bases, and largely disconnected from the robot-learning ecosystem. In this work, we introduce the \\orca~learning stack, an open-source research stack for dexterity as a first-class robot learning domain. Our \\orca~stack unifies low-level control, simulation, teleoperation from a range of consumer platforms, and hand retargeting, behind a single interface, and integrates natively with popular robot-learning frameworks such as \\lerobot, so dexterous hand researchers can leverage the same data, training, and evaluation pipelines used for non-dexterous robot learning. We demonstrate a complete end-to-end workflow, collecting expert demonstrations of an in-hand reorientation task by teleoperation with a consumer-grade VR headset, training an autonomous policy with \\lerobot, and evaluating the learned policy in a fully reproducible and observable setup. We open-source the entire stack as a shared, reproducible foundation for dexterous-manipulation research.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2606.14561v1_ORCA_A_Platform_for_Open_Source_Dexterity_Research.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.14561", "fig": "figures/x-orcaaplatformforopensourcedexterityresea.webp"}, {"id": "x-omnidirectionaldualarmaerialmanipulatorw", "title": "Omnidirectional Dual-Arm Aerial Manipulator with Proprioceptive Contact Localization for Landing on Slanted Roofs", "tree": "systems", "branch": "perception", "authors": ["Martijn B. J. Brummelhuis", "Nathan F. Lepora", "Salua Hamaza"], "first_author": "Martijn B. J. Brummelhuis", "corresponding": [], "institutions": ["TU Munich"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2026, "date": "2026-02-11", "topics": ["perception & estimation", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2", "wam-tactile"], "related": [], "arxiv": "2602.10703", "doi": "", "url": "https://arxiv.org/abs/2602.10703", "code": "", "site": "", "abstract": "Operating drones in urban environments often means they need to land on rooftops, which can have different geometries and surface irregularities. Accurately detecting roof inclination using conventional sensing methods, such as vision-based or acoustic techniques, can be unreliable, as measurement quality is strongly influenced by external factors including weather conditions and surface materials. To overcome these challenges, we propose a novel unmanned aerial manipulator morphology featuring a dual-arm aerial manipulator with an omnidirectional 3D workspace and extended reach. Building on this design, we develop a proprioceptive contact detection and contact localization strategy based on a momentum-based torque observer. This enables the UAM to infer the inclination of slanted surfaces blindly - through physical interaction - prior to touchdown. We validate the approach in flight experiments, demonstrating robust landings on surfaces with inclinations of up to 30.5 degrees and achieving an average surface inclination estimation error of 2.87 degrees over 9 experiments at different incline angles.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2026_2602.10703v1_Omnidirectional_Dual_Arm_Aerial_Manipulator_with_Propriocept.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2602.10703", "fig": "figures/x-omnidirectionaldualarmaerialmanipulatorw.webp"}, {"id": "x-onoptimaleventtriggereddistributedcontro", "title": "On Optimal Event-Triggered Distributed Control for Stochastic Multi-Agent Systems via Reinforcement Learning", "tree": "systems", "branch": "planning-control", "authors": ["Ziming Wang", "Bingbing Li", "Karl H. Johansson", "Apostolos I. Rikos"], "first_author": "Ziming Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2026, "date": "2026-07-20", "topics": ["planning & control", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.17635", "doi": "", "url": "https://arxiv.org/abs/2607.17635", "code": "", "site": "", "abstract": "We propose a reinforcement learning (RL) based optimal distributed control algorithm for the multi-agent systems (MASs) with stochastic uncertainties. Unlike existing methods, during the optimized backstepping design process, we use the actor-critic-identifier structure. The actor neural network is used to reflect control behavior, the critic neural network works to evaluate control performance and the unknown stochastic uncertainties are handled by identifier neural network. Furthermore, a low-pass filter effectively suppresses problems stemming from non-affine nonlinear faults and a hybrid event-triggered control (ETC) strategy is proposed to reduce control frequency. We analyze our algorithm's operation, and we provide a Lyapunov-based stability proof that guarantees all errors are bounded, ensuring precise tracking between the leader and followers. We validate its correctness in a single-axis robotic manipulator simulation and finally, we compare against the non-optimal control algorithm highlighting our optimal control algorithm's operational advantages.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2607.17635v1_On_Optimal_Event_Triggered_Distributed_Control_for_Stochasti.pdf", "source": "library", "fig": "figures/x-onoptimaleventtriggereddistributedcontro.webp"}, {"id": "x-onthegeneralizationcapabilitiesdesigncho", "title": "On the Generalization Capabilities, Design Choices and Limitations of Keypoint Imitation Learning", "tree": "il", "branch": "vla", "authors": ["Thomas Lips", "Marco Moletta", "Michael C. Welle", "Danica Kragic", "Francis wyffels"], "first_author": "Thomas Lips", "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2026, "date": "2026-05-26", "topics": ["VLA", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2605.26649", "doi": "", "url": "https://arxiv.org/abs/2605.26649", "code": "", "site": "", "abstract": "RGB-based imitation learning requires many demonstrations to generalize to unseen objects or scenes, motivating research into intermediate representations to improve generalization for robotic manipulation. Visual foundation models enable one-shot extraction of keypoints to provide such representation. However, it remains unclear how to integrate them into imitation learning optimally and when they outperform alternative representations. We combine approaches from previous works on keypoint imitation learning (KIL) and investigate several design choices to provide practical guidelines. Using over 2000 real-world rollouts, we also assess the generalization capabilities of KIL to unseen objects and scene variations. KIL achieves a 75% overall success rate across five tasks, significantly outperforming the RGB baseline (47%) and performing on par with S2-diffusion (73%). Finally, we explore the limitations of the foundation models used for keypoint extraction and extend KIL to tasks with multiple object instances. Our results confirm KIL as a data-efficient approach for robot learning, though it does not outperform alternative representations and inherits limitations of the foundation models used for keypoint extraction. All rollout videos, demonstrations, and results are available at https://kil-manipulation.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2026_2605.26649v1_On_the_Generalization_Capabilities_Design_Choices_and_Limita.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.26649", "fig": "figures/x-onthegeneralizationcapabilitiesdesigncho.webp"}, {"id": "x-openheartopeningheterogeneousarticulated", "title": "OpenHEART: Opening Heterogeneous Articulated Objects with a Legged Manipulator", "tree": "rl", "branch": "loco-manip", "authors": ["Seonghyeon Lim", "Hyeonwoo Lee", "Seunghyun Lee", "I Made Aswin Nahrendra", "Hyun Myung"], "first_author": "Seonghyeon Lim", "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2026, "date": "2026-03-06", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2603.05830", "doi": "", "url": "https://arxiv.org/abs/2603.05830", "code": "", "site": "", "abstract": "Legged manipulators offer high mobility and versatile manipulation. However, robust interaction with heterogeneous articulated objects, such as doors, drawers, and cabinets, remains challenging because of the diverse articulation types of the objects and the complex dynamics of the legged robot. Existing reinforcement learning (RL)-based approaches often rely on high-dimensional sensory inputs, leading to sample inefficiency. In this paper, we propose a robust and sample-efficient framework for opening heterogeneous articulated objects with a legged manipulator. In particular, we propose Sampling-based Abstracted Feature Extraction (SAFE), which encodes handle and panel geometry into a compact low-dimensional representation, improving cross-domain generalization. Additionally, Articulation Information Estimator (ArtIEst) is introduced to adaptively mix proprioception with exteroception to estimate opening direction and range of motion for each object. The proposed framework was deployed to manipulate various heterogeneous articulated objects in simulation and real-world robot systems. Videos can be found on the project website: https://openheart-icra.github.io/OpenHEART/", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2603.05830v1_OpenHEART_Opening_Heterogeneous_Articulated_Objects_with_a_L.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2603.05830", "fig": "figures/x-openheartopeningheterogeneousarticulated.webp"}, {"id": "x-paintpartneragnosticintentawarecooperati", "title": "PAINT: Partner-Agnostic Intent-Aware Cooperative Transport with Legged Robots", "tree": "rl", "branch": "loco-manip", "authors": ["Zhihao Cao", "Tianxu An", "Chenhao Li", "Stelian Coros", "Marco Hutter"], "first_author": "Zhihao Cao", "corresponding": [], "institutions": ["ETH Zürich"], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2026, "date": "2026-04-14", "topics": ["loco-manipulation", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2604.12852", "doi": "10.1109/lra.2026.3730304", "url": "https://doi.org/10.1109/lra.2026.3730304", "code": "", "site": "", "abstract": "Collaborative transport requires robots to infer partner intent through physical interaction while maintaining stable loco-manipulation. This becomes particularly challenging in complex environments, where interaction signals are difficult to capture and model. We present PAINT, a hierarchical learning framework for partner-agnostic intent-aware collaborative legged transport that represents partner intent as an explicit interaction wrench and recovers it from payload-coupled proprioceptive histories. PAINT decouples intent understanding from terrain-robust locomotion: A high-level policy uses the inferred interaction wrench for transport and reconstructs it through teacher-student training, while a low-level locomotion backbone ensures robust execution. This enables lightweight deployment without external force-torque sensing or payload tracking. Extensive simulation and real-world experiments demonstrate compliant cooperative transport across diverse terrains, payloads, and partners. Furthermore, we show that PAINT can reuse the same single-agent policy for decentralized multi-robot transport by mechanically combining team interactions into a common wrench space, and supports heterogeneous-team transport. Our results suggest that payload-coupled proprioceptive interaction provides a physically grounded interface for partner-agnostic intent-aware collaborative transport.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2604.12852v1_PAINT_Partner_Agnostic_Intent_Aware_Cooperative_Transport_wi.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2604.12852", "fig": "figures/x-paintpartneragnosticintentawarecooperati.webp"}, {"id": "pds-joint", "title": "PDS Joint: A Parametric Double-Spiral Joint Tailored for Dexterous Hands", "tree": "systems", "branch": "hands", "authors": ["(see PDF)"], "corresponding": [], "institutions": ["Beijing Institute of Technology"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-24", "topics": ["joint mechanism", "dexterous hand design"], "paradigm": ["hardware"], "method": [], "projects": ["m2"], "related": ["midas-hand", "orca-hand"], "arxiv": "2606.24377", "doi": "", "code": "", "site": "", "note": "From your M2 library. Mechanism-level design detail for high-DOF hands.", "aka": "", "stars": 1, "curated": true, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2606.24377v1_PDS_Joint_A_Parametric_Double_Spiral_Joint_Tailored_for_Dext.pdf", "abstract": "Compliant joints can embed safety and adaptability into dexterous hands, but achieving large-stroke anthropomorphic motion while maintaining joint-specific, directiondependent stiffness and reliable proprioception remains challenging. This paper presents the PDS joint, a parametric doublespiral (PDS) compliant joint that enables systematic shaping of directional stiffness across multiple deformation modes, including flexion/extension, abduction/adduction, and pronation/supination. We instantiate the joint using Archimedean and logarithmic spiral templates for different hand joints and introduce an asymmetry ratio to tailor stiffness distributions for both grasp stability and hyperextension resistance. To make the joint practically usable under large deformation, we co-design embedded inductive proprioception and propose a learningbased calibration pipeline that maps raw inductive signals to joint states using ArUco-marker tracking. Experiments characterize the stiffness landscapes across geometric parameters and demonstrate a non-monotonic dependence of lateral support on asymmetry, indicating the importance of principled parameter tuning. For joint-state estimation in the most challenging abduction/adduction motion, a learned multilayer-perceptron (MLP) mapping reduces the error compared with conventional curve fitting by 41.6%. Finally, we integrate the proposed joints into an open-source dexterous hand as a demonstration platform, on which the hand grasps a set of nine everyday objects and performs safe, contact-rich human-involved interactions.", "source": "curated", "first_author": "(see PDF)", "url": "https://arxiv.org/abs/2606.24377", "arxiv_url": "https://arxiv.org/abs/2606.24377", "fig": "figures/pds-joint.webp", "figs": ["figures/pds-joint-1.webp", "figures/pds-joint-2.webp"]}, {"id": "x-pimbotaselfadaptiveattackframeworkforadv", "title": "PIMbot: A Self-Adaptive Attack Framework for Adversarial Manipulation of Multi-Robot Reinforcement Learning", "tree": "rl", "branch": "marl", "authors": ["Zexin Li", "Ziliang Zhang", "Hyoseung Kim", "Cong Liu"], "first_author": "Zexin Li", "corresponding": [], "institutions": ["NVIDIA"], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2026, "date": "2026-05-21", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.23027", "doi": "", "url": "https://arxiv.org/abs/2605.23027", "code": "", "site": "", "abstract": "Recent research has demonstrated the potential of reinforcement learning in effective multi-robot collaboration, particularly in social dilemmas where robots face a trade-off between self-interest and collective benefits. However, environmental factors such as miscommunication and adversarial robots can impact cooperation, making it crucial to explore how multi-robot communication can be manipulated to achieve different outcomes. This paper presents PIMbot, a framework that manipulates outcomes via two complementary levers: (i) incentive manipulation of the reward channel and (ii) policy manipulation of an agent's own actions. An adaptive multi-objective controller balances these levers in an online manner. Our work introduces a novel approach to manipulation in recent multi-agent RL social dilemmas that utilize a unique reward function for incentivization. By utilizing our proposed PIMbot mechanisms, a robot is able to manipulate the social dilemma environment effectively. Comprehensive experimental results demonstrate the effectiveness of our proposed methods in the Gazebo-simulated multi-robot environment. Moreover, a real embedded device case study on NVIDIA Jetson Orin Nano quantifies system cost and validates PIMbot's effectiveness on realistic autonomous embedded systems scenarios beyond simulation. Together, these results position PIMbot as a rigorous stress-test tool exposing critical vulnerabilities in multi-robot cooperative tasks.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2605.23027v1_PIMbot_A_Self_Adaptive_Attack_Framework_for_Adversarial_Mani.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.23027", "fig": "figures/x-pimbotaselfadaptiveattackframeworkforadv.webp"}, {"id": "x-paralleldynamicprogrammingforconiclinear", "title": "Parallel Dynamic Programming for Conic Linear Quadratic Control", "tree": "systems", "branch": "planning-control", "authors": ["Luyao Zhang", "Gabriel Bravo-Palacios", "Brian Plancher", "Sergio Grammatico"], "first_author": "Luyao Zhang", "corresponding": [], "institutions": ["Columbia University", "TU Delft"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-23", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.24632", "doi": "", "url": "https://arxiv.org/abs/2606.24632", "code": "", "site": "", "abstract": "Linear Quadratic (LQ) control problems are at the heart of linear control theory and Model Predictive Control (MPC). While performant, standard approaches to solving such problems are inherently serial, limiting real-time scalability despite the parallel computing power available on modern multi-core CPUs. Contributing to addressing this challenge and motivated by ``divide and conquer'' strategies, we present a parallel-in-time approach that solves computationally demanding conic optimal control problems through the use of the alternating direction method of multipliers (ADMM). In particular, we formulate the inner primal update of ADMM as an LQ problem and split the reformulated problem along the time horizon. This enables us to derive a variant of the Riccati recursion using dynamic programming to solve each subproblem in parallel. Numerical benchmarks on two real-world applications demonstrate as much as a 5x speedup compared to existing related approaches on multi-core CPU hardware.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2606.24632v1_Parallel_Dynamic_Programming_for_Conic_Linear_Quadratic_Cont.pdf", "source": "library", "fig": "figures/x-paralleldynamicprogrammingforconiclinear.webp"}, {"id": "x-realtimeprojectedadaptivecontrolforclose", "title": "Real-Time Projected Adaptive Control for Closed-Chain Co-Manipulative Continuum Robots", "tree": "systems", "branch": "sim-bench", "authors": ["Rana Danesh", "Farrokh Janabi-Sharifi", "Farhad Aghili"], "first_author": "Rana Danesh", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-05", "topics": ["simulation & benchmarks", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2604.04286", "doi": "", "url": "https://arxiv.org/abs/2604.04286", "code": "", "site": "", "abstract": "In co-manipulative continuum robots (CCRs), multiple continuum arms cooperate by grasping a common flexible object, forming a closed-chain deformable mechanical system. The closed-chain coupling induces strong dynamic interactions and internal reaction forces. Moreover, in practical tasks, the flexible object's physical parameters are often unknown and vary between operations, rendering nominal model-based controllers inadequate. This paper presents a projected adaptive control framework for CCRs formulated at the dynamic level. The coupled dynamics are expressed using the Geometric Variable Strain (GVS) representation, yielding a finite-dimensional model that accurately represents the system, preserves the linear-in-parameters structure required for adaptive control, and is suitable for real-time implementation. Closed-chain interactions are enforced through Pfaffian velocity constraints, and an orthogonal projection is used to express the dynamics in the constraint-consistent motion subspace. Based on the projected dynamics, an adaptive control law is developed to compensate online for uncertain dynamic parameters of both the continuum robots and the manipulated flexible object. Lyapunov analysis establishes closed-loop stability and convergence of the task-space tracking errors to zero. Simulation and experiments on a tendon-driven CCR platform validate the proposed framework in task-space regulation and trajectory tracking.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2604.04286v1_Real_Time_Projected_Adaptive_Control_for_Closed_Chain_Co_Man.pdf", "source": "library", "fig": "figures/x-realtimeprojectedadaptivecontrolforclose.webp"}, {"id": "x-roboreactagenticskilldistillationfromgen", "title": "RoboReact: Agentic Skill Distillation from Generated Egocentric Videos for Generalizable Whole-Body Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Shuliang He", "Shuai Wang", "Bo Yue", "Junchi Teng", "Changyu Wang", "Guiliang Liu"], "first_author": "Shuliang He", "corresponding": [], "institutions": ["Tsinghua University", "CUHK", "HKU"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-04", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.03387", "doi": "", "url": "https://arxiv.org/abs/2608.03387", "code": "", "site": "", "abstract": "Humanoid robots have the potential to perform dexterous manipulation in human environments, yet acquiring diverse and generalizable skills remains costly due to expensive hardware data collection and labor-intensive annotation. Recent advances in video generative models provide a promising opportunity to synthesize rich manipulation experiences from visual observations, but transferring such imagined behaviors into executable whole-body humanoid skills remains largely unexplored. In this work, we present RoboReact, a framework that automatically synthesizes whole-body humanoid manipulation skills from a single egocentric RGB-D observation. RoboReact generates human manipulation videos, extracts geometry-preserving interaction keyframes through depth-aware 3D reconstruction, and retargets them to high-DoF humanoid platforms while preserving hand-object interaction geometry. To bridge the gap between imagined plans and physical execution, RoboReact performs online object-centric re-grounding and leverages a vision-language model-guided refinement loop to adapt skills under geometric mismatch and execution deviations. The refined skills are executed through a whole-body controller, enabling coordinated whole-body manipulation and dexterous interaction. Experiments on real humanoid robots demonstrate that RoboReact generalizes across diverse object configurations and robustly recovers from execution disturbances without requiring teleoperation or human demonstrations. These results highlight the potential of combining generative models, vision-language reasoning, and closed-loop control for scalable humanoid skill acquisition.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.03387v2_RoboReact_Agentic_Skill_Distillation_from_Generated_Egocentr.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.03387", "fig": "figures/x-roboreactagenticskilldistillationfromgen.webp"}, {"id": "x-robostrikerlatentspacestrategicgamesfora", "title": "RoboStriker: Latent-Space Strategic Games for Autonomous Humanoid Boxing", "tree": "rl", "branch": "manipulation", "authors": ["Kangning Yin", "Kaige Liu", "Zhe Cao", "Wentao Dong", "Weishuai Zeng", "Tianyi Zhang", "Qiang Zhang", "Jingbo Wang", "Jiangmiao Pang", "Yang Li", "Ming Zhou", "Weinan Zhang"], "first_author": "Kangning Yin", "corresponding": [], "institutions": ["Shanghai Jiao Tong University", "Shanghai AI Laboratory", "USTC"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-17", "topics": ["dexterous manipulation", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.16195", "doi": "10.48550/arxiv.2608.16195", "url": "https://arxiv.org/abs/2608.16195", "code": "", "site": "", "abstract": "Achieving human-level competitive intelligence and physical agility in humanoid robots remains a profound challenge, particularly in contact-rich and highly dynamic tasks such as boxing. While Multi-Agent Reinforcement Learning offers a principled framework for strategic interaction, its direct application to unstructured raw motor spaces inevitably leads to joint-level physical collapse, preventing the emergence of any viable combat tactics. To resolve this fundamental conflict between strategic exploration and physical feasibility, we formulate the humanoid combat task as a novel two-player latent-space zero-sum Markov game. Under standard regularity and approximate best-response assumptions, we show that the latent formulation induces an equivalent game over the decoder-reachable action manifold, providing an approximate-Nash interpretation of the resulting self-play dynamics. To instantiate this theoretical formulation, we propose RoboStriker, a hierarchical framework that decouples high-level reasoning from low-level execution. It first distills the tracking expertise of predefined boxing motions into a topologically bounded latent manifold. This structured latent foundation subsequently drives multi-agent co-evolution via Latent-Space Neural Fictitious Self-Play. Extensive experimental results demonstrate that gaming within this structured latent space substantially outperforms direct exploration. By constraining strategic exploration through a pretrained motion decoder, RoboStriker substantially reduces the catastrophic balance failures observed in raw action-space methods and achieves superior tactical performance in both competitive win rates and striking efficiency. Finally, we successfully deploy and validate our learned combat policies on real-world humanoid robots. Our code and video and supplementary materials are available at RoboStriker.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.16195v1_RoboStriker_Latent_Space_Strategic_Games_for_Autonomous_Huma.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.16195", "fig": "figures/x-robostrikerlatentspacestrategicgamesfora.webp"}, {"id": "x-robotsthatcollaboratesequentialasymmetri", "title": "Robots that Collaborate: Sequential Asymmetric Imitation for Learning Coupled Robot Policies", "tree": "il", "branch": "retargeting", "authors": ["Yincong Chen", "Ranpeng Qiu", "Zihao Li", "Yanan Zhou", "Guoqiang Ren", "Weiming Zhi"], "first_author": "Yincong Chen", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-15", "topics": ["humanoid platforms", "mobile bimanual platforms", "teleoperation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.16490", "doi": "10.48550/arxiv.2606.16490", "url": "https://arxiv.org/abs/2606.16490", "code": "", "site": "", "abstract": "Collaborative mobile manipulation requires robots to coordinate with a partially observed partner while physically interacting through shared objects. This is difficult because failures often arise not from poor local skills, but from mistimed waiting, yielding, pulling, releasing, or repositioning. We study this problem with two bimanual mobile manipulators coupled through rigid and deformable objects. We propose Sequential Asymmetric Imitation (SAI), a single-teleoperator curriculum for learning coupled multi-robot behaviors without synchronized dual-operator demonstrations or explicit inter-robot communication. SAI trains Robot A from unilateral demonstrations with a compliant human partner, trains Robot B against the deployed Robot A policy, and then refines Robot A using sparse interventions near coordination failures. This staged process exposes the policies to increasingly realistic partner behaviors, including delay, phase mismatch,insufficient yielding, and interaction conflict. Across real-world dual-robot manipulation tasks, SAI improves task success, phase synchronization, and partner-contingent yielding over independent imitation and curriculum-ablation baselines. These results suggest that physically coupled collaboration can be learned through the structure of the imitation curriculum, rather than through synchronized multi-operator demonstrations or explicit coordination mechanisms. More videos on project page:http://cyc0429.github.io/sai-project-page/", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2606.16490v1_Robots_that_Collaborate_Sequential_Asymmetric_Imitation_for.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.16490", "fig": "figures/x-robotsthatcollaboratesequentialasymmetri.webp"}, {"id": "x-safepolytopeinpolytopemotionplanningandc", "title": "Safe Polytope-in-Polytope Motion Planning and Control with Control Barrier Functions", "tree": "systems", "branch": "planning-control", "authors": ["Alejandro González-García", "Dries Dirckx", "Jan Swevers", "Wilm Decré"], "first_author": "Alejandro González-García", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-08", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.09719", "doi": "", "url": "https://arxiv.org/abs/2606.09719", "code": "", "site": "", "abstract": "Autonomous mobile robots operating in tight environments require motion planning frameworks that account for the physical footprint of the robot. Simplifying the geometry to a point or a circle is conservative and discards information needed to successfully and safely traverse narrow passages. This work proposes a safe local motion planning and control method that guarantees that a polytopic robot footprint stays inside a continuously updated convex free-space region. The containment condition is formulated as a set of discrete-time control barrier function constraints within a model predictive controller. The number of safety constraints depends on the complexity of the local free-space geometry and the robot shape, instead of the number of obstacles. The proposed free-space formulation does not need any obstacle detection or segmentation. A comparative analysis against a polytope-based obstacle avoidance formulation confirms favorable scaling up to a reduction of 91$\\times$ in computation time as the number of obstacles increases. The approach is validated in simulation with an autonomous surface vehicle and on hardware with a non-holonomic mobile robot, using both occupancy grids and LiDAR sensing. The experiments demonstrate safe real-time motion planning and control at 10~Hz on an onboard embedded computer, including reactive avoidance of dynamic obstacles.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2606.09719v1_Safe_Polytope_in_Polytope_Motion_Planning_and_Control_with_C.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.09719", "fig": "figures/x-safepolytopeinpolytopemotionplanningandc.webp"}, {"id": "x-scaleinvariantsamplinginmultiarmbanditmo", "title": "Scale-Invariant Sampling in Multi-Arm Bandit Motion Planning for Object Extraction", "tree": "systems", "branch": "planning-control", "authors": ["Servet B. Bayraktar", "Andreas Orthey", "Marc Toussaint"], "first_author": "Servet B. Bayraktar", "corresponding": [], "institutions": ["Technische Universität Berlin"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-15", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2604.14026", "doi": "", "url": "https://arxiv.org/abs/2604.14026", "code": "", "site": "", "abstract": "Object extraction tasks often occur in disassembly problems, where bolts, screws, or pins have to be removed from tight, narrow spaces. In such problems, the distance to the environment is often on the millimeter scale. Sampling-based planners can solve such problems and provide completeness guarantees. However, sampling becomes a bottleneck, since almost all motions will result in collisions with the environment. To overcome this problem, we propose a novel scale-invariant sampling strategy which explores the configuration space using a grow-shrink search to find useful, high-entropy sampling scales. Once a useful sampling scale has been found, our framework exploits this scale by using a principal components analysis (PCA) to find useful directions for object extraction. We embed this sampler into a multi-arm bandit rapidly-exploring random tree (MAB-RRT) planner and test it on eight challenging 3D object extraction scenarios, involving bolts, gears, rods, pins, and sockets. To evaluate our framework, we compare it with classical sampling strategies like uniform sampling, obstacle-based sampling, and narrow-passage sampling, and with modern strategies like mate vectors, physics-based planning, and disassembly breadth first search. Our experiments show that scale-invariant sampling improves success rate by one order of magnitude on 7 out of 8 scenarios. This demonstrates that scale-invariant sampling is an important concept for general purpose object extraction in disassembly tasks.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2604.14026v1_Scale_Invariant_Sampling_in_Multi_Arm_Bandit_Motion_Planning.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2604.14026", "fig": "figures/x-scaleinvariantsamplinginmultiarmbanditmo.webp"}, {"id": "x-shapeformationforthecooperativetransport", "title": "Shape Formation for the Cooperative Transportation of Arbitrary Objects Using Multi-Agent Reinforcement Learning", "tree": "rl", "branch": "marl", "authors": ["Mohamed Sayed", "Wolfram Burgard", "Tanja Katharina Kaiser"], "first_author": "Mohamed Sayed", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-08", "topics": ["multi-agent", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.09610", "doi": "", "url": "https://arxiv.org/abs/2606.09610", "code": "", "site": "", "abstract": "Cooperative object transportation is essential in numerous domains, including industrial to domestic services. A popular transportation strategy is to carry objects on top of multi-robot systems. The corresponding task is typically solved by decomposing it into three interconnected subproblems: formation control, cooperative navigation, and collision avoidance. A particular challenge posed by real-world objects is their potentially arbitrary shape and non-uniform mass distribution, necessitating robot formations that securely support the object. In this work, we address the challenge of pattern formation control for transporting such real-world objects by proposing a novel multi-agent reinforcement learning approach. Our approach enables a multi-robot system to autonomously position itself underneath an object to support its weight while avoiding obstacles during the formation process. Our evaluations with diverse environments and varying numbers of robots show that our approach leads to policies that reliably produce balanced formations and generalize to cluttered scenes and objects with complex geometry and non-uniform mass distribution.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2606.09610v1_Shape_Formation_for_the_Cooperative_Transportation_of_Arbitr.pdf", "source": "library", "fig": "figures/x-shapeformationforthecooperativetransport.webp"}, {"id": "x-sim2seasimtorealpolicytransferformaritim", "title": "Sim2Sea: Sim-to-Real Policy Transfer for Maritime Vessel Navigation in Congested Waters", "tree": "systems", "branch": "sim-bench", "authors": ["Xinyu Cui", "Xuanfa Jin", "Xue Yan", "Yongcheng Zeng", "Luoyang Sun", "Siying Wei", "Ruizhi Zhang (752030)", "Jian Zhao", "Hailong Zhang", "Yi-Xiang Wang"], "first_author": "Xinyu Cui", "corresponding": [], "institutions": ["UCL"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-04", "topics": ["simulation & benchmarks", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r", "wam-tactile"], "related": [], "arxiv": "2603.04057", "doi": "10.65109/CMRP6518", "url": "https://arxiv.org/abs/2603.04057", "code": "", "site": "", "abstract": "Autonomous navigation in congested maritime environments is a critical capability for a wide range of real-world applications. However, it remains an unresolved challenge due to complex vessel interactions and significant environmental uncertainties. Existing methods often fail in practical deployment due to a substantial sim-to-real gap, which stems from imprecise simulation, inadequate situational awareness, and unsafe exploration strategies. To address these, we propose \\textbf{Sim2Sea}, a comprehensive framework designed to bridge simulation and real-world execution. Sim2Sea advances in three key aspects. First, we develop a GPU-accelerated parallel simulator for scalable and accurate maritime scenario simulation. Second, we design a dual-stream spatiotemporal policy that handles complex dynamics and multi-modal perception, augmented with a velocity-obstacle-guided action masking mechanism to ensure safe and efficient exploration. Finally, a targeted domain randomization scheme helps bridge the sim-to-real gap. Simulation results demonstrate that our method achieves faster convergence and safer trajectories than established baselines. In addition, our policy trained purely in simulation successfully transfers zero-shot to a 17-ton unmanned vessel operating in real-world congested waters. These results validate the effectiveness of Sim2Sea in achieving reliable sim-to-real transfer for practical autonomous maritime navigation.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2603.04057v1_Sim2Sea_Sim_to_Real_Policy_Transfer_for_Maritime_Vessel_Navi.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2603.04057", "fig": "figures/x-sim2seasimtorealpolicytransferformaritim.webp"}, {"id": "x-sociallyconsistentmultirobotnavigationus", "title": "Socially Consistent Multi-Robot Navigation Using Decoupled Planning and Trajectory Coordination", "tree": "rl", "branch": "marl", "authors": ["Matthew M. Sato", "Kincho H. Law"], "first_author": "Matthew M. Sato", "corresponding": [], "institutions": ["Stanford University"], "lab": "", "venue": "Civil Engineering Sciences", "venue_type": "preprint", "year": 2026, "date": "2026-07-22", "topics": ["multi-agent", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.20772", "doi": "10.34133/cesci.0025", "url": "https://doi.org/10.34133/cesci.0025", "code": "", "site": "", "abstract": "The successful integration of mobile robots in human-centric environments requires navigation that is not only safe and efficient, but also consistent and compliant with social conventions: key precursors for human comfort and acceptance. Most human-aware navigation research targets the local planner, generating short-horizon, sensing-dependent reactive behavior around nearby humans. However, some social conventions are long-horizon routing choices that are better established at the global planning level, where the positions of individual humans are either unobservable or likely to be stale by the time the robot arrives. We develop a partially decentralized global planning system that encodes social conventions and coordinates multiple robots, complementing local planners that react to dynamically sensed humans. First, we introduce a general mechanism for embedding sensing-independent social conventions into a modified A* cost function. Planned paths are shared across the fleet to collaboratively build a shared social graph of established routes, enforcing path consistency and reducing future planning effort. Second, we leverage the resulting structure of these socially consistent paths to formulate multi-robot trajectory coordination as a mixed-integer convex program. While the underlying optimization is solved centrally, the constraint generation is distributed across the fleet, enabling efficient computation of conflict-free trajectories. Through simulation and hardware experiments, we demonstrate that viewing global social path planning and multi-robot trajectory coordination as a single system, rather than independent problems, produces socially consistent, repeatable paths and simplifies multi-robot coordination.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2607.20772v1_Socially_Consistent_Multi_Robot_Navigation_Using_Decoupled_P.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.20772", "fig": "figures/x-sociallyconsistentmultirobotnavigationus.webp"}, {"id": "x-structuredllmreasoningforzeroshothumanro", "title": "Structured LLM Reasoning for Zero-Shot Human--Robot Coordination Under Hidden Goals", "tree": "rl", "branch": "marl", "authors": ["Dong Hae Mangalindan", "Anand Gokhale", "Francesco Bullo", "Vaibhav Srivastava"], "first_author": "Dong Hae Mangalindan", "corresponding": [], "institutions": ["Michigan State University", "University of California, Santa Barbara"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-05", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.04309", "doi": "", "url": "https://arxiv.org/abs/2608.04309", "code": "", "site": "", "abstract": "We present a structured large-language-model (LLM) architecture for zero-shot human--robot coordination in a cooperative construction task with private goal views. Guided by a Dec-POMDP formulation, the architecture decomposes decision-making into (i) action-conditioned Theory-of-Mind (ToM) inference, (ii) hierarchical planning, (iii) conversation interpretation, (iv) action verification, and (v) feedback-based replanning. We compare the proposed method with an ablation without ToM inference and a multi-agent reinforcement-learning policy trained offline over many goal pairs. In human-participant experiments, the proposed method required fewer interaction steps and yielded higher post-interaction trust ratings than both baselines. These results suggest that systematically decomposing the team decision problem, using LLMs as tractable surrogates for otherwise intractable inference and planning computations, and retaining conventional verification for physical feasibility can improve both task coordination and the human experience.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.04309v1_Structured_LLM_Reasoning_for_Zero_Shot_Human_Robot_Coordinat.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.04309", "fig": "figures/x-structuredllmreasoningforzeroshothumanro.webp"}, {"id": "x-submodularpolicylearningfordistributedta", "title": "Submodular Policy Learning for Distributed Task Allocation in Open Multi-Agent Systems", "tree": "systems", "branch": "sim-bench", "authors": ["Jing Liu", "Luca Ballotta", "Yangyang Yang", "Fangfei Li", "Yang Tang", "Ruggero Carli"], "first_author": "Jing Liu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-14", "topics": ["simulation & benchmarks", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.14390", "doi": "10.48550/arxiv.2608.14390", "url": "https://arxiv.org/abs/2608.14390", "code": "", "site": "", "abstract": "This paper studies policy learning for distributed task allocation in open multi-agent systems, where agents may join and leave in a time-varying fashion, with submodular stage team utilities. At each time, the active agents select actions from local categorical policies such that the feasible joint agent-action pairs form a partition matroid. Standard continuous relaxations of submodular set functions are based on independent Bernoulli sampling, making them inconsistent with agents' policies.To solve this mismatch, we propose the \\emph{partition multilinear extension} (PME), a policy-based relaxation whose continuous support matches feasible actions under categorical policies.We prove that the marginal gains of the stage utility provide an unbiased estimator of the gradient of the PME and that maximizing the PME over action distributions is equivalent to maximizing the stage utilities over agent actions, which are critical to devise principled policy gradient.Building on this, we design \\emph{SubMAPL}, a centralized-training decentralized-execution KL-mirror policy-learning method that uses local marginal gains as stochastic PME gradients during training. KL-mirror updates preserve categorical feasibility without Euclidean projection.In the case where agents run tabular-softmax policies, we introduce open policy migration and an open-system KL tracking variation to handle agent arrivals and departures. Using dynamic regret analysis, we establish a lower bound on the cumulative utility which accounts for the openness of the environment and for the gap between optimal stage-wise and global utilities. Simulations on multi-agent coverage demonstrate that SubMAPL outperforms policy-gradient and online-learning baselines.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.14390v1_Submodular_Policy_Learning_for_Distributed_Task_Allocation_i.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.14390", "fig": "figures/x-submodularpolicylearningfordistributedta.webp"}, {"id": "x-thrivetherapeutichumanoidrobotinvirtuale", "title": "THRIVE: Therapeutic Humanoid Robot In Virtual Environment", "tree": "systems", "branch": "humanoid-hw", "authors": ["Jin Xu", "Yuping Chen", "Ayanna Howard"], "first_author": "Jin Xu", "corresponding": [], "institutions": ["Georgia Tech"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-14", "topics": ["wearable robotics", "humanoid platforms"], "paradigm": [], "method": [], "projects": ["exo", "m2"], "related": [], "arxiv": "2608.14462", "doi": "10.48550/arxiv.2608.14462", "url": "https://arxiv.org/abs/2608.14462", "code": "", "site": "", "abstract": "This paper presents THRIVE (Therapeutic Humanoid Robot In Virtual Environment), an at-home rehabilitation platform that integrates a suite of virtual-reality upper-body rehabilitation games, a real-time camera-based motion-tracking system, and a socially interactive robot therapist. The system is designed for therapy and intervention in children with upper-limb motor impairments, which can be improved through consistent, task-specific practice. THRIVE features a set of newly designed, engaging games that target functional reaching, grasping, and object-manipulation movements through customizable popping, hitting, catching, and grabbing tasks, while the camera-based tracking system captures the child's kinematic performance during play. A robot therapist - deployable either as a physical robotic coach or as a remote-presence virtual agent - delivers adaptive, dynamic feedback to motivate the child and guide their movements toward therapeutic goals. THRIVE decouples the therapeutic games from the robot embodiment, extending the platform to support various embodiments and different robots within one modular system. This robot-agnostic design makes THRIVE affordable, scalable, and readily adaptable for sustained use in the home, offering a practical pathway to more consistent and engaging upper-limb therapy for children with motor function impairments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2608.14462v1_THRIVE_Therapeutic_Humanoid_Robot_In_Virtual_Environment.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.14462", "fig": "figures/x-thrivetherapeutichumanoidrobotinvirtuale.webp"}, {"id": "x-tac4locolearningspatiotemporalplantarpre", "title": "Tac4Loco: Learning Spatiotemporal Plantar Pressure Representations for Humanoid Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Ziyun Liu", "Sikai Guo", "Zheng Li", "Jiahang Cao", "Haichao Liu", "Pei Qu", "Yinghong Zhang", "Jinni Zhou", "Jun Ma"], "first_author": "Ziyun Liu", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2026, "date": "2026-08-16", "topics": ["locomotion", "humanoid platforms"], "paradigm": [], "method": [], "projects": ["m2", "wam-tactile"], "related": [], "arxiv": "2608.15766", "doi": "10.48550/arxiv.2608.15766", "url": "https://arxiv.org/abs/2608.15766", "code": "", "site": "", "abstract": "Humanoid robots are expected to traverse complex terrains, where the plantar support may vary dramatically due to foot placement errors, ground properties, and transient dynamics. To achieve robust locomotion, the robots are required to adapt to uneven terrain and uncertain foot--ground interactions. Existing locomotion policies rely primarily on proprioception or exteroceptive terrain perception, where the former provides only indirect evidence of plantar support, while the latter predicts contact conditions before touchdown but cannot observe the actual support in real-time. Although some studies incorporate plantar contacts as an auxiliary perception, they rely mainly on summary statistics, overlooking the spatial topology of plantar pressure, which provides a more direct characterization of the realized contact state. To bridge this gap, we present Tac4Loco, a tactile-perceptive framework that incorporates multi-array plantar pressure as direct feedback for humanoid locomotion. We formulate a topology-preserving ordinal representation to map simulated and physical sensor signals into a shared observation space, with a dual-branch encoder for extracting their spatial and temporal representations. Subsequently, the learned spatiotemporal features are integrated with augmented proprioception including terrain estimation cues, and provided to an asymmetric actor-critic architecture for policy learning. Extensive simulation and real-world experiments demonstrate improved tracking performance and support adaptation on terrains with inclined, partial, asymmetric, and changing support. We further demonstrate its zero-shot deployment on unseen compliant and unstructured terrains, including a foam platform and a gravel road. All code and experimental configurations will be released as open-source to facilitate reproducibility.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/humanoid_platforms/2026_2608.15766v1_Tac4Loco_Learning_Spatiotemporal_Plantar_Pressure_Representa.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.15766", "fig": "figures/x-tac4locolearningspatiotemporalplantarpre.webp"}, {"id": "x-taskagnosticanddeviceagnosticexoskeleton", "title": "Task-Agnostic and Device-Agnostic Exoskeleton Control via Reinforcement Learning and Mixture of Experts", "tree": "systems", "branch": "humanoid-hw", "authors": [], "first_author": "", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2026, "date": "", "topics": ["wearable robotics", "MOE"], "paradigm": [], "method": [], "projects": ["exo"], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MOE/2026_Hou_MoE_exo.pdf", "source": "library", "fig": "figures/x-taskagnosticanddeviceagnosticexoskeleton.webp"}, {"id": "x-taskorientedrobothumanhandoversonleggedm", "title": "Task-Oriented Robot-Human Handovers on Legged Manipulators", "tree": "rl", "branch": "loco-manip", "authors": ["Andreea Tulbure", "Carmen Scheidemann", "Elias Steiner", "Marco Hutter"], "first_author": "Andreea Tulbure", "corresponding": [], "institutions": ["ETH Zürich"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-05", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2602.05760", "doi": "10.1145/3757279.3785629", "url": "https://arxiv.org/abs/2602.05760", "code": "", "site": "", "abstract": "Task-oriented handovers (TOH) are fundamental to effective human-robot collaboration, requiring robots to present objects in a way that supports the human's intended post-handover use. Existing approaches are typically based on object- or task-specific affordances, but their ability to generalize to novel scenarios is limited. To address this gap, we present AFT-Handover, a framework that integrates large language model (LLM)-driven affordance reasoning with efficient texture-based affordance transfer to achieve zero-shot, generalizable TOH. Given a novel object-task pair, the method retrieves a proxy exemplar from a database, establishes part-level correspondences via LLM reasoning, and texturizes affordances for feature-based point cloud transfer. We evaluate AFT-Handover across diverse task-object pairs, showing improved handover success rates and stronger generalization compared to baselines. In a comparative user study, our framework is significantly preferred over the current state-of-the-art, effectively reducing human regrasping before tool use. Finally, we demonstrate TOH on legged manipulators, highlighting the potential of our framework for real-world robot-human handovers.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2602.05760v2_Task_Oriented_Robot_Human_Handovers_on_Legged_Manipulators.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2602.05760", "fig": "figures/x-taskorientedrobothumanhandoversonleggedm.webp"}, {"id": "x-teleopitafullembodimenthumanoidteleopera", "title": "Teleopit: A Full-Embodiment Humanoid Teleoperation System", "tree": "rl", "branch": "locomotion", "authors": ["Bingqian Wu", "Zicheng Xu", "Xianghui Fan", "Dayu Li", "Xiangru Huang"], "first_author": "Bingqian Wu", "corresponding": [], "institutions": ["Institute for Human Development", "Westlake University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-03", "topics": ["locomotion", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.01834", "doi": "", "url": "https://arxiv.org/abs/2608.01834", "code": "", "site": "", "abstract": "Humanoid teleoperation for demonstration collection requires coordinated whole-body motion, continuous dexterous hand control, and viewpoint control. Existing systems either simplify hand commands or depend on dedicated wearable sensors for fine-grained hand motion. We introduce Teleopit, a full-embodiment teleoperation system that maps body, hand, and head signals from VR to a humanoid body, configurable dexterous hands, and a 2-DoF active vision module. A history encoder and failure-aware rewind sampling improve the motion tracker on both motion-capture and live VR references. An optimization-based hand retargeter combines normalized finger directions, fingertip closure, and thumb-frame alignment to map human hand motion to different dexterous hands without tuning hand-specific objective or solver hyperparameters. Component experiments evaluate tracking success rate and retargeting behavior, while real-robot teleoperation demonstrates coordinated locomotion, manipulation, and viewpoint control. ACT and GR00T N1.7 policies trained on 96 successful demonstrations collected with Teleopit achieve task success rates of 90.0% and 95.0%, respectively, when deployed on the humanoid. The project page is available at https://botrunner64.github.io/teleopit-page.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.01834v1_Teleopit_A_Full_Embodiment_Humanoid_Teleoperation_System.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.01834", "fig": "figures/x-teleopitafullembodimenthumanoidteleopera.webp"}, {"id": "x-tetherawaredynamiccollisionavoidanceforu", "title": "Tether-Aware Dynamic Collision Avoidance for USV-HROV Systems", "tree": "systems", "branch": "planning-control", "authors": ["Yang Gu", "Ziyang Hong", "Xuanlin Chen", "Hao Wei", "Cheng Wang", "Shujie Yang", "Yulin Si"], "first_author": "Yang Gu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-05-31", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.01112", "doi": "", "url": "https://arxiv.org/abs/2606.01112", "code": "", "site": "", "abstract": "Heterogeneous marine robotic systems composed of an unmanned surface vehicle (USV) and a hybrid remotely operated vehicle (HROV) have shown great potential for subsea cable inspection. In such missions, the USV tracks the HROV at the surface while supplying power and communication through an umbilical tether. However, dynamic collision avoidance for the USV during HROV tracking is challenging because the submerged tether may scrape against passing vessels, while evasive maneuvers can enlarge the USV--HROV separation, thereby increasing the likelihood of tether tautness and compromising HROV operations. To address these challenges, this work proposes a tether-aware dynamic collision avoidance method for a USV tracking an HROV. First, a tether safety-aware planar domain is introduced to represent the three-dimensional collision risk between the tether and obstacle vessels without an explicit tether shape model. Second, a tether tautness-aware velocity obstacle method is developed to achieve safe avoidance while reducing the likelihood of tether tautness. Finally, the method is integrated with line-of-sight guidance to coordinate HROV tracking and collision avoidance. Gazebo-based simulations show that the proposed method avoids dynamic obstacle vessels while maintaining tether safety and reducing the likelihood of tether tautness during USV evasive maneuvers.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2606.01112v1_Tether_Aware_Dynamic_Collision_Avoidance_for_USV_HROV_System.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.01112", "fig": "figures/x-tetherawaredynamiccollisionavoidanceforu.webp"}, {"id": "x-tinysdprealtimesemidefiniteoptimizationf", "title": "TinySDP: Real Time Semidefinite Optimization for Certifiable and Agile Edge Robotics", "tree": "systems", "branch": "planning-control", "authors": ["Ishaan Mahajan", "Jon Arrizabalaga", "Andrea Grillo", "Fausto Vega", "Zac Manchester", "James Anderson", "Brian Plancher"], "first_author": "Ishaan Mahajan", "corresponding": [], "institutions": ["MIT", "Columbia University", "EPFL"], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2026, "date": "2026-05-13", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2605.13748", "doi": "10.15607/rss.2026.xxii.110", "url": "https://arxiv.org/abs/2605.13748", "code": "", "site": "", "abstract": "Semidefinite programming (SDP) provides a principled framework for convex relaxations of nonconvex geometric constraints in motion planning, yet existing solvers are too computationally expensive for real-time control, particularly on resource-constrained embedded systems. To address this gap, we introduce TinySDP, the first semidefinite programming solver designed for embedded systems, enabling real-time model-predictive control (MPC) on microcontrollers for problems with nonconvex obstacle constraints. Our approach integrates positive-semidefinite cone projections into a cached-Riccati-based ADMM solver, leveraging computational structure for embedded tractability. We pair this solver with an a posteriori rank-1 certificate that converts relaxed solutions into explicit geometric guarantees at each timestep. On challenging benchmarks, e.g., cul-de-sac and dynamic obstacle avoidance scenarios that induce failures in local methods, TinySDP achieves collision-free navigation with up to 73% shorter paths than state-of-the-art baselines. We validate our approach on a Crazyflie quadrotor, demonstrating that semidefinite constraints can be enforced at real-time rates for agile embedded robotics.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2026_2605.13748v1_TinySDP_Real_Time_Semidefinite_Optimization_for_Certifiable.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2605.13748", "fig": "figures/x-tinysdprealtimesemidefiniteoptimizationf.webp"}, {"id": "x-towardsmultiobjectnonprehensiletransport", "title": "Towards Multi-Object Nonprehensile Transportation via Shared Teleoperation: A Framework Based on Virtual Object Model Predictive Control", "tree": "il", "branch": "retargeting", "authors": ["Xinyang Fan", "Zhaoyang Chen", "Shu Xin", "Yi Ren", "Zainan Jiang", "Fenglei Ni", "Hong Liu"], "first_author": "Xinyang Fan", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-04-08", "topics": ["teleoperation", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2604.06932", "doi": "10.48550/arxiv.2604.06932", "url": "https://arxiv.org/abs/2604.06932", "code": "", "site": "", "abstract": "Multi-object nonprehensile transportation in teleoperation demands simultaneous trajectory tracking and tray orientation control. Existing methods often struggle with model dependency, uncertain parameters, and multi-object adaptability. We propose a shared teleoperation framework where humans and robots share positioning control, while the robot autonomously manages orientation to satisfy dynamic constraints. Key contributions include: 1) A theoretical dynamic constraint analysis utilizing a novel virtual object (VO)-based method to simplify constraints for trajectory planning. 2) An MPC-based trajectory smoothing algorithm that enforces real-time constraints and coordinates user tracking with orientation control. 3) Validations demonstrating stable manipulation of nine objects at accelerations up to 2.4 m/s2. Compared to the baseline, our approach reduces sliding distance by 72.45% and eliminates tip-overs (0% vs. 13.9%), proving robust adaptability in complex scenarios.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2026_2604.06932v1_Towards_Multi_Object_Nonprehensile_Transportation_via_Shared.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2604.06932", "fig": "figures/x-towardsmultiobjectnonprehensiletransport.webp"}, {"id": "x-turbompcfastscalableanddifferentiablemod", "title": "TurboMPC: Fast, Scalable, and Differentiable Model Predictive Control on the GPU", "tree": "systems", "branch": "planning-control", "authors": ["Gabriel Bravo-Palacios", "Jianghan Zhang", "Zachary Pestrikov", "Brian Plancher", "Thomas Lew"], "first_author": "Gabriel Bravo-Palacios", "corresponding": [], "institutions": ["Toyota Research Institute"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-23", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2606.24039", "doi": "10.48550/arxiv.2606.24039", "url": "https://arxiv.org/abs/2606.24039", "code": "", "site": "", "abstract": "Robotics increasingly relies on GPUs for parallel simulation, large-scale learning, and neural-network inference. For model predictive control (MPC) to scale with this paradigm, solvers must run efficiently on this hardware while remaining fast, differentiable, and compatible with expressive MPC formulations used in robotics. We present TurboMPC, a differentiable MPC solver that runs entirely on the GPU and supports state and control inequality constraints, implicit integrators, cross-time-coupled costs, and slack variables. TurboMPC combines sequential quadratic programming (SQP), an alternating direction method of multipliers (ADMM) inner solver, implicit differentiation, and a co-designed JAX-CUDA implementation for efficiency and ease of use. In simulation, we validate TurboMPC on constrained planning, humanoid imitation learning, and reinforcement learning with neural-network cost function tasks, achieving up to $15\\times$ and $58\\times$ speedups over state-of-the-art CPU and GPU differentiable solvers, respectively. We deploy TurboMPC on a full-scale car for minimum-time racing and find that batched, GPU-accelerated tuning of MPC parameters via Bayesian optimization yields significantly faster driving than a hand-tuned baseline. TurboMPC also scales to planning horizons of over $8000$ knot points while maintaining control of the vehicle. We open-source TurboMPC at: https://github.com/ToyotaResearchInstitute/turbompc", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2606.24039v1_TurboMPC_Fast_Scalable_and_Differentiable_Model_Predictive_C.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2606.24039", "fig": "figures/x-turbompcfastscalableanddifferentiablemod.webp"}, {"id": "x-unveilingcomplexcollectivebehaviorsfroms", "title": "Unveiling Complex Collective Behaviors from Simple Rewards", "tree": "rl", "branch": "marl", "authors": ["Yize Mi", "Jianan Li", "Liang Li", "Shiyu Zhao"], "first_author": "Yize Mi", "corresponding": [], "institutions": ["SAIC-GM (China)", "Westlake University"], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2026, "date": "2026-07-14", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.12861", "doi": "", "url": "https://arxiv.org/abs/2607.12861", "code": "", "site": "", "abstract": "Multi-agent Reinforcement Learning (MARL) holds great potential for robot swarms, but the black-box nature of neural policies complicates strategic analysis, limiting multi-robot applications. Furthermore, complex swarm behaviors can surprisingly emerge from simple rewards without explicit aggregation incentives. Unveiling the mechanisms behind this emergence is critical, but the disconnection between simple rewards and collective behaviors exacerbates interpretability challenges. This paper aims to reveal the hidden mechanisms in this process. We propose a two-stage EEC (\\LinkIII) explanatory framework. This includes a novel analytical tool called the Agent Response Map (ARM), which reveals agents' decision-making patterns across space and identifies regions of aggregation and avoidance. ARM reveals that the robots implicitly learn the geometric fields of the environment and utilize these structures as desired targets for coordinated movement. We validate this finding across two distinct tasks: a cooperative multi-robot shape assembly and a competitive predator-prey pursuit-evasion. 1) In the cooperative task, ARM identifies the unoccupied target interior as the desired destination for robot navigation. As the center becomes occupied, this target region automatically shifts toward the boundary, demonstrating the robots' capacity to autonomously explore unoccupied areas. 2) In the competitive task, ARM surprisingly identifies the boundary of the predators' Voronoi diagram as the convergence destination for prey agents. Together, these two tasks demonstrate the capability of ARM to discover the hidden geometric structures underlying MARL policies in robot swarms.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2607.12861v1_Unveiling_Complex_Collective_Behaviors_from_Simple_Rewards.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.12861", "fig": "figures/x-unveilingcomplexcollectivebehaviorsfroms.webp"}, {"id": "x-vibrationsuppressionincollaborativeflexi", "title": "Vibration Suppression in Collaborative Flexible Payload Manipulation Using Passive Force Control", "tree": "systems", "branch": "sim-bench", "authors": ["Alaa Abderrahim", "Antonio Rosales", "Ferdinando Milella", "Markku Suomalainen", "Shuai Li"], "first_author": "Alaa Abderrahim", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-14", "topics": ["simulation & benchmarks", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.14244", "doi": "10.48550/arxiv.2608.14244", "url": "https://arxiv.org/abs/2608.14244", "code": "", "site": "", "abstract": "In large and heavy structures, vibrations arise during motion, posing significant challenges for precise manipulation. To accomplish the desired motion, control algorithms must effectively suppress these structural vibrations. In cutting edge projects, such as remote maintenance of future fusion energy reactors (tokamaks), the manipulation of this type of structure is defined as a crucial task. This paper presents a control strategy to suppress transverse vibrations in flexible payloads during motion using a collaborative payload manipulation approach. Two different industrial robot arms are arranged in a leader follower configuration for the manipulation strategy. The leader robot guides the motion with shaped velocity commands, while the follower robot ensures compliance with the estimated external forces applied by the leader on the payload through an admittance controller. Unlike existing methods, the proposed approach enables collaborative manipulation of heavier and larger flexible objects, addressing additional challenges such as vibration suppression and heterogeneous robot specifications. The dynamics of the leader follower payload system are modeled using an equivalent mass spring damper model, and it is shown that, with appropriate admittance parameters, the total energy of the system is passively dissipated. A stability proof is also provided. Numerical simulations validate the proposed method, and experimental results demonstrate its effectiveness.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2026_2608.14244v1_Vibration_Suppression_in_Collaborative_Flexible_Payload_Mani.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.14244", "fig": "figures/x-vibrationsuppressionincollaborativeflexi.webp"}, {"id": "x-video2doortraversalpushdoortraversalvias", "title": "Video2DoorTraversal: Push Door Traversal via Simulated Door Twins", "tree": "rl", "branch": "loco-manip", "authors": ["Xincheng Tang", "Yiji Chen", "Youhan Xie", "Wanyu Li", "Zhengjie Shu", "Lai Jiang", "Wenkang Hu", "Yitong Li", "Jinchuang Zhang", "Xibin Song", "Ruigang Yang"], "first_author": "Xincheng Tang", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-20", "topics": ["loco-manipulation", "whole body loco manipulation"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2608.20251", "doi": "10.48550/arxiv.2608.20251", "url": "https://arxiv.org/abs/2608.20251", "code": "", "site": "", "abstract": "Door opening and traversal is a long-horizon loco-manipulation task that requires precise handle interaction and coordinated base-arm control. We present Video2DoorTraversal, a single-video real-to-sim-to-real framework for wheel-legged mobile manipulators. Given one RGB video of a real door, DoorTwin reconstructs an instance-aligned, articulated, and simulation-ready door twin with realistic geometry and appearance. A simulation-in-the-loop agent converts the recovered articulation into a parameterized skill program and iteratively refines failed rollouts to generate physically executable demonstrations. These demonstrations are used to train ArticuACT, a dual-depth policy that predicts coordinated base, arm, and gripper commands using robot-centric camera conditioning and interaction-aware supervision. With all perception and policy inference running onboard, the system achieves a 96.57% average success rate across five real doors and an 80.95% zero-shot success rate on structurally similar unseen doors, while completing the full approach, opening, and traversal sequence in approximately 13s on average. Project Page: https://video2doortraversal.github.io/.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/whole_body_loco_manipulation/2026_2608.20251v1_Video2DoorTraversal_Push_Door_Traversal_via_Simulated_Door_T.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2608.20251", "fig": "figures/x-video2doortraversalpushdoortraversalvias.webp"}, {"id": "vla-jumpstart", "title": "Vision-Language-Action Jump-Starting for Reinforcement Learning Robotic Agents", "tree": "rl", "branch": "post-training", "authors": ["Angelo Moroncelli", "Roberto Zanetti", "Marco Maccarini", "Loris Roveda"], "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2026, "date": "2026-04-15", "topics": ["VLA as RL initialization", "jump-starting"], "paradigm": ["RL", "VLA"], "method": [], "projects": [], "related": ["expo-ft", "pld"], "arxiv": "2604.13733", "doi": "", "code": "", "site": "", "note": "ICRA 2026 workshop paper. The simplest version of the idea — the VLA is the exploration prior, RL does the rest.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Reinforcement learning (RL) enables high-frequency, closed-loop control for robotic manipulation, but scaling to long-horizon tasks with sparse or imperfect rewards remains difficult due to inefficient exploration and poor credit assignment. Vision-Language-Action (VLA) models leverage large-scale multimodal pretraining to provide generalist, task-level reasoning, but current limitations hinder their direct use in fast and precise manipulation. In this paper, we propose Vision-Language-Action Jump-Starting (VLAJS), a method that bridges sparse VLA guidance with on-policy RL to improve exploration and learning efficiency. VLAJS treats VLAs as transient sources of high-level action suggestions that bias early exploration and improve credit assignment, while preserving the high-frequency, state-based control of RL. Our approach augments Proximal Policy Optimization (PPO) with a directional action-consistency regularization that softly aligns the RL agent's actions with VLA guidance during early training, without enforcing strict imitation, requiring demonstrations, or relying on continuous teacher queries. VLA guidance is applied sparsely and annealed over time, allowing the agent to adapt online and ultimately surpass the guiding policy. We evaluate VLAJS on six challenging manipulation tasks: lifting, pick-and-place, peg reorientation, peg insertion, poking, and pushing in simulation, and validate a subset on a real Franka Panda robot. VLAJS consistently outperforms PPO and distillation-style baselines in sample efficiency, reducing required environment interactions by over 50% in several tasks. Real-world experiments demonstrate zero-shot sim-to-real transfer and robust execution under clutter, object variation, and external perturbations.", "source": "curated", "first_author": "Angelo Moroncelli", "url": "https://arxiv.org/abs/2604.13733", "fig": "figures/vla-jumpstart.webp", "figs": ["figures/vla-jumpstart-1.webp"]}, {"id": "x-warpmpclargebatchmpcongpuviaadmmwithunro", "title": "WarpMPC: Large-Batch MPC on GPU via ADMM with Unrolled $LDL^\\top$ Factorization", "tree": "systems", "branch": "planning-control", "authors": ["Henrik Hose", "Se Hwan Jeon", "Charles Khazoom", "Sangbae Kim", "Sebastian Trimpe"], "first_author": "Henrik Hose", "corresponding": [], "institutions": ["RWTH Aachen University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-13", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.11603", "doi": "", "url": "https://arxiv.org/abs/2607.11603", "code": "", "site": "", "abstract": "This paper introduces numerical optimizations for maximizing throughput on GPU when solving large batches (10,000 to over 100,000) of sequential quadratic programming (SQP) iterations, where all problems have the same structure. The optimizations are implemented in a toolbox WarpMPC for model-predictive control (MPC) in JAX and Warp. Based on the insight that all MPC problem instances in a batch share the same sparsity in time, cost, and constraints, we propose unrolling sparse linear factorizations and solves, which dominate alternating direction method of multipliers (ADMM) solver runtime. We avoid memory access bottlenecks and wasting computations via optimized memory layout, padding-reducing segmentation of the unrolled factorization, and dependency level scheduled backsolves, additionally accelerating sensitivity computation. We achieve throughputs of 8,000 to 250,000 SQP iterations per second on nonlinear cartpole, quadrotor, and humanoid robot benchmarks, outperforming baselines by 3$\\times$ to 25$\\times$. We illustrate practical usefulness by synthesizing a dataset and training a neural network approximation of an MPC in under 4 minutes that stabilizes a nano quadrotor in hardware experiments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2026_2607.11603v1_WarpMPC_Large_Batch_MPC_on_GPU_via_ADMM_with_Unrolled_LDL_to.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.11603", "fig": "figures/x-warpmpclargebatchmpcongpuviaadmmwithunro.webp"}, {"id": "x-whendoeslegacydatastarttohelpemergenttra", "title": "When Does Legacy Data Start to Help? Emergent Transfer in Cross-Configuration Robot Learning", "tree": "il", "branch": "visuomotor", "authors": ["Tao Wang", "Hudson Hou", "Yingdong Hu", "Yufeng Liu", "Qinghai Li", "Yingjie Jiang", "Yingzhi Wang", "Cheng Ma", "Richard Wang", "Yang Gao"], "first_author": "Tao Wang", "corresponding": [], "institutions": ["Tsinghua University", "Peking University", "Shanghai Jiao Tong University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-07-28", "topics": ["mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2607.25593", "doi": "", "url": "https://arxiv.org/abs/2607.25593", "code": "", "site": "", "abstract": "Robotic hardware evolves over time, but demonstration data is often tied to a specific sensor and actuator configuration. This raises a practical and underexplored question: when does legacy data begin to benefit an upgraded robot? We study this question on a wheeled humanoid platform across two hardware generations, where both the camera and gripper are changed while the overall morphology remains fixed. Contrary to the common assumption that more cross-configuration data is always helpful, we observe a grokking-like transition: legacy data remains ineffective until the upgraded configuration acquires a minimum level of task competence, after which co-training gains rise sharply before diminishing near saturation. We hypothesize that this task-dependent transition is governed by a transfer threshold and characterize the resulting three-phase pattern. Across real-robot manipulation tasks, we observe all three phases: no measurable benefit at low competence ($10.0\\% \\rightarrow 10.0\\%$), a sharp gain after crossing the threshold ($23.3\\% \\rightarrow 86.7\\%$ on flower insertion), and diminishing returns at high competence ($85.0\\% \\rightarrow 93.3\\%$ on pen insertion). We provide a theoretical account based on gradient alignment and residual policy uncertainty, and derive a phase-aware rule for deciding when to collect more new-hardware data and when to reuse legacy demonstrations. We further validate this three-phase pattern on a mobile dual-arm watering task, with results consistent with our predictions.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2026_2607.25593v1_When_Does_Legacy_Data_Start_to_Help_Emergent_Transfer_in_Cro.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2607.25593", "fig": "figures/x-whendoeslegacydatastarttohelpemergenttra.webp"}, {"id": "x-xaiguidedconservativedecentralizedexecut", "title": "XAI-Guided Conservative Decentralized Execution for Offline Multi-Agent Network Slicing", "tree": "rl", "branch": "marl", "authors": ["Eslam Eldeeb", "Hatim Chergui", "Merouane Debbah"], "first_author": "Eslam Eldeeb", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-08-14", "topics": ["multi-agent", "multi agent rl"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2608.13982", "doi": "10.13039/501100011033/FEDER", "url": "https://arxiv.org/abs/2608.13982", "code": "", "site": "", "abstract": "The recent advances toward sixth-generation (6G) and beyond-6G networks have accelerated the need for intelligent resource management mechanisms capable of supporting heterogeneous services under shared infrastructures in network slicing. However, resource allocation in network slicing naturally forms a resource-coupled cooperative optimization problem with competing slice demands. Slices compete for limited resources to minimize individual latencies while coordinating to avoid conflicts and underutilization. Although multi-agent reinforcement learning (MARL) has shown promising performance in such settings, existing online formulations remain costly, unsafe, and difficult to deploy due to their reliance on environmental interactions and communication among agents. In this work, we present explainable artificial intelligence (XAI)-guided conservative decentralized execution (X-CODE). X-CODE is an explainable offline MARL that operates offline without environmental interaction, nor inter-agent communication. It exploits explainability-aware reward shaping to modify the relative preference among joint offline transitions during centralized training to improve decentralized resource-allocation behavior. In deployment, the agents operate independently without signaling exchange among the agents. Simulation results demonstrate that the proposed approach achieves zero observed resource-conflict events in the evaluated test episodes while minimizing per-slice latencies. Moreover, the proposed framework exhibits lower signaling overhead and reduces effective inference latency by 88 % under the considered communication-delay model compared to the online baselines. Source codes and datasets are available through: https://github.com/Eslam211/xcode-ran-slicing.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_agent_rl/2026_2608.13982v1_XAI_Guided_Conservative_Decentralized_Execution_for_Offline.pdf", "source": "library", "fig": "figures/x-xaiguidedconservativedecentralizedexecut.webp"}, {"id": "dist-r2s2r", "title": "A Distributional Treatment of Real2Sim2Real for Object-Centric Agent Adaptation in Vision-Driven Deformable Linear Object Manipulation", "tree": "rl", "branch": "sim2real", "authors": ["Georgios Kamaras", "Subramanian Ramamoorthy"], "corresponding": [], "institutions": [], "lab": "", "venue": "In IEEE Robotics and Automation Letters,", "venue_type": "journal", "year": 2025, "date": "2025-02-25", "topics": ["deformable linear objects", "distributional real2sim2real"], "paradigm": ["real2sim"], "method": [], "projects": ["r2s2r"], "related": ["real2sim", "simweaver", "video2sim2real"], "arxiv": "2502.18615", "doi": "10.1109/LRA.2025.3581744", "code": "", "site": "", "note": "Real2Sim2Real treated distributionally rather than as point identification, on deformable linear objects. Relevant to both the real2sim2real loop and the soft-object gap.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "We present an integrated (or end-to-end) framework for the Real2Sim2Real problem of manipulating deformable linear objects (DLOs) based on visual perception. Working with a parameterised set of DLOs, we use likelihood-free inference (LFI) to compute the posterior distributions for the physical parameters using which we can approximately simulate the behaviour of each specific DLO. We use these posteriors for domain randomisation while training, in simulation, object-specific visuomotor policies (i.e. assuming only visual and proprioceptive sensory) for a DLO reaching task, using model-free reinforcement learning. We demonstrate the utility of this approach by deploying sim-trained DLO manipulation policies in the real world in a zero-shot manner, i.e. without any further fine-tuning. In this context, we evaluate the capacity of a prominent LFI method to perform fine classification over the parametric set of DLOs, using only visual and proprioceptive data obtained in a dynamic manipulation trajectory. We then study the implications of the resulting domain distributions in sim-based policy learning and real-world performance.", "source": "curated", "first_author": "Georgios Kamaras", "url": "https://arxiv.org/abs/2502.18615", "fig": "figures/dist-r2s2r.webp"}, {"id": "reconfig-wheelleg", "title": "A Reconfigured Wheel-Legged Robot for Enhanced Steering and Adaptability", "tree": "systems", "branch": "codesign", "authors": ["Song", "Zhicheng", "Xu", "Jinglan", "Zheng", "Chunxin", "Li", "Yulin", "Bi", "Zhihai", "Ma", "Jun"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-07-30", "topics": ["reconfigurable wheel-leg", "steering and adaptability"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["swheg", "x2n"], "arxiv": "2507.22345", "doi": "10.48550/arxiv.2507.22345", "code": "", "site": "", "note": "A reconfigured wheel-legged design focused on steering and terrain adaptability. Useful as a mechanism survey point rather than for its learning content.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Wheel-legged robots integrate leg agility on rough terrain with wheel efficiency on flat ground. However, most existing designs do not fully capitalize on the benefits of both legged and wheeled structures, which limits overall system flexibility and efficiency. We present FLORES, a novel wheel-legged robot design featuring a distinctive front-leg configuration that sets it beyond standard design approaches. Specifically, FLORES replaces the conventional hip-roll degree of freedom (DoF) of the front leg with hip-yaw DoFs, and this allows for efficient movement on flat surfaces while ensuring adaptability when navigating complex terrains. This innovative design facilitates seamless transitions between different locomotion modes (i.e., legged locomotion and wheeled locomotion) and optimizes the performance across varied environments. To fully exploit \\flores's mechanical capabilities, we develop a tailored reinforcement learning (RL) controller that adapts the Hybrid Internal Model (HIM) with a customized reward structure optimized for our unique mechanical configuration. This framework enables the generation of adaptive, multi-modal locomotion strategies that facilitate smooth transitions between wheeled and legged movements. Furthermore, our distinctive joint design enables the robot to exhibit novel and highly efficient locomotion gaits that capitalize on the synergistic advantages of both locomotion modes. Through comprehensive experiments, we demonstrate FLORES's enhanced steering capabilities, improved navigation efficiency, and versatile locomotion across various terrains. The open-source project can be found at https://github.com/ZhichengSong6/FLORES.", "source": "curated", "first_author": "Song", "url": "https://arxiv.org/abs/2507.22345", "fig": "figures/reconfig-wheelleg.webp", "figs": ["figures/reconfig-wheelleg-1.webp"]}, {"id": "x-asurveyonimprovinghumanrobotcollaboratio", "title": "A Survey on Improving Human Robot Collaboration through Vision-and-Language Navigation", "tree": "rl", "branch": "marl", "authors": ["Nivedan Yakolli", "Avinash Gautam", "Abhijit Das", "Yuankai Qi", "Virendra Singh Shekhawat"], "first_author": "Nivedan Yakolli", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-06", "topics": ["multi-agent", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2512.00027", "doi": "", "url": "https://arxiv.org/abs/2512.00027", "code": "", "site": "", "abstract": "Vision-and-Language Navigation (VLN) is a multi-modal, cooperative task requiring agents to interpret human instructions, navigate 3D environments, and communicate effectively under ambiguity. This paper presents a comprehensive review of recent VLN advancements in robotics and outlines promising directions to improve multi-robot coordination. Despite progress, current models struggle with bidirectional communication, ambiguity resolution, and collaborative decision-making in the multi-agent systems. We review approximately 200 relevant articles to provide an in-depth understanding of the current landscape. Through this survey, we aim to provide a thorough resource that inspires further research at the intersection of VLN and robotics. We advocate that the future VLN systems should support proactive clarification, real-time feedback, and contextual reasoning through advanced natural language understanding (NLU) techniques. Additionally, decentralized decision-making frameworks with dynamic role assignment are essential for scalable, efficient multi-robot collaboration. These innovations can significantly enhance human-robot interaction (HRI) and enable real-world deployment in domains such as healthcare, logistics, and disaster response.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2025_2512.00027v1_A_Survey_on_Improving_Human_Robot_Collaboration_through_Visi.pdf", "source": "library", "fig": "figures/x-asurveyonimprovinghumanrobotcollaboratio.webp"}, {"id": "x-activepusheractivelearningandplanningwit", "title": "ActivePusher: Active Learning and Planning with Residual Physics for Nonprehensile Manipulation", "tree": "systems", "branch": "sim-bench", "authors": ["Zhuoyun Zhong", "Seyed Jafar Golestaneh", "Constantinos Chamzas"], "first_author": "Zhuoyun Zhong", "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2025-06-05", "topics": ["simulation & benchmarks", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2506.04646", "doi": "10.48550/arxiv.2506.04646", "url": "https://arxiv.org/abs/2506.04646", "code": "", "site": "", "abstract": "Planning with learned dynamics models offers a promising approach toward versatile real-world manipulation, particularly in nonprehensile settings such as pushing or rolling, where accurate analytical models are difficult to obtain. However, collecting training data for learning-based methods can be costly and inefficient, as it often relies on randomly sampled interactions that are not necessarily the most informative. Furthermore, learned models tend to exhibit high uncertainty in underexplored regions of the skill space, undermining the reliability of long-horizon planning. To address these challenges, we propose ActivePusher, a novel framework that combines residual-physics modeling with uncertainty-based active learning, to focus data acquisition on the most informative skill parameters. Additionally, ActivePusher seamlessly integrates with model-based kinodynamic planners, leveraging uncertainty estimates to bias control sampling toward more reliable actions. We evaluate our approach in both simulation and real-world environments, and demonstrate that it consistently improves data efficiency and achieves higher planning success rates in comparison to baseline methods. The source code is available at https://github.com/elpis-lab/ActivePusher.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2025_2506.04646v4_ActivePusher_Active_Learning_and_Planning_with_Residual_Phys.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2506.04646", "fig": "figures/x-activepusheractivelearningandplanningwit.webp"}, {"id": "x-affordancebaseddisambiguationofsurgicali", "title": "Affordance-Based Disambiguation of Surgical Instructions for Collaborative Robot-Assisted Surgery", "tree": "systems", "branch": "agentic", "authors": ["Ana Davila", "Jacinto Colan", "Yasuhisa Hasegawa"], "first_author": "Ana Davila", "corresponding": [], "institutions": [], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-09-18", "topics": ["LLM agent", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2509.14967", "doi": "10.48550/arxiv.2509.14967", "url": "https://arxiv.org/abs/2509.14967", "code": "", "site": "", "abstract": "Effective human-robot collaboration in surgery is affected by the inherent ambiguity of verbal communication. This paper presents a framework for a robotic surgical assistant that interprets and disambiguates verbal instructions from a surgeon by grounding them in the visual context of the operating field. The system employs a two-level affordance-based reasoning process that first analyzes the surgical scene using a multimodal vision-language model and then reasons about the instruction using a knowledge base of tool capabilities. To ensure patient safety, a dual-set conformal prediction method is used to provide a statistically rigorous confidence measure for robot decisions, allowing it to identify and flag ambiguous commands. We evaluated our framework on a curated dataset of ambiguous surgical requests from cholecystectomy videos, demonstrating a general disambiguation rate of 60% and presenting a method for safer human-robot interaction in the operating room.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2025_2509.14967v2_Affordance_Based_Disambiguation_of_Surgical_Instructions_for.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2509.14967", "fig": "figures/x-affordancebaseddisambiguationofsurgicali.webp"}, {"id": "x-anyplacelearninggeneralizableobjectplace", "title": "AnyPlace: Learning Generalized Object Placement for Robot Manipulation", "tree": "il", "branch": "visuomotor", "authors": ["Yuchi Zhao", "Miroslav Bogdanovic", "Chengyuan Luo", "Steven Tohme", "Kourosh Darvish", "Alán Aspuru-Guzik", "Florian Shkurti", "Animesh Garg"], "first_author": "Yuchi Zhao", "corresponding": [], "institutions": ["Georgia Tech", "University of Toronto", "Shanghai Jiao Tong University"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "", "topics": ["proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2502.04531", "doi": "", "url": "https://arxiv.org/abs/2502.04531", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/425_AnyPlace_Learning_Generali.pdf", "source": "library", "fig": "figures/x-anyplacelearninggeneralizableobjectplace.webp", "arxiv_url": "https://arxiv.org/abs/2502.04531"}, {"id": "booster-rl", "title": "Booster Gym: An End-to-End Reinforcement Learning Framework for Humanoid Robot Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Yushi Wang", "Penghui Chen", "Xinyu Han", "Feng Wu", "Mingguo Zhao"], "corresponding": [], "institutions": ["Tsinghua University", "Booster Robotics"], "lab": "", "venue": "Lecture Notes in Electrical Engineering", "venue_type": "preprint", "year": 2025, "date": "2025-06-18", "topics": ["humanoid locomotion", "open framework"], "paradigm": ["RL"], "method": ["sim-to-real pipeline", "domain randomization"], "projects": ["mabel"], "related": ["isaaclab", "rudin2021"], "arxiv": "2506.15132", "doi": "10.1007/978-981-95-6825-3_19", "code": "", "site": "", "note": "In your library. Useful as a concrete reference stack for a humanoid RL bring-up.", "aka": "", "stars": 1, "curated": true, "local": "~/Desktop/papers/RL/booster_robot_rl.pdf", "abstract": "Recent advancements in reinforcement learning (RL) have led to significant progress in humanoid robot locomotion, simplifying the design and training of motion policies in simulation. However, the numerous implementation details make transferring these policies to real-world robots a challenging task. To address this, we have developed a comprehensive code framework that covers the entire process from training to deployment, incorporating common RL training methods, domain randomization, reward function design, and solutions for handling parallel structures. This library is made available as a community resource, with detailed descriptions of its design and experimental results. We validate the framework on the Booster T1 robot, demonstrating that the trained policies seamlessly transfer to the physical platform, enabling capabilities such as omnidirectional walking, disturbance resistance, and terrain adaptability. We hope this work provides a convenient tool for the robotics community, accelerating the development of humanoid robots. The code can be found in https://github.com/BoosterRobotics/booster_gym.", "source": "curated", "first_author": "Yushi Wang", "url": "https://doi.org/10.1007/978-981-95-6825-3_19", "arxiv_url": "https://arxiv.org/abs/2506.15132", "fig": "figures/booster-rl.webp", "figs": ["figures/booster-rl-1.webp", "figures/booster-rl-2.webp"]}, {"id": "x-classcontrastivelearningviaactionsequenc", "title": "CLASS: Contrastive Learning via Action Sequence Supervision for Robot Manipulation", "tree": "il", "branch": "visuomotor", "authors": ["Sung-Wook Lee", "Xuhui Kang", "Brandon Yang", "Yen-Ling Kuo"], "first_author": "Sung-Wook Lee", "corresponding": [], "institutions": ["University of Virginia"], "lab": "", "venue": "", "venue_type": "", "year": 2025, "date": "2025-08-03", "topics": ["proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2508.01600", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/803_CLASS_Contrastive_Learning.pdf", "source": "library", "fig": "figures/x-classcontrastivelearningviaactionsequenc.webp"}, {"id": "x-may19232025atlantausacatchitlearningtoca", "title": "Catch It! Learning to Catch in Flight with Mobile Dexterous Hands", "tree": "il", "branch": "visuomotor", "authors": ["Yuanhang Zhang", "Tianhai Liang", "Zhenyang Chen", "Yanjie Ze", "Huazhe Xu"], "first_author": "Yuanhang Zhang", "corresponding": [], "institutions": ["Georgia Institute of Technology", "ShangHai JiAi Genetics & IVF Institute", "Stanford University", "Tsinghua University"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2025-05-19", "topics": ["RL"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/ICRA55743.2025.11127596", "url": "https://doi.org/10.1109/ICRA55743.2025.11127596", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/catch_it.pdf", "source": "library", "fig": "figures/x-may19232025atlantausacatchitlearningtoca.webp"}, {"id": "x-combininghighlevelschedulingandlowlevelc", "title": "Combining High Level Scheduling and Low Level Control to Manage Fleets of Mobile Robots", "tree": "systems", "branch": "planning-control", "authors": ["Sabino Francesco Roselli", "Ze Zhang", "Knut Åkesson"], "first_author": "Sabino Francesco Roselli", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-27", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2510.23129", "doi": "", "url": "https://arxiv.org/abs/2510.23129", "code": "", "site": "", "abstract": "The deployment of mobile robots for material handling in industrial environments requires scalable coordination of large fleets in dynamic settings. This paper presents a two-layer framework that combines high-level scheduling with low-level control. Tasks are assigned and scheduled using the compositional algorithm ComSat, which generates time-parameterized routes for each robot. These schedules are then used by a distributed Model Predictive Control (MPC) system in real time to compute local reference trajectories, accounting for static and dynamic obstacles. The approach ensures safe, collision-free operation, and supports rapid rescheduling in response to disruptions such as robot failures or environmental changes. We evaluate the method in simulated 2D environments with varying road capacities and traffic conditions, demonstrating high task completion rates and robust behavior even under congestion. The modular structure of the framework allows for computational tractability and flexibility, making it suitable for deployment in complex, real-world industrial scenarios.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2025_2510.23129v2_Combining_High_Level_Scheduling_and_Low_Level_Control_to_Man.pdf", "source": "library", "fig": "figures/x-combininghighlevelschedulingandlowlevelc.webp"}, {"id": "x-continuoustimestateestimationmethodsinro", "title": "Continuous-Time State Estimation Methods in Robotics: A Survey", "tree": "systems", "branch": "perception", "authors": ["William Henry Fox Talbot", "Julian Nubert", "Turcan Tuna", "César Cadena", "Frederike Dümbgen", "Jesus Tordesillas", "Timothy D. Barfoot", "Marco Hutter"], "first_author": "William Henry Fox Talbot", "corresponding": [], "institutions": ["Département d'Informatique", "Institut national de recherche en sciences et technologies du numérique", "Robotic Technology (United States)", "Robotics Research (United States)", "Universidad Pontificia Comillas", "École Normale Supérieure - PSL"], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2025, "date": "2025-01-01", "topics": ["perception & estimation", "Survey"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/tro.2025.3593079", "url": "https://doi.org/10.1109/tro.2025.3593079", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Survey/Continuous-Time_State_Estimation_Methods_in_Robotics_A_Survey.pdf", "source": "library", "fig": "figures/x-continuoustimestateestimationmethodsinro.webp"}, {"id": "x-deepreactivepolicylearningreactivemanipu", "title": "Deep Reactive Policy: Learning Reactive Manipulator Motion Planning for Dynamic Environments", "tree": "systems", "branch": "planning-control", "authors": ["Jiahui Yang", "Jason Jingzhou Liu", "Yulong Li", "Youssef Khaky", "Kenneth Shaw", "Deepak Pathak"], "first_author": "Jiahui Yang", "corresponding": [], "institutions": ["Carnegie Mellon University"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-09-08", "topics": ["planning & control", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2509.06953", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/145_Deep_Reactive_Policy_Learn.pdf", "source": "library", "fig": "figures/x-deepreactivepolicylearningreactivemanipu.webp"}, {"id": "x-diffusiondynamicsmodelswithgenerativesta", "title": "Diffusion Dynamics Models with Generative State Estimation for Cloth Manipulation", "tree": "systems", "branch": "perception", "authors": ["Tongxuan Tian", "Haoyang Li", "Bo Ai", "Xiaodi Yuan", "Zhiao Huang", "Hao Su"], "first_author": "Tongxuan Tian", "corresponding": [], "institutions": [], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2025, "date": "2025-03-15", "topics": ["perception & estimation", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2503.11999", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/497_Diffusion_Dynamics_Models_.pdf", "source": "library", "fig": "figures/x-diffusiondynamicsmodelswithgenerativesta.webp"}, {"id": "x-distributedswitchingmodelpredictivecontr", "title": "Distributed Switching Model Predictive Control Meets Koopman Operator for Dynamic Obstacle Avoidance", "tree": "systems", "branch": "planning-control", "authors": ["Ali Azarbahram", "Chrystian Pool Yuca Huanca", "Gian Paolo Incremona", "Patrizio Colaneri"], "first_author": "Ali Azarbahram", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-11-21", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2511.17186", "doi": "", "url": "https://arxiv.org/abs/2511.17186", "code": "", "site": "", "abstract": "This paper introduces a Koopman-enhanced distributed switched model predictive control (SMPC) framework for safe and scalable navigation of quadrotor unmanned aerial vehicles (UAVs) in dynamic environments with moving obstacles. The proposed method integrates switched motion modes and data-driven prediction to enable real-time, collision-free coordination. A localized Koopman operator approximates nonlinear obstacle dynamics as linear models based on online measurements, enabling accurate trajectory forecasting. These predictions are embedded into a distributed SMPC structure, where each UAV makes autonomous decisions using local and cluster-based information. This computationally efficient architecture is particularly promising for applications in surface transportation, including coordinated vehicle flows, shared infrastructure with pedestrians or cyclists, and urban UAV traffic. Simulation results demonstrate reliable formation control and real-time obstacle avoidance, highlighting the frameworks broad relevance for intelligent and cooperative mobility systems.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2511.17186v2_Distributed_Switching_Model_Predictive_Control_Meets_Koopman.pdf", "source": "library", "fig": "figures/x-distributedswitchingmodelpredictivecontr.webp"}, {"id": "dyna", "title": "Dyna Robotics: DYNA-1 Autonomous Dexterous Model", "tree": "systems", "branch": "agentic", "authors": ["Dyna Robotics"], "corresponding": [], "institutions": ["Dyna Robotics"], "lab": "", "venue": "company blog", "venue_type": "industry", "year": 2025, "date": "", "topics": ["long-horizon autonomy", "commercial dexterity"], "paradigm": ["VLA"], "method": [], "projects": [], "related": ["dyna2", "generalist-ai", "pi06"], "arxiv": "", "doi": "", "code": "", "site": "https://www.dyna.co/", "note": "Claims long-duration autonomous commercial task execution. Unreviewed — useful mainly as a marker of what industry considers the current reliability bar.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Dyna Robotics", "url": "https://www.dyna.co/"}, {"id": "x-emergentactiveperceptionanddexterityofsi", "title": "Emergent Active Perception and Dexterity of Simulated Humanoids from Visual Reinforcement Learning", "tree": "rl", "branch": "loco-manip", "authors": ["Zhengyi Luo", "Chen Tessler", "Toru Lin", "Ye Yuan", "Tairan He", "Wenli Xiao", "Yunrong Guo", "Gal Chechik", "Kris Kitani", "Linxi Fan", "Yuke Zhu"], "first_author": "Zhengyi Luo", "corresponding": [], "institutions": ["UC Berkeley", "Carnegie Mellon University", "NVIDIA"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2025, "date": "2025-05-18", "topics": ["loco-manipulation", "RL"], "paradigm": [], "method": [], "projects": ["r2s2r"], "related": [], "arxiv": "2505.12278", "doi": "", "url": "https://arxiv.org/abs/2505.12278", "code": "", "site": "", "abstract": "Human behavior is fundamentally shaped by visual perception -- our ability to interact with the world depends on actively gathering relevant information and adapting our movements accordingly. Behaviors like searching for objects, reaching, and hand-eye coordination naturally emerge from the structure of our sensory system. Inspired by these principles, we introduce Perceptive Dexterous Control (PDC), a framework for vision-driven dexterous whole-body control with simulated humanoids. PDC operates solely on egocentric vision for task specification, enabling object search, target placement, and skill selection through visual cues, without relying on privileged state information (e.g., 3D object positions and geometries). This perception-as-interface paradigm enables learning a single policy to perform multiple household tasks, including reaching, grasping, placing, and articulated object manipulation. We also show that training from scratch with reinforcement learning can produce emergent behaviors such as active search. These results demonstrate how vision-driven control and complex tasks induce human-like behaviors and can serve as the key ingredients in closing the perception-action loop for animation, robotics, and embodied AI.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/visual_rl.pdf", "source": "library", "fig": "figures/x-emergentactiveperceptionanddexterityofsi.webp"}, {"id": "x-eventtriggerednonlinearmodelpredictiveco", "title": "Event-Triggered Nonlinear Model Predictive Control for Cooperative Cable-Suspended Payload Transportation with Multi-Quadrotors", "tree": "systems", "branch": "bimanual", "authors": ["Tohid Kargar Tasooji", "Sakineh Khodadadi", "Guangjun Liu"], "first_author": "Tohid Kargar Tasooji", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-26", "topics": ["open hardware", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2504.03123", "doi": "", "url": "https://arxiv.org/abs/2504.03123", "code": "", "site": "", "abstract": "Autonomous Micro Aerial Vehicles (MAVs), particularly quadrotors, have shown significant potential in assisting humans with tasks such as construction and package delivery. These applications benefit greatly from the use of cables for manipulation mechanisms due to their lightweight, low-cost, and simple design. However, designing effective control and planning strategies for cable-suspended systems presents several challenges, including indirect load actuation, nonlinear configuration space, and highly coupled system dynamics. In this paper, we introduce a novel event-triggered distributed Nonlinear Model Predictive Control (NMPC) method specifically designed for cooperative transportation involving multiple quadrotors manipulating a cable-suspended payload. This approach addresses key challenges such as payload manipulation, inter-robot separation, obstacle avoidance, and trajectory tracking, all while optimizing the use of computational and communication resources. By integrating an event-triggered mechanism, our NMPC method reduces unnecessary computations and communication, enhancing energy efficiency and extending the operational range of MAVs. The proposed method employs a lightweight state vector parametrization that focuses on payload states in all six degrees of freedom, enabling efficient planning of trajectories on the SE(3) manifold. This not only reduces planning complexity but also ensures real-time computational feasibility. Our approach is validated through extensive simulation, demonstrating its efficacy in dynamic and resource-constrained environments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2025_2504.03123v1_Event_Triggered_Nonlinear_Model_Predictive_Control_for_Coope.pdf", "source": "library", "fig": "figures/x-eventtriggerednonlinearmodelpredictiveco.webp"}, {"id": "x-forecastdrivenmpcfordecentralizedmultiro", "title": "Forecast-Driven MPC for Decentralized Multi-Robot Collision Avoidance", "tree": "systems", "branch": "planning-control", "authors": ["Hadush Hailu", "Bruk Gebregziabher", "Prudhvi Raj"], "first_author": "Hadush Hailu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-07-25", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2508.08264", "doi": "", "url": "https://arxiv.org/abs/2508.08264", "code": "", "site": "", "abstract": "The Iterative Forecast Planner (IFP) is a geometric planning approach that offers lightweight computations, scalable, and reactive solutions for multi-robot path planning in decentralized, communication-free settings. However, it struggles in symmetric configurations, where mirrored interactions often lead to collisions and deadlocks. We introduce eIFP-MPC, an optimized and extended version of IFP that improves robustness and path consistency in dense, dynamic environments. The method refines threat prioritization using a time-to-collision heuristic, stabilizes path generation through cost-based via-point selection, and ensures dynamic feasibility by incorporating model predictive control (MPC) into the planning process. These enhancements are tightly integrated into the IFP to preserve its efficiency while improving its adaptability and stability. Extensive simulations across symmetric and high-density scenarios show that eIFP-MPC significantly reduces oscillations, ensures collision-free motion, and improves trajectory efficiency. The results demonstrate that geometric planners can be strengthened through optimization, enabling robust performance at scale in complex multi-agent environments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2025_2508.08264v1_Forecast_Driven_MPC_for_Decentralized_Multi_Robot_Collision.pdf", "source": "library", "fig": "figures/x-forecastdrivenmpcfordecentralizedmultiro.webp"}, {"id": "x-heatsahierarchicalframeworkforefficienta", "title": "HEATS: A Hierarchical Framework for Efficient Autonomous Target Search with Mobile Manipulators", "tree": "systems", "branch": "bimanual", "authors": ["Hao Zhang", "Yifei Wang", "Weifan Zhang", "Yu Wang", "Haoyao Chen"], "first_author": "Hao Zhang", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-11", "topics": ["open hardware", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2503.07986", "doi": "", "url": "https://arxiv.org/abs/2503.07986", "code": "", "site": "", "abstract": "Utilizing robots for autonomous target search in complex and unknown environments can greatly improve the efficiency of search and rescue missions. However, existing methods have shown inadequate performance due to hardware platform limitations, inefficient viewpoint selection strategies, and conservative motion planning. In this work, we propose HEATS, which enhances the search capability of mobile manipulators in complex and unknown environments. We design a target viewpoint planner tailored to the strengths of mobile manipulators, ensuring efficient and comprehensive viewpoint planning. Supported by this, a whole-body motion planner integrates global path search with local IPC optimization, enabling the mobile manipulator to safely and agilely visit target viewpoints, significantly improving search performance. We present extensive simulated and real-world tests, in which our method demonstrates reduced search time, higher target search completeness, and lower movement cost compared to classic and state-of-the-art approaches. Our method will be open-sourced for community benefit.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2025_2503.07986v1_HEATS_A_Hierarchical_Framework_for_Efficient_Autonomous_Targ.pdf", "source": "library", "fig": "figures/x-heatsahierarchicalframeworkforefficienta.webp"}, {"id": "x-may19232025atlantausahoverversatileneura", "title": "HOVER: Versatile Neural Whole-Body Controller for Humanoid Robots", "tree": "rl", "branch": "loco-manip", "authors": ["Tairan He", "Wenli Xiao", "Toru Lin", "Zhengyi Luo", "Zhenjia Xu", "Zhenyu Jiang", "Jan Kautz", "Changliu Liu", "Guanya Shi", "Xiaolong Wang", "Linxi Fan", "Yuke Zhu"], "first_author": "Tairan He", "corresponding": [], "institutions": ["Carnegie Mellon University"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2025-05-19", "topics": ["loco-manipulation", "RL"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/ICRA55743.2025.11128549", "url": "https://doi.org/10.1109/ICRA55743.2025.11128549", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/hover.pdf", "source": "library", "fig": "figures/x-may19232025atlantausahoverversatileneura.webp"}, {"id": "x-heavyliftingtasksviahapticteleoperationo", "title": "Heavy lifting tasks via haptic teleoperation of a wheeled humanoid", "tree": "rl", "branch": "motion-imitation", "authors": ["Amartya Purushottam", "Jack Yan", "Christopher Yu", "Joao Ramos"], "first_author": "Amartya Purushottam", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2025, "date": "2025-05-26", "topics": ["motion imitation", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2505.19530", "doi": "", "url": "https://arxiv.org/abs/2505.19530", "code": "", "site": "", "abstract": "Humanoid robots can support human workers in physically demanding environments by performing tasks that require whole-body coordination, such as lifting and transporting heavy objects.These tasks, which we refer to as Dynamic Mobile Manipulation (DMM), require the simultaneous control of locomotion, manipulation, and posture under dynamic interaction forces. This paper presents a teleoperation framework for DMM on a height-adjustable wheeled humanoid robot for carrying heavy payloads. A Human-Machine Interface (HMI) enables whole-body motion retargeting from the human pilot to the robot by capturing the motion of the human and applying haptic feedback. The pilot uses body motion to regulate robot posture and locomotion, while arm movements guide manipulation.Real time haptic feedback delivers end effector wrenches and balance related cues, closing the loop between human perception and robot environment interaction. We evaluate the different telelocomotion mappings that offer varying levels of balance assistance, allowing the pilot to either manually or automatically regulate the robot's lean in response to payload-induced disturbances. The system is validated in experiments involving dynamic lifting of barbells and boxes up to 2.5 kg (21% of robot mass), demonstrating coordinated whole-body control, height variation, and disturbance handling under pilot guidance. Video demo can be found at: https://youtu.be/jF270_bG1h8?feature=shared", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2025_2505.19530v1_Heavy_lifting_tasks_via_haptic_teleoperation_of_a_wheeled_hu.pdf", "source": "library", "fig": "figures/x-heavyliftingtasksviahapticteleoperationo.webp"}, {"id": "artic-tools", "title": "In-Hand Manipulation of Articulated Tools with Dexterous Robot Hands with Sim-to-Real Transfer", "tree": "rl", "branch": "manipulation", "authors": ["Soofiyan Atar", "Daniel Huang", "Florian Richter", "Michael Yip"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-27", "topics": ["articulated tools", "in-hand manipulation", "sim-to-real"], "paradigm": ["RL", "sim2real"], "method": [], "projects": ["geodex"], "related": ["chen-visual-inhand", "clutterdexgrasp"], "arxiv": "2509.23075", "doi": "", "code": "", "site": "", "note": "In-hand manipulation of articulated tools — a harder object class than the rigid blocks most in-hand papers use.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Reinforcement learning (RL) and sim-to-real transfer have advanced rigid-object manipulation. However, policies remain brittle for articulated mechanisms due to contact-rich dynamics that require both stable grasping and simultaneous free in-hand articulation. Furthermore, articulated objects and robot hands exhibit under-modeled joint phenomena such as friction, stiction, and backlash in real life that can increase the sim-to-real gap, and robot hands still fall short of idealized tactile sensing, both in terms of coverage, sensitivity, and specificity. In this paper, we present an original approach to learning dexterous in-hand manipulation of articulated tools that has reduced articulation and kinematic redundancy relative to the human hand. Our approach augments a simulation-trained base policy with a sensor-driven refinement learned from hardware demonstrations. This refinement conditions on proprioception and target articulation states while fusing whole-hand tactile and force-torque feedback with the policy's action intent through cross-attention. The resulting controller adapts online to instance-specific articulation properties, stabilizes contact interactions, and regulates internal forces under perturbations. We validate our method across diverse real-world tools, including scissors, pliers, minimally invasive surgical instruments, and staplers, demonstrating robust sim-to-real transfer, improved disturbance resilience, and generalization across structurally related articulated tools without precise physical modeling.", "source": "curated", "first_author": "Soofiyan Atar", "url": "https://arxiv.org/abs/2509.23075", "fig": "figures/artic-tools.webp", "figs": ["figures/artic-tools-1.webp"]}, {"id": "x-learninglonghorizonrobotmanipulationskil", "title": "Learning Long-Horizon Robot Manipulation Skills via Privileged Action", "tree": "il", "branch": "visuomotor", "authors": ["Xiaofeng Mao", "Yucheng Xu", "Zhaole Sun", "Elle Miller", "Daniel Layeghi", "Michael Mistry"], "first_author": "Xiaofeng Mao", "corresponding": [], "institutions": ["University of Edinburgh"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-02-21", "topics": ["proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2502.15442", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/196_Learning_Long_Horizon_Robo.pdf", "source": "library", "fig": "figures/x-learninglonghorizonrobotmanipulationskil.webp"}, {"id": "x-learningaunifiedpolicyforpositionandforc", "title": "Learning a Unified Policy for Position and Force Control in Legged Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Peiyuan Zhi", "Peiyang Li", "Jianqin Yin", "Baoxiong Jia", "Siyuan Huang"], "first_author": "Peiyuan Zhi", "corresponding": [], "institutions": [], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-05-27", "topics": ["loco-manipulation", "RL"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2505.20829", "doi": "", "url": "https://arxiv.org/abs/2505.20829", "code": "", "site": "", "abstract": "Robotic loco-manipulation tasks often involve contact-rich interactions with the environment, requiring the joint modeling of contact force and robot position. However, recent visuomotor policies often focus solely on learning position or force control, overlooking their co-learning. In this work, we propose the first unified policy for legged robots that jointly models force and position control learned without reliance on force sensors. By simulating diverse combinations of position and force commands alongside external disturbance forces, we use reinforcement learning to learn a policy that estimates forces from historical robot states and compensates for them through position and velocity adjustments. This policy enables a wide range of manipulation behaviors under varying force and position inputs, including position tracking, force application, force tracking, and compliant interactions. Furthermore, we demonstrate that the learned policy enhances trajectory-based imitation learning pipelines by incorporating essential contact information through its force estimation module, achieving approximately 39.5% higher success rates across four challenging contact-rich manipulation tasks compared to position-control policies. Extensive experiments on both a quadrupedal manipulator and a humanoid robot validate the versatility and robustness of the proposed policy across diverse scenarios.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/force_control_corl_best_paper.pdf", "source": "library", "fig": "figures/x-learningaunifiedpolicyforpositionandforc.webp"}, {"id": "x-mimicfuncimitatingtoolmanipulationfromas", "title": "MimicFunc: Imitating Tool Manipulation from a Single Human Video via Functional Correspondence", "tree": "il", "branch": "human-video", "authors": ["Chao Tang", "Anxing Xiao", "Yuhong Deng", "Tianrun Hu", "Wenlong Dong", "Hanbo Zhang", "David Hsu", "Hong Zhang"], "first_author": "Chao Tang", "corresponding": [], "institutions": ["NUS"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-08-19", "topics": ["human video", "proceedings 2025"], "paradigm": [], "method": [], "projects": ["r2s2r"], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2508.13534", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/565_MimicFunc_Imitating_Tool_M.pdf", "source": "library", "fig": "figures/x-mimicfuncimitatingtoolmanipulationfromas.webp"}, {"id": "x-moelocomixtureofexpertsformultitasklocom", "title": "MoE-Loco: Mixture of Experts for Multitask Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Runhan Huang", "Shaoting Zhu", "Yilun Du", "Hang Zhao"], "first_author": "Runhan Huang", "corresponding": [], "institutions": ["Harvard University", "Tsinghua University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-03-11", "topics": ["locomotion", "MOE"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2503.08564", "doi": "", "url": "https://arxiv.org/abs/2503.08564", "code": "", "site": "", "abstract": "We present MoE-Loco, a Mixture of Experts (MoE) framework for multitask locomotion for legged robots. Our method enables a single policy to handle diverse terrains, including bars, pits, stairs, slopes, and baffles, while supporting quadrupedal and bipedal gaits. Using MoE, we mitigate the gradient conflicts that typically arise in multitask reinforcement learning, improving both training efficiency and performance. Our experiments demonstrate that different experts naturally specialize in distinct locomotion behaviors, which can be leveraged for task migration and skill composition. We further validate our approach in both simulation and real-world deployment, showcasing its robustness and adaptability.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MOE/2025_Huang_MoE_locomotion.pdf", "source": "library", "fig": "figures/x-moelocomixtureofexpertsformultitasklocom.webp"}, {"id": "x-morphologicallysymmetricreinforcementlea", "title": "Morphologically Symmetric Reinforcement Learning for Ambidextrous Bimanual Manipulation", "tree": "rl", "branch": "foundations", "authors": ["Zechu Li", "Yufeng Jin", "Daniel Ordonez Apraez", "Claudio Semini", "Puze Liu", "Georgia Chalvatzaki"], "first_author": "Zechu Li", "corresponding": [], "institutions": ["TU Darmstadt", "Honda Research Institute"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "2025-05-08", "topics": ["RL", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2505.05287", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/574_Morphologically_Symmetric_.pdf", "source": "library", "fig": "figures/x-morphologicallysymmetricreinforcementlea.webp"}, {"id": "multi-loco", "title": "Multi-Loco: Unifying Multi-Embodiment Legged Locomotion via Reinforcement Learning Augmented Diffusion", "tree": "rl", "branch": "locomotion", "authors": ["Shunpeng Yang", "Zhen Fu", "Zhefeng Cao", "Guo Junde", "Patrick Wensing", "Wei Zhang", "Hua Chen"], "corresponding": [], "institutions": ["Southern University of Science and Technology", "University of Notre Dame"], "lab": "", "venue": "Rare & Special e-Zone (The Hong Kong Univers", "venue_type": "preprint", "year": 2025, "date": "2025-06-13", "topics": ["multi-embodiment", "cross-morphology locomotion"], "paradigm": ["RL", "diffusion"], "method": ["diffusion prior over gaits + RL residual"], "projects": [], "related": ["crossformer", "rudin2021"], "arxiv": "2506.11470", "doi": "", "code": "", "site": "", "note": "In your ~/Desktop/papers/RL. Relevant to the cross-embodiment thread — one policy, many leg counts.", "aka": "", "stars": 1, "curated": true, "local": "~/Desktop/papers/RL/Multi-Loco_2025_Chen.pdf", "abstract": "Generalizing locomotion policies across diverse legged robots with varying morphologies is a key challenge due to differences in observation/action dimensions and system dynamics. In this work, we propose Multi-Loco, a novel unified framework combining a morphology-agnostic generative diffusion model with a lightweight residual policy optimized via reinforcement learning (RL). The diffusion model captures morphology-invariant locomotion patterns from diverse cross-embodiment datasets, improving generalization and robustness. The residual policy is shared across all embodiments and refines the actions generated by the diffusion model, enhancing task-aware performance and robustness for real-world deployment. We evaluated our method with a rich library of four legged robots in both simulation and real-world experiments. Compared to a standard RL framework with PPO, our approach -- replacing the Gaussian policy with a diffusion model and residual term -- achieves a 10.35% average return improvement, with gains up to 13.57% in wheeled-biped locomotion tasks. These results highlight the benefits of cross-embodiment data and composite generative architectures in learning robust, generalized locomotion skills.", "source": "curated", "first_author": "Shunpeng Yang", "url": "https://repository.hkust.edu.hk/ir/usage-metrics/download/1783.1-171732", "arxiv_url": "https://arxiv.org/abs/2506.11470", "fig": "figures/multi-loco.webp", "figs": ["figures/multi-loco-1.webp"]}, {"id": "x-multidirectionalsaferectanglecorridorbas", "title": "Multi-directional Safe Rectangle Corridor-Based MPC for Nonholonomic Robots Navigation in Cluttered Environment", "tree": "systems", "branch": "planning-control", "authors": ["Yanzhang Qu", "Yunxiang Li", "Shanlin Zhong"], "first_author": "Yanzhang Qu", "corresponding": [], "institutions": ["Beijing Institute of Technology", "Chinese Academy of Sciences", "Institute of Automation"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-15", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2512.13215", "doi": "10.1109/ICARM65671.2025.11293633", "url": "https://arxiv.org/abs/2512.13215", "code": "", "site": "", "abstract": "Autonomous Mobile Robots (AMRs) have become indispensable in industrial applications due to their operational flexibility and efficiency. Navigation serves as a crucial technical foundation for accomplishing complex tasks. However, navigating AMRs in dense, cluttered, and semi-structured environments remains challenging, primarily due to nonholonomic vehicle dynamics, interactions with mixed static/dynamic obstacles, and the non-convex constrained nature of such operational spaces. To solve these problems, this paper proposes an Improved Sequential Model Predictive Control (ISMPC) navigation framework that systematically reformulates navigation tasks as sequential switched optimal control problems. The framework addresses the aforementioned challenges through two key innovations: 1) Implementation of a Multi-Directional Safety Rectangular Corridor (MDSRC) algorithm, which encodes the free space through rectangular convex regions to avoid collision with static obstacles, eliminating redundant computational burdens and accelerating solver convergence; 2) A sequential MPC navigation framework that integrates corridor constraints with barrier function constraints is proposed to achieve static and dynamic obstacle avoidance. The ISMPC navigation framework enables direct velocity generation for AMRs, simplifying traditional navigation algorithm architectures. Comparative experiments demonstrate the framework's superiority in free-space utilization ( an increase of 41.05$\\%$ in the average corridor area) while maintaining real-time computational performance (average corridors generation latency of 3 ms).", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2512.13215v1_Multi_directional_Safe_Rectangle_Corridor_Based_MPC_for_Nonh.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2512.13215", "fig": "figures/x-multidirectionalsaferectanglecorridorbas.webp"}, {"id": "x-o3affordoneshot3dobjecttoobjectaffordanc", "title": "O$^3$Afford: One-Shot 3D Object-to-Object Affordance Grounding for Generalizable Robotic Manipulation", "tree": "il", "branch": "visuomotor", "authors": [], "first_author": "", "corresponding": [], "institutions": ["University of Virginia"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2025, "date": "", "topics": ["proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2509.06233", "doi": "", "url": "https://arxiv.org/abs/2509.06233", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/187_O_3_Afford_One_Shot_3D_Obj.pdf", "source": "library", "fig": "figures/x-o3affordoneshot3dobjecttoobjectaffordanc.webp", "arxiv_url": "https://arxiv.org/abs/2509.06233"}, {"id": "x-olafbringingananimatedcharactertolifeint", "title": "Olaf: Bringing an Animated Character to Life in the Physical World", "tree": "systems", "branch": "sim-bench", "authors": ["David Müller", "Espen Knoop", "Dario Mylonopoulos", "Agon Serifi", "Michael A. Hopkins", "Ruben Grandia", "Moritz Bächer"], "first_author": "David Müller", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE RA-L", "venue_type": "preprint", "year": 2025, "date": "2025-12-18", "topics": ["simulation & benchmarks", "CRV iris"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2512.16705", "doi": "10.1109/lra.2026.3685938", "url": "https://doi.org/10.1109/lra.2026.3685938", "code": "", "site": "", "abstract": "Animated characters often move in non-physical ways and have proportions that are far from a typical walking robot. This provides an ideal platform for innovation in both mechanical design and stylized motion control. In this paper, we bring Olaf to life in the physical world, relying on reinforcement learning guided by animation references for control. To create the illusion of Olaf's feet moving along his body, we hide two asymmetric legs under a soft foam skirt. To fit actuators inside the character, we use spherical and planar linkages in the arms, mouth, and eyes. Because the walk cycle results in harsh contact sounds, we introduce additional rewards that noticeably reduce impact noise. The large head, driven by small actuators in the character's slim neck, creates a risk of overheating, amplified by the costume. To keep actuators from overheating, we feed temperature values as additional inputs to policies, introducing new rewards to keep them within bounds. We validate the efficacy of our modeling in simulation and on hardware, demonstrating an unmatched level of believability for a costumed robotic character.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/Olaf.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2512.16705", "fig": "figures/x-olafbringingananimatedcharactertolifeint.webp"}, {"id": "x-oneviewmanyworldssingleimageto3dobjectme", "title": "One View, Many Worlds: Single-Image to 3D Object Meets Generative Domain Randomization for One-Shot 6D Pose Estimation", "tree": "systems", "branch": "sim-bench", "authors": ["Zheng Geng", "Nan Wang", "Shaocong Xu", "Chongjie Ye", "Bohan Li", "Zhaoxi Chen", "Sida Peng", "Hao Zhao"], "first_author": "Zheng Geng", "corresponding": [], "institutions": ["Tsinghua University", "Shanghai Jiao Tong University", "Zhejiang University"], "lab": "", "venue": "", "venue_type": "", "year": 2025, "date": "2025-09-09", "topics": ["simulation & benchmarks", "proceedings 2025"], "paradigm": [], "method": [], "projects": ["r2s2r"], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2509.07978", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/262_One_View_Many_Worlds_Singl.pdf", "source": "library", "fig": "figures/x-oneviewmanyworldssingleimageto3dobjectme.webp"}, {"id": "pwm", "title": "PWM: Policy Learning with Multi-Task World Models", "tree": "rl", "branch": "model-based", "authors": ["Ignat Georgiev", "Varun Giridhar", "Nicklas Hansen", "Animesh Garg"], "corresponding": [], "institutions": ["Georgia Tech", "UC San Diego"], "lab": "", "venue": "ICLR", "venue_type": "conference", "year": 2025, "date": "2024-07-02", "topics": ["differentiable world models", "multi-task control"], "paradigm": ["model-based RL"], "method": [], "projects": ["wam-tactile"], "related": ["dreamer-v1", "dreamerv3", "td-mpc", "tdmpc2"], "arxiv": "2407.02466", "doi": "", "code": "", "site": "", "note": "First-order gradients through a learned smooth model — relevant if you want the residual-physics world model to be differentiable end to end.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Reinforcement Learning (RL) has made significant strides in complex tasks but struggles in multi-task settings with different embodiments. World model methods offer scalability by learning a simulation of the environment but often rely on inefficient gradient-free optimization methods for policy extraction. In contrast, gradient-based methods exhibit lower variance but fail to handle discontinuities. Our work reveals that well-regularized world models can generate smoother optimization landscapes than the actual dynamics, facilitating more effective first-order optimization. We introduce Policy learning with multi-task World Models (PWM), a novel model-based RL algorithm for continuous control. Initially, the world model is pre-trained on offline data, and then policies are extracted from it using first-order optimization in less than 10 minutes per task. PWM effectively solves tasks with up to 152 action dimensions and outperforms methods that use ground-truth dynamics. Additionally, PWM scales to an 80-task setting, achieving up to 27% higher rewards than existing baselines without relying on costly online planning. Visualizations and code are available at https://www.imgeorgiev.com/pwm/.", "source": "curated", "first_author": "Ignat Georgiev", "url": "https://arxiv.org/abs/2407.02466", "fig": "figures/pwm.webp", "figs": ["figures/pwm-1.webp"]}, {"id": "x-physicsbasedmotionimitationwithadversari", "title": "Physics-Based Motion Imitation with Adversarial Differential Discriminators", "tree": "rl", "branch": "motion-imitation", "authors": ["Ziyu Zhang", "Sergey Bashkirov", "Dun Yang", "Yi Shi", "Michael Taylor", "Xue Bin Peng"], "first_author": "Ziyu Zhang", "corresponding": [], "institutions": ["Simon Fraser University", "Sony Corporation (United States)"], "lab": "", "venue": "SIGGRAPH Asia", "venue_type": "conference", "year": 2025, "date": "2025-12-08", "topics": ["motion imitation", "RL"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1145/3757377.3763819", "url": "https://doi.org/10.1145/3757377.3763819", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/add.pdf", "source": "library", "fig": "figures/x-physicsbasedmotionimitationwithadversari.webp"}, {"id": "x-planningandcontrolfordeformablelinearobj", "title": "Planning and Control for Deformable Linear Object Manipulation", "tree": "systems", "branch": "planning-control", "authors": ["Burak Aksoy", "John T. Wen"], "first_author": "Burak Aksoy", "corresponding": [], "institutions": ["Intuitive Surgical (United States)", "Rensselaer Polytechnic Institute"], "lab": "", "venue": "IEEE Transactions on Automation Science and En", "venue_type": "preprint", "year": 2025, "date": "2025-03-06", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2503.04007", "doi": "10.1109/tase.2025.3635685", "url": "https://doi.org/10.1109/tase.2025.3635685", "code": "", "site": "", "abstract": "Manipulating a deformable linear object (DLO) such as wire, cable, and rope is a common yet challenging task due to their high degrees of freedom and complex deformation behaviors, especially in an environment with obstacles. Existing local control methods are efficient but prone to failure in complex scenarios, while precise global planners are computationally intensive and difficult to deploy. This paper presents an efficient, easy-to-deploy framework for collision-free DLO manipulation using mobile manipulators. We demonstrate the effectiveness of leveraging standard planning tools for high-dimensional DLO manipulation without requiring custom planners or extensive data-driven models. Our approach combines an off-the-shelf global planner with a real-time local controller. The global planner approximates the DLO as a series of rigid links connected by spherical joints, enabling rapid path planning without the need for problem-specific planners or large datasets. The local controller employs control barrier functions (CBFs) to enforce safety constraints, maintain the DLO integrity, prevent overstress, and handle obstacle avoidance. It compensates for modeling inaccuracies by using a state-of-the-art position-based dynamics technique that approximates physical properties like Young's and shear moduli. We validate our framework through extensive simulations and real-world demonstrations. In complex obstacle scenarios-including tent pole transport, corridor navigation, and tasks requiring varied stiffness-our method achieves a 100% success rate over thousands of trials, with significantly reduced planning times compared to state-of-the-art techniques. Real-world experiments include transportation of a tent pole and a rope using mobile manipulators. We share our ROS-based implementation to facilitate adoption in various applications.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2503.04007v1_Planning_and_Control_for_Deformable_Linear_Object_Manipulati.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2503.04007", "fig": "figures/x-planningandcontrolfordeformablelinearobj.webp"}, {"id": "x-primaryexperimentalfeedbackonacomanipula", "title": "Primary Experimental Feedback on a Co-manipulated Robotic System for Assisted Cervical Surgery", "tree": "il", "branch": "visuomotor", "authors": ["Seifeddine Sellemi", "Abdelbadiâ Chaker", "Tanguy Vendeuvre", "Térence Essomba", "Med Amine Laribi"], "first_author": "Seifeddine Sellemi", "corresponding": [], "institutions": ["Institut Pprime"], "lab": "", "venue": "Mechanisms and machine science", "venue_type": "preprint", "year": 2025, "date": "2025-10-04", "topics": ["collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2602.06541", "doi": "10.1007/978-3-032-05466-1_15", "url": "https://doi.org/10.1007/978-3-032-05466-1_15", "code": "", "site": "", "abstract": "Robotic-assisted surgery has emerged as a promising approach to improve surgical ergonomics, precision, and workflow efficiency, particularly in complex procedures such as cervical spine surgery. In this study, we evaluate the performance of a collaborative robotic system designed to assist surgeons in drilling tasks by assessing its accuracy in executing predefined trajectories. A total of 14 drillings were performed by eight experienced cervical surgeons, utilizing a robotic-assisted setup aimed at ensuring stability and alignment. The primary objective of this study is to quantify the deviations in the position and orientation of the drilling tool relative to the planned trajectory, providing insights into the system's reliability and potential impact on clinical outcomes. While the primary function of robotic assistance in surgery is to enhance surgeon comfort and procedural guidance rather than solely optimizing precision, understanding the system's accuracy remains crucial for its effective integration into surgical practices part of this primary experimental feedback, the study offers an in-depth analysis of the co-manipulated robotic system's performance, focusing on the experimental setup and error evaluation methods. The findings of this study will contribute to the ongoing development of robotic-assisted cervical surgery, highlighting both its advantages and areas for improvement in achieving safer and more efficient surgical workflows", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2026_2602.06541v1_Primary_Experimental_Feedback_on_a_Co_manipulated_Robotic_Sy.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2602.06541", "fig": "figures/x-primaryexperimentalfeedbackonacomanipula.webp"}, {"id": "x-pushsmarternotharderhierarchicalrldiffus", "title": "Push Smarter, Not Harder: Hierarchical RL-Diffusion Policy for Efficient Nonprehensile Manipulation", "tree": "systems", "branch": "sim-bench", "authors": ["Steven Caro", "Stephen L. Smith"], "first_author": "Steven Caro", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-12-10", "topics": ["simulation & benchmarks", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2512.10099", "doi": "10.1109/lra.2026.3692041/mm1", "url": "https://doi.org/10.1109/lra.2026.3692041/mm1", "code": "", "site": "", "abstract": "Nonprehensile manipulation, such as pushing objects across cluttered environments, presents a challenging control problem due to complex contact dynamics and long-horizon planning requirements. In this work, we propose HeRD, a hierarchical reinforcement learning-diffusion policy that decomposes pushing tasks into two levels: high-level goal selection and low-level trajectory generation. We employ a high-level reinforcement learning (RL) agent to select intermediate spatial goals, and a low-level goal-conditioned diffusion model to generate feasible, efficient trajectories to reach them. This architecture combines the long-term reward maximizing behaviour of RL with the generative capabilities of diffusion models. We evaluate our method in a 2D simulation environment and show that it outperforms the state-of-the-art baseline in success rate, path efficiency, and generalization across multiple environment configurations. Our results suggest that hierarchical control with generative low-level planning is a promising direction for scalable, goal-directed nonprehensile manipulation. Code, documentation, and trained models are available: https://github.com/carosteven/HeRD.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2025_2512.10099v1_Push_Smarter_Not_Harder_Hierarchical_RL_Diffusion_Policy_for.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2512.10099", "fig": "figures/x-pushsmarternotharderhierarchicalrldiffus.webp"}, {"id": "x-quadraticprogrammingbasedcontrolofmultir", "title": "Quadratic-Programming-based Control of Multi-Robot Systems for Cooperative Object Transport", "tree": "systems", "branch": "planning-control", "authors": ["Si Wu", "Zhengyan Qin", "Tengfei Liu", "Zhong-Ping Jiang"], "first_author": "Si Wu", "corresponding": [], "institutions": [], "lab": "", "venue": "European Journal of Control", "venue_type": "preprint", "year": 2025, "date": "2025-12-14", "topics": ["planning & control", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2512.12601", "doi": "10.1016/j.ejcon.2026.101598", "url": "https://doi.org/10.1016/j.ejcon.2026.101598", "code": "", "site": "", "abstract": "This paper investigates the control problem of steering a group of spherical mobile robots to cooperatively transport a spherical object. By controlling the movements of the robots to exert appropriate contact (pushing) forces, it is desired that the object follows a velocity command. To solve the problem, we first treat the robots' positions as virtual control inputs of the object, and propose a velocity-tracking controller based on quadratic programming (QP), enabling the robots to cooperatively generate desired contact forces while minimizing the sum of the contact-force magnitudes. Then, we design position-tracking controllers for the robots. By appropriately designing the objective function and the constraints for the QP, it is guaranteed that the QP admits a unique solution and the QP-based velocity-tracking controller is Lipschitz continuous. Finally, we consider the closed-loop system as an interconnection of two subsystems, corresponding to the velocity-tracking error of the object and the position-tracking error of the robots, and employ nonlinear small-gain techniques for stability analysis. The effectiveness of the proposed design is demonstrated through numerical simulations.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2025_2512.12601v1_Quadratic_Programming_based_Control_of_Multi_Robot_Systems_f.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2512.12601", "fig": "figures/x-quadraticprogrammingbasedcontrolofmultir.webp"}, {"id": "x-rlvrworldtrainingworldmodelswithreinforc", "title": "RLVR-World: Training World Models with Reinforcement Learning", "tree": "il", "branch": "world-models", "authors": ["Jialong Wu", "Shaofeng Yin", "Ningya Feng", "Mingsheng Long"], "first_author": "Jialong Wu", "corresponding": [], "institutions": ["Tsinghua University"], "lab": "", "venue": "NeurIPS", "venue_type": "conference", "year": 2025, "date": "2025-05-20", "topics": ["world model", "RL"], "paradigm": [], "method": [], "projects": ["wam-tactile"], "related": [], "arxiv": "2505.13934", "doi": "10.52202/085713-4177", "url": "https://doi.org/10.52202/085713-4177", "code": "", "site": "", "abstract": "World models predict state transitions in response to actions and are increasingly developed across diverse modalities. However, standard training objectives such as maximum likelihood estimation (MLE) often misalign with task-specific goals of world models, i.e., transition prediction metrics like accuracy or perceptual quality. In this paper, we present RLVR-World, a unified framework that leverages reinforcement learning with verifiable rewards (RLVR) to directly optimize world models for such metrics. Despite formulating world modeling as autoregressive prediction of tokenized sequences, RLVR-World evaluates metrics of decoded predictions as verifiable rewards. We demonstrate substantial performance gains on both language- and video-based world models across domains, including text games, web navigation, and robot manipulation. Our work indicates that, beyond recent advances in reasoning language models, RLVR offers a promising post-training paradigm for enhancing the utility of generative models more broadly. Code, datasets, models, and video samples are available at the project website: https://thuml.github.io/RLVR-World.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/world_model.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2505.13934", "fig": "figures/x-rlvrworldtrainingworldmodelswithreinforc.webp"}, {"id": "x-riskawaresafecontrolwithmultimodalsensin", "title": "Risk Aware Safe Control with Multi-Modal Sensing for Dynamic Obstacle Avoidance", "tree": "systems", "branch": "planning-control", "authors": ["Pei Yu Chang", "Qizhe Xu", "Vishnu Renganathan", "Qadeer Ahmed"], "first_author": "Pei Yu Chang", "corresponding": [], "institutions": [], "lab": "", "venue": "Control Engineering Practice", "venue_type": "preprint", "year": 2025, "date": "2025-11-03", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2511.01403", "doi": "10.1016/j.conengprac.2026.107114", "url": "https://doi.org/10.1016/j.conengprac.2026.107114", "code": "", "site": "", "abstract": "Safe control in dynamic traffic environments remains a major challenge for autonomous vehicles (AVs), as ego vehicle and obstacle states are inherently affected by sensing noise and estimation uncertainty. However, existing studies have not sufficiently addressed how uncertain multi-modal sensing information can be systematically incorporated into tail-risk-aware safety-critical control. To address this gap, this paper proposes a risk-aware safe control framework that integrates probabilistic state estimation with a conditional value-at-risk (CVaR) control barrier function (CBF) safety filter. Obstacle detections from cameras, LiDAR, and vehicle-to-everything (V2X) communication are combined using a Wasserstein barycenter (WB) to obtain a probabilistic state estimate. A model predictive controller generates the nominal control, which is then filtered through a CVaR-CBF quadratic program to enforce risk-aware safety constraints. The approach is evaluated through numerical studies and further validated on a full-scale AV. Results demonstrate improved safety and robustness over a baseline MPC-CBF design, with an average improvement of 12.7\\% in success rate across the evaluated scenarios.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2511.01403v2_Risk_Aware_Safe_Control_with_Multi_Modal_Sensing_for_Dynamic.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2511.01403", "fig": "figures/x-riskawaresafecontrolwithmultimodalsensin.webp"}, {"id": "x-robomatchaunifiedmobilemanipulationteleo", "title": "RoboMatch: A Unified Mobile-Manipulation Teleoperation Platform with Auto-Matching Network Architecture for Long-Horizon Tasks", "tree": "il", "branch": "retargeting", "authors": ["Hanyu Liu", "Yunsheng Ma", "Jiaxin Huang", "Keqiang Ren", "Jiayi Wen", "Yilin Zheng", "Haoru Luan", "Baishu Wan", "Pan Li", "Jiejun Hou", "Zhihua Wang", "Zhigong Song"], "first_author": "Hanyu Liu", "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2025, "date": "2025-09-10", "topics": ["teleoperation", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2509.08522", "doi": "", "url": "https://arxiv.org/abs/2509.08522", "code": "", "site": "", "abstract": "This paper presents RoboMatch, a novel unified teleoperation platform for mobile manipulation with an auto-matching network architecture, designed to tackle long-horizon tasks in dynamic environments. Our system enhances teleoperation performance, data collection efficiency, task accuracy, and operational stability. The core of RoboMatch is a cockpit-style control interface that enables synchronous operation of the mobile base and dual arms, significantly improving control precision and data collection. Moreover, we introduce the Proprioceptive-Visual Enhanced Diffusion Policy (PVE-DP), which leverages Discrete Wavelet Transform (DWT) for multi-scale visual feature extraction and integrates high-precision IMUs at the end-effector to enrich proprioceptive feedback, substantially boosting fine manipulation performance. Furthermore, we propose an Auto-Matching Network (AMN) architecture that decomposes long-horizon tasks into logical sequences and dynamically assigns lightweight pre-trained models for distributed inference. Experimental results demonstrate that our approach improves data collection efficiency by over 20%, increases task success rates by 20-30% with PVE-DP, and enhances long-horizon inference performance by approximately 40% with AMN, offering a robust solution for complex manipulation tasks. Project website: https://robomatch.github.io", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2025_2509.08522v3_RoboMatch_A_Unified_Mobile_Manipulation_Teleoperation_Platfo.pdf", "source": "library", "fig": "figures/x-robomatchaunifiedmobilemanipulationteleo.webp"}, {"id": "x-roboswapagandrivenvideodiffusionframewor", "title": "RoboSwap: A GAN-driven Video Diffusion Framework For Unsupervised Robot Arm Swapping", "tree": "systems", "branch": "sim-bench", "authors": ["Yang Bai", "Liudi Yang", "George Eskandar", "Fengyi Shen", "Dong Chen", "Mohammad Altillawi", "Ziyuan Liu", "Gitta Kutyniok"], "first_author": "Yang Bai", "corresponding": [], "institutions": ["TU Munich", "University of Freiburg"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-06-10", "topics": ["simulation & benchmarks", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2506.08632", "doi": "", "url": "https://arxiv.org/abs/2506.08632", "code": "", "site": "", "abstract": "Recent advancements in generative models have revolutionized video synthesis and editing. However, the scarcity of diverse, high-quality datasets continues to hinder video-conditioned robotic learning, limiting cross-platform generalization. In this work, we address the challenge of swapping a robotic arm in one video with another: a key step for crossembodiment learning. Unlike previous methods that depend on paired video demonstrations in the same environmental settings, our proposed framework, RoboSwap, operates on unpaired data from diverse environments, alleviating the data collection needs. RoboSwap introduces a novel video editing pipeline integrating both GANs and diffusion models, combining their isolated advantages. Specifically, we segment robotic arms from their backgrounds and train an unpaired GAN model to translate one robotic arm to another. The translated arm is blended with the original video background and refined with a diffusion model to enhance coherence, motion realism and object interaction. The GAN and diffusion stages are trained independently. Our experiments demonstrate that RoboSwap outperforms state-of-the-art video and image editing models on three benchmarks in terms of both structural coherence and motion consistency, thereby offering a robust solution for generating reliable, cross-embodiment data in robotic learning.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2025_2506.08632v2_RoboSwap_A_GAN_driven_Video_Diffusion_Framework_For_Unsuperv.pdf", "source": "library", "fig": "figures/x-roboswapagandrivenvideodiffusionframewor.webp"}, {"id": "x-roboticworldmodelaneuralnetworksimulator", "title": "Robotic World Model: A Neural Network Simulator for Robust Policy Optimization in Robotics", "tree": "il", "branch": "world-models", "authors": ["Chenhao Li", "Andreas Krause", "Marco Hutter"], "first_author": "Chenhao Li", "corresponding": [], "institutions": ["ETH Zürich"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-01-17", "topics": ["world model", "World Model"], "paradigm": [], "method": [], "projects": ["r2s2r", "wam-tactile"], "related": [], "arxiv": "2501.10100", "doi": "", "url": "https://arxiv.org/abs/2501.10100", "code": "", "site": "", "abstract": "Learning robust and generalizable world models is crucial for enabling efficient and scalable robotic control in real-world environments. In this work, we introduce a novel framework for learning world models that accurately capture complex, partially observable, and stochastic dynamics. The proposed method employs a dual-autoregressive mechanism and self-supervised training to achieve reliable long-horizon predictions without relying on domain-specific inductive biases, ensuring adaptability across diverse robotic tasks. We further propose a policy optimization framework that leverages world models for efficient training in imagined environments and seamless deployment in real-world systems. This work advances model-based reinforcement learning by addressing the challenges of long-horizon prediction, error accumulation, and sim-to-real transfer. By providing a scalable and robust framework, the introduced methods pave the way for adaptive and efficient robotic systems in real-world applications.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/World Model/2501.10100v4.pdf", "source": "library", "fig": "figures/x-roboticworldmodelaneuralnetworksimulator.webp"}, {"id": "x-robustvisualservoingunderhumansupervisio", "title": "Robust Visual Servoing under Human Supervision for Assembly Tasks", "tree": "rl", "branch": "manipulation", "authors": ["Victor Nan Fernandez-Ayala", "Jorge Silva", "Meng Guo", "Dimos V. Dimarogonas"], "first_author": "Victor Nan Fernandez-Ayala", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-04-16", "topics": ["dexterous manipulation", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2504.12506", "doi": "", "url": "https://arxiv.org/abs/2504.12506", "code": "", "site": "", "abstract": "We propose a framework enabling mobile manipulators to reliably complete pick-and-place tasks for assembling structures from construction blocks. The picking uses an eye-in-hand visual servoing controller for object tracking with Control Barrier Functions (CBFs) to ensure fiducial markers in the blocks remain visible. An additional robot with an eye-to-hand setup ensures precise placement, critical for structural stability. We integrate human-in-the-loop capabilities for flexibility and fault correction and analyze robustness to camera pose errors, proposing adapted barrier functions to handle them. Lastly, experiments validate the framework on 6-DoF mobile arms.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2504.12506v1_Robust_Visual_Servoing_under_Human_Supervision_for_Assembly.pdf", "source": "library", "fig": "figures/x-robustvisualservoingunderhumansupervisio.webp"}, {"id": "x-sharedobjectmanipulationwithateamofcolla", "title": "Shared Object Manipulation with a Team of Collaborative Quadrupeds", "tree": "rl", "branch": "loco-manip", "authors": ["Shengzhi Wang", "Niels Dehio", "Xuanqi Zeng", "Xian Yang", "Lingwei Zhang", "Yun-Hui Liu", "K. W. Samuel Au"], "first_author": "Shengzhi Wang", "corresponding": [], "institutions": ["HKU"], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2025, "date": "2025-10-01", "topics": ["loco-manipulation", "collaborative transport"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2510.00682", "doi": "", "url": "https://arxiv.org/abs/2510.00682", "code": "", "site": "", "abstract": "Utilizing teams of multiple robots is advantageous for handling bulky objects. Many related works focus on multi-manipulator systems, which are limited by workspace constraints. In this paper, we extend a classical hybrid motion-force controller to a team of legged manipulator systems, enabling collaborative loco-manipulation of rigid objects with a force-closed grasp. Our novel approach allows the robots to flexibly coordinate their movements, achieving efficient and stable object co-manipulation and transport, validated through extensive simulations and real-world experiments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/collaborative_transport/2025_2510.00682v1_Shared_Object_Manipulation_with_a_Team_of_Collaborative_Quad.pdf", "source": "library", "fig": "figures/x-sharedobjectmanipulationwithateamofcolla.webp"}, {"id": "x-sharingtheloadautonomousmultirovercargot", "title": "Sharing the Load: Autonomous Multi-Rover Cargo Transport", "tree": "il", "branch": "visuomotor", "authors": ["Alexander Krawciw", "Luka Antonyshyn", "Sven Lilge", "Nicolas Olmedo", "Faizan Rehmatullah", "Maxime Desjardins-Goulet", "Pascal Toupin", "Timothy D. Barfoot"], "first_author": "Alexander Krawciw", "corresponding": [], "institutions": ["University of Toronto"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-10-21", "topics": ["distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2510.18766", "doi": "10.1109/XXXX.2022.1234567", "url": "https://arxiv.org/abs/2510.18766", "code": "", "site": "", "abstract": "A future lunar habitat, as part of the Artemis program, will require a significant amount of logistics infrastructure. Cargo that is transported to the Moon will need to be moved from a landing site to other key locations that may be up to 5 km away. Teach and repeat navigation is well suited to this task as utility rovers will need to repeat these cargo routes many times. One of the most significant challenges involves the modules that will be assembled together to form the habitat. Canada is studying potential Lunar Utility Vehicle (LUV) designs to carry these large payloads between the landing site and the location of the habitat. As the details of the cargo continue to evolve, using two, smaller LUVs to carry cargo together would provide high capacity and mission flexibility. In this paper, we develop and implement a distributed model-predictive controller that allows vehicles to carry cargo that is shared between them. The algorithm is compared to baselines in small-scale before being implemented onboard two 800 kg path-to-flight rovers and field tested carrying a 475 kg cargo between them. A custom cargo coupling decouples the kinematics of each vehicle while fully supporting the cargo's mass. In our field test, the rovers maintain a relative separation error of 9.2 cm and maximum error of 33.4 cm. This multi-vehicle control architecture retains the high-quality path tracking of lidar teach and repeat for each rover. We demonstrate that kinematic freedom of the vehicles allows a single controller to provide mission improvements for other operations as well.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2025_2510.18766v2_Sharing_the_Load_Autonomous_Multi_Rover_Cargo_Transport.pdf", "source": "library", "fig": "figures/x-sharingtheloadautonomousmultirovercargot.webp"}, {"id": "skild", "title": "Skild AI: Omni-bodied Robot Brain", "tree": "systems", "branch": "agentic", "authors": ["Skild AI"], "corresponding": [], "institutions": ["Skild AI"], "lab": "Skild AI (CMU spin-off)", "venue": "company blog", "venue_type": "industry", "year": 2025, "date": "", "topics": ["general robot brain", "cross-embodiment"], "paradigm": ["VLA"], "method": [], "projects": [], "related": ["crossformer", "generalist-ai", "pi05", "skild-scale"], "arxiv": "", "doi": "", "code": "", "site": "https://www.skild.ai/", "note": "CMU spin-off (Pathak/Gupta) pitching one model across embodiments. Unreviewed; included for completeness of the startup landscape.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Skild AI", "url": "https://www.skild.ai/"}, {"id": "x-skyvlnvisionandlanguagenavigationandnmpc", "title": "SkyVLN: Vision-and-Language Navigation and NMPC Control for UAVs in Urban Environments", "tree": "systems", "branch": "planning-control", "authors": ["Tianshun Li", "Tianyi Huai", "Zhen Li", "Yichun Gao", "Haoang Li", "Xinhu Zheng"], "first_author": "Tianshun Li", "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2025, "date": "2025-07-09", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2507.06564", "doi": "", "url": "https://arxiv.org/abs/2507.06564", "code": "", "site": "", "abstract": "Unmanned Aerial Vehicles (UAVs) have emerged as versatile tools across various sectors, driven by their mobility and adaptability. This paper introduces SkyVLN, a novel framework integrating vision-and-language navigation (VLN) with Nonlinear Model Predictive Control (NMPC) to enhance UAV autonomy in complex urban environments. Unlike traditional navigation methods, SkyVLN leverages Large Language Models (LLMs) to interpret natural language instructions and visual observations, enabling UAVs to navigate through dynamic 3D spaces with improved accuracy and robustness. We present a multimodal navigation agent equipped with a fine-grained spatial verbalizer and a history path memory mechanism. These components allow the UAV to disambiguate spatial contexts, handle ambiguous instructions, and backtrack when necessary. The framework also incorporates an NMPC module for dynamic obstacle avoidance, ensuring precise trajectory tracking and collision prevention. To validate our approach, we developed a high-fidelity 3D urban simulation environment using AirSim, featuring realistic imagery and dynamic urban elements. Extensive experiments demonstrate that SkyVLN significantly improves navigation success rates and efficiency, particularly in new and unseen environments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2025_2507.06564v1_SkyVLN_Vision_and_Language_Navigation_and_NMPC_Control_for_U.pdf", "source": "library", "fig": "figures/x-skyvlnvisionandlanguagenavigationandnmpc.webp"}, {"id": "x-slotlevelroboticplacementviavisualimitat", "title": "Slot-Level Robotic Placement via Visual Imitation from Single Human Video", "tree": "il", "branch": "vla", "authors": ["Dandan Shan", "Kaichun Mo", "Wei Yang", "Yu-Wei Chao", "David Fouhey", "Dieter Fox", "Arsalan Mousavian"], "first_author": "Dandan Shan", "corresponding": [], "institutions": ["NVIDIA", "New York University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-04-02", "topics": ["VLA", "learning from human video"], "paradigm": [], "method": [], "projects": ["m2", "r2s2r"], "related": [], "arxiv": "2504.01959", "doi": "", "url": "https://arxiv.org/abs/2504.01959", "code": "", "site": "", "abstract": "The majority of modern robot learning methods focus on learning a set of pre-defined tasks with limited or no generalization to new tasks. Extending the robot skillset to novel tasks involves gathering an extensive amount of training data for additional tasks. In this paper, we address the problem of teaching new tasks to robots using human demonstration videos for repetitive tasks (e.g., packing). This task requires understanding the human video to identify which object is being manipulated (the pick object) and where it is being placed (the placement slot). In addition, it needs to re-identify the pick object and the placement slots during inference along with the relative poses to enable robot execution of the task. To tackle this, we propose SLeRP, a modular system that leverages several advanced visual foundation models and a novel slot-level placement detector Slot-Net, eliminating the need for expensive video demonstrations for training. We evaluate our system using a new benchmark of real-world videos. The evaluation results show that SLeRP outperforms several baselines and can be deployed on a real robot.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/learning_from_human_video/2025_2504.01959v1_Slot_Level_Robotic_Placement_via_Visual_Imitation_from_Singl.pdf", "source": "library", "fig": "figures/x-slotlevelroboticplacementviavisualimitat.webp"}, {"id": "x-stableobjectplacementplanningfromcontact", "title": "Stable Object Placement Planning From Contact Point Robustness", "tree": "il", "branch": "visuomotor", "authors": ["Philippe Nadeau", "Jonathan Kelly"], "first_author": "Philippe Nadeau", "corresponding": [], "institutions": ["University of Toronto"], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2025, "date": "2025-01-01", "topics": ["State Esimation "], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/tro.2025.3577049", "url": "https://doi.org/10.1109/tro.2025.3577049", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/State Esimation /Stable_Object_Placement_Planning_From_Contact_Point_Robustness.pdf", "source": "library", "fig": "figures/x-stableobjectplacementplanningfromcontact.webp"}, {"id": "x-trajboosterboostinghumanoidwholebodymani", "title": "TrajBooster: Boosting Humanoid Whole-Body Manipulation via Trajectory-Centric Learning", "tree": "il", "branch": "vla", "authors": ["Jiacheng Liu", "Pengxiang Ding", "Qihang Zhou", "Yuxuan Wu", "Da Huang", "Zimian Peng", "Wei Xiao", "Weinan Zhang", "Lixin Yang", "Cewu Lu", "Donglin Wang"], "first_author": "Jiacheng Liu", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2025, "date": "2025-09-15", "topics": ["VLA", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2509.11839", "doi": "", "url": "https://arxiv.org/abs/2509.11839", "code": "", "site": "", "abstract": "Recent Vision-Language-Action models show potential to generalize across embodiments but struggle to quickly align with a new robot's action space when high-quality demonstrations are scarce, especially for bipedal humanoids. We present TrajBooster, a cross-embodiment framework that leverages abundant wheeled-humanoid data to boost bipedal VLA. Our key idea is to use end-effector trajectories as a morphology-agnostic interface. TrajBooster (i) extracts 6D dual-arm end-effector trajectories from real-world wheeled humanoids, (ii) retargets them in simulation to Unitree G1 with a whole-body controller trained via a heuristic-enhanced harmonized online DAgger to lift low-dimensional trajectory references into feasible high-dimensional whole-body actions, and (iii) forms heterogeneous triplets that couple source vision/language with target humanoid-compatible actions to post-pre-train a VLA, followed by only 10 minutes of teleoperation data collection on the target humanoid domain. Deployed on Unitree G1, our policy achieves beyond-tabletop household tasks, enabling squatting, cross-height manipulation, and coordinated whole-body motion with markedly improved robustness and generalization. Results show that TrajBooster allows existing wheeled-humanoid data to efficiently strengthen bipedal humanoid VLA performance, reducing reliance on costly same-embodiment data while enhancing action space understanding and zero-shot skill transfer capabilities. For more details, For more details, please refer to our \\href{https://jiachengliu3.github.io/TrajBooster/}.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2025_2509.11839v3_TrajBooster_Boosting_Humanoid_Whole_Body_Manipulation_via_Tr.pdf", "source": "library", "fig": "figures/x-trajboosterboostinghumanoidwholebodymani.webp"}, {"id": "dvla", "title": "dVLA: Diffusion Vision-Language-Action Model with Multimodal Chain-of-Thought", "tree": "il", "branch": "vla", "authors": ["Junjie Wen", "Minjie Zhu", "Jiaming Liu", "Zhiyuan Liu", "Yicun Yang", "Linfeng Zhang", "Shanghang Zhang", "Yichen Zhu", "Yi Xu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "2025-09-30", "topics": ["diffusion VLA", "multimodal chain-of-thought"], "paradigm": ["VLA", "diffusion"], "method": [], "projects": [], "related": ["cosmos-reason", "dexvla", "pi0"], "arxiv": "2509.25681", "doi": "", "code": "", "site": "", "note": "Diffusion VLA with explicit multimodal chain-of-thought — the reasoning-plus-diffusion combination that several 2026 systems converge on.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "Vision-Language-Action (VLA) models are emerging as a next-generation paradigm for robotics. We introduce dVLA, a diffusion-based VLA that leverages a multimodal chain-of-thought to unify visual perception, language reasoning, and robotic control in a single system. dVLA jointly optimizes perception, language understanding, and action under a single diffusion objective, enabling stronger cross-modal reasoning and better generalization to novel instructions and objects. For practical deployment, we mitigate inference latency by incorporating two acceleration strategies, a prefix attention mask and KV caching, yielding up to around times speedup at test-time inference. We evaluate dVLA in both simulation and the real world: on the LIBERO benchmark, it achieves state-of-the-art performance with a 96.4% average success rate, consistently surpassing both discrete and continuous action policies; on a real Franka robot, it succeeds across a diverse task suite, including a challenging bin-picking task that requires multi-step planning, demonstrating robust real-world performance. Together, these results underscore the promise of unified diffusion frameworks for practical, high-performance VLA robotics.", "source": "curated", "first_author": "Junjie Wen", "url": "https://arxiv.org/abs/2509.25681", "fig": "figures/dvla.webp", "figs": ["figures/dvla-1.webp"]}, {"id": "x-archhierarchicalhybridlearningforlonghor", "title": "ARCH: Hierarchical Hybrid Learning for Long-Horizon Contact-Rich Robotic Assembly", "tree": "rl", "branch": "manipulation", "authors": ["Jiankai Sun", "Aidan Curtis", "Yang You", "Yan Xu", "Michael Koehle", "Qianzhong Chen", "Suning Huang", "Leonidas Guibas", "Sachin Chitta", "Mac Schwager", "Hui Li"], "first_author": "Jiankai Sun", "corresponding": [], "institutions": ["Stanford University", "MIT", "University of Michigan"], "lab": "", "venue": "", "venue_type": "", "year": 2024, "date": "2024-09-24", "topics": ["dexterous manipulation", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2409.16451", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/729_ARCH_Hierarchical_Hybrid_L.pdf", "source": "library", "fig": "figures/x-archhierarchicalhybridlearningforlonghor.webp"}, {"id": "x-acquiringdiverseskillsusingcurriculumrei", "title": "Acquiring Diverse Skills using Curriculum Reinforcement Learning with Mixture of Experts", "tree": "systems", "branch": "sim-bench", "authors": ["Onur Celik", "Aleksandar Taranovic", "Gerhard Neumann"], "first_author": "Onur Celik", "corresponding": [], "institutions": ["KIT"], "lab": "", "venue": "ICML", "venue_type": "conference", "year": 2024, "date": "2024-03-11", "topics": ["simulation & benchmarks", "MOE"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2403.06966", "doi": "", "url": "https://arxiv.org/abs/2403.06966", "code": "", "site": "", "abstract": "Reinforcement learning (RL) is a powerful approach for acquiring a good-performing policy. However, learning diverse skills is challenging in RL due to the commonly used Gaussian policy parameterization. We propose \\textbf{Di}verse \\textbf{Skil}l \\textbf{L}earning (Di-SkilL\\footnote{Videos and code are available on the project webpage: \\url{https://alrhub.github.io/di-skill-website/}}), an RL method for learning diverse skills using Mixture of Experts, where each expert formalizes a skill as a contextual motion primitive. Di-SkilL optimizes each expert and its associate context distribution to a maximum entropy objective that incentivizes learning diverse skills in similar contexts. The per-expert context distribution enables automatic curricula learning, allowing each expert to focus on its best-performing sub-region of the context space. To overcome hard discontinuities and multi-modalities without any prior knowledge of the environment's unknown context probability space, we leverage energy-based models to represent the per-expert context distributions and demonstrate how we can efficiently train them using the standard policy gradient objective. We show on challenging robot simulation tasks that Di-SkilL can learn diverse and performant skills.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MOE/2024_Celik_MoE.pdf", "source": "library", "fig": "figures/x-acquiringdiverseskillsusingcurriculumrei.webp"}, {"id": "x-may13172024yokohamajapanactformerscalabl", "title": "ActFormer: Scalable Collaborative Perception via Active Queries", "tree": "il", "branch": "visuomotor", "authors": ["Suozhi Huang", "Juexiao Zhang", "Yiming Li", "Chen Feng"], "first_author": "Suozhi Huang", "corresponding": [], "institutions": ["New York University"], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2024, "date": "2024-05-13", "topics": ["transformer"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/ICRA57147.2024.10610997", "url": "https://doi.org/10.1109/ICRA57147.2024.10610997", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/transformer/ActFormer_Scalable_Collaborative_Perception_via_Active_Queries.pdf", "source": "library", "fig": "figures/x-may13172024yokohamajapanactformerscalabl.webp"}, {"id": "x-autonomousmultipletrolleycollectionsyste", "title": "Autonomous Multiple-Trolley Collection System with Nonholonomic Robots: Design, Control, and Implementation", "tree": "systems", "branch": "planning-control", "authors": ["Peijia Xie", "Bingyi Xia", "Anjun Hu", "Ziqi Zhao", "Lingxiao Meng", "Zhirui Sun", "Xuheng Gao", "Jiankun Wang", "Max Q. -H. Meng"], "first_author": "Peijia Xie", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-01-16", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2401.08433", "doi": "", "url": "https://arxiv.org/abs/2401.08433", "code": "", "site": "", "abstract": "The intricate and multi-stage task in dynamic public spaces like luggage trolley collection in airports presents both a promising opportunity and an ongoing challenge for automated service robots. Previous research has primarily focused on handling a single trolley or individual functional components, creating a gap in providing cost-effective and efficient solutions for practical scenarios. In this paper, we propose a mobile manipulation robot incorporated with an autonomy framework for the collection and transportation of multiple trolleys that can significantly enhance operational efficiency. We address the key challenges in the trolley collection problem through the novel design of the mechanical system and the vision-based control strategy. We design a lightweight manipulator and docking mechanism, optimized for the sequential stacking and transportation of multiple trolleys. Additionally, based on the Control Lyapunov Function and Control Barrier Function, we propose a novel vision-based control with the online Quadratic Programming which significantly improves the accuracy and efficiency of the collection process. The practical application of our system is demonstrated in real world scenarios, where it successfully executes multiple-trolley collection tasks.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2024_2401.08433v1_Autonomous_Multiple_Trolley_Collection_System_with_Nonholono.pdf", "source": "library", "fig": "figures/x-autonomousmultipletrolleycollectionsyste.webp"}, {"id": "x-cusadiagpuparallelizationframeworkforsym", "title": "CusADi: A GPU Parallelization Framework for Symbolic Expressions and Optimal Control", "tree": "systems", "branch": "sim-bench", "authors": ["Se Hwan Jeon", "Seungwoo Hong", "Ho Jae Lee", "Charles Khazoom", "Sangbae Kim"], "first_author": "Se Hwan Jeon", "corresponding": [], "institutions": ["MIT", "NVIDIA"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-08-19", "topics": ["simulation & benchmarks", "Humaniod Benchmark"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2408.09662", "doi": "", "url": "https://arxiv.org/abs/2408.09662", "code": "", "site": "", "abstract": "The parallelism afforded by GPUs presents significant advantages in training controllers through reinforcement learning (RL). However, integrating model-based optimization into this process remains challenging due to the complexity of formulating and solving optimization problems across thousands of instances. In this work, we present CusADi, an extension of the CasADi symbolic framework to support the parallelization of arbitrary closed-form expressions on GPUs with CUDA. We also formulate a closed-form approximation for solving general optimal control problems, enabling large-scale parallelization and evaluation of MPC controllers. Our results show a ten-fold speedup relative to similar MPC implementation on the CPU, and we demonstrate the use of CusADi for various applications, including parallel simulation, parameter sweeps, and policy training.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/CusADi_Se Hwan Jeon 2024.pdf", "source": "library", "fig": "figures/x-cusadiagpuparallelizationframeworkforsym.webp"}, {"id": "x-dcubedlatentdiffusiontrajectoryoptimisat", "title": "D-Cubed: Latent Diffusion Trajectory Optimisation for Dexterous Deformable Manipulation", "tree": "systems", "branch": "planning-control", "authors": ["Jun Yamada", "Shaohong Zhong", "Jack Collins", "Ingmar Posner"], "first_author": "Jun Yamada", "corresponding": [], "institutions": ["University of Oxford"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-03-19", "topics": ["planning & control", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2403.12861", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/355_D_Cubed_Latent_Diffusion_T.pdf", "source": "library", "fig": "figures/x-dcubedlatentdiffusiontrajectoryoptimisat.webp"}, {"id": "x-eyesighthanddesignofafullyactuateddexter", "title": "EyeSight Hand: Design of a Fully-Actuated Dexterous Robot Hand with Integrated Vision-Based Tactile Sensors and Compliant Actuation", "tree": "rl", "branch": "manipulation", "authors": ["Branden Romero", "Hao-Shu Fang", "Pulkit Agrawal", "Edward Adelson"], "first_author": "Branden Romero", "corresponding": [], "institutions": ["MIT"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-08-12", "topics": ["dexterous manipulation", "Bionic Hand Motor"], "paradigm": [], "method": [], "projects": ["wam-tactile"], "related": [], "arxiv": "2408.06265", "doi": "", "url": "https://arxiv.org/abs/2408.06265", "code": "", "site": "", "abstract": "In this work, we introduce the EyeSight Hand, a novel 7 degrees of freedom (DoF) humanoid hand featuring integrated vision-based tactile sensors tailored for enhanced whole-hand manipulation. Additionally, we introduce an actuation scheme centered around quasi-direct drive actuation to achieve human-like strength and speed while ensuring robustness for large-scale data collection. We evaluate the EyeSight Hand on three challenging tasks: bottle opening, plasticine cutting, and plate pick and place, which require a blend of complex manipulation, tool use, and precise force application. Imitation learning models trained on these tasks, with a novel vision dropout strategy, showcase the benefits of tactile feedback in enhancing task success rates. Our results reveal that the integration of tactile sensing dramatically improves task performance, underscoring the critical role of tactile information in dexterous manipulation.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Bionic Hand Motor/Hao-Shu Fang_eyeslight_hand 2024.pdf", "source": "library", "fig": "figures/x-eyesighthanddesignofafullyactuateddexter.webp"}, {"id": "x-ffhflowdiverseanduncertaintyawaredextero", "title": "FFHFlow: Diverse and Uncertainty-Aware Dexterous Grasp Generation via Flow Variational Inference", "tree": "rl", "branch": "manipulation", "authors": ["Qian Feng", "Jianxiang Feng", "Zhaopeng Chen", "Rudolph Triebel", "Alois Knoll"], "first_author": "Qian Feng", "corresponding": [], "institutions": ["DLR", "TU Munich", "KIT"], "lab": "", "venue": "", "venue_type": "", "year": 2024, "date": "2024-07-21", "topics": ["dexterous manipulation", "proceedings 2025"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "https://arxiv.org/abs/2407.15161", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CoRL_hand/proceedings_2025/752_FFHFlow_Diverse_and_Uncert.pdf", "source": "library", "fig": "figures/x-ffhflowdiverseanduncertaintyawaredextero.webp"}, {"id": "x-hierarchicaladaptivemotionplanningwithno", "title": "Hierarchical Adaptive Motion Planning with Nonlinear Model Predictive Control for Safety-Critical Collaborative Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": ["Mohsen Sombolestan", "Quan Nguyen"], "first_author": "Mohsen Sombolestan", "corresponding": [], "institutions": ["USC"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-11-16", "topics": ["loco-manipulation", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2411.10699", "doi": "", "url": "https://arxiv.org/abs/2411.10699", "code": "", "site": "", "abstract": "As legged robots take on roles in industrial and autonomous construction, collaborative loco-manipulation is crucial for handling large and heavy objects that exceed the capabilities of a single robot. However, ensuring the safety of these multi-robot tasks is essential to prevent accidents and guarantee reliable operation. This paper presents a hierarchical control system for object manipulation using a team of quadrupedal robots. The combination of the motion planner and the decentralized locomotion controller in a hierarchical structure enables safe, adaptive planning for teams in complex scenarios. A high-level nonlinear model predictive control planner generates collision-free paths by incorporating control barrier functions, accounting for static and dynamic obstacles. This process involves calculating contact points and forces while adapting to unknown objects and terrain properties. The decentralized loco-manipulation controller then ensures each robot maintains stable locomotion and manipulation based on the planner's guidance. The effectiveness of our method is carefully examined in simulations under various conditions and validated in real-life setups with robot hardware. By modifying the object's configuration, the robot team can maneuver unknown objects through an environment containing both static and dynamic obstacles. We have made our code publicly available in an open-source repository at \\url{https://github.com/DRCL-USC/collaborative_loco_manipulation}.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2024_2411.10699v1_Hierarchical_Adaptive_Motion_Planning_with_Nonlinear_Model_P.pdf", "source": "library", "fig": "figures/x-hierarchicaladaptivemotionplanningwithno.webp"}, {"id": "x-integratingmodelbasedfootstepplanningwit", "title": "Integrating Model-Based Footstep Planning with Model-Free Reinforcement Learning for Dynamic Legged Locomotion", "tree": "rl", "branch": "locomotion", "authors": ["Ho Jae Lee", "Seungwoo Hong", "Sangbae Kim"], "first_author": "Ho Jae Lee", "corresponding": [], "institutions": ["MIT"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-08-05", "topics": ["locomotion", "Humaniod Benchmark"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2408.02662", "doi": "", "url": "https://arxiv.org/abs/2408.02662", "code": "", "site": "", "abstract": "In this work, we introduce a control framework that combines model-based footstep planning with Reinforcement Learning (RL), leveraging desired footstep patterns derived from the Linear Inverted Pendulum (LIP) dynamics. Utilizing the LIP model, our method forward predicts robot states and determines the desired foot placement given the velocity commands. We then train an RL policy to track the foot placements without following the full reference motions derived from the LIP model. This partial guidance from the physics model allows the RL policy to integrate the predictive capabilities of the physics-informed dynamics and the adaptability characteristics of the RL controller without overfitting the policy to the template model. Our approach is validated on the MIT Humanoid, demonstrating that our policy can achieve stable yet dynamic locomotion for walking and turning. We further validate the adaptability and generalizability of our policy by extending the locomotion task to unseen, uneven terrain. During the hardware deployment, we have achieved forward walking speeds of up to 1.5 m/s on a treadmill and have successfully performed dynamic locomotion maneuvers such as 90-degree and 180-degree turns.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/Ho Jae Lee 2024.pdf", "source": "library", "fig": "figures/x-integratingmodelbasedfootstepplanningwit.webp"}, {"id": "koch", "title": "Koch v1.1 Low-Cost Robot Arm", "tree": "systems", "branch": "bimanual", "authors": ["Alexander Koch", "Jess Moss", "Rémi Cadène"], "corresponding": [], "institutions": ["community", "Hugging Face"], "lab": "", "venue": "open hardware", "venue_type": "software", "year": 2024, "date": "", "topics": ["low-cost arm", "leader-follower"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["aloha", "so100"], "arxiv": "", "doi": "", "code": "https://github.com/jess-moss/koch-v1-1", "site": "", "note": "The direct predecessor of SO-100 in the LeRobot ecosystem. Historical value mostly.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Alexander Koch", "url": "https://github.com/jess-moss/koch-v1-1"}, {"id": "x-learningbasedleggedlocomotionstateofthea", "title": "Learning-based legged locomotion; state of the art and future perspectives", "tree": "rl", "branch": "locomotion", "authors": ["Sehoon Ha", "Joonho Lee", "Michiel van de Panne", "Zhaoming Xie", "Wenhao Yu", "Majid Khadiv"], "first_author": "Sehoon Ha", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Humanoids", "venue_type": "conference", "year": 2024, "date": "2024-06-03", "topics": ["locomotion", "Locomotion Survey"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2406.01152", "doi": "10.1177/ToBeAssigned", "url": "https://arxiv.org/abs/2406.01152", "code": "", "site": "", "abstract": "Legged locomotion holds the premise of universal mobility, a critical capability for many real-world robotic applications. Both model-based and learning-based approaches have advanced the field of legged locomotion in the past three decades. In recent years, however, a number of factors have dramatically accelerated progress in learning-based methods, including the rise of deep learning, rapid progress in simulating robotic systems, and the availability of high-performance and affordable hardware. This article aims to give a brief history of the field, to summarize recent efforts in learning locomotion skills for quadrupeds, and to provide researchers new to the area with an understanding of the key issues involved. With the recent proliferation of humanoid robots, we further outline the rapid rise of analogous methods for bipedal locomotion. We conclude with a discussion of open problems as well as related societal impact.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Locomotion Survey/Sehoon_2017.pdf", "source": "library", "fig": "figures/x-learningbasedleggedlocomotionstateofthea.webp"}, {"id": "x-contentmaychangepriortofinalpublicationc", "title": "NavFormer: A Transformer Architecture for Robot Target-Driven Navigation in Unknown and Dynamic Environments", "tree": "il", "branch": "visuomotor", "authors": ["Haitong Wang", "Aaron Hao Tan", "Goldie Nejat"], "first_author": "Haitong Wang", "corresponding": [], "institutions": ["University of Toronto"], "lab": "", "venue": "IEEE RA-L", "venue_type": "journal", "year": 2024, "date": "2024-06-11", "topics": ["transformer"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/lra.2024.3412638", "url": "https://doi.org/10.1109/lra.2024.3412638", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/transformer/navFormer.pdf", "source": "library", "fig": "figures/x-contentmaychangepriortofinalpublicationc.webp"}, {"id": "x-robustdynamiccontrolbarrierfunctionbased", "title": "Robust Dynamic Control Barrier Function Based Trajectory Planning for Mobile Manipulator", "tree": "systems", "branch": "planning-control", "authors": ["Lihao Xu", "Xiaogang Xiong", "Bai Yang", "Yunjiang Lou"], "first_author": "Lihao Xu", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-06-22", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2406.15806", "doi": "", "url": "https://arxiv.org/abs/2406.15806", "code": "", "site": "", "abstract": "High-dimensional robot dynamic trajectory planning poses many challenges for traditional planning algorithms. Existing planning methods suffer from issues such as long computation times, limited capacity to address intricate obstacle models, and lack of consideration for external disturbances and measurement inaccuracies in these high-dimensional systems. To tackle these challenges, this paper proposes a novel trajectory planning approach that combines Dynamic Control Barrier Function (DCBF) with a disturbance observer to create a Robust Dynamic Control Barrier Function (RDCBF) planner. This approach successfully plans trajectories in environments with complex dynamic obstacles while accounting for external disturbances and measurement uncertainties, ensuring system safety and enabling precise obstacle avoidance. Experimental results on a mobile manipulator demonstrate outstanding performance of the proposed approach.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2024_2406.15806v1_Robust_Dynamic_Control_Barrier_Function_Based_Trajectory_Pla.pdf", "source": "library", "fig": "figures/x-robustdynamiccontrolbarrierfunctionbased.webp"}, {"id": "x-safeexpeditiouswholebodycontrolofmobilem", "title": "Safe Expeditious Whole-Body Control of Mobile Manipulators for Collision Avoidance", "tree": "rl", "branch": "loco-manip", "authors": ["Bingjie Chen", "Yancong Wei", "Rihao Liu", "Chenxi Han", "Houde Liu", "Chongkun Xia", "Liang Han", "Bin Liang"], "first_author": "Bingjie Chen", "corresponding": [], "institutions": ["Tsinghua University"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-09-23", "topics": ["loco-manipulation", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2409.14775", "doi": "", "url": "https://arxiv.org/abs/2409.14775", "code": "", "site": "", "abstract": "Whole-body reactive obstacle avoidance for mobile manipulators (MM) remains an open research problem. Control Barrier Functions (CBF), combined with Quadratic Programming (QP), have become a popular approach for reactive control with safety guarantees. However, traditional CBF methods often face issues such as pseudo-equilibrium problems (PEP) and are ineffective in handling dynamic obstacles. To overcome these challenges, we introduce the Adaptive Cyclic Inequality (ACI) method. ACI takes into account both the obstacle's velocity and the robot's nominal control to define a directional safety constraint. When added to the CBF-QP, ACI helps avoid PEP and enables reliable collision avoidance in dynamic environments. We validate our approach on a MM that includes a low-dimensional mobile base and a high-dimensional manipulator, demonstrating the generality of the framework. In addition, we integrate a simple yet effective method for avoiding self-collisions, allowing the robot enabling comprehensive whole-body collision-free operation. Extensive benchmark comparisons and experiments demonstrate that our method performs well in unknown and dynamic scenarios, including difficult tasks like avoiding sticks swung by humans and rapidly thrown objects.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2024_2409.14775v3_Safe_Expeditious_Whole_Body_Control_of_Mobile_Manipulators_f.pdf", "source": "library", "fig": "figures/x-safeexpeditiouswholebodycontrolofmobilem.webp"}, {"id": "x-safeforcepositiontrackingcontrolviacontr", "title": "Safe Force/Position Tracking Control via Control Barrier Functions for Floating Base Mobile Manipulator Systems", "tree": "systems", "branch": "planning-control", "authors": ["Maryam Sharifi", "Shahab Heshmati-Alamdari"], "first_author": "Maryam Sharifi", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-04-21", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2404.13626", "doi": "", "url": "https://arxiv.org/abs/2404.13626", "code": "", "site": "", "abstract": "This paper introduces a safe force/position tracking control strategy designed for Free-Floating Mobile Manipulator Systems (MMSs) engaging in compliant contact with planar surfaces. The strategy uniquely integrates the Control Barrier Function (CBF) to manage operational limitations and safety concerns. It effectively addresses safety-critical aspects in the kinematic as well as dynamic level, such as manipulator joint limits, system velocity constraints, and inherent system dynamic uncertainties. The proposed strategy remains robust to the uncertainties of the MMS dynamic model, external disturbances, or variations in the contact stiffness model. The proposed control method has low computational demand ensures easy implementation on onboard computing systems, endorsing real-time operations. Simulation results verify the strategy's efficacy, reflecting enhanced system performance and safety.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2024_2404.13626v1_Safe_Force_Position_Tracking_Control_via_Control_Barrier_Fun.pdf", "source": "library", "fig": "figures/x-safeforcepositiontrackingcontrolviacontr.webp"}, {"id": "x-theoneringaroboticindoornavigationgenera", "title": "The One RING: a Robotic Indoor Navigation Generalist", "tree": "systems", "branch": "sim-bench", "authors": ["Ainaz Eftekhar", "Rose Hendrix", "Luca Weihs", "Jiafei Duan", "Ege Caglar", "Jordi Salvador", "Alvaro Herrasti", "Winson Han", "Eli VanderBil", "Aniruddha Kembhavi", "Ali Farhadi", "Ranjay Krishna", "Kiana Ehsani", "Kuo-Hao Zeng"], "first_author": "Ainaz Eftekhar", "corresponding": [], "institutions": ["University of Washington", "Allen Institute for AI"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-12-18", "topics": ["simulation & benchmarks", "mobile bimanual platforms"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2412.14401", "doi": "", "url": "https://arxiv.org/abs/2412.14401", "code": "", "site": "", "abstract": "Modern robots vary significantly in shape, size, and sensor configurations used to perceive and interact with their environments. However, most navigation policies are embodiment-specific--a policy trained on one robot typically fails to generalize to another, even with minor changes in body size or camera viewpoint. As custom hardware becomes increasingly common, there is a growing need for a single policy that generalizes across embodiments, eliminating the need to retrain for each specific robot. In this paper, we introduce RING (Robotic Indoor Navigation Generalist), an embodiment-agnostic policy that turns any mobile robot into an effective indoor semantic navigator. Trained entirely in simulation, RING leverages large-scale randomization over robot embodiments to enable robust generalization to many real-world platforms. To support this, we augment the AI2-THOR simulator to instantiate robots with controllable configurations, varying in body size, rotation pivot point, and camera parameters. On the visual object-goal navigation task, RING achieves strong cross-embodiment (XE) generalization--72.1% average success rate across five simulated embodiments (a 16.7% absolute improvement on the Chores-S benchmark) and 78.9% across four real-world platforms, including Stretch RE-1, LoCoBot, and Unitree Go1--matching or even surpassing embodiment-specific policies. We further deploy RING on the RB-Y1 wheeled humanoid in a real-world kitchen environment, showcasing its out-of-the-box potential for mobile manipulation platforms. (Project website: https://one-ring-policy.allen.ai)", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/mobile_bimanual_platforms/2024_2412.14401v2_The_One_RING_a_Robotic_Indoor_Navigation_Generalist.pdf", "source": "library", "fig": "figures/x-theoneringaroboticindoornavigationgenera.webp"}, {"id": "x-topologydrivenparalleltrajectoryoptimiza", "title": "Topology-Driven Parallel Trajectory Optimization in Dynamic Environments", "tree": "systems", "branch": "planning-control", "authors": ["Oscar de Groot", "Laura Ferranti", "Dariu M. Gavrila", "Javier Alonso-Mora"], "first_author": "Oscar de Groot", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2024, "date": "2024-01-11", "topics": ["planning & control", "dynamic obstacle avoidance"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2401.06021", "doi": "", "url": "https://arxiv.org/abs/2401.06021", "code": "", "site": "", "abstract": "Ground robots navigating in complex, dynamic environments must compute collision-free trajectories to avoid obstacles safely and efficiently. Nonconvex optimization is a popular method to compute a trajectory in real-time. However, these methods often converge to locally optimal solutions and frequently switch between different local minima, leading to inefficient and unsafe robot motion. In this work, We propose a novel topology-driven trajectory optimization strategy for dynamic environments that plans multiple distinct evasive trajectories to enhance the robot's behavior and efficiency. A global planner iteratively generates trajectories in distinct homotopy classes. These trajectories are then optimized by local planners working in parallel. While each planner shares the same navigation objectives, they are locally constrained to a specific homotopy class, meaning each local planner attempts a different evasive maneuver. The robot then executes the feasible trajectory with the lowest cost in a receding horizon manner. We demonstrate, on a mobile robot navigating among pedestrians, that our approach leads to faster and safer trajectories than existing planners.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/dynamic_obstacle_avoidance/2024_2401.06021v2_Topology_Driven_Parallel_Trajectory_Optimization_in_Dynamic.pdf", "source": "library", "fig": "figures/x-topologydrivenparalleltrajectoryoptimiza.webp"}, {"id": "x-integratedplanningandcontrolofroboticsur", "title": "Integrated Planning and Control of Robotic Surgical Instruments for Tasks Autonomy", "tree": "systems", "branch": "sim-bench", "authors": ["Fangxun Zhong", "Yun-Hui Liu"], "first_author": "Fangxun Zhong", "corresponding": [], "institutions": ["KIT"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-05-13", "topics": ["simulation & benchmarks", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2305.09470", "doi": "10.1177/ToBeAssigned", "url": "https://arxiv.org/abs/2305.09470", "code": "", "site": "", "abstract": "Agile maneuvers are essential for robot-enabled complex tasks such as surgical procedures. Prior explorations on surgery autonomy are limited to feasibility study of completing a single task without systematically addressing generic manipulation safety across different tasks. We present an integrated planning and control framework for 6-DoF robotic instruments for pipeline automation of surgical tasks.We leverage the geometry of a robotic instrument and propose the nodal state space (NSS) to represent the robot state in SE(3) space. Each elementary robot motion could be encoded by regulation of the state parameters via a dynamical system. This theoretically ensures that every in-process trajectory is globally feasible and stably reached to an admissible target, and the controller is of closed-form without computing 6-DoF inverse kinematics. Then, to plan the motion steps reliably, we propose an interactive (instant) goal state of the robot that transforms manipulation planning through desired path constraints into a goal-varying manipulation (GVM) problem. We detail how GVM could adaptively and smoothly plan the procedure (could proceed or rewind the process as needed) based on on-the-fly situations under dynamic or disturbed environment. Finally, we extend the above policy to characterize complete pipelines of various surgical tasks. Simulations show that our framework could smoothly solve twisted maneuvers while avoiding collisions. Physical experiments using the da Vinci Research Kit (dVRK) validates the capability of automating individual tasks including tissue debridement, dissection, and wound suturing. The results confirm good task-level consistency and reliability compared to state-of-the-art automation algorithms.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2023_2305.09470v1_Integrated_Planning_and_Control_of_Robotic_Surgical_Instrume.pdf", "source": "library", "fig": "figures/x-integratedplanningandcontrolofroboticsur.webp"}, {"id": "x-multirobotlocalmotionplanningusingdynami", "title": "Multi-Robot Local Motion Planning Using Dynamic Optimization Fabrics", "tree": "systems", "branch": "planning-control", "authors": ["Saray Bakker", "Luzia Knoedler", "Max Spahn", "Wendelin Böhmer", "Javier Alonso-Mora"], "first_author": "Saray Bakker", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-10-19", "topics": ["planning & control", "multi robot collaboration"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2310.12816", "doi": "", "url": "https://arxiv.org/abs/2310.12816", "code": "", "site": "", "abstract": "In this paper, we address the problem of real-time motion planning for multiple robotic manipulators that operate in close proximity. We build upon the concept of dynamic fabrics and extend them to multi-robot systems, referred to as Multi-Robot Dynamic Fabrics (MRDF). This geometric method enables a very high planning frequency for high-dimensional systems at the expense of being reactive and prone to deadlocks. To detect and resolve deadlocks, we propose Rollout Fabrics where MRDF are forward simulated in a decentralized manner. We validate the methods in simulated close-proximity pick-and-place scenarios with multiple manipulators, showing high success rates and real-time performance.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/multi_robot_collaboration/2023_2310.12816v1_Multi_Robot_Local_Motion_Planning_Using_Dynamic_Optimization.pdf", "source": "library", "fig": "figures/x-multirobotlocalmotionplanningusingdynami.webp"}, {"id": "x-nonholonomicmotionplanningasefficientasp", "title": "Nonholonomic Motion Planning as Efficient as Piano Mover's", "tree": "systems", "branch": "planning-control", "authors": ["David Nister", "Jaikrishna Soundararajan", "Yizhou Wang", "Harshad Sane"], "first_author": "David Nister", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-06-02", "topics": ["planning & control", "motion planning constrained"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2306.01301", "doi": "", "url": "https://arxiv.org/abs/2306.01301", "code": "", "site": "", "abstract": "We present an algorithm for non-holonomic motion planning (or 'parking a car') that is as computationally efficient as a simple approach to solving the famous Piano-mover's problem, where the non-holonomic constraints are ignored. The core of the approach is a graph-discretization of the problem. The graph-discretization is provably accurate in modeling the non-holonomic constraints, and yet is nearly as small as the straightforward regular grid discretization of the Piano-mover's problem into a 3D volume of 2D position plus angular orientation. Where the Piano mover's graph has one vertex and edges to six neighbors each, we have three vertices with a total of ten edges, increasing the graph size by less than a factor of two, and this factor does not depend on spatial or angular resolution. The local edge connections are organized so that they represent globally consistent turn and straight segments. The graph can be used with Dijkstra's algorithm, A*, value iteration or any other graph algorithm. Furthermore, the graph has a structure that lends itself to processing with deterministic massive parallelism. The turn and straight curves divide the configuration space into many parallel groups. We use this to develop a customized 'kernel-style' graph processing method. It results in an N-turn planner that requires no heuristics or load balancing and is as efficient as a simple solution to the Piano mover's problem even in sequential form. In parallel form it is many times faster than the sequential processing of the graph, and can run many times a second on a consumer grade GPU while exploring a configuration space pose grid with very high spatial and angular resolution. We prove approximation quality and computational complexity and demonstrate that it is a flexible, practical, reliable, and efficient component for a production solution.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/motion_planning_constrained/2023_2306.01301v1_Nonholonomic_Motion_Planning_as_Efficient_as_Piano_Mover_s.pdf", "source": "library", "fig": "figures/x-nonholonomicmotionplanningasefficientasp.webp"}, {"id": "x-nonlinearmodelpredictivecontrolforcooper", "title": "Nonlinear Model Predictive Control for Cooperative Transportation and Manipulation of Cable Suspended Payloads with Multiple Quadrotors", "tree": "systems", "branch": "bimanual", "authors": ["Guanrui Li", "Giuseppe Loianno"], "first_author": "Guanrui Li", "corresponding": [], "institutions": ["New York University"], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2023, "date": "2023-03-10", "topics": ["open hardware", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2303.06165", "doi": "10.1109/IROS55552.2023.10341785", "url": "https://arxiv.org/abs/2303.06165", "code": "", "site": "", "abstract": "Autonomous Micro Aerial Vehicles (MAVs) such as quadrotors equipped with manipulation mechanisms have the potential to assist humans in tasks such as construction and package delivery. Cables are a promising option for manipulation mechanisms due to their low weight, low cost, and simple design. However, designing control and planning strategies for cable mechanisms presents challenges due to indirect load actuation, nonlinear configuration space, and highly coupled system dynamics. In this paper, we propose a novel Nonlinear Model Predictive Control (NMPC) method that enables a team of quadrotors to manipulate a rigid-body payload in all 6 degrees of freedom via suspended cables. Our approach can concurrently exploit, as part of the receding horizon optimization, the available mechanical system redundancies to perform additional tasks such as inter-robot separation and obstacle avoidance while respecting payload dynamics and actuator constraints. To address real-time computational requirements and scalability, we employ a lightweight state vector parametrization that includes only payload states in all six degrees of freedom. This also enables the planning of trajectories on the $SE(3)$ manifold load configuration space, thereby also reducing planning complexity. We validate the proposed approach through simulation and real-world experiments.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2023_2303.06165v2_Nonlinear_Model_Predictive_Control_for_Cooperative_Transport.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/2303.06165", "fig": "figures/x-nonlinearmodelpredictivecontrolforcooper.webp"}, {"id": "x-planningoptimaltrajectoriesformobilemani", "title": "Planning Optimal Trajectories for Mobile Manipulators under End-effector Trajectory Continuity Constraint", "tree": "systems", "branch": "planning-control", "authors": ["Quang-Nam Nguyen", "Quang-Cuong Pham"], "first_author": "Quang-Nam Nguyen", "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2023, "date": "2023-09-21", "topics": ["planning & control", "whole body control"], "paradigm": [], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2309.12251", "doi": "", "url": "https://arxiv.org/abs/2309.12251", "code": "", "site": "", "abstract": "Mobile manipulators have been employed in many applications that are traditionally performed by either multiple fixed-base robots or a large robotic system. This capability is enabled by the mobility of the mobile base. However, the mobile base also brings redundancy to the system, which makes mobile manipulator motion planning more challenging. In this paper, we tackle the mobile manipulator motion planning problem under the end-effector trajectory continuity constraint in which the end-effector is required to traverse a continuous task-space trajectory (time-parametrized path), such as in mobile printing or spraying applications. Our method decouples the problem into: (1) planning an optimal base trajectory subject to geometric task constraints, end-effector trajectory continuity constraint, collision avoidance, and base velocity constraint; which ensures that (2) a manipulator trajectory is computed subsequently based on the obtained base trajectory. To validate our method, we propose a discrete optimal base trajectory planning algorithm to solve several mobile printing tasks in hardware experiment and simulations.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/MABEL/literatures/whole_body_control/2309.12251v2.pdf", "source": "library", "fig": "figures/x-planningoptimaltrajectoriesformobilemani.webp"}, {"id": "x-rlbasedvariablehorizonmodelpredictivecon", "title": "RL-based Variable Horizon Model Predictive Control of Multi-Robot Systems using Versatile On-Demand Collision Avoidance", "tree": "systems", "branch": "planning-control", "authors": ["Shreyash Gupta", "Abhinav Kumar", "Niladri S. Tripathy", "Suril V. Shah"], "first_author": "Shreyash Gupta", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-08-14", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2308.07071", "doi": "", "url": "https://arxiv.org/abs/2308.07071", "code": "", "site": "", "abstract": "Multi-robot systems have become very popular in recent years because of their wide spectrum of applications, ranging from surveillance to cooperative payload transportation. Model Predictive Control (MPC) is a promising controller for multi-robot control because of its preview capability and ability to handle constraints easily. The performance of the MPC widely depends on many parameters, among which the prediction horizon is the major contributor. Increasing the prediction horizon beyond a limit drastically increases the computation cost. Tuning the value of the prediction horizon can be very time-consuming, and the tuning process must be repeated for every task. Moreover, instead of using a fixed horizon for an entire task, a better balance between performance and computation cost can be established if different prediction horizons can be employed for every robot at each time step. Further, for such variable prediction horizon MPC for multiple robots, on-demand collision avoidance is the key requirement. We propose Versatile On-demand Collision Avoidance (VODCA) strategy to comply with the variable horizon model predictive control. We also present a framework for learning the prediction horizon for the multi-robot system as a function of the states of the robots using the Soft Actor-Critic (SAC) RL algorithm. The results are illustrated and validated numerically for different multi-robot tasks.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2023_2308.07071v1_RL_based_Variable_Horizon_Model_Predictive_Control_of_Multi.pdf", "source": "library", "fig": "figures/x-rlbasedvariablehorizonmodelpredictivecon.webp"}, {"id": "task2morph", "title": "Task2Morph: Differentiable Task-Inspired Framework for Contact-Aware Robot Design", "tree": "systems", "branch": "codesign", "authors": ["Xiaohan Zhang", "Yingying Yu", "Zhangjie Cao", "et al."], "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2023, "date": "", "topics": ["differentiable co-design", "contact-aware"], "paradigm": ["co-design"], "method": [], "projects": [], "related": ["codesign-rl", "diff-contact-design", "robogrammar"], "arxiv": "", "doi": "10.1109/iros55552.2023.10341360", "code": "", "site": "", "note": "Differentiable co-design with contact. Relevant if you want gradients rather than evolutionary search in the dexterity co-design loop.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Xiaohan Zhang", "url": "https://doi.org/10.1109/iros55552.2023.10341360"}, {"id": "x-datadrivenrisksensitivemodelpredictiveco", "title": "Data-Driven Risk-sensitive Model Predictive Control for Safe Navigation in Multi-Robot Systems", "tree": "systems", "branch": "planning-control", "authors": ["Atharva Navsalkar", "Ashish R. Hota"], "first_author": "Atharva Navsalkar", "corresponding": [], "institutions": [], "lab": "", "venue": "Nature", "venue_type": "journal", "year": 2022, "date": "2022-09-16", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2209.07793", "doi": "", "url": "https://arxiv.org/abs/2209.07793", "code": "", "site": "", "abstract": "Safe navigation is a fundamental challenge in multi-robot systems due to the uncertainty surrounding the future trajectory of the robots that act as obstacles for each other. In this work, we propose a principled data-driven approach where each robot repeatedly solves a finite horizon optimization problem subject to collision avoidance constraints with latter being formulated as distributionally robust conditional value-at-risk (CVaR) of the distance between the agent and a polyhedral obstacle geometry. Specifically, the CVaR constraints are required to hold for all distributions that are close to the empirical distribution constructed from observed samples of prediction error collected during execution. The generality of the approach allows us to robustify against prediction errors that arise under commonly imposed assumptions in both distributed and decentralized settings. We derive tractable finite-dimensional approximations of this class of constraints by leveraging convex and minmax duality results for Wasserstein distributionally robust optimization problems. The effectiveness of the proposed approach is illustrated in a multi-drone navigation setting implemented in Gazebo platform.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2022_2209.07793v1_Data_Driven_Risk_sensitive_Model_Predictive_Control_for_Safe.pdf", "source": "library", "fig": "figures/x-datadrivenrisksensitivemodelpredictiveco.webp"}, {"id": "x-distributedsafelearningandplanningformul", "title": "Distributed Safe Learning and Planning for Multi-robot Systems", "tree": "systems", "branch": "planning-control", "authors": ["Zhenyuan Yuan", "Minghui Zhu"], "first_author": "Zhenyuan Yuan", "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2022, "date": "2022-07-16", "topics": ["planning & control", "distributed mpc"], "paradigm": [], "method": [], "projects": ["m2"], "related": [], "arxiv": "2207.07824", "doi": "", "url": "https://arxiv.org/abs/2207.07824", "code": "", "site": "", "abstract": "This paper considers the problem of online multi-robot motion planning with general nonlinear dynamics subject to unknown external disturbances. We propose dSLAP, a distributed safe learning and planning framework that allows the robots to safely navigate through the environments by coupling online learning and motion planning. Gaussian process regression is used to online learn the disturbances with uncertainty quantification. The planning algorithm ensures collision avoidance against the learning uncertainty and utilizes set-valued analysis to achieve fast adaptation in response to the newly learned models. A set-valued model predictive control problem is then formulated and solved to return a control policy that balances between actively exploring the unknown disturbances and reaching goal regions. Sufficient conditions are established to guarantee the safety of the robots in the absence of backup policy. Monte Carlo simulations are conducted for evaluation.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/M2/literature/distributed_mpc/2022_2207.07824v8_Distributed_Safe_Learning_and_Planning_for_Multi_robot_Syste.pdf", "source": "library", "fig": "figures/x-distributedsafelearningandplanningformul.webp"}, {"id": "x-specializingversatileskilllibrariesusing", "title": "Specializing Versatile Skill Libraries using Local Mixture of Experts", "tree": "systems", "branch": "sim-bench", "authors": ["Onur Celik", "Dongzhuoran Zhou", "Ge Li", "Philipp Becker", "Gerhard Neumann"], "first_author": "Onur Celik", "corresponding": [], "institutions": ["KIT"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2021, "date": "2021-12-08", "topics": ["simulation & benchmarks", "MOE"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "2112.04216", "doi": "", "url": "https://arxiv.org/abs/2112.04216", "code": "", "site": "", "abstract": "A long-cherished vision in robotics is to equip robots with skills that match the versatility and precision of humans. For example, when playing table tennis, a robot should be capable of returning the ball in various ways while precisely placing it at the desired location. A common approach to model such versatile behavior is to use a Mixture of Experts (MoE) model, where each expert is a contextual motion primitive. However, learning such MoEs is challenging as most objectives force the model to cover the entire context space, which prevents specialization of the primitives resulting in rather low-quality components. Starting from maximum entropy reinforcement learning (RL), we decompose the objective into optimizing an individual lower bound per mixture component. Further, we introduce a curriculum by allowing the components to focus on a local context region, enabling the model to learn highly accurate skill representations. To this end, we use local context distributions that are adapted jointly with the expert primitives. Our lower bound advocates an iterative addition of new components, where new components will concentrate on local context regions not covered by the current MoE. This local and incremental learning results in a modular MoE model of high accuracy and versatility, where both properties can be scaled by adding more components on the fly. We demonstrate this by an extensive ablation and on two challenging simulated robot skill learning tasks. We compare our achieved performance to LaDiPS and HiREPS, a known hierarchical policy search method for learning diverse skills.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MOE/2022_Celik_MoE.pdf", "source": "library", "fig": "figures/x-specializingversatileskilllibrariesusing.webp"}, {"id": "hsmal", "title": "hSMAL: Detailed Horse Shape and Pose Reconstruction for Motion Pattern Recognition", "tree": "il", "branch": "human-video", "authors": ["Ci Li", "Nima Ghorbani", "Sofia Broomé", "Maheen Rashid", "Michael J. Black", "Elin Hernlund", "Hedvig Kjellström", "Silvia Zuffi"], "corresponding": [], "institutions": ["KTH Royal Institute of Technology", "Max Planck Society", "National Research Council", "Swedish University of Agricultural Sciences"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2021, "date": "2021-06-18", "topics": ["horse shape and pose", "motion pattern recognition"], "paradigm": ["representation learning"], "method": [], "projects": [], "related": ["animal-avatars", "barc", "dogmo"], "arxiv": "2106.10102", "doi": "10.48550/arxiv.2106.10102", "code": "", "site": "", "note": "The horse instance of the SMAL family. Useful mainly as evidence that the parametric-quadruped approach generalises across species, which matters if you want to co-design for a morphology between a dog and a cheetah.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "In this paper we present our preliminary work on model-based behavioral analysis of horse motion. Our approach is based on the SMAL model, a 3D articulated statistical model of animal shape. We define a novel SMAL model for horses based on a new template, skeleton and shape space learned from $37$ horse toys. We test the accuracy of our hSMAL model in reconstructing a horse from 3D mocap data and images. We apply the hSMAL model to the problem of lameness detection from video, where we fit the model to images to recover 3D pose and train an ST-GCN network on pose data. A comparison with the same network trained on mocap points illustrates the benefit of our approach.", "source": "curated", "first_author": "Ci Li", "url": "https://arxiv.org/abs/2106.10102", "fig": "figures/hsmal.webp", "figs": ["figures/hsmal-1.webp"]}, {"id": "x-imagebaseddeepreinforcementlearningforau", "title": "Image-based Deep Reinforcement Learning for Autonomous Lunar Landing", "tree": "rl", "branch": "foundations", "authors": ["Andrea Scorsoglio", "Roberto Furfaro", "Richard Linares", "Brian Gaudet"], "first_author": "Andrea Scorsoglio", "corresponding": [], "institutions": ["Massachusetts Institute of Technology", "University of Arizona"], "lab": "", "venue": "AIAA Scitech 2020 Forum", "venue_type": "", "year": 2020, "date": "2020-01-05", "topics": ["RL", "Space Robot"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.2514/6.2020-1910", "url": "https://doi.org/10.2514/6.2020-1910", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Space Robot/2020_scitech04.pdf", "source": "library", "fig": "figures/x-imagebaseddeepreinforcementlearningforau.webp"}, {"id": "x-animallocomotioncopyright2020theauthorss", "title": "Multi-expert learning of adaptive legged locomotion", "tree": "rl", "branch": "locomotion", "authors": [], "first_author": "", "corresponding": [], "institutions": [], "lab": "", "venue": "Science Robotics", "venue_type": "journal", "year": 2020, "date": "2020-10-14", "topics": ["locomotion", "MOE"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1126/scirobotics.abb2174", "url": "https://doi.org/10.1126/scirobotics.abb2174", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MOE/2020_Yang_SR_MoE_locomotion.pdf", "source": "library", "fig": "figures/x-animallocomotioncopyright2020theauthorss.webp"}, {"id": "x-delayedoutputfeedbackcontrolforgaitassis", "title": "Delayed Output Feedback Control for Gait Assistance With a Robotic Hip Exoskeleton", "tree": "rl", "branch": "locomotion", "authors": ["Bokman Lim", "Jusuk Lee", "Junwon Jang", "Kyungrock Kim", "Young Jin Park", "Keehong Seo", "Youngbo Shim"], "first_author": "Bokman Lim", "corresponding": [], "institutions": ["Samsung (South Korea)"], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2019, "date": "2019-05-16", "topics": ["locomotion", "Exo Controllers"], "paradigm": [], "method": [], "projects": ["exo"], "related": [], "arxiv": "", "doi": "10.1109/tro.2019.2913318", "url": "https://doi.org/10.1109/tro.2019.2913318", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Exo Controllers/samsung_controller.pdf", "source": "library", "fig": "figures/x-delayedoutputfeedbackcontrolforgaitassis.webp"}, {"id": "x-onlinereinforcementlearningcontrolforthe", "title": "Online Reinforcement Learning Control for the Personalization of a Robotic Knee Prosthesis", "tree": "systems", "branch": "humanoid-hw", "authors": ["Yue Wen", "Jennie Si", "Andrea Brandt", "Xiang Gao", "He Huang"], "first_author": "Yue Wen", "corresponding": [], "institutions": ["Arizona State University", "North Carolina State University", "UNC/NCSU Joint Department of Biomedical Engineering", "University of North Carolina at Chapel Hill"], "lab": "", "venue": "IEEE T-Cybernetics", "venue_type": "journal", "year": 2019, "date": "2019-01-16", "topics": ["wearable robotics", "Exo and learning"], "paradigm": [], "method": [], "projects": ["exo"], "related": [], "arxiv": "", "doi": "10.1109/tcyb.2019.2890974", "url": "https://doi.org/10.1109/tcyb.2019.2890974", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Exo and learning/Online_Reinforcement_Learning_Control_for_the_Personalization_of_a_Robotic_Knee_Prosthesis.pdf", "source": "library", "fig": "figures/x-onlinereinforcementlearningcontrolforthe.webp"}, {"id": "x-quasidirectdriveforlowcostcompliantrobot", "title": "Quasi-Direct Drive for Low-Cost Compliant Robotic Manipulation", "tree": "systems", "branch": "bimanual", "authors": ["David V. Gealy", "Stephen McKinley", "Brent Yi", "Philipp Wu", "Phillip R. Downey", "Greg Balke", "Allan Zhao", "Menglong Guo", "Rachel Thomasson", "Anthony Sinclair", "Peter Cuellar", "Zoe McCarthy", "Pieter Abbeel"], "first_author": "David V. Gealy", "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2019, "date": "2019-04-08", "topics": ["hardware", "imitation", "open hardware"], "paradigm": [], "method": [], "projects": ["mabel"], "related": [], "arxiv": "1904.03815", "doi": "10.1109/icra.2019.8794236", "url": "https://doi.org/10.1109/icra.2019.8794236", "code": "", "site": "", "abstract": "Robots must cost less and be force-controlled to enable widespread, safe deployment in unconstrained human environments. We propose Quasi-Direct Drive actuation as a capable paradigm for robotic force-controlled manipulation in human environments at low-cost. Our prototype - Blue - is a human scale 7 Degree of Freedom arm with 2kg payload. Blue can cost less than $5000. We show that Blue has dynamic properties that meet or exceed the needs of human operators: the robot has a nominal position-control bandwidth of 7.5Hz and repeatability within 4mm. We demonstrate a Virtual Reality based interface that can be used as a method for telepresence and collecting robot training demonstrations. Manufacturability, scaling, and potential use-cases for the Blue system are also addressed. Videos and additional information can be found online at berkeleyopenarms.github.io", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/imitation/1904.03815v2.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/1904.03815", "fig": "figures/x-quasidirectdriveforlowcostcompliantrobot.webp"}, {"id": "x-anintroductiontodeepreinforcementlearnin", "title": "An Introduction to Deep Reinforcement Learning", "tree": "rl", "branch": "foundations", "authors": ["Vincent François-Lavet", "Peter Henderson", "Riashat Islam", "Marc G. Bellemare", "Joëlle Pineau"], "first_author": "Vincent François-Lavet", "corresponding": [], "institutions": ["McGill University"], "lab": "", "venue": "Foundations and Trends® in Machine Learning", "venue_type": "preprint", "year": 2018, "date": "2018-11-30", "topics": ["RL", "RL"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "1811.12560", "doi": "10.1561/2200000071", "url": "https://doi.org/10.1561/2200000071", "code": "", "site": "", "abstract": "Deep reinforcement learning is the combination of reinforcement learning (RL) and deep learning. This field of research has been able to solve a wide range of complex decision-making tasks that were previously out of reach for a machine. Thus, deep RL opens up many new applications in domains such as healthcare, robotics, smart grids, finance, and many more. This manuscript provides an introduction to deep reinforcement learning models, algorithms and techniques. Particular focus is on the aspects related to generalization and how deep RL can be used for practical applications. We assume the reader is familiar with basic machine learning concepts.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/RL/textbook_intro_drl.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/1811.12560", "fig": "figures/x-anintroductiontodeepreinforcementlearnin.webp"}, {"id": "x-journalhomepagewwwelseviercomlocatejbiom", "title": "Comparison of the human-exosuit interaction using ankle moment and ankle positive power inspired walking assistance", "tree": "il", "branch": "visuomotor", "authors": ["Martin Grimmer", "Brendan Quinlivan", "Sangjun Lee", "Philippe Malcolm", "Denise Martineli Rossi", "Christopher Siviy", "Conor J. Walsh"], "first_author": "Martin Grimmer", "corresponding": ["Martin Grimmer"], "institutions": ["Harvard University", "Harvard University Press", "Technische Universität Darmstadt", "Universidade de São Paulo", "University of Nebraska at Omaha"], "lab": "", "venue": "J. Biomechanics", "venue_type": "journal", "year": 2018, "date": "2018-11-20", "topics": ["Exo and learning"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1016/j.jbiomech.2018.11.023", "url": "https://doi.org/10.1016/j.jbiomech.2018.11.023", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Exo and learning/1-s2.0-S0021929018308583-main-2.pdf", "source": "library", "fig": "figures/x-journalhomepagewwwelseviercomlocatejbiom.webp"}, {"id": "x-deepreinforcementlearningforsixdegreeoff", "title": "Deep Reinforcement Learning for Six Degree-of-Freedom Planetary Powered Descent and Landing", "tree": "systems", "branch": "sim-bench", "authors": ["Brian Gaudet", "Richard Linares", "Roberto Furfaro"], "first_author": "Brian Gaudet", "corresponding": [], "institutions": ["Massachusetts Institute of Technology", "University of Arizona"], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2018, "date": "2018-10-20", "topics": ["simulation & benchmarks", "Space Robot"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "1810.08719", "doi": "", "url": "https://arxiv.org/abs/1810.08719", "code": "", "site": "", "abstract": "Future Mars missions will require advanced guidance, navigation, and control algorithms for the powered descent phase to target specific surface locations and achieve pinpoint accuracy (landing error ellipse $<$ 5 m radius). The latter requires both a navigation system capable of estimating the lander's state in real-time and a guidance and control system that can map the estimated lander state to a commanded thrust for each lander engine. In this paper, we present a novel integrated guidance and control algorithm designed by applying the principles of reinforcement learning theory. The latter is used to learn a policy mapping the lander's estimated state directly to a commanded thrust for each engine, with the policy resulting in accurate and fuel-efficient trajectories. Specifically, we use proximal policy optimization, a policy gradient method, to learn the policy. Another contribution of this paper is the use of different discount rates for terminal and shaping rewards, which significantly enhances optimization performance. We present simulation results demonstrating the guidance and control system's performance in a 6-DOF simulation environment and demonstrate robustness to noise and system parameter uncertainty.", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Space Robot/1810.08719v1-2.pdf", "source": "library", "arxiv_url": "https://arxiv.org/abs/1810.08719", "fig": "figures/x-deepreinforcementlearningforsixdegreeoff.webp"}, {"id": "x-anthropomorphiclowinertiahighstiffnessma", "title": "Anthropomorphic Low-Inertia High-Stiffness Manipulator for High-Speed Safe Interaction", "tree": "systems", "branch": "sim-bench", "authors": ["Yong-Jae Kim"], "first_author": "Yong-Jae Kim", "corresponding": ["Yong-Jae Kim"], "institutions": ["Korea University of Technology and Education"], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": 2017, "date": "2017-08-25", "topics": ["simulation & benchmarks", "Humaniod Benchmark"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/tro.2017.2732354", "url": "https://doi.org/10.1109/tro.2017.2732354", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/cable_arm_Yong-Jae Kim 2015.pdf", "source": "library", "fig": "figures/x-anthropomorphiclowinertiahighstiffnessma.webp"}, {"id": "letrons", "title": "Letrons: a drivable BMW that converts into a 4.5 m robot", "tree": "systems", "branch": "codesign", "authors": ["Letvision (Levent Erenler)"], "corresponding": [], "institutions": ["Letvision"], "lab": "Letvision", "venue": "industry", "venue_type": "industry", "year": 2016, "date": "2016-09-01", "topics": ["car-robot conversion", "show robot"], "paradigm": ["hardware"], "method": [], "projects": [], "related": ["jdeite", "robosen"], "arxiv": "", "doi": "", "code": "", "site": "https://www.letrons.com/", "note": "Twelve engineers, eight months, a real BMW that stands up and articulates its arms and fingers — but it <b>does not walk</b>, and it is remote-operated. Useful as a mechanical-packaging reference and as a reminder of where the difficulty actually sits: standing up is not the hard part, balancing and walking afterwards is.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Letvision (Levent Erenler)", "url": "https://www.letrons.com/"}, {"id": "x-biomedresearchinternational", "title": "Human Locomotion under Reduced Gravity Conditions: Biomechanical and Neurophysiological Considerations", "tree": "il", "branch": "visuomotor", "authors": ["Francesca Sylos‐Labini", "Francesco Lacquaniti", "Yuri P. Ivanenko"], "first_author": "Francesca Sylos‐Labini", "corresponding": ["Yuri P. Ivanenko"], "institutions": ["Fondazione Santa Lucia", "University of Rome Tor Vergata"], "lab": "", "venue": "BioMed Research International", "venue_type": "", "year": 2014, "date": "2014-01-01", "topics": ["Space Robot"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1155/2014/547242", "url": "https://doi.org/10.1155/2014/547242", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Space Robot/BMRI2014-547242.pdf", "source": "library", "fig": "figures/x-biomedresearchinternational.webp"}, {"id": "x-convexprogrammingapproachtopowereddescen", "title": "Convex Programming Approach to Powered Descent Guidance for Mars Landing", "tree": "systems", "branch": "planning-control", "authors": ["Behçet Açıkmeşe", "Scott Ploen"], "first_author": "Behçet Açıkmeşe", "corresponding": [], "institutions": ["Analysis Group (United States)", "California Institute of Technology", "Jet Propulsion Laboratory"], "lab": "", "venue": "JGCD", "venue_type": "journal", "year": 2007, "date": "2007-08-22", "topics": ["planning & control", "Space Robot"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.2514/1.27553", "url": "https://doi.org/10.2514/1.27553", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Space Robot/_Convexprogramming_pdgforMarslanding.pdf", "source": "library", "fig": "figures/x-convexprogrammingapproachtopowereddescen.webp"}, {"id": "x-atargetbasedextrinsiccalibrationframewor", "title": "A Target-Based Extrinsic Calibration Framework for Non-Overlapping Camera-Lidar Systems Using a Motion Capture System", "tree": "systems", "branch": "perception", "authors": ["Huaiyuan Weng Nicholas Charron"], "first_author": "Huaiyuan Weng Nicholas Charron", "corresponding": [], "institutions": ["UCLA", "University of Toronto", "University of Waterloo"], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["perception & estimation", "proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.58a61852", "url": "https://doi.org/10.21428/d82e957c.58a61852", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/7_A_Target_Based_Extrinsic_Cal-31747929083285-2.pdf", "source": "library", "fig": "figures/x-atargetbasedextrinsiccalibrationframewor.webp"}, {"id": "x-manipulationcopyright2025theauthorssomer", "title": "A review of learning-­based dynamics models for robotic manipulation", "tree": "il", "branch": "world-models", "authors": ["Bo Ai", "Stephen Tian", "Haochen Shi", "Yixuan Wang", "Tobias Pfaff", "Cheston Tan", "Henrik I. Christensen", "Hao Su", "Jiajun Wu", "Yunzhu Li"], "first_author": "Bo Ai", "corresponding": [], "institutions": ["New York University"], "lab": "", "venue": "Science Robotics", "venue_type": "journal", "year": null, "date": "", "topics": ["world model", "World Model"], "paradigm": [], "method": [], "projects": ["wam-tactile"], "related": [], "arxiv": "", "doi": "10.1126/scirobotics.adt1497", "url": "https://doi.org/10.1126/scirobotics.adt1497", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/World Model/scirobotics.adt1497-3.pdf", "source": "library", "fig": "figures/x-manipulationcopyright2025theauthorssomer.webp"}, {"id": "x-chosencontrastivehypothesisselectionform", "title": "CHOSEN: Contrastive Hypothesis Selection for Multi-View Depth Refinement", "tree": "systems", "branch": "perception", "authors": ["Di Qiu", "Yinda Zhang", "Thabo Beeler", "Vladimir Tankovich", "Christian Häne", "Sean Fanello", "Christoph Rhemann", "Sergio Orts-Escolano"], "first_author": "Di Qiu", "corresponding": [], "institutions": [], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["perception & estimation", "proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.024f76dd", "url": "https://doi.org/10.21428/d82e957c.024f76dd", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/3_CHOSEN_Contrastive_Hypothesi-01747095731708-3.pdf", "source": "library", "fig": "figures/x-chosencontrastivehypothesisselectionform.webp"}, {"id": "x-catheterpciinterventionsim2real", "title": "Catheter PCI Intervention Sim2Real", "tree": "systems", "branch": "sim-bench", "authors": [], "first_author": "", "corresponding": [], "institutions": [], "lab": "", "venue": "", "venue_type": "", "year": null, "date": "", "topics": ["simulation & benchmarks", "Surgrical Robot"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Surgrical Robot/Catheter_Sim2Real.pdf", "source": "library", "fig": "figures/x-catheterpciinterventionsim2real.webp"}, {"id": "x-changeofsceneryunsupervisedlidarchangede", "title": "Change of Scenery: Unsupervised LiDAR Change Detection for Mobile Robots", "tree": "systems", "branch": "perception", "authors": ["Alexander D. Krawciw", "Jordy Sehn", "Timothy D. Barfoot"], "first_author": "Alexander D. Krawciw", "corresponding": [], "institutions": [], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["perception & estimation", "proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.276daa5a", "url": "https://doi.org/10.21428/d82e957c.276daa5a", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/oia-51715789797384.pdf", "source": "library", "fig": "figures/x-changeofsceneryunsupervisedlidarchangede.webp"}, {"id": "x-dtmdifferencebasedtemporalmoduleformonoc", "title": "DTM: Difference-Based Temporal Module for Monocular Category-Level 6 DoF Object Pose Tracking", "tree": "il", "branch": "visuomotor", "authors": ["Zishen Chen", "Jochen Lang"], "first_author": "Zishen Chen", "corresponding": [], "institutions": [], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.f508d99e", "url": "https://doi.org/10.21428/d82e957c.f508d99e", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/7ri-21715788674051.pdf", "source": "library", "fig": "figures/x-dtmdifferencebasedtemporalmoduleformonoc.webp"}, {"id": "x-deepmimicexampleguideddeepreinforcementl", "title": "DeepMimic: Example-Guided Deep Reinforcement Learning of Physics-Based Character Skills", "tree": "rl", "branch": "motion-imitation", "authors": [], "first_author": "", "corresponding": [], "institutions": ["UC Berkeley", "Columbia University", "UBC"], "lab": "", "venue": "SIGGRAPH", "venue_type": "conference", "year": null, "date": "", "topics": ["motion imitation", "Humaniod Benchmark"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1145/3197517.3201311", "url": "https://doi.org/10.1145/3197517.3201311", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/DeepMimic_XUE BIN PENG 2018.pdf", "source": "library", "fig": "figures/x-deepmimicexampleguideddeepreinforcementl.webp"}, {"id": "x-designandmodelingofbioinspiredshoulderex", "title": "Design and Modeling of Bio-Inspired Shoulder Exoskeleton with Cable-Driven Quasi-Direct Drive Actuation: Misalignment Compensation and Interference Mitigation", "tree": "systems", "branch": "humanoid-hw", "authors": ["shoulder exoskeletons must be fully portable"], "first_author": "shoulder exoskeletons must be fully portable", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": null, "date": "", "topics": ["wearable robotics", "TRO Weibo Shoulder Exo"], "paradigm": [], "method": [], "projects": ["exo"], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/TRO Weibo Shoulder Exo/bio_inspired_design_of_shoulder_exo_TRO-2.pdf", "source": "library", "fig": "figures/x-designandmodelingofbioinspiredshoulderex.webp"}, {"id": "x-internationalconferenceonadvancedintelli", "title": "Design, Modeling, and Analysis of a Liquid Cooled Proprioceptive Actuator for Legged Robots", "tree": "il", "branch": "visuomotor", "authors": ["Taoyuanmin Zhu", "Joshua Hooks", "Dennis Hong the authors present a knee", "ankle that utilize propriocep-"], "first_author": "Taoyuanmin Zhu", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE/ASME International Conference on Advanc", "venue_type": "", "year": null, "date": "", "topics": ["Bionic Hand Motor"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/aim.2019.8868596", "url": "https://doi.org/10.1109/aim.2019.8868596", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Bionic Hand Motor/Taoyuanmin Zhu 2019 AIM.pdf", "source": "library", "fig": "figures/x-internationalconferenceonadvancedintelli.webp"}, {"id": "x-endtoendassistivetorquecontrolforlowerli", "title": "End-to-End Assistive Torque Control for Lower-Limb Exoskeleton via Sim-to-Real Reinforcement Learning and Curriculum Learning", "tree": "systems", "branch": "humanoid-hw", "authors": ["has significantly constrained the adaptability", "scalability of"], "first_author": "has significantly constrained the adaptability", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE T-RO", "venue_type": "journal", "year": null, "date": "", "topics": ["wearable robotics", "TRO End-to-End"], "paradigm": [], "method": [], "projects": ["exo", "r2s2r"], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/TRO End-to-End/IEEE_TRO_End_to_end_Exo_Control.pdf", "source": "library", "fig": "figures/x-endtoendassistivetorquecontrolforlowerli.webp"}, {"id": "x-fromsim2real10to40forhumanoidwholebodyco", "title": "From Sim2Real 1.0 to 4.0 for Humanoid Whole-Body Control and Loco-Manipulation", "tree": "rl", "branch": "loco-manip", "authors": [], "first_author": "", "corresponding": [], "institutions": ["Carnegie Mellon University"], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": null, "date": "", "topics": ["loco-manipulation", "Locomotion Survey"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Locomotion Survey/sim2real1-4_ICRA_workshop.pdf", "source": "library", "fig": "figures/x-fromsim2real10to40forhumanoidwholebodyco.webp"}, {"id": "x-imagespacepathfollowingcontrolusingvisua", "title": "Image Space Path Following Control Using Visual Servoing", "tree": "systems", "branch": "perception", "authors": ["Martin Jagersand Cole Dewis"], "first_author": "Martin Jagersand Cole Dewis", "corresponding": [], "institutions": ["University of Alberta"], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["perception & estimation", "proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.4ade25c5", "url": "https://doi.org/10.21428/d82e957c.4ade25c5", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/33_Image_Space_Path_Following_-41747945503536.pdf", "source": "library", "fig": "figures/x-imagespacepathfollowingcontrolusingvisua.webp"}, {"id": "x-lixieeejournaltemplateforlatex1modelingc", "title": "Modeling, Control, and Learning for Quadrupedal and Bipedal Locomotion: Perspective Review", "tree": "rl", "branch": "locomotion", "authors": ["Yan Gu", "I-Chia Chang", "Ivan Lopez-Sanchez", "Jerry: Qilong Cheng", "Fan Shi", "Guanya Shi", "Koushil Sreenath", "climb stairs"], "first_author": "Yan Gu", "corresponding": [], "institutions": [], "lab": "", "venue": "", "venue_type": "", "year": null, "date": "", "topics": ["locomotion", "Locomotion Survey"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "", "url": "", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Locomotion Survey/Control_Learning__Planning_of_Quadrupedal_and_Bipedal_Robots__Perspective_Review-8.pdf", "source": "library", "fig": "figures/x-lixieeejournaltemplateforlatex1modelingc.webp"}, {"id": "x-pathfollowingcontrollerdesignsforautonom", "title": "Path-Following Controller Designs for Autonomous and Semi-Autonomous Industrial Motor Graders", "tree": "il", "branch": "visuomotor", "authors": ["Anthony Beca Joshua A. Marshall"], "first_author": "Anthony Beca Joshua A. Marshall", "corresponding": [], "institutions": [], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.4194d20c", "url": "https://doi.org/10.21428/d82e957c.4194d20c", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/41_Path_Following_Controller_D-61747961932795.pdf", "source": "library", "fig": "figures/x-pathfollowingcontrollerdesignsforautonom.webp"}, {"id": "x-scalablemuscleactuatedhumansimulationand", "title": "Scalable Muscle-Actuated Human Simulation and Control", "tree": "systems", "branch": "sim-bench", "authors": ["Seunghwan Lee", "Moonseok Park", "Kyoungmin Lee", "Jehee Lee"], "first_author": "Seunghwan Lee", "corresponding": [], "institutions": ["Seoul National University"], "lab": "", "venue": "SIGGRAPH", "venue_type": "conference", "year": null, "date": "", "topics": ["simulation & benchmarks", "Humaniod Benchmark"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1145/3306346.3322972", "url": "https://doi.org/10.1145/3306346.3322972", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/Archieve/Humaniod Benchmark/muscle_sim_SEUNGHWAN LEE 2019.pdf", "source": "library", "fig": "figures/x-scalablemuscleactuatedhumansimulationand.webp"}, {"id": "x-visionbasedautonomousbloodsuctionwithaco", "title": "Vision-based Autonomous Blood Suction with a Concentric Tube Continuum Robot", "tree": "il", "branch": "visuomotor", "authors": ["Jinjie Sun", "Courtney Amm", "Jessica Burgner-Kahrs", "Lueder A. Kahrs"], "first_author": "Jinjie Sun", "corresponding": [], "institutions": ["University of Toronto"], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.e9be1327", "url": "https://doi.org/10.21428/d82e957c.e9be1327", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/38_Vision_based_Autonomous_Blo-11747946341023-2.pdf", "source": "library", "fig": "figures/x-visionbasedautonomousbloodsuctionwithaco.webp"}, {"id": "x-xnavlearningendtoendcrossembodimentnavig", "title": "X-Nav: Learning End-to-End Cross-Embodiment Navigation for Mobile Robots", "tree": "il", "branch": "visuomotor", "authors": ["Haitong Wang", "Aaron Hao Tan", "Angus Fung", "Goldie Nejat"], "first_author": "Haitong Wang", "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE RA-L", "venue_type": "", "year": null, "date": "", "topics": ["MC Lab Papers"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.1109/lra.2025.3632119", "url": "https://doi.org/10.1109/lra.2025.3632119", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/MC Lab Papers/xNav_haitong.pdf", "source": "library", "fig": "figures/x-xnavlearningendtoendcrossembodimentnavig.webp"}, {"id": "x-autopathunifiedplanningandcontrolforauto", "title": "aUToPath: Unified Planning and Control for Autonomous Vehicles in Urban Environments Using Hybrid Lattice and Free-Space Search", "tree": "il", "branch": "visuomotor", "authors": ["Tanmay P. Patel∗", "Connor Wilson∗", "Ellina R. Zhang∗", "Morgan Tran", "Chang Keun Paik", "Steven L. Waslander", "Timothy D. Barfoot"], "first_author": "Tanmay P. Patel∗", "corresponding": [], "institutions": ["University of Toronto"], "lab": "", "venue": "Conference on Robots and Vision", "venue_type": "", "year": null, "date": "", "topics": ["proceedings"], "paradigm": [], "method": [], "projects": [], "related": [], "arxiv": "", "doi": "10.21428/d82e957c.409284ce", "url": "https://doi.org/10.21428/d82e957c.409284ce", "code": "", "site": "", "abstract": "", "note": "", "stars": 1, "curated": false, "local": "~/Desktop/papers/CRV_iris/proceedings/27_aUToPath_Unified_Planning_a-41747944791063.pdf", "source": "library", "fig": "figures/x-autopathunifiedplanningandcontrolforauto.webp"}, {"id": "hrl-artic-tool", "title": "Hierarchical Reinforcement Learning for Articulated Tool Manipulation with Multifingered Hand", "tree": "rl", "branch": "manipulation", "authors": ["Wei Xu", "Yanchao Zhao", "Weichao Guo", "Xinjun Sheng"], "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2025, "date": "2025-07-09", "topics": ["articulated tools", "tweezers", "tool affordance state", "point clouds"], "paradigm": ["RL", "hierarchical"], "method": ["hierarchical RL", "point-cloud encoder"], "projects": ["geodex"], "related": ["artic-tools", "sylink-hand"], "arxiv": "2507.06822", "doi": "", "code": "", "site": "", "note": "The closest published analog to chopsticks. A low-level policy sets the tool's configuration while a high-level policy picks the tool goal state and moves the arm; a point-cloud encoder trained on synthetic data reads the tool's affordance state — the tweezer opening angle against the object size. <b>70.8%</b> on real tweezer-like picking across diverse objects. Says out loud what the field has been doing: prior work treats chopsticks as a two-finger gripper.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Wei Xu", "url": "https://arxiv.org/abs/2507.06822"}, {"id": "sylink-hand", "title": "SyLink Hand: A Synergy-Inspired Linkage-Driven Anthropomorphic Hand for Human-Like Dexterity", "tree": "systems", "branch": "hands", "authors": ["Hao Wu", "Yanzhe Wang", "Yu Feng", "Yitong Li", "Jingxiang Guo", "Jian Liu", "Jianshu Zhou"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-06-12", "topics": ["linkage-driven hand", "synergies", "scissors"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["craft-hand", "artic-tools"], "arxiv": "2606.14250", "doi": "", "code": "", "site": "", "note": "Hardware paper, useful here for one result: paper-cutting with <b>scissors</b> on a UR5e, thumb and index in the handles. Executed by scripted motion primitives, not learning — so read it as a statement of what the mechanism can do, not of what a policy can do.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Hao Wu", "url": "https://arxiv.org/abs/2606.14250"}, {"id": "craft-hand", "title": "CRAFT: A Tendon-Driven Hand with Hybrid Hard-Soft Compliance", "tree": "systems", "branch": "hands", "authors": ["Leo Lin", "Shivansh Patel", "Jay Moon", "Svetlana Lazebnik", "Unnat Jain"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-03-12", "topics": ["tendon-driven hand", "hybrid compliance"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["sylink-hand"], "arxiv": "2603.12120", "doi": "", "code": "", "site": "", "note": "Shows a <b>static chopstick hold</b> in its Figure 1. No grasping result with them — a capability photograph, and a compliance design worth reading for the anchor-stick problem.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Leo Lin", "url": "https://arxiv.org/abs/2603.12120"}, {"id": "chopsticks-styles", "title": "Learning to Use Chopsticks in Diverse Gripping Styles", "tree": "rl", "branch": "manipulation", "authors": ["Zeshi Yang", "KangKang Yin", "Libin Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "SIGGRAPH / TOG", "venue_type": "journal", "year": 2022, "date": "2022-05-28", "topics": ["chopsticks", "gripping styles", "holding-pose search"], "paradigm": ["RL", "optimization"], "method": ["Bayesian optimization", "deep RL"], "projects": ["geodex"], "related": ["chopsticks-grasp", "artic-tools"], "arxiv": "2205.14313", "doi": "", "code": "", "site": "", "note": "The method reference for chopsticks. Bayesian optimization discovers valid <b>holding poses</b> across grip styles and hand morphologies; a kinematic chopstick planner then drives deep-RL finger controllers. Simulation only — which is exactly the gap a hardware result would close.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Zeshi Yang", "url": "https://arxiv.org/abs/2205.14313"}, {"id": "chopsticks-grasp", "title": "Grasping with Chopsticks: Combating Covariate Shift in Model-free Imitation Learning for Fine Manipulation", "tree": "il", "branch": "visuomotor", "authors": ["Liyiming Ke", "Jingqiang Wang", "Tapomayukh Bhattacharjee", "Byron Boots", "Siddhartha Srinivasa"], "corresponding": [], "institutions": [], "lab": "", "venue": "ICRA", "venue_type": "conference", "year": 2021, "date": "2020-11-13", "topics": ["chopsticks", "covariate shift", "fine manipulation"], "paradigm": ["IL"], "method": ["imitation learning"], "projects": ["geodex"], "related": ["chopsticks-teleop", "cherry-picking-rl", "chopsticks-styles"], "arxiv": "2011.06719", "doi": "", "code": "", "site": "", "note": "A real 6-DoF chopstick robot, 500 teleoperated demonstrations per object, and the evaluation protocol worth stealing: success = grasp + lift + <b>hold for 1 s</b>, 25 trials on a 5&times;5 grid, on a 1 cm cube and 1.4/2 cm balls. The chopsticks are the end effector, not something a hand holds — which is the baseline any in-hand result has to beat.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Liyiming Ke", "url": "https://arxiv.org/abs/2011.06719"}, {"id": "chopsticks-teleop", "title": "Telemanipulation with Chopsticks: Analyzing Human Factors in User Demonstrations", "tree": "il", "branch": "human-video", "authors": ["Liyiming Ke", "Ajinkya Kamat", "Jingqiang Wang", "Tapomayukh Bhattacharjee", "Christoforos Mavrogiannis", "Siddhartha S. Srinivasa"], "corresponding": [], "institutions": [], "lab": "", "venue": "IROS", "venue_type": "conference", "year": 2020, "date": "2020-07-31", "topics": ["chopsticks", "teleoperation", "human factors"], "paradigm": ["teleop"], "method": [], "projects": ["geodex"], "related": ["chopsticks-grasp"], "arxiv": "2008.00101", "doi": "", "code": "", "site": "", "note": "Where the demonstrations in the ICRA paper came from, and what humans actually do with chopsticks under teleoperation. The data-collection half of the same programme.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Liyiming Ke", "url": "https://arxiv.org/abs/2008.00101"}, {"id": "cherry-picking-rl", "title": "Cherry-Picking with Reinforcement Learning: Robust Dynamic Grasping in Unstable Conditions", "tree": "rl", "branch": "manipulation", "authors": ["Yunchu Zhang", "Liyiming Ke", "Abhay Deshpande", "Abhishek Gupta", "Siddhartha Srinivasa"], "corresponding": [], "institutions": [], "lab": "", "venue": "RSS", "venue_type": "conference", "year": 2023, "date": "2023-03-09", "topics": ["chopsticks", "dynamic grasping", "unstable objects"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["chopsticks-grasp"], "arxiv": "2303.05508", "doi": "", "code": "", "site": "", "note": "The same chopstick platform pushed into dynamic, unstable grasps with RL. Read it for how far a tool-as-end-effector system can be taken, which sets the bar for doing it in-hand.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Yunchu Zhang", "url": "https://arxiv.org/abs/2303.05508"}, {"id": "hashi", "title": "HASHI: Highly Adaptable Seafood Handling Instrument for Manipulation in Industrial Settings", "tree": "systems", "branch": "interfaces", "authors": ["Austin Allison", "Nathaniel Hanson", "Sebastian Wicke", "Taşkın Padır"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2023, "date": "2023-11-03", "topics": ["chopstick-like end effector", "food handling"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["chopsticks-grasp"], "arxiv": "2311.02277", "doi": "", "code": "", "site": "", "note": "A chopstick-shaped industrial end effector for delicate food. Another data point for the tool-as-mechanism camp.", "aka": "", "stars": 1, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Austin Allison", "url": "https://arxiv.org/abs/2311.02277"}, {"id": "simtoolreal", "title": "SimToolReal: An Object-Centric Policy for Zero-Shot Dexterous Tool Manipulation", "tree": "rl", "branch": "sim2real", "authors": ["Kushal Kedia", "Tyler Ga Wei Lum", "Jeannette Bohg", "C. Karen Liu"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-18", "topics": ["tool use", "object-centric goals", "zero-shot sim-to-real"], "paradigm": ["RL"], "method": ["object-centric goal tracking"], "projects": ["geodex"], "related": ["grasp-to-act", "dex-functional-grasp"], "arxiv": "2602.16863", "doi": "", "code": "", "site": "", "note": "One sim RL policy, object-centric goal-pose tracking, zero-shot to real, code released. The open question for articulated tools is whether the object-centric formulation survives a tool with an internal DoF — goal becomes <b>tool pose plus opening angle</b>, and the object is no longer rigid.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Kushal Kedia", "url": "https://arxiv.org/abs/2602.16863"}, {"id": "grasp-to-act", "title": "Grasp to Act: Dexterous Grasping for Tool Use in Dynamic Settings", "tree": "rl", "branch": "manipulation", "authors": ["Harsh Gupta", "Mohammad Amin Mirzaee", "Wenzhen Yuan"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "2026-02-24", "topics": ["tool use", "wrench-space grasp selection", "in-grip drift"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["simtoolreal"], "arxiv": "2602.20466", "doi": "", "code": "", "site": "", "note": "Wrench-space grasp selection with RL finger corrections. The finding that matters for tools: they <b>drift in the grip</b> without active adaptation — which for chopsticks is not drift but failure, since the anchor stick moving is the whole problem.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Harsh Gupta", "url": "https://arxiv.org/abs/2602.20466"}, {"id": "dex-functional-grasp", "title": "Dexterous Functional Grasping", "tree": "rl", "branch": "manipulation", "authors": ["Ananye Agarwal", "Shagun Uppal", "Kenneth Shaw", "Deepak Pathak"], "corresponding": [], "institutions": [], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2023, "date": "2023-12-05", "topics": ["functional grasps", "tool use", "eigengrasps"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["simtoolreal", "foundational-pose"], "arxiv": "2312.02975", "doi": "", "code": "", "site": "", "note": "Grasping tools the way they are meant to be held rather than however is stable. The prerequisite for articulated tools: you cannot drive a tool's DoF from a grasp that is merely secure.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Ananye Agarwal", "url": "https://arxiv.org/abs/2312.02975"}, {"id": "pen-spinning", "title": "Lessons from Learning to Spin \"Pens\"", "tree": "rl", "branch": "manipulation", "authors": ["Jun Wang", "Ying Yuan", "Haichuan Che", "Haozhi Qi", "Yi Ma", "Jitendra Malik", "Xiaolong Wang"], "corresponding": [], "institutions": [], "lab": "", "venue": "CoRL", "venue_type": "conference", "year": 2024, "date": "2024-07-26", "topics": ["thin long objects", "in-hand rotation", "sim-to-real priors"], "paradigm": ["RL"], "method": [], "projects": ["geodex"], "related": ["anyrotate", "chopsticks-styles"], "arxiv": "2407.18902", "doi": "", "code": "", "site": "", "note": "The sim-to-real priors for <b>thin, long, slippery</b> objects — which is what a chopstick is before it is a tool. Note the id: this is 2407.18902. arXiv 2405.07391 is AnyRotate, a different paper.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Jun Wang", "url": "https://arxiv.org/abs/2407.18902"}, {"id": "anyrotate", "title": "AnyRotate: Gravity-Invariant In-Hand Object Rotation with Sim-to-Real Touch", "tree": "rl", "branch": "manipulation", "authors": ["Max Yang", "Chenghua Lu", "Alex Church", "Yijiong Lin", "Chris Ford", "Haoran Li", "Efi Psomopoulou", "David A.W. Barton", "Nathan F. Lepora"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-05-12", "topics": ["in-hand rotation", "tactile sim-to-real", "gravity invariance"], "paradigm": ["RL"], "method": ["tactile RL"], "projects": ["geodex"], "related": ["pen-spinning"], "arxiv": "2405.07391", "doi": "", "code": "", "site": "", "note": "Dense tactile in-hand rotation in any orientation. Filed here mostly to keep the record straight: this is what 2405.07391 actually is.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Max Yang", "url": "https://arxiv.org/abs/2405.07391"}, {"id": "foundational-pose", "title": "The Foundational Pose as a Selection Mechanism for the Design of Tool-Wielding Multi-Finger Robotic Hands", "tree": "systems", "branch": "codesign", "authors": ["Sunyu Wang", "Jean Oh", "Nancy S. Pollard"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2024, "date": "2024-09-21", "topics": ["tool-wielding hands", "hand design", "pose selection"], "paradigm": ["design"], "method": [], "projects": ["geodex"], "related": ["dex-functional-grasp"], "arxiv": "2409.14158", "doi": "", "code": "", "site": "", "note": "Design-side: which hand poses make a tool wieldable at all, as a selection mechanism for hand design. Unverified whether it covers <b>articulated</b> tools or real hardware — check before citing it as such.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Sunyu Wang", "url": "https://arxiv.org/abs/2409.14158"}, {"id": "soft-chopstick-hand", "title": "A Biologically Inspired Soft Robotic Hand Using Chopsticks for Grasping Tasks", "tree": "systems", "branch": "hands", "authors": ["Mariia Chepisheva", "Utku Culha", "Fumiya Iida"], "corresponding": [], "institutions": [], "lab": "", "venue": "SAB", "venue_type": "conference", "year": 2016, "date": "2016-08-01", "topics": ["soft hand", "chopsticks", "underactuation"], "paradigm": ["hardware"], "method": [], "projects": ["geodex"], "related": ["craft-hand", "chopsticks-styles"], "arxiv": "", "doi": "10.1007/978-3-319-43488-9_18", "code": "", "site": "", "note": "The only real hand-plus-chopsticks-plus-grasping result found: a 4-servo soft underactuated hand where the chopstick behaviour comes mostly from <b>passive structure</b> rather than control. Ten years old, and still the state of the art for the hardest taxonomy class — which is the whole argument for the project.", "aka": "", "stars": 2, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Mariia Chepisheva", "url": "https://doi.org/10.1007/978-3-319-43488-9_18"}, {"id": "dyna-2-1", "title": "Dyna-2.1: A Physical Agent for End-to-End Workflows", "tree": "rl", "branch": "loco-manip", "authors": ["Dyna Robotics"], "corresponding": [], "institutions": [], "lab": "", "venue": "Technical report", "venue_type": "industry", "year": 2026, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "", "doi": "", "code": "", "site": "https://www.dyna.co/dyna-2.1", "note": "Industrial report on Taku, a folding mobile bimanual platform, with an RL task-space tracking layer. See the hardware and control sections; this is a company report rather than a reproducible benchmark.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Dyna Robotics", "url": "https://www.dyna.co/dyna-2.1"}, {"id": "astribot-suite", "title": "Towards Human-level Intelligence via Human-like Whole-Body Manipulation", "tree": "systems", "branch": "mobile-manip", "authors": ["Guang Gao", "Jianan Wang", "Jinbo Zuo", "Junnan Jiang", "Jingfan Zhang", "Xianwen Zeng", "Yuejiang Zhu", "Lianyang Ma", "Ke Chen", "Minhua Sheng", "Ruirui Zhang", "Zhaohui An"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["hardware"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2507.17141", "doi": "", "code": "", "site": "", "note": "Astribot Suite connects an articulated torso, whole-body teleoperation and imitation learning. Useful platform reference; not evidence of an all-RL low-level stack.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Guang Gao", "url": "https://arxiv.org/abs/2507.17141"}, {"id": "whole-moma", "title": "Whole-Body Mobile Manipulation using Offline Reinforcement Learning on Sub-optimal Controllers", "tree": "rl", "branch": "loco-manip", "authors": ["Snehal Jauhri", "Vignesh Prasad", "Georgia Chalvatzaki"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2026, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2604.12509", "doi": "", "code": "", "site": "", "note": "WHOLE-MoMa improves randomized-controller demonstrations using offline RL, with real TIAGo++ bimanual articulated-object tasks. Its action-chunked policy is not a demonstrated liquid-stabilization controller.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Snehal Jauhri", "url": "https://arxiv.org/abs/2604.12509"}, {"id": "keep-it-upright", "title": "Keep it Upright: Model Predictive Control for Nonprehensile Object Transportation with Obstacle Avoidance on a Mobile Manipulator", "tree": "systems", "branch": "planning-control", "authors": ["Adam Heins", "Angela P. Schoellig"], "corresponding": [], "institutions": [], "lab": "", "venue": "IEEE Robotics and Automation Letters", "venue_type": "journal", "year": 2023, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["control"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2305.17484", "doi": "", "code": "https://github.com/learnsyslab/upright", "site": "https://github.com/learnsyslab/upright", "note": "Whole-body MPC for fast tray-like transport with static/dynamic obstacle avoidance. The paper reports end-effector speeds and accelerations; these must not be interpreted as base limits.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Adam Heins", "url": "https://arxiv.org/abs/2305.17484"}, {"id": "mobile-wbc-e2e", "title": "Whole-Body Control of a Mobile Manipulator using End-to-End Reinforcement Learning", "tree": "rl", "branch": "loco-manip", "authors": ["Julien Kindle", "Fadri Furrer", "Tonci Novkovic", "Jen Jen Chung", "Roland Siegwart", "Juan Nieto"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2020, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["RL"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2003.02637", "doi": "", "code": "", "site": "", "note": "Early coordinated mobile-manipulator RL, validated on RoyalPanda in narrow environments. Useful wheeled reference, distinct from bimanual payload stabilization.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Julien Kindle", "url": "https://arxiv.org/abs/2003.02637"}, {"id": "morph-pretraining", "title": "Accelerated co-design of robots through morphological pretraining", "tree": "systems", "branch": "codesign", "authors": ["Luke Strgar", "Sam Kriegman"], "corresponding": [], "institutions": [], "lab": "", "venue": "arXiv", "venue_type": "preprint", "year": 2025, "date": "", "topics": ["whole-body control", "mobile manipulation"], "paradigm": ["co-design"], "method": [], "projects": ["mabel"], "related": [], "arxiv": "2502.10862", "doi": "", "code": "", "site": "https://lukestrgar.com/codesign-mpt-project-page/", "note": "Studies controller reuse during morphology search and diversity collapse. Morphological pretraining uses differentiable simulation; transfer to mobile torso design remains a proposed application.", "aka": "", "stars": 3, "curated": true, "local": "", "abstract": "", "source": "curated", "first_author": "Luke Strgar", "url": "https://arxiv.org/abs/2502.10862"}]}