{ "schema_version": "1.0.0", "license": "CC-BY-4.0", "attribution": "CanIEmbody (https://caniembody.pages.dev)", "dataset_version": "1.0.0-seed", "generated": "2026-10-25", "data": [ { "id": "act", "name": "ACT", "aliases": [ "Action Chunking with Transformers" ], "family": "bc", "summary": "Action Chunking with Transformers: a CVAE encoder-decoder transformer with a ResNet-18 image backbone that predicts chunks of future joint actions. Single-task, trained from scratch on ~50 demonstrations; LeRobot's recommended starting policy.", "authors": "Stanford University, with Meta and UC Berkeley (Zhao, Kumar, Levine, Finn — ALOHA project)", "paper_url": "https://arxiv.org/abs/2304.13705", "params": 80000000, "params_note": "Authors state \"around 80M\" for their configuration. LeRobot's default implementation is smaller: the LeRobot paper (Table 2) lists ACT at 52M parameters.", "params_source_ref": "ev-act-paper-params", "backbone": "ResNet-18 (ImageNet-pretrained) + transformer encoder-decoder (CVAE)", "action_output": "regression", "action_chunking": true, "arch_source_ref": "ev-lerobot-act-docs-arch", "license": "Apache-2.0 (LeRobot implementation)", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "policy_type": "act", "docs_url": "https://huggingface.co/docs/lerobot/act", "note": "Present at v0.3.2, the earliest tagged LeRobot release. `since` left blank: the only earlier PyPI release (0.1.0, 2024-03, a prototype) was not inspected. Not listed by the RTC docs (RTC targets flow-matching policies).", "source_ref": "ev-lerobot-act-config" } }, "checkpoints": [ { "hub_id": "lerobot/act_aloha_sim_transfer_cube_human", "note": "ALOHA sim transfer-cube checkpoint (Apache-2.0 on the Hub)." }, { "hub_id": "lerobot/act_aloha_sim_insertion_human", "note": "ALOHA sim insertion checkpoint (Apache-2.0 on the Hub)." } ], "requirements": { "train": { "vram_gb": { "min": 2, "max": 6 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Light BC\" group (act, vqbet, tdmpc) in the LeRobot compute hardware guide. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "reference_time": "5 epochs on a ~45k-frame dataset (640×480): ~30–60 min at batch 8 on a single RTX 4090/3090; ~1–2 h at batch 8 on a single L4/A10G; ~6–14 h at batch 4 on Apple M1/M2/M3 Max (MPS) (indicative, ±50%).", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "measured": 0.21124 }, "precision": "fp32", "condition": "Peak memory at inference measured by the LeRobot paper (Table 2): 211.24 MB on both RTX 4090 and A100, full fp32. (Same table: 817.4MB CPU-only, 462MB on Apple MPS.)", "source_ref": "ev-lerobot-paper-mem" } }, "observation_space": { "state": "current robot joint positions (code requires at least one image or an environment-state input)", "language": "unsupported", "source_ref": "ev-lerobot-act-docs-inputs" }, "notes": "The ACT docs recommend starting at batch size 8 and expect \"a few hours for 100k training steps on a single GPU\". The original paper reports ~5 h training on a single 11 GB RTX 2080 Ti and ~0.01 s inference on the same machine.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-act-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2304.13705", "date": "2023-04-23", "verified_on": "2026-10-24", "summary": "ACT paper (Appendix D) states the model size, training time and inference time.", "quote": "The model has around 80M parameters, and we train from scratch for each task. The training takes around 5 hours on a single 11G RTX 2080 Ti GPU, and the inference time is around 0.01 seconds on the same machine." }, { "id": "ev-lerobot-act-docs-arch", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/act.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot ACT docs describe the ResNet-18 backbone and transformer encoder/decoder that outputs an action chunk.", "quote": "**Vision Backbone**: ResNet-18 processes images from multiple camera viewpoints" }, { "id": "ev-lerobot-act-docs-inputs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/act.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "ACT inputs are multi-view RGB images, joint positions and a latent z; the rollout example marks --task as skippable for ACT.", "quote": "--task=\"Your task description\" \\ # can be skipped for ACT" }, { "id": "ev-lerobot-act-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/act/configuration_act.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `act` (chunk_size 100 by default).", "quote": "@PreTrainedConfig.register_subclass(\"act\")" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Light BC (`act`, `vqbet`, `tdmpc`) peak VRAM ~2–6GB at BS 8 with AdamW; figures are indicative.", "quote": "| Light BC | `act`, `vqbet`, `tdmpc` | ~2–6GB | Laptop GPU (RTX 3060), L4, A10G |" }, { "id": "ev-lerobot-paper-mem", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2602.22818", "date": "2026-02-26", "verified_on": "2026-10-24", "summary": "LeRobot paper Table 2: ACT (52M) peak memory 817.4MB CPU, 462MB MPS, 211.24 MB RTX 4090, 211.24 MB A100, fp32.", "quote": "Table 2: Peak memory consumption of policy models currently supported by lerobot . All models are run in full precision (fp32). Diffusion and Flow Models are run with 10 denoising steps at inference." } ] }, { "id": "diffusion", "name": "Diffusion Policy", "aliases": [ "diffusion_policy" ], "family": "diffusion", "summary": "Visuomotor policy that represents the action distribution as a conditional denoising diffusion process, predicting a sequence of future actions from recent observations and executing it with receding-horizon control. LeRobot's implementation uses a ResNet-18 image encoder and a 1D convolutional U-Net denoiser.", "authors": "Columbia University and Toyota Research Institute (Chi, Xu, Feng, Cousineau, Du, Burchfiel, Tedrake, Song)", "paper_url": "https://arxiv.org/abs/2303.04137", "params_note": "The authors do not state a single parameter count. The LeRobot paper (Table 2) lists LeRobot's Diffusion Policy at 263M parameters.", "params_source_ref": "ev-lerobot-paper-mem", "backbone": "ResNet-18 image encoder (ImageNet-pretrained weights by default in LeRobot v0.6.1) + conditional 1D conv U-Net denoiser", "action_output": "diffusion", "action_chunking": true, "arch_source_ref": "ev-diffusion-paper-arch", "license": "Apache-2.0 (LeRobot implementation)", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "policy_type": "diffusion", "docs_url": "https://huggingface.co/docs/lerobot/policy_diffusion_README", "note": "Present at v0.3.2, the earliest tagged LeRobot release. `since` left blank: the only earlier PyPI release (0.1.0, 2024-03, a prototype whose README mentions diffusion_policy) was not inspected. Defaults: horizon 64, n_action_steps 32, n_obs_steps 2.", "source_ref": "ev-lerobot-diffusion-config" } }, "checkpoints": [ { "hub_id": "lerobot/diffusion_pusht", "note": "PushT simulation checkpoint (Apache-2.0 on the Hub)." } ], "requirements": { "train": { "vram_gb": { "min": 8, "max": 14 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Diffusion\" group (diffusion, multi_task_dit) in the LeRobot compute hardware guide. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "reference_time": "5 epochs on a ~45k-frame dataset (640×480): ~2–4 h at batch 8 on a single RTX 4090/3090; ~30–60 min at batch 32 on a 4× H100 80 GB cluster (indicative, ±50%). Measured anchor in the same guide: 5000 steps, batch 32, 4× H100, 16m 17s.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "measured": 1.12 }, "precision": "fp32", "condition": "Peak memory at inference measured by the LeRobot paper (Table 2): 1.12 GB on both RTX 4090 and A100, full fp32, 10 denoising steps. (Same table: 1.22GB CPU-only, 224MB on Apple MPS.)", "source_ref": "ev-lerobot-paper-mem" } }, "observation_space": { "state": "robot state vector; code requires at least one image or an environment-state input, and all images must share one shape", "source_ref": "ev-lerobot-diffusion-config-features" }, "verified_on": "2026-10-24", "sources": [ { "id": "ev-diffusion-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2303.04137", "date": "2023-03-07", "verified_on": "2026-10-24", "summary": "Diffusion Policy models the visuomotor policy as a conditional denoising diffusion process over action sequences.", "quote": "This paper introduces Diffusion Policy, a new way of generating robot behavior by representing a robot’s visuomotor policy as a conditional denoising diffusion process." }, { "id": "ev-lerobot-diffusion-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/diffusion/configuration_diffusion.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `diffusion`; defaults resnet18 backbone, horizon 64, n_action_steps 32.", "quote": "@PreTrainedConfig.register_subclass(\"diffusion\")" }, { "id": "ev-lerobot-diffusion-config-features", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/diffusion/configuration_diffusion.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Feature validation requires at least one image or an environment-state input.", "quote": "raise ValueError(\"You must provide at least one image or the environment state among the inputs.\")" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Diffusion group (`diffusion`, `multi_task_dit`) peak VRAM ~8–14GB at BS 8 with AdamW; figures are indicative.", "quote": "| Diffusion | `diffusion`, `multi_task_dit` | ~8–14GB | RTX 4070+ / L4 / A10G |" }, { "id": "ev-lerobot-paper-mem", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2602.22818", "date": "2026-02-26", "verified_on": "2026-10-24", "summary": "LeRobot paper Table 2: Diffusion Policy (263M) peak memory 1.22GB CPU, 224MB MPS, 1.12 GB RTX 4090, 1.12 GB A100, fp32.", "quote": "Table 2: Peak memory consumption of policy models currently supported by lerobot . All models are run in full precision (fp32). Diffusion and Flow Models are run with 10 denoising steps at inference." } ] }, { "id": "eo1", "name": "EO-1", "aliases": [ "EO1", "EmbodiedOneVision", "eo-1-3b", "IPEC-COMMUNITY/EO-1-3B" ], "family": "vla", "summary": "Unified embodied foundation model from the EO-Robotics project: a Qwen2.5-VL-3B backbone trained on interleaved vision-text-action data. LeRobot integrates the robot-control path: a continuous flow-matching action head that predicts action chunks (default 8 steps) from images, EO-1 state tokens and a language task.", "authors": "EO-Robotics (Delin Qu, Haoming Song et al.)", "paper_url": "https://arxiv.org/abs/2508.21112", "params": 3000000000, "params_note": "Authors describe EO-1 as comprising 3B parameters (rounded).", "params_source_ref": "ev-eo1-paper-params", "backbone": "Qwen2.5-VL-3B-Instruct", "action_output": "flow-matching", "action_chunking": true, "arch_source_ref": "ev-lerobot-eo1-docs", "license": "Apache-2.0 (LeRobot integration)", "weights_license": "MIT", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "eo1", "docs_url": "https://huggingface.co/docs/lerobot/eo1", "note": "Added in v0.6.0 (\"feat(policies): add EO-1 model\"). Install with the `eo1` extra.", "source_ref": "ev-lerobot-eo1-config" } }, "checkpoints": [ { "hub_id": "lerobot/eo1-base", "note": "LeRobot-org checkpoint; Hub license tag MIT." }, { "hub_id": "IPEC-COMMUNITY/EO-1-3B", "note": "Upstream release from the EO-1 project; Hub license tag MIT." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Multimodal\" group (groot, eo1) in the LeRobot compute hardware guide; suggested starter GPU A100 40 GB+. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "min": 6.5 }, "condition": "Upstream EO-1 repository README: loading the pretrained model with transformers + lerobot and running inference \"requires ~6.5GB GPU memory\". Refers to the upstream EO-1 code path, not measured on the LeRobot v0.6.1 port; precision and batch size not stated.", "source_ref": "ev-eo1-upstream-readme" } }, "observation_space": { "cameras": { "min": 1 }, "state": "robot state (observation.state), represented as EO-1 state tokens; padded to max_state_dim 32", "language": "required", "source_ref": "ev-lerobot-eo1-docs-data" }, "notes": "Family vla: the LeRobot docs call EO-1 a \"Vision-Language-Action policy\". Upstream EO-1 also does interleaved autoregressive text/reasoning decoding (hybrid), but the LeRobot integration covers only the continuous flow-matching robot-control path. rtc_compatible and peft_supported are left unset: the LeRobot RTC and PEFT docs do not name EO-1. The upstream EO-Robotics/EO1 GitHub repo had no license detected by the GitHub API on 2026-10-24, so the upstream code license is unknown; the LeRobot integration is Apache-2.0.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-eo1-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/eo1.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot EO-1 docs: Qwen2.5-VL backbone plus continuous flow-matching action head; needs ≥1 camera, state and a language task.", "quote": "EO-1 uses a Qwen2.5-VL backbone for vision-language understanding and adds a continuous flow-matching action head for robot control." }, { "id": "ev-lerobot-eo1-docs-data", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/eo1.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "EO-1 data requirements list at least one image, observation.state, action and a language task.", "quote": "At least one visual observation, for example `observation.images.image`" }, { "id": "ev-lerobot-eo1-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/eo1/configuration_eo1.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `eo1`.", "quote": "@PreTrainedConfig.register_subclass(\"eo1\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "EO-1 was added in LeRobot v0.6.0 (absent from the v0.5.1 policies tree).", "quote": "* feat(policies): add EO-1 model by @HaomingSong in https://github.com/huggingface/lerobot/pull/3403" }, { "id": "ev-eo1-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2508.21112v1", "date": "2025-08-28", "verified_on": "2026-10-24", "summary": "EO-1 paper states the model comprises 3B parameters.", "quote": "a unified embodied foundation model comprising 3B parameters" }, { "id": "ev-eo1-upstream-readme", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/EO-Robotics/EO1/blob/451c363d7dff682700789afae0b9b17c004e8f8c/README.md", "date": "2025-11-12", "verified_on": "2026-10-24", "summary": "Upstream EO-1 README gives ~6.5GB GPU memory for inference with the pretrained model.", "quote": "you can load the model and run inference directly with just a few lines of code (requires ~6.5GB GPU memory)" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Multimodal group (`groot`, `eo1`) peak VRAM ~24–40GB at BS 8 with AdamW; figures are indicative.", "quote": "| Multimodal | `groot`, `eo1` | ~24–40GB | A100 40 GB+ |" }, { "id": "ev-hub-eo1-checkpoints", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/lerobot/eo1-base", "verified_on": "2026-10-24", "summary": "Hub API: lerobot/eo1-base and IPEC-COMMUNITY/EO-1-3B exist, both tagged license:mit." } ] }, { "id": "evo1", "name": "Evo-1", "aliases": [ "EVO1", "Evo1", "evo-1", "MINT-SJTU/Evo-1" ], "family": "vla", "summary": "Lightweight vision-language-action model from MINT-SJTU: an InternVL3-1B backbone (14 language layers kept) with a continuous flow-matching action head predicting 50-step action chunks from one to three camera views, state and a language task. Trained in two stages (action head only, then VLM fine-tuning).", "authors": "MINT-SJTU (School of AI, Shanghai Jiao Tong University) et al.", "paper_url": "https://arxiv.org/abs/2511.04555", "params": 770000000, "params_note": "Authors state 0.77B parameters (rounded).", "params_source_ref": "ev-evo1-paper-params", "backbone": "InternVL3-1B (OpenGVLab/InternVL3-1B-hf), 14 language layers", "action_output": "flow-matching", "action_chunking": true, "rtc_compatible": true, "arch_source_ref": "ev-lerobot-evo1-docs", "license": "MIT (upstream MINT-SJTU/Evo-1); LeRobot integration Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "evo1", "docs_url": "https://huggingface.co/docs/lerobot/evo1", "note": "Added in v0.6.0 (\"feat(policies): add EVO1 policy\"). Docs show optional `lerobot-rollout --inference.type=rtc`; EVO1Policy.supports_rtc() returns True at v0.6.1.", "source_ref": "ev-lerobot-evo1-config" } }, "checkpoints": [ { "hub_id": "zuoxingdong/evo1_libero", "note": "LeRobot-format Stage-2 LIBERO checkpoint referenced by the LeRobot docs (96.65% LIBERO average at step 70k, single run)." }, { "hub_id": "MINT-SJTU/Evo1_LIBERO", "note": "Upstream (non-LeRobot-format) LIBERO weights from the Evo-1 repo." }, { "hub_id": "MINT-SJTU/Evo1_MetaWorld", "note": "Upstream (non-LeRobot-format) Meta-World weights from the Evo-1 repo." } ], "observation_space": { "cameras": { "min": 1, "max": 3 }, "state": "observation.state, padded to max_state_dim 24", "language": "required", "source_ref": "ev-lerobot-evo1-docs-data" }, "notes": "The LeRobot docs' reference LIBERO training run used two H100 GPUs (DDP, batch 64 per process, global batch 128); no VRAM figure is given and Evo-1 is not listed in the LeRobot compute hardware guide, so requirements are left unset. Camera max is the default `max_views` (3), configurable. peft_supported left unset (not in the PEFT docs). Weights license not stated on the Hub for the MINT-SJTU repos.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-evo1-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/evo1.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot EVO1 docs describe an InternVL3 backbone with a continuous flow-matching action head.", "quote": "EVO1 is a Vision-Language-Action policy for robot control built around an InternVL3 backbone and a continuous flow-matching action head." }, { "id": "ev-lerobot-evo1-docs-data", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/evo1.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "EVO1 expects one to `policy.max_views` (default 3) cameras, observation.state, action and a language task.", "quote": "One to `policy.max_views` visual observations, for example `observation.images.image`" }, { "id": "ev-lerobot-evo1-docs-rtc", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/evo1.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "EVO1 rollout example enables RTC inference as an option.", "quote": "--inference.type=rtc \\ # optional" }, { "id": "ev-lerobot-evo1-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/evo1/configuration_evo1.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `evo1`; default max_views 3, max_state_dim 24, VLM OpenGVLab/InternVL3-1B-hf.", "quote": "@PreTrainedConfig.register_subclass(\"evo1\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "EVO1 was added in LeRobot v0.6.0.", "quote": "* feat(policies): add EVO1 policy by @imstevenpmwork in https://github.com/huggingface/lerobot/pull/3908" }, { "id": "ev-evo1-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2511.04555v1", "date": "2025-11-06", "verified_on": "2026-10-24", "summary": "Evo-1 paper states 0.77B parameters.", "quote": "We propose Evo-1, a lightweight VLA architecture with only 0.77B parameters" }, { "id": "ev-evo1-upstream-repo", "tier": "community-reported", "type": "source-code", "url": "https://github.com/MINT-SJTU/Evo-1/tree/5fd14b015013c4fd0aacf5f8f48f868ca9b870a2", "date": "2026-08-18", "verified_on": "2026-10-24", "summary": "GitHub API reports the upstream MINT-SJTU/Evo-1 repo license as MIT; README links the MINT-SJTU/Evo1_LIBERO and Evo1_MetaWorld weights." }, { "id": "ev-hub-evo1-checkpoints", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/zuoxingdong/evo1_libero", "verified_on": "2026-10-24", "summary": "Hub API: zuoxingdong/evo1_libero (license tag 'other'), MINT-SJTU/Evo1_LIBERO and MINT-SJTU/Evo1_MetaWorld (no license tag) exist." } ] }, { "id": "fastwam", "name": "Fast-WAM", "aliases": [ "fastwam", "FastWAM", "Fast-WAM" ], "family": "world-model", "summary": "World Action Model on a Wan2.2-TI2V-5B video backbone plus a 1B action expert, trained with a joint flow-matching objective over action chunks and future video latents. Keeps video modeling during training but predicts actions directly at inference, without generating future frames.", "authors": "Tianyuan Yuan, Zibin Dong, Yicheng Liu, Hang Zhao (IIIS, Tsinghua University; Galaxea AI)", "paper_url": "https://arxiv.org/abs/2603.16666", "params": 6000000000, "params_note": "Authors state a total model size of 6B parameters (Wan2.2-5B backbone + 1B action expert), rounded.", "params_source_ref": "ev-fastwam-paper-params", "backbone": "Wan2.2-TI2V-5B (video DiT, text encoder, video VAE) + 1B action expert", "action_output": "flow-matching", "action_chunking": true, "arch_source_ref": "ev-fastwam-paper-arch", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "fastwam", "docs_url": "https://huggingface.co/docs/lerobot/fastwam", "note": "Added in v0.6.0 (\"feat(policies): Add FastWAM Policy\"). Default action_horizon 32. DiT uses PyTorch SDPA only (flash-attn has no effect).", "source_ref": "ev-lerobot-fastwam-config" } }, "checkpoints": [ { "hub_id": "lerobot/fastwam_base", "note": "LeRobot-org base checkpoint (Hub license tag apache-2.0)." }, { "hub_id": "lerobot/fastwam_libero_uncond_2cam224", "note": "LeRobot-org LIBERO checkpoint, 2 cameras at 224 px (no Hub license tag)." }, { "hub_id": "lerobot/fastwam_robotwin_uncond_3cam_384", "note": "LeRobot-org RoboTwin checkpoint, 3 cameras (no Hub license tag)." }, { "hub_id": "ZibinDong/fastwam_libero_uncond_2cam224", "note": "Checkpoint used for the LIBERO results in the LeRobot docs (96.4% average over 2000 episodes)." }, { "hub_id": "yuanty/fastwam", "note": "Released upstream checkpoints." } ], "observation_space": { "cameras": { "min": 1 }, "state": "observation.state when policy.proprio_dim is set (optional)", "language": "required", "source_ref": "ev-lerobot-fastwam-docs-data" }, "notes": "Family world-model: the LeRobot docs call FastWAM a \"World Action Model policy\" and the paper frames it against imagine-then-execute world action models; the video branch is trained but not rolled out at inference. Language: a task instruction is required, or precomputed `context`/`context_mask` text tensors. Multiple cameras are concatenated along width to `policy.image_size` (default one 224x448 image). The LeRobot docs' LIBERO reproduction ran in float32 on 1x H20 140 GB; that is a reference setup, not a stated minimum, so requirements are unset (not in the LeRobot compute hardware guide). Code license: the GitHub API reports NOASSERTION for yuantianyuan01/FastWAM, so it is left unknown; weights license is unknown for the upstream repo. Not named by the RTC or PEFT docs.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-fastwam-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/fastwam.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot docs describe FastWAM as a World Action Model that predicts actions directly at inference.", "quote": "FastWAM keeps video modeling during training, but uses direct action prediction at inference time instead of iteratively generating future observations." }, { "id": "ev-lerobot-fastwam-docs-data", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/fastwam.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Data requirements: one or more cameras, optional state (proprio_dim), action, and a language task or precomputed context.", "quote": "a language task instruction through the dataset task field, or precomputed `context` and `context_mask` tensors" }, { "id": "ev-lerobot-fastwam-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/fastwam/configuration_fastwam.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `fastwam`; default action_horizon 32.", "quote": "@PreTrainedConfig.register_subclass(\"fastwam\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "FastWAM was added in LeRobot v0.6.0.", "quote": "* feat(policies): Add FastWAM Policy by @Maximellerbach in https://github.com/huggingface/lerobot/pull/3834" }, { "id": "ev-fastwam-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2603.16666v1", "date": "2026-03-17", "verified_on": "2026-10-24", "summary": "Fast-WAM paper states a 1B action expert and 6B total parameters on a Wan2.2-5B backbone.", "quote": "resulting in a 1B action expert and a total model size of 6B parameters." }, { "id": "ev-fastwam-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2603.16666v1", "date": "2026-03-17", "verified_on": "2026-10-24", "summary": "Fast-WAM uses a joint flow-matching objective over action tokens and future video latents.", "quote": "Fast-WAM is trained with a joint flow matching objective over action tokens and future video latents." }, { "id": "ev-hub-fastwam-checkpoints", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/lerobot/fastwam_base", "verified_on": "2026-10-24", "summary": "Hub API: lerobot/fastwam_base (license:apache-2.0), lerobot/fastwam_libero_uncond_2cam224, lerobot/fastwam_robotwin_uncond_3cam_384, ZibinDong/fastwam_libero_uncond_2cam224 and yuanty/fastwam exist (no license tags on the latter four)." } ] }, { "id": "groot-n1-7", "name": "GR00T N1.7", "aliases": [ "groot", "gr00t", "GR00T", "Isaac GR00T N1.7", "GR00T-N1.7-3B", "nvidia/GR00T-N1.7-3B", "gr00t-n1.7" ], "family": "vla", "summary": "NVIDIA's open cross-embodiment foundation model for humanoid and manipulation skills: a Cosmos-Reason2-2B (Qwen3-VL architecture) vision-language backbone feeding a flow-matching diffusion-transformer (DiT) action head that denoises continuous action chunks conditioned on vision, language and proprioception.", "authors": "NVIDIA (GEAR)", "paper_url": "https://arxiv.org/abs/2503.14734", "params": 3000000000, "params_note": "Model card states 3,000,000,000; the Isaac-GR00T README calls it the 'Base model (3B params)'. Both are rounded figures.", "params_source_ref": "ev-groot-n17-card", "backbone": "Cosmos-Reason2-2B (Qwen3-VL architecture) VLM with SigLip2 vision transformer", "action_output": "flow-matching", "action_chunking": true, "rtc_compatible": true, "arch_source_ref": "ev-groot-n17-card-arch", "license": "Apache-2.0", "weights_license": "NVIDIA Open Model License", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "groot", "docs_url": "https://huggingface.co/docs/lerobot/groot", "note": "`groot` is GR00T N1.7 from v0.6.0. LeRobot v0.4.0–v0.5.1 shipped GR00T N1.5 under the same `groot` type (default base model nvidia/GR00T-N1.5-3B); N1.5 was removed in v0.6.0 and N1.5 checkpoints are rejected. To keep N1.5, pin lerobot==0.5.1. groot.mdx shows rollout with --inference.type=rtc.", "source_ref": "ev-lerobot-v060-notes" }, "isaac-groot": { "support": "upstream-repo", "since": "n1.7-release", "repo_url": "https://github.com/NVIDIA/Isaac-GR00T", "note": "N1.7 is an Early Access release (tag n1.7-release, 2026-04-18); older generations live on the n1.6-release / n1.5-release tags.", "source_ref": "ev-isaac-groot-readme-ea" } }, "checkpoints": [ { "hub_id": "nvidia/GR00T-N1.7-3B", "note": "Base model; NVIDIA Open Model License." }, { "hub_id": "nvidia/gr00t17-lerobot-libero_spatial-640", "note": "LeRobot-trained LIBERO Spatial checkpoint (95% in LeRobot docs)." }, { "hub_id": "nvidia/gr00t17-lerobot-libero_object-640" }, { "hub_id": "nvidia/gr00t17-lerobot-libero_goal-640" }, { "hub_id": "nvidia/gr00t17-lerobot-libero_10-640" }, { "hub_id": "nvidia/GR00T-N1.7-LIBERO", "note": "Upstream (Isaac-GR00T format) LIBERO checkpoints." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Multimodal\" group (`groot`, `eo1`) in the LeRobot compute hardware guide; suggested starter GPU \"A100 40 GB+\". The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "min": 16 }, "condition": "Upstream Isaac-GR00T N1.7 README hardware requirement for inference with NVIDIA's own gr00t package: 1 GPU with 16 GB+ VRAM (e.g. RTX 4090, L40, H100, Jetson AGX Thor/Orin, DGX Spark). Not a LeRobot measurement; precision and batch size not stated. The same README recommends 40 GB+ GPUs for fine-tuning.", "source_ref": "ev-isaac-groot-readme-hw" } }, "observation_space": { "state": "robot proprioception, MLP-encoded per embodiment ID and padded to a configurable max length (LeRobot default max_state_dim 132)", "source_ref": "ev-groot-n17-card-proprio" }, "notes": "Naming: the LeRobot policy type is `groot` (no version in the string); the version is determined by the LeRobot release (N1.5 in 0.4.0–0.5.1, N1.7 from 0.6.0). The GR00T N1 technical report (arXiv 2503.14734) covers the N1.x family; there is no separate N1.7 paper. The v0.6.1 GR00T config keeps deprecated LoRA fields that its own comment calls \"never-wired\", so PEFT support is left unknown. Upstream code is Apache-2.0; weights are under the NVIDIA Open Model License.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-groot-n17-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/nvidia/GR00T-N1.7-3B", "verified_on": "2026-10-24", "summary": "N1.7 model card gives the parameter count and the flow-matching DiT action head over SigLip2 vision features.", "quote": "**Number of Model Parameters:** 3,000,000,000" }, { "id": "ev-groot-n17-card-arch", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/nvidia/GR00T-N1.7-3B", "verified_on": "2026-10-24", "summary": "Model card describes the flow matching action transformer that models a chunk of actions.", "quote": "uses a flow matching action transformer to model a chunk of actions conditioned on vision, language and proprioception." }, { "id": "ev-groot-n17-card-proprio", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/nvidia/GR00T-N1.7-3B", "verified_on": "2026-10-24", "summary": "Model card describes proprioception encoding and padding.", "quote": "To handle variable-dimension proprioception, inputs are padded to a configurable max length before feeding into the MLP." }, { "id": "ev-lerobot-groot-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/groot.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot integrates GR00T N1.7 only; N1.5 was removed and needs lerobot==0.5.1.", "quote": "To keep using an N1.5 checkpoint, pin the last release that supports it: `pip install 'lerobot==0.5.1'`." }, { "id": "ev-lerobot-groot-rtc", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/groot.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot GR00T docs deploy with RTC inference via lerobot-rollout.", "quote": "--inference.rtc.enabled=True \\ # set to False if it causes inference instability" }, { "id": "ev-lerobot-groot-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/groot/configuration_groot.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `groot`.", "quote": "@PreTrainedConfig.register_subclass(\"groot\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "v0.6.0 replaced GR00T N1.5 with N1.7 (PR", "quote": "GR00T N1.5 is replaced by N1.7 (pin `lerobot==0.5.1` if you need N1.5)." }, { "id": "ev-lerobot-v040-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.0", "date": "2025-10-23", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.0" }, "summary": "The `groot` policy (then GR00T N1.5) first shipped in v0.4.0.", "quote": "feat(policies): add Nvidia Gr00t N1.5 model by @imstevenpmwork" }, { "id": "ev-isaac-groot-readme-ea", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/NVIDIA/Isaac-GR00T/blob/n1.7-release/README.md", "date": "2026-04-18", "verified_on": "2026-10-24", "summary": "Isaac-GR00T n1.7-release README describes N1.7 as an Early Access release with a Cosmos-Reason2-2B / Qwen3-VL backbone.", "quote": "We just released GR00T N1.7 Early Access, the latest version of GR00T N1 with a new VLM backbone (Cosmos-Reason2-2B / Qwen3-VL) and improved performance." }, { "id": "ev-isaac-groot-readme-hw", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/NVIDIA/Isaac-GR00T/blob/n1.7-release/README.md", "date": "2026-04-18", "verified_on": "2026-10-24", "summary": "Upstream hardware requirements for N1.7 inference (16 GB+) and fine-tuning (40 GB+ recommended).", "quote": "**Inference:** 1 GPU with 16 GB+ VRAM (e.g., RTX 4090, L40, H100, Jetson AGX Thor/Orin, DGX Spark)." }, { "id": "ev-isaac-groot-readme-license", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/NVIDIA/Isaac-GR00T/blob/n1.7-release/README.md", "date": "2026-04-18", "verified_on": "2026-10-24", "summary": "Code is Apache 2.0; model weights are under the NVIDIA Open Model License.", "quote": "- **Model weights:** [NVIDIA Open Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/)" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Multimodal group (`groot`, `eo1`) peak VRAM ~24–40GB at BS 8 with AdamW; figures are indicative.", "quote": "| Multimodal | `groot`, `eo1` | ~24–40GB | A100 40 GB+ |" } ] }, { "id": "hil-serl", "name": "HIL-SERL", "aliases": [ "hilserl", "Human-in-the-Loop Sample-Efficient Robotic Reinforcement Learning", "gaussian_actor" ], "family": "rl", "summary": "Human-in-the-loop, sample-efficient real-robot reinforcement learning: a few teleoperated demonstrations and an optional vision-based reward classifier bootstrap a distributed actor/learner SAC loop, with a human able to intervene during on-robot exploration. In LeRobot v0.6.x the trained policy is a `gaussian_actor` driven by the `sac` algorithm.", "authors": "UC Berkeley (Luo, Xu, Wu, Levine)", "paper_url": "https://arxiv.org/abs/2410.21845", "backbone": "Gaussian actor (tanh-squashed diagonal Gaussian head) over image + state encoders; critics live in the SAC algorithm", "action_output": "other", "arch_source_ref": "ev-lerobot-gaussian-actor-config", "license": "Apache-2.0 (LeRobot implementation)", "frameworks": { "lerobot": { "support": "native", "since": "0.3.2", "policy_type": "gaussian_actor", "docs_url": "https://huggingface.co/docs/lerobot/hilserl", "note": "v0.6.0 rebuilt the RL stack: the policy is now `--policy.type=gaussian_actor` (policy.type=\"gaussian_actor\" in the JSON train config) with a top-level `algorithm` block of type \"sac\"; from v0.3.2 through v0.5.1 the policy type was `sac`. Training runs as two processes, `python -m lerobot.rl.learner` and `python -m lerobot.rl.actor`. An optional reward classifier (type \"reward_classifier\", lerobot/rewards/classifier) predicts success from camera images. v0.3.2 is the earliest tagged LeRobot release and already ships the `sac` policy and HIL-SERL guide; the only earlier PyPI release (0.1.0, 2024-03) predates the HIL-SERL paper.", "source_ref": "ev-lerobot-hilserl-docs-policy" } }, "requirements": { "train": { "condition": "The LeRobot compute hardware guide gives no VRAM figure for RL: its RL row (still labelled `sac`, the pre-v0.6.0 type name) reads \"config-dep.\" and points to the HIL-SERL guide. The HIL-SERL guide lists \"A Nvidia GPU\" as a requirement and suggests 64×64 instead of 128×128 image inputs to save GPU memory.", "source_ref": "ev-lerobot-hw-guide" } }, "observation_space": { "state": "robot joint state, optionally with joint velocities, motor currents and end-effector pose; code requires state or an image input", "source_ref": "ev-lerobot-hilserl-docs-images" }, "notes": "The guide recommends square 128×128 (default) or 64×64 image crops. The HIL-SERL paper's core RL algorithm is RLPD. The LeRobot compute hardware guide still names the RL policy `sac` although v0.6.x registers `gaussian_actor`.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-hilserl-docs-policy", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hilserl.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "HIL-SERL guide: the policy is configured with type gaussian_actor and the algorithm with type sac.", "quote": "1. Configure the policy settings (`type=\"gaussian_actor\"`, `device`, etc.)" }, { "id": "ev-lerobot-hilserl-docs-images", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hilserl.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "HIL-SERL guide recommends square 128×128 or 64×64 image inputs.", "quote": "Most vision-based policies have been validated on square inputs of either **128×128** (default) or **64×64** pixels." }, { "id": "ev-lerobot-gaussian-actor-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/gaussian_actor/configuration_gaussian_actor.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "`gaussian_actor` is registered as a policy type; its docstring places critics/temperature on the SAC algorithm side.", "quote": "This configures the policy-side (actor + observation encoder) of a Gaussian policy, as used by SAC and related maximum-entropy continuous-control algorithms." }, { "id": "ev-lerobot-v060-release", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "v0.6.0 renamed the RL policy type from sac to gaussian_actor.", "quote": "The RL stack was rebuilt: the `sac` policy type is now `gaussian_actor` under the new modular RL API." }, { "id": "ev-lerobot-sac-v032", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.3.2/src/lerobot/policies/sac/configuration_sac.py", "date": "2025-08-01", "verified_on": "2026-10-24", "versions": { "lerobot": "0.3.2" }, "summary": "The `sac` policy type (used by HIL-SERL) is registered at v0.3.2, the earliest tagged release.", "quote": "@PreTrainedConfig.register_subclass(\"sac\")" }, { "id": "ev-hilserl-paper", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2410.21845", "date": "2024-10-29", "verified_on": "2026-10-24", "summary": "HIL-SERL paper builds on RLPD as its core RL algorithm.", "quote": "the core underlying RL algorithm that we build on is RLPD" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide RL row: `sac`, VRAM config-dependent, see HIL-SERL guide.", "quote": "| RL | `sac` | config-dep. | See [HIL-SERL guide](./hilserl) |" } ] }, { "id": "lingbot-va", "name": "LingBot-VA", "aliases": [ "lingbot_va", "lingbot-va-base", "robbyant/lingbot-va" ], "family": "world-model", "summary": "Autoregressive video-action world-model policy from Robbyant built on the Wan2.2 video-diffusion stack: a dual-stream mixture-of-transformers that interleaves prediction of future video latents and robot actions, each denoised by flow matching, with a KV cache and observed-keyframe feedback across chunks. Predicts end-effector poses + gripper.", "authors": "Robbyant", "paper_url": "https://arxiv.org/abs/2601.21998", "params": 5300000000, "params_note": "Authors state a total model size of 5.3B parameters (approx. 350M of them the action stream). The LeRobot checkpoint stores only the ~5B trainable transformer; the frozen Wan2.2 VAE + UMT5-XXL text encoder (~20 GB) are pulled separately.", "params_source_ref": "ev-lingbotva-paper-params", "backbone": "Wan2.2 video DiT (dual-stream mixture-of-transformers, 30 shared blocks) + Wan2.2 VAE + UMT5-XXL text encoder (frozen)", "action_output": "flow-matching", "action_chunking": true, "peft_supported": true, "arch_source_ref": "ev-lerobot-lingbotva-docs", "license": "Apache-2.0", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "lingbot_va", "docs_url": "https://huggingface.co/docs/lerobot/lingbot_va", "note": "Added in v0.6.0 (\"LingBot-VA: video-action world model\"). Streaming inference is single-environment (`--eval.batch_size=1`); training needs `--policy.attn_mode=flex`. LoRA via the policy's own `--policy.use_peft=true` flag (documented on the LingBot-VA page, not in the generic PEFT guide).", "source_ref": "ev-lerobot-lingbotva-config" } }, "checkpoints": [ { "hub_id": "lerobot/lingbot_va_base", "note": "Pretrained base, converted to LeRobot format from robbyant/lingbot-va-base." }, { "hub_id": "lerobot/lingbot_va_libero_long", "note": "LIBERO-Long post-train (LeRobot format)." }, { "hub_id": "lerobot/lingbot_va_robotwin", "note": "RoboTwin post-train (LeRobot format)." }, { "hub_id": "robbyant/lingbot-va-base", "note": "Original upstream release." } ], "requirements": { "train": { "condition": "LeRobot docs: the full 5B DiT does not fit a single 24–32 GB GPU under AdamW; fine-tune with LoRA (`--policy.use_peft=true`) and/or optimizer offload. No VRAM figure is given for either path; not covered by the LeRobot compute hardware guide.", "source_ref": "ev-lerobot-lingbotva-docs-train" }, "inference": { "vram_gb": { "min": 18, "max": 24 }, "batch_size": 1, "condition": "LeRobot docs: inference needs roughly 18–24 GB of VRAM, with the UMT5-XXL text encoder on CPU by default (`config.text_encoder_device`) so the 5B transformer + VAE fit on one GPU; single-environment streaming inference. Precision not stated.", "source_ref": "ev-lerobot-lingbotva-docs-vram" } }, "observation_space": { "cameras": { "min": 2, "max": 3 }, "language": "required", "source_ref": "ev-lerobot-lingbotva-docs-cams" }, "notes": "Family world-model: the LeRobot docs describe it as an \"autoregressive video-action world-model policy\" that predicts future video latents alongside actions (closed-loop world modeling). Camera counts are those of the documented configurations (LIBERO 2: agentview + wrist; RoboTwin 3: head + two wrists); camera order is fixed and order-sensitive. Joint-space datasets must be remapped into the EEF-pose action schema before fine-tuning the released checkpoints. Upstream repo README gives ~24GB (RoboTwin eval) and ~18GB (i2av inference) with VAE and text encoder offloaded to CPU.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-lingbotva-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/lingbot_va.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot docs: autoregressive video-action world-model policy on Wan2.2; video and action streams use flow-matching schedulers.", "quote": "LingBot-VA is an **autoregressive video-action world-model policy** built on the **Wan2.2** video-diffusion stack." }, { "id": "ev-lerobot-lingbotva-docs-train", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/lingbot_va.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Full AdamW fine-tuning does not fit 24–32 GB; LoRA via --policy.use_peft=true and/or optimizer offload.", "quote": "The full 5B DiT does not fit a single 24–32 GB GPU under AdamW; fine-tune with **LoRA** (`--policy.use_peft=true`) and/or optimizer offload." }, { "id": "ev-lerobot-lingbotva-docs-vram", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/lingbot_va.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Inference VRAM roughly 18–24 GB.", "quote": "inference needs roughly 18–24 GB of VRAM." }, { "id": "ev-lerobot-lingbotva-docs-cams", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/lingbot_va.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Camera order is fixed (LIBERO 2 cameras, RoboTwin 3); first camera is exterior/head, rest are wrist views.", "quote": "The first camera is the exterior/head view and the rest are wrist views." }, { "id": "ev-lerobot-lingbotva-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/lingbot_va/configuration_lingbot_va.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `lingbot_va`.", "quote": "@PreTrainedConfig.register_subclass(\"lingbot_va\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "LingBot-VA was added in LeRobot v0.6.0.", "quote": "* LingBot-VA: video-action world model by @pkooij in https://github.com/huggingface/lerobot/pull/3731" }, { "id": "ev-lingbotva-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2601.21998v1", "date": "2026-01-29", "verified_on": "2026-10-24", "summary": "LingBot-VA paper (\"Causal World Modeling for Robot Control\") states a total model size of 5.3B parameters.", "quote": "resulting in approximately 350M additional parameters and a total model size of 5.3B parameters." }, { "id": "ev-lingbotva-upstream-readme", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Robbyant/lingbot-va/blob/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb/README.md", "date": "2026-07-09", "verified_on": "2026-10-24", "summary": "Upstream README memory notes with VAE and text encoder offloaded to CPU; Apache 2.0 license.", "quote": "> **GPU Memory Requirements**: Approximately **18GB VRAM** for single-GPU i2av inference with offload mode enabled (VAE and text_encoder offloaded to CPU)." }, { "id": "ev-hub-lingbotva-checkpoints", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/lerobot/lingbot_va_base", "verified_on": "2026-10-24", "summary": "Hub API: lerobot/lingbot_va_{base,libero_long,robotwin} and robbyant/lingbot-va-base exist, all tagged license:apache-2.0." } ] }, { "id": "molmoact2", "name": "MolmoAct2", "aliases": [ "molmoact2", "MolmoAct 2", "allenai/MolmoAct2" ], "family": "vla", "summary": "Ai2's action reasoning model: the Molmo2-ER embodied-reasoning VLM grafted onto a DiT-style flow-matching continuous action expert via per-layer key-value conditioning. Pre-training uses discrete MolmoAct2-FAST action tokens, so the policy can emit either continuous (flow-matching) or discrete-token actions.", "authors": "Allen Institute for AI (Ai2), with University of Washington and others", "paper_url": "https://arxiv.org/abs/2605.02881", "backbone": "Molmo2-ER VLM (from the Molmo2-4B mid-training checkpoint; SigLIP2 ViT) + flow-matching DiT action expert", "action_output": "hybrid", "action_chunking": true, "peft_supported": true, "arch_source_ref": "ev-molmoact2-paper-arch", "license": "Apache-2.0", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "molmoact2", "docs_url": "https://huggingface.co/docs/lerobot/molmoact2", "note": "Covers the regular MolmoAct2 model; MolmoAct2-Think is not included yet (docs). Training target is chosen with `policy.action_mode` (continuous | discrete | both) and rollout requires `policy.inference_action_mode`. LoRA is a policy-specific flag (`policy.enable_lora_vlm`, uses the peft package), not the generic `--peft.*` path.", "source_ref": "ev-lerobot-v060-notes-molmo" }, "molmoact2": { "support": "upstream-repo", "repo_url": "https://github.com/allenai/molmoact2", "note": "Original training code used for the paper's experiments (no tagged releases; pin by commit).", "source_ref": "ev-lerobot-molmoact2-docs" } }, "checkpoints": [ { "hub_id": "allenai/MolmoAct2", "note": "Post-trained multi-embodiment base for fine-tuning (original HF format; load via policy.checkpoint_path)." }, { "hub_id": "allenai/MolmoAct2-LIBERO" }, { "hub_id": "allenai/MolmoAct2-LIBERO-LeRobot", "note": "LeRobot fine-tuned LIBERO checkpoint (98.25% avg in LeRobot docs)." }, { "hub_id": "lerobot/MolmoAct2-SO100_101-LeRobot", "note": "LeRobot-format SO-100/101 checkpoint with the joint-frame correction for LeRobot >= 0.5.0 calibration." }, { "hub_id": "lerobot/MolmoAct2-LIBERO-LeRobot" }, { "hub_id": "lerobot/MolmoAct2-DROID-LeRobot" }, { "hub_id": "lerobot/MolmoAct2-BimanualYAM-LeRobot" } ], "requirements": { "train": { "vram_gb": { "measured": 48.3 }, "precision": "bf16", "batch_size": 8, "condition": "LeRobot MolmoAct2 docs: full-model fine-tuning, both action modes, batch 8, peak 48.3 GiB (GiB as reported; about 51.9 GB), single H100 80GB, bf16 model loading, LIBERO with two RGB cameras, chunk_size 10, 8 flow timesteps, gradient_checkpointing on, includes optimizer state. Peak GPU memory sampled with nvidia-smi. Other rows in the same table: action-expert-only 16.5 GiB, LoRA VLM 20.2 GiB (bs 8); full model 49.8 GiB (bs 16), 60.1 GiB (bs 32).", "source_ref": "ev-lerobot-molmoact2-mem" }, "inference": { "vram_gb": { "measured": 12.1 }, "precision": "bf16", "batch_size": 1, "condition": "LeRobot MolmoAct2 docs: inference, continuous action mode, CUDA graph enabled, batch 1, peak 12.1 GiB (GiB as reported), single H100 80GB, bf16 model loading, LIBERO with two RGB cameras, peak sampled with nvidia-smi.", "source_ref": "ev-lerobot-molmoact2-mem-inf" } }, "observation_space": { "state": "robot state vector, quantile-normalised by default", "source_ref": "ev-lerobot-molmoact2-docs-lang" }, "notes": "RTC: v0.6.1 code returns supports_rtc() true only when inference_action_mode == \"continuous\", but the LeRobot docs (rtc.mdx, molmoact2.mdx) do not mention RTC for MolmoAct2, so rtc_compatible is left unset. Parameter count: the paper does not state a total, so params is left out. For exact paper replication the docs point to the v0.5.1-pinned allenai/lerobot `molmoact2-hf-inference` branch. The LeRobot port loads bf16 by default; the original stack loads fp32 with mixed-precision training.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-molmoact2-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2605.02881", "date": "2026-05-04", "verified_on": "2026-10-24", "summary": "MolmoAct2 grafts a discrete-token VLM onto a flow-matching continuous action expert via per-layer KV conditioning.", "quote": "We design a new VLA architecture to graft the discrete-token VLM into the flow-matching continuous-action expert via per-layer key-value (KV) conditioning." }, { "id": "ev-molmoact2-paper-backbone", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2605.02881", "date": "2026-05-04", "verified_on": "2026-10-24", "summary": "Molmo2-ER is built from the Molmo2-4B mid-training checkpoint.", "quote": "Starting from the Molmo2 –4B mid-training checkpoint" }, { "id": "ev-lerobot-molmoact2-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/molmoact2.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot docs point to allenai/molmoact2 as the original training code.", "quote": "For the original MolmoAct2 training code used for the experiments reported in" }, { "id": "ev-lerobot-molmoact2-docs-lang", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/molmoact2.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "State and action use quantile normalisation by default.", "quote": "MolmoAct2 defaults to quantile normalization for state and action features." }, { "id": "ev-lerobot-molmoact2-lora", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/molmoact2.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot MolmoAct2 exposes LoRA on the VLM (and optionally the action expert).", "quote": "- `policy.enable_lora_vlm`: enables LoRA on VLM linear layers." }, { "id": "ev-lerobot-molmoact2-mem", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/molmoact2.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Peak memory table (H100 80GB, bf16, LIBERO, 2 cameras): inference 12.1 GiB (bs 1); full fine-tune 48.3 GiB (bs 8).", "quote": "| Fine-tuning, full model, both action modes | 48.3 GiB | 49.8 GiB | 60.1 GiB |" }, { "id": "ev-lerobot-molmoact2-mem-inf", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/molmoact2.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Inference peak memory row and measurement setup.", "quote": "| Inference, continuous, CUDA graph enabled (bs=1) | 12.1 GiB | - | - |" }, { "id": "ev-lerobot-molmoact2-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/molmoact2/configuration_molmoact2.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `molmoact2`.", "quote": "@PreTrainedConfig.register_subclass(\"molmoact2\")" }, { "id": "ev-lerobot-molmoact2-rtc-code", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/molmoact2/modeling_molmoact2.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "supports_rtc() is conditional on continuous inference mode.", "quote": "return self.config.inference_action_mode == \"continuous\"" }, { "id": "ev-lerobot-v060-notes-molmo", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "MolmoAct2 was added in LeRobot v0.6.0 (PR", "quote": "Add MolmoAct2 policy by @hq-fang" }, { "id": "ev-molmoact2-readme-license", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/allenai/molmoact2/blob/6070080a20321b4f498ab30f28e1d09ac465edb7/README.md", "date": "2026-10-02", "verified_on": "2026-10-24", "summary": "Upstream README licenses the model under Apache 2.0 (research and educational use per Ai2 guidelines).", "quote": "This model is licensed under Apache 2.0." } ] }, { "id": "multitask-dit", "name": "Multitask DiT", "aliases": [ "multi_task_dit", "Multitask Diffusion Transformer Policy", "Multi-Task DiT" ], "family": "diffusion", "summary": "Multitask Diffusion Transformer policy: an evolution of Diffusion Policy that conditions a diffusion transformer on CLIP vision features from multiple cameras and a CLIP-encoded language instruction. Diffusion (DDPM/DDIM) is the default objective; flow matching is selectable.", "authors": "Bryson Jones (open-source LeRobot implementation), after Toyota Research Institute's Large Behavior Model DiT policy", "paper_url": "https://arxiv.org/abs/2507.05331", "params": 450000000, "params_note": "\"~450M\" per the LeRobot docs (approximate; the default uses CLIP ViT-B/16 vision and text encoders, and the count changes with encoder choice and num_layers).", "params_source_ref": "ev-lerobot-mtdit-docs-params", "backbone": "CLIP vision encoder + CLIP text encoder (frozen, learnable projection) + diffusion transformer (DiT)", "action_output": "diffusion", "action_chunking": true, "arch_source_ref": "ev-lerobot-mtdit-docs-arch", "license": "Apache-2.0 (LeRobot implementation)", "frameworks": { "lerobot": { "support": "native", "since": "0.5.1", "policy_type": "multi_task_dit", "docs_url": "https://huggingface.co/docs/lerobot/multi_task_dit", "note": "Install with `pip install lerobot[multi_task_dit]`. Defaults: objective diffusion (DDPM), horizon 32, n_action_steps 24, n_obs_steps 2.", "source_ref": "ev-lerobot-mtdit-config" } }, "requirements": { "train": { "vram_gb": { "min": 8, "max": 14 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Diffusion\" group (diffusion, multi_task_dit) in the LeRobot compute hardware guide. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated. Note the policy's own docs call batch 64 the minimum for stable training and recommend 256–320, so real runs need more memory than this BS-8 envelope.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "condition": "No VRAM figure published. The policy docs recommend at least an RTX 5070 Ti (or equivalent GPU) for reasonable inference speed.", "source_ref": "ev-lerobot-mtdit-docs-gpu" } }, "observation_space": { "language": "optional", "source_ref": "ev-lerobot-mtdit-processor" }, "notes": "Family is \"diffusion\": the docs describe it as \"an evolution of the original Diffusion Policy architecture\" and diffusion is the default objective (flow matching is an option). Uses multiple camera views (all images must share one shape); separate per-camera encoders are optional.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-mtdit-docs-arch", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/multi_task_dit.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Multitask DiT is a DiT evolution of Diffusion Policy with text and vision conditioning, supporting diffusion and flow matching.", "quote": "Multitask Diffusion Transformer (DiT) Policy is an evolution of the original Diffusion Policy architecture, which leverages a large DiT with text and vision conditioning for multitask robot learning." }, { "id": "ev-lerobot-mtdit-docs-params", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/multi_task_dit.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Docs give an approximate parameter count.", "quote": "VLAs, with only ~450M parameters and significantly less training." }, { "id": "ev-lerobot-mtdit-docs-gpu", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/multi_task_dit.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Docs recommend at least an RTX 5070 Ti for inference and batch sizes >64 for training.", "quote": "**Inference**: At least an RTX 5070 Ti (or equivalent GPU) is recommended for reasonable speed performance" }, { "id": "ev-lerobot-mtdit-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/multi_task_dit/configuration_multi_task_dit.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `multi_task_dit`; default objective is diffusion.", "quote": "@PreTrainedConfig.register_subclass(\"multi_task_dit\")" }, { "id": "ev-lerobot-mtdit-processor", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/multi_task_dit/processor_multi_task_dit.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "The preprocessor tokenizes the language task description only if present.", "quote": "3. Tokenizing the language task description (if present)." }, { "id": "ev-lerobot-v051-release", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.5.1", "date": "2026-04-07", "verified_on": "2026-10-24", "versions": { "lerobot": "0.5.1" }, "summary": "Multitask DiT policy was added in v0.5.1 (absent from v0.5.0's policies directory).", "quote": "Feature/add multitask diffusion transformer policy implementation by @brysonjones" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Diffusion group (`diffusion`, `multi_task_dit`) peak VRAM ~8–14GB at BS 8 with AdamW; figures are indicative.", "quote": "| Diffusion | `diffusion`, `multi_task_dit` | ~8–14GB | RTX 4070+ / L4 / A10G |" } ] }, { "id": "openvla-oft", "name": "OpenVLA-OFT", "aliases": [ "openvla_oft", "OpenVLA-OFT+", "moojink/openvla-oft" ], "family": "vla", "summary": "Optimized Fine-Tuning (OFT) recipe for OpenVLA from Stanford: replaces autoregressive token decoding with parallel decoding of continuous action chunks trained with an L1 regression objective (optionally FiLM for stronger language grounding, \"OFT+\"), improving inference speed and task success when adapting OpenVLA to new robots.", "authors": "Stanford University (Moo Jin Kim, Chelsea Finn, Percy Liang)", "paper_url": "https://arxiv.org/abs/2502.19645", "backbone": "OpenVLA (Prismatic VLM, Llama 2 language model) with an MLP action head and proprio projector", "action_output": "regression", "action_chunking": true, "arch_source_ref": "ev-oft-paper-recipe", "license": "MIT", "weights_license": "MIT", "open_weights": true, "frameworks": { "openvla-oft": { "support": "upstream-repo", "repo_url": "https://github.com/moojink/openvla-oft", "note": "Official code; not part of LeRobot v0.6.1 (no openvla/oft policy directory).", "source_ref": "ev-oft-readme-hw" } }, "checkpoints": [ { "hub_id": "moojink/openvla-7b-oft-finetuned-libero-spatial", "note": "Used in the README quick-start (MIT tag on the Hub)." }, { "hub_id": "moojink/openvla-7b-oft-finetuned-libero-10" }, { "hub_id": "moojink/openvla-7b-oft-finetuned-libero-spatial-object-goal-10" } ], "requirements": { "train": { "vram_gb": { "min": 27, "max": 80 }, "precision": "bf16", "condition": "Upstream openvla-oft README: between 1 and 8 GPUs with 27–80 GB each, depending on the training setup, with the default bfloat16 data type. Not a LeRobot figure; batch size not stated in the README.", "source_ref": "ev-oft-readme-train" }, "inference": { "vram_gb": { "min": 16 }, "condition": "Upstream openvla-oft README: 1 GPU with ~16 GB VRAM for LIBERO sim tasks (~18 GB for ALOHA robot tasks). Precision not stated in this line; the quick-start loads without 8/4-bit quantization.", "source_ref": "ev-oft-readme-hw" } }, "notes": "Not integrated in LeRobot v0.6.1. Parameter count: the paper does not restate a total for the OFT model (it builds on the 7B OpenVLA plus an action head), so params is left out. The README quick-start uses two images (third-person + wrist) and proprioceptive state for LIBERO.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-oft-paper-recipe", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2502.19645", "date": "2025-02-27", "verified_on": "2026-10-24", "summary": "OFT integrates parallel decoding, action chunking, continuous actions and an L1 regression objective.", "quote": "an instantiation of an Optimized Fine-Tuning (OFT) recipe that integrates parallel decoding and action chunking, continuous action representations, and an L1 regression objective" }, { "id": "ev-oft-readme-hw", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/moojink/openvla-oft/blob/e4287e94541f459edc4feabc4e181f537cd569a8/README.md", "date": "2025-09-09", "verified_on": "2026-10-24", "summary": "Upstream inference requirement for LIBERO sim tasks (~16 GB; ALOHA ~18 GB).", "quote": "* 1 GPU with ~16 GB VRAM for LIBERO sim benchmark tasks" }, { "id": "ev-oft-readme-train", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/moojink/openvla-oft/blob/e4287e94541f459edc4feabc4e181f537cd569a8/README.md", "date": "2025-09-09", "verified_on": "2026-10-24", "summary": "Upstream training requirement, 1-8 GPUs with 27-80 GB in bfloat16.", "quote": "* Between 1-8 GPUs with 27-80 GB, depending on the desired training setup (with default bfloat16 data type)." } ] }, { "id": "openvla", "name": "OpenVLA", "aliases": [ "openvla-7b", "openvla/openvla-7b", "OpenVLA-7B" ], "family": "vla", "summary": "Open-source 7B-class VLA from Stanford, UC Berkeley, Google DeepMind and TRI: a Prismatic VLM (fused DINOv2 + SigLIP vision encoder, Llama 2 language model) fine-tuned on Open X-Embodiment to emit robot actions as discrete language tokens (256 bins per action dimension), one 7-DoF end-effector delta per step.", "authors": "Stanford, UC Berkeley, Google DeepMind, Toyota Research Institute (OpenVLA team)", "paper_url": "https://arxiv.org/abs/2406.09246", "params": 7500000000, "params_note": "README says fully fine-tuning touches 'all 7.5 billion parameters'; the paper calls it a '7B-parameter' VLA.", "params_source_ref": "ev-openvla-readme-params", "backbone": "Prismatic VLM (DINOv2 + SigLIP fused vision encoder, Llama 2 language model)", "action_output": "autoregressive-tokens", "action_chunking": false, "arch_source_ref": "ev-openvla-paper-tokens", "license": "MIT", "weights_license": "MIT (model card); README notes the weights are derived from Llama 2 and subject to the Llama Community License", "open_weights": true, "frameworks": { "openvla": { "support": "upstream-repo", "repo_url": "https://github.com/openvla/openvla", "note": "Official code and checkpoints; loads through Hugging Face transformers (trust_remote_code). Not part of LeRobot v0.6.1 (no openvla policy directory).", "source_ref": "ev-openvla-readme-params" } }, "checkpoints": [ { "hub_id": "openvla/openvla-7b", "note": "Flagship model from the paper (MIT tag on the Hub)." } ], "requirements": { "train": { "vram_gb": { "min": 27, "max": 72 }, "condition": "Upstream openvla README, LoRA fine-tuning of openvla-7b with vla-scripts/finetune.py (LoRA rank 32): batch_size 16 with grad_accumulation_steps 1 needs ~72 GB; a smaller GPU works if it has at least ~27 GB, by reducing the batch size. Full fine-tuning instead uses FSDP (e.g. a node of 8 A100s). Precision not stated in that section.", "source_ref": "ev-openvla-readme-lora" }, "inference": { "vram_gb": { "measured": 15 }, "precision": "bf16", "condition": "OpenVLA paper: 15 GB GPU memory when loaded in bfloat16 without quantization; runs at about 6 Hz on one RTX 4090 without compilation or other speed-ups. Batch size not stated.", "source_ref": "ev-openvla-paper-mem" } }, "observation_space": { "cameras": { "min": 1, "max": 1 }, "state": "not used (inputs are one camera image and a language instruction)", "language": "required", "source_ref": "ev-openvla-card-io" }, "notes": "Not integrated in LeRobot v0.6.1 (no `openvla` policy type). Action chunking is not part of the original model; OpenVLA-OFT (separate record) adds parallel decoding and chunking. The README's LoRA section uses Hugging Face PEFT.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-openvla-readme-params", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/openvla/openvla/blob/c8f03f48af692657d3060c19588038c7220e9af9/README.md", "date": "2025-03-23", "verified_on": "2026-10-24", "summary": "Upstream README states the full model has 7.5 billion parameters.", "quote": "fully fine-tuning OpenVLA (all 7.5 billion parameters) via native PyTorch Fully Sharded Data Parallel (FSDP)" }, { "id": "ev-openvla-readme-lora", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/openvla/openvla/blob/c8f03f48af692657d3060c19588038c7220e9af9/README.md", "date": "2025-03-23", "verified_on": "2026-10-24", "summary": "LoRA fine-tuning needs ~27 GB minimum; batch 16 without accumulation needs ~72 GB.", "quote": "requires ~72 GB GPU memory. If you have a smaller GPU, you should reduce `--batch_size` and increase `--grad_accumulation_steps`" }, { "id": "ev-openvla-readme-lora-min", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/openvla/openvla/blob/c8f03f48af692657d3060c19588038c7220e9af9/README.md", "date": "2025-03-23", "verified_on": "2026-10-24", "summary": "LoRA fine-tuning works on GPUs with at least ~27 GB.", "quote": "(You can also fine-tune with a smaller GPU, as long as it has at least ~27 GB of memory," }, { "id": "ev-openvla-readme-license", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/openvla/openvla/blob/c8f03f48af692657d3060c19588038c7220e9af9/README.md", "date": "2025-03-23", "verified_on": "2026-10-24", "summary": "Code is MIT; pretrained models derive from Llama 2 and inherit the Llama Community License restrictions.", "quote": "above models are derived from Llama-2, and as such are subject to the" }, { "id": "ev-openvla-paper-tokens", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2406.09246", "date": "2024-06-13", "verified_on": "2026-10-24", "summary": "OpenVLA maps each continuous action dimension to one of 256 discrete token bins.", "quote": "we discretize each dimension of the robot actions separately into one of 256 bins." }, { "id": "ev-openvla-paper-mem", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2406.09246", "date": "2024-06-13", "verified_on": "2026-10-24", "summary": "Inference needs 15 GB in bfloat16 and runs at ~6 Hz on an RTX 4090.", "quote": "During inference, OpenVLA requires 15GB of GPU memory when loaded in bfloat16 precision (i.e., without quantization) and runs at approximately 6Hz on one NVIDIA RTX 4090 GPU" }, { "id": "ev-openvla-card-io", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/openvla/openvla-7b", "verified_on": "2026-10-24", "summary": "Model card - inputs are a language instruction and one camera image; outputs 7-DoF end-effector deltas.", "quote": "OpenVLA models take a language instruction and a camera image of a robot workspace as input, and predict (normalized) robot actions consisting of 7-DoF end-effector deltas" } ] }, { "id": "pi0-fast", "name": "π0-FAST", "aliases": [ "pi0_fast", "pi0fast", "pi0-fast", "Pi0-FAST", "π₀-FAST", "PiFast", "lerobot/pi0fast-base" ], "family": "vla", "summary": "Autoregressive variant of π0 from Physical Intelligence: the PaliGemma VLM predicts discrete FAST action tokens (DCT + BPE compression of action chunks) that are decoded back to continuous actions.", "authors": "Physical Intelligence", "paper_url": "https://arxiv.org/abs/2501.09747", "backbone": "PaliGemma (SigLIP vision tower + Gemma 2B)", "action_output": "autoregressive-tokens", "action_chunking": true, "arch_source_ref": "ev-lerobot-pi0fast-docs", "license": "Apache-2.0", "weights_license": "Gemma (Hub license tag `gemma` on lerobot/pi0fast-base)", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.4.3", "policy_type": "pi0_fast", "docs_url": "https://huggingface.co/docs/lerobot/pi0fast", "note": "Current implementation added in v0.4.3 (\"add autoregressive VLAs with tokenization PiFast\"). An earlier implementation registered as `pi0fast` existed in v0.3.2–v0.3.3 and was deprecated/removed in v0.4.0 (\"deprecate pi0fast\"); the directory is absent at v0.4.0–v0.4.2. Default FAST tokenizer: lerobot/fast-action-tokenizer.", "source_ref": "ev-lerobot-v043-notes" }, "openpi": { "support": "upstream-repo", "repo_url": "https://github.com/Physical-Intelligence/openpi", "docs_url": "https://github.com/Physical-Intelligence/openpi", "note": "Official release; base checkpoint gs://openpi-assets/checkpoints/pi0_fast_base, plus π0-FAST-DROID.", "source_ref": "ev-openpi-readme-fast" } }, "checkpoints": [ { "hub_id": "lerobot/pi0fast-base", "note": "LeRobot-format base model; Hub license tag gemma." }, { "hub_id": "lerobot/pi0fast-libero", "note": "LIBERO fine-tune used for the docs' results (82.5% average); no license tag on the Hub." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Large VLA\" group (pi0, pi0_fast, pi05, xvla, wall_x) in the LeRobot compute hardware guide; suggested A100 40 GB+, \"24 GB tight at BS 1\". The guide calls its figures indicative, order-of-magnitude numbers, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "min": 8 }, "condition": "openpi README single-GPU requirement for inference (\"> 8 GB\", example RTX 4090), stated for all openpi models and the openpi runtime, not LeRobot. Precision not stated. No LeRobot measurement for π0-FAST found.", "source_ref": "ev-openpi-requirements" } }, "observation_space": { "state": "proprio/state vector, padded to max_state_dim 32", "language": "optional", "source_ref": "ev-pi0fast-base-card" }, "notes": "RTC: not listed in LeRobot's RTC docs. In v0.6.1 code PI0FastConfig has an rtc_config field, but the policy does not override supports_rtc() (base default False) and select_action asserts RTC is disabled; left unset here pending a documented statement. KV-caching is on by default (use_kv_cache=true).", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-pi0fast-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/pi0fast.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot π0-FAST docs; PaliGemma (SigLIP + Gemma 2B) predicting discrete FAST tokens decoded to continuous actions.", "quote": "The model takes images, text instructions, and robot state as input, and outputs discrete FAST tokens that are decoded back to continuous actions." }, { "id": "ev-lerobot-pi0fast-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/pi0_fast/configuration_pi0_fast.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `pi0_fast` (also registered at v0.4.3); chunk_size default 50.", "quote": "@PreTrainedConfig.register_subclass(\"pi0_fast\")" }, { "id": "ev-lerobot-v043-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.3", "date": "2026-01-22", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.3" }, "summary": "v0.4.3 added the current π0-FAST (pi0_fast) implementation.", "quote": "* feat(policies): add autoregressive VLAs with tokenization PiFast by @jadechoghari in https://github.com/huggingface/lerobot/pull/2734" }, { "id": "ev-lerobot-v040-notes-deprecate", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.0", "date": "2025-10-23", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.0" }, "summary": "The earlier `pi0fast` implementation (v0.3.x) was deprecated in v0.4.0.", "quote": "* chore(policies): deprecate pi0fast by @imstevenpmwork in https://github.com/huggingface/lerobot/pull/2203" }, { "id": "ev-fast-paper", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2501.09747", "date": "2025-01-16", "verified_on": "2026-10-24", "summary": "FAST paper introduces the DCT-based action tokenizer used with the π0 VLA.", "quote": "Our tokenization approach, Frequency-space Action Sequence Tokenization (FAST), enables us to train autoregressive VLAs for highly dexterous and high-frequency tasks where standard discretization methods fail completely." }, { "id": "ev-openpi-readme-fast", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi (official PI repo) ships π0-FAST with base checkpoints.", "quote": "- the [π₀-FAST model](https://www.physicalintelligence.company/research/fast), an autoregressive VLA, based on the FAST action tokenizer." }, { "id": "ev-openpi-requirements", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi single-GPU memory requirements: inference >8 GB, LoRA fine-tuning >22.5 GB, full fine-tuning >70 GB.", "quote": "| Inference | > 8 GB | RTX 4090 |" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Large VLA group (incl. `pi0_fast`) ~24–40GB at BS 8 with AdamW; indicative figures.", "quote": "| Large VLA | `pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x` | ~24–40GB | A100 40 GB+ (24 GB tight at BS 1) |" }, { "id": "ev-pi0fast-base-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/lerobot/pi0fast-base", "date": "2026-01-09", "verified_on": "2026-10-24", "summary": "lerobot/pi0fast-base card lists inputs, next-token objective and FAST-token action representation; license gemma.", "quote": "- **Inputs:** images (multi-view), proprio/state, optional language instruction" } ] }, { "id": "pi0", "name": "π0", "aliases": [ "pi0", "pi_0", "Pi0", "π₀", "lerobot/pi0_base" ], "family": "vla", "summary": "Physical Intelligence's flow-matching vision-language-action model: a PaliGemma VLM plus a 300M-parameter action expert that predicts 50-step action chunks. Open weights via openpi; LeRobot ships a PyTorch port.", "authors": "Physical Intelligence", "paper_url": "https://arxiv.org/abs/2410.24164", "params": 3300000000, "params_note": "Authors state 3.3 billion (PaliGemma 3B + 300M action expert). The LeRobot paper's Table 2 lists π0 as 3.5B.", "params_source_ref": "ev-pi0-paper-params", "backbone": "PaliGemma (SigLIP + Gemma 2B), with a Gemma-300M action expert", "action_output": "flow-matching", "action_chunking": true, "rtc_compatible": true, "peft_supported": true, "arch_source_ref": "ev-pi0-paper-arch", "license": "Apache-2.0", "weights_license": "Gemma (Hub license tag `gemma` on lerobot/pi0_base)", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.3.2", "policy_type": "pi0", "docs_url": "https://huggingface.co/docs/lerobot/pi0", "note": "`pi0` is registered at v0.3.2, the first tagged LeRobot release (the only earlier PyPI release, 0.1.0 of 2024-03, predates π0). The v0.4.0 release notes list \"Add OpenPi, Pi0 and Pi0.5\" (PR #1910); the docs describe the current implementation as adapted from openpi. RTC per rtc.mdx; PEFT per peft_training.mdx.", "source_ref": "ev-lerobot-pi0-config-v032" }, "openpi": { "support": "upstream-repo", "repo_url": "https://github.com/Physical-Intelligence/openpi", "docs_url": "https://github.com/Physical-Intelligence/openpi", "note": "Official Physical Intelligence release (JAX, plus PyTorch since Sept 2025); base checkpoint gs://openpi-assets/checkpoints/pi0_base.", "source_ref": "ev-openpi-readme" } }, "checkpoints": [ { "hub_id": "lerobot/pi0_base", "note": "LeRobot-format base model; Hub license tag gemma." }, { "hub_id": "lerobot/pi0_libero_base", "note": "LIBERO-trained variant. The v0.6.1 pi0 docs link lerobot/pi0_libero, which the Hub redirects here." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Large VLA\" group (pi0, pi0_fast, pi05, xvla, wall_x) in the LeRobot compute hardware guide; suggested A100 40 GB+, \"24 GB tight at BS 1\". The guide calls its figures indicative, order-of-magnitude numbers, not exact predictions. Precision not stated.", "reference_time": "~4–8 h for 5 epochs on a ~45k-frame dataset at batch 4 on a single A100 40 GB (indicative, ±50%).", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "measured": 13.32 }, "precision": "fp32", "condition": "Peak memory at inference measured by the LeRobot paper (Table 2): 13.32 GB on both RTX 4090 and A100, all models in full fp32 precision, 10 flow-matching denoising steps. (Same table: 4.13GB CPU-only, 97MB on Apple MPS, where unified memory and swap make figures unreliable per the paper.)", "source_ref": "ev-lerobot-paper-mem" } }, "observation_space": { "state": "proprio/state vector, padded to max_state_dim 32", "language": "optional", "source_ref": "ev-pi0-base-card" }, "notes": "openpi's own README (JAX/openpi runtime, not LeRobot) states single-GPU needs of >8 GB for inference, >22.5 GB for LoRA fine-tuning and >70 GB for full fine-tuning (ev-openpi-requirements). In LeRobot, train_expert_only=true freezes the VLM to reduce fine-tuning memory.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-pi0-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2410.24164", "date": "2024-10-31", "verified_on": "2026-10-24", "summary": "π0 paper states total parameter count (PaliGemma 3B + 300M action expert).", "quote": "We add 300M parameters for the action expert (which is initialized from scratch) for a total of 3.3 billion parameters." }, { "id": "ev-pi0-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2410.24164", "date": "2024-10-31", "verified_on": "2026-10-24", "summary": "π0 builds on PaliGemma and supervises action-chunk tokens with conditional flow matching (H = 50).", "quote": "During training, we supervise these action tokens using a conditional flow matching loss" }, { "id": "ev-lerobot-pi0-config-v032", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.3.2/src/lerobot/policies/pi0/configuration_pi0.py", "date": "2025-08-01", "verified_on": "2026-10-24", "versions": { "lerobot": "0.3.2" }, "summary": "`pi0` policy type is registered at v0.3.2 (and unchanged at v0.6.1).", "quote": "@PreTrainedConfig.register_subclass(\"pi0\")" }, { "id": "ev-lerobot-v040-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.0", "date": "2025-10-23", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.0" }, "summary": "v0.4.0 added the openpi-based Pi0 and Pi0.5 implementations.", "quote": "* Add OpenPi, Pi0 and Pi0.5 by @pkooij in https://github.com/huggingface/lerobot/pull/1910" }, { "id": "ev-lerobot-pi0-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/pi0.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot π0 docs; implementation adapted from openpi, Apache 2.0.", "quote": "The LeRobot implementation is adapted from their open source [OpenPI](https://github.com/Physical-Intelligence/openpi) repository." }, { "id": "ev-openpi-readme", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi is Physical Intelligence's official repo containing π0 with base checkpoints.", "quote": "- the [π₀ model](https://www.physicalintelligence.company/blog/pi0), a flow-based vision-language-action model (VLA)." }, { "id": "ev-openpi-requirements", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi single-GPU memory requirements: inference >8 GB, LoRA fine-tuning >22.5 GB, full fine-tuning >70 GB.", "quote": "| Fine-Tuning (Full) | > 70 GB | A100 (80GB) / H100 |" }, { "id": "ev-lerobot-rtc-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/rtc.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot RTC docs name Pi0 as a supported flow-matching policy.", "quote": "allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion" }, { "id": "ev-lerobot-peft-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/peft_training.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot PEFT guide names π₀ among the pretrained policies it adapts (PI0Policy also defines default LoRA targets).", "quote": "large pretrained models such as pre-trained policies (e.g., SmolVLA, π₀, ...) to new tasks without training all" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Large VLA group (incl. `pi0`) peak VRAM ~24–40GB at BS 8 with AdamW; indicative figures.", "quote": "| Large VLA | `pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x` | ~24–40GB | A100 40 GB+ (24 GB tight at BS 1) |" }, { "id": "ev-lerobot-paper-mem", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2602.22818", "date": "2026-02-26", "verified_on": "2026-10-24", "summary": "LeRobot paper Table 2: π0 (3.5B) peak memory 4.13GB CPU, 97MB MPS, 13.32 GB RTX 4090, 13.32 GB A100, fp32.", "quote": "Table 2: Peak memory consumption of policy models currently supported by lerobot . All models are run in full precision (fp32). Diffusion and Flow Models are run with 10 denoising steps at inference." }, { "id": "ev-pi0-base-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/lerobot/pi0_base", "date": "2025-09-09", "verified_on": "2026-10-24", "summary": "lerobot/pi0_base card lists inputs (multi-view images, state, optional language), flow-matching objective, license gemma.", "quote": "- **Inputs:** images (multi-view), proprio/state, optional language instruction" } ] }, { "id": "pi05", "name": "π0.5", "aliases": [ "pi05", "pi0.5", "pi_05", "Pi0.5", "π₀.₅", "lerobot/pi05_base" ], "family": "vla", "summary": "Physical Intelligence's successor to π0, co-trained on heterogeneous robot, web and instruction data for open-world generalization. The open release (openpi, and LeRobot's port) supports only the flow-matching action head.", "authors": "Physical Intelligence", "paper_url": "https://arxiv.org/abs/2504.16054", "backbone": "PaliGemma-based VLM with a 300M-parameter action expert", "action_output": "flow-matching", "action_chunking": true, "rtc_compatible": true, "peft_supported": true, "arch_source_ref": "ev-openpi-readme-pi05", "license": "Apache-2.0", "weights_license": "Gemma (Hub license tag `gemma` on lerobot/pi05_base)", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.4.0", "policy_type": "pi05", "docs_url": "https://huggingface.co/docs/lerobot/pi05", "note": "Added in v0.4.0 (\"Add OpenPi, Pi0 and Pi0.5\"). Uses the gated google/paligemma-3b-pt-224 tokenizer (accept its license on the Hub). Normalizes STATE/ACTION with quantiles by default, so datasets need q01/q99 stats. RTC per rtc.mdx; PEFT default LoRA targets defined in PI05Policy (source code; not covered by a dedicated doc example).", "source_ref": "ev-lerobot-v040-notes" }, "openpi": { "support": "upstream-repo", "repo_url": "https://github.com/Physical-Intelligence/openpi", "docs_url": "https://github.com/Physical-Intelligence/openpi", "note": "Official release; base checkpoint gs://openpi-assets/checkpoints/pi05_base. openpi supports only the flow-matching head for π0.5.", "source_ref": "ev-openpi-readme-pi05" } }, "checkpoints": [ { "hub_id": "lerobot/pi05_base", "note": "LeRobot-format base model; Hub license tag gemma." }, { "hub_id": "lerobot/pi05_libero_base", "note": "LIBERO-trained base for LIBERO fine-tuning (per the v0.6.1 pi05 docs)." }, { "hub_id": "lerobot/pi05_libero_finetuned_v044", "note": "Checkpoint the docs' LIBERO results (97.5% average) were measured on." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Large VLA\" group (pi0, pi0_fast, pi05, xvla, wall_x) in the LeRobot compute hardware guide; suggested A100 40 GB+, \"24 GB tight at BS 1\". The guide calls its figures indicative, order-of-magnitude numbers, not exact predictions. Precision not stated.", "reference_time": "~4–8 h for 5 epochs on a ~45k-frame dataset at batch 4 on a single A100 40 GB (indicative, ±50%); measured 3h 41m for 5000 steps at batch 32 on 4× H100 80 GB with gradient_checkpointing=true, dtype=bfloat16, from lerobot/pi05_base.", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "min": 8 }, "condition": "openpi README single-GPU requirement for inference (\"> 8 GB\", example RTX 4090), stated for all openpi models and the openpi runtime, not LeRobot. Precision not stated. No LeRobot measurement for π0.5 found.", "source_ref": "ev-openpi-requirements" } }, "observation_space": { "state": "proprio/state vector; in the LeRobot implementation it is discretized into 256 bins and written into the prompt", "language": "optional", "source_ref": "ev-pi05-base-card" }, "notes": "The v0.6.1 docs size their full LIBERO fine-tune command (batch 64, bf16, gradient checkpointing) \"for a single 80 GB GPU\"; freezing the VLM (train_expert_only=true) reduces memory at some cost in success rate. openpi README: LoRA fine-tuning >22.5 GB, full fine-tuning >70 GB (openpi runtime).", "verified_on": "2026-10-24", "sources": [ { "id": "ev-openpi-readme-pi05", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi (official PI repo) ships π0.5 and supports only its flow-matching head.", "quote": "Note that, in this repository, we currently only support the flow matching head for both $\\pi_{0.5}$ training and inference." }, { "id": "ev-pi05-paper", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2504.16054", "date": "2025-04-22", "verified_on": "2026-10-24", "summary": "π0.5 paper (arXiv HTML v1); the final model predicts continuous actions with flow matching, like π0.", "quote": "we use flow-matching [ 50 ] to predict continuous actions in the final model." }, { "id": "ev-lerobot-v040-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.0", "date": "2025-10-23", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.0" }, "summary": "v0.4.0 is the first release with Pi0.5 (`pi05` registered at that tag).", "quote": "* Add OpenPi, Pi0 and Pi0.5 by @pkooij in https://github.com/huggingface/lerobot/pull/1910" }, { "id": "ev-lerobot-pi05-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/pi05/configuration_pi05.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `pi05`.", "quote": "@PreTrainedConfig.register_subclass(\"pi05\")" }, { "id": "ev-lerobot-pi05-peft-code", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/pi05/modeling_pi05.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "PI05Policy defines default PEFT (LoRA) target modules for the generic --peft.* training path.", "quote": "\"\"\"Return default PEFT target modules for PI0.5 fine-tuning.\"\"\"" }, { "id": "ev-lerobot-pi05-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/pi05.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot π0.5 docs; state is discretized into the prompt, quantile normalization by default.", "quote": "| `observation.state` | 8 | discretized into 256 bins and written into the prompt |" }, { "id": "ev-openpi-requirements", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/Physical-Intelligence/openpi/blob/215abfb217dbac7d5f1273282331b9b1866c0479/README.md", "verified_on": "2026-10-24", "versions": { "openpi": "215abfb" }, "summary": "openpi single-GPU memory requirements: inference >8 GB, LoRA fine-tuning >22.5 GB, full fine-tuning >70 GB.", "quote": "| Inference | > 8 GB | RTX 4090 |" }, { "id": "ev-lerobot-rtc-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/rtc.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot RTC docs name Pi0.5 as a supported flow-matching policy.", "quote": "allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Large VLA group (incl. `pi05`) ~24–40GB at BS 8 with AdamW; pi05 4×H100 reference run 3h 41m.", "quote": "| Large VLA | `pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x` | ~24–40GB | A100 40 GB+ (24 GB tight at BS 1) |" }, { "id": "ev-pi05-base-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/lerobot/pi05_base", "date": "2025-09-09", "verified_on": "2026-10-24", "summary": "lerobot/pi05_base card lists inputs and notes only the flow-matching head is supported.", "quote": "- **Inputs:** images (multi-view), proprio/state, optional language instruction" } ] }, { "id": "robometer", "name": "Robometer", "aliases": [ "ROBOMETER", "Robometer-4B", "lerobot/Robometer-4B" ], "family": "reward-model", "in_grid": false, "summary": "General-purpose video-language robotic reward model on a Qwen3-VL-4B backbone with progress, success and preference heads. Predicts dense per-frame task progress and success from a trajectory video and task description; in LeRobot it is used zero-shot (inference-only) and returns one scalar reward per sample.", "authors": "Anthony Liang, Jesse Zhang et al. (University of Southern California, University of Washington, Ai2, NVIDIA and others)", "paper_url": "https://arxiv.org/abs/2603.02115", "backbone": "Qwen3-VL-4B-Instruct + progress / success / preference heads", "action_output": "scalar-reward", "arch_source_ref": "ev-lerobot-robometer-docs", "license": "Apache-2.0 (LeRobot integration)", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "robometer", "docs_url": "https://huggingface.co/docs/lerobot/robometer", "note": "A reward model, not a policy: lives in src/lerobot/rewards/robometer, registered with RewardModelConfig.register_subclass(\"robometer\") and selected as `reward_model.type=robometer` (not --policy.type). Inference-only in LeRobot (no training or preference-pair training). Ships a compute_rabc_weights module for offline progress labelling.", "source_ref": "ev-lerobot-robometer-config" } }, "checkpoints": [ { "hub_id": "lerobot/Robometer-4B", "note": "Published ROBOMETER checkpoint used by the LeRobot integration (Hub license tag apache-2.0)." } ], "observation_space": { "cameras": { "min": 1, "max": 1 }, "state": "not used (video + task description only)", "language": "required", "source_ref": "ev-lerobot-robometer-docs-inputs" }, "notes": "in_grid false: reward model, not a robot-control policy. reward_output=\"progress\" (default) returns last-frame progress clamped to [0, 1]; \"success\" returns a thresholded last-frame success. Default 8 frames from one configured camera key (image_key, default observation.images.top). The docs strongly recommend GPU inference but give no VRAM figure; params not stated as an exact total. Upstream robometer/robometer code: no license detected by the GitHub API on 2026-10-24.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-robometer-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/robometer.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "ROBOMETER builds on Qwen3-VL-4B with progress, success and preference heads; LeRobot integration is inference-only.", "quote": "ROBOMETER builds on `Qwen/Qwen3-VL-4B-Instruct` and adds three lightweight prediction heads:" }, { "id": "ev-lerobot-robometer-docs-inputs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/robometer.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Video-language only; no privileged robot state.", "quote": "ROBOMETER is video-language based; it does not use privileged robot state such as contact forces or object poses." }, { "id": "ev-lerobot-robometer-docs-inference", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/robometer.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot integration is inference-only.", "quote": "The LeRobot integration is currently **inference-only**." }, { "id": "ev-lerobot-robometer-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/rewards/robometer/configuration_robometer.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Registered as reward-model type `robometer`.", "quote": "@RewardModelConfig.register_subclass(\"robometer\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "ROBOMETER was added in LeRobot v0.6.0.", "quote": "* feat(rewards): add ROBOMETER reward model by @s1lent4gnt in https://github.com/huggingface/lerobot/pull/3627" }, { "id": "ev-hub-robometer-checkpoint", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/lerobot/Robometer-4B", "verified_on": "2026-10-24", "summary": "Hub API: lerobot/Robometer-4B exists, tagged license:apache-2.0." } ] }, { "id": "sarm", "name": "SARM", "aliases": [ "Stage-Aware Reward Modeling", "sarm_reward_model" ], "family": "reward-model", "in_grid": false, "summary": "Stage-Aware Reward Modeling: a video-based reward model for long-horizon manipulation that jointly predicts the current task stage and within-stage progress, normalised to task progress in [0, 1]. In LeRobot it is trained on your own dataset and its progress predictions can weight policy training via Reward-Aligned Behavior Cloning (RA-BC).", "authors": "Qianzhong Chen et al. (Stanford University, UC Berkeley, xdof.ai)", "paper_url": "https://arxiv.org/abs/2509.25358", "backbone": "frozen CLIP ViT-B/32 image/text encoders + transformer stage/progress heads", "action_output": "scalar-reward", "arch_source_ref": "ev-lerobot-sarm-docs-inputs", "license": "Apache-2.0 (LeRobot)", "frameworks": { "lerobot": { "support": "native", "since": "0.4.3", "policy_type": "sarm", "docs_url": "https://huggingface.co/docs/lerobot/sarm", "note": "A reward model, not a policy: at v0.6.1 it lives in src/lerobot/rewards/sarm and is registered with RewardModelConfig.register_subclass(\"sarm\") (the train config has a separate `reward_model` field). Added in v0.4.3 under src/lerobot/policies/sarm; moved to src/lerobot/rewards in v0.6.0 (\"Reward models refactor\"). The v0.6.1 sarm.mdx training examples still pass `--policy.type=sarm`. Used to compute RA-BC sample weights (`--sample_weighting.type=rabc`), which the docs say currently supports PI0, PI0.5 and SmolVLA.", "source_ref": "ev-lerobot-sarm-config" } }, "observation_space": { "cameras": { "min": 1, "max": 1 }, "state": "robot state padded/truncated to max_state_dim 32 (state_key, default observation.state)", "language": "required", "source_ref": "ev-lerobot-sarm-docs-inputs" }, "notes": "in_grid false: reward model, not a robot-control policy. Three annotation modes (single_stage needs no annotations; dense_only and dual use VLM subtask annotations, default Qwen/Qwen3-VL-30B-A3B-Instruct). No official pretrained SARM checkpoint was found on the Hub under the lerobot org; users train their own, so open_weights, checkpoints and requirements are left unset. Camera count reflects the single configurable `image_key` (default observation.images.top).", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-sarm-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/sarm.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "SARM is a video-based reward modeling framework, optionally used with RA-BC.", "quote": "SARM (Stage-Aware Reward Modeling) is a video-based reward modeling framework for long-horizon robot manipulation tasks." }, { "id": "ev-lerobot-sarm-docs-inputs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/sarm.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "SARM processor encodes images and task text with CLIP ViT-B/32 and pads robot state.", "quote": "- **Encodes** images and task text with CLIP (ViT-B/32) into `video_features` and `text_features`" }, { "id": "ev-lerobot-sarm-docs-rabc", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/sarm.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "RA-BC weighting currently supports PI0, PI0.5 and SmolVLA.", "quote": "Currently PI0, PI0.5 and SmolVLA are supported with RA-BC:" }, { "id": "ev-lerobot-sarm-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/rewards/sarm/configuration_sarm.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "SARM is registered as reward-model type `sarm` (RewardModelConfig, not PreTrainedConfig).", "quote": "@RewardModelConfig.register_subclass(\"sarm\")" }, { "id": "ev-lerobot-v043-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.3", "date": "2026-01-22", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.3" }, "summary": "SARM was first added in LeRobot v0.4.3.", "quote": "* Add sarm by @pkooij in https://github.com/huggingface/lerobot/pull/2639" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "v0.6.0 refactored reward models into src/lerobot/rewards.", "quote": "* Reward models refactor by @s1lent4gnt in https://github.com/huggingface/lerobot/pull/3142" } ] }, { "id": "smolvla", "name": "SmolVLA", "aliases": [ "smolvla_base", "lerobot/smolvla_base" ], "family": "vla", "summary": "Hugging Face's compact vision-language-action model: a SmolVLM-2 backbone with a flow-matching action expert that outputs 50-step action chunks. Designed to be fine-tuned on LeRobot datasets on a single GPU.", "authors": "Hugging Face (LeRobot team)", "paper_url": "https://arxiv.org/abs/2506.01844", "params": 450000000, "params_note": "Authors state 450 million parameters (rounded), ~100M of which are the action expert.", "params_source_ref": "ev-smolvla-paper-params", "backbone": "SmolVLM-2 (SigLIP vision encoder + SmolLM2 decoder), upper layers skipped", "action_output": "flow-matching", "action_chunking": true, "rtc_compatible": true, "peft_supported": true, "arch_source_ref": "ev-smolvla-paper-arch", "license": "Apache-2.0", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.3.2", "policy_type": "smolvla", "docs_url": "https://huggingface.co/docs/lerobot/smolvla", "note": "Present in v0.3.2, the first tagged LeRobot release (the only earlier PyPI release, 0.1.0 of 2024-03, predates SmolVLA). RTC per rtc.mdx; PEFT/LoRA example in peft_training.mdx.", "source_ref": "ev-lerobot-smolvla-config" } }, "checkpoints": [ { "hub_id": "lerobot/smolvla_base", "note": "Pretrained base for fine-tuning (Apache-2.0 on the Hub)." } ], "requirements": { "train": { "vram_gb": { "min": 10, "max": 16 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Small VLA\" group in the LeRobot compute hardware guide. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "reference_time": "~3–6 h for 5 epochs on a ~45k-frame dataset at batch 4 on a single L4/A10G 24 GB; ~1–2 h at batch 16 on a single A100 40 GB (indicative, ±50%).", "source_ref": "ev-lerobot-hw-guide" }, "inference": { "vram_gb": { "measured": 1.75 }, "precision": "fp32", "condition": "Peak memory at inference measured by the LeRobot paper (Table 2): 1.75 GB on both RTX 4090 and A100, all models in full fp32 precision, 10 flow-matching denoising steps. (Same table: 1.69 GB CPU-only, 555 MB on Apple MPS.)", "source_ref": "ev-lerobot-paper-mem" } }, "observation_space": { "state": "robot sensorimotor state (proprioception), padded to max_state_dim 32", "language": "optional", "source_ref": "ev-smolvla-base-card" }, "notes": "The SmolVLA docs give a separate anchor: 20k fine-tuning steps take roughly 4 h on a single A100. The RTC docs recommend max_guidance_weight 10.0 for SmolVLA's 10-step flow matching.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-smolvla-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2506.01844", "date": "2025-06-02", "verified_on": "2026-10-24", "summary": "SmolVLA paper states the main model's parameter count.", "quote": "Our main model, contains 450 million parameters, with approximately 100 million dedicated to the action expert." }, { "id": "ev-smolvla-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2506.01844", "date": "2025-06-02", "verified_on": "2026-10-24", "summary": "SmolVLA is a compact VLM plus a flow-matching action expert outputting 50-action chunks.", "quote": "SmolVLA is a lightweight VLA composed of a compact pretrained VLM, and an action expert trained with flow matching." }, { "id": "ev-lerobot-smolvla-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/smolvla/configuration_smolvla.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `smolvla`; the same registration exists at tag v0.3.2.", "quote": "@PreTrainedConfig.register_subclass(\"smolvla\")" }, { "id": "ev-lerobot-rtc-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/rtc.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot RTC docs name SmolVLA as a supported flow-matching policy.", "quote": "allows large, flow-matching based robotic policies, such as [Pi0](./pi0), [Pi0.5](./pi05), and [SmolVLA](./smolvla), to produce smooth, continuous, and reactive motion" }, { "id": "ev-lerobot-peft-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/peft_training.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot PEFT guide walks through LoRA fine-tuning of lerobot/smolvla_base.", "quote": "In this section we'll show you how to train a pre-trained SmolVLA policy with PEFT on the libero dataset." }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Small VLA (`smolvla`) peak VRAM ~10–16GB at BS 8 with AdamW; figures are indicative.", "quote": "| Small VLA | `smolvla` | ~10–16GB | RTX 4080+ / L4 / A10G |" }, { "id": "ev-lerobot-paper-mem", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2602.22818", "date": "2026-02-26", "verified_on": "2026-10-24", "summary": "LeRobot paper Table 2: SmolVLA (450M) peak memory 1.69GB CPU, 555MB MPS, 1.75 GB RTX 4090, 1.75 GB A100, fp32.", "quote": "Table 2: Peak memory consumption of policy models currently supported by lerobot . All models are run in full precision (fp32). Diffusion and Flow Models are run with 10 denoising steps at inference." }, { "id": "ev-smolvla-base-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/lerobot/smolvla_base", "date": "2025-06-01", "verified_on": "2026-10-24", "summary": "Model card lists inputs (multi-view images, state, optional language) and the flow-matching objective.", "quote": "- **Inputs:** images (multi-view), proprio/state, optional language instruction" }, { "id": "ev-smolvla-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/smolvla.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "SmolVLA docs give a 20k-step fine-tuning time anchor on one A100.", "quote": "Training the model for 20k steps will roughly take ~4 hrs on a single A100 GPU." } ] }, { "id": "tdmpc", "name": "TD-MPC", "aliases": [ "Temporal Difference Learning for Model Predictive Control" ], "family": "rl", "summary": "Model-based reinforcement learning: a task-oriented latent dynamics model and terminal value function learned jointly by temporal-difference learning, with MPPI trajectory optimisation in latent space at each decision step.", "authors": "UC San Diego (Hansen, Wang, Su)", "paper_url": "https://arxiv.org/abs/2203.04955", "backbone": "Task-oriented latent dynamics model (TOLD) + terminal value function, MPPI planner", "action_output": "other", "arch_source_ref": "ev-tdmpc-paper-arch", "license": "Apache-2.0 (LeRobot implementation)", "frameworks": { "lerobot": { "support": "native", "policy_type": "tdmpc", "docs_url": "https://huggingface.co/docs/lerobot/policy_tdmpc_README", "note": "Present at v0.3.2, the earliest tagged LeRobot release. `since` left blank: the only earlier PyPI release (0.1.0, 2024-03, a TD-MPC-centric prototype) was not inspected. Defaults: use_mpc true, horizon 5, n_action_steps 1. action_output is \"other\": actions come from MPPI planning over the learned model, not a direct policy head.", "source_ref": "ev-lerobot-tdmpc-config" } }, "requirements": { "train": { "vram_gb": { "min": 2, "max": 6 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Light BC\" group (act, vqbet, tdmpc) in the LeRobot compute hardware guide; the guide groups tdmpc with the light BC policies by size. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" } }, "observation_space": { "cameras": { "min": 0, "max": 1 }, "source_ref": "ev-lerobot-tdmpc-config-features" }, "notes": "LeRobot's TD-MPC accepts at most one image, and it must be square.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-tdmpc-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2203.04955", "date": "2022-03-09", "verified_on": "2026-10-24", "summary": "TD-MPC learns a latent dynamics model and terminal value function by TD learning and plans with them.", "quote": "a framework for data-driven MPC using a task-oriented latent dynamics model and terminal value function learned jointly by temporal difference (TD) learning." }, { "id": "ev-lerobot-tdmpc-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/tdmpc/configuration_tdmpc.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `tdmpc`.", "quote": "@PreTrainedConfig.register_subclass(\"tdmpc\")" }, { "id": "ev-lerobot-tdmpc-config-features", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/tdmpc/configuration_tdmpc.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Feature validation allows at most one image and requires it to be square.", "quote": "f\"{self.__class__.__name__} handles at most one image for now. Got image keys {self.image_features}.\"" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Light BC (`act`, `vqbet`, `tdmpc`) peak VRAM ~2–6GB at BS 8 with AdamW; figures are indicative.", "quote": "| Light BC | `act`, `vqbet`, `tdmpc` | ~2–6GB | Laptop GPU (RTX 3060), L4, A10G |" } ] }, { "id": "topreward", "name": "TOPReward", "aliases": [ "topreward", "TOP-Reward" ], "family": "reward-model", "in_grid": false, "summary": "Zero-shot reward model that reads the token log-probability of \"True\" from a frozen vision-language model asked whether a trajectory video completes a task instruction. No fine-tuned weights: the LeRobot port wraps Qwen3-VL (default Qwen3-VL-8B-Instruct) and returns one scalar log-prob reward per sample.", "authors": "Shirui Chen et al. (University of Washington, Allen Institute for AI)", "paper_url": "https://arxiv.org/abs/2602.19313", "backbone": "frozen Qwen3-VL (default Qwen/Qwen3-VL-8B-Instruct)", "action_output": "scalar-reward", "arch_source_ref": "ev-lerobot-topreward-docs", "license": "Apache-2.0 (LeRobot integration); MIT (upstream TOPReward/TOPReward)", "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "topreward", "docs_url": "https://huggingface.co/docs/lerobot/topreward", "note": "A reward model, not a policy: lives in src/lerobot/rewards/topreward, registered with RewardModelConfig.register_subclass(\"topreward\") and selected as `reward_model.type=topreward` (not --policy.type). Inference-only and zero-shot; only the Qwen3-VL client is ported. Includes an offline labelling script that writes a SARM-compatible topreward_progress.parquet for RA-BC.", "source_ref": "ev-lerobot-topreward-config" } }, "checkpoints": [ { "hub_id": "Qwen/Qwen3-VL-8B-Instruct", "note": "Default frozen VLM (vlm_name); TOPReward has no weights of its own — save_pretrained writes only config.json." } ], "observation_space": { "cameras": { "min": 1, "max": 1 }, "state": "not used (video + task instruction only)", "language": "required", "source_ref": "ev-lerobot-topreward-docs" }, "notes": "in_grid false: reward model, not a robot-control policy. open_weights left unset because there are no TOPReward-specific weights (it uses an off-the-shelf open VLM). Docs note the first run downloads ~16 GB of Qwen3-VL-8B-Instruct weights and strongly recommend a GPU, but give no VRAM requirement. Default 16 frames at 2 fps from one configured camera key (image_key, default observation.images.top).", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-topreward-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/topreward.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "TOPReward extracts token log-probabilities from an off-the-shelf VLM given a video and task instruction; zero-shot.", "quote": "TOPReward is a **zero-shot reward model** that extracts token log-probabilities from an off-the-shelf vision-language model (VLM) as a robotic reward signal." }, { "id": "ev-lerobot-topreward-docs-weights", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/topreward.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "No fine-tuned weights to host; LeRobot wraps Qwen3VLForConditionalGeneration.", "quote": "Because the method only depends on a frozen VLM, TOPReward is **zero-shot**: there are no fine-tuned weights to host." }, { "id": "ev-lerobot-topreward-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/rewards/topreward/configuration_topreward.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Registered as reward-model type `topreward`.", "quote": "@RewardModelConfig.register_subclass(\"topreward\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "TOPReward was added in LeRobot v0.6.0.", "quote": "* feat(rewards): add TOPReward reward model by @s1lent4gnt in https://github.com/huggingface/lerobot/pull/3629" }, { "id": "ev-topreward-upstream-repo", "tier": "community-reported", "type": "source-code", "url": "https://github.com/TOPReward/TOPReward", "verified_on": "2026-10-24", "summary": "GitHub API reports the upstream TOPReward/TOPReward repo license as MIT." } ] }, { "id": "vla-jepa", "name": "VLA-JEPA", "aliases": [ "vla_jepa", "VLAJEPA", "ginwind/VLA-JEPA" ], "family": "vla", "summary": "Vision-language-action model that pairs a Qwen3-VL-2B backbone with a flow-matching DiT action head, and adds a V-JEPA2 latent video world model as an auxiliary self-supervised loss during training only. At inference only Qwen and the action head run, predicting 7-step action chunks.", "authors": "Jingwen Sun et al. (University of Science and Technology of China and collaborators)", "paper_url": "https://arxiv.org/abs/2602.10098", "backbone": "Qwen3-VL-2B-Instruct + DiT-B flow-matching action head (V-JEPA2 world model, training only)", "action_output": "flow-matching", "action_chunking": true, "arch_source_ref": "ev-lerobot-vlajepa-docs", "license": "Apache-2.0", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.6.0", "policy_type": "vla_jepa", "docs_url": "https://huggingface.co/docs/lerobot/vla_jepa", "note": "Added in v0.6.0 (\"feat(policies): add VLA-JEPA\"). Default chunk_size 7, 4 Euler denoising steps.", "source_ref": "ev-lerobot-vlajepa-config" } }, "checkpoints": [ { "hub_id": "lerobot/VLA-JEPA-Pretrain", "note": "Converted from ginwind/VLA-JEPA; DROID 1.0.1, 2 exterior views, 7-dim actions." }, { "hub_id": "lerobot/VLA-JEPA-LIBERO", "note": "LIBERO-10; 2 cameras (agentview + wrist), 7-dim actions." }, { "hub_id": "lerobot/VLA-JEPA-SimplerEnv", "note": "OXE Bridge / RT-1; 1 view duplicated x2, 7-dim actions." }, { "hub_id": "ginwind/VLA-JEPA", "note": "Original upstream release (Apache-2.0)." } ], "observation_space": { "state": "proprioceptive state via the action head's state encoder", "language": "required", "source_ref": "ev-lerobot-vlajepa-docs" }, "notes": "Family vla rather than world-model: the LeRobot docs call it a Vision-Language-Action model, and the V-JEPA2 world model is a training-time auxiliary loss that is not used at inference. Released checkpoints use 1–2 camera views; datasets with more views pass all views to Qwen but only the first `jepa_tubelet_size` (default 2) to the world model, so no hard camera max is stated. Params not stated as a total by the authors; not in the LeRobot compute hardware guide (requirements unset); not named by the RTC or PEFT docs. The LeRobot docs' LIBERO results table at v0.6.1 is internally inconsistent (e.g. libero_10 \"96\" successes of 100 shown as 93.0%).", "verified_on": "2026-10-24", "sources": [ { "id": "ev-lerobot-vlajepa-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/vla_jepa.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot VLA-JEPA docs: Qwen3-VL backbone, V-JEPA2 video world model, flow-matching DiT action head; world model unused at inference.", "quote": "a Vision-Language-Action model that combines a Qwen3-VL language backbone with a self-supervised video world model (V-JEPA2) and a flow-matching DiT action head." }, { "id": "ev-lerobot-vlajepa-docs-inference", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/vla_jepa.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Only the Qwen backbone and action head run at inference.", "quote": "Only Qwen + the action head are used. The world model is not needed at inference time." }, { "id": "ev-lerobot-vlajepa-docs-license", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/vla_jepa.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Weights follow ginwind/VLA-JEPA (Apache 2.0); integration code Apache 2.0.", "quote": "The LeRobot integration code follows the **Apache 2.0 License**." }, { "id": "ev-lerobot-vlajepa-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/vla_jepa/configuration_vla_jepa.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `vla_jepa`; default chunk_size 7.", "quote": "@PreTrainedConfig.register_subclass(\"vla_jepa\")" }, { "id": "ev-lerobot-v060-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.6.0", "date": "2026-07-06", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.0" }, "summary": "VLA-JEPA was added in LeRobot v0.6.0.", "quote": "* feat(policies): add VLA-JEPA by @Maximellerbach in https://github.com/huggingface/lerobot/pull/3568" }, { "id": "ev-hub-vlajepa-checkpoints", "tier": "community-reported", "type": "hub-checkpoint", "url": "https://huggingface.co/lerobot/VLA-JEPA-Pretrain", "verified_on": "2026-10-24", "summary": "Hub API: lerobot/VLA-JEPA-{Pretrain,LIBERO,SimplerEnv} and ginwind/VLA-JEPA exist, all tagged license:apache-2.0." } ] }, { "id": "vqbet", "name": "VQ-BeT", "aliases": [ "Vector-Quantized Behavior Transformer" ], "family": "bc", "summary": "Vector-Quantized Behavior Transformer: a GPT-style behavior transformer that tokenizes continuous action chunks with a hierarchical (residual) vector-quantization module, predicts the discrete codes plus a continuous offset, and so captures multimodal action distributions.", "authors": "New York University and Seoul National University (Lee, Wang, Etukuru, Kim, Shafiullah, Pinto)", "paper_url": "https://arxiv.org/abs/2403.03181", "backbone": "ResNet-18 image encoder (ImageNet-pretrained weights by default in LeRobot v0.6.1) + GPT transformer over residual-VQ action tokens", "action_output": "vq-tokens", "action_chunking": true, "arch_source_ref": "ev-vqbet-paper-arch", "license": "Apache-2.0 (LeRobot implementation)", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "policy_type": "vqbet", "docs_url": "https://huggingface.co/docs/lerobot/policy_vqbet_README", "note": "Present at v0.3.2, the earliest tagged LeRobot release. `since` left blank: the only earlier PyPI release (0.1.0, 2024-03-09, a prototype) was not inspected. Training first fits the VQ-VAE (n_vqvae_training_steps 20000 by default), then the transformer. Defaults: n_obs_steps 5, action_chunk_size 5.", "source_ref": "ev-lerobot-vqbet-config" } }, "checkpoints": [ { "hub_id": "lerobot/vqbet_pusht", "note": "PushT simulation checkpoint (Apache-2.0 on the Hub)." } ], "requirements": { "train": { "vram_gb": { "min": 2, "max": 6 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Light BC\" group (act, vqbet, tdmpc) in the LeRobot compute hardware guide. The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" } }, "observation_space": { "cameras": { "min": 1, "max": 1 }, "state": "robot state vector", "source_ref": "ev-lerobot-vqbet-config-features" }, "verified_on": "2026-10-24", "sources": [ { "id": "ev-vqbet-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2403.03181", "date": "2024-03-05", "verified_on": "2026-10-24", "summary": "VQ-BeT tokenizes continuous actions with a hierarchical vector quantization module.", "quote": "VQ-BeT augments BeT by tokenizing continuous actions with a hierarchical vector quantization module." }, { "id": "ev-lerobot-vqbet-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/vqbet/configuration_vqbet.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `vqbet` (same registration at tag v0.3.2).", "quote": "@PreTrainedConfig.register_subclass(\"vqbet\")" }, { "id": "ev-lerobot-vqbet-config-features", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/vqbet/configuration_vqbet.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Feature validation requires exactly one image input.", "quote": "raise ValueError(\"You must provide only one image among the inputs.\")" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Light BC (`act`, `vqbet`, `tdmpc`) peak VRAM ~2–6GB at BS 8 with AdamW; figures are indicative.", "quote": "| Light BC | `act`, `vqbet`, `tdmpc` | ~2–6GB | Laptop GPU (RTX 3060), L4, A10G |" } ] }, { "id": "wall-oss", "name": "WALL-OSS", "aliases": [ "wall_x", "wallx", "Wall-X", "WallX", "wall-oss-flow", "wall-oss-fast", "x-square-robot/wall-oss-flow", "x-square-robot/wall-oss-fast" ], "family": "vla", "summary": "X Square Robot's open embodied foundation model: a Qwen2.5-VL-3B backbone with mixture-of-experts action routing, trained in two stages (discrete FAST action tokens, then flow matching). It ships as two checkpoints, a flow-matching one (wall-oss-flow) and a FAST-token one (wall-oss-fast).", "authors": "X Square Robot", "paper_url": "https://arxiv.org/abs/2509.11766", "backbone": "Qwen2.5-VL-3B with MoE action FFN (flow-matching head or FAST token head)", "action_output": "hybrid", "action_chunking": true, "arch_source_ref": "ev-walloss-paper-arch", "license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.4.3", "policy_type": "wall_x", "docs_url": "https://huggingface.co/docs/lerobot/walloss", "note": "Policy type `wall_x` (directory src/lerobot/policies/wall_x, extra `.[wallx]`); docs page and README title it WALL-OSS. `--policy.prediction_mode` selects `diffusion` (flow matching, default) or `fast` (next-token prediction). Default chunk_size 32. Only attn_implementation='eager' is accepted in v0.6.1.", "source_ref": "ev-lerobot-v043-notes" } }, "checkpoints": [ { "hub_id": "x-square-robot/wall-oss-flow", "note": "Flow-matching checkpoint; default `pretrained_name_or_path` in the LeRobot wall_x config. No license tag on the Hub." }, { "hub_id": "x-square-robot/wall-oss-fast", "note": "FAST-token checkpoint." }, { "hub_id": "x-square-robot/wall-oss-flow-0.1" } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Large VLA\" group (`pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x`) in the LeRobot compute hardware guide; \"A100 40 GB+ (24 GB tight at BS 1)\". The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" } }, "observation_space": { "cameras": { "min": 1 }, "state": "proprioception, padded to max_state_dim 20", "source_ref": "ev-lerobot-wallx-config" }, "notes": "Naming, reconciled: WALL-OSS is the model (paper \"Igniting VLMs toward the Embodied Space\", Hub org x-square-robot). Wall-X / WallX is X Square Robot's training and inference codebase for the WALL model series (github.com/X-Square-Robot/wall-x, Apache-2.0), which has no framework record here. LeRobot names the policy type, directory and pip extra after the codebase (`wall_x`, `wallx`) and the docs after the model (WALL-OSS). action_output is `hybrid` because one policy offers two selectable heads: flow matching (default) or FAST discrete tokens. Parameter count: the paper states only the 3B backbone, not a total, so params is left out. Weights license: not stated on the Hub model cards, so it is left out. Wall-OSS-0.5 exists only on LeRobot main (unreleased) and is out of scope.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-walloss-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2509.11766", "date": "2025-09-15", "verified_on": "2026-10-24", "summary": "WALL-OSS uses Qwen2.5-VL-3B as its main backbone.", "quote": "We adopt QwenVL2.5-3B as the main backbone." }, { "id": "ev-walloss-paper-flow", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2509.11766", "date": "2025-09-15", "verified_on": "2026-10-24", "summary": "The Integration stage swaps discrete action prediction for flow-matching continuous actions.", "quote": "we replace discrete action prediction with continuous action modeling via flow matching" }, { "id": "ev-lerobot-walloss-readme", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/policy_walloss_README.md", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot WALL-OSS README lists both action prediction modes.", "quote": "| Action Prediction | Flow Matching (diffusion) or FAST (discrete tokens) |" }, { "id": "ev-lerobot-walloss-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/walloss.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot docs describe WALL-OSS as adapted from the WallX repository; license Apache 2.0.", "quote": "The LeRobot implementation is adapted from their open-source [WallX](https://github.com/X-Square-Robot/wall-x) repository." }, { "id": "ev-lerobot-wallx-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/wall_x/configuration_wall_x.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type `wall_x`; requires at least one visual input feature; max_state_dim 20; chunk_size 32.", "quote": "\"Wall-X policy requires at least one visual input feature. \"" }, { "id": "ev-lerobot-wallx-register", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/wall_x/configuration_wall_x.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `wall_x`.", "quote": "@PreTrainedConfig.register_subclass(\"wall_x\")" }, { "id": "ev-lerobot-v043-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.3", "date": "2026-01-22", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.3" }, "summary": "WALL-OSS (wall_x) was added in LeRobot v0.4.3 (PR", "quote": "feat: support wallx model by @wut19" }, { "id": "ev-wallx-readme", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/X-Square-Robot/wall-x/blob/ce2a1ae31d9ee1028b97ca40e3f66034cf8c0e5f/README.md", "date": "2026-09-11", "verified_on": "2026-10-24", "summary": "Wall-X is the codebase for the WALL model series and lists the WALL-OSS checkpoints.", "quote": "This repository provides the training and inference code for the WALL series" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Large VLA group peak VRAM ~24–40GB at BS 8 with AdamW; figures are indicative.", "quote": "| Large VLA | `pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x` | ~24–40GB | A100 40 GB+ (24 GB tight at BS 1) |" } ] }, { "id": "xvla", "name": "X-VLA", "aliases": [ "xvla", "X-VLA-0.9B", "lerobot/xvla-base", "2toINF/X-VLA" ], "family": "vla", "summary": "Soft-prompted, flow-matching VLA: a Florence-Large vision-language encoder plus a 24-layer transformer action generator, with small sets of learnable per-embodiment \"soft prompt\" embeddings that absorb differences between robots, cameras and datasets. The released X-VLA-0.9B was pretrained on 290K episodes from seven platforms.", "authors": "Institute for AI Industry Research (AIR), Tsinghua University, et al.", "paper_url": "https://arxiv.org/abs/2510.10274", "params": 900000000, "params_note": "Authors state 0.9B (rounded; model is named X-VLA-0.9B).", "params_source_ref": "ev-xvla-paper-params", "backbone": "Florence-Large vision-language encoder + 24-layer transformer (hidden size 1024) with soft prompts", "action_output": "flow-matching", "action_chunking": true, "arch_source_ref": "ev-xvla-paper-arch", "license": "Apache-2.0", "weights_license": "Apache-2.0", "open_weights": true, "frameworks": { "lerobot": { "support": "native", "since": "0.4.3", "policy_type": "xvla", "docs_url": "https://huggingface.co/docs/lerobot/xvla", "note": "Default chunk_size 32 (docs). Docs recommend --policy.dtype=bfloat16 when fine-tuning to avoid OOM.", "source_ref": "ev-lerobot-v043-notes" } }, "checkpoints": [ { "hub_id": "lerobot/xvla-base", "note": "0.9B base model (Apache-2.0 on the Hub)." }, { "hub_id": "lerobot/xvla-libero" }, { "hub_id": "lerobot/xvla-widowx" }, { "hub_id": "lerobot/xvla-folding" }, { "hub_id": "lerobot/xvla-agibot-world" }, { "hub_id": "lerobot/xvla-google-robot" }, { "hub_id": "2toINF/X-VLA-Pt", "note": "Authors' own pretrained checkpoint (upstream repo 2toINF/X-VLA)." } ], "requirements": { "train": { "vram_gb": { "min": 24, "max": 40 }, "batch_size": 8, "condition": "Peak training VRAM at batch size 8 with AdamW (LeRobot default), \"Large VLA\" group (`pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x`) in the LeRobot compute hardware guide; \"A100 40 GB+ (24 GB tight at BS 1)\". The guide calls its figures indicative, order-of-magnitude numbers for picking hardware, not exact predictions. Precision not stated.", "source_ref": "ev-lerobot-hw-guide" } }, "observation_space": { "state": "proprioceptive state (use_proprio, padded to max_state_dim 32)", "language": "optional", "source_ref": "ev-xvla-base-card" }, "notes": "Upstream code: github.com/2toINF/X-VLA (Apache-2.0); no framework record exists for it here, so only the LeRobot integration is listed. Action spaces are handled by an action registry (`--policy.action_mode`, `auto` recommended) and embodiments by domain IDs. Inference memory: no primary figure found.", "verified_on": "2026-10-24", "sources": [ { "id": "ev-xvla-paper-params", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2510.10274", "date": "2025-10-11", "verified_on": "2026-10-24", "summary": "X-VLA paper names its released instantiation X-VLA-0.9B.", "quote": "We implement X-VLA-0.9B , a 0.9B instantiation of X-VLA , trained with a carefully designed data processing and learning recipe." }, { "id": "ev-xvla-paper-arch", "tier": "upstream-documented", "type": "paper", "url": "https://arxiv.org/abs/2510.10274", "date": "2025-10-11", "verified_on": "2026-10-24", "summary": "Architecture is Florence-Large encoder + 24-layer transformer (hidden 1024) for action generation.", "quote": "we adopt Florence-Large ( Xiao et al., 2024 ) as the vision–language encoder and employ a standard Transformer backbone with 24 layers and a hidden size of 1024 for action generation." }, { "id": "ev-lerobot-xvla-docs", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/xvla.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "LeRobot docs describe X-VLA as a soft-prompted flow-matching VLA.", "quote": "**X-VLA** is a soft-prompted, flow-matching VLA framework that treats each hardware setup as a \"task\" and encodes it using a small set of learnable embeddings." }, { "id": "ev-lerobot-xvla-config", "tier": "upstream-documented", "type": "source-code", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/src/lerobot/policies/xvla/configuration_xvla.py", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Policy type string is `xvla`.", "quote": "@PreTrainedConfig.register_subclass(\"xvla\")" }, { "id": "ev-lerobot-v043-notes", "tier": "upstream-documented", "type": "release-notes", "url": "https://github.com/huggingface/lerobot/releases/tag/v0.4.3", "date": "2026-01-22", "verified_on": "2026-10-24", "versions": { "lerobot": "0.4.3" }, "summary": "X-VLA was added in LeRobot v0.4.3 (PR", "quote": "feat(policies): Add X-VLA by @jadechoghari" }, { "id": "ev-xvla-base-card", "tier": "upstream-documented", "type": "model-card", "url": "https://huggingface.co/lerobot/xvla-base", "verified_on": "2026-10-24", "summary": "Model card lists inputs (multi-view images, state, optional language) and flow-matching objective.", "quote": "- **Inputs:** images (multi-view), proprio/state, optional language instruction" }, { "id": "ev-lerobot-hw-guide", "tier": "upstream-documented", "type": "official-docs", "url": "https://github.com/huggingface/lerobot/blob/v0.6.1/docs/source/hardware_guide.mdx", "date": "2026-08-03", "verified_on": "2026-10-24", "versions": { "lerobot": "0.6.1" }, "summary": "Compute hardware guide: Large VLA group peak VRAM ~24–40GB at BS 8 with AdamW; figures are indicative.", "quote": "| Large VLA | `pi0`, `pi0_fast`, `pi05`, `xvla`, `wall_x` | ~24–40GB | A100 40 GB+ (24 GB tight at BS 1) |" } ] } ] }