eric-haibin-lin · Jul 5, 2025 · Jul 5, 2025 · Jul 5, 2025 · Jul 5, 2025
diff --git a/.github/workflows/e2e_ppo_trainer.yml b/.github/workflows/e2e_ppo_trainer.yml
@@ -97,6 +97,11 @@ jobs:
         run: |
           ray stop --force
           python3 examples/data_preprocess/gsm8k.py
+      # HF sanity
+      - name: Running GSM8K E2E training tests on 1 L20 GPU with hf for santiy
+        run: |
+          ray stop --force
+          bash tests/special_e2e/ppo_trainer/run_single_gpu.sh
       # Function RM
       - name: Running GSM8K E2E training tests on 8 L20 GPUs with rmpad using function rm with validation and saving (FSDP_SIZE=8)
         run: |

diff --git a/docs/amd_tutorial/amd_build_dockerfile_page.rst b/docs/amd_tutorial/amd_build_dockerfile_page.rst
@@ -162,7 +162,6 @@ PPO
      trainer.project_name=$YOUR_PROJECT_NAME \
      trainer.experiment_name=$YOUR_RUN_NAME \
      trainer.val_before_train=False \
-     trainer.default_hdfs_dir=null \
      trainer.n_gpus_per_node=$GPUS_PER_NODE \
      trainer.nnodes=1 \
      trainer.save_freq=10 \

@@ -525,7 +525,7 @@ Trainer
      val_before_train: True
      test_freq: 2
      critic_warmup: 0
-     default_hdfs_dir: ~/experiments/gsm8k/ppo/${trainer.experiment_name} # hdfs checkpoint path
+     default_hdfs_dir: null # hdfs checkpoint path
      default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name} # local checkpoint path
      resume_mode: auto # or disable or resume_path if resume_from_path is set
      resume_from_path: null

@@ -92,7 +92,6 @@ We also provide various training scripts for SFT on GSM8K dataset in `gsm8k sft
        data.response_key=answer \
        data.micro_batch_size_per_gpu=8 \
        model.partial_pretrain=deepseek-ai/deepseek-coder-6.7b-instruct \
-       trainer.default_hdfs_dir=hdfs://user/verl/experiments/gsm8k/deepseek-coder-6.7b-instruct/ \
        trainer.project_name=gsm8k-sft \
        trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \
        trainer.total_epochs=4 \

@@ -175,7 +175,6 @@ Now you can submit the training job to the Ray cluster which is available at ``l
         trainer.project_name=ppo_training \
         trainer.experiment_name=qwen-2.5-7B \
         trainer.val_before_train=False \
-        trainer.default_hdfs_dir=null \
         trainer.n_gpus_per_node=8 \
         trainer.nnodes=2 \
         trainer.default_local_dir=/checkpoints \

@@ -102,7 +102,6 @@ You may set ``VERL_USE_MODELSCOPE=True`` to download models from `modelscope <ht
     algorithm.kl_ctrl.kl_coef=0.001 \
     trainer.logger=['console'] \
     trainer.val_before_train=False \
-    trainer.default_hdfs_dir=null \
     trainer.n_gpus_per_node=1 \
     trainer.nnodes=1 \
     trainer.save_freq=10 \
@@ -116,7 +115,7 @@ You are expected to see the following logs, indicating training in progress. The
     step:0 - timing/gen:21.470 - timing/ref:4.360 - timing/values:5.800 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty_coeff:0.001 - timing/adv:0.109 - timing/update_critic:15.664 - critic/vf_loss:14.947 - critic/vf_clipfrac:0.000 - critic/vpred_mean:-2.056 - critic/grad_norm:1023.278 - critic/lr(1e-4):0.100 - timing/update_actor:20.314 - actor/entropy_loss:0.433 - actor/pg_loss:-0.005 - actor/pg_clipfrac:0.000 - actor/ppo_kl:0.000 - actor/grad_norm:1.992 - actor/lr(1e-4):0.010 - critic/score/mean:0.004 - critic/score/max:1.000 - critic/score/min:0.000 - critic/rewards/mean:0.004 - critic/rewards/max:1.000 - critic/rewards/min:0.000 - critic/advantages/mean:-0.000 - critic/advantages/max:2.360 - critic/advantages/min:-2.280 - critic/returns/mean:0.003 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.045 - critic/values/max:9.500 - critic/values/min:-14.000 - response_length/mean:239.133 - response_length/max:256.000 - response_length/min:77.000 - prompt_length/mean:104.883 - prompt_length/max:175.000 - prompt_length/min:68.000
     step:1 - timing/gen:23.020 - timing/ref:4.322 - timing/values:5.953 - actor/reward_kl_penalty:0.000 - actor/reward_kl_penalty:0.001 - timing/adv:0.118 - timing/update_critic:15.646 - critic/vf_loss:18.472 - critic/vf_clipfrac:0.384 - critic/vpred_mean:1.038 - critic/grad_norm:942.924 - critic/lr(1e-4):0.100 - timing/update_actor:20.526 - actor/entropy_loss:0.440 - actor/pg_loss:0.000 - actor/pg_clipfrac:0.002 - actor/ppo_kl:0.000 - actor/grad_norm:2.060 - actor/lr(1e-4):0.010 - critic/score/mean:0.000 - critic/score/max:0.000 - critic/score/min:0.000 - critic/rewards/mean:0.000 - critic/rewards/max:0.000 - critic/rewards/min:0.000 - critic/advantages/mean:0.000 - critic/advantages/max:2.702 - critic/advantages/min:-2.616 - critic/returns/mean:0.000 - critic/returns/max:0.000 - critic/returns/min:0.000 - critic/values/mean:-2.280 - critic/values/max:11.000 - critic/values/min:-16.000 - response_length/mean:232.242 - response_length/max:256.000 - response_length/min:91.000 - prompt_length/mean:102.398 - prompt_length/max:185.000 - prompt_length/min:70.000
 
-Checkout :ref:`algo-baseline-page` for full training and validation logs for reference.
+Checkout ``Algorithm Baselines`` page for full training and validation logs for reference.
 
 The checkpoint is saved at the following dir by default: ``checkpoints/${trainer.project_name}/${trainer.experiment_name}``. You can merge the saved checkpoints to huggingface model using ``verl.model_merger`` module, for example:
 

@@ -78,7 +78,6 @@ We use Qwen/Qwen2-7B-Instruct on the gsm8k dataset for a simple test.
         algorithm.kl_ctrl.kl_coef=0.001 \
         trainer.logger=['console'] \
         trainer.val_before_train=False \
-        trainer.default_hdfs_dir=null \
         trainer.n_gpus_per_node=4 \
         trainer.nnodes=1 \
         trainer.save_freq=-1 \
@@ -231,7 +230,6 @@ You can see that the cluster has two nodes with 16 GPUs:
         trainer.critic_warmup=0 \
         trainer.logger=['console'] \
         trainer.val_before_train=True \
-        trainer.default_hdfs_dir=null \
         trainer.n_gpus_per_node=8 \
         trainer.nnodes=2 \
         trainer.save_freq=-1 \

@@ -99,7 +99,6 @@ python3 -m verl.trainer.main_ppo --config-path=./config --config-name='ppo_megat
     actor_rollout_ref.ref.megatron.use_dist_checkpointing=True \
     actor_rollout_ref.actor.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \
     actor_rollout_ref.ref.megatron.dist_checkpointing_path=$DIST_CKPT_PATH \
-    trainer.default_hdfs_dir=null \
     trainer.default_local_dir=$CKPT_DIR \
     trainer.val_before_train=False \
     trainer.total_epochs=100 $@
diff --git a/examples/sft/gsm8k/run_deepseek_6b7.sh b/examples/sft/gsm8k/run_deepseek_6b7.sh
@@ -25,5 +25,4 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-deepseek-coder-6.7b-instruct \
     trainer.total_epochs=4 \
-    trainer.logger=['console','wandb'] \
-    trainer.default_hdfs_dir=null $@
+    trainer.logger=['console','wandb'] $@
diff --git a/examples/sft/gsm8k/run_gemma_2b.sh b/examples/sft/gsm8k/run_gemma_2b.sh
@@ -27,5 +27,4 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-gemma-2b-it \
     trainer.total_epochs=2 \
-    trainer.logger=['console','wandb'] \
-    trainer.default_hdfs_dir=null $@
+    trainer.logger=['console','wandb'] $@
diff --git a/examples/sft/gsm8k/run_gemma_7b.sh b/examples/sft/gsm8k/run_gemma_7b.sh
@@ -23,5 +23,4 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-gemma-1.1-7b-it \
     trainer.total_epochs=4 \
-    trainer.logger=['console','wandb'] \
-    trainer.default_hdfs_dir=null $@
+    trainer.logger=['console','wandb'] $@
diff --git a/examples/sft/gsm8k/run_qwen2_5_05b_sft_peft_sp2_npu.sh b/examples/sft/gsm8k/run_qwen2_5_05b_sft_peft_sp2_npu.sh
@@ -26,8 +26,7 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \
     trainer.logger=['console'] \
-    trainer.total_epochs=2 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_epochs=2 $@ \
     model.lora_rank=32 \
     model.lora_alpha=16 \
     model.target_modules=all-linear \

diff --git a/examples/sft/gsm8k/run_qwen_05_peft.sh b/examples/sft/gsm8k/run_qwen_05_peft.sh
@@ -28,8 +28,7 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \
     trainer.logger=['console'] \
-    trainer.total_epochs=1 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_epochs=1 $@ \
     model.lora_rank=32\
     model.lora_alpha=16 \
     model.target_modules=all-linear

diff --git a/examples/sft/gsm8k/run_qwen_05_sp2.sh b/examples/sft/gsm8k/run_qwen_05_sp2.sh
@@ -26,7 +26,6 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2 \
     trainer.logger=['console'] \
-    trainer.total_training_steps=1 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_training_steps=1 $@ \
     ulysses_sequence_parallel_size=2 \
     use_remove_padding=true
diff --git a/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh b/examples/sft/gsm8k/run_qwen_05_sp2_liger.sh
@@ -26,7 +26,6 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.default_local_dir=$save_path \
     trainer.project_name=gsm8k-sft \
     trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct-sp2-liger \
-    trainer.logger=['console'] \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.logger=['console'] $@ \
     ulysses_sequence_parallel_size=2 \
     use_remove_padding=true
diff --git a/examples/sft/multiturn/run_qwen_05_sp2.sh b/examples/sft/multiturn/run_qwen_05_sp2.sh
@@ -24,7 +24,6 @@ torchrun --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=multiturn-sft \
     trainer.experiment_name=multiturn-sft-qwen-2.5-0.5b-instruct-sp2 \
     trainer.logger=['console'] \
-    trainer.total_training_steps=1 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_training_steps=1 $@ \
     ulysses_sequence_parallel_size=2 \
     use_remove_padding=true
diff --git a/examples/slurm/ray_on_slurm.slurm b/examples/slurm/ray_on_slurm.slurm
@@ -91,7 +91,6 @@ PYTHONUNBUFFERED=1 srun --overlap --nodes=1 --ntasks=1 -w "$head_node" \
     algorithm.use_kl_in_reward=False \
     trainer.logger=['console'] \
     trainer.val_before_train=False \
-    trainer.default_hdfs_dir=null \
     trainer.n_gpus_per_node="${SLURM_GPUS_PER_NODE}" \
     trainer.nnodes="${SLURM_NNODES}" \
     trainer.save_freq=10 \

diff --git a/recipe/char_count/train_sft.sh b/recipe/char_count/train_sft.sh
@@ -18,5 +18,4 @@ torchrun --standalone --nnodes=1 --nproc_per_node=$nproc_per_node \
     trainer.project_name=char_count-sft \
     trainer.experiment_name=char_count-sft-SmolLM2-135M-Instruct \
     trainer.total_epochs=3 \
-    trainer.logger=['console'] \
-    trainer.default_hdfs_dir=null
+    trainer.logger=['console']
diff --git a/recipe/retool/run_qwen2-32b_sft.sh b/recipe/retool/run_qwen2-32b_sft.sh
@@ -55,6 +55,5 @@ torchrun --nnodes=$ARNOLD_WORKER_NUM \
     trainer.experiment_name=$experiment_name \
     trainer.logger=['console','wandb'] \
     trainer.total_epochs=6 \
-    trainer.default_hdfs_dir=null \
     ulysses_sequence_parallel_size=4 \
     use_remove_padding=true
diff --git a/recipe/retool/run_qwen2.5_32b_sp8.sh b/recipe/retool/run_qwen2.5_32b_sp8.sh
@@ -28,7 +28,6 @@ torchrun --nnodes=1 --nproc_per_node=8 \
     trainer.project_name=retool-multiturn-sft \
     trainer.experiment_name=$EXPERIMENT_NAME \
     trainer.logger=['console','wandb'] \
-    trainer.total_epochs=12 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_epochs=12 $@ \
     ulysses_sequence_parallel_size=8 \
     use_remove_padding=true
diff --git a/recipe/retool/run_qwen2.5_7b_sp4.sh b/recipe/retool/run_qwen2.5_7b_sp4.sh
@@ -28,7 +28,6 @@ torchrun --nnodes=1 --nproc_per_node=8 \
     trainer.project_name=retool-multiturn-sft \
     trainer.experiment_name=$EXPERIMENT_NAME \
     trainer.logger=['console','wandb'] \
-    trainer.total_epochs=8 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_epochs=8 $@ \
     ulysses_sequence_parallel_size=4 \
     use_remove_padding=true
diff --git a/recipe/retool/run_qwen3_4b_sp4.sh b/recipe/retool/run_qwen3_4b_sp4.sh
@@ -26,7 +26,6 @@ torchrun --nnodes=1 --nproc_per_node=8 \
     trainer.project_name=retool-multiturn-sft \
     trainer.experiment_name=$EXPERIMENT_NAME \
     trainer.logger=['console','wandb'] \
-    trainer.total_epochs=12 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_epochs=12 $@ \
     ulysses_sequence_parallel_size=4 \
     use_remove_padding=true
diff --git a/recipe/spin/run_spin.sh b/recipe/spin/run_spin.sh
@@ -20,7 +20,6 @@ CUDA_VISIBLE_DEVICES=${VISIBLE_DEVICES} python3 -m recipe.spin.main_spin \
   algorithm.kl_ctrl.kl_coef=0.001 \
   trainer.logger=['console'] \
   trainer.val_before_train=True \
-  trainer.default_hdfs_dir=null \
   trainer.n_gpus_per_node=4 \
   trainer.nnodes=1 \
   trainer.save_freq=-1 \

diff --git a/tests/special_e2e/ppo_trainer/run_single_gpu.sh b/tests/special_e2e/ppo_trainer/run_single_gpu.sh
@@ -0,0 +1,24 @@
+PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \
+  data.train_files=$HOME/data/gsm8k/train.parquet \
+  data.val_files=$HOME/data/gsm8k/test.parquet \
+  data.train_batch_size=256  \
+  data.max_prompt_length=512 \
+  data.max_response_length=256  \
+  actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \
+  actor_rollout_ref.actor.optim.lr=1e-6 \
+  actor_rollout_ref.actor.ppo_mini_batch_size=64 \
+  actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4  \
+  actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \
+  actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
+  actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \
+  actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \
+  critic.optim.lr=1e-5 \
+  critic.model.path=Qwen/Qwen2.5-0.5B-Instruct \
+  critic.ppo_micro_batch_size_per_gpu=4 \
+  algorithm.kl_ctrl.kl_coef=0.001 \
+  trainer.logger=['console'] \
+  trainer.val_before_train=False \
+  trainer.n_gpus_per_node=1 \
+  trainer.nnodes=1 \
+  actor_rollout_ref.rollout.name=hf \
+  trainer.total_training_steps=2
diff --git a/tests/special_e2e/run_spin.sh b/tests/special_e2e/run_spin.sh
@@ -21,7 +21,6 @@ CUDA_VISIBLE_DEVICES=${VISIBLE_DEVICES} python3 -m recipe.spin.main_spin \
   algorithm.kl_ctrl.kl_coef=0.001 \
   trainer.logger=['console'] \
   trainer.val_before_train=False \
-  trainer.default_hdfs_dir=null \
   trainer.n_gpus_per_node=4 \
   trainer.nnodes=1 \
   trainer.save_freq=-1 \

diff --git a/tests/special_e2e/sft/run_sft.sh b/tests/special_e2e/sft/run_sft.sh
@@ -50,7 +50,6 @@ torchrun --standalone --nnodes=1 --nproc_per_node=${NUM_GPUS} ${ENTRYPOINT} \
     trainer.project_name="${project_name}" \
     trainer.experiment_name="${exp_name}" \
     trainer.total_training_steps=1 \
-    trainer.logger=['console'] \
-    trainer.default_hdfs_dir=null $@
+    trainer.logger=['console'] $@
 
 rm -rf "${ckpts_home:?}/*"
diff --git a/tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh b/tests/special_npu/run_qwen2_5_05b_sft_peft_sp2.sh
@@ -18,8 +18,7 @@ torchrun --standalone --nnodes=1 --nproc_per_node=8 \
     trainer.experiment_name=gsm8k-sft-qwen-2.5-0.5b-instruct \
     trainer.logger=['console'] \
     trainer.total_epochs=1 \
-    trainer.total_training_steps=1 \
-    trainer.default_hdfs_dir=null $@ \
+    trainer.total_training_steps=1 $@ \
     model.lora_rank=32 \
     model.lora_alpha=16 \
     model.target_modules=all-linear \

diff --git a/verl/workers/sharding_manager/base.py b/verl/workers/sharding_manager/base.py
@@ -19,6 +19,9 @@
 
 
 class BaseShardingManager:
+    def __init__(self):
+        self.timing = {}
+
     def __enter__(self):
         pass