# 入口（Hydra 配置加载）
@hydra.main(config_path="config", config_name="ppo_trainer")   # main_ppo.py:29
def main(config): run_ppo(config)

# run_ppo() 初始化 Ray 集群，然后启动 TaskRunner  # main_ppo.py:34-51
runner = TaskRunner.remote()
ray.get(runner.run.remote(config))

# TaskRunner.run() 完成以下初始化步骤              # main_ppo.py:56-188
# 1. 下载模型检查点
local_path = copy_to_local(config.actor_rollout_ref.model.path)

# 2. 初始化环境（含技能记忆模块）
envs, val_envs = make_envs(config)                # main_ppo.py:71

# 3. 初始化分词器与处理器
tokenizer = hf_tokenizer(local_path)              # main_ppo.py:77
processor = hf_processor(local_path)              # main_ppo.py:78

# 4. 创建多轮轨迹收集器
traj_collector = TrajectoryCollector(config, tokenizer, processor)  # main_ppo.py:162

# 5. 创建训练/验证数据集
train_dataset = create_rl_dataset(...)            # main_ppo.py:166
val_dataset   = create_rl_dataset(...)            # main_ppo.py:167

# 6. 初始化 PPO 训练器并启动
trainer = RayPPOTrainer(...)                      # main_ppo.py:169-186
trainer.init_workers()                            # main_ppo.py:187
trainer.fit()                                     # main_ppo.py:188

class TrajectoryCollector:                        # rollout_loop.py:29
    def multi_turn_loop(
        self, gen_batch, actor_rollout_wg, envs, is_train
    ) -> DataProto:

class EpisodeRewardManager:                       # episode.py:20
    def __call__(self, data: DataProto):
        # 取 episode 最终奖励（0/1 成功标志），
        # 放到最后一个有效 response token 的位置
        reward_tensor[i, valid_response_length - 1] = episode_rewards[i]  # episode.py:79

# 无效动作惩罚（可选）
apply_invalid_action_penalty(data, coef)          # ray_trainer.py:200
    reward_tensor[i, valid_response_length-1] -= coef * action_invalids

# KL 惩罚（可选，use_kl_in_reward=True 时）
apply_kl_penalty(data, kl_ctrl)                   # ray_trainer.py:152
    kld = kl_penalty(old_log_probs, ref_log_prob)
    token_level_rewards = token_level_scores - beta * kld

def compute_advantage(data, adv_estimator, gamma, lam, ...):  # ray_trainer.py:244

# Critic 更新（仅 GAE 估计器，需要 value 网络）       # ray_trainer.py:1452-1457
if self.use_critic:
    critic_output = self.critic_wg.update_critic(batch)

# Actor 更新（PPO clip 损失）                         # ray_trainer.py:1459-1466
if self.config.trainer.critic_warmup <= self.global_steps:
    actor_output = self.actor_rollout_wg.update_actor(batch)
# 输入: input_ids, attention_mask, old_log_probs, advantages, returns
# 输出: actor_loss, entropy, pg_clipfrac, ...

# 检查点保存                                          # ray_trainer.py:1125-1156
_save_checkpoint()
# 保存 actor weights + critic weights + dataloader 状态

class SkillsOnlyMemory(BaseMemory):               # skills_only_memory.py:36
    def __init__(self, skills_json_path, retrieval_mode="template",
                 embedding_model_path=None, task_specific_top_k=None):
        # 加载技能库 JSON                          # skills_only_memory.py:86-88
        with open(skills_json_path) as f:
            self.skills = json.load(f)
        # embedding 模式下预计算技能向量            # skills_only_memory.py:108-109
        if retrieval_mode == "embedding":
            self._compute_skill_embeddings()

# env_manager.py:90-98 (SearchEnvironmentManager.reset 为例)
for task in self.tasks:
    memories = self.retrieval_memory.retrieve(
        task_description=task, top_k=top_k, ...)
    self.retrieved_memories.append(memories)
# retrieved_memories 之后在 build_text_obs() 中拼入 prompt

def _update_skills_from_validation(
    self, sample_inputs, sample_outputs, sample_scores, success_rate
):                                                # ray_trainer.py:837

    # 1. 检测低成功率任务类型
    for task_key, rate in success_rate.items():
        if rate < threshold:
            low_success_tasks.append(task_type)  # ray_trainer.py:855-860

    # 2. 收集失败轨迹
    failed_trajectories = _collect_failed_trajectories(...)  # ray_trainer.py:869

    # 3. 调用 o3 API 分析失败并生成新技能
    new_skills = skill_updater.analyze_failures(
        failed_trajectories=failed_trajectories,
        current_skills=retrieval_memory.skills,
        evolution_variant=evolution_variant,     # v0/v2/v3/v4
        frozen_layers=frozen_layers,             # ray_trainer.py:898-903
    )

    # 4. 注入新技能到训练环境（不回流验证环境，防止数据泄漏）
    self.envs.retrieval_memory.add_skills(new_skills, category='general')  # ray_trainer.py:913

    # 5. 保存更新后的技能库 JSON
    train_memory.save_skills(save_path)          # ray_trainer.py:927

class EnvironmentManagerBase:
    def reset(self, kwargs) -> (obs, infos)     # 重置并返回初始观测（含技能上下文）
    def step(self, text_actions) -> (obs, rewards, dones, infos)
    def build_text_obs() -> List[str]           # 将 obs + 检索技能拼装成 LLM prompt