fix(gepa): collect and save trajectory feedback in optimization results (#819)

TimeToBuildBob · web-flow · commit 603ccfefc2bf · 2025-11-05T18:27:18.000+01:00
Fixes Issue ErikBjare/bob#128 Problem: - TrajectoryAnalyzer and trajectory_feedback_metric existed - GEPA received and used trajectory metric internally - But _evaluate_prompt didn't collect trajectory feedback - Results saved without trajectory_feedback field Root cause: - DSPy GEPA uses trajectory metric during optimization but doesn't expose feedback - _evaluate_prompt only calculated scores, no trajectory feedback collection Solution: - Store trajectory_metric in PromptOptimizer when creating GEPA optimizer - In _evaluate_prompt: collect trajectory feedback for each validation example - Add trajectory_feedback to results dict with score + feedback per example This enables: - Phase 3.2 Tests 2-4 validation (previously blocked) - Phase 3.3-4 performance validation - Trajectory-based prompt optimization Testing: - All 17 dspy tests pass - 1 test fails due to missing API key (expected, not code issue)
diff --git a/gptme/eval/dspy/prompt_optimizer.py b/gptme/eval/dspy/prompt_optimizer.py
@@ -268,6 +268,7 @@ def _create_optimizer(self, eval_specs: list[EvalSpec]):
             )
         elif self.optimizer_type.lower() == "gepa":
             trajectory_metric = create_trajectory_feedback_metric(eval_specs=eval_specs)
+            self._trajectory_metric = trajectory_metric  # Store for evaluation
             reflection_model = ModelNameMapper.get_reflection_model(self.model)
             reflection_lm = dspy.LM(reflection_model)
 
@@ -314,6 +315,7 @@ def _evaluate_prompt(self, prompt: str, val_data: PromptDataset) -> dict[str, An
         task_scores = []
         tool_scores = []
         judge_scores = []
+        trajectory_feedbacks = []
         module = GptmeModule(prompt, self.model)
 
         for example in val_data:
@@ -328,6 +330,18 @@ def _evaluate_prompt(self, prompt: str, val_data: PromptDataset) -> dict[str, An
             tool_scores.append(tool_metric(example, pred, None))
             judge_scores.append(judge_metric(example, pred, None))
 
+            # If trajectory metric exists (GEPA), collect feedback
+            if hasattr(self, "_trajectory_metric"):
+                trajectory_result = self._trajectory_metric(
+                    example, pred, None, None, None
+                )
+                trajectory_feedbacks.append(
+                    {
+                        "score": trajectory_result.score,
+                        "feedback": trajectory_result.feedback,
+                    }
+                )
+
         # Calculate averages
         avg_task = sum(task_scores) / len(task_scores) if task_scores else 0.0
         avg_tool = sum(tool_scores) / len(tool_scores) if tool_scores else 0.0
@@ -340,7 +354,7 @@ def _evaluate_prompt(self, prompt: str, val_data: PromptDataset) -> dict[str, An
             for t, tool, j in zip(task_scores, tool_scores, judge_scores)
         ]
 
-        return {
+        results = {
             "average_score": avg_composite,
             "task_success_rate": avg_task,
             "tool_usage_score": avg_tool,
@@ -353,6 +367,12 @@ def _evaluate_prompt(self, prompt: str, val_data: PromptDataset) -> dict[str, An
             "optimized_prompt": prompt,
         }
 
+        # Add trajectory feedback if available (GEPA only)
+        if trajectory_feedbacks:
+            results["trajectory_feedback"] = trajectory_feedbacks
+
+        return results
+
     def compare_prompts(
         self,
         prompts: dict[str, str],
