Merge pull request #787 from UiPath/akshaya/runtime_details

feat(ReturnAgentExecution): return agent execution details as part of evaluation output

Author: akshaylive

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.112"
+version = "2.1.113"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

samples/calculator/evals/eval-sets/legacy.json

@@ -21,7 +21,7 @@
       "expectedOutput": {
         "result": 2
       },
-      "expectedAgentBehavior": "",
+      "expectedAgentBehavior": "The operation should produce the right output.",
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",
       "updatedAt": "2025-09-04T18:55:55.416Z"
@@ -37,7 +37,7 @@
       "expectedOutput": {
         "result": 2
       },
-      "expectedAgentBehavior": "",
+      "expectedAgentBehavior": "The operation should produce the right output.",
       "mockingStrategy": {
         "type": "mockito",
         "behaviors": [
@@ -73,7 +73,7 @@
       "expectedOutput": {
         "result": 2
       },
-      "expectedAgentBehavior": "",
+      "expectedAgentBehavior": "The operation should produce the right output.",
       "mockingStrategy": {
         "type": "llm",
         "prompt": "The random operator is '+'.",
@@ -98,7 +98,7 @@
       "expectedOutput": {
         "result": 35
       },
-      "expectedAgentBehavior": "",
+      "expectedAgentBehavior": "The operation should produce the right output.",
       "inputMockingStrategy": {
         "prompt": "Generate a multiplication calculation where the first number is 5 and the second number is 7",
         "model": {

samples/calculator/evals/evaluators/legacy-trajectory.json

@@ -6,7 +6,7 @@
   "category": 3,
   "type": 7,
   "prompt": "Evaluate the agent's execution trajectory based on the expected behavior.\n\nExpected Agent Behavior: {{ExpectedAgentBehavior}}\nAgent Run History: {{AgentRunHistory}}\n\nProvide a score from 0-100 based on how well the agent followed the expected trajectory.",
-  "model": "gpt-4o-mini",
+  "model": "gpt-4.1-2025-04-14",
   "targetOutputKey": "*",
   "createdAt": "2025-06-26T17:45:39.651Z",
   "updatedAt": "2025-06-26T17:45:39.651Z"

src/uipath/_cli/_evals/_models/_output.py

@@ -8,7 +8,11 @@
 from pydantic_core import core_schema
 
 from uipath._cli._runtime._contracts import UiPathRuntimeResult
-from uipath.eval.models.models import EvaluationResult, ScoreType
+from uipath.eval.models.models import (
+    EvaluationResult,
+    ScoreType,
+    TrajectoryEvaluationTrace,
+)
 
 
 class UiPathEvalRunExecutionOutput(BaseModel):
@@ -22,6 +26,22 @@ class UiPathEvalRunExecutionOutput(BaseModel):
     result: UiPathRuntimeResult
 
 
+class UiPathSerializableEvalRunExecutionOutput(BaseModel):
+    execution_time: float
+    trace: TrajectoryEvaluationTrace
+    result: UiPathRuntimeResult
+
+
+def convert_eval_execution_output_to_serializable(
+    output: UiPathEvalRunExecutionOutput,
+) -> UiPathSerializableEvalRunExecutionOutput:
+    return UiPathSerializableEvalRunExecutionOutput(
+        execution_time=output.execution_time,
+        result=output.result,
+        trace=TrajectoryEvaluationTrace.from_readable_spans(output.spans),
+    )
+
+
 class EvaluationResultDto(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
@@ -67,19 +87,13 @@ class EvaluationRunResultDto(BaseModel):
     evaluator_id: str
     result: EvaluationResultDto
 
-    @model_serializer(mode="wrap")
-    def serialize_model(self, serializer, info):
-        data = serializer(self)
-        if isinstance(data, dict):
-            data.pop("evaluatorId", None)
-        return data
-
 
 class EvaluationRunResult(BaseModel):
     model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
 
     evaluation_name: str
     evaluation_run_results: List[EvaluationRunResultDto]
+    agent_execution_output: Optional[UiPathSerializableEvalRunExecutionOutput] = None
 
     @property
     def score(self) -> float:

src/uipath/_cli/_evals/_runtime.py

@@ -38,6 +38,7 @@
 )
 from .._runtime._logging import ExecutionLogHandler
 from .._utils._eval_set import EvalHelpers
+from ..models.runtime_schema import Entrypoint
 from ._evaluator_factory import EvaluatorFactory
 from ._models._evaluation_set import (
     AnyEvaluationItem,
@@ -53,6 +54,7 @@
     EvaluationRunResultDto,
     UiPathEvalOutput,
     UiPathEvalRunExecutionOutput,
+    convert_eval_execution_output_to_serializable,
 )
 from ._span_collection import ExecutionSpanCollector
 from .mocks.mocks import (
@@ -147,6 +149,7 @@ class UiPathEvalContext(UiPathRuntimeContext):
     workers: Optional[int] = 1
     eval_set: Optional[str] = None
     eval_ids: Optional[List[str]] = None
+    verbose: bool = False
 
 
 class UiPathEvalRuntime(UiPathBaseRuntime, Generic[T, C]):
@@ -173,6 +176,15 @@ def __init__(
 
         self.logs_exporter: ExecutionLogsExporter = ExecutionLogsExporter()
         self.execution_id = str(uuid.uuid4())
+        self.entrypoint: Optional[Entrypoint] = None
+
+    async def get_entrypoint(self):
+        if not self.entrypoint:
+            temp_runtime = self.factory.new_runtime(
+                entrypoint=self.context.entrypoint, runtime_dir=os.getcwd()
+            )
+            self.entrypoint = await temp_runtime.get_entrypoint()
+        return self.entrypoint
 
     @classmethod
     def from_eval_context(
@@ -187,13 +199,6 @@ async def execute(self) -> UiPathRuntimeResult:
         if self.context.eval_set is None:
             raise ValueError("eval_set must be provided for evaluation runs")
 
-        # Get entrypoint from a temporary runtime
-        temp_context = self.factory.new_context(
-            entrypoint=self.context.entrypoint, runtime_dir=os.getcwd()
-        )
-        temp_runtime = self.factory.from_context(temp_context)
-        self.entrypoint = await temp_runtime.get_entrypoint()
-
         event_bus = self.event_bus
 
         # Load eval set (path is already resolved in cli_eval.py)
@@ -360,6 +365,12 @@ async def _execute_eval(
 
         try:
             agent_execution_output = await self.execute_runtime(eval_item, execution_id)
+            if self.context.verbose:
+                evaluation_run_results.agent_execution_output = (
+                    convert_eval_execution_output_to_serializable(
+                        agent_execution_output
+                    )
+                )
             evaluation_item_results: list[EvalItemResult] = []
 
             for evaluator in evaluators:
@@ -477,7 +488,9 @@ async def _generate_input_for_eval(
         self, eval_item: AnyEvaluationItem
     ) -> AnyEvaluationItem:
         """Use LLM to generate a mock input for an evaluation item."""
-        generated_input = await generate_llm_input(eval_item, self.entrypoint.input)
+        generated_input = await generate_llm_input(
+            eval_item, (await self.get_entrypoint()).input
+        )
         updated_eval_item = eval_item.model_copy(update={"inputs": generated_input})
         return updated_eval_item
 

tests/cli/eval/test_evaluate.py

@@ -2,6 +2,7 @@
 from typing import Any
 
 from uipath._cli._evals._evaluate import evaluate
+from uipath._cli._evals._models._output import UiPathEvalOutput
 from uipath._cli._evals._runtime import UiPathEvalContext
 from uipath._cli._runtime._contracts import UiPathRuntimeContext, UiPathRuntimeFactory
 from uipath._cli._runtime._runtime import UiPathRuntime
@@ -42,11 +43,18 @@ def __init__(self):
     # Act
     result = await evaluate(MyFactory(), context, event_bus)
 
-    # Assert
+    # Assert that the output is json-serializable
+    UiPathEvalOutput.model_validate(result.output).model_dump_json()
     assert result.output
     assert (
         result.output["evaluationSetResults"][0]["evaluationRunResults"][0]["result"][
             "score"
         ]
         == 100.0
     )
+    assert (
+        result.output["evaluationSetResults"][0]["evaluationRunResults"][0][
+            "evaluatorId"
+        ]
+        == "equality"
+    )