Skip to content

Commit d1fdd39

Browse files
Chibionosclaude
andcommitted
fix(eval): let coded agents pick the tool simulation model for BYOM tenants [AE-2294]
Tool and input simulation fell back to a hardcoded gpt-4.1-mini whenever the strategy had no model. Coded agents never carry a model in their runtime schema, so every coded agent simulation hit the UiPath-hosted default, which BYOM-only governance policies block. UIPATH_SIMULATION_MODEL now fills that gap, and the resolved model is actually sent instead of only being logged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 4e7440e commit d1fdd39

6 files changed

Lines changed: 146 additions & 25 deletions

File tree

‎packages/uipath/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "uipath"
3-
version = "2.14.25"
3+
version = "2.14.26"
44
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
55
readme = { file = "README.md", content-type = "text/markdown" }
66
requires-python = ">=3.11"

‎packages/uipath/src/uipath/eval/mocks/_input_mocker.py‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@
1010
from uipath.core.tracing import traced
1111
from uipath.platform import UiPath
1212
from uipath.platform.chat import UiPathLlmChatService
13-
from uipath.platform.chat._llm_gateway_service import ChatModels
1413

1514
from .._execution_context import eval_set_run_id_context
1615
from ._mock_context import cache_manager_context
1716
from ._mocker import UiPathInputMockingError, format_exception_message
17+
from ._simulation_model import simulation_completion_kwargs
1818
from ._structured_output import coerce_to_schema, generate_structured_output
1919
from ._types import (
2020
InputMockingStrategy,
@@ -127,17 +127,12 @@ async def generate_llm_input(
127127

128128
prompt = get_input_mocking_prompt(**prompt_generation_args)
129129

130-
model_parameters = mocking_strategy.model if mocking_strategy else None
131-
completion_kwargs = (
132-
model_parameters.model_dump(by_alias=False, exclude_none=True)
133-
if model_parameters
134-
else {}
130+
completion_kwargs = simulation_completion_kwargs(
131+
mocking_strategy.model if mocking_strategy else None
135132
)
136-
137-
simulation_model = completion_kwargs.get(
138-
"model", ChatModels.gpt_4_1_mini_2025_04_14
133+
logger.info(
134+
f"Simulating input generation using model: {completion_kwargs['model']}"
139135
)
140-
logger.info(f"Simulating input generation using model: {simulation_model}")
141136

142137
if cache_manager is not None:
143138
cache_key_data = {

‎packages/uipath/src/uipath/eval/mocks/_llm_mocker.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from uipath.core.tracing import traced
1212
from uipath.platform import UiPath
1313
from uipath.platform.chat import UiPathLlmChatService
14-
from uipath.platform.chat._llm_gateway_service import ChatModels, _cleanup_schema
14+
from uipath.platform.chat._llm_gateway_service import _cleanup_schema
1515

1616
from .._execution_context import (
1717
eval_set_run_id_context,
@@ -30,6 +30,7 @@
3030
UiPathNoMockFoundError,
3131
format_exception_message,
3232
)
33+
from ._simulation_model import simulation_completion_kwargs
3334
from ._structured_output import generate_structured_output
3435
from ._types import (
3536
ExampleCall,
@@ -176,18 +177,12 @@ async def response(
176177
k: json.dumps(pydantic_to_dict_safe(v), default=serialize_defaults)
177178
for k, v in prompt_input.items()
178179
}
179-
model_parameters = self.context.strategy.model
180-
completion_kwargs = (
181-
model_parameters.model_dump(by_alias=False, exclude_none=True)
182-
if model_parameters
183-
else {}
184-
)
185-
186-
simulation_model = completion_kwargs.get(
187-
"model", ChatModels.gpt_4_1_mini_2025_04_14
180+
completion_kwargs = simulation_completion_kwargs(
181+
self.context.strategy.model
188182
)
189183
logger.info(
190-
f"Simulating tool '{function_name}' using model: {simulation_model}"
184+
f"Simulating tool '{function_name}' using model: "
185+
f"{completion_kwargs['model']}"
191186
)
192187

193188
formatted_prompt = PROMPT.format(**prompt_generation_args)
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"""Model resolution for LLM-backed simulations."""
2+
3+
import os
4+
from typing import Any
5+
6+
from uipath.platform.chat._llm_gateway_service import ChatModels
7+
8+
from ._types import ModelSettings
9+
10+
SIMULATION_MODEL_ENV = "UIPATH_SIMULATION_MODEL"
11+
12+
13+
def simulation_completion_kwargs(
14+
model_settings: ModelSettings | None,
15+
) -> dict[str, Any]:
16+
"""Build completion kwargs for a simulation call, always naming the model."""
17+
completion_kwargs = (
18+
model_settings.model_dump(by_alias=False, exclude_none=True)
19+
if model_settings
20+
else {}
21+
)
22+
# Coded agents have no model in their schema; BYOM tenants block the default.
23+
completion_kwargs.setdefault(
24+
"model",
25+
os.environ.get(SIMULATION_MODEL_ENV) or ChatModels.gpt_4_1_mini_2025_04_14,
26+
)
27+
return completion_kwargs
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
from unittest.mock import MagicMock
2+
3+
import pytest
4+
from _pytest.monkeypatch import MonkeyPatch
5+
from pytest_httpx import HTTPXMock
6+
7+
from uipath.eval.mocks import mockable
8+
from uipath.eval.mocks._cache_manager import CacheManager
9+
from uipath.eval.mocks._mock_runtime import (
10+
clear_execution_context,
11+
set_execution_context,
12+
)
13+
from uipath.eval.mocks._simulation_model import (
14+
SIMULATION_MODEL_ENV,
15+
simulation_completion_kwargs,
16+
)
17+
from uipath.eval.mocks._types import (
18+
LLMMockingStrategy,
19+
MockingContext,
20+
ModelSettings,
21+
ToolSimulation,
22+
)
23+
24+
BYOM_MODEL = "Siemens-SDC-gpt-4o"
25+
26+
27+
def test_defaults_to_gateway_model_without_settings_or_env(monkeypatch: MonkeyPatch):
28+
monkeypatch.delenv(SIMULATION_MODEL_ENV, raising=False)
29+
30+
assert simulation_completion_kwargs(None) == {"model": "gpt-4.1-mini-2025-04-14"}
31+
32+
33+
def test_env_model_applies_when_settings_have_no_model(monkeypatch: MonkeyPatch):
34+
monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
35+
36+
assert simulation_completion_kwargs(None) == {"model": BYOM_MODEL}
37+
38+
39+
def test_configured_model_wins_over_env(monkeypatch: MonkeyPatch):
40+
monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
41+
settings = ModelSettings(model="gpt-4o-2024-11-20", temperature=0.2)
42+
43+
assert simulation_completion_kwargs(settings) == {
44+
"model": "gpt-4o-2024-11-20",
45+
"temperature": 0.2,
46+
}
47+
48+
49+
@pytest.mark.httpx_mock(assert_all_responses_were_requested=False)
50+
def test_tool_simulation_sends_env_model(
51+
httpx_mock: HTTPXMock, monkeypatch: MonkeyPatch
52+
):
53+
monkeypatch.setenv("UIPATH_URL", "https://example.com")
54+
monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "1234567890")
55+
monkeypatch.setenv(SIMULATION_MODEL_ENV, BYOM_MODEL)
56+
monkeypatch.setattr(CacheManager, "get", lambda *args, **kwargs: None)
57+
monkeypatch.setattr(CacheManager, "set", lambda *args, **kwargs: None)
58+
59+
@mockable()
60+
def web_search(*args, **kwargs) -> str:
61+
raise NotImplementedError()
62+
63+
for service in ("agenthub_", "orchestrator_"):
64+
httpx_mock.add_response(
65+
url=f"https://example.com/{service}/llm/api/capabilities", json={}
66+
)
67+
httpx_mock.add_response(
68+
url="https://example.com/llm/api/chat/completions"
69+
"?api-version=2024-08-01-preview",
70+
json={
71+
"id": "response-id",
72+
"object": "",
73+
"created": 0,
74+
"model": BYOM_MODEL,
75+
"choices": [
76+
{
77+
"index": 0,
78+
"message": {"role": "ai", "content": '"ok"', "tool_calls": None},
79+
"finish_reason": "EOS",
80+
}
81+
],
82+
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
83+
},
84+
)
85+
# Mirrors what cli_run/cli_debug build for a coded agent: no model anywhere.
86+
set_execution_context(
87+
MockingContext(
88+
strategy=LLMMockingStrategy(
89+
prompt="",
90+
tools_to_simulate=[ToolSimulation(name="web_search")],
91+
),
92+
name="debug-simulation",
93+
),
94+
MagicMock(),
95+
"test-execution-id",
96+
)
97+
try:
98+
assert web_search() == "ok"
99+
finally:
100+
clear_execution_context()
101+
102+
request = httpx_mock.get_request(method="POST")
103+
assert request is not None
104+
assert request.headers["X-UiPath-LlmGateway-NormalizedApi-ModelName"] == BYOM_MODEL

‎packages/uipath/uv.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)