From 8eae4ed9d08fffae152daceec2ea9e1a00fd85d1 Mon Sep 17 00:00:00 2001 From: Jason Wang <56037774+JasonW404@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:09:18 +0800 Subject: [PATCH 01/26] =?UTF-8?q?=F0=9F=90=9B=20Bugfix:=20Guard=20memory?= =?UTF-8?q?=5Fuser=5Fconfig=20against=20None=20in=20store/search=20memory?= =?UTF-8?q?=20tools=20(#3359)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Bugfix: Guard memory_user_config against None in store/search memory tools Add None checks before accessing memory_user_config attributes in StoreMemoryTool and SearchMemoryTool. When config is unavailable, apply conservative default (agent_share_option='never') to prevent unintended cross-agent memory sharing. * fix(memory): restore hidden tabs and hide Tenant Shared in speed mode - Fix nested array syntax bug that hid Tenant Shared and Agent Shared tabs - Conditionally hide Tenant Shared tab when isSpeedMode=true (no tenant concept) - All 5 tabs now properly visible in non-speed mode - 4 tabs visible in speed mode (Tenant Shared hidden) * test: add coverage for memory None guard, duplicate cleanup, and system tool filtering - test_store_memory_tool: add None config, _run_coroutine tests - test_search_memory_tool: add None config, _run_coroutine tests - test_memory_config_service: add duplicate record cleanup tests - test_tool_configuration_service: add SYSTEM_MANAGED_TOOL_NAMES filter test --- backend/agents/create_agent_info.py | 3 ++ backend/consts/tool_labels.py | 2 + backend/services/memory_config_service.py | 11 ++++ .../services/tool_configuration_service.py | 4 ++ frontend/app/[locale]/memory/page.tsx | 50 +++++++++---------- sdk/nexent/core/tools/search_memory_tool.py | 39 +++++++++++---- sdk/nexent/core/tools/store_memory_tool.py | 35 ++++++++++--- .../services/test_memory_config_service.py | 32 ++++++++++++ .../test_tool_configuration_service.py | 26 ++++++++++ .../sdk/core/tools/test_search_memory_tool.py | 46 ++++++++++++++++- test/sdk/core/tools/test_store_memory_tool.py | 45 ++++++++++++++++- 11 files changed, 247 insertions(+), 46 deletions(-) diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index c02addff8..d58932946 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -762,6 +762,9 @@ async def create_agent_config( "agent_id": memory_context.agent_id, } + memory_tool_names = {"store_memory", "search_memory"} + tool_list = [t for t in tool_list if t.name not in memory_tool_names] + store_tool_config = ToolConfig( class_name="StoreMemoryTool", name="store_memory", diff --git a/backend/consts/tool_labels.py b/backend/consts/tool_labels.py index 2f7228bbe..f1edebb37 100644 --- a/backend/consts/tool_labels.py +++ b/backend/consts/tool_labels.py @@ -52,3 +52,5 @@ BUILTIN_LABEL_MAP.update(_category_email) BUILTIN_LABEL_MAP.update(_category_memory) BUILTIN_LABEL_MAP.update(_category_terminal) + +SYSTEM_MANAGED_TOOL_NAMES = frozenset({"store_memory", "search_memory"}) diff --git a/backend/services/memory_config_service.py b/backend/services/memory_config_service.py index 9841caf8d..40cdc5b0e 100644 --- a/backend/services/memory_config_service.py +++ b/backend/services/memory_config_service.py @@ -94,6 +94,17 @@ def _update_single_config(user_id: str, config_key: str, config_value: str) -> b f"update_config_by_id failed, user_id={user_id}, key={config_key}, value={config_value}" ) return False + + # Handle duplicate records: delete all except the first one + if len(record_list) > 1: + logger.warning( + f"Found {len(record_list)} duplicate records for user_id={user_id}, key={config_key}. Cleaning up..." + ) + for duplicate in record_list[1:]: + dup_id = duplicate["config_id"] + delete_result = delete_config_by_config_id(dup_id, updated_by=user_id) + if not delete_result: + logger.error(f"Failed to delete duplicate config_id={dup_id}") return True diff --git a/backend/services/tool_configuration_service.py b/backend/services/tool_configuration_service.py index 32460fcee..88b9cbb2b 100644 --- a/backend/services/tool_configuration_service.py +++ b/backend/services/tool_configuration_service.py @@ -15,6 +15,7 @@ from consts.const import DATA_PROCESS_SERVICE, LOCAL_MCP_SERVER, MCP_MANAGEMENT_API from consts.exceptions import MCPConnectionError, NotFoundException, ToolExecutionException from consts.model import ToolInstanceInfoRequest, ToolInfo, ToolSourceEnum, ToolValidateRequest +from consts.tool_labels import SYSTEM_MANAGED_TOOL_NAMES from database.outer_api_tool_db import ( upsert_openapi_service, query_openapi_services_by_tenant, @@ -506,6 +507,9 @@ async def list_all_tools(tenant_id: str, labels: Optional[List[str]] = None): for tool in tools_info: tool_name = tool.get("name") + if tool_name in SYSTEM_MANAGED_TOOL_NAMES: + continue + # Always use SDK inputs for local tools to stay in sync with current tool code is_local = tool.get("source") == "local" sdk_info = local_tool_descriptions.get(tool_name) if is_local else None diff --git a/frontend/app/[locale]/memory/page.tsx b/frontend/app/[locale]/memory/page.tsx index 62d1d52fa..c0913d2d0 100644 --- a/frontend/app/[locale]/memory/page.tsx +++ b/frontend/app/[locale]/memory/page.tsx @@ -297,32 +297,30 @@ export default function MemoryContent() { ), children: renderBaseSettings(), }, - [ - { - key: MEMORY_TAB_KEYS.TENANT, - label: ( - - - {t("memoryManageModal.tenantShareTab")} - - ), - children: renderSingleList(memory.tenantSharedGroup), - disabled: !memory.memoryEnabled, - }, - { - key: MEMORY_TAB_KEYS.AGENT_SHARED, - label: ( - - - {t("memoryManageModal.agentShareTab")} - - ), - children: renderMemoryWithMenu(memory.agentSharedGroups, true), - disabled: - !memory.memoryEnabled || - memory.shareOption === MEMORY_SHARE_STRATEGY.NEVER, - }, - ], + ...(!isSpeedMode ? [{ + key: MEMORY_TAB_KEYS.TENANT, + label: ( + + + {t("memoryManageModal.tenantShareTab")} + + ), + children: renderSingleList(memory.tenantSharedGroup), + disabled: !memory.memoryEnabled, + }] : []), + { + key: MEMORY_TAB_KEYS.AGENT_SHARED, + label: ( + + + {t("memoryManageModal.agentShareTab")} + + ), + children: renderMemoryWithMenu(memory.agentSharedGroups, true), + disabled: + !memory.memoryEnabled || + memory.shareOption === MEMORY_SHARE_STRATEGY.NEVER, + }, { key: MEMORY_TAB_KEYS.USER_PERSONAL, label: ( diff --git a/sdk/nexent/core/tools/search_memory_tool.py b/sdk/nexent/core/tools/search_memory_tool.py index 4819a7625..8ffe2bc49 100644 --- a/sdk/nexent/core/tools/search_memory_tool.py +++ b/sdk/nexent/core/tools/search_memory_tool.py @@ -1,4 +1,5 @@ import asyncio +import concurrent.futures import logging from typing import Any @@ -11,6 +12,15 @@ logger = logging.getLogger("search_memory_tool") +def _run_coroutine(coro): + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + class SearchMemoryTool(Tool): name = "search_memory" description = ( @@ -63,25 +73,34 @@ def __init__( self.running_prompt_en = "Searching memory..." self.running_prompt_zh = "搜索记忆中..." + def _resolve_memory_levels(self) -> list[str]: + """Determine which memory levels to search based on user preferences. + + Conservative default: when config is unavailable, assume "never" + (no agent-level sharing) to protect user privacy. + """ + levels = ["tenant", "user", "agent", "user_agent"] + if not self.memory_user_config or self.memory_user_config.agent_share_option == "never": + levels.remove("agent") + if self.memory_user_config and self.agent_id in getattr(self.memory_user_config, "disable_agent_ids", []): + if "agent" in levels: + levels.remove("agent") + if self.memory_user_config and self.agent_id in getattr(self.memory_user_config, "disable_user_agent_ids", []): + if "user_agent" in levels: + levels.remove("user_agent") + return levels + def forward(self, query: str, top_k: int = 5) -> str: logger.info(f"[ACTIVE MEMORY] SearchMemoryTool invoked: query={query[:200]}, top_k={top_k}, user_id={self.user_id}, agent_id={self.agent_id}") if self.observer: running_prompt = self.running_prompt_zh if self.observer.lang == "zh" else self.running_prompt_en self.observer.add_message("", ProcessType.TOOL, running_prompt) - memory_levels = ["tenant", "user", "agent", "user_agent"] - if self.memory_user_config.agent_share_option == "never": - memory_levels.remove("agent") - if self.agent_id in getattr(self.memory_user_config, "disable_agent_ids", []): - if "agent" in memory_levels: - memory_levels.remove("agent") - if self.agent_id in getattr(self.memory_user_config, "disable_user_agent_ids", []): - if "user_agent" in memory_levels: - memory_levels.remove("user_agent") + memory_levels = self._resolve_memory_levels() try: from ...memory.memory_service import search_memory_in_levels - result = asyncio.run(search_memory_in_levels( + result = _run_coroutine(search_memory_in_levels( query_text=query, memory_config=self.memory_config, tenant_id=self.tenant_id, diff --git a/sdk/nexent/core/tools/store_memory_tool.py b/sdk/nexent/core/tools/store_memory_tool.py index 29dbc4a8f..47d3bd8c5 100644 --- a/sdk/nexent/core/tools/store_memory_tool.py +++ b/sdk/nexent/core/tools/store_memory_tool.py @@ -1,4 +1,5 @@ import asyncio +import concurrent.futures import logging from typing import Any @@ -11,6 +12,15 @@ logger = logging.getLogger("store_memory_tool") +def _run_coroutine(coro): + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + class StoreMemoryTool(Tool): name = "store_memory" description = ( @@ -58,6 +68,21 @@ def __init__( self.running_prompt_en = "Saving to memory..." self.running_prompt_zh = "保存到记忆中..." + def _resolve_memory_levels(self) -> list[str]: + """Determine which memory levels to write to based on user preferences. + + Conservative default: when config is unavailable, assume "never" + (no agent-level sharing) to protect user privacy. + """ + levels = ["user_agent", "agent"] + if not self.memory_user_config or self.memory_user_config.agent_share_option == "never": + levels.remove("agent") + if self.memory_user_config and self.agent_id in getattr(self.memory_user_config, "disable_user_agent_ids", []): + levels = [l for l in levels if l != "user_agent"] + if self.memory_user_config and self.agent_id in getattr(self.memory_user_config, "disable_agent_ids", []): + levels = [l for l in levels if l != "agent"] + return levels + def forward(self, content: str) -> str: logger.info(f"[ACTIVE MEMORY] StoreMemoryTool invoked: content={content[:200]}, user_id={self.user_id}, agent_id={self.agent_id}, store_count={self.store_count}/{self.max_stores_per_run}") if self.observer: @@ -67,19 +92,13 @@ def forward(self, content: str) -> str: if self.store_count >= self.max_stores_per_run: return "Memory storage limit reached for this conversation. Information will be saved automatically at the end." - levels = ["user_agent", "agent"] - if self.memory_user_config.agent_share_option == "never": - levels.remove("agent") - if self.agent_id in getattr(self.memory_user_config, "disable_user_agent_ids", []): - levels = [l for l in levels if l != "user_agent"] - if self.agent_id in getattr(self.memory_user_config, "disable_agent_ids", []): - levels = [l for l in levels if l != "agent"] + levels = self._resolve_memory_levels() if not levels: return "No memory levels available (all disabled by user preferences)." try: from ...memory.memory_service import add_memory_in_levels - result = asyncio.run(add_memory_in_levels( + result = _run_coroutine(add_memory_in_levels( messages=[{"role": "user", "content": content}], memory_config=self.memory_config, tenant_id=self.tenant_id, diff --git a/test/backend/services/test_memory_config_service.py b/test/backend/services/test_memory_config_service.py index e5b5e16bc..2f2679152 100644 --- a/test/backend/services/test_memory_config_service.py +++ b/test/backend/services/test_memory_config_service.py @@ -188,6 +188,38 @@ def test_update_single_config_insert_branch_fail(self, m_get_info, m_insert): ok = _update_single_config(self.user_id, "MEMORY_SWITCH", "Y") self.assertFalse(ok) + @patch("backend.services.memory_config_service.delete_config_by_config_id", return_value=True) + @patch("backend.services.memory_config_service.update_config_by_id", return_value=True) + @patch("backend.services.memory_config_service.get_memory_config_info") + def test_update_single_config_cleans_up_duplicates(self, m_get_info, m_update, m_del): + m_get_info.return_value = [ + {"config_id": 10, "config_value": "N"}, + {"config_id": 20, "config_value": "Y"}, + {"config_id": 30, "config_value": "N"}, + ] + from backend.services.memory_config_service import _update_single_config + + ok = _update_single_config(self.user_id, "MEMORY_SWITCH", "Y") + self.assertTrue(ok) + m_update.assert_called_once() + self.assertEqual(m_del.call_count, 2) + m_del.assert_any_call(20, updated_by=self.user_id) + m_del.assert_any_call(30, updated_by=self.user_id) + + @patch("backend.services.memory_config_service.delete_config_by_config_id", return_value=False) + @patch("backend.services.memory_config_service.update_config_by_id", return_value=True) + @patch("backend.services.memory_config_service.get_memory_config_info") + def test_update_single_config_duplicate_cleanup_failure_logged(self, m_get_info, m_update, m_del): + m_get_info.return_value = [ + {"config_id": 10, "config_value": "N"}, + {"config_id": 20, "config_value": "Y"}, + ] + from backend.services.memory_config_service import _update_single_config + + ok = _update_single_config(self.user_id, "MEMORY_SWITCH", "Y") + self.assertTrue(ok) + m_del.assert_called_once_with(20, updated_by=self.user_id) + # ------------------------------ _add_multi_value ------------------------------ @patch("backend.services.memory_config_service.insert_config", return_value=True) @patch("backend.services.memory_config_service.get_memory_config_info", return_value=[]) diff --git a/test/backend/services/test_tool_configuration_service.py b/test/backend/services/test_tool_configuration_service.py index ea2dbbe42..cdc10f135 100644 --- a/test/backend/services/test_tool_configuration_service.py +++ b/test/backend/services/test_tool_configuration_service.py @@ -1105,6 +1105,32 @@ async def test_list_all_tools_with_labels(self, mock_query_by_labels, mock_descr assert result[0]["tool_id"] == 2 mock_query_by_labels.assert_called_once_with("tenant1", ["database", "file"]) + @patch('backend.services.tool_configuration_service.get_local_tools_description_zh') + @patch('backend.services.tool_configuration_service.query_all_tools') + async def test_list_all_tools_filters_system_managed_tools(self, mock_query, mock_descriptions): + """list_all_tools filters out tools in SYSTEM_MANAGED_TOOL_NAMES.""" + mock_query.return_value = [ + {"tool_id": 1, "name": "tavily_search", "description": "d1", "source": "local", + "params": [], "inputs": "{}", "is_available": True, "create_time": "", "usage": ""}, + {"tool_id": 2, "name": "store_memory", "description": "d2", "source": "local", + "params": [], "inputs": "{}", "is_available": True, "create_time": "", "usage": ""}, + {"tool_id": 3, "name": "search_memory", "description": "d3", "source": "local", + "params": [], "inputs": "{}", "is_available": True, "create_time": "", "usage": ""}, + {"tool_id": 4, "name": "postgres_database", "description": "d4", "source": "local", + "params": [], "inputs": "{}", "is_available": True, "create_time": "", "usage": ""}, + ] + mock_descriptions.return_value = {} + + from backend.services.tool_configuration_service import list_all_tools + result = await list_all_tools("tenant1") + + result_names = [t["name"] for t in result] + assert "store_memory" not in result_names + assert "search_memory" not in result_names + assert "tavily_search" in result_names + assert "postgres_database" in result_names + assert len(result) == 2 + # test the fixture and helper function @pytest.fixture diff --git a/test/sdk/core/tools/test_search_memory_tool.py b/test/sdk/core/tools/test_search_memory_tool.py index c17f2f14d..c044ed903 100644 --- a/test/sdk/core/tools/test_search_memory_tool.py +++ b/test/sdk/core/tools/test_search_memory_tool.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch, AsyncMock from sdk.nexent.core.utils.observer import MessageObserver, ProcessType -from sdk.nexent.core.tools.search_memory_tool import SearchMemoryTool +from sdk.nexent.core.tools.search_memory_tool import SearchMemoryTool, _run_coroutine @pytest.fixture @@ -207,3 +207,47 @@ def test_forward_exception_returns_friendly_error(search_memory_tool): assert "Memory search failed" in result assert "Elasticsearch timeout" in result assert "Continuing without memory results" in result + + +def test_levels_none_config_conservative_default(mock_observer): + tool = SearchMemoryTool( + memory_config={"test": "config"}, + tenant_id="tenant_1", + user_id="user_1", + agent_id="agent_1", + memory_user_config=None, + observer=mock_observer, + ) + + with patch( + "sdk.nexent.memory.memory_service.search_memory_in_levels", + new_callable=AsyncMock, + return_value={"results": []}, + ) as mock_search: + tool.forward("query") + + call_kwargs = mock_search.call_args[1] + assert "agent" not in call_kwargs["memory_levels"] + assert "tenant" in call_kwargs["memory_levels"] + assert "user" in call_kwargs["memory_levels"] + assert "user_agent" in call_kwargs["memory_levels"] + + +def test_run_coroutine_no_running_loop(): + async def sample_coro(): + return "result" + + result = _run_coroutine(sample_coro()) + assert result == "result" + + +def test_run_coroutine_with_running_loop(): + async def sample_coro(): + return "result" + + async def run_with_loop(): + return _run_coroutine(sample_coro()) + + import asyncio + result = asyncio.run(run_with_loop()) + assert result == "result" diff --git a/test/sdk/core/tools/test_store_memory_tool.py b/test/sdk/core/tools/test_store_memory_tool.py index a3cfde9ef..85d441964 100644 --- a/test/sdk/core/tools/test_store_memory_tool.py +++ b/test/sdk/core/tools/test_store_memory_tool.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch, AsyncMock from sdk.nexent.core.utils.observer import MessageObserver, ProcessType -from sdk.nexent.core.tools.store_memory_tool import StoreMemoryTool +from sdk.nexent.core.tools.store_memory_tool import StoreMemoryTool, _run_coroutine @pytest.fixture @@ -283,3 +283,46 @@ def test_forward_exception_does_not_increment_counter(store_memory_tool): store_memory_tool.forward("some content") assert store_memory_tool.store_count == 0 + + +def test_levels_none_config_conservative_default(mock_observer): + """When memory_user_config is None, apply conservative default (no agent-level sharing).""" + tool = StoreMemoryTool( + memory_config={"test": "config"}, + tenant_id="tenant_1", + user_id="user_1", + agent_id="agent_1", + memory_user_config=None, + observer=mock_observer, + ) + + with patch( + "sdk.nexent.memory.memory_service.add_memory_in_levels", + new_callable=AsyncMock, + return_value={"results": [{"event": "ADD", "memory": "fact"}]}, + ) as mock_add: + tool.forward("some content") + + call_kwargs = mock_add.call_args[1] + assert call_kwargs["memory_levels"] == ["user_agent"] + assert "agent" not in call_kwargs["memory_levels"] + + +def test_run_coroutine_no_running_loop(): + async def sample_coro(): + return "result" + + result = _run_coroutine(sample_coro()) + assert result == "result" + + +def test_run_coroutine_with_running_loop(): + async def sample_coro(): + return "result" + + async def run_with_loop(): + return _run_coroutine(sample_coro()) + + import asyncio + result = asyncio.run(run_with_loop()) + assert result == "result" From 2bae2f3388242bd31e221ca7e1fc306f9fa117c1 Mon Sep 17 00:00:00 2001 From: DongJiBao2001 <120021235+DongJiBao2001@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:50:01 +0800 Subject: [PATCH 02/26] =?UTF-8?q?=E2=9C=A8Feat:Integrate=20openjiuwen's=20?= =?UTF-8?q?intelligent=20agent=20evaluation=20function=20close#3188=20(#33?= =?UTF-8?q?56)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨Feat：add prompt optimization * 🐛Bugfix: dockerbuild failed when running pipefail in python3_11 * 🔨Optimize： Optimize prompt optimization display page and interaction methods * ✨Feat:add agent evaluator * 🐛Bugfix: fix fronted and report download * 🐛 Bugfix: Add xlrd dependency and improve openjiuwen SDK adapter to handle circular imports more effectively. Update import paths in frontend components for consistency. * ♻️Refactor: frontend style refact * 🔨Rename evaluator sql * ♻️ Refactor: Optimize the frontend page for agent evaluation * ♻️Refactor: update modal mask properties for consistency * ♻️ Refactor: Simplify locale handling and clean up imports in OAuthCompletePage and UserManageComp * ✨ Feature: Enhance agent evaluation with Chinese output directive and new database migrations * 🔧 Update: Change button variant from "outline" to "outlined" in EvaluationConfigCard and EvaluationReportCard components; enhance datetime handling in test_client.py to ensure correct timezone representation. * 🧪Test: Add unit tests for the jiuwen_sdk_adapter to verify module-level behavior and side effects. * 🔧 Update: Refactor the mock invocation in test_jiuwen_sdk_adapter to use an async function for better compatibility with asynchronous tests. * 🧪 Test: Add comprehensive unit tests for evaluation set functionality, including CRUD operations and Excel parsing, to ensure robust behavior and error handling. * 🧪 Test: Add extensive unit tests for Jiuwen SDK adapter and evaluation services, covering lazy imports, log envelope parsing, and case validation to enhance error handling and ensure robust functionality. * 🔧 Update: Consolidate judge_model_id addition into agent_evaluation_t migration, removing the separate migration file for improved installation simplicity and ensuring idempotency in existing environments. * 🔧 Update: Introduce a consolidated migration file for agent evaluation, combining previous separate migrations into a single idempotent script for improved installation simplicity and clarity. This includes the addition of evaluation tables and the pass_status column, ensuring all necessary structures are created in one step. --- backend/adapters/jiuwen_sdk_adapter.py | 360 +++- backend/apps/agent_evaluation_app.py | 147 ++ backend/apps/config_app.py | 4 + backend/apps/evaluation_set_app.py | 198 ++ backend/database/agent_evaluation_db.py | 345 ++++ backend/database/client.py | 6 +- backend/database/db_models.py | 163 +- backend/database/evaluation_set_db.py | 149 ++ backend/pyproject.toml | 3 + backend/services/agent_evaluation_service.py | 866 ++++++++ backend/services/evaluation_set_service.py | 193 ++ backend/utils/evaluation_set_excel_utils.py | 214 ++ .../v2.3.0_0628_add_agent_evaluation_full.sql | 220 ++ .../components/AgentRepositoryCopyDialog.tsx | 2 +- .../components/AgentRepositoryDetailModal.tsx | 2 +- .../agent-space/components/MineAgentsView.tsx | 9 + .../agent-space/components/MyAgentCard.tsx | 6 +- frontend/app/[locale]/agent-space/page.tsx | 10 +- .../agents/components/a2a/A2AChatModal.tsx | 2 +- .../agentConfig/skill/SkillConfigModal.tsx | 2 +- .../agentConfig/tool/LabelManagementModal.tsx | 2 +- .../agentConfig/tool/SelectToolsDialog.tsx | 4 +- .../agentInfo/AgentGenerateDetail.tsx | 46 +- .../components/agentInfo/ExpandEditModal.tsx | 19 +- .../agentInfo/PromptOptimizeModal.tsx | 9 +- frontend/app/[locale]/agents/page.tsx | 7 +- .../components/UserManageComp.tsx | 19 +- .../components/resources/McpList.tsx | 2 +- .../components/EvaluationConclusion.tsx | 31 + .../components/EvaluationConfigCard.tsx | 278 +++ .../components/EvaluationHistoryCard.tsx | 205 ++ .../components/EvaluationReportCard.tsx | 116 ++ .../evaluate/components/MetricCard.tsx | 37 + .../components/ResultDistributionBar.tsx | 54 + .../components/TestCaseLibraryModal.tsx | 258 +++ .../evaluate/internal/AgentEvaluateClient.tsx | 199 ++ .../space/agents/[agentId]/evaluate/page.tsx | 12 + .../skill/InstallOfficialSkillsModal.tsx | 2 +- frontend/hooks/auth/useAuthorization.ts | 26 +- .../hooks/evaluation/useEvaluationHistory.ts | 132 ++ .../hooks/evaluation/useEvaluationReport.ts | 35 + .../hooks/evaluation/useEvaluationSets.ts | 55 + .../hooks/evaluation/useStartEvaluation.ts | 40 + frontend/hooks/useConfirmModal.ts | 18 +- frontend/lib/utils.ts | 22 +- frontend/public/locales/en/common.json | 100 + frontend/public/locales/zh/common.json | 103 + frontend/services/api.ts | 17 + frontend/services/evaluationService.ts | 200 ++ frontend/types/agentEvaluation.ts | 83 + .../adapters/test_jiuwen_sdk_adapter.py | 1835 +++++++++++++++++ .../backend/app/test_evaluation_delete_app.py | 352 ++++ test/backend/app/test_evaluation_set_app.py | 457 ++++ .../database/test_agent_evaluation_db.py | 610 ++++++ test/backend/database/test_client.py | 21 +- .../database/test_evaluation_set_db.py | 235 +++ .../services/test_agent_evaluation_service.py | 1296 ++++++++++++ .../services/test_evaluation_set_service.py | 489 +++++ .../utils/test_evaluation_set_excel_utils.py | 479 +++++ test/conftest.py | 41 + 60 files changed, 10693 insertions(+), 154 deletions(-) create mode 100644 backend/apps/agent_evaluation_app.py create mode 100644 backend/apps/evaluation_set_app.py create mode 100644 backend/database/agent_evaluation_db.py create mode 100644 backend/database/evaluation_set_db.py create mode 100644 backend/services/agent_evaluation_service.py create mode 100644 backend/services/evaluation_set_service.py create mode 100644 backend/utils/evaluation_set_excel_utils.py create mode 100644 deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationConclusion.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationConfigCard.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationHistoryCard.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationReportCard.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/MetricCard.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/ResultDistributionBar.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/components/TestCaseLibraryModal.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/internal/AgentEvaluateClient.tsx create mode 100644 frontend/app/[locale]/space/agents/[agentId]/evaluate/page.tsx create mode 100644 frontend/hooks/evaluation/useEvaluationHistory.ts create mode 100644 frontend/hooks/evaluation/useEvaluationReport.ts create mode 100644 frontend/hooks/evaluation/useEvaluationSets.ts create mode 100644 frontend/hooks/evaluation/useStartEvaluation.ts create mode 100644 frontend/services/evaluationService.ts create mode 100644 frontend/types/agentEvaluation.ts create mode 100644 test/backend/adapters/test_jiuwen_sdk_adapter.py create mode 100644 test/backend/app/test_evaluation_delete_app.py create mode 100644 test/backend/app/test_evaluation_set_app.py create mode 100644 test/backend/database/test_agent_evaluation_db.py create mode 100644 test/backend/database/test_evaluation_set_db.py create mode 100644 test/backend/services/test_agent_evaluation_service.py create mode 100644 test/backend/services/test_evaluation_set_service.py create mode 100644 test/backend/utils/test_evaluation_set_excel_utils.py diff --git a/backend/adapters/jiuwen_sdk_adapter.py b/backend/adapters/jiuwen_sdk_adapter.py index f62ce9d06..c70194c99 100644 --- a/backend/adapters/jiuwen_sdk_adapter.py +++ b/backend/adapters/jiuwen_sdk_adapter.py @@ -1,13 +1,13 @@ """ openjiuwen SDK adapter for Nexent. -This module must be imported lazily (not at module load time) because -openjiuwen 0.1.13 has circular import bugs in its __init__.py files that -prevent the SDK from loading unless we bypass them. +This module works around circular import bugs in openjiuwen 0.1.13 by: + 1. Stubbing react_agent_evolve (causes the circular import) + 2. Installing a meta-path finder that blocks broken __init__.py files + 3. All of the above runs at MODULE LOAD TIME (before openjiuwen imports) -Import flow: - backend/adapters/__init__.py -> try/except -> JiuwenSDKAdapter = None - -> when needed: _install_jiuwen_bypasser() -> openjiuwen imports work +The adapters/__init__.py wraps the import in try/except so the server +starts even when openjiuwen is not installed. """ import asyncio import importlib.abc @@ -17,21 +17,73 @@ import os import sys import types -from typing import Any, List, Literal, Optional - -logger = logging.getLogger("jiuwen_adapter") - -from adapters.exception import JiuwenSDKError - +from typing import Any, List, Literal, Optional, Tuple # ---------------------------------------------------------------------- -# Circular import bypasser for openjiuwen 0.1.13 -# -# openjiuwen has broken __init__.py files that create circular import chains: -# tune/__init__.py -> tune.optimizer -> core.operator -> agent_evolving -> ... -# This bypasser prevents those __init__.py files from executing while still -# allowing regular .py submodule files to load normally. +# MUST install bypasser + stubs before any openjiuwen import. +# The module-level `from openjiuwen.dev_tools.tune.base import ...` below +# triggers the entire circular import chain. Installing the bypasser here +# (before that line executes) breaks the cycle. # ---------------------------------------------------------------------- + +# Stub react_agent_evolve so single_agent.__init__.py can import it without +# hitting the circular core.operator dependency. +if "openjiuwen.core.single_agent.agents.react_agent_evolve" not in sys.modules: + _stub = types.ModuleType("openjiuwen.core.single_agent.agents.react_agent_evolve") + _stub.__file__ = "" + + # Provide a dummy ReActAgentEvolve class. The real one lives in the + # actual react_agent_evolve.py which we cannot load at this stage + # because it imports ToolCallOperator from core.operator (still blocked). + # The stub class satisfies the import in single_agent.__init__.py. + class _DummyReActAgentEvolve: # noqa: N801 + pass + + _stub.ReActAgentEvolve = _DummyReActAgentEvolve + _stub.__all__ = ["ReActAgentEvolve"] + sys.modules["openjiuwen.core.single_agent.agents.react_agent_evolve"] = _stub + +# NOTE: do NOT stub openjiuwen.core.single_agent. Its real __init__.py must +# run so that `from .schema.agent_card import AgentCard` and the other +# relative submodule imports resolve correctly. We only need react_agent_evolve +# stubbed (its real file imports ToolCallOperator from core.operator, which +# the bypasser below blocks). + +# Stub missing optional dependencies before openjiuwen import chain reaches them +for _name, _attrs in [ + ("pymilvus", {"is_successful": lambda *a, **kw: True}), + ("dashscope", {}), + ("pdfplumber", {}), +]: + if _name not in sys.modules: + _mod = types.ModuleType(_name) + _mod.__path__ = [] + for _k, _v in _attrs.items(): + setattr(_mod, _k, _v) + sys.modules[_name] = _mod + +for _name in ["pymilvus.client", "pymilvus.client.utils"]: + if _name not in sys.modules: + _m = types.ModuleType(_name) + _m.__path__ = [] + if _name == "pymilvus.client.utils": + _m.is_successful = lambda *a, **kw: True + sys.modules[_name] = _m + +for _name, _attrs in [ + ("dashscope.api_entities", {}), + ("dashscope.api_entities.data", {}), + ("dashscope.api_entities.dashscope_response", {"DashScopeAPIResponse": object}), + ("dashscope.common", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), + ("dashscope.common.constants", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), +]: + if _name not in sys.modules: + _m = types.ModuleType(_name) + _m.__path__ = [] + for _k, _v in _attrs.items(): + setattr(_m, _k, _v) + sys.modules[_name] = _m + _CIRCULAR_CHAIN = { "openjiuwen.agent_evolving", "openjiuwen.agent_evolving.trainer", @@ -57,30 +109,24 @@ class _JiuwenInitBypasser(importlib.abc.MetaPathFinder, importlib.abc.Loader): def find_spec(self, fullname: str, path: Any, target: Any = None) -> Any: if not fullname.startswith("openjiuwen") or fullname == "openjiuwen": return None - try: import openjiuwen as _oj pkg_root = _oj.__path__[0] except ImportError: return None - parts = fullname.split(".")[1:] file_path = pkg_root for p in parts: file_path = os.path.join(file_path, p) - is_package = os.path.isdir(file_path) if not is_package: return None - init_path = os.path.join(file_path, "__init__.py") if not os.path.exists(init_path): return None - if fullname not in _CIRCULAR_CHAIN: return None - spec = importlib.machinery.ModuleSpec( fullname, self, is_package=True, origin="" ) @@ -106,7 +152,6 @@ def __getattr__(self, name: str) -> Any: import openjiuwen as _oj import importlib - # Prevent recursion when Python scans sys.meta_path for find_distributions etc. if name in ( "find_distributions", "find_module", @@ -118,87 +163,55 @@ def __getattr__(self, name: str) -> Any: "__spec__", ): raise AttributeError(name) - pkg_root = _oj.__path__[0] parts = self.__name__.split(".")[1:] + [name] file_path = pkg_root for p in parts: file_path = os.path.join(file_path, p) - - # If it's a package directory, import it as a submodule if os.path.isdir(file_path) and os.path.exists(os.path.join(file_path, "__init__.py")): return importlib.import_module(f"{self.__name__}.{name}") - # If it's a regular .py file if os.path.exists(file_path + ".py"): return importlib.import_module(f"{self.__name__}.{name}") raise AttributeError(name) -_bypasser_installed = False +# Install the bypasser into sys.meta_path so it intercepts openjiuwen imports. +for _finder in sys.meta_path: + if isinstance(_finder, _JiuwenInitBypasser): + break +else: + sys.meta_path.insert(0, _JiuwenInitBypasser()) + +# No-op: bypasser is already installed above. +# Kept for backward compatibility with code that calls it. +_bypasser_installed = True def _install_jiuwen_bypasser() -> bool: - """ - Install the circular import bypasser for openjiuwen. - Returns True if installed, False if already installed or openjiuwen not available. - """ - global _bypasser_installed - if _bypasser_installed: - return True - - # Stub missing optional dependencies before openjiuwen import chain reaches them - _stubbed = [ - ("pymilvus", {"is_successful": lambda *args, **kwargs: True}), - ("dashscope", {}), - ("pdfplumber", {}), - ] - for _name, _attrs in _stubbed: - if _name not in sys.modules: - _mod = types.ModuleType(_name) - for _k, _v in _attrs.items(): - setattr(_mod, _k, _v) - sys.modules[_name] = _mod - _mod.__path__ = [] - - # Pre-create nested stub modules for pymilvus.client.utils chain - if "pymilvus.client" not in sys.modules: - _client_mod = types.ModuleType("pymilvus.client") - _client_mod.__path__ = [] - sys.modules["pymilvus.client"] = _client_mod - if "pymilvus.client.utils" not in sys.modules: - _utils_mod = types.ModuleType("pymilvus.client.utils") - _utils_mod.is_successful = lambda *args, **kwargs: True - sys.modules["pymilvus.client.utils"] = _utils_mod - - # Stub dashscope sub-modules that may be imported lazily - _dashscope_subs = [ - ("dashscope.api_entities", {}), - ("dashscope.api_entities.data", {}), - ("dashscope.api_entities.dashscope_response", {"DashScopeAPIResponse": object}), - ("dashscope.common", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ("dashscope.common.constants", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ] - for _name, _attrs in _dashscope_subs: - if _name not in sys.modules: - _m = types.ModuleType(_name) - _m.__path__ = [] - for _k, _v in _attrs.items(): - setattr(_m, _k, _v) - sys.modules[_name] = _m + """No-op: bypasser is installed at module load time. Kept for compatibility.""" + return True +# Now safe to import openjiuwen — bypasser is active. +from openjiuwen.dev_tools.tune.base import Case as _Case, EvaluatedCase as _EvaluatedCase + + +def _case_from_inputs_label(inputs: dict, label: dict) -> "_Case": + # Keep stable keys for schema; allow extra fields to exist. + return _Case(inputs=inputs, label=label) + + +def _extract_score_reason(evaluated: Any) -> Tuple[float, str]: + """Try to normalize Jiuwen EvaluatedCase into (score, reason).""" try: - import openjiuwen # noqa: F401 - except ImportError: - return False + score = float(getattr(evaluated, "score", 0.0) or 0.0) + except Exception: + score = 0.0 + reason = getattr(evaluated, "reason", "") or "" + return score, str(reason) - for finder in sys.meta_path: - if isinstance(finder, _JiuwenInitBypasser): - _bypasser_installed = True - return True +logger = logging.getLogger("jiuwen_adapter") - sys.meta_path.insert(0, _JiuwenInitBypasser()) - _bypasser_installed = True - return True +from adapters.exception import JiuwenSDKError # ---------------------------------------------------------------------- @@ -247,8 +260,8 @@ def run_in_thread(): # ---------------------------------------------------------------------- # Jiuwen SDK lazy import helpers # ---------------------------------------------------------------------- -def _lazy_import_jiuwen_config(): - """Lazily import only lightweight Jiuwen config classes.""" +def _lazy_import_jiuwen(): + """延迟导入 Jiuwen SDK，避免 openjiuwen 未装时模块级 ImportError""" _install_jiuwen_bypasser() try: @@ -261,8 +274,33 @@ def _lazy_import_jiuwen_config(): ModelClientConfig, ProviderType, ) + # Optional: allow adjusting SDK internal client timeout via config if supported. + # We keep import local to avoid breaking on SDK version differences. + try: + from openjiuwen.core.foundation.llm import ModelClientOptions # type: ignore + except Exception: # pragma: no cover + ModelClientOptions = None # type: ignore + from openjiuwen.dev_tools.prompt_builder.builder.feedback_prompt_builder import ( + FeedbackPromptBuilder, + ) + from openjiuwen.dev_tools.prompt_builder.builder.badcase_prompt_builder import ( + BadCasePromptBuilder, + ) + from openjiuwen.dev_tools.tune.base import Case, EvaluatedCase - return ModelRequestConfig, ModelClientConfig, ProviderType + # Evaluator metrics (avoid importing openjiuwen.dev_tools.tune package root) + from openjiuwen.agent_evolving.evaluator.metrics.llm_as_judge import LLMAsJudgeMetric + + return ( + ModelRequestConfig, + ModelClientConfig, + ProviderType, + FeedbackPromptBuilder, + BadCasePromptBuilder, + Case, + EvaluatedCase, + LLMAsJudgeMetric, + ) def build_jiuwen_model_configs(model_id: int, tenant_id: str): @@ -270,7 +308,7 @@ def build_jiuwen_model_configs(model_id: int, tenant_id: str): from database.model_management_db import get_model_by_model_id from utils.config_utils import get_model_name_from_config - ModelRequestConfig, ModelClientConfig, ProviderType = _lazy_import_jiuwen_config() + ModelRequestConfig, ModelClientConfig, ProviderType, _, _, _, _, _ = _lazy_import_jiuwen() model_config = get_model_by_model_id(model_id, tenant_id) if not model_config: @@ -307,6 +345,40 @@ def build_jiuwen_model_configs(model_id: int, tenant_id: str): return request_config, client_config +def _build_openai_client( + client_config: "ModelClientConfig", request_config: "ModelRequestConfig" +) -> Any: + """ + 直接创建 OpenAIModelClient，绕过 FeedbackPromptBuilder 的深导入链。 + """ + _install_jiuwen_bypasser() + from openjiuwen.core.foundation.llm.model_clients.openai_model_client import ( + OpenAIModelClient, + ) + + class _DirectClient: + """对 OpenAIModelClient 的薄封装，只暴露 invoke() 方法""" + + def __init__(self, inner: Any): + self._inner = inner + self._model = request_config.model_name + + async def invoke(self, messages: list[dict]) -> Any: + return await self._inner.invoke( + model=self._model, + messages=messages, + temperature=request_config.temperature, + top_p=request_config.top_p, + ) + + client = OpenAIModelClient( + model_config=request_config, model_client_config=client_config + ) + return _DirectClient(client) + + +def _build_message(role: str, content: str) -> dict: + return {"role": role, "content": content} def _lazy_import_jiuwen_builders(): """Lazily import prompt builders only when optimization paths need them.""" _install_jiuwen_bypasser() @@ -366,16 +438,9 @@ def _unwrap_prompt_response(text: str) -> str: return text -def _lazy_import_jiuwen_tune_types(): - """Lazily import Jiuwen tune types only when badcase flow needs them.""" - _install_jiuwen_bypasser() - from openjiuwen.dev_tools.tune.base import Case, EvaluatedCase - return Case, EvaluatedCase - - def to_jiuwen_evaluated_case(bad_case) -> Any: """将 nexent BadCase 转换为 Jiuwen EvaluatedCase""" - Case, EvaluatedCase = _lazy_import_jiuwen_tune_types() + _, _, _, _, _, Case, EvaluatedCase, _ = _lazy_import_jiuwen() case = Case( inputs={"question": bad_case.question}, @@ -508,6 +573,103 @@ def optimize_badcase( self.logger.error(f"Jiuwen BadCasePromptBuilder 调用失败: {e}") raise JiuwenSDKError(f"BadCasePromptBuilder 调用失败: {e}") from e + def evaluate_semantic_consistency( + self, + *, + question: str, + expected_answer: str, + model_answer: str, + user_metrics: str = "", + ) -> Tuple[float, str]: + """LLM-as-judge semantic consistency scoring. + + Returns: + (score, reason) + + Notes: + openjiuwen 0.1.13 implements LLMAsJudgeMetric as a binary metric + (1.0 pass / 0.0 fail) by parsing a JSON result from the judge model. + ``metric.compute()`` only returns the numeric score; the actual judge + verdict (with ``result`` and ``reason``) lives in the LLM response. + We invoke the model directly here so we can extract **both** the score + and the human-readable reason from the same response. + """ + self._ensure_available() + + try: + from openjiuwen.agent_evolving.evaluator.metrics.llm_as_judge import ( + LLMAsJudgeMetric, + ) + except Exception as exc: + raise JiuwenSDKError(f"Failed to import LLMAsJudgeMetric: {exc}") from exc + + request_config, client_config = build_jiuwen_model_configs(self.model_id, self.tenant_id) + + metric = LLMAsJudgeMetric( + model_config=request_config, + model_client_config=client_config, + user_metrics=user_metrics or "", + ) + + # Build the same prompt that ``metric.compute()`` uses. + messages = metric._template.format({ + "question": str(question or ""), + "expected_answer": str(expected_answer), + "model_answer": str(model_answer), + }).to_messages() + + # Append a directive requiring Chinese output for the evaluation reason. + # This ensures consistent Chinese language in the report regardless of + # the judge model's default language. + chinese_directive = ( + "\n\nIMPORTANT: You MUST respond in Chinese for the 'reason' field. " + "The reason must be a clear explanation in Simplified Chinese. " + "重要提示：'reason' 字段必须使用中文撰写，用简洁的中文解释评判结果。" + ) + + if messages: + last = messages[-1] + last_role = getattr(last, "role", None) + # Default template returns a single UserMessage with role="user" + if last_role == "user": + last.content = (last.content or "") + chinese_directive + else: + # Append a new user message with the directive + from openjiuwen.core.foundation.llm import UserMessage + messages.append(UserMessage(content=chinese_directive.lstrip("\n"))) + + try: + response = asyncio.run(metric._model.invoke(messages)).content + except Exception as exc: + raise JiuwenSDKError(f"Judge LLM invoke failed: {exc}") from exc + + try: + from openjiuwen.agent_evolving.utils import TuneUtils + data = TuneUtils.parse_json_from_llm_response(response) + except Exception as exc: + raise JiuwenSDKError(f"Failed to parse judge response: {exc}") from exc + + if not isinstance(data, dict): + raise JiuwenSDKError(f"Judge response is not a JSON object: {response!r}") + + result = data.get("result") + if result is True or (isinstance(result, str) and result.strip().lower() == "true"): + score = 1.0 + else: + score = 0.0 + + raw_reason = data.get("reason") + if isinstance(raw_reason, str): + reason = raw_reason.strip() + elif isinstance(raw_reason, dict): + reason = raw_reason.get("reason") or "" + else: + reason = "" + if not reason: + reason = "通过" if score >= 1.0 else "失败" + + return score, reason + def generate(self, **kwargs) -> dict: """调用 Jiuwen 提示词生成能力""" self._ensure_available() diff --git a/backend/apps/agent_evaluation_app.py b/backend/apps/agent_evaluation_app.py new file mode 100644 index 000000000..b163babe0 --- /dev/null +++ b/backend/apps/agent_evaluation_app.py @@ -0,0 +1,147 @@ +import logging +from http import HTTPStatus +from typing import Optional + +from fastapi import APIRouter, Body, Header, HTTPException, Query +from fastapi.responses import JSONResponse, StreamingResponse + +from services.agent_evaluation_service import ( + create_agent_evaluation_run_impl, + delete_agent_evaluation_run_impl, + generate_agent_evaluation_report_impl, + get_agent_evaluation_run_impl, + list_agent_evaluation_cases_impl, + list_agent_evaluations_by_agent_impl, +) +from utils.auth_utils import get_current_user_id + +logger = logging.getLogger("agent_evaluation_app") + +router = APIRouter(prefix="/agent-evaluations") + + +@router.post("") +async def create_agent_evaluation_api( + agent_id: int = Body(...), + evaluation_set_id: int = Body(...), + judge_model_id: int = Body(..., description="Model id used for judging (Jiuwen)"), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + run = create_agent_evaluation_run_impl( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + evaluation_set_id=evaluation_set_id, + judge_model_id=judge_model_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": run}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Create agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Create agent evaluation error") + + +@router.get("") +async def list_agent_evaluations_by_agent_api( + agent_id: int = Query(...), + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_agent_evaluations_by_agent_impl( + agent_id=agent_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List agent evaluations error: %r", exc) + raise HTTPException(status_code=500, detail="List agent evaluations error") + + +@router.get("/{agent_evaluation_id}") +async def get_agent_evaluation_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = get_agent_evaluation_run_impl(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("Get agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Get agent evaluation error") + + +@router.get("/{agent_evaluation_id}/cases") +async def list_agent_evaluation_cases_api( + agent_evaluation_id: int, + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_agent_evaluation_cases_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List agent evaluation cases error: %r", exc) + raise HTTPException(status_code=500, detail="List agent evaluation cases error") + + +@router.get("/{agent_evaluation_id}/report") +async def download_agent_evaluation_report_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data, fail_count = generate_agent_evaluation_report_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + ) + suffix = "_failed.xlsx" if fail_count > 0 else "_all.xlsx" + return StreamingResponse( + iter([data]), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={ + "Content-Disposition": f"attachment; filename=evaluation_report_{agent_evaluation_id}{suffix}" + }, + ) + except ValueError as ve: + raise HTTPException(status_code=404, detail=str(ve)) + except Exception as exc: + logger.exception("Download agent evaluation report error: %r", exc) + raise HTTPException(status_code=500, detail="Download agent evaluation report error") + + +@router.delete("/{agent_evaluation_id}") +async def delete_agent_evaluation_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + """Soft-delete an evaluation run. Only the creator may delete.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + delete_agent_evaluation_run_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + user_id=user_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Delete agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Delete agent evaluation error") diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index 9ffadfe5e..f8f40341f 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -33,6 +33,8 @@ from apps.monitoring_app import router as monitoring_router from apps.a2a_server_app import router as a2a_server_router from apps.haotian_app import router as haotian_router +from apps.evaluation_set_app import router as evaluation_set_router +from apps.agent_evaluation_app import router as agent_evaluation_router from apps.aidp_app import router as aidp_router from apps.cas_app import router as cas_router from consts.const import IS_SPEED_MODE @@ -93,4 +95,6 @@ async def sync_default_prompt_template_on_startup(): app.include_router(a2a_client_router) app.include_router(a2a_server_router) app.include_router(haotian_router) +app.include_router(evaluation_set_router) +app.include_router(agent_evaluation_router) app.include_router(aidp_router) diff --git a/backend/apps/evaluation_set_app.py b/backend/apps/evaluation_set_app.py new file mode 100644 index 000000000..d9d870a72 --- /dev/null +++ b/backend/apps/evaluation_set_app.py @@ -0,0 +1,198 @@ +import io +import json +import logging +from http import HTTPStatus +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Body, File, Form, Header, HTTPException, Query, UploadFile +from fastapi.responses import JSONResponse, StreamingResponse + +from services.evaluation_set_service import ( + create_evaluation_set_from_cases, + create_evaluation_set_from_jsonl, + delete_evaluation_set_impl, + get_evaluation_set_impl, + list_evaluation_set_cases_impl, + list_evaluation_sets_impl, +) +from utils.auth_utils import get_current_user_id +from utils.evaluation_set_excel_utils import build_evaluation_set_excel_template_bytes, parse_evaluation_cases_from_excel + +logger = logging.getLogger("evaluation_set_app") + +router = APIRouter(prefix="/evaluation-sets") + + +@router.get("") +async def list_evaluation_sets_api( + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + data = list_evaluation_sets_impl(tenant_id=tenant_id, limit=limit, offset=offset) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List evaluation sets error: %r", exc) + raise HTTPException(status_code=500, detail="List evaluation sets error") + + +@router.post("") +async def create_evaluation_set_api( + name: str = Body(...), + description: Optional[str] = Body(None), + source_filename: Optional[str] = Body(None), + jsonl_text: str = Body(..., description="Raw JSONL content"), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + meta = create_evaluation_set_from_jsonl( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + jsonl_text=jsonl_text, + created_by=user_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Create evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Create evaluation set error") + + +@router.post("/upload") +async def upload_evaluation_set_api( + name: str = Form(...), + description: Optional[str] = Form(None), + files: List[UploadFile] = File(...), + authorization: Optional[str] = Header(None, alias="Authorization"), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + if not files: + raise ValueError("At least one file is required") + + all_cases: List[Dict[str, Any]] = [] + source_filenames: List[str] = [] + + for file in files: + raw = await file.read() + filename = file.filename or "" + source_filenames.append(filename) + lower = filename.lower() + + if lower.endswith(".xlsx") or lower.endswith(".xls"): + cases = parse_evaluation_cases_from_excel(filename=filename, raw=raw) + all_cases.extend(cases) + else: + # Backward compatible: still accept JSONL upload + try: + jsonl_text = raw.decode("utf-8") + except Exception: + jsonl_text = raw.decode("utf-8", errors="ignore") + + # Parse JSONL into cases + for line in jsonl_text.strip().splitlines(): + line = line.strip() + if not line: + continue + obj = json.loads(line) + all_cases.append({ + "query": obj.get("query", ""), + "answer": obj.get("answer", ""), + "context": obj.get("context"), + "case_id": obj.get("case_id"), + }) + + if not all_cases: + raise ValueError("No valid cases found in uploaded files") + + meta = create_evaluation_set_from_cases( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=", ".join(source_filenames), + cases=all_cases, + created_by=user_id, + ) + + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Upload evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Upload evaluation set error") + + +@router.get("/template") +async def download_evaluation_set_template_api(): + """Download Excel template for evaluation set upload.""" + data = build_evaluation_set_excel_template_bytes() + headers = { + "Content-Disposition": 'attachment; filename="evaluation_set_template.xlsx"' + } + return StreamingResponse( + io.BytesIO(data), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers=headers, + ) + + +@router.get("/{evaluation_set_id}") +async def get_evaluation_set_api( + evaluation_set_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = get_evaluation_set_impl(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("Get evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Get evaluation set error") + + +@router.get("/{evaluation_set_id}/cases") +async def list_evaluation_set_cases_api( + evaluation_set_id: int, + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_evaluation_set_cases_impl( + evaluation_set_id=evaluation_set_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List evaluation set cases error: %r", exc) + raise HTTPException(status_code=500, detail="List evaluation set cases error") + + +@router.delete("/{evaluation_set_id}") +async def delete_evaluation_set_api( + evaluation_set_id: int, + authorization: Optional[str] = Header(None), +): + """Soft-delete an evaluation set. + + Blocked when any active evaluation run still references the set, so + historical runs never lose their context. + """ + try: + user_id, tenant_id = get_current_user_id(authorization) + delete_evaluation_set_impl(evaluation_set_id, tenant_id, user_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Delete evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Delete evaluation set error") diff --git a/backend/database/agent_evaluation_db.py b/backend/database/agent_evaluation_db.py new file mode 100644 index 000000000..0688319cd --- /dev/null +++ b/backend/database/agent_evaluation_db.py @@ -0,0 +1,345 @@ +import logging +from typing import Any, Dict, List, Optional + +from sqlalchemy import case as sql_case, func +from database.client import as_dict, get_db_session +from database.db_models import AgentEvaluation, AgentEvaluationCase, AgentInfo, EvaluationSet, ModelRecord + +logger = logging.getLogger("agent_evaluation_db") + + +def create_agent_evaluation( + tenant_id: str, + agent_id: int, + agent_version_no: int, + evaluation_set_id: int, + total: int, + judge_model_id: Optional[int], + created_by: Optional[str], +) -> Dict[str, Any]: + with get_db_session() as session: + rec = AgentEvaluation( + tenant_id=tenant_id, + agent_id=agent_id, + agent_version_no=agent_version_no, + evaluation_set_id=evaluation_set_id, + status="PENDING", + progress_total=total, + progress_done=0, + judge_model_id=judge_model_id, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + session.flush() + + es_row = ( + session.query(EvaluationSet.name) + .filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + ) + .scalar() + ) + + judge_model_name = None + if judge_model_id is not None: + judge_model_name = ( + session.query(ModelRecord.display_name) + .filter( + ModelRecord.model_id == judge_model_id, + ModelRecord.tenant_id == tenant_id, + ) + .scalar() + ) + + result = as_dict(rec) + result["evaluation_set_name"] = es_row + result["judge_model_name"] = judge_model_name + return result + + +def update_agent_evaluation_status( + agent_evaluation_id: int, + tenant_id: str, + status: str, + updated_by: Optional[str] = None, + error_message: Optional[str] = None, + score_overall: Optional[float] = None, + progress_done: Optional[int] = None, +) -> None: + updates: Dict[str, Any] = {"status": status, "updated_by": updated_by} + if error_message is not None: + updates["error_message"] = error_message + if score_overall is not None: + updates["score_overall"] = score_overall + if progress_done is not None: + updates["progress_done"] = progress_done + + with get_db_session() as session: + session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).update(updates, synchronize_session=False) + + +def get_agent_evaluation(agent_evaluation_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).first() + if not rec: + raise ValueError("agent evaluation not found") + result = as_dict(rec) + + evaluation_set_name = ( + session.query(EvaluationSet.name) + .filter( + EvaluationSet.evaluation_set_id == rec.evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + ) + .scalar() + ) + result["evaluation_set_name"] = evaluation_set_name + + agent_name = ( + session.query(AgentInfo.display_name, AgentInfo.name) + .filter( + AgentInfo.agent_id == rec.agent_id, + AgentInfo.tenant_id == tenant_id, + ) + .order_by(AgentInfo.version_no.desc()) + .first() + ) + if agent_name is not None: + display_name, programmatic_name = agent_name + result["agent_name"] = display_name or programmatic_name or "" + else: + result["agent_name"] = "" + + judge_model_name = None + if rec.judge_model_id is not None: + judge_model_name = ( + session.query(ModelRecord.display_name, ModelRecord.model_name) + .filter( + ModelRecord.model_id == rec.judge_model_id, + ModelRecord.tenant_id == tenant_id, + ) + .first() + ) + if judge_model_name is not None: + judge_display, judge_repo = judge_model_name + judge_model_name = judge_display or judge_repo + result["judge_model_name"] = judge_model_name + return result + + +def list_agent_evaluations_by_agent( + agent_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + # ``case((, 1), else_=0)`` translates to SQL + # ``CASE WHEN THEN 1 ELSE 0 END``, which is summed per row. + # The previous ``func.cast(, Integer)`` produced a single + # constant (the Python-side truthiness of the whole comparison), making + # every pass_count return 0. See CRITICAL-1 in the audit report. + pass_count_expr = func.sum( + sql_case( + (AgentEvaluationCase.pass_status == "pass", 1), + else_=0, + ) + ).label("pass_count") + + q = ( + session.query( + AgentEvaluation, + EvaluationSet.name.label("evaluation_set_name"), + ModelRecord.display_name.label("judge_model_name"), + func.count(AgentEvaluationCase.agent_evaluation_case_id).label("case_count"), + pass_count_expr, + ) + .outerjoin( + EvaluationSet, + (AgentEvaluation.evaluation_set_id == EvaluationSet.evaluation_set_id) + & (AgentEvaluation.tenant_id == EvaluationSet.tenant_id), + ) + .outerjoin( + ModelRecord, + (AgentEvaluation.judge_model_id == ModelRecord.model_id) + & (AgentEvaluation.tenant_id == ModelRecord.tenant_id), + ) + .outerjoin( + AgentEvaluationCase, + AgentEvaluation.agent_evaluation_id == AgentEvaluationCase.agent_evaluation_id, + ) + .filter( + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.agent_id == agent_id, + AgentEvaluation.delete_flag == "N", + ) + .group_by( + AgentEvaluation.agent_evaluation_id, + EvaluationSet.name, + ModelRecord.display_name, + ) + .order_by(AgentEvaluation.create_time.desc()) + .offset(offset) + .limit(limit) + ) + rows = q.all() + results = [] + for eval_row, evaluation_set_name, judge_model_name, case_count, pass_count in rows: + rec = as_dict(eval_row) + rec["evaluation_set_name"] = evaluation_set_name + rec["judge_model_name"] = judge_model_name + rec["case_count"] = case_count or 0 + rec["pass_count"] = pass_count or 0 + rec["fail_count"] = (case_count or 0) - (pass_count or 0) + results.append(rec) + return results + + +def create_agent_evaluation_cases( + tenant_id: str, + agent_evaluation_id: int, + set_cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> int: + with get_db_session() as session: + inserted = 0 + for sc in set_cases: + rec = AgentEvaluationCase( + tenant_id=tenant_id, + agent_evaluation_id=agent_evaluation_id, + evaluation_set_case_id=sc["evaluation_set_case_id"], + inputs=sc["inputs"], + label=sc["label"], + predict=None, + score=None, + reason=None, + status="PENDING", + error_message=None, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + inserted += 1 + session.flush() + return inserted + + +def update_agent_evaluation_case_result( + agent_evaluation_case_id: int, + tenant_id: str, + status: str, + predict: Optional[Dict[str, Any]] = None, + score: Optional[float] = None, + reason: Optional[str] = None, + error_message: Optional[str] = None, + pass_status: Optional[str] = None, + updated_by: Optional[str] = None, +) -> None: + """Update a case result. + + Storage policy: when a case is judged as ``pass`` (either via an explicit + ``pass_status="pass"`` argument or an observed ``score == 1``), the heavy + detail fields (``predict``, ``reason``, ``label.answer``) are cleared to + save space. Only failed cases retain the full detail for debugging. + """ + updates: Dict[str, Any] = {"status": status, "updated_by": updated_by} + + is_pass = (pass_status == "pass") or (score == 1) + + if not is_pass: + if predict is not None: + updates["predict"] = predict + if reason is not None: + updates["reason"] = reason + else: + # Pass case: trim heavy fields regardless of what was passed in. + updates["predict"] = None + updates["reason"] = None + updates["label"] = {"answer": ""} + + if score is not None: + updates["score"] = score + if pass_status is not None: + updates["pass_status"] = pass_status + if error_message is not None: + updates["error_message"] = error_message + + with get_db_session() as session: + rows = session.query(AgentEvaluationCase).filter( + AgentEvaluationCase.agent_evaluation_case_id == agent_evaluation_case_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ).update(updates, synchronize_session=False) + if rows == 0: + logger.warning( + "agent_evaluation_case not updated: id=%s, tenant=%s", + agent_evaluation_case_id, + tenant_id, + ) + + +def list_agent_evaluation_cases( + agent_evaluation_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(AgentEvaluationCase) + .filter( + AgentEvaluationCase.agent_evaluation_id == agent_evaluation_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ) + .order_by(AgentEvaluationCase.agent_evaluation_case_id.asc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_agent_evaluation_case(agent_evaluation_case_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(AgentEvaluationCase).filter( + AgentEvaluationCase.agent_evaluation_case_id == agent_evaluation_case_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ).first() + if not rec: + raise ValueError("agent evaluation case not found") + return as_dict(rec) + + +def soft_delete_agent_evaluation( + agent_evaluation_id: int, + tenant_id: str, + deleted_by: str, +) -> None: + """Soft-delete an evaluation run by setting delete_flag='Y'. + + Raises ``ValueError`` when the run is not found or has already been deleted. + """ + with get_db_session() as session: + rows = session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).update( + {"delete_flag": "Y", "updated_by": deleted_by}, + synchronize_session=False, + ) + if rows == 0: + raise ValueError("agent evaluation not found or already deleted") diff --git a/backend/database/client.py b/backend/database/client.py index e095c5636..28cc483ae 100644 --- a/backend/database/client.py +++ b/backend/database/client.py @@ -330,8 +330,12 @@ def as_dict(obj): for c in class_mapper(obj.__class__).columns: value = getattr(obj, c.key) # Convert datetime to ISO format string for JSON serialization + # Naive datetime (TIMESTAMP WITHOUT TIME ZONE) is treated as local time. + # Append 'Z' so browsers interpret it as UTC-equivalent local time + # without shifting. if isinstance(value, datetime): - result[c.key] = value.isoformat() + iso = value.isoformat() + result[c.key] = iso if value.tzinfo else iso + "Z" else: result[c.key] = value return result diff --git a/backend/database/db_models.py b/backend/database/db_models.py index ca2f898e1..b0d6d710f 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -1373,5 +1373,164 @@ class A2AArtifact(SimpleTableBase): extensions = Column(JSON, doc="Extension URI list") # Timestamp - create_time = Column(TIMESTAMP( - timezone=False), server_default=func.now(), doc="Artifact creation timestamp") + create_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), doc="Artifact creation timestamp") + + +# ----------------------------------------------------------------------------- +# Agent Evaluation (offline) tables +# ----------------------------------------------------------------------------- +class EvaluationSet(TableBase): + """Evaluation set metadata.""" + + __tablename__ = "evaluation_set_t" + __table_args__ = {"schema": SCHEMA} + + evaluation_set_id = Column( + BigInteger, + Sequence("evaluation_set_t_evaluation_set_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + name = Column(String(255), nullable=False, doc="Evaluation set name") + description = Column(Text, doc="Evaluation set description") + + source_filename = Column(String(255), doc="Original uploaded filename") + case_count = Column(Integer, default=0, doc="Total number of cases") + + __table_args__ = ( + Index("ix_eval_set_tenant_id", "tenant_id"), + Index("ix_eval_set_name", "tenant_id", "name"), + {"schema": SCHEMA}, + ) + + +class EvaluationSetCase(TableBase): + """Evaluation cases belonging to a set.""" + + __tablename__ = "evaluation_set_case_t" + __table_args__ = {"schema": SCHEMA} + + evaluation_set_case_id = Column( + BigInteger, + Sequence("evaluation_set_case_t_evaluation_set_case_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + evaluation_set_id = Column(BigInteger, nullable=False, doc="Evaluation set id") + + case_id = Column(String(128), doc="External case_id from JSONL (optional)") + + inputs = Column(JSONB, nullable=False, doc="Case inputs JSON") + label = Column(JSONB, nullable=False, doc="Case label JSON") + + order_no = Column(Integer, default=0, doc="Case order in the set") + + __table_args__ = ( + Index("ix_eval_set_case_set_id", "evaluation_set_id"), + Index("ix_eval_set_case_tenant_id", "tenant_id"), + {"schema": SCHEMA}, + ) + + +class AgentEvaluation(TableBase): + """An evaluation run for a specific agent and evaluation set.""" + + __tablename__ = "agent_evaluation_t" + __table_args__ = {"schema": SCHEMA} + + agent_evaluation_id = Column( + BigInteger, + Sequence("agent_evaluation_t_agent_evaluation_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + + agent_id = Column(Integer, nullable=False, doc="Agent id") + agent_version_no = Column(Integer, nullable=False, doc="Published agent version_no used for evaluation") + + evaluation_set_id = Column(BigInteger, nullable=False, doc="Evaluation set id") + + status = Column( + String(30), + nullable=False, + default="PENDING", + doc="Run status: PENDING/RUNNING/COMPLETED/FAILED", + ) + + progress_total = Column(Integer, default=0, doc="Total cases") + progress_done = Column(Integer, default=0, doc="Completed cases") + + judge_model_id = Column( + Integer, + doc=( + "Model id used by the judge. Persisted so the background worker can " + "recover it after restart and the frontend can resolve judge_model_name." + ), + ) + + score_overall = Column(Float, doc="Overall score (0-1)") + + error_message = Column(Text, doc="Failure reason") + + __table_args__ = ( + Index("ix_agent_eval_tenant_id", "tenant_id"), + Index("ix_agent_eval_agent_id", "tenant_id", "agent_id"), + Index("ix_agent_eval_set_id", "tenant_id", "evaluation_set_id"), + Index("ix_agent_eval_judge_model_id", "tenant_id", "judge_model_id"), + {"schema": SCHEMA}, + ) + + +class AgentEvaluationCase(TableBase): + """Per-case evaluation details within an evaluation run.""" + + __tablename__ = "agent_evaluation_case_t" + __table_args__ = {"schema": SCHEMA} + + agent_evaluation_case_id = Column( + BigInteger, + Sequence("agent_evaluation_case_t_agent_evaluation_case_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + + agent_evaluation_id = Column(BigInteger, nullable=False, doc="Evaluation run id") + evaluation_set_case_id = Column(BigInteger, nullable=False, doc="Evaluation set case id") + + inputs = Column(JSONB, nullable=False, doc="Case inputs snapshot (query only for pass cases)") + label = Column(JSONB, nullable=False, doc="Case label snapshot (cleared to {answer:''} for pass cases)") + predict = Column(JSONB, doc="Predict JSON (answer/raw); NULL for pass cases") + + score = Column(Float, doc="Case score (0-1)") + reason = Column(Text, doc="Judge reason; NULL for pass cases") + pass_status = Column( + String(16), + doc="Judge result: pass / fail. Pass cases have predict/reason/label.answer cleared to save space.", + ) + + status = Column( + String(30), + nullable=False, + default="PENDING", + doc="Case status: PENDING/RUNNING/COMPLETED/FAILED", + ) + error_message = Column(Text, doc="Per-case failure reason") + + __table_args__ = ( + Index("ix_agent_eval_case_eval_id", "agent_evaluation_id"), + Index("ix_agent_eval_case_tenant_id", "tenant_id"), + Index("ix_agent_eval_case_pass_status", "tenant_id", "agent_evaluation_id", "pass_status"), + {"schema": SCHEMA}, + ) diff --git a/backend/database/evaluation_set_db.py b/backend/database/evaluation_set_db.py new file mode 100644 index 000000000..ad2cbc88b --- /dev/null +++ b/backend/database/evaluation_set_db.py @@ -0,0 +1,149 @@ +import json +import logging +from typing import Any, Dict, List, Optional, Tuple + +from database.client import as_dict, filter_property, get_db_session +from database.db_models import EvaluationSet, EvaluationSetCase + +logger = logging.getLogger("evaluation_set_db") + + +def create_evaluation_set( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + created_by: Optional[str], +) -> Dict[str, Any]: + with get_db_session() as session: + rec = EvaluationSet( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + session.flush() + return as_dict(rec) + + +def update_evaluation_set_case_count(evaluation_set_id: int, case_count: int, updated_by: Optional[str] = None) -> None: + with get_db_session() as session: + session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.delete_flag == "N", + ).update({"case_count": case_count, "updated_by": updated_by}, synchronize_session=False) + + +def list_evaluation_sets(tenant_id: str, limit: int = 50, offset: int = 0) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSet) + .filter(EvaluationSet.tenant_id == tenant_id, EvaluationSet.delete_flag == "N") + .order_by(EvaluationSet.update_time.desc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_evaluation_set(evaluation_set_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + EvaluationSet.delete_flag == "N", + ).first() + if not rec: + raise ValueError("evaluation set not found") + return as_dict(rec) + + +def insert_evaluation_set_cases( + tenant_id: str, + evaluation_set_id: int, + cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> int: + """Insert cases. Each case must have: inputs(dict), label(dict), optional case_id(str). + + Returns inserted count. + """ + with get_db_session() as session: + inserted = 0 + for i, c in enumerate(cases): + rec = EvaluationSetCase( + tenant_id=tenant_id, + evaluation_set_id=evaluation_set_id, + case_id=c.get("case_id"), + inputs=c["inputs"], + label=c["label"], + order_no=int(c.get("order_no", i)), + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + inserted += 1 + session.flush() + return inserted + + +def list_evaluation_set_cases( + evaluation_set_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSetCase) + .filter( + EvaluationSetCase.evaluation_set_id == evaluation_set_id, + EvaluationSetCase.tenant_id == tenant_id, + EvaluationSetCase.delete_flag == "N", + ) + .order_by(EvaluationSetCase.order_no.asc(), EvaluationSetCase.evaluation_set_case_id.asc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_evaluation_set_cases_all(evaluation_set_id: int, tenant_id: str) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSetCase) + .filter( + EvaluationSetCase.evaluation_set_id == evaluation_set_id, + EvaluationSetCase.tenant_id == tenant_id, + EvaluationSetCase.delete_flag == "N", + ) + .order_by(EvaluationSetCase.order_no.asc(), EvaluationSetCase.evaluation_set_case_id.asc()) + ) + return [as_dict(x) for x in q.all()] + + +def soft_delete_evaluation_set( + evaluation_set_id: int, + tenant_id: str, + deleted_by: str, +) -> None: + """Soft-delete an evaluation set by setting delete_flag='Y'. + + Raises ``ValueError`` when the set is not found or has already been deleted. + """ + with get_db_session() as session: + rows = session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + EvaluationSet.delete_flag == "N", + ).update( + {"delete_flag": "Y", "updated_by": deleted_by}, + synchronize_session=False, + ) + if rows == 0: + raise ValueError("evaluation set not found or already deleted") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b8f51dd4c..9481cdbc7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "openjiuwen>=0.1.0", "pydantic-settings>=2.0.0", "python-docx>=1.1.0", + "xlrd>=2.0.1", ] [project.optional-dependencies] @@ -38,6 +39,8 @@ data-process = [ "flower>=2.0.1", "nest_asyncio>=1.5.6", "unstructured[csv,docx,pdf,pptx,xlsx,md]==0.18.14", + "openpyxl>=3.1.5", + "xlrd>=2.0.1", "huggingface_hub>=0.30.0,<1.0" ] test = [ diff --git a/backend/services/agent_evaluation_service.py b/backend/services/agent_evaluation_service.py new file mode 100644 index 000000000..49c199623 --- /dev/null +++ b/backend/services/agent_evaluation_service.py @@ -0,0 +1,866 @@ +import asyncio +import io +import json +import logging +from statistics import mean +from typing import Any, Dict, List, Optional, Tuple + +from adapters.exception import JiuwenSDKError, JiuwenSDKUnavailableError + +try: + from adapters.jiuwen_sdk_adapter import JiuwenSDKAdapter +except ModuleNotFoundError: + JiuwenSDKAdapter = None # type: ignore[assignment, misc] +from consts.model import AgentRequest +from database.agent_evaluation_db import ( + create_agent_evaluation, + create_agent_evaluation_cases, + get_agent_evaluation, + list_agent_evaluation_cases, + list_agent_evaluations_by_agent, + soft_delete_agent_evaluation, + update_agent_evaluation_case_result, + update_agent_evaluation_status, +) +from database.evaluation_set_db import get_evaluation_set_cases_all +from services.evaluation_set_service import resolve_latest_published_version_no +from services.agent_service import prepare_agent_run +from utils.thread_utils import pool +from openpyxl import Workbook +from openpyxl.styles import Font, PatternFill, Alignment +import re + +logger = logging.getLogger(__name__) + + +# Log records emitted during SDK invocations may bleed into the ``reason`` +# field as ``"[HH:MM:SS LEVEL logger_name] {...payload...}"``. Extract the +# embedded JSON ``reason`` from those polluted strings so the report shows the +# judge's actual explanation rather than the surrounding log envelope. +_LOG_PREFIX_RE = re.compile( + r"(?:" + # Short form: ``[HH:MM:SS LEVEL logger] `` + r"\[(\d{2}:\d{2}:\d{2}|\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\w+\s+[\w.]+\]" + # Long form: ``YYYY-MM-DD HH:MM:SS | logger_name | trace_id | LEVEL | `` + r"|(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})" + r"(?:\s*\|\s*[\w.]+){1,3}\s*\|?" + r")\s*" +) +_MARKDOWN_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL) + + +def _extract_clean_reason(raw: Any) -> str: + """Best-effort extraction of the judge model's reason text. + + The ``reason`` column persisted by the judge pipeline can take three + shapes depending on which step produced the value: + + 1. The plain judge verdict: ``"pass"`` or ``"fail"``. + 2. The judge's free-form explanation (when the SDK upgrades to return it). + 3. A captured log record of the form + ``"[HH:MM:SS LEVEL llm] {event_id:..., response_content: '```json\\n{result, reason}\\n```', ...}"`` + where the judge JSON is nested inside ``response_content``. + + This helper unwraps case 3 by parsing the outer JSON, pulling + ``response_content``, stripping the markdown code fence, and returning the + inner ``reason`` field. Cases 1 and 2 are returned as-is. + """ + if raw is None: + return "" + text = str(raw).strip() + if not text: + return "" + + # Strip leading "[HH:MM:SS LEVEL logger] " log envelope(s) that may have + # been prepended one or more times by the root-logger StreamHandler when + # the SDK redirected ``response`` into a log call. + stripped = _LOG_PREFIX_RE.sub("", text).strip() + if not stripped: + return text + + # Try the full string first; if it is not parseable JSON, fall back to + # the inner match. This handles both well-formed outer objects and strings + # that have stray characters around a single JSON object. + parsed: Optional[Dict[str, Any]] = None + try: + parsed = json.loads(stripped) + except (ValueError, TypeError): + match = _JSON_OBJECT_RE.search(stripped) + if match: + try: + parsed = json.loads(match.group(0)) + except (ValueError, TypeError): + parsed = None + + if not isinstance(parsed, dict): + # Not a JSON envelope at all — leave the stripped text in place so + # plain "pass"/"fail" and free-form explanations still render. + return stripped + + response_content = parsed.get("response_content") + if isinstance(response_content, str): + # Strip the ```json ... ``` fence the judge model wraps its verdict in. + fence_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", response_content, re.DOTALL) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + if isinstance(inner, dict) and isinstance(inner.get("reason"), str): + return inner["reason"].strip() + except (ValueError, TypeError): + pass + + reason_field = parsed.get("reason") + if isinstance(reason_field, str): + return reason_field.strip() + + return stripped + + +def _iter_log_envelopes(text: str): + """Yield ``(log_prefix, json_payload)`` for every + ``"[HH:MM:SS LEVEL logger] {...}"`` envelope in *text*. + + The SDK's root logger may emit several records per evaluation + (``"OpenAI API response received"``, ``"Before parse response with output + parser"``, ``"Before parse content with parser"``, ...). Some of them carry + the judge's verdict inside their JSON payload, others do not. The caller + picks the envelope that contains ``result`` + ``reason``. + """ + cursor = 0 + while cursor < len(text): + # Skip any whitespace / newlines separating envelopes. + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + if cursor >= len(text): + break + match = _LOG_PREFIX_RE.match(text, cursor) + if not match: + break + prefix_end = match.end() + # Find the balanced JSON object that begins at prefix_end. If the + # payload is not valid JSON (e.g. truncated) we still want to advance + # the cursor past the broken prefix so the loop can find the next + # envelope. + depth = 0 + payload_start = -1 + payload_end = -1 + in_string = False + escape = False + for idx in range(prefix_end, len(text)): + ch = text[idx] + if in_string: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + continue + if ch == "{": + if depth == 0: + payload_start = idx + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0 and payload_start != -1: + payload_end = idx + 1 + break + if payload_start == -1 or payload_end == -1: + # No balanced JSON after this prefix — skip the prefix and keep + # scanning for the next log envelope. + cursor = prefix_end + continue + yield text[cursor:prefix_end], text[payload_start:payload_end] + cursor = payload_end + + +def _reason_from_json_envelope(payload: str) -> Optional[str]: + """Pull the judge's ``reason`` text out of one log-envelope JSON payload. + + Tries the following strategies, in order: + + 1. ``payload["response_content"]`` wrapped in a markdown code fence — the + shape produced by the judge LLM's first ``llm_call_end`` event. + 2. ``payload["response_content"]`` being a plain JSON string. + 3. ``payload["response"]["choices"][0]["message"]["content"]`` — the + OpenAI ``ChatCompletion`` repr captured by the request-side log. + 4. ``payload["reason"]`` at the top level — fallback. + """ + try: + data = json.loads(payload) + except (ValueError, TypeError): + return None + if not isinstance(data, dict): + return None + + response_content = data.get("response_content") + if isinstance(response_content, str): + fence_match = _MARKDOWN_FENCE_RE.search(response_content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + # ``response_content`` may be the raw JSON string itself. + try: + inner = json.loads(response_content) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + + # OpenAI ChatCompletion repr — unwrap the ``message.content`` field. + # The SDK captures the OpenAI response by calling ``repr(chat_completion)`` + # so the ``response`` field stores a Python repr (single-quoted strings, + # ``\n`` literal characters, ...) rather than a JSON document. Use a + # targeted regex to pull the ``content='...'`` blob out of the repr, then + # unwrap the markdown fence the judge model wraps its verdict in. + # The field can live either at the top level (``payload["response"]``) or + # nested under ``payload["metadata"]["response"]`` depending on which SDK + # code path emitted the log record. + response = data.get("response") + if response is None: + metadata = data.get("metadata") + if isinstance(metadata, dict): + response = metadata.get("response") + if isinstance(response, str) and "ChatCompletion" in response: + # Use a non-greedy match that ends at the next ``, refusal`` token. + # The repr is single-quoted and the ``content`` field is followed by + # ``, refusal=`` so this anchor avoids trying to balance quotes in + # the repr string — which is fragile once ``\n`` has been unescaped + # by ``json.loads``. + content_match = re.search( + r"ChatCompletionMessage\(content='(.*?)', refusal=", + response, + re.DOTALL, + ) + if content_match: + # The captured group is the raw repr body; turn the literal + # ``\n`` / ``\"`` / ``\\`` escape sequences back into real + # characters, then look for the markdown code fence inside. + try: + content = content_match.group(1).encode("utf-8").decode("unicode_escape") + except UnicodeDecodeError: + content = content_match.group(1) + if isinstance(content, str): + fence_match = _MARKDOWN_FENCE_RE.search(content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + # The judge verdict may live directly on ``response`` if the + # repr was stored without the ``metadata.response`` wrapping. + try: + inner = json.loads(content) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + elif isinstance(response, str): + # Some SDKs serialise the response as a plain JSON string. + try: + response_obj = json.loads(response) + except (ValueError, TypeError): + response_obj = None + if isinstance(response_obj, dict): + choices = response_obj.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + content = message.get("content") + if isinstance(content, str): + fence_match = _MARKDOWN_FENCE_RE.search(content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + + top_reason = data.get("reason") + if isinstance(top_reason, str): + return top_reason.strip() + + return None + + +def _extract_clean_reason_v2(raw: Any) -> str: + """Walk every log envelope in *raw* and pull out the judge reason. + + See ``_extract_clean_reason`` for the original entry point; this helper + handles the noisy multi-envelope shape where several SDK log records + (with the verdict buried in the first one) are concatenated. + """ + if raw is None: + return "" + text = str(raw).strip() + if not text: + return "" + + # Fast path: the string is already a single JSON object (no log envelope) + # containing the judge verdict. + standalone = _reason_from_json_envelope(text) + if standalone is not None: + return standalone + + # Slow path: walk every ``[time LEVEL logger] {...}`` envelope and take + # the first one whose payload yields a reason. + for _, payload in _iter_log_envelopes(text): + reason = _reason_from_json_envelope(payload) + if reason is not None: + return reason + + # No JSON envelope produced a reason. Return the input with leading log + # prefixes stripped so plain "pass" / "fail" and free-form explanations + # render without the surrounding noise. + stripped = text + while True: + match = _LOG_PREFIX_RE.match(stripped) + if not match: + break + stripped = stripped[match.end():].lstrip() + return stripped or text + + +def _is_llm_related_error(exc: Exception) -> bool: + """Check if an exception is related to LLM API calls.""" + error_str = str(exc).lower() + llm_keywords = [ + "openai", "api", "llm", "model", "completion", "chat", + "connection", "timeout", "rate limit", "authentication", + "invalid response", "async invoke", "jiuwen", "sdk", + "schedule new futures", "interpreter shutdown", + ] + return any(keyword in error_str for keyword in llm_keywords) + + +def _generate_friendly_error_message(exc: Exception, default_msg: str) -> str: + """Generate a friendly error message for LLM-related errors using another LLM.""" + if not _is_llm_related_error(exc): + return default_msg + + # Only call LLM for LLM-related errors + try: + import os + from openai import AsyncOpenAI + + client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + error_snippet = str(exc)[:500] + + response = asyncio.run( + client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "system", + "content": ( + "You are a helpful assistant that explains technical errors " + "to end users in simple Chinese. Be concise and actionable. " + "Focus on what the user can do to fix the problem." + ), + }, + { + "role": "user", + "content": ( + f"以下是智能体评估时的错误信息，请用简单的中文解释给用户，并给出建议的操作：\n\n{error_snippet}" + ), + }, + ], + max_tokens=200, + temperature=0.3, + ) + ) + return response.choices[0].message.content or default_msg + except Exception as llm_exc: + logger.warning("Failed to generate friendly error message: %r", llm_exc) + return default_msg + +logger = logging.getLogger("agent_evaluation_service") + + +def _make_background_done_callback( + tenant_id: str, + user_id: str, + agent_evaluation_id: int, +): + """Return a done-callback that marks the run FAILED if the worker raised.""" + + def callback(future): + exc = future.exception() + if exc is not None: + logger.exception( + "Background evaluation run failed (id=%s): %r", + agent_evaluation_id, + exc, + ) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + try: + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="FAILED", + updated_by=user_id, + error_message=friendly_msg, + ) + except Exception as update_exc: + logger.error( + "Failed to write FAILED status for evaluation id=%s: %r", + agent_evaluation_id, + update_exc, + ) + + return callback + + +def _build_case_for_jiuwen( + inputs: Dict[str, Any], + label: Dict[str, Any], +) -> Dict[str, Any]: + # Jiuwen Case(inputs, label) will accept arbitrary keys, but we keep the stable schema + return { + "inputs": { + "query": inputs.get("query", ""), + }, + "label": {"answer": label.get("answer", "")}, + } + + +async def _run_agent_to_final_answer( + agent_id: int, + tenant_id: str, + user_id: str, + query: str, + version_no: int, +) -> str: + """Run agent once and aggregate final answer text.""" + + # Build a single-turn AgentRequest. We do not persist messages for offline eval. + agent_request = AgentRequest( + query=query, + conversation_id=0, + history=None, + minio_files=None, + agent_id=agent_id, + version_no=version_no, + is_debug=True, + ) + + agent_run_info, memory_context = await prepare_agent_run( + agent_request=agent_request, + user_id=user_id, + tenant_id=tenant_id, + allow_memory_search=False, + ) + + # Stream chunks from core agent runner and extract final_answer content. + from nexent.core.agents.run_agent import agent_run + + final_answer_parts: List[str] = [] + async for chunk in agent_run(agent_run_info): + try: + if isinstance(chunk, str): + data = json.loads(chunk) + if isinstance(data, dict) and data.get("type") == "final_answer": + content = data.get("content") + if isinstance(content, str): + final_answer_parts.append(content) + except Exception: + continue + + return "".join(final_answer_parts).strip() + + +def create_agent_evaluation_run_impl( + tenant_id: str, + user_id: str, + agent_id: int, + evaluation_set_id: int, + judge_model_id: int, +) -> Dict[str, Any]: + set_cases = get_evaluation_set_cases_all(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + if not set_cases: + raise ValueError("evaluation set has no cases") + + agent_version_no = resolve_latest_published_version_no(agent_id=agent_id, tenant_id=tenant_id) + + run = create_agent_evaluation( + tenant_id=tenant_id, + agent_id=agent_id, + agent_version_no=agent_version_no, + evaluation_set_id=evaluation_set_id, + total=len(set_cases), + judge_model_id=judge_model_id, + created_by=user_id, + ) + + create_agent_evaluation_cases( + tenant_id=tenant_id, + agent_evaluation_id=run["agent_evaluation_id"], + set_cases=set_cases, + created_by=user_id, + ) + + # Kick off background execution — attach a callback so that any uncaught + # exception inside the worker thread is surfaced as a RUN→FAILED transition + # instead of silently leaving the run stuck at RUNNING forever. + future = pool.submit( + execute_agent_evaluation_run, + tenant_id, + user_id, + run["agent_evaluation_id"], + judge_model_id, + ) + future.add_done_callback( + _make_background_done_callback( + tenant_id, + user_id, + run["agent_evaluation_id"], + ) + ) + + return run + + +def execute_agent_evaluation_run( + tenant_id: str, + user_id: str, + agent_evaluation_id: int, + judge_model_id: Optional[int] = None, +): + """Background execution entry point (sync). + + ``judge_model_id`` is normally supplied by ``submit`` when the run is + first created. If the worker process restarts mid-run and the queued + payload is lost, we fall back to whatever was persisted on the run + record so the run can still recover. + """ + try: + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + ) + + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + agent_id = int(run["agent_id"]) + agent_version_no = int(run["agent_version_no"]) + if judge_model_id is None: + judge_model_id = run.get("judge_model_id") + if judge_model_id is None: + raise ValueError("judge_model_id is required but neither passed in nor persisted on the run") + judge_model_id = int(judge_model_id) + + if JiuwenSDKAdapter is None: + raise JiuwenSDKUnavailableError("Jiuwen SDK adapter is unavailable. Please install optional dependencies for openjiuwen.") + + adapter = JiuwenSDKAdapter(model_id=judge_model_id, tenant_id=tenant_id) + + cases = list_agent_evaluation_cases(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id, limit=100000, offset=0) + scores: List[float] = [] + + for idx, c in enumerate(cases, start=1): + case_id = c["agent_evaluation_case_id"] + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + ) + + inputs = c["inputs"] or {} + label = c["label"] or {} + + query = inputs.get("query", "") + + try: + answer_text = asyncio.run( + _run_agent_to_final_answer( + agent_id=agent_id, + tenant_id=tenant_id, + user_id=user_id, + query=query, + version_no=agent_version_no, + ) + ) + + predict = {"answer": answer_text} + + # Judge with openjiuwen LLM-as-judge metric (binary 1/0) + expected = (label.get("answer") or "").strip() + + score, reason = adapter.evaluate_semantic_consistency( + question=query, + expected_answer=expected, + model_answer=answer_text, + ) + + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="COMPLETED", + predict=predict, + score=score, + pass_status="pass" if score == 1 else "fail", + reason=reason, + updated_by=user_id, + ) + + scores.append(score) + + except Exception as exc: + logger.exception("Evaluation case failed: %r", exc) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="FAILED", + pass_status="fail", + error_message=friendly_msg, + updated_by=user_id, + ) + + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + progress_done=idx, + ) + + overall = float(mean(scores)) if scores else 0.0 + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="COMPLETED", + updated_by=user_id, + score_overall=overall, + progress_done=len(cases), + ) + + except Exception as exc: + logger.exception("Evaluation run failed: %r", exc) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="FAILED", + updated_by=user_id, + error_message=friendly_msg, + ) + + +def get_agent_evaluation_run_impl(agent_evaluation_id: int, tenant_id: str) -> Dict[str, Any]: + return get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + + +def list_agent_evaluations_by_agent_impl( + agent_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_agent_evaluations_by_agent(agent_id=agent_id, tenant_id=tenant_id, limit=limit, offset=offset) + + +def list_agent_evaluation_cases_impl( + agent_evaluation_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_agent_evaluation_cases(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id, limit=limit, offset=offset) + + +def delete_agent_evaluation_run_impl( + agent_evaluation_id: int, + tenant_id: str, + user_id: str, +) -> None: + """Soft-delete an evaluation run. + + Only the creator can delete the run. Tenant admins are handled at the + service layer when permissions are extended. + """ + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + if run.get("created_by") != user_id: + raise ValueError("Only the creator can delete this evaluation run") + soft_delete_agent_evaluation(agent_evaluation_id, tenant_id, user_id) + + +def generate_agent_evaluation_report_impl( + agent_evaluation_id: int, + tenant_id: str, +) -> Tuple[bytes, int]: + """Build the evaluation report workbook. + + Returns ``(excel_bytes, failed_count)``. ``failed_count`` lets the API + layer name the downloaded file ``_all.xlsx`` vs ``_failed.xlsx`` so a + clean run does not download a file whose name suggests failure. + """ + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + all_cases = list_agent_evaluation_cases( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + limit=100000, + offset=0, + ) + + failed_cases = [ + c for c in all_cases + if c.get("status") == "FAILED" + or c.get("score") == 0 + or c.get("pass_status") == "fail" + ] + pass_count = sum( + 1 for c in all_cases + if c.get("status") != "FAILED" and c.get("score") == 1 + ) + fail_count = len(failed_cases) + total = len(all_cases) + pass_rate = f"{(pass_count / total * 100):.2f}%" if total else "-" + + wb = Workbook() + ws_summary = wb.active + ws_summary.title = "概要" + + header_font = Font(bold=True) + header_fill = PatternFill(start_color="E8F4FD", end_color="E8F4FD", fill_type="solid") + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + # Excel's default cell style is right-aligned for numbers; force every + # value cell to render flush-left so mixed types (numbers, percentages, + # timestamps, error messages) line up consistently in the report. + left_align = Alignment(horizontal="left", vertical="center", wrap_text=True) + + # The three "ID" rows (评估ID / Agent ID / Evaluation Set ID) expose the + # human-readable name (judge model / agent / evaluation set) instead of + # the opaque numeric ID. Falls back to the raw ID when the name lookup + # returned nothing, so the cell is never blank. + judge_model_label = run.get("judge_model_name") or run.get("judge_model_id") or "-" + agent_label = run.get("agent_name") or run.get("agent_id") or "-" + evaluation_set_label = run.get("evaluation_set_name") or run.get("evaluation_set_id") or "-" + + summary_rows = [ + ("测评模型", judge_model_label), + ("智能体名称", agent_label), + ("智能体版本", run.get("agent_version_no", "")), + ("评测集名称", evaluation_set_label), + ("状态", run.get("status", "")), + ("用例总数", total), + ("通过用例数", pass_count), + ("失败用例数", fail_count), + ("通过率", pass_rate), + ("综合得分", f"{run.get('score_overall', 0):.4f}" if run.get('score_overall') is not None else "-"), + ("错误信息", run.get("error_message") or "-"), + ("创建时间", run.get("create_time") or "-"), + ("报告范围", "失败用例"), + ] + + ws_summary.append(["字段", "值"]) + for cell in ws_summary[1]: + cell.font = header_font + cell.fill = header_fill + cell.alignment = center + + # Pre-build the data rows so we can apply the left alignment to every + # value cell in one pass (rows start at index 3 after the header in row 1 + # and the empty default row that ``append`` does not create; openpyxl + # places our header at row 1 and the data rows begin at row 2). + for field, value in summary_rows: + ws_summary.append([field, value]) + + for row_idx in range(2, 2 + len(summary_rows)): + ws_summary.cell(row=row_idx, column=1).alignment = left_align + ws_summary.cell(row=row_idx, column=2).alignment = left_align + + ws_summary.column_dimensions["A"].width = 24 + ws_summary.column_dimensions["B"].width = 60 + + ws_cases = wb.create_sheet("失败用例") + + # All columns in the failed-cases sheet are localized to Chinese to match + # the rest of the report. + case_headers = [ + "用例ID", + "问题", + "期望答案", + "模型答案", + "得分", + "评测理由", + "用例状态", + "错误信息", + ] + ws_cases.append(case_headers) + for cell in ws_cases[1]: + cell.font = header_font + cell.fill = header_fill + cell.alignment = center + + for c in failed_cases: + inputs = c.get("inputs") or {} + label = c.get("label") or {} + predict = c.get("predict") or {} + score = c.get("score") + if score == 0: + score_str = "0.0000" + elif isinstance(score, (int, float)): + score_str = f"{score:.4f}" + elif score is not None: + score_str = str(score) + else: + score_str = "-" + + # error_message is only meaningful when the case is FAILED + # (execution crashed before/during model call). For scored cases + # that simply scored 0 it is usually empty. + error_msg = c.get("error_message") or "" + + ws_cases.append([ + c.get("agent_evaluation_case_id", ""), + inputs.get("query", ""), + label.get("answer", ""), + predict.get("answer", ""), + score_str, + _extract_clean_reason_v2(c.get("reason")), + c.get("status", ""), + error_msg, + ]) + + # Apply left alignment + wrap to every data cell so the row stays readable + # when any column overflows the configured width. + if failed_cases: + last_data_row = 1 + len(failed_cases) + for row in ws_cases.iter_rows( + min_row=2, max_row=last_data_row, min_col=1, max_col=len(case_headers) + ): + for cell in row: + cell.alignment = Alignment(horizontal="left", vertical="top", wrap_text=True) + + widths = {"A": 14, "B": 50, "C": 40, "D": 40, "E": 10, "F": 50, "G": 12, "H": 40} + for col, w in widths.items(): + ws_cases.column_dimensions[col].width = w + + out = io.BytesIO() + wb.save(out) + return out.getvalue(), fail_count diff --git a/backend/services/evaluation_set_service.py b/backend/services/evaluation_set_service.py new file mode 100644 index 000000000..888ea4e47 --- /dev/null +++ b/backend/services/evaluation_set_service.py @@ -0,0 +1,193 @@ +import json +import logging +import uuid +from typing import Any, Dict, List, Optional, Tuple + +from consts.model import AgentRequest +from database.agent_version_db import query_version_list +from database.client import get_db_session +from database.db_models import AgentEvaluation +from database.evaluation_set_db import ( + create_evaluation_set, + get_evaluation_set, + get_evaluation_set_cases_all, + insert_evaluation_set_cases, + list_evaluation_set_cases, + list_evaluation_sets, + soft_delete_evaluation_set, + update_evaluation_set_case_count, +) + +logger = logging.getLogger("evaluation_set_service") + + +def _validate_single_turn_case(obj: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(obj, dict): + raise ValueError("case must be an object") + + inputs = obj.get("inputs") + label = obj.get("label") + + if not isinstance(inputs, dict): + raise ValueError("inputs must be an object") + if not isinstance(label, dict): + raise ValueError("label must be an object") + + query = inputs.get("query") + if not isinstance(query, str) or not query.strip(): + raise ValueError("inputs.query must be a non-empty string") + + context = inputs.get("context") + if context is not None and not isinstance(context, str): + raise ValueError("inputs.context must be a string when provided") + + answer = label.get("answer") + if not isinstance(answer, str) or not answer.strip(): + raise ValueError("label.answer must be a non-empty string") + + case_id = obj.get("case_id") + if case_id is not None and not isinstance(case_id, str): + raise ValueError("case_id must be a string when provided") + + return { + "case_id": case_id, + "inputs": {"query": query, **({"context": context} if context is not None else {})}, + "label": {"answer": answer}, + } + + +def parse_jsonl_cases(jsonl_text: str) -> List[Dict[str, Any]]: + cases: List[Dict[str, Any]] = [] + for idx, line in enumerate((jsonl_text or "").splitlines(), start=1): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception as exc: + raise ValueError(f"Invalid JSON at line {idx}: {exc}") from exc + + normalized = _validate_single_turn_case(obj) + normalized["order_no"] = len(cases) + cases.append(normalized) + + if not cases: + raise ValueError("JSONL contains no cases") + + return cases + + +def create_evaluation_set_from_cases( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> Dict[str, Any]: + if not cases: + raise ValueError("cases is empty") + + meta = create_evaluation_set( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + created_by=created_by, + ) + + inserted = insert_evaluation_set_cases( + tenant_id=tenant_id, + evaluation_set_id=meta["evaluation_set_id"], + cases=cases, + created_by=created_by, + ) + + update_evaluation_set_case_count(meta["evaluation_set_id"], inserted, updated_by=created_by) + meta["case_count"] = inserted + return meta + + +def create_evaluation_set_from_jsonl( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + jsonl_text: str, + created_by: Optional[str], +) -> Dict[str, Any]: + cases = parse_jsonl_cases(jsonl_text) + return create_evaluation_set_from_cases( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + cases=cases, + created_by=created_by, + ) + + +def list_evaluation_sets_impl(tenant_id: str, limit: int = 50, offset: int = 0) -> List[Dict[str, Any]]: + return list_evaluation_sets(tenant_id=tenant_id, limit=limit, offset=offset) + + +def get_evaluation_set_impl(evaluation_set_id: int, tenant_id: str) -> Dict[str, Any]: + return get_evaluation_set(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + + +def list_evaluation_set_cases_impl( + evaluation_set_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_evaluation_set_cases( + evaluation_set_id=evaluation_set_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + + +def resolve_latest_published_version_no(agent_id: int, tenant_id: str) -> int: + """Return latest published version_no for the agent. + + Raises ValueError if no published version exists. + """ + versions = query_version_list(agent_id, tenant_id) + if not versions: + raise ValueError("agent has no published versions") + # query_version_list returns latest first in existing code usage + latest = versions[0].get("version_no") + if latest is None: + raise ValueError("failed to resolve latest published version") + return int(latest) + + +def count_active_runs_using_set(evaluation_set_id: int, tenant_id: str) -> int: + """Return the number of active (non-soft-deleted) evaluation runs referencing the set.""" + with get_db_session() as session: + return session.query(AgentEvaluation).filter( + AgentEvaluation.evaluation_set_id == evaluation_set_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).count() + + +def delete_evaluation_set_impl( + evaluation_set_id: int, + tenant_id: str, + user_id: str, +) -> None: + """Soft-delete an evaluation set. + + Blocked when any active evaluation run still references the set, so historical + runs never lose their context. Use ``count_active_runs_using_set`` first if + the caller wants to surface the referenced count to the user before deletion. + """ + referenced = count_active_runs_using_set(evaluation_set_id, tenant_id) + if referenced > 0: + raise ValueError( + f"evaluation set is referenced by {referenced} evaluation run(s); cannot delete" + ) + soft_delete_evaluation_set(evaluation_set_id, tenant_id, user_id) diff --git a/backend/utils/evaluation_set_excel_utils.py b/backend/utils/evaluation_set_excel_utils.py new file mode 100644 index 000000000..c9f7de4b6 --- /dev/null +++ b/backend/utils/evaluation_set_excel_utils.py @@ -0,0 +1,214 @@ +import io +from typing import List, Optional, Dict, Any + +import xlrd +from openpyxl import Workbook, load_workbook +from openpyxl.styles import Font, PatternFill + + +REQUIRED_HEADERS = ["query", "answer"] +OPTIONAL_HEADERS = ["case_id"] +ALL_HEADERS = REQUIRED_HEADERS + OPTIONAL_HEADERS + + +def _normalize_header(v: Any) -> str: + if v is None: + return "" + return str(v).strip().lower() + + +def build_evaluation_set_excel_template_bytes() -> bytes: + """Build a downloadable XLSX template. + + Column order puts required fields first. + """ + wb = Workbook() + ws = wb.active + ws.title = "evaluation_cases" + + headers = [ + "序号", + "问题*", + "答案*", + ] + + ws.append(headers) + ws.freeze_panes = "A2" + + # Styling + bold = Font(bold=True) + required_fill = PatternFill(start_color="FFF7E6", end_color="FFF7E6", fill_type="solid") + + for col_idx, title in enumerate(headers, start=1): + cell = ws.cell(row=1, column=col_idx) + cell.font = bold + if title.endswith("*"): + cell.fill = required_fill + + # Column widths + ws.column_dimensions["A"].width = 12 # 序号 (case_id) + ws.column_dimensions["B"].width = 50 # 问题 (query) + ws.column_dimensions["C"].width = 50 # 答案 (answer) + + # Example rows + ws.append([ + "c1", + "1+1等于几？", + "2", + ]) + ws.append([ + "c2", + "中国首都是哪里？", + "北京", + ]) + + out = io.BytesIO() + wb.save(out) + return out.getvalue() + + +def parse_evaluation_cases_from_excel(filename: str, raw: bytes) -> List[Dict[str, Any]]: + """Parse evaluation cases from .xlsx or .xls. + + Expected headers (case-insensitive): query, answer, case_id (or aliases). + A trailing '*' in header is allowed (e.g. query*). + Chinese aliases are also recognized so the template round-trip works: + 序号 / case_id / case_id -> case_id + 问题 / query / query -> query + 答案 / answer / answer -> answer + + Returns normalized case dicts compatible with insert_evaluation_set_cases. + """ + + HEADER_ALIASES = { + "query": "query", + "问题": "query", + "answer": "answer", + "答案": "answer", + "case_id": "case_id", + "序号": "case_id", + "caseid": "case_id", + "id": "case_id", + } + + def _canonical_header(v: Any) -> Optional[str]: + key = _normalize_header(v).rstrip("*") + if not key: + return None + return HEADER_ALIASES.get(key) + + lower_name = (filename or "").lower() + if lower_name.endswith(".xlsx"): + wb = load_workbook(io.BytesIO(raw), read_only=True, data_only=True) + ws = wb.active + rows = ws.iter_rows(values_only=True) + header_row = next(rows, None) + if not header_row: + raise ValueError("Excel contains no header row") + + header_map: Dict[str, int] = {} + for idx, v in enumerate(header_row): + canon = _canonical_header(v) + if canon: + header_map[canon] = idx + + for h in REQUIRED_HEADERS: + if h not in header_map: + raise ValueError(f"Missing required column: {h}") + + cases: List[Dict[str, Any]] = [] + for excel_row_idx, row in enumerate(rows, start=2): + if row is None: + continue + + def get_col(col: str) -> Optional[str]: + if col not in header_map: + return None + v = row[header_map[col]] if header_map[col] < len(row) else None + if v is None: + return None + s = str(v).strip() + return s if s != "" else None + + query = get_col("query") + answer = get_col("answer") + case_id = get_col("case_id") + + # Skip fully empty rows + if not any([query, answer, case_id]): + continue + + if not query: + raise ValueError(f"Row {excel_row_idx}: 问题 is required") + if not answer: + raise ValueError(f"Row {excel_row_idx}: 答案 is required") + + normalized: Dict[str, Any] = { + "case_id": case_id, + "inputs": {"query": query}, + "label": {"answer": answer}, + "order_no": len(cases), + } + cases.append(normalized) + + if not cases: + raise ValueError("Excel contains no cases") + + return cases + + if lower_name.endswith(".xls"): + book = xlrd.open_workbook(file_contents=raw) + sheet = book.sheet_by_index(0) + if sheet.nrows < 1: + raise ValueError("Excel contains no header row") + + header_row = sheet.row_values(0) + header_map: Dict[str, int] = {} + for idx, v in enumerate(header_row): + canon = _canonical_header(v) + if canon: + header_map[canon] = idx + + for h in REQUIRED_HEADERS: + if h not in header_map: + raise ValueError(f"Missing required column: {h}") + + cases: List[Dict[str, Any]] = [] + for r in range(1, sheet.nrows): + excel_row_idx = r + 1 + + def get_cell(col: str) -> Optional[str]: + if col not in header_map: + return None + v = sheet.cell_value(r, header_map[col]) + if v is None: + return None + s = str(v).strip() + return s if s != "" else None + + query = get_cell("query") + answer = get_cell("answer") + case_id = get_cell("case_id") + + if not any([query, answer, case_id]): + continue + + if not query: + raise ValueError(f"Row {excel_row_idx}: 问题 is required") + if not answer: + raise ValueError(f"Row {excel_row_idx}: 答案 is required") + + normalized: Dict[str, Any] = { + "case_id": case_id, + "inputs": {"query": query}, + "label": {"answer": answer}, + "order_no": len(cases), + } + cases.append(normalized) + + if not cases: + raise ValueError("Excel contains no cases") + + return cases + + raise ValueError("Unsupported file type. Please upload .xlsx or .xls") diff --git a/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql b/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql new file mode 100644 index 000000000..3ec4e1dbf --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql @@ -0,0 +1,220 @@ +-- ============================================================================= +-- Agent evaluation (offline) - full bundle +-- ============================================================================= +-- Version: v2.3.0 +-- Date: 2026-06-30 +-- Description: Single-file bundle for the v2.3.0 agent evaluation feature. +-- Combines what were originally three separate migration drafts (0628, 0630 +-- pass_status, 0630 route grant) before any of them had been applied to +-- any environment. Use this file on fresh installs. +-- +-- Idempotency: every DDL statement in this file is safe to run multiple times. +-- - CREATE TABLE IF NOT EXISTS +-- - ALTER TABLE ... ADD COLUMN IF NOT EXISTS +-- - CREATE INDEX IF NOT EXISTS +-- - COMMENT ON (overwrites previous value, no-op if identical) +-- - INSERT ... ON CONFLICT (role_permission_id) DO NOTHING +-- +-- Sections: +-- 1. evaluation_set_t / evaluation_set_case_t / agent_evaluation_t / +-- agent_evaluation_case_t (incl. judge_model_id on agent_evaluation_t) +-- 2. pass_status column on agent_evaluation_case_t + composite index +-- 3. LEFT_NAV_MENU '/space' grant for roles that have /agent-space +-- +-- Design decisions (see PR review 2026-06-30): +-- * No standalone (tenant_id) index on any table. Every case-level and +-- run-level read is scoped by PK or by a foreign key into a parent that +-- itself is already tenant-scoped at the application layer. A bare +-- (tenant_id) index has no real query plan and only inflates write cost. +-- * No (tenant_id, judge_model_id) index. judge_model_id is read alongside +-- the row via PK; "list runs by judge model" is not a supported query. +-- * No (tenant_id, evaluation_set_id) on evaluation_set_case_t. The set +-- itself is tenant-scoped at the app layer, and the existing +-- (evaluation_set_id) index already covers set-case listing. +-- * ix_agent_eval_case_pass_status is (agent_evaluation_id, pass_status) +-- rather than (tenant_id, agent_evaluation_id, pass_status): case-level +-- reads never filter on tenant_id directly, and dropping the leading +-- tenant_id column keeps the most common "list failed cases for run X" +-- query on a single composite index. +-- * Section 3 INSERT must include parent_key (see 0622 menu migration) so +-- a future renderer that joins on parent_key does not leave this batch +-- as orphans. /space is a first-level entry for the route guard only, +-- so parent_key is NULL. +-- ============================================================================= + +SET search_path TO nexent; + +BEGIN; + + +-- ----------------------------------------------------------------------------- +-- Section 1: Evaluation set & evaluation run tables +-- ----------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS nexent.evaluation_set_t ( + evaluation_set_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + name VARCHAR(255) NOT NULL, + description TEXT, + + source_filename VARCHAR(255), + case_count INTEGER DEFAULT 0, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_eval_set_name ON nexent.evaluation_set_t(tenant_id, name); + +COMMENT ON TABLE nexent.evaluation_set_t IS 'Offline evaluation sets (JSONL single-turn cases).'; +COMMENT ON COLUMN nexent.evaluation_set_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; +COMMENT ON COLUMN nexent.evaluation_set_t.source_filename IS 'Original uploaded filename'; +COMMENT ON COLUMN nexent.evaluation_set_t.case_count IS 'Total number of cases'; + + +CREATE TABLE IF NOT EXISTS nexent.evaluation_set_case_t ( + evaluation_set_case_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + evaluation_set_id BIGINT NOT NULL, + + case_id VARCHAR(128), + inputs JSONB NOT NULL, + label JSONB NOT NULL, + order_no INTEGER DEFAULT 0, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_eval_set_case_set_id ON nexent.evaluation_set_case_t(evaluation_set_id); + +COMMENT ON TABLE nexent.evaluation_set_case_t IS 'Cases within evaluation sets.'; +COMMENT ON COLUMN nexent.evaluation_set_case_t.inputs IS 'Case inputs JSON: {query: string, context?: string}'; +COMMENT ON COLUMN nexent.evaluation_set_case_t.label IS 'Case label JSON: {answer: string}'; + + +CREATE TABLE IF NOT EXISTS nexent.agent_evaluation_t ( + agent_evaluation_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + agent_id INTEGER NOT NULL, + agent_version_no INTEGER NOT NULL, + + evaluation_set_id BIGINT NOT NULL, + + status VARCHAR(30) NOT NULL DEFAULT 'PENDING', + + progress_total INTEGER DEFAULT 0, + progress_done INTEGER DEFAULT 0, + + score_overall DOUBLE PRECISION, + error_message TEXT, + + judge_model_id INTEGER, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_agent_eval_agent_id ON nexent.agent_evaluation_t(tenant_id, agent_id); +CREATE INDEX IF NOT EXISTS ix_agent_eval_set_id ON nexent.agent_evaluation_t(tenant_id, evaluation_set_id); + +COMMENT ON TABLE nexent.agent_evaluation_t IS 'Offline evaluation runs for an agent.'; +COMMENT ON COLUMN nexent.agent_evaluation_t.status IS 'Run status: PENDING/RUNNING/COMPLETED/FAILED'; +COMMENT ON COLUMN nexent.agent_evaluation_t.judge_model_id IS + 'Model id used by the judge. Persisted so the background worker can recover it after restart and so the frontend can display judge_model_name.'; + + +CREATE TABLE IF NOT EXISTS nexent.agent_evaluation_case_t ( + agent_evaluation_case_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + agent_evaluation_id BIGINT NOT NULL, + evaluation_set_case_id BIGINT NOT NULL, + + inputs JSONB NOT NULL, + label JSONB NOT NULL, + predict JSONB, + + score DOUBLE PRECISION, + reason TEXT, + + status VARCHAR(30) NOT NULL DEFAULT 'PENDING', + error_message TEXT, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_agent_eval_case_eval_id ON nexent.agent_evaluation_case_t(agent_evaluation_id); + +COMMENT ON TABLE nexent.agent_evaluation_case_t IS 'Per-case evaluation results.'; +COMMENT ON COLUMN nexent.agent_evaluation_case_t.predict IS 'Predict JSON: {answer: string, raw?: any}'; +COMMENT ON COLUMN nexent.agent_evaluation_case_t.status IS 'Case status: PENDING/RUNNING/COMPLETED/FAILED'; + + +-- ----------------------------------------------------------------------------- +-- Section 2: pass_status on agent_evaluation_case_t +-- ----------------------------------------------------------------------------- +-- Stores the binary judge result ("pass" / "fail") for each case. +-- Enables fast filtering for failed-case reports and storage optimization: +-- passed cases have predict/reason/label.answer cleared to save space, +-- while only failed cases retain full detail. + +ALTER TABLE nexent.agent_evaluation_case_t +ADD COLUMN IF NOT EXISTS pass_status VARCHAR(16); + +COMMENT ON COLUMN nexent.agent_evaluation_case_t.pass_status IS + 'Judge result per case: pass / fail. pass cases have predict/reason/label.answer cleared to save space.'; + +-- Composite index to support failed-case listing and "only failed" reports. +-- Scoped by (agent_evaluation_id, pass_status) only; tenant_id is enforced +-- at the application layer via the parent run's tenant. +CREATE INDEX IF NOT EXISTS ix_agent_eval_case_pass_status + ON nexent.agent_evaluation_case_t (agent_evaluation_id, pass_status); + + +-- ----------------------------------------------------------------------------- +-- Section 3: Grant /space LEFT_NAV_MENU so the evaluation page is reachable +-- ----------------------------------------------------------------------------- +-- The agent evaluation page lives at the route prefix /space/agents/{id}/evaluate. +-- The previous menu migration (v2.2.2_0622_update_left_nav_menu.sql) removed +-- the legacy /space entry when it refactored the menu structure. As a result +-- the frontend route guard (which uses accessibleRoutes prefix matching) blocks +-- any user from entering the evaluation page with "no access permission". +-- +-- This section adds LEFT_NAV_MENU = '/space' for every role that already has +-- access to the resource-space (i.e. /agent-space). This entry is NOT rendered +-- in the side navigation (SideNavigation uses exact-match against ROUTE_CONFIG) +-- but it IS picked up by the backend as part of accessibleRoutes, so the route +-- guard will allow /space/agents/{id}/evaluate and its sub-paths. + +-- Roles that already have /resource-space (and thus /agent-space) get /space. +-- Mirrors v2.2.2_0622_update_left_nav_menu.sql IDs (16xx range) to keep the +-- scheme consistent. parent_key is NULL: /space is a top-level entry used +-- only by the backend route guard (prefix match on accessibleRoutes) and +-- is not rendered by SideNavigation. +INSERT INTO nexent.role_permission_t + (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) +VALUES + (1600, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1601, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1602, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1603, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1604, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL) +ON CONFLICT (role_permission_id) DO NOTHING; + + +COMMIT; diff --git a/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx b/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx index e840ad052..89dffcbc6 100644 --- a/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx +++ b/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx @@ -167,7 +167,7 @@ export function AgentRepositoryCopyDialog({ title={t("agentRepository.copy.title", { name: listingTitle })} centered width={520} - destroyOnClose + destroyOnHidden footer={
diff --git a/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx b/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx index 98bc649de..b62c944bc 100644 --- a/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx +++ b/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx @@ -299,7 +299,7 @@ export function AgentRepositoryDetailModal({ footer={null} width={720} centered - destroyOnClose + destroyOnHidden title={null} className="agent-repository-detail-modal" styles={{ body: { padding: 0 } }} diff --git a/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx b/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx index b4964e820..9a5c3102d 100644 --- a/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx +++ b/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx @@ -136,6 +136,14 @@ export function MineAgentsView({ router.push(`/${locale}/agents?agent_id=${agentId}`); }; + const handleEvaluate = (agent: MyEditableAgentItem) => { + const versionNo = agent.current_version_no ?? 0; + if (versionNo <= 0) { + return; + } + router.push(`/${locale}/space/agents/${agent.agent_id}/evaluate?back_tab=mine`); + }; + const closeReviewModal = () => { setReviewModalOpen(false); setReviewModalAgent(null); @@ -353,6 +361,7 @@ export function MineAgentsView({ } onApplyListing={() => handleApplyListing(agent)} onViewReview={(mode) => handleViewReview(agent, mode)} + onEvaluate={() => handleEvaluate(agent)} isApplying={ applyingAgentId === agent.agent_id && createListingMutation.isPending diff --git a/frontend/app/[locale]/agent-space/components/MyAgentCard.tsx b/frontend/app/[locale]/agent-space/components/MyAgentCard.tsx index e6be0485c..257459091 100644 --- a/frontend/app/[locale]/agent-space/components/MyAgentCard.tsx +++ b/frontend/app/[locale]/agent-space/components/MyAgentCard.tsx @@ -27,6 +27,7 @@ interface MyAgentCardProps { onView: () => void; onApplyListing: () => void; onViewReview: (mode: "review" | "reviewUpdate") => void; + onEvaluate: () => void; isApplying?: boolean; } @@ -54,6 +55,7 @@ export function MyAgentCard({ onView, onApplyListing, onViewReview, + onEvaluate, isApplying = false, }: MyAgentCardProps) { const { t } = useTranslation("common"); @@ -70,6 +72,7 @@ export function MyAgentCard({ const versionLabel = agent.version_label; const canEdit = agent.permission !== "READ_ONLY"; const canView = (agent.current_version_no ?? 0) > 0; + const canEvaluate = canView; const menuActions = getMineCardMenuActions(agent); const menuItems: MenuProps["items"] = menuActions.map((action) => { @@ -212,7 +215,8 @@ export function MyAgentCard({ type="default" className="min-w-0 flex-1" icon={} - disabled + onClick={onEvaluate} + disabled={!canEvaluate} > {t("agentRepository.mine.evaluate")} diff --git a/frontend/app/[locale]/agent-space/page.tsx b/frontend/app/[locale]/agent-space/page.tsx index 8fdde1c08..00eff3c5e 100644 --- a/frontend/app/[locale]/agent-space/page.tsx +++ b/frontend/app/[locale]/agent-space/page.tsx @@ -1,6 +1,7 @@ "use client"; import { useEffect, useMemo, useState } from "react"; +import { useSearchParams } from "next/navigation"; import { App, Button, @@ -62,10 +63,17 @@ const agentRepositoryTheme = { export default function AgentRepositoryPage() { const { t } = useTranslation("common"); const { pageVariants, pageTransition } = useSetupFlow(); + const searchParams = useSearchParams(); const { user } = useAuthorizationContext(); const isAdmin = user?.role === USER_ROLES.ADMIN; - const [tab, setTab] = useState(AgentRepositoryTab.REPOSITORY); + const [tab, setTab] = useState(() => { + const backTab = searchParams.get("back_tab"); + if (backTab === "mine") return AgentRepositoryTab.MINE; + if (backTab === "repository") return AgentRepositoryTab.REPOSITORY; + if (backTab === "review") return AgentRepositoryTab.REVIEW; + return AgentRepositoryTab.REPOSITORY; + }); const [searchQuery, setSearchQuery] = useState(""); const [repositoryPage, setRepositoryPage] = useState(1); const [selectedCategoryId, setSelectedCategoryId] = useState(null); diff --git a/frontend/app/[locale]/agents/components/a2a/A2AChatModal.tsx b/frontend/app/[locale]/agents/components/a2a/A2AChatModal.tsx index a6f544ae9..0b40f9d7c 100644 --- a/frontend/app/[locale]/agents/components/a2a/A2AChatModal.tsx +++ b/frontend/app/[locale]/agents/components/a2a/A2AChatModal.tsx @@ -192,7 +192,7 @@ export default function A2AChatModal({ onCancel={onClose} footer={null} width={600} - destroyOnClose + destroyOnHidden >
{/* Agent Info Header */} diff --git a/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx b/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx index 9729007e2..9e3f9ba7b 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/skill/SkillConfigModal.tsx @@ -540,7 +540,7 @@ export default function SkillConfigModal({ open={isOpen} onCancel={onCancel} width={600} - destroyOnClose + destroyOnHidden footer={
diff --git a/frontend/app/[locale]/agents/components/agentConfig/tool/LabelManagementModal.tsx b/frontend/app/[locale]/agents/components/agentConfig/tool/LabelManagementModal.tsx index 2cac4ee67..a8a117967 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/tool/LabelManagementModal.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/tool/LabelManagementModal.tsx @@ -139,7 +139,7 @@ export default function LabelManagementModal({ title={t("toolConfig.title.manageLabels")} open={open} onCancel={onClose} - maskClosable={true} + mask={{ closable: true }} footer={null} width={1100} zIndex={1100} diff --git a/frontend/app/[locale]/agents/components/agentConfig/tool/SelectToolsDialog.tsx b/frontend/app/[locale]/agents/components/agentConfig/tool/SelectToolsDialog.tsx index f45f589f7..bc81e30f8 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/tool/SelectToolsDialog.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/tool/SelectToolsDialog.tsx @@ -306,8 +306,8 @@ export default function SelectToolsDialog({ footer={null} width={1100} zIndex={1000} - maskClosable={true} - destroyOnClose + mask={{ closable: true }} + destroyOnHidden > diff --git a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx index aa08f40f8..d62e9ca3d 100644 --- a/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx +++ b/frontend/app/[locale]/agents/components/agentInfo/AgentGenerateDetail.tsx @@ -105,6 +105,26 @@ export default function AgentGenerateDetail({}) { // Streaming field values (accumulated from SSE, bypasses Form disabled state) + // Track form values for modal props to avoid synchronous getFieldValue during render + const [watchedPromptTemplateId, setWatchedPromptTemplateId] = useState( + form.getFieldValue("promptTemplateId") + ); + const [watchedBusinessDescription, setWatchedBusinessDescription] = useState( + form.getFieldValue("businessDescription") || "" + ); + const [watchedBusinessLogicModelId, setWatchedBusinessLogicModelId] = useState( + form.getFieldValue("businessLogicModelId") + ); + const [watchedDutyPrompt, setWatchedDutyPrompt] = useState( + form.getFieldValue("dutyPrompt") || "" + ); + const [watchedConstraintPrompt, setWatchedConstraintPrompt] = useState( + form.getFieldValue("constraintPrompt") || "" + ); + const [watchedFewShotsPrompt, setWatchedFewShotsPrompt] = useState( + form.getFieldValue("fewShotsPrompt") || "" + ); + // Modal states const [expandModalOpen, setExpandModalOpen] = useState(false); const [expandModalType, setExpandModalType] = useState<'duty' | 'constraint' | 'few-shots' | null>(null); @@ -117,6 +137,16 @@ export default function AgentGenerateDetail({}) { clearExpiredGenerationCaches(); }, []); + // Sync watched form values with state to avoid synchronous getFieldValue during render + useEffect(() => { + setWatchedPromptTemplateId(form.getFieldValue("promptTemplateId")); + setWatchedBusinessDescription(form.getFieldValue("businessDescription") || ""); + setWatchedBusinessLogicModelId(form.getFieldValue("businessLogicModelId")); + setWatchedDutyPrompt(form.getFieldValue("dutyPrompt") || ""); + setWatchedConstraintPrompt(form.getFieldValue("constraintPrompt") || ""); + setWatchedFewShotsPrompt(form.getFieldValue("fewShotsPrompt") || ""); + }, [form]); + // (e.g. business_description from a previously edited agent) useEffect(() => { @@ -278,6 +308,10 @@ export default function AgentGenerateDetail({}) { if (!editable || isGenerating || !modelId) { return; } + // Sync watched values before opening modal to ensure they're available on render + setWatchedDutyPrompt(form.getFieldValue("dutyPrompt") || ""); + setWatchedConstraintPrompt(form.getFieldValue("constraintPrompt") || ""); + setWatchedFewShotsPrompt(form.getFieldValue("fewShotsPrompt") || ""); setOptimizeModalType(type); setOptimizeModalOpen(true); }; @@ -1253,7 +1287,7 @@ export default function AgentGenerateDetail({}) { open={promptTemplateManagerOpen} editable={editable} templates={promptTemplates} - selectedTemplateId={form.getFieldValue("promptTemplateId") || editedAgent.prompt_template_id || 0} + selectedTemplateId={watchedPromptTemplateId ?? editedAgent.prompt_template_id ?? 0} onClose={() => setPromptTemplateManagerOpen(false)} onSelectTemplate={handleSelectPromptTemplate} onTemplatesChanged={invalidatePromptTemplates} @@ -1271,11 +1305,15 @@ export default function AgentGenerateDetail({}) { sectionType={ optimizeModalType === "few-shots" ? "few_shots" : optimizeModalType } - taskDescription={form.getFieldValue("businessDescription") || editedAgent.business_description || ""} + taskDescription={watchedBusinessDescription ?? editedAgent.business_description ?? ""} currentContent={ - form.getFieldValue(getPromptFieldKey(optimizeModalType)) || "" + optimizeModalType === "duty" + ? watchedDutyPrompt ?? "" + : optimizeModalType === "constraint" + ? watchedConstraintPrompt ?? "" + : watchedFewShotsPrompt ?? "" } - modelId={form.getFieldValue("businessLogicModelId")} + modelId={watchedBusinessLogicModelId ?? 0} agentId={currentAgentId ?? 0} toolIds={ Array.isArray(editedAgent.tools) diff --git a/frontend/app/[locale]/agents/components/agentInfo/ExpandEditModal.tsx b/frontend/app/[locale]/agents/components/agentInfo/ExpandEditModal.tsx index 4ef03289d..3d11e6013 100644 --- a/frontend/app/[locale]/agents/components/agentInfo/ExpandEditModal.tsx +++ b/frontend/app/[locale]/agents/components/agentInfo/ExpandEditModal.tsx @@ -1,6 +1,6 @@ import { useState, useEffect } from "react"; import { useTranslation } from "react-i18next"; -import { Modal, Input, Badge, Button } from "antd"; +import { Modal, Input, Badge, Button, Space } from "antd"; export interface ExpandEditModalProps { open: boolean; @@ -53,16 +53,17 @@ export default function ExpandEditModal({ footer={ readOnly ? ( ) : ( - + + + + ) } width={1000} diff --git a/frontend/app/[locale]/agents/components/agentInfo/PromptOptimizeModal.tsx b/frontend/app/[locale]/agents/components/agentInfo/PromptOptimizeModal.tsx index 897c523c3..c829acf22 100644 --- a/frontend/app/[locale]/agents/components/agentInfo/PromptOptimizeModal.tsx +++ b/frontend/app/[locale]/agents/components/agentInfo/PromptOptimizeModal.tsx @@ -16,7 +16,7 @@ import { Tooltip, Alert, } from "antd"; -import { MousePointer2 } from "lucide-react"; +import { MousePointer2, Sparkles } from "lucide-react"; import log from "@/lib/logger"; import { optimizePromptSection } from "@/services/promptService"; @@ -200,7 +200,12 @@ export default function PromptOptimizeModal({ return ( + + {title} +
+ } open={open} onCancel={onClose} width={1200} diff --git a/frontend/app/[locale]/agents/page.tsx b/frontend/app/[locale]/agents/page.tsx index 2ef7692d4..4e625eb3d 100644 --- a/frontend/app/[locale]/agents/page.tsx +++ b/frontend/app/[locale]/agents/page.tsx @@ -43,13 +43,14 @@ export default function AgentSetupOrchestrator() { // Handle auto-create mode from URL params useEffect(() => { - const create = searchParams.get('create'); - if (create === 'true') { + const create = searchParams?.get("create"); + if (create === "true") { setTimeout(() => { enterCreateMode(); }, 100); } - }, [searchParams, enterCreateMode]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [enterCreateMode]); // Handle auto-select agent from URL params (agent_id) useEffect(() => { diff --git a/frontend/app/[locale]/resource-manage/components/UserManageComp.tsx b/frontend/app/[locale]/resource-manage/components/UserManageComp.tsx index cfff26fe9..0b499bf62 100644 --- a/frontend/app/[locale]/resource-manage/components/UserManageComp.tsx +++ b/frontend/app/[locale]/resource-manage/components/UserManageComp.tsx @@ -19,7 +19,7 @@ import { Alert, Space, Divider, - Tooltip + Tooltip, } from "antd"; import { Users, @@ -1040,7 +1040,7 @@ export default function UserManageComp() { const { user } = useAuthorizationContext(); const { isSpeedMode } = useDeployment(); const params = useParams(); - const locale = (params.locale as string) || "en"; + const locale = (params as { locale?: string })?.locale || "en"; // Check if user is super admin (speed mode or admin role) const isSuperAdmin = isSpeedMode || user?.role === USER_ROLES.SU; @@ -1258,11 +1258,10 @@ export default function UserManageComp() {
)} -
- +
- +
{tenantId ? ( , }, { @@ -1312,7 +1312,9 @@ export default function UserManageComp() { }, { key: "invitations", - label: t("tenantResources.invitation.tab") || "Invitations", + label: + t("tenantResources.invitation.tab") || + "Invitations", children: ( )}
-
@@ -1358,4 +1359,4 @@ export default function UserManageComp() { max-width: 100%; overflow: hidden; } -`} +`}; diff --git a/frontend/app/[locale]/resource-manage/components/resources/McpList.tsx b/frontend/app/[locale]/resource-manage/components/resources/McpList.tsx index 412ff402f..29eda7e78 100644 --- a/frontend/app/[locale]/resource-manage/components/resources/McpList.tsx +++ b/frontend/app/[locale]/resource-manage/components/resources/McpList.tsx @@ -816,7 +816,7 @@ export default function McpList({ tenantId }: { tenantId: string | null }) { onCancel={() => !actionsLocked && setAddModalVisible(false)} footer={null} width={800} - destroyOnClose + destroyOnHidden > = 80) return "excellent"; + if (passRate >= 50) return "medium"; + return "poor"; +} + +export default function EvaluationConclusion({ passRate }: EvaluationConclusionProps) { + const { t } = useTranslation("common"); + const key = getConclusion(passRate); + return ( +
+ + {t("agentEvaluation.report.conclusionTitle")} + + + {t(`agentEvaluation.report.conclusion.${key}`)} + +
+ ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationConfigCard.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationConfigCard.tsx new file mode 100644 index 000000000..e6455ce54 --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationConfigCard.tsx @@ -0,0 +1,278 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button, Flex, Select, Typography, message } from "antd"; +import { BookOpen, PlayCircle, Download, FileSpreadsheet, Upload } from "lucide-react"; +import { useModelList } from "@/hooks/model/useModelList"; +import { useStartEvaluation } from "@/hooks/evaluation/useStartEvaluation"; +import { evaluationService } from "@/services/evaluationService"; +import type { AgentEvaluationRun, EvaluationSet } from "@/types/agentEvaluation"; + +const { Text } = Typography; + +interface EvaluationConfigCardProps { + agentId: number; + selectedSet: EvaluationSet | null; + onOpenLibrary: () => void; + onOpenUpload: () => void; + onEvaluationStarted: (run: AgentEvaluationRun) => void; + runningProgress?: { done: number; total: number }; +} + +export default function EvaluationConfigCard({ + agentId, + selectedSet, + onOpenLibrary, + onOpenUpload, + onEvaluationStarted, + runningProgress, +}: EvaluationConfigCardProps) { + const { t } = useTranslation("common"); + const { availableLlmModels } = useModelList(); + const { startEvaluation, starting } = useStartEvaluation(); + + const [judgeModelId, setJudgeModelId] = useState(null); + const [agentVersionNo, setAgentVersionNo] = useState(null); + const [agentVersions, setAgentVersions] = useState<{ label: string; value: number }[]>([]); + + const modelOptions = useMemo( + () => + availableLlmModels.map((m) => ({ + label: m.displayName || m.name, + value: m.id, + })), + [availableLlmModels] + ); + + useEffect(() => { + if (modelOptions.length && judgeModelId == null) { + setJudgeModelId(modelOptions[0].value); + } + }, [modelOptions, judgeModelId]); + + useEffect(() => { + fetch(`/api/agent/${agentId}/versions`) + .then((r) => r.json()) + .then((res) => { + const versions = (res.items || []).map((v: { version_no: number }) => ({ + label: `v${v.version_no}`, + value: v.version_no, + })); + setAgentVersions(versions); + if (versions.length && agentVersionNo == null) { + setAgentVersionNo(versions[0].value); + } + }) + .catch(() => {}); + }, [agentId]); + + const handleStart = async () => { + if (!judgeModelId) { + message.error(t("agentEvaluation.selectJudgeModelFirst")); + return; + } + if (!selectedSet) { + message.error(t("agentEvaluation.step1.selectFirst")); + return; + } + const result = await startEvaluation({ + agentId, + evaluationSetId: selectedSet.evaluation_set_id, + judgeModelId, + }); + if (result) { + onEvaluationStarted(result); + } + }; + + const handleDownloadTemplate = async () => { + try { + const blob = await evaluationService.downloadEvaluationSetTemplate(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "agent_evaluation_template.xlsx"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } catch (err: any) { + message.error(err?.message || t("agentEvaluation.downloadTemplateFailed")); + } + }; + + const isRunning = runningProgress != null; + + return ( +
+ {/* Step 1 header with number circle */} +
+ +
+ 1 +
+ + {t("agentEvaluation.step1.title")} + + +
+ + {/* Content */} +
+ + {/* Config area: model + version in a gray box */} +
+ + + + {t("agentEvaluation.step1.judgeModel")} + + + + {t("agentEvaluation.step1.agentVersionHint2")} + + + +
+ + {/* Download template + Test case selector — equal height stretch */} +
+ {/* Download template */} +
+
+ +
+ + {t("agentEvaluation.step1.downloadTemplate")} + + + {t("agentEvaluation.step1.downloadTemplateHint")} + +
+ +
+
+ + {/* Test case selector */} +
+
+ +
+ + {t("agentEvaluation.step1.selectTestCaseSet")} + + {selectedSet ? ( + + + {selectedSet.name} + + + + ) : ( + + {t("agentEvaluation.step1.selectTestCaseSetHint")} + + )} + + + + +
+
+ + {/* Start button */} + + {isRunning ? ( + + + {t("agentEvaluation.step1.runningHint", { + done: runningProgress?.done ?? 0, + total: runningProgress?.total ?? 0, + })} + +
+
+
+ + {Math.round( + ((runningProgress?.done ?? 0) / (runningProgress?.total ?? 1)) * 100 + )}% + + + ) : ( + + {selectedSet + ? t("agentEvaluation.step1.ready") + : t("agentEvaluation.step1.selectFirst")} + + )} + + + +
+
+ ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationHistoryCard.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationHistoryCard.tsx new file mode 100644 index 000000000..b9dba0005 --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationHistoryCard.tsx @@ -0,0 +1,205 @@ +"use client"; + +import { useState } from "react"; +import { Button, Empty, Flex, Modal, Pagination, Spin, Typography } from "antd"; +import { Trash2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import type { EvaluationHistoryItem } from "@/types/agentEvaluation"; +import { formatDateTime } from "@/lib/utils"; + +const { Text } = Typography; + +const PAGE_SIZE = 3; + +interface EvaluationHistoryCardProps { + history: EvaluationHistoryItem[]; + loading: boolean; + deletingId: number | null; + selectedId: number | null; + onSelect: (item: EvaluationHistoryItem) => void; + onDelete: (evaluationId: number) => Promise<{ success: boolean }>; +} + +export default function EvaluationHistoryCard({ + history, + loading, + deletingId, + selectedId, + onSelect, + onDelete, +}: EvaluationHistoryCardProps) { + const { t } = useTranslation("common"); + const [page, setPage] = useState(1); + const total = history.length; + const pageData = history.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE); + + const passRate = (item: EvaluationHistoryItem) => { + const pass = item.pass_count ?? 0; + const fail = item.fail_count ?? 0; + const total = pass + fail; + if (total === 0) return "-"; + return `${Math.round((pass / total) * 100)}%`; + }; + + const caseCount = (item: EvaluationHistoryItem) => { + return (item.pass_count ?? 0) + (item.fail_count ?? 0); + }; + + const STATUS_LABELS: Record = { + PENDING: t("agentEvaluation.status.pending"), + RUNNING: t("agentEvaluation.status.running"), + COMPLETED: t("agentEvaluation.status.completed"), + FAILED: t("agentEvaluation.status.failed"), + }; + + if (loading) { + return ( +
+
+ + {t("agentEvaluation.history.title")} + {history.length > 0 && ( + + ({t("agentEvaluation.history.count", { n: history.length })}) + + )} + +
+ + + +
+ ); + } + + return ( +
+ {/* Header */} +
+ + + {t("agentEvaluation.history.title")} + {history.length > 0 && ( + + ({t("agentEvaluation.history.count", { n: history.length })}) + + )} + + +
+ + {/* Body */} +
+ {history.length === 0 ? ( + + {t("agentEvaluation.history.empty")} + } + image={Empty.PRESENTED_IMAGE_SIMPLE} + /> + + ) : ( + + {pageData.map((item) => { + const rate = passRate(item); + const total = caseCount(item); + return ( + onSelect(item)} + > + {/* Row 1: version tag + set name + status badge */} + + + + v{item.agent_version_no} + + + {item.evaluation_set_name || t("agentEvaluation.history.noSet")} + + + + {STATUS_LABELS[item.status] || item.status} + + + + {/* Row 2: judge model name + pass rate (right) */} + + {item.judge_model_name ? ( + + {item.judge_model_name} + + ) : ( + - + )} + + + 通过率 + + + {rate} + + + + + {/* Row 3: time + delete */} + + + {item.create_time ? formatDateTime(item.create_time) : "-"} + +
+
+ ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationReportCard.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationReportCard.tsx new file mode 100644 index 000000000..810af26b7 --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/EvaluationReportCard.tsx @@ -0,0 +1,116 @@ +"use client"; + +import { Button, Flex, Spin, Typography } from "antd"; +import { Download } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { useEvaluationReport } from "@/hooks/evaluation/useEvaluationReport"; +import MetricCard from "./MetricCard"; +import ResultDistributionBar from "./ResultDistributionBar"; +import EvaluationConclusion from "./EvaluationConclusion"; +import type { EvaluationHistoryItem } from "@/types/agentEvaluation"; + +const { Text } = Typography; + +interface EvaluationReportCardProps { + run: EvaluationHistoryItem | null; + loading?: boolean; +} + +export default function EvaluationReportCard({ run, loading }: EvaluationReportCardProps) { + const { t } = useTranslation("common"); + const { exportReport, exporting } = useEvaluationReport(); + + if (loading) { + return ( +
+
+ +
+ 2 +
+ + {t("agentEvaluation.report.title")} + + +
+ + + +
+ ); + } + + if (!run) { + return ( +
+
+ +
+ 2 +
+ + {t("agentEvaluation.report.title")} + + +
+ + + {t("agentEvaluation.report.empty", "请选择一条测评历史查看报告")} + + +
+ ); + } + + const passCount = run.pass_count ?? 0; + const failCount = run.fail_count ?? 0; + const total = passCount + failCount; + const passRate = total > 0 ? Math.round((passCount / total) * 100) : 0; + const score = run.score_overall != null ? run.score_overall.toFixed(1) : "-"; + + return ( +
+ {/* Header with number circle */} +
+ + +
+ 2 +
+ + {t("agentEvaluation.report.title")} + + + + +
+ + {/* Content */} +
+ + {/* Metrics row */} + + + + + + + {/* Distribution bar */} + + + {/* Conclusion */} + + +
+
+ ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/MetricCard.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/MetricCard.tsx new file mode 100644 index 000000000..d193b245e --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/MetricCard.tsx @@ -0,0 +1,37 @@ +"use client"; + +import { Flex, Typography } from "antd"; + +const { Text } = Typography; + +interface MetricCardProps { + label: string; + value: string | number; + highlight?: boolean; +} + +export default function MetricCard({ label, value, highlight }: MetricCardProps) { + return ( + + + {label} + + + {value} + + + ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/ResultDistributionBar.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/ResultDistributionBar.tsx new file mode 100644 index 000000000..032d9eedf --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/ResultDistributionBar.tsx @@ -0,0 +1,54 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { Flex, Typography } from "antd"; + +const { Text } = Typography; + +interface ResultDistributionBarProps { + passCount: number; + failCount: number; +} + +export default function ResultDistributionBar({ passCount, failCount }: ResultDistributionBarProps) { + const { t } = useTranslation("common"); + const total = passCount + failCount; + const passPct = total > 0 ? Math.round((passCount / total) * 100) : 0; + + return ( +
+ + {t("agentEvaluation.report.distributionTitle")} + +
+ {passCount > 0 && ( +
+ )} + {failCount > 0 && ( +
+ )} +
+ + +
+ + {t("agentEvaluation.report.distPass", { n: passCount })} + + + | + +
+ + {t("agentEvaluation.report.distFail", { n: failCount })} + + + +
+ ); +} diff --git a/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/TestCaseLibraryModal.tsx b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/TestCaseLibraryModal.tsx new file mode 100644 index 000000000..211dd1a6d --- /dev/null +++ b/frontend/app/[locale]/space/agents/[agentId]/evaluate/components/TestCaseLibraryModal.tsx @@ -0,0 +1,258 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + App, + Button, + Empty, + Flex, + Modal, + Table, + Typography, + Upload, + Input, +} from "antd"; +const { TextArea } = Input; +import { Trash2 } from "lucide-react"; +import type { ColumnsType } from "antd/es/table"; +import { evaluationService } from "@/services/evaluationService"; +import { useEvaluationSets } from "@/hooks/evaluation/useEvaluationSets"; +import type { EvaluationSet } from "@/types/agentEvaluation"; + +const { Text } = Typography; + +interface TestCaseLibraryModalProps { + open: boolean; + onClose: () => void; + selectedSetId: number | null; + onSelect: (set: EvaluationSet | null) => void; + uploadOpen?: boolean; + onUploadOpenChange?: (open: boolean) => void; +} + +export default function TestCaseLibraryModal({ + open, + onClose, + selectedSetId, + onSelect, + uploadOpen: uploadOpenProp, + onUploadOpenChange, +}: TestCaseLibraryModalProps) { + const { t } = useTranslation("common"); + const { message: messageApi } = App.useApp(); + const { sets, loading, deletingId, loadSets, deleteSet } = useEvaluationSets(); + + const [uploadOpenInternal, setUploadOpenInternal] = useState(false); + const uploadOpen = uploadOpenProp !== undefined ? uploadOpenProp : uploadOpenInternal; + const setUploadOpen = onUploadOpenChange ?? setUploadOpenInternal; + const [uploadName, setUploadName] = useState(""); + const [uploadDesc, setUploadDesc] = useState(""); + const [uploadFiles, setUploadFiles] = useState([]); + const [uploading, setUploading] = useState(false); + const [localSelectedId, setLocalSelectedId] = useState(null); + const [changed, setChanged] = useState(false); + + useEffect(() => { + if (open) { + setLocalSelectedId(selectedSetId); + setChanged(false); + loadSets(); + } + }, [open, selectedSetId, loadSets]); + + const handleClose = () => { + if (changed) { + const next = + localSelectedId !== null + ? sets.find((s) => s.evaluation_set_id === localSelectedId) ?? null + : null; + onSelect(next); + } + onClose(); + }; + + const handleToggle = (record: EvaluationSet) => { + const isCurrent = record.evaluation_set_id === localSelectedId; + setLocalSelectedId(isCurrent ? null : record.evaluation_set_id); + setChanged(true); + }; + + const handleUpload = async () => { + const name = uploadName.trim(); + if (!name) { + messageApi.error(t("agentEvaluation.createSetModal.nameRequired")); + return; + } + if (!uploadFiles.length) { + messageApi.error(t("agentEvaluation.createSetModal.fileRequired")); + return; + } + setUploading(true); + try { + await evaluationService.uploadEvaluationSetExcel({ + name, + description: uploadDesc || undefined, + files: uploadFiles, + }); + messageApi.success(t("agentEvaluation.message.createSetSuccess")); + setUploadOpen(false); + setUploadName(""); + setUploadDesc(""); + setUploadFiles([]); + loadSets(); + } catch (err: any) { + messageApi.error(err?.message || t("agentEvaluation.message.createSetFailed")); + } finally { + setUploading(false); + } + }; + + const handleDelete = async (setId: number, setName: string) => { + Modal.confirm({ + title: t("agentEvaluation.lib.deleteConfirm", { name: setName }), + okText: t("common.confirm"), + cancelText: t("common.cancel"), + async onOk() { + await deleteSet(setId); + }, + }); + }; + + const columns: ColumnsType = [ + { + title: t("common.name"), + dataIndex: "name", + key: "name", + render: (name) => {name}, + }, + { + title: t("agentEvaluation.lib.caseCount"), + key: "case_count", + width: 100, + render: (_, record) => ( + {record.case_count ?? 0} + ), + }, + { + title: t("common.description"), + dataIndex: "description", + key: "description", + ellipsis: true, + render: (v) => {v || "-"}, + }, + { + title: t("common.actions"), + key: "actions", + width: 120, + render: (_, record) => { + const isCurrent = record.evaluation_set_id === localSelectedId; + return ( + + +