diff --git a/.dockerignore b/.dockerignore index f66110780..6792bb71d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -62,4 +62,4 @@ assets/ .Spotlight-V100 .Trashes ehthumbs.db -Thumbs.db +Thumbs.db diff --git a/.github/workflows/build-offline-package.yml b/.github/workflows/build-offline-package.yml index ae67ed4ca..54a6c4caf 100644 --- a/.github/workflows/build-offline-package.yml +++ b/.github/workflows/build-offline-package.yml @@ -6,15 +6,7 @@ on: version: description: 'Image version tag, e.g. v2.2.0 or latest' required: false - default: '' - platform: - description: 'Target platform' - required: false - default: 'amd64' - type: choice - options: - - amd64 - - arm64 + default: 'latest' image_source: description: 'Image source' required: false @@ -23,15 +15,6 @@ on: options: - general - mainland - target: - description: 'Package target' - required: false - default: 'all' - type: choice - options: - - docker - - k8s - - all include_source: description: 'Include source code in the package' required: false @@ -41,6 +24,12 @@ on: jobs: build-offline-package: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - amd64 + - arm64 steps: - name: Free disk space @@ -63,7 +52,7 @@ jobs: - name: Set version and platform variables id: set-vars run: | - PLATFORM="${{ inputs.platform }}" + PLATFORM="${{ matrix.platform }}" REF_TYPE="${{ github.ref_type }}" REF_NAME="${{ github.ref_name }}" @@ -81,9 +70,14 @@ jobs: VERSION="latest" fi + SOURCE_SUFFIX="" + if [ "${{ inputs.include_source }}" = "true" ]; then + SOURCE_SUFFIX="-with-source" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT echo "platform=$PLATFORM" >> $GITHUB_OUTPUT - echo "package-name=nexent-offline-${{ inputs.target }}-${PLATFORM}-${VERSION}" >> $GITHUB_OUTPUT + echo "package-name=nexent-${VERSION}-${PLATFORM}${SOURCE_SUFFIX}" >> $GITHUB_OUTPUT - name: Set deployment components id: set-components @@ -104,24 +98,26 @@ jobs: --include-source "${{ inputs.include_source }}" \ --image-source "${{ inputs.image_source }}" \ --components "${{ steps.set-components.outputs.components }}" \ - --target "${{ inputs.target }}" \ - --compress true - - + --target all \ + --compress false - - name: Show zip package + - name: Show offline package run: | PACKAGE_NAME="${{ steps.set-vars.outputs.package-name }}" - echo "Package created by build script: ${PACKAGE_NAME}.zip" + echo "Package directory created by build script: ./offline-output" + echo "GitHub Actions will publish the final downloadable artifact as ${PACKAGE_NAME}.zip" - ls -lh "${PACKAGE_NAME}.zip" + du -sh ./offline-output + find ./offline-output -maxdepth 2 -type f | sort | head -50 - name: Upload artifact uses: actions/upload-artifact@v4 with: name: ${{ steps.set-vars.outputs.package-name }} - path: ${{ steps.set-vars.outputs.package-name }}.zip + path: ./offline-output + if-no-files-found: error + include-hidden-files: true retention-days: 30 - name: Summary @@ -133,6 +129,7 @@ jobs: echo "Version: ${{ steps.set-vars.outputs.version }}" echo "Platform: ${{ steps.set-vars.outputs.platform }}" echo "Package: ${{ steps.set-vars.outputs.package-name }}.zip" + echo "Note: the downloaded artifact zip contains the offline package contents directly." echo "Target: ${{ inputs.target }}" echo "Components: ${{ steps.set-components.outputs.components }}" echo "Image source: ${{ inputs.image_source }}" diff --git a/.gitignore b/.gitignore index ece66528e..ebce75ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ docker/openssh-server docker/volumes/db/data docker/.env docker/.env.generated -deploy/docker/assets/monitoring/monitoring.env +deploy/env/monitoring.env docker/.run docker/deploy.options k8s/helm/deploy.options @@ -84,4 +84,4 @@ _doc/ /deploy/env/.env /deploy/env/.env.bak -agent_repository_frontend \ No newline at end of file +agent_repository_frontend diff --git a/README.md b/README.md index dd78f2763..de8fffeb4 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,7 @@ Nexent is a zero-code platform for auto-generating production-grade AI agents, b > ⭐ Before you get started, please star us on [GitHub](https://github.com/ModelEngine-Group/nexent) — your support drives us forward! -## Option 1: Try Our Official Demo - -No installation required — jump right in with our **[online demo environment](http://60.204.251.153:3000/en)** to experience Nexent's capabilities instantly. - -## Option 2: Deploy on Your Own +## Deploy on Your Own If you need to run Nexent locally or in your private infrastructure, we offer two deployment options: @@ -50,13 +46,13 @@ cd nexent bash deploy.sh docker ``` -The root `deploy.sh` only forwards to the target deploy script; the native Docker implementation is `bash deploy/docker/deploy.sh`. The Docker and Kubernetes deploy scripts share the same deployment configuration model. Interactive runs show Bash TUI menus for component selection, port policy, and image source. `infrastructure` is required; `application`, `data-process`, and `supabase` are selected by default and can be disabled when you want a smaller deployment. Use `b`/Backspace to return to the previous TUI step and `q` to quit. Non-interactive runs can pass the same choices with `--version`, `--components`, `--port-policy development|production`, and `--image-source general|mainland|local-latest`. Successful deployments save non-sensitive choices to each deploy directory's `deploy.options` for reuse on the next run. +The root `deploy.sh` only forwards to the target deploy script; the native Docker implementation is `bash deploy/docker/deploy.sh`. The Docker and Kubernetes deploy scripts share the same deployment configuration model. Interactive runs show Bash TUI menus for component selection, port policy, and image source. `infrastructure` is required; `application`, `data-process`, and `supabase` are selected by default and can be disabled when you want a smaller deployment. Use `b`/Backspace to return to the previous TUI step and `q` to quit. Use `--defaults` to skip the TUI and deploy with saved `deploy.options` or built-in defaults. Non-interactive runs can also pass the same choices with `--version`, `--components`, `--port-policy development|production`, and `--image-source general|mainland|local-latest`. Successful deployments save non-sensitive choices to each deploy directory's `deploy.options` for reuse on the next run. -Docker and Kubernetes both use `deploy/env/.env` as the runtime configuration file. Existing `deploy/env/.env` is kept as-is. If it does not exist, the deploy scripts first reuse `docker/.env`, then fall back to `deploy/env/.env.example`. +Docker and Kubernetes both use `deploy/env/.env` as the runtime configuration file. Existing `deploy/env/.env` is kept as-is. If it does not exist, the deploy scripts first reuse `docker/.env`, then fall back to `deploy/env/.env.example`. Monitoring-specific settings are generated from `deploy/env/monitoring.env.example` into `deploy/env/monitoring.env`. Docker uninstall is handled by `bash uninstall.sh docker`. It can preserve or delete data volumes: run it interactively, pass `--delete-volumes true|false`, or use `bash uninstall.sh docker delete-all` to remove containers and persistent data. -Offline image packages can be built with `bash deploy/offline/build_offline_package.sh --target docker --compress true`. The package includes image tar files, `load-images.sh`, root deploy/uninstall entrypoints, deployment scripts, SQL files, `manifest.yaml`, and `checksums.txt`; deploy it with `bash deploy.sh --load-images docker ...` on the target host. +Offline image packages can be built with `bash build.sh --package --target docker --compress true` or `bash deploy/offline/build_offline_package.sh --target docker --compress true`. The package includes image tar files, `load-images.sh`, `push-images.sh`, root deploy/uninstall entrypoints, deployment scripts, SQL files, `manifest.yaml`, and `checksums.txt`. Package deploys use saved `deploy.options` or built-in defaults without opening the TUI; add `--config` to configure interactively. Deploy with `bash deploy.sh --load-images docker ...` on the target host, or use `bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent docker ...` to push loaded images to an internal registry and deploy with that image prefix. When `--push-images` is used without a prefix, `deploy.sh` asks for it before `push-images.sh` prompts for the registry username and password. For detailed deployment instructions, see [Docker Installation](https://modelengine-group.github.io/nexent/en/quick-start/installation.html). @@ -74,7 +70,7 @@ The native Kubernetes implementation is `bash deploy/k8s/deploy.sh`. It reads th Kubernetes uninstall is handled by `bash uninstall.sh k8s`. It removes the Helm release first, then can optionally delete the namespace and local PV data. Use `--delete-namespace true|false`, `--delete-local-data true|false`, or `bash uninstall.sh k8s delete-all`; pass `--keep-local-data` with `delete-all` to preserve local volume contents. -Kubernetes offline packages use the same builder with `--target k8s` or `--target all`. Run `load-images.sh` on every cluster node that needs the images, or push the loaded images to an internal registry before deploying with the same version and image-source options used during packaging. +Kubernetes offline packages use the same builder with `--target k8s` or `--target all`. Run `load-images.sh` on every cluster node that needs the images, or use `--push-images --image-registry-prefix registry.example.com/nexent` to push the images to an internal registry before deploying with the same version, image source, and image registry prefix. For detailed deployment instructions, see [Kubernetes Installation](https://modelengine-group.github.io/nexent/en/quick-start/kubernetes-installation.html). diff --git a/README_CN.md b/README_CN.md index 0368dad24..1a4052b8c 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,11 +20,7 @@ Nexent 是一个基于 **Harness Engineering** 原则打造的零代码智能体 > ⭐ 在您开始使用前，请您顺手在 [GitHub](https://github.com/ModelEngine-Group/nexent) 为我们点个 Star，您的支持是我们前进的动力！ -## 方式一：使用官方体验环境 - -无需安装，直接访问我们的 **[在线体验环境](http://60.204.251.153:3000/zh)**，快速体验 Nexent 的强大功能。 - -## 方式二：自行部署 +## 自行部署 如果需要在本地或私有环境中部署 Nexent，我们提供两种部署方式： @@ -50,13 +46,13 @@ cd nexent bash deploy.sh docker ``` -根目录 `deploy.sh` 只负责转发到目标部署脚本；Docker 真实实现为 `bash deploy/docker/deploy.sh`。Docker 和 Kubernetes 使用同一套部署配置模型；交互式运行会通过 Bash TUI 选择组件、端口策略和镜像源。`infrastructure` 必选，`application`、`data-process`、`supabase` 默认选中，也可以取消以部署更小的组合。非交互部署可传入 `--version`、`--components`、`--port-policy development|production`、`--image-source general|mainland|local-latest`。 +根目录 `deploy.sh` 只负责转发到目标部署脚本；Docker 真实实现为 `bash deploy/docker/deploy.sh`。Docker 和 Kubernetes 使用同一套部署配置模型；交互式运行会通过 Bash TUI 选择组件、端口策略和镜像源。`infrastructure` 必选，`application`、`data-process`、`supabase` 默认选中，也可以取消以部署更小的组合。使用 `--defaults` 可跳过 TUI，并复用已保存的 `deploy.options` 或内置默认值。非交互部署也可传入 `--version`、`--components`、`--port-policy development|production`、`--image-source general|mainland|local-latest`。 -Docker 与 Kubernetes 统一使用 `deploy/env/.env` 作为运行配置文件；已有 `deploy/env/.env` 会原样保留。如果 `deploy/env/.env` 不存在，部署脚本会优先复用已有的 `docker/.env`，再回退到 `deploy/env/.env.example`。 +Docker 与 Kubernetes 统一使用 `deploy/env/.env` 作为运行配置文件；已有 `deploy/env/.env` 会原样保留。如果 `deploy/env/.env` 不存在，部署脚本会优先复用已有的 `docker/.env`，再回退到 `deploy/env/.env.example`。监控相关配置会从 `deploy/env/monitoring.env.example` 生成到 `deploy/env/monitoring.env`。 Docker 卸载入口为 `bash uninstall.sh docker`，默认交互确认是否删除持久化数据；也可以通过 `--delete-volumes true|false` 控制，或使用 `bash uninstall.sh docker delete-all` 同时删除容器和持久化数据。 -离线镜像包可通过 `bash deploy/offline/build_offline_package.sh --target docker --compress true` 构建。包内包含镜像 tar、`load-images.sh`、根目录部署/卸载入口、部署脚本、SQL 文件、`manifest.yaml` 和 `checksums.txt`；在目标机器上使用 `bash deploy.sh --load-images docker ...` 加载镜像并部署。 +离线镜像包可通过 `bash build.sh --package --target docker --compress true` 或 `bash deploy/offline/build_offline_package.sh --target docker --compress true` 构建。包内包含镜像 tar、`load-images.sh`、`push-images.sh`、根目录部署/卸载入口、部署脚本、SQL 文件、`manifest.yaml` 和 `checksums.txt`。包内部署会复用已保存的 `deploy.options` 或内置默认值，默认不进入 TUI；添加 `--config` 可交互配置。在目标机器上使用 `bash deploy.sh --load-images docker ...` 加载镜像并部署，或使用 `bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent docker ...` 推送到内部仓库并使用该镜像前缀部署。启用 `--push-images` 且未传前缀时，`deploy.sh` 会先询问镜像仓库前缀，随后 `push-images.sh` 询问仓库账号和密码。 详细部署指南请参考 [Docker 安装部署](https://modelengine-group.github.io/nexent/zh/quick-start/installation.html)。 @@ -74,7 +70,7 @@ Kubernetes 真实实现为 `bash deploy/k8s/deploy.sh`。它会读取同一个`d 根目录卸载入口为 `bash uninstall.sh docker ...` 或 `bash uninstall.sh k8s ...`，具体实现仍分别在 `deploy/docker/uninstall.sh` 和 `deploy/k8s/uninstall.sh`。 -Kubernetes 离线包使用同一个构建脚本，传入 `--target k8s` 或 `--target all`。部署前需要在每个需要运行 Pod 的节点上执行 `load-images.sh`，或将镜像推送到集群可访问的内部镜像仓库，再使用与打包时一致的版本和镜像源参数部署。 +Kubernetes 离线包使用同一个构建脚本，传入 `--target k8s` 或 `--target all`。部署前需要在每个需要运行 Pod 的节点上执行 `load-images.sh`，或使用 `--push-images --image-registry-prefix registry.example.com/nexent` 将镜像推送到集群可访问的内部镜像仓库，再使用与打包时一致的版本、镜像源和镜像仓库前缀部署。 详细部署指南请参考 [Kubernetes 安装部署](https://modelengine-group.github.io/nexent/zh/quick-start/kubernetes-installation.html)。 diff --git a/backend/adapters/jiuwen_sdk_adapter.py b/backend/adapters/jiuwen_sdk_adapter.py index f62ce9d06..c70194c99 100644 --- a/backend/adapters/jiuwen_sdk_adapter.py +++ b/backend/adapters/jiuwen_sdk_adapter.py @@ -1,13 +1,13 @@ """ openjiuwen SDK adapter for Nexent. -This module must be imported lazily (not at module load time) because -openjiuwen 0.1.13 has circular import bugs in its __init__.py files that -prevent the SDK from loading unless we bypass them. +This module works around circular import bugs in openjiuwen 0.1.13 by: + 1. Stubbing react_agent_evolve (causes the circular import) + 2. Installing a meta-path finder that blocks broken __init__.py files + 3. All of the above runs at MODULE LOAD TIME (before openjiuwen imports) -Import flow: - backend/adapters/__init__.py -> try/except -> JiuwenSDKAdapter = None - -> when needed: _install_jiuwen_bypasser() -> openjiuwen imports work +The adapters/__init__.py wraps the import in try/except so the server +starts even when openjiuwen is not installed. """ import asyncio import importlib.abc @@ -17,21 +17,73 @@ import os import sys import types -from typing import Any, List, Literal, Optional - -logger = logging.getLogger("jiuwen_adapter") - -from adapters.exception import JiuwenSDKError - +from typing import Any, List, Literal, Optional, Tuple # ---------------------------------------------------------------------- -# Circular import bypasser for openjiuwen 0.1.13 -# -# openjiuwen has broken __init__.py files that create circular import chains: -# tune/__init__.py -> tune.optimizer -> core.operator -> agent_evolving -> ... -# This bypasser prevents those __init__.py files from executing while still -# allowing regular .py submodule files to load normally. +# MUST install bypasser + stubs before any openjiuwen import. +# The module-level `from openjiuwen.dev_tools.tune.base import ...` below +# triggers the entire circular import chain. Installing the bypasser here +# (before that line executes) breaks the cycle. # ---------------------------------------------------------------------- + +# Stub react_agent_evolve so single_agent.__init__.py can import it without +# hitting the circular core.operator dependency. +if "openjiuwen.core.single_agent.agents.react_agent_evolve" not in sys.modules: + _stub = types.ModuleType("openjiuwen.core.single_agent.agents.react_agent_evolve") + _stub.__file__ = "" + + # Provide a dummy ReActAgentEvolve class. The real one lives in the + # actual react_agent_evolve.py which we cannot load at this stage + # because it imports ToolCallOperator from core.operator (still blocked). + # The stub class satisfies the import in single_agent.__init__.py. + class _DummyReActAgentEvolve: # noqa: N801 + pass + + _stub.ReActAgentEvolve = _DummyReActAgentEvolve + _stub.__all__ = ["ReActAgentEvolve"] + sys.modules["openjiuwen.core.single_agent.agents.react_agent_evolve"] = _stub + +# NOTE: do NOT stub openjiuwen.core.single_agent. Its real __init__.py must +# run so that `from .schema.agent_card import AgentCard` and the other +# relative submodule imports resolve correctly. We only need react_agent_evolve +# stubbed (its real file imports ToolCallOperator from core.operator, which +# the bypasser below blocks). + +# Stub missing optional dependencies before openjiuwen import chain reaches them +for _name, _attrs in [ + ("pymilvus", {"is_successful": lambda *a, **kw: True}), + ("dashscope", {}), + ("pdfplumber", {}), +]: + if _name not in sys.modules: + _mod = types.ModuleType(_name) + _mod.__path__ = [] + for _k, _v in _attrs.items(): + setattr(_mod, _k, _v) + sys.modules[_name] = _mod + +for _name in ["pymilvus.client", "pymilvus.client.utils"]: + if _name not in sys.modules: + _m = types.ModuleType(_name) + _m.__path__ = [] + if _name == "pymilvus.client.utils": + _m.is_successful = lambda *a, **kw: True + sys.modules[_name] = _m + +for _name, _attrs in [ + ("dashscope.api_entities", {}), + ("dashscope.api_entities.data", {}), + ("dashscope.api_entities.dashscope_response", {"DashScopeAPIResponse": object}), + ("dashscope.common", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), + ("dashscope.common.constants", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), +]: + if _name not in sys.modules: + _m = types.ModuleType(_name) + _m.__path__ = [] + for _k, _v in _attrs.items(): + setattr(_m, _k, _v) + sys.modules[_name] = _m + _CIRCULAR_CHAIN = { "openjiuwen.agent_evolving", "openjiuwen.agent_evolving.trainer", @@ -57,30 +109,24 @@ class _JiuwenInitBypasser(importlib.abc.MetaPathFinder, importlib.abc.Loader): def find_spec(self, fullname: str, path: Any, target: Any = None) -> Any: if not fullname.startswith("openjiuwen") or fullname == "openjiuwen": return None - try: import openjiuwen as _oj pkg_root = _oj.__path__[0] except ImportError: return None - parts = fullname.split(".")[1:] file_path = pkg_root for p in parts: file_path = os.path.join(file_path, p) - is_package = os.path.isdir(file_path) if not is_package: return None - init_path = os.path.join(file_path, "__init__.py") if not os.path.exists(init_path): return None - if fullname not in _CIRCULAR_CHAIN: return None - spec = importlib.machinery.ModuleSpec( fullname, self, is_package=True, origin="" ) @@ -106,7 +152,6 @@ def __getattr__(self, name: str) -> Any: import openjiuwen as _oj import importlib - # Prevent recursion when Python scans sys.meta_path for find_distributions etc. if name in ( "find_distributions", "find_module", @@ -118,87 +163,55 @@ def __getattr__(self, name: str) -> Any: "__spec__", ): raise AttributeError(name) - pkg_root = _oj.__path__[0] parts = self.__name__.split(".")[1:] + [name] file_path = pkg_root for p in parts: file_path = os.path.join(file_path, p) - - # If it's a package directory, import it as a submodule if os.path.isdir(file_path) and os.path.exists(os.path.join(file_path, "__init__.py")): return importlib.import_module(f"{self.__name__}.{name}") - # If it's a regular .py file if os.path.exists(file_path + ".py"): return importlib.import_module(f"{self.__name__}.{name}") raise AttributeError(name) -_bypasser_installed = False +# Install the bypasser into sys.meta_path so it intercepts openjiuwen imports. +for _finder in sys.meta_path: + if isinstance(_finder, _JiuwenInitBypasser): + break +else: + sys.meta_path.insert(0, _JiuwenInitBypasser()) + +# No-op: bypasser is already installed above. +# Kept for backward compatibility with code that calls it. +_bypasser_installed = True def _install_jiuwen_bypasser() -> bool: - """ - Install the circular import bypasser for openjiuwen. - Returns True if installed, False if already installed or openjiuwen not available. - """ - global _bypasser_installed - if _bypasser_installed: - return True - - # Stub missing optional dependencies before openjiuwen import chain reaches them - _stubbed = [ - ("pymilvus", {"is_successful": lambda *args, **kwargs: True}), - ("dashscope", {}), - ("pdfplumber", {}), - ] - for _name, _attrs in _stubbed: - if _name not in sys.modules: - _mod = types.ModuleType(_name) - for _k, _v in _attrs.items(): - setattr(_mod, _k, _v) - sys.modules[_name] = _mod - _mod.__path__ = [] - - # Pre-create nested stub modules for pymilvus.client.utils chain - if "pymilvus.client" not in sys.modules: - _client_mod = types.ModuleType("pymilvus.client") - _client_mod.__path__ = [] - sys.modules["pymilvus.client"] = _client_mod - if "pymilvus.client.utils" not in sys.modules: - _utils_mod = types.ModuleType("pymilvus.client.utils") - _utils_mod.is_successful = lambda *args, **kwargs: True - sys.modules["pymilvus.client.utils"] = _utils_mod - - # Stub dashscope sub-modules that may be imported lazily - _dashscope_subs = [ - ("dashscope.api_entities", {}), - ("dashscope.api_entities.data", {}), - ("dashscope.api_entities.dashscope_response", {"DashScopeAPIResponse": object}), - ("dashscope.common", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ("dashscope.common.constants", {"REQUEST_TIMEOUT_KEYWORD": "timeout"}), - ] - for _name, _attrs in _dashscope_subs: - if _name not in sys.modules: - _m = types.ModuleType(_name) - _m.__path__ = [] - for _k, _v in _attrs.items(): - setattr(_m, _k, _v) - sys.modules[_name] = _m + """No-op: bypasser is installed at module load time. Kept for compatibility.""" + return True +# Now safe to import openjiuwen — bypasser is active. +from openjiuwen.dev_tools.tune.base import Case as _Case, EvaluatedCase as _EvaluatedCase + + +def _case_from_inputs_label(inputs: dict, label: dict) -> "_Case": + # Keep stable keys for schema; allow extra fields to exist. + return _Case(inputs=inputs, label=label) + + +def _extract_score_reason(evaluated: Any) -> Tuple[float, str]: + """Try to normalize Jiuwen EvaluatedCase into (score, reason).""" try: - import openjiuwen # noqa: F401 - except ImportError: - return False + score = float(getattr(evaluated, "score", 0.0) or 0.0) + except Exception: + score = 0.0 + reason = getattr(evaluated, "reason", "") or "" + return score, str(reason) - for finder in sys.meta_path: - if isinstance(finder, _JiuwenInitBypasser): - _bypasser_installed = True - return True +logger = logging.getLogger("jiuwen_adapter") - sys.meta_path.insert(0, _JiuwenInitBypasser()) - _bypasser_installed = True - return True +from adapters.exception import JiuwenSDKError # ---------------------------------------------------------------------- @@ -247,8 +260,8 @@ def run_in_thread(): # ---------------------------------------------------------------------- # Jiuwen SDK lazy import helpers # ---------------------------------------------------------------------- -def _lazy_import_jiuwen_config(): - """Lazily import only lightweight Jiuwen config classes.""" +def _lazy_import_jiuwen(): + """延迟导入 Jiuwen SDK，避免 openjiuwen 未装时模块级 ImportError""" _install_jiuwen_bypasser() try: @@ -261,8 +274,33 @@ def _lazy_import_jiuwen_config(): ModelClientConfig, ProviderType, ) + # Optional: allow adjusting SDK internal client timeout via config if supported. + # We keep import local to avoid breaking on SDK version differences. + try: + from openjiuwen.core.foundation.llm import ModelClientOptions # type: ignore + except Exception: # pragma: no cover + ModelClientOptions = None # type: ignore + from openjiuwen.dev_tools.prompt_builder.builder.feedback_prompt_builder import ( + FeedbackPromptBuilder, + ) + from openjiuwen.dev_tools.prompt_builder.builder.badcase_prompt_builder import ( + BadCasePromptBuilder, + ) + from openjiuwen.dev_tools.tune.base import Case, EvaluatedCase - return ModelRequestConfig, ModelClientConfig, ProviderType + # Evaluator metrics (avoid importing openjiuwen.dev_tools.tune package root) + from openjiuwen.agent_evolving.evaluator.metrics.llm_as_judge import LLMAsJudgeMetric + + return ( + ModelRequestConfig, + ModelClientConfig, + ProviderType, + FeedbackPromptBuilder, + BadCasePromptBuilder, + Case, + EvaluatedCase, + LLMAsJudgeMetric, + ) def build_jiuwen_model_configs(model_id: int, tenant_id: str): @@ -270,7 +308,7 @@ def build_jiuwen_model_configs(model_id: int, tenant_id: str): from database.model_management_db import get_model_by_model_id from utils.config_utils import get_model_name_from_config - ModelRequestConfig, ModelClientConfig, ProviderType = _lazy_import_jiuwen_config() + ModelRequestConfig, ModelClientConfig, ProviderType, _, _, _, _, _ = _lazy_import_jiuwen() model_config = get_model_by_model_id(model_id, tenant_id) if not model_config: @@ -307,6 +345,40 @@ def build_jiuwen_model_configs(model_id: int, tenant_id: str): return request_config, client_config +def _build_openai_client( + client_config: "ModelClientConfig", request_config: "ModelRequestConfig" +) -> Any: + """ + 直接创建 OpenAIModelClient，绕过 FeedbackPromptBuilder 的深导入链。 + """ + _install_jiuwen_bypasser() + from openjiuwen.core.foundation.llm.model_clients.openai_model_client import ( + OpenAIModelClient, + ) + + class _DirectClient: + """对 OpenAIModelClient 的薄封装，只暴露 invoke() 方法""" + + def __init__(self, inner: Any): + self._inner = inner + self._model = request_config.model_name + + async def invoke(self, messages: list[dict]) -> Any: + return await self._inner.invoke( + model=self._model, + messages=messages, + temperature=request_config.temperature, + top_p=request_config.top_p, + ) + + client = OpenAIModelClient( + model_config=request_config, model_client_config=client_config + ) + return _DirectClient(client) + + +def _build_message(role: str, content: str) -> dict: + return {"role": role, "content": content} def _lazy_import_jiuwen_builders(): """Lazily import prompt builders only when optimization paths need them.""" _install_jiuwen_bypasser() @@ -366,16 +438,9 @@ def _unwrap_prompt_response(text: str) -> str: return text -def _lazy_import_jiuwen_tune_types(): - """Lazily import Jiuwen tune types only when badcase flow needs them.""" - _install_jiuwen_bypasser() - from openjiuwen.dev_tools.tune.base import Case, EvaluatedCase - return Case, EvaluatedCase - - def to_jiuwen_evaluated_case(bad_case) -> Any: """将 nexent BadCase 转换为 Jiuwen EvaluatedCase""" - Case, EvaluatedCase = _lazy_import_jiuwen_tune_types() + _, _, _, _, _, Case, EvaluatedCase, _ = _lazy_import_jiuwen() case = Case( inputs={"question": bad_case.question}, @@ -508,6 +573,103 @@ def optimize_badcase( self.logger.error(f"Jiuwen BadCasePromptBuilder 调用失败: {e}") raise JiuwenSDKError(f"BadCasePromptBuilder 调用失败: {e}") from e + def evaluate_semantic_consistency( + self, + *, + question: str, + expected_answer: str, + model_answer: str, + user_metrics: str = "", + ) -> Tuple[float, str]: + """LLM-as-judge semantic consistency scoring. + + Returns: + (score, reason) + + Notes: + openjiuwen 0.1.13 implements LLMAsJudgeMetric as a binary metric + (1.0 pass / 0.0 fail) by parsing a JSON result from the judge model. + ``metric.compute()`` only returns the numeric score; the actual judge + verdict (with ``result`` and ``reason``) lives in the LLM response. + We invoke the model directly here so we can extract **both** the score + and the human-readable reason from the same response. + """ + self._ensure_available() + + try: + from openjiuwen.agent_evolving.evaluator.metrics.llm_as_judge import ( + LLMAsJudgeMetric, + ) + except Exception as exc: + raise JiuwenSDKError(f"Failed to import LLMAsJudgeMetric: {exc}") from exc + + request_config, client_config = build_jiuwen_model_configs(self.model_id, self.tenant_id) + + metric = LLMAsJudgeMetric( + model_config=request_config, + model_client_config=client_config, + user_metrics=user_metrics or "", + ) + + # Build the same prompt that ``metric.compute()`` uses. + messages = metric._template.format({ + "question": str(question or ""), + "expected_answer": str(expected_answer), + "model_answer": str(model_answer), + }).to_messages() + + # Append a directive requiring Chinese output for the evaluation reason. + # This ensures consistent Chinese language in the report regardless of + # the judge model's default language. + chinese_directive = ( + "\n\nIMPORTANT: You MUST respond in Chinese for the 'reason' field. " + "The reason must be a clear explanation in Simplified Chinese. " + "重要提示：'reason' 字段必须使用中文撰写，用简洁的中文解释评判结果。" + ) + + if messages: + last = messages[-1] + last_role = getattr(last, "role", None) + # Default template returns a single UserMessage with role="user" + if last_role == "user": + last.content = (last.content or "") + chinese_directive + else: + # Append a new user message with the directive + from openjiuwen.core.foundation.llm import UserMessage + messages.append(UserMessage(content=chinese_directive.lstrip("\n"))) + + try: + response = asyncio.run(metric._model.invoke(messages)).content + except Exception as exc: + raise JiuwenSDKError(f"Judge LLM invoke failed: {exc}") from exc + + try: + from openjiuwen.agent_evolving.utils import TuneUtils + data = TuneUtils.parse_json_from_llm_response(response) + except Exception as exc: + raise JiuwenSDKError(f"Failed to parse judge response: {exc}") from exc + + if not isinstance(data, dict): + raise JiuwenSDKError(f"Judge response is not a JSON object: {response!r}") + + result = data.get("result") + if result is True or (isinstance(result, str) and result.strip().lower() == "true"): + score = 1.0 + else: + score = 0.0 + + raw_reason = data.get("reason") + if isinstance(raw_reason, str): + reason = raw_reason.strip() + elif isinstance(raw_reason, dict): + reason = raw_reason.get("reason") or "" + else: + reason = "" + if not reason: + reason = "通过" if score >= 1.0 else "失败" + + return score, reason + def generate(self, **kwargs) -> dict: """调用 Jiuwen 提示词生成能力""" self._ensure_available() diff --git a/backend/agents/agent_run_manager.py b/backend/agents/agent_run_manager.py index eca8c2fa4..33703fafa 100644 --- a/backend/agents/agent_run_manager.py +++ b/backend/agents/agent_run_manager.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, Union from nexent.core.agents.agent_model import AgentRunInfo +from services.runtime_state_service import runtime_state_service if TYPE_CHECKING: from nexent.core.agents.agent_context import ContextManager, ContextManagerConfig @@ -45,8 +46,9 @@ def register_agent_run(self, conversation_id: Union[int, str], agent_run_info, u self._conversation_run_counts[conv_key] = self._conversation_run_counts.get(conv_key, 0) + 1 logger.info( f"register agent run instance, user_id: {user_id}, conversation_id: {conversation_id}") + runtime_state_service.register_run(user_id=user_id, conversation_id=conversation_id) - def unregister_agent_run(self, conversation_id: Union[int, str], user_id: str): + def unregister_agent_run(self, conversation_id: Union[int, str], user_id: str, status: str = "completed"): """unregister agent run instance""" with self._lock: run_key = self._get_run_key(conversation_id, user_id) @@ -61,6 +63,7 @@ def unregister_agent_run(self, conversation_id: Union[int, str], user_id: str): else: logger.info( f"no agent run instance found for user_id: {user_id}, conversation_id: {conversation_id}") + runtime_state_service.mark_run_finished(user_id=user_id, conversation_id=conversation_id, status=status) def get_agent_run_info(self, conversation_id: Union[int, str], user_id: str): """get agent run instance""" @@ -69,13 +72,17 @@ def get_agent_run_info(self, conversation_id: Union[int, str], user_id: str): def stop_agent_run(self, conversation_id: Union[int, str], user_id: str) -> bool: """stop agent run for specified conversation_id and user_id""" + remote_signal_set = runtime_state_service.set_cancel_signal( + user_id=user_id, + conversation_id=conversation_id, + ) agent_run_info = self.get_agent_run_info(conversation_id, user_id) if agent_run_info is not None: agent_run_info.stop_event.set() logger.info( f"agent run stopped, user_id: {user_id}, conversation_id: {conversation_id}") return True - return False + return remote_signal_set def get_or_create_context_manager( self, diff --git a/backend/agents/create_agent_info.py b/backend/agents/create_agent_info.py index c02addff8..e5708904d 100644 --- a/backend/agents/create_agent_info.py +++ b/backend/agents/create_agent_info.py @@ -762,6 +762,9 @@ async def create_agent_config( "agent_id": memory_context.agent_id, } + memory_tool_names = {"store_memory", "search_memory"} + tool_list = [t for t in tool_list if t.name not in memory_tool_names] + store_tool_config = ToolConfig( class_name="StoreMemoryTool", name="store_memory", @@ -859,12 +862,14 @@ async def create_agent_config( skills = _get_skills_for_template(agent_id, tenant_id, version_no) is_manager = len(managed_agents) > 0 or len(external_a2a_agents) > 0 + builtin_tools = _get_skill_script_tools(agent_id, tenant_id, version_no) + available_tools = tool_list + builtin_tools render_kwargs = { "duty": duty_prompt, "constraint": constraint_prompt, "few_shots": few_shots_prompt, - "tools": {tool.name: tool for tool in tool_list}, + "tools": {tool.name: tool for tool in available_tools}, "skills": skills, "managed_agents": {agent.name: agent for agent in managed_agents}, "external_a2a_agents": {agent.agent_id: agent for agent in external_a2a_agents}, @@ -955,6 +960,7 @@ async def create_agent_config( token_threshold=context_token_threshold, soft_input_budget_tokens=soft_input_budget_tokens, hard_input_budget_tokens=hard_input_budget_tokens, + strategy="full", ) agent_config = AgentConfig( name="undefined" if agent_info["name"] is None else agent_info["name"], @@ -965,7 +971,7 @@ async def create_agent_config( language=language, agent_id=agent_id ), - tools=tool_list + _get_skill_script_tools(agent_id, tenant_id, version_no), + tools=available_tools, max_steps=agent_info.get("max_steps", 15), requested_output_tokens=requested_output_tokens, model_name=model_name, @@ -1089,7 +1095,7 @@ async def create_tool_config_list( f"No embedding model found for index '{index_names[0]}'. " f"Please configure an embedding model for this knowledge base.") tool_config.metadata["embedding_model"] = embedding_model - elif tool_config.class_name in ["DifySearchTool", "DataMateSearchTool"]: + elif tool_config.class_name in ["DifySearchTool", "DataMateSearchTool", "RAGFlowSearchTool"]: rerank = tool_config.params.get("rerank", False) rerank_model_name = tool_config.params.get("rerank_model_name", "") rerank_model = None diff --git a/backend/apps/agent_evaluation_app.py b/backend/apps/agent_evaluation_app.py new file mode 100644 index 000000000..b163babe0 --- /dev/null +++ b/backend/apps/agent_evaluation_app.py @@ -0,0 +1,147 @@ +import logging +from http import HTTPStatus +from typing import Optional + +from fastapi import APIRouter, Body, Header, HTTPException, Query +from fastapi.responses import JSONResponse, StreamingResponse + +from services.agent_evaluation_service import ( + create_agent_evaluation_run_impl, + delete_agent_evaluation_run_impl, + generate_agent_evaluation_report_impl, + get_agent_evaluation_run_impl, + list_agent_evaluation_cases_impl, + list_agent_evaluations_by_agent_impl, +) +from utils.auth_utils import get_current_user_id + +logger = logging.getLogger("agent_evaluation_app") + +router = APIRouter(prefix="/agent-evaluations") + + +@router.post("") +async def create_agent_evaluation_api( + agent_id: int = Body(...), + evaluation_set_id: int = Body(...), + judge_model_id: int = Body(..., description="Model id used for judging (Jiuwen)"), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + run = create_agent_evaluation_run_impl( + tenant_id=tenant_id, + user_id=user_id, + agent_id=agent_id, + evaluation_set_id=evaluation_set_id, + judge_model_id=judge_model_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": run}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Create agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Create agent evaluation error") + + +@router.get("") +async def list_agent_evaluations_by_agent_api( + agent_id: int = Query(...), + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_agent_evaluations_by_agent_impl( + agent_id=agent_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List agent evaluations error: %r", exc) + raise HTTPException(status_code=500, detail="List agent evaluations error") + + +@router.get("/{agent_evaluation_id}") +async def get_agent_evaluation_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = get_agent_evaluation_run_impl(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("Get agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Get agent evaluation error") + + +@router.get("/{agent_evaluation_id}/cases") +async def list_agent_evaluation_cases_api( + agent_evaluation_id: int, + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_agent_evaluation_cases_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List agent evaluation cases error: %r", exc) + raise HTTPException(status_code=500, detail="List agent evaluation cases error") + + +@router.get("/{agent_evaluation_id}/report") +async def download_agent_evaluation_report_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data, fail_count = generate_agent_evaluation_report_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + ) + suffix = "_failed.xlsx" if fail_count > 0 else "_all.xlsx" + return StreamingResponse( + iter([data]), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={ + "Content-Disposition": f"attachment; filename=evaluation_report_{agent_evaluation_id}{suffix}" + }, + ) + except ValueError as ve: + raise HTTPException(status_code=404, detail=str(ve)) + except Exception as exc: + logger.exception("Download agent evaluation report error: %r", exc) + raise HTTPException(status_code=500, detail="Download agent evaluation report error") + + +@router.delete("/{agent_evaluation_id}") +async def delete_agent_evaluation_api( + agent_evaluation_id: int, + authorization: Optional[str] = Header(None), +): + """Soft-delete an evaluation run. Only the creator may delete.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + delete_agent_evaluation_run_impl( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + user_id=user_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Delete agent evaluation error: %r", exc) + raise HTTPException(status_code=500, detail="Delete agent evaluation error") diff --git a/backend/apps/agent_repository_app.py b/backend/apps/agent_repository_app.py index df96e576c..f7f063282 100644 --- a/backend/apps/agent_repository_app.py +++ b/backend/apps/agent_repository_app.py @@ -26,10 +26,6 @@ async def list_agent_repository_listings_api( status: Optional[str] = Query(None, description="Filter by listing status"), agent_id: Optional[int] = Query(None, description="Filter by source agent ID"), - category_id: Optional[int] = Query( - None, - description="Filter by marketplace category ID", - ), page: Annotated[int, Query(ge=1, description="Page number starting from 1")] = 1, page_size: Annotated[ int, Query(ge=1, le=100, description="Page size from 1 to 100") @@ -46,7 +42,6 @@ async def list_agent_repository_listings_api( tenant_id, status=status, agent_id=agent_id, - category_id=category_id, page=page, page_size=page_size, search=search, diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index 9ffadfe5e..b8998f19f 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -3,11 +3,13 @@ from apps.app_factory import create_app from apps.agent_app import agent_config_router as agent_router from apps.agent_repository_app import agent_repository_router +from apps.skill_repository_app import skill_repository_router from apps.config_sync_app import router as config_sync_router from apps.datamate_app import router as datamate_router from apps.vectordatabase_app import router as vectordatabase_router from apps.dify_app import router as dify_router from apps.idata_app import router as idata_router +from apps.ragflow_app import router as ragflow_router from apps.file_management_app import ( file_management_config_router as file_manager_router, ) @@ -33,6 +35,8 @@ from apps.monitoring_app import router as monitoring_router from apps.a2a_server_app import router as a2a_server_router from apps.haotian_app import router as haotian_router +from apps.evaluation_set_app import router as evaluation_set_router +from apps.agent_evaluation_app import router as agent_evaluation_router from apps.aidp_app import router as aidp_router from apps.cas_app import router as cas_router from consts.const import IS_SPEED_MODE @@ -58,6 +62,7 @@ async def sync_default_prompt_template_on_startup(): app.include_router(config_sync_router) app.include_router(agent_router) app.include_router(agent_repository_router) +app.include_router(skill_repository_router) app.include_router(vectordatabase_router) app.include_router(datamate_router) app.include_router(voice_router) @@ -66,6 +71,7 @@ async def sync_default_prompt_template_on_startup(): app.include_router(tool_config_router) app.include_router(dify_router) app.include_router(idata_router) +app.include_router(ragflow_router) app.include_router(monitoring_router) # Choose user management router based on IS_SPEED_MODE @@ -93,4 +99,6 @@ async def sync_default_prompt_template_on_startup(): app.include_router(a2a_client_router) app.include_router(a2a_server_router) app.include_router(haotian_router) +app.include_router(evaluation_set_router) +app.include_router(agent_evaluation_router) app.include_router(aidp_router) diff --git a/backend/apps/evaluation_set_app.py b/backend/apps/evaluation_set_app.py new file mode 100644 index 000000000..d9d870a72 --- /dev/null +++ b/backend/apps/evaluation_set_app.py @@ -0,0 +1,198 @@ +import io +import json +import logging +from http import HTTPStatus +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Body, File, Form, Header, HTTPException, Query, UploadFile +from fastapi.responses import JSONResponse, StreamingResponse + +from services.evaluation_set_service import ( + create_evaluation_set_from_cases, + create_evaluation_set_from_jsonl, + delete_evaluation_set_impl, + get_evaluation_set_impl, + list_evaluation_set_cases_impl, + list_evaluation_sets_impl, +) +from utils.auth_utils import get_current_user_id +from utils.evaluation_set_excel_utils import build_evaluation_set_excel_template_bytes, parse_evaluation_cases_from_excel + +logger = logging.getLogger("evaluation_set_app") + +router = APIRouter(prefix="/evaluation-sets") + + +@router.get("") +async def list_evaluation_sets_api( + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + data = list_evaluation_sets_impl(tenant_id=tenant_id, limit=limit, offset=offset) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List evaluation sets error: %r", exc) + raise HTTPException(status_code=500, detail="List evaluation sets error") + + +@router.post("") +async def create_evaluation_set_api( + name: str = Body(...), + description: Optional[str] = Body(None), + source_filename: Optional[str] = Body(None), + jsonl_text: str = Body(..., description="Raw JSONL content"), + authorization: Optional[str] = Header(None), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + meta = create_evaluation_set_from_jsonl( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + jsonl_text=jsonl_text, + created_by=user_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Create evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Create evaluation set error") + + +@router.post("/upload") +async def upload_evaluation_set_api( + name: str = Form(...), + description: Optional[str] = Form(None), + files: List[UploadFile] = File(...), + authorization: Optional[str] = Header(None, alias="Authorization"), +): + try: + user_id, tenant_id = get_current_user_id(authorization) + if not files: + raise ValueError("At least one file is required") + + all_cases: List[Dict[str, Any]] = [] + source_filenames: List[str] = [] + + for file in files: + raw = await file.read() + filename = file.filename or "" + source_filenames.append(filename) + lower = filename.lower() + + if lower.endswith(".xlsx") or lower.endswith(".xls"): + cases = parse_evaluation_cases_from_excel(filename=filename, raw=raw) + all_cases.extend(cases) + else: + # Backward compatible: still accept JSONL upload + try: + jsonl_text = raw.decode("utf-8") + except Exception: + jsonl_text = raw.decode("utf-8", errors="ignore") + + # Parse JSONL into cases + for line in jsonl_text.strip().splitlines(): + line = line.strip() + if not line: + continue + obj = json.loads(line) + all_cases.append({ + "query": obj.get("query", ""), + "answer": obj.get("answer", ""), + "context": obj.get("context"), + "case_id": obj.get("case_id"), + }) + + if not all_cases: + raise ValueError("No valid cases found in uploaded files") + + meta = create_evaluation_set_from_cases( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=", ".join(source_filenames), + cases=all_cases, + created_by=user_id, + ) + + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": meta}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Upload evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Upload evaluation set error") + + +@router.get("/template") +async def download_evaluation_set_template_api(): + """Download Excel template for evaluation set upload.""" + data = build_evaluation_set_excel_template_bytes() + headers = { + "Content-Disposition": 'attachment; filename="evaluation_set_template.xlsx"' + } + return StreamingResponse( + io.BytesIO(data), + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers=headers, + ) + + +@router.get("/{evaluation_set_id}") +async def get_evaluation_set_api( + evaluation_set_id: int, + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = get_evaluation_set_impl(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("Get evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Get evaluation set error") + + +@router.get("/{evaluation_set_id}/cases") +async def list_evaluation_set_cases_api( + evaluation_set_id: int, + limit: int = Query(50, ge=1, le=200), + offset: int = Query(0, ge=0), + authorization: Optional[str] = Header(None), +): + try: + _, tenant_id = get_current_user_id(authorization) + data = list_evaluation_set_cases_impl( + evaluation_set_id=evaluation_set_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success", "data": data}) + except Exception as exc: + logger.exception("List evaluation set cases error: %r", exc) + raise HTTPException(status_code=500, detail="List evaluation set cases error") + + +@router.delete("/{evaluation_set_id}") +async def delete_evaluation_set_api( + evaluation_set_id: int, + authorization: Optional[str] = Header(None), +): + """Soft-delete an evaluation set. + + Blocked when any active evaluation run still references the set, so + historical runs never lose their context. + """ + try: + user_id, tenant_id = get_current_user_id(authorization) + delete_evaluation_set_impl(evaluation_set_id, tenant_id, user_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"message": "Success"}) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as exc: + logger.exception("Delete evaluation set error: %r", exc) + raise HTTPException(status_code=500, detail="Delete evaluation set error") diff --git a/backend/apps/mcp_management_app.py b/backend/apps/mcp_management_app.py index cfb0c292a..f94e03e1f 100644 --- a/backend/apps/mcp_management_app.py +++ b/backend/apps/mcp_management_app.py @@ -1,5 +1,5 @@ -import logging -from typing import Optional +import logging +from typing import Annotated, Optional from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request from fastapi.responses import JSONResponse @@ -7,6 +7,7 @@ from consts.exceptions import ( MCPConnectionError, + McpNameConflictError, McpNotFoundError, McpValidationError, UnauthorizedError, @@ -15,17 +16,25 @@ RegistryListQuery, CommunityListRequest, CommunityPublishRequest, + CommunityReviewActionRequest, + CommunityReviewListRequest, + CommunityStatusUpdateRequest, CommunityUpdateRequest, ) from services.mcp_management_service import ( list_community_mcp_services, list_community_mcp_tag_stats, + approve_community_mcp_service, + change_mcp_market_status, + list_community_mcp_review_services, list_my_community_mcp_services, list_registry_mcp_services, publish_community_mcp_service, + reject_community_mcp_service, update_community_mcp_service, delete_community_mcp_service, ) +from database.market_mcp_db import increment_mcp_market_download_count from utils.auth_utils import get_current_user_info router = APIRouter(prefix="/mcp-tools") @@ -42,12 +51,9 @@ async def list_registry_mcp_services_api( authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - List MCP services from the official MCP Registry. - """ + """List MCP services from the official MCP Registry.""" try: get_current_user_info(authorization, http_request) - data = await list_registry_mcp_services( search=query.search, include_deleted=query.include_deleted, @@ -56,27 +62,21 @@ async def list_registry_mcp_services_api( cursor=query.cursor, limit=query.limit, ) - return JSONResponse( - status_code=HTTPStatus.OK, - content=data, - ) + return JSONResponse(status_code=HTTPStatus.OK, content=data) except UnauthorizedError as exc: - raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), - ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: logger.error(f"Failed to list MCP registry services: {exc}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to list MCP registry services" + detail="Failed to list MCP registry services", ) # --------------------------------------------------------------------------- -# Community Endpoints +# Community Endpoints — Public listing # --------------------------------------------------------------------------- @router.get("/community/list") @@ -85,34 +85,27 @@ async def list_community_mcp_services_api( authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - List public community MCP services. - """ + """List public community MCP services (shared only).""" try: - get_current_user_info(authorization, http_request) + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) data = await list_community_mcp_services( + tenant_id=tenant_id, search=query.search, tag=query.tag, transport_type=query.transport_type, cursor=query.cursor, limit=query.limit, ) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"status": "success", "data": data}, - ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success", "data": data}) except UnauthorizedError as exc: - raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), - ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: logger.error(f"Failed to list MCP community services: {exc}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to list MCP community services" + detail="Failed to list MCP community services", ) @@ -121,182 +114,422 @@ async def list_community_mcp_tag_stats_api( authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - Get community MCP tag statistics. - """ + """Get community MCP tag statistics.""" try: - get_current_user_info(authorization, http_request) - stats = list_community_mcp_tag_stats() - return JSONResponse( - status_code=HTTPStatus.OK, - content={"status": "success", "data": stats}, + _, tenant_id, _ = get_current_user_info(authorization, http_request) + stats = list_community_mcp_tag_stats(tenant_id=tenant_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success", "data": stats}) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to list community MCP tag stats: {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to list community MCP tag stats", + ) + + +@router.get("/community/mine") +async def list_my_community_mcp_services_api( + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """List MCP services published by the current user to the community.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + data = await list_my_community_mcp_services( + tenant_id=tenant_id, + user_id=user_id, ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success", "data": data}) except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to list my MCP community services: {exc}") raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to list my MCP community services", ) + + +# --------------------------------------------------------------------------- +# Community Endpoints — Review queue +# --------------------------------------------------------------------------- + +@router.get("/community/review/list") +async def list_community_mcp_review_services_api( + query: CommunityReviewListRequest = Depends(), + authorization: Annotated[Optional[str], Header()] = None, + http_request: Request = None, +): + """List MCP community submissions for administrator review.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + data = await list_community_mcp_review_services( + tenant_id=tenant_id, + user_id=user_id, + status=query.status, + search=query.search, + tag=query.tag, + transport_type=query.transport_type, + cursor=query.cursor, + limit=query.limit, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success", "data": data}) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: - logger.error(f"Failed to list community MCP tag stats: {exc}") + logger.exception("Failed to list MCP community review services") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to list community MCP tag stats" + detail="Failed to list MCP community review services", ) +# --------------------------------------------------------------------------- +# Legacy endpoints (deprecated — defined before path-param routes for routing) +# --------------------------------------------------------------------------- + @router.post("/community/publish") async def publish_community_mcp_service_api( payload: CommunityPublishRequest, authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - Publish a local MCP service to the community. - """ + """[Deprecated] Publish a local MCP service to the community. Use POST /community.""" try: user_id, tenant_id, _ = get_current_user_info(authorization, http_request) - community_id = await publish_community_mcp_service( + market_id = await publish_community_mcp_service( tenant_id=tenant_id, user_id=user_id, mcp_id=payload.mcp_id, name=payload.name, description=payload.description, - version=payload.version, tags=payload.tags, mcp_server=payload.mcp_server, config_json=payload.config_json, ) return JSONResponse( status_code=HTTPStatus.OK, - content={"status": "success", "data": {"community_id": community_id}}, + content={"status": "success", "data": {"market_id": market_id}}, ) except McpNotFoundError as exc: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) except McpValidationError as exc: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except McpNameConflictError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) except UnauthorizedError as exc: - raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), - ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: logger.error(f"Failed to publish MCP community service: {exc}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to publish MCP community service" + detail="Failed to publish MCP community service", ) @router.put("/community/update") -async def update_community_mcp_service_api( +async def update_community_mcp_service_legacy_api( payload: CommunityUpdateRequest, authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - Update a community MCP service. - """ + """[Deprecated] Update a community MCP service. Use PUT /community/{market_id}.""" try: user_id, tenant_id, _ = get_current_user_info(authorization, http_request) await update_community_mcp_service( tenant_id=tenant_id, user_id=user_id, - community_id=payload.community_id, + market_id=payload.market_id, name=payload.name, description=payload.description, tags=payload.tags, - version=payload.version, registry_json=payload.registry_json, + mcp_server=payload.mcp_server, + config_json=payload.config_json, + transport_type=payload.transport_type, ) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"status": "success"}, - ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) except McpNotFoundError as exc: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) except McpValidationError as exc: raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except McpNameConflictError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) except UnauthorizedError as exc: - raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), - ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: logger.error(f"Failed to update MCP community service: {exc}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to update MCP community service" + detail="Failed to update MCP community service", ) @router.delete("/community/delete") -async def delete_community_mcp_service_api( - community_id: int = Query(gt=0), +async def delete_community_mcp_service_legacy_api( + market_id: int = Query(gt=0), authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - Delete a community MCP service. - """ + """[Deprecated] Delete a market MCP service. Use DELETE /community/{market_id}.""" try: user_id, tenant_id, _ = get_current_user_info(authorization, http_request) await delete_community_mcp_service( tenant_id=tenant_id, user_id=user_id, - community_id=community_id, + market_id=market_id, ) - return JSONResponse( - status_code=HTTPStatus.OK, - content={"status": "success"}, + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to delete MCP community service: {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to delete MCP community service", + ) + + +@router.post("/community/review/approve") +async def approve_community_mcp_service_api( + payload: CommunityReviewActionRequest, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """[Deprecated] Approve an MCP community submission. Use PATCH /community/{id}/status.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await approve_community_mcp_service( + tenant_id=tenant_id, + user_id=user_id, + market_id=payload.review_id, ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) except McpNotFoundError as exc: raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to approve MCP community service: {exc}") raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to approve MCP community service", + ) + + +@router.post("/community/review/reject") +async def reject_community_mcp_service_api( + payload: CommunityReviewActionRequest, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """[Deprecated] Reject an MCP community submission. Use PATCH /community/{id}/status.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await reject_community_mcp_service( + tenant_id=tenant_id, + user_id=user_id, + market_id=payload.review_id, ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: - logger.error(f"Failed to delete MCP community service: {exc}") + logger.exception("Failed to reject MCP community service") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to delete MCP community service" + detail="Failed to reject MCP community service", ) -@router.get("/community/mine") -async def list_my_community_mcp_services_api( +# --------------------------------------------------------------------------- +# Community Endpoints — RESTful resource operations +# --------------------------------------------------------------------------- + +@router.post("/community") +async def create_community_mcp_service_api( + payload: CommunityPublishRequest, authorization: Optional[str] = Header(None), http_request: Request = None, ): - """ - List MCP services published by the current user to the community. - """ + """Create a new community MCP listing (submits for review).""" try: - _, tenant_id, _ = get_current_user_info(authorization, http_request) - data = await list_my_community_mcp_services(tenant_id=tenant_id) + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + market_id = await publish_community_mcp_service( + tenant_id=tenant_id, + user_id=user_id, + mcp_id=payload.mcp_id, + name=payload.name, + description=payload.description, + tags=payload.tags, + mcp_server=payload.mcp_server, + config_json=payload.config_json, + ) return JSONResponse( status_code=HTTPStatus.OK, - content={"status": "success", "data": data}, + content={"status": "success", "data": {"market_id": market_id}}, ) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except McpValidationError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except McpNameConflictError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to create community MCP listing: {exc}") raise HTTPException( - status_code=HTTPStatus.UNAUTHORIZED, - detail=str(exc), + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to create community MCP listing", + ) + + +@router.put("/community/{market_id}") +async def update_community_mcp_service_api( + market_id: int, + payload: CommunityUpdateRequest, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """Update a community MCP listing and submit for re-review.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await update_community_mcp_service( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + name=payload.name, + description=payload.description, + tags=payload.tags, + registry_json=payload.registry_json, + mcp_server=payload.mcp_server, + config_json=payload.config_json, + transport_type=payload.transport_type, ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except McpValidationError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except McpNameConflictError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) except HTTPException: raise except Exception as exc: - logger.error(f"Failed to list my MCP community services: {exc}") + logger.error(f"Failed to update community MCP listing: {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to update community MCP listing", + ) + + +@router.delete("/community/{market_id}") +async def delete_community_mcp_service_api( + market_id: int, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """Delete a community MCP listing.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await delete_community_mcp_service( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to delete community MCP listing: {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to delete community MCP listing", + ) + + +@router.patch("/community/{market_id}/status") +async def change_community_mcp_status_api( + market_id: int, + payload: CommunityStatusUpdateRequest, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """Change MCP listing status (approve, reject, withdraw, unshare).""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await change_mcp_market_status( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + new_status=payload.status, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success"}) + except McpNotFoundError as exc: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(exc)) + except McpNameConflictError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except ValueError as exc: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)) + except HTTPException: + raise + except Exception as exc: + logger.error(f"Failed to change MCP listing status: {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to change MCP listing status", + ) + + +@router.post("/community/{market_id}/download") +async def increment_community_mcp_download_count_api( + market_id: int, + authorization: Optional[str] = Header(None), + http_request: Request = None, +): + """Increment the download counter when a user installs a community MCP.""" + try: + get_current_user_info(authorization, http_request) + increment_mcp_market_download_count(market_id) + return JSONResponse(status_code=HTTPStatus.OK, content={"status": "success", "data": None}) + except UnauthorizedError as exc: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(exc)) + except Exception as exc: + logger.error(f"Failed to increment download count: {exc}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Failed to list my MCP community services" + detail="Failed to increment download count", ) diff --git a/backend/apps/northbound_knowledge_app.py b/backend/apps/northbound_knowledge_app.py index 02739d138..a5b190ee8 100644 --- a/backend/apps/northbound_knowledge_app.py +++ b/backend/apps/northbound_knowledge_app.py @@ -11,7 +11,8 @@ LimitExceededError, UnauthorizedError, ) -from consts.model import ProcessParams +from consts.model import HybridSearchRequest, ProcessParams +from database.knowledge_db import get_index_name_by_knowledge_name from services.file_management_service import ( upload_files_impl, get_file_url_impl, @@ -20,7 +21,11 @@ ) from services.northbound_service import NorthboundContext from services.redis_service import get_redis_service -from services.vectordatabase_service import ElasticSearchService, get_vector_db_core +from services.vectordatabase_service import ( + ElasticSearchService, + KnowledgeBaseNeedsModelConfigError, + get_vector_db_core, +) from utils.auth_utils import generate_session_jwt from utils.file_management_utils import trigger_data_process @@ -221,6 +226,132 @@ async def get_index_files( detail="Error getting index files") +@router.post("/indices/{index_name}/chunks") +async def get_index_chunks( + request: Request, + index_name: Annotated[str, Path(..., description="Name of the index")], + page: Annotated[ + Optional[int], + Query(description="Page number (1-based) for pagination"), + ] = None, + page_size: Annotated[ + Optional[int], + Query(description="Number of records per page for pagination"), + ] = None, + path_or_url: Annotated[ + Optional[str], + Query(description="Filter chunks by document path_or_url"), + ] = None, +): + """Get chunks from the specified index, with optional pagination. + + Restricted to asset administrators (same auth as get_list_indices). + """ + try: + ctx = await _require_asset_owner_context(request) + vdb_core = get_vector_db_core(db_type=VectorDatabaseType.ELASTICSEARCH) + + if path_or_url is not None and not check_file_access( + path_or_url, ctx.user_id, ctx.tenant_id + ): + raise HTTPException( + status_code=HTTPStatus.FORBIDDEN, + detail="You don't have permission to access this file", + ) + + return ElasticSearchService.get_index_chunks( + index_name=index_name, + page=page, + page_size=page_size, + path_or_url=path_or_url, + vdb_core=vdb_core, + ) + except ValueError as e: + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, detail=str(e) + ) + except LimitExceededError as e: + logger.exception("Rate limit exceeded while getting chunks") + raise HTTPException( + status_code=HTTPStatus.TOO_MANY_REQUESTS, + detail=RATE_LIMIT_EXCEEDED_DETAIL) + except UnauthorizedError as e: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except HTTPException: + raise + except Exception: + logger.exception("Error getting chunks for index") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Error getting chunks") + + +@router.post("/indices/search/hybrid") +async def hybrid_search( + request: Request, + payload: HybridSearchRequest, +): + """Run a hybrid (accurate + semantic) search across indices. + + Restricted to asset administrators (same auth as get_list_indices). + """ + try: + ctx = await _require_asset_owner_context(request) + vdb_core = get_vector_db_core(db_type=VectorDatabaseType.ELASTICSEARCH) + + resolved_index_names: List[str] = [] + for requested_name in payload.index_names: + try: + resolved_name = get_index_name_by_knowledge_name( + requested_name, ctx.tenant_id + ) + except Exception: + resolved_name = requested_name + resolved_index_names.append(resolved_name) + + return ElasticSearchService.search_hybrid( + index_names=resolved_index_names, + query=payload.query, + tenant_id=ctx.tenant_id, + top_k=payload.top_k, + weight_accurate=payload.weight_accurate, + vdb_core=vdb_core, + ) + except KnowledgeBaseNeedsModelConfigError as exc: + raise HTTPException( + status_code=HTTPStatus.CONFLICT, + detail={ + "error_type": "KNOWLEDGE_BASE_NEEDS_MODEL_CONFIG", + "index_name": exc.index_name, + "message": exc.message, + "suggestion": ( + "Please select an embedding model for this knowledge base " + "before searching." + ), + }, + ) + except ValueError as exc: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, detail=str(exc) + ) + except LimitExceededError as e: + logger.exception("Rate limit exceeded while running hybrid search") + raise HTTPException( + status_code=HTTPStatus.TOO_MANY_REQUESTS, + detail=RATE_LIMIT_EXCEEDED_DETAIL) + except UnauthorizedError as e: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except HTTPException: + raise + except Exception: + logger.exception("Error executing hybrid search") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Error executing hybrid search") + + @router.delete("/indices/{index_name}/documents") async def delete_documents( request: Request, diff --git a/backend/apps/oauth_app.py b/backend/apps/oauth_app.py index f05102d0c..b36e9b3db 100644 --- a/backend/apps/oauth_app.py +++ b/backend/apps/oauth_app.py @@ -20,6 +20,7 @@ generate_pending_oauth_token, get_authorize_url, get_enabled_providers, + get_oauth_config, get_pending_oauth_info, get_provider_user_info, list_linked_accounts, @@ -37,6 +38,14 @@ router = APIRouter(prefix="/user/oauth", tags=["oauth"]) +@router.get("/config") +async def get_config(): + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "success", "data": get_oauth_config()}, + ) + + @router.get("/providers") async def get_providers(): providers = get_enabled_providers() diff --git a/backend/apps/ragflow_app.py b/backend/apps/ragflow_app.py new file mode 100644 index 000000000..eab899c14 --- /dev/null +++ b/backend/apps/ragflow_app.py @@ -0,0 +1,45 @@ +""" +RAGFlow App Layer +FastAPI endpoints for RAGFlow knowledge base operations. +""" +import logging +from http import HTTPStatus +from typing import Annotated, Optional + +from fastapi import APIRouter, Header, HTTPException, Query +from fastapi.responses import JSONResponse + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from services.ragflow_service import fetch_ragflow_datasets_impl + +router = APIRouter(prefix="/ragflow") +logger = logging.getLogger("ragflow_app") + + +@router.get("/datasets") +async def fetch_ragflow_datasets_api( + ragflow_api_base: Annotated[str, Query(description="RAGFlow API base URL")], + api_key: Annotated[str, Query(description="RAGFlow API key")], + authorization: Annotated[Optional[str], Header()] = None +): + """ + Fetch datasets (knowledge bases) from RAGFlow API. + """ + ragflow_api_base = ragflow_api_base.rstrip('/') + + try: + result = fetch_ragflow_datasets_impl( + ragflow_api_base=ragflow_api_base, + api_key=api_key, + ) + return JSONResponse( + status_code=HTTPStatus.OK, + content=result + ) + except AppException: + raise + except Exception as e: + logger.exception("Failed to fetch RAGFlow datasets") + raise AppException(ErrorCode.RAGFLOW_SERVICE_ERROR, + f"Failed to fetch RAGFlow datasets: {str(e)}") diff --git a/backend/apps/remote_mcp_app.py b/backend/apps/remote_mcp_app.py index 3993e24ce..aef1348b4 100644 --- a/backend/apps/remote_mcp_app.py +++ b/backend/apps/remote_mcp_app.py @@ -1,6 +1,6 @@ import logging import json -from typing import Optional +from typing import Annotated, Optional from fastapi import APIRouter, Header, HTTPException, UploadFile, File, Form, Query, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -15,6 +15,7 @@ McpValidationError, McpNameConflictError, McpPortConflictError, + UnauthorizedError, ) from consts.model import ( MCPConfigRequest, @@ -24,6 +25,7 @@ EnableMcpServiceRequest, DisableMcpServiceRequest, HealthcheckMcpServiceRequest, + TestMcpConnectionRequest, ListMcpServicesQuery, ) from services.remote_mcp_service import ( @@ -35,12 +37,14 @@ attach_mcp_container_permissions, get_mcp_record_by_id, list_mcp_service_tools_by_id, + refresh_mcp_service_tool_count, add_mcp_service, add_container_mcp_service, update_mcp_service, update_mcp_service_enabled, delete_mcp_service, check_mcp_service_health, + test_mcp_connection, check_container_port_conflict, suggest_container_port, ) @@ -51,6 +55,8 @@ router = APIRouter(prefix="/mcp") logger = logging.getLogger("remote_mcp_app") +_MCP_SERVICE_ID_DESC = Query(..., description="MCP service ID") + # --------------------------------------------------------------------------- # Tools Endpoint @@ -58,7 +64,7 @@ @router.get("/tools") async def get_tools_from_mcp( - mcp_id: int = Query(..., description="MCP service ID"), + mcp_id: int = _MCP_SERVICE_ID_DESC, authorization: Optional[str] = Header(None), http_request: Request = None ): @@ -96,6 +102,46 @@ async def get_tools_from_mcp( ) +# --------------------------------------------------------------------------- +# Tool Count Refresh Endpoint +# --------------------------------------------------------------------------- + +@router.post("/refresh-tools") +async def refresh_mcp_tools_endpoint( + mcp_id: int = _MCP_SERVICE_ID_DESC, + authorization: Optional[str] = Header(None), + http_request: Request = None +): + """Connect to the MCP server, fetch tool names, and persist them to the record.""" + try: + user_id, tenant_id, _ = get_current_user_info(authorization, http_request) + await refresh_mcp_service_tool_count( + tenant_id=tenant_id, + user_id=user_id, + mcp_id=mcp_id, + ) + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "Tool count refreshed", "status": "success"} + ) + except McpNotFoundError as e: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) + except McpValidationError as e: + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + except MCPConnectionError as e: + logger.exception(f"Failed to refresh tool count for mcp_id={mcp_id}") + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail="MCP connection failed" + ) + except Exception as e: + logger.error(f"Failed to refresh MCP tool count: {e}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Failed to refresh MCP tool count" + ) + + # --------------------------------------------------------------------------- # Add Endpoints # --------------------------------------------------------------------------- @@ -125,6 +171,8 @@ async def add_mcp_service_endpoint( custom_headers=payload.custom_headers, container_config=payload.container_config, registry_json=payload.registry_json, + config_json=payload.config_json, + market_id=payload.market_id, enabled=payload.enabled if payload.enabled is not None else False, ) @@ -171,6 +219,8 @@ async def add_container_mcp_service_endpoint( tags=payload.tags, authorization_token=payload.authorization_token, registry_json=payload.registry_json, + version=payload.version, + market_id=payload.market_id, port=payload.port, mcp_config=payload.mcp_config, ) @@ -241,7 +291,9 @@ async def update_mcp_service_endpoint( server_url=payload.server_url, authorization_token=payload.authorization_token, custom_headers=payload.custom_headers, + config_json=payload.config_json, tags=payload.tags, + market_id=payload.market_id, ) return JSONResponse( @@ -385,6 +437,11 @@ async def get_mcp_list( "status": "success" } ) + except UnauthorizedError as e: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail=str(e) + ) except Exception as e: logger.error(f"Failed to get MCP list: {e}") raise HTTPException( @@ -553,7 +610,7 @@ async def generate_log_stream(): @router.get("/healthcheck") async def check_mcp_health( - mcp_id: int = Query(..., description="MCP service ID"), + mcp_id: int = _MCP_SERVICE_ID_DESC, authorization: Optional[str] = Header(None), http_request: Request = None ): @@ -589,6 +646,43 @@ async def check_mcp_health( ) +# --------------------------------------------------------------------------- +# Test Connection Endpoint +# --------------------------------------------------------------------------- + + +@router.post("/test-connection") +async def test_mcp_connection_endpoint( + payload: TestMcpConnectionRequest, + authorization: Annotated[Optional[str], Header()] = None, + http_request: Request = None +): + """Lightweight MCP connectivity test. Performs only the initialize handshake. + + This is faster than a full health check (which calls list_tools()) and is + intended for pre-install validation in the Quick Add modal. + """ + try: + get_current_user_info(authorization, http_request) + + success = await test_mcp_connection( + server_url=payload.server_url, + authorization_token=payload.authorization_token, + custom_headers=payload.custom_headers, + ) + + return JSONResponse( + status_code=HTTPStatus.OK, + content={"success": success} + ) + except Exception as e: + logger.exception("MCP test connection failed") + return JSONResponse( + status_code=HTTPStatus.OK, + content={"success": False, "error": str(e) or "Connection failed"} + ) + + # --------------------------------------------------------------------------- # Port Management Endpoints # --------------------------------------------------------------------------- diff --git a/backend/apps/skill_app.py b/backend/apps/skill_app.py index 5a67cafd5..7cc5b9f82 100644 --- a/backend/apps/skill_app.py +++ b/backend/apps/skill_app.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, Field from consts.const import APP_VERSION, STREAMABLE_CONTENT_TYPES -from consts.exceptions import SkillException, UnauthorizedError +from consts.exceptions import ForbiddenError, SkillException, UnauthorizedError from services.skill_service import ( SkillService, skill_creation_task_manager, @@ -25,6 +25,7 @@ ASSET_OWNER_SKILL_VIEW_DENIED = {"content": "您无权限查看"} logger = logging.getLogger(__name__) +_NOT_FOUND_TEXT = "not found" router = APIRouter(prefix="/skills", tags=["skills"]) skill_creator_router = APIRouter(prefix="/skills", tags=["nl2skill"]) @@ -37,6 +38,25 @@ def _asset_owner_skill_view_denied_response(skill: Optional[Dict[str, Any]], ten return None +def _build_skill_update_data(request: SkillUpdateRequest) -> Dict[str, Any]: + update_data: Dict[str, Any] = {} + for field_name in ( + "name", + "description", + "content", + "tags", + "source", + "config_schemas", + "config_values", + ): + value = getattr(request, field_name) + if value is not None: + update_data[field_name] = value + if request.files is not None: + update_data["files"] = [f.model_dump() for f in request.files] + return update_data + + # List routes first (no path parameters) @router.get("") async def list_skills( @@ -169,7 +189,7 @@ async def create_skill_from_file( file: UploadFile = File(..., description="SKILL.md file or ZIP archive"), skill_name: Optional[str] = Form( None, description="Optional skill name override"), - source: Optional[str] = Form("自定义", description="Skill source"), + source: Optional[str] = Form("custom", description="Skill source"), authorization: Optional[str] = Header(None) ) -> JSONResponse: """Create a skill from file upload. @@ -323,7 +343,7 @@ async def update_skill_from_file( except UnauthorizedError as e: raise HTTPException(status_code=401, detail=str(e)) except SkillException as e: - if "not found" in str(e).lower(): + if _NOT_FOUND_TEXT in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=400, detail=str(e)) except Exception as e: @@ -496,6 +516,72 @@ async def scan_and_update_skill(authorization: Optional[str] = Header(None)): status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Failed to update skill") +@router.get("/{skill_id:int}") +async def get_skill_by_id(skill_id: int, authorization: Optional[str] = Header(None)) -> JSONResponse: + """Get a specific skill by ID.""" + try: + _, tenant_id = get_current_user_id(authorization) + service = SkillService(tenant_id=tenant_id) + skill = service.get_skill_by_id(skill_id, tenant_id=tenant_id) + if not skill: + raise HTTPException( + status_code=404, detail=f"Skill not found: {skill_id}") + return JSONResponse(content=skill) + except HTTPException: + raise + except SkillException as e: + raise HTTPException(status_code=500, detail=str(e)) + except Exception as e: + logger.exception("Error getting skill by ID %s", skill_id) + raise HTTPException(status_code=500, detail="Internal server error") + + +@router.put( + "/{skill_id:int}", + responses={ + 400: {"description": "No fields to update or invalid skill data"}, + 401: {"description": "Unauthorized"}, + 403: {"description": "Not authorized to update this skill"}, + 404: {"description": "Skill not found"}, + 500: {"description": "Internal server error"}, + }, +) +async def update_skill_by_id( + skill_id: int, + request: SkillUpdateRequest, + authorization: Optional[str] = Header(None) +) -> JSONResponse: + """Update an existing skill by ID.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + service = SkillService(tenant_id=tenant_id) + update_data = _build_skill_update_data(request) + + if not update_data: + raise HTTPException(status_code=400, detail="No fields to update") + + skill = service.update_skill_by_id( + skill_id, + update_data, + tenant_id=tenant_id, + user_id=user_id, + ) + return JSONResponse(content=skill) + except UnauthorizedError as e: + raise HTTPException(status_code=401, detail=str(e)) + except ForbiddenError as e: + raise HTTPException(status_code=403, detail=str(e)) + except SkillException as e: + if _NOT_FOUND_TEXT in str(e).lower(): + raise HTTPException(status_code=404, detail=str(e)) + raise HTTPException(status_code=400, detail=str(e)) + except HTTPException: + raise + except Exception as e: + logger.exception("Error updating skill by ID %s", skill_id) + raise HTTPException(status_code=500, detail="Internal server error") + + @router.get("/{skill_name}") async def get_skill(skill_name: str, authorization: Optional[str] = Header(None)) -> JSONResponse: """Get a specific skill by name.""" @@ -558,7 +644,7 @@ async def update_skill( except UnauthorizedError as e: raise HTTPException(status_code=401, detail=str(e)) except SkillException as e: - if "not found" in str(e).lower(): + if _NOT_FOUND_TEXT in str(e).lower(): raise HTTPException(status_code=404, detail=str(e)) raise HTTPException(status_code=400, detail=str(e)) except HTTPException: diff --git a/backend/apps/skill_repository_app.py b/backend/apps/skill_repository_app.py new file mode 100644 index 000000000..108642f6a --- /dev/null +++ b/backend/apps/skill_repository_app.py @@ -0,0 +1,263 @@ +import logging +from http import HTTPStatus +from typing import Annotated, Optional + +from fastapi import APIRouter, Body, Header, HTTPException, Query +from starlette.responses import JSONResponse + +from consts.exceptions import ForbiddenError, SkillDuplicateError, UnauthorizedError +from consts.model import SkillRepositoryInstallRequest, SkillRepositoryListingCreateRequest +from services.skill_repository_service import ( + create_skill_repository_listing_impl, + get_skill_repository_listing_detail_impl, + install_skill_from_repository_impl, + list_my_editable_skills_impl, + list_skill_repository_listings_impl, + update_skill_repository_status_impl, +) +from utils.auth_utils import get_current_user_id + +logger = logging.getLogger(__name__) +skill_repository_router = APIRouter(prefix="/repository/skill") + + +@skill_repository_router.get("") +async def list_skill_repository_listings_api( + status: Optional[str] = Query( + None, description="Filter by listing status"), + skill_id: Optional[int] = Query( + None, description="Filter by source skill ID"), + category_id: Optional[int] = Query( + None, + description="Filter by marketplace category ID", + ), + page: Annotated[int, Query( + ge=1, description="Page number starting from 1")] = 1, + page_size: Annotated[ + int, Query(ge=1, le=100, description="Page size from 1 to 100") + ] = 10, + search: Optional[str] = Query( + None, description="Filter by name, description, source, submitter, or tags" + ), + sort_by_update_time: bool = Query( + False, description="Sort by repository update time descending" + ), + authorization: str = Header(None), +): + """List all skill marketplace repository listings with optional filters.""" + try: + _, tenant_id = get_current_user_id(authorization) + result = list_skill_repository_listings_impl( + tenant_id, + status=status, + skill_id=skill_id, + category_id=category_id, + page=page, + page_size=page_size, + search=search, + sort_by_update_time=sort_by_update_time, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized skill repository listings access attempt: {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + logger.warning( + f"Invalid skill repository listings request parameters: {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + + +@skill_repository_router.get("/mine") +async def list_my_editable_skills_api( + ownership: Optional[str] = Query( + "all", + description="Filter by ownership: all / created / others", + ), + page: Annotated[int, Query( + ge=1, description="Page number starting from 1")] = 1, + page_size: Annotated[ + int, Query(ge=1, le=100, description="Page size from 1 to 100") + ] = 10, + search: Optional[str] = Query( + None, description="Filter by skill name, description, source, creator, or tags" + ), + new_skill_padding: bool = Query( + False, + description="Reserve first slot on page 1 for create-skill placeholder", + ), + authorization: str = Header(None), +): + """List editable skills for the current user with repository listing info.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + result = list_my_editable_skills_impl( + tenant_id=tenant_id, + user_id=user_id, + ownership=ownership or "all", + page=page, + page_size=page_size, + search=search, + new_skill_padding=new_skill_padding, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized my editable skills access attempt: {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + logger.warning( + f"Invalid my editable skills request parameters (ownership={ownership}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + + +@skill_repository_router.get("/{skill_repository_id}") +async def get_skill_repository_listing_detail_api( + skill_repository_id: int, + authorization: str = Header(None), +): + """Get detailed skill marketplace repository listing by primary key.""" + try: + _, tenant_id = get_current_user_id(authorization) + result = get_skill_repository_listing_detail_impl( + skill_repository_id, + tenant_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized skill repository listing detail access attempt " + f"(id={skill_repository_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + logger.warning( + f"Skill repository listing not found (id={skill_repository_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) + + +@skill_repository_router.patch("/{skill_repository_id}/status") +async def update_skill_repository_status_api( + skill_repository_id: int, + status: str = Body( + ..., + embed=True, + description=( + "New status: not_shared / pending_review / rejected / shared" + ), + ), + authorization: str = Header(None), +): + """Update skill marketplace repository listing status.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + result = update_skill_repository_status_impl( + skill_repository_id=skill_repository_id, + status=status, + user_id=user_id, + tenant_id=tenant_id, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized skill repository status update attempt " + f"(id={skill_repository_id}, status={status}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ForbiddenError as e: + logger.warning( + f"Forbidden skill repository status update attempt " + f"(id={skill_repository_id}, status={status}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=str(e)) + except ValueError as e: + logger.warning( + f"Invalid skill repository status update " + f"(id={skill_repository_id}, status={status}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + + +@skill_repository_router.post("/{skill_id}") +async def create_skill_repository_listing_api( + skill_id: int, + payload: Optional[SkillRepositoryListingCreateRequest] = Body(None), + authorization: str = Header(None), +): + """Create or update a marketplace repository listing from a skill snapshot.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + card_fields = payload.model_dump( + exclude_none=True) if payload else None + result = create_skill_repository_listing_impl( + skill_id=skill_id, + tenant_id=tenant_id, + user_id=user_id, + card_fields=card_fields, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized skill repository listing creation attempt " + f"(skill_id={skill_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ForbiddenError as e: + logger.warning( + f"Forbidden skill repository listing creation attempt " + f"(skill_id={skill_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=str(e)) + except ValueError as e: + logger.warning( + f"Invalid skill repository listing creation parameters " + f"(skill_id={skill_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(e)) + + +@skill_repository_router.post("/{skill_repository_id}/install") +async def install_skill_from_repository_api( + skill_repository_id: int, + payload: Optional[SkillRepositoryInstallRequest] = Body(None), + authorization: Optional[str] = Header(None), +): + """Install a skill from a shared marketplace repository listing.""" + try: + user_id, tenant_id = get_current_user_id(authorization) + result = install_skill_from_repository_impl( + skill_repository_id=skill_repository_id, + tenant_id=tenant_id, + user_id=user_id, + target_name=payload.target_name if payload else None, + ) + return JSONResponse(status_code=HTTPStatus.OK, content=result) + except UnauthorizedError as e: + logger.warning( + f"Unauthorized skill repository install attempt " + f"(id={skill_repository_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, detail=str(e)) + except ValueError as e: + logger.warning( + f"Skill repository listing not found for install " + f"(id={skill_repository_id}): {str(e)}" + ) + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail=str(e)) + except SkillDuplicateError as e: + logger.warning( + f"Duplicate skill repository install attempt " + f"(id={skill_repository_id}, duplicates={e.duplicate_names})" + ) + raise HTTPException( + status_code=HTTPStatus.CONFLICT, + detail={ + "type": "skill_duplicate", + "duplicate_skills": e.duplicate_names, + }, + ) diff --git a/backend/consts/agent_unavailable_reasons.py b/backend/consts/agent_unavailable_reasons.py index 4e710ee7d..04d1230c3 100644 --- a/backend/consts/agent_unavailable_reasons.py +++ b/backend/consts/agent_unavailable_reasons.py @@ -20,6 +20,7 @@ class AgentUnavailableReason: # Tool issues TOOL_UNAVAILABLE = "tool_unavailable" ALL_TOOLS_DISABLED = "all_tools_disabled" + MCP_MODEL_UNAVAILABLE = "mcp_model_unavailable" # Agent issues AGENT_NOT_FOUND = "agent_not_found" @@ -35,6 +36,7 @@ def all_reasons(cls) -> list[str]: cls.TOOL_UNAVAILABLE, cls.ALL_TOOLS_DISABLED, cls.AGENT_NOT_FOUND, + cls.MCP_MODEL_UNAVAILABLE, ] @classmethod diff --git a/backend/consts/const.py b/backend/consts/const.py index cf950d109..eabf6a8b3 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -88,6 +88,11 @@ class VectorDatabaseType(str, Enum): OAUTH_CALLBACK_BASE_URL = os.getenv("OAUTH_CALLBACK_BASE_URL", "") OAUTH_SSL_VERIFY = os.getenv("OAUTH_SSL_VERIFY", "true").lower() == "true" OAUTH_CA_BUNDLE = os.getenv("OAUTH_CA_BUNDLE", "") +# OAuth login mode: +# - disabled: hide OAuth login entries and disable automatic OAuth redirects. +# - button: show configured OAuth providers as optional login entries. +# - force: automatically redirect when exactly one OAuth provider is configured. +OAUTH_LOGIN_MODE = os.getenv("OAUTH_LOGIN_MODE", "button").lower() # CAS SSO Configuration @@ -204,6 +209,16 @@ class VectorDatabaseType(str, Enum): REDIS_URL = os.getenv("REDIS_URL") REDIS_BACKEND_URL = os.getenv("REDIS_BACKEND_URL") REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +RUNTIME_STATE_REDIS_URL = os.getenv("RUNTIME_STATE_REDIS_URL") or REDIS_URL +RUNTIME_STREAM_TTL_SECONDS = int(os.getenv("RUNTIME_STREAM_TTL_SECONDS", "86400")) +RUNTIME_STREAM_MAX_LEN = int(os.getenv("RUNTIME_STREAM_MAX_LEN", "10000")) +RUNTIME_RUN_TTL_SECONDS = int(os.getenv("RUNTIME_RUN_TTL_SECONDS", "86400")) +RUNTIME_CANCEL_TTL_SECONDS = int(os.getenv("RUNTIME_CANCEL_TTL_SECONDS", "86400")) +RUNTIME_COMPLETED_TTL_SECONDS = int(os.getenv("RUNTIME_COMPLETED_TTL_SECONDS", "300")) +RUNTIME_CANCEL_POLL_INTERVAL_SECONDS = float(os.getenv("RUNTIME_CANCEL_POLL_INTERVAL_SECONDS", "1.0")) +NORTHBOUND_IDEMPOTENCY_TTL_SECONDS = int(os.getenv("NORTHBOUND_IDEMPOTENCY_TTL_SECONDS", "600")) +NORTHBOUND_RATE_LIMIT_ENABLED = os.getenv("NORTHBOUND_RATE_LIMIT_ENABLED", "true").lower() == "true" +NORTHBOUND_RATE_LIMIT_PER_MINUTE = int(os.getenv("NORTHBOUND_RATE_LIMIT_PER_MINUTE", "120")) FLOWER_PORT = int(os.getenv("FLOWER_PORT", "5555")) DP_REDIS_CHUNKS_WAIT_TIMEOUT_S = int( os.getenv("DP_REDIS_CHUNKS_WAIT_TIMEOUT_S", "30")) diff --git a/backend/consts/error_code.py b/backend/consts/error_code.py index 9f5849ea7..a6326668f 100644 --- a/backend/consts/error_code.py +++ b/backend/consts/error_code.py @@ -195,6 +195,14 @@ class ErrorCode(Enum): AIDP_CONNECTION_ERROR = "130503" # AIDP connection error AIDP_AUTH_ERROR = "130504" # AIDP auth error + # 06 - RAGFlow Service + RAGFLOW_SERVICE_ERROR = "130507" # RAGFlow service error + RAGFLOW_CONFIG_INVALID = "130508" # Invalid RAGFlow configuration + RAGFLOW_CONNECTION_ERROR = "130509" # RAGFlow connection error + RAGFLOW_AUTH_ERROR = "130510" # RAGFlow auth error + RAGFLOW_RATE_LIMIT = "130511" # RAGFlow rate limit + RAGFLOW_RESPONSE_ERROR = "130512" # RAGFlow response error + # ==================== 14 Northbound / 北向接口 ==================== # 01 - Request NORTHBOUND_REQUEST_FAILED = "140101" # Northbound request failed @@ -263,6 +271,12 @@ class ErrorCode(Enum): ErrorCode.IDATA_CONNECTION_ERROR: 502, ErrorCode.IDATA_RESPONSE_ERROR: 502, ErrorCode.IDATA_RATE_LIMIT: 429, + # RAGFlow (module 13) + ErrorCode.RAGFLOW_CONFIG_INVALID: 400, + ErrorCode.RAGFLOW_AUTH_ERROR: 502, + ErrorCode.RAGFLOW_CONNECTION_ERROR: 502, + ErrorCode.RAGFLOW_RESPONSE_ERROR: 502, + ErrorCode.RAGFLOW_RATE_LIMIT: 429, # AIDP (module 13) ErrorCode.AIDP_CONFIG_INVALID: 400, ErrorCode.AIDP_AUTH_ERROR: 502, diff --git a/backend/consts/exceptions.py b/backend/consts/exceptions.py index e5e4c7a89..54fce6d82 100644 --- a/backend/consts/exceptions.py +++ b/backend/consts/exceptions.py @@ -95,6 +95,12 @@ class UnauthorizedError(Exception): pass +class ForbiddenError(Exception): + """Raised when an authenticated user lacks permission.""" + + pass + + class SignatureValidationError(Exception): """Raised when X-Signature header is missing or does not match the expected HMAC value.""" @@ -286,7 +292,6 @@ class UnsupportedOperationError(Exception): # Common aliases ParameterInvalidError = ValidationError -ForbiddenError = Exception # Generic fallback ServiceUnavailableError = Exception # Generic fallback DatabaseError = Exception # Generic fallback TimeoutError = TimeoutException diff --git a/backend/consts/mcp_market.py b/backend/consts/mcp_market.py new file mode 100644 index 000000000..edfd9532d --- /dev/null +++ b/backend/consts/mcp_market.py @@ -0,0 +1,15 @@ +"""Domain constants for MCP marketplace listing statuses.""" + +# Listing status: not_shared (未共享), pending_review (待审核), +# rejected (审核驳回), shared (已共享) +STATUS_NOT_SHARED = "not_shared" +STATUS_PENDING_REVIEW = "pending_review" +STATUS_REJECTED = "rejected" +STATUS_SHARED = "shared" + +VALID_MARKET_STATUSES = frozenset({ + STATUS_NOT_SHARED, + STATUS_PENDING_REVIEW, + STATUS_REJECTED, + STATUS_SHARED, +}) diff --git a/backend/consts/model.py b/backend/consts/model.py index bdcde905e..981bed6f4 100644 --- a/backend/consts/model.py +++ b/backend/consts/model.py @@ -547,7 +547,7 @@ class AgentInfoRequest(BaseModel): business_description: Optional[str] = None author: Optional[str] = None model_ids: Optional[List[int]] = None - max_steps: Optional[int] = Field(default=None, ge=1, le=30) + max_steps: Optional[int] = Field(default=None, ge=1) requested_output_tokens: Optional[int] = Field(default=None, gt=0) provide_run_summary: Optional[bool] = None duty_prompt: Optional[str] = None @@ -717,19 +717,11 @@ class AgentRepositoryListingCreateRequest(BaseModel): icon: Optional[str] = Field(None, description="Marketplace card icon (emoji or URL)") downloads: int = Field(0, ge=0, description="Initial download/copy count for card display") tags: Optional[List[str]] = Field(None, description="Marketplace tags") - category_id: Optional[int] = Field(0, description="Optional marketplace category ID") tool_count: Optional[int] = Field( None, ge=0, description="Total tool count across all agents in the bundle" ) -class AgentRepositoryCategoryItem(BaseModel): - """Marketplace category option for agent repository filtering.""" - id: int - key: str - name: str - - class AgentRepositoryListingDetailResponse(BaseModel): """Detailed marketplace listing payload for repository detail view.""" agent_repository_id: int @@ -748,6 +740,39 @@ class AgentRepositoryListingDetailResponse(BaseModel): tools: List[str] = Field(default_factory=list) +class SkillRepositoryListingCreateRequest(BaseModel): + """Request body for creating a marketplace listing from a skill snapshot.""" + icon: Optional[str] = Field(None, description="Marketplace card icon (emoji or URL)") + downloads: int = Field(0, ge=0, description="Initial download count for card display") + tags: Optional[List[str]] = Field(None, description="Marketplace tags") + category_id: Optional[int] = Field(0, description="Optional marketplace category ID") + + +class SkillRepositoryInstallRequest(BaseModel): + """Request body for installing a repository skill into current tenant.""" + target_name: Optional[str] = Field(None, description="Target skill name in current tenant") + + +class SkillRepositoryListingDetailResponse(BaseModel): + """Detailed marketplace listing payload for skill repository detail view.""" + skill_repository_id: int + skill_id: Optional[int] = None + name: str + description: Optional[str] = None + source: Optional[str] = None + submitted_by: Optional[str] = None + icon: Optional[str] = None + status: str + category_id: Optional[int] = None + tags: List[str] = Field(default_factory=list) + downloads: int = 0 + created_at: Optional[str] = None + content: Optional[str] = None + config_schemas: Optional[Dict[str, Any]] = None + config_values: Optional[Dict[str, Any]] = None + tool_ids: List[int] = Field(default_factory=list) + + class SkillZipEntry(BaseModel): """A skill bundled inside an agent export ZIP.""" skill_name: str @@ -1268,6 +1293,7 @@ class SkillFileData(BaseModel): class SkillUpdateRequest(BaseModel): """Request model for updating a skill.""" + name: Optional[str] = None description: Optional[str] = None content: Optional[str] = None tool_ids: Optional[List[int]] = None @@ -1330,9 +1356,12 @@ class AddMcpServiceRequest(BaseModel): custom_headers: Optional[Dict[str, Any]] = Field(None, description="Custom HTTP headers as JSON object") container_config: Optional[Dict[str, Any]] = Field(None, description="Container configuration") registry_json: Optional[Dict[str, Any]] = Field(None, description="Registry metadata JSON") + config_json: Optional[Dict[str, Any]] = Field(None, description="MCP configuration JSON (e.g. OpenAPI spec for API-type MCP)") + version: Optional[str] = Field(None, description="MCP version") + market_id: Optional[int] = Field(None, gt=0, description="Linked market record ID") enabled: Optional[bool] = Field(default=False, description="Whether the MCP is enabled after creation") - @field_validator("name", "server_url", "description", "authorization_token", mode="before") + @field_validator("name", "server_url", "description", "authorization_token", "version", mode="before") @classmethod def _strip_text(cls, value: Any): if isinstance(value, str): @@ -1348,10 +1377,12 @@ class AddContainerMcpServiceRequest(BaseModel): tags: List[str] = Field(default_factory=list, description="MCP tags") authorization_token: Optional[str] = Field(None, description="Authorization token for MCP server") registry_json: Optional[Dict[str, Any]] = Field(None, description="Registry metadata JSON") + version: Optional[str] = Field(None, description="MCP version") + market_id: Optional[int] = Field(None, gt=0, description="Linked market record ID") port: int = Field(..., ge=1, le=65535, description="Host port for the container") mcp_config: MCPConfigRequest = Field(..., description="MCP server configuration") - @field_validator("name", "description", "authorization_token", mode="before") + @field_validator("name", "description", "authorization_token", "version", mode="before") @classmethod def _strip_text(cls, value: Any): if isinstance(value, str): @@ -1368,8 +1399,11 @@ class UpdateMcpServiceRequest(BaseModel): tags: List[str] = Field(default_factory=list, description="MCP tags") authorization_token: Optional[str] = Field(None, description="Authorization token for MCP server") custom_headers: Optional[Dict[str, Any]] = Field(None, description="Custom HTTP headers as JSON object") + config_json: Optional[Dict[str, Any]] = Field(None, description="MCP configuration JSON") + version: Optional[str] = Field(None, description="MCP version") + market_id: Optional[int] = Field(None, gt=0, description="Linked market record ID") - @field_validator("name", "server_url", "description", "authorization_token", mode="before") + @field_validator("name", "server_url", "description", "authorization_token", "version", mode="before") @classmethod def _strip_text(cls, value: Any): if isinstance(value, str): @@ -1392,6 +1426,13 @@ class HealthcheckMcpServiceRequest(BaseModel): mcp_id: int = Field(..., gt=0, description="MCP record ID to health check") +class TestMcpConnectionRequest(BaseModel): + """Request model for testing MCP server connectivity (lightweight handshake)""" + server_url: str = Field(..., min_length=1, description="MCP server URL to test") + authorization_token: Optional[str] = Field(None, description="Authorization token for MCP server") + custom_headers: Optional[Dict[str, Any]] = Field(None, description="Custom HTTP headers as JSON object") + + class ListMcpToolsRequest(BaseModel): """Request model for listing MCP service tools""" mcp_id: int = Field(..., gt=0, description="MCP record ID") @@ -1450,18 +1491,35 @@ def _strip_text(cls, value: Any): return value +class CommunityReviewListRequest(CommunityListRequest): + """Request model for listing MCP community review submissions""" + status: Optional[str] = Field(None, description="Review status filter") + + @field_validator("status", mode="before") + @classmethod + def _strip_status(cls, value: Any): + if isinstance(value, str): + stripped = value.strip() + return stripped or None + return value + + +class CommunityReviewActionRequest(BaseModel): + """Request model for approving or rejecting an MCP community submission""" + review_id: int = Field(..., gt=0, description="Review record ID") + + class CommunityPublishRequest(BaseModel): """Publish a local MCP to the community; optional fields override the snapshot.""" mcp_id: int = Field(..., gt=0, description="MCP record ID to publish") name: Optional[str] = Field(None, description="Community display name override") description: Optional[str] = Field(None, description="Description override") - version: Optional[str] = Field(None, description="Version override") tags: Optional[List[str]] = Field(None, description="Tags override") mcp_server: Optional[str] = Field(None, max_length=500, description="Remote MCP server URL override (URL / HTTP / SSE transports)") config_json: Optional[Dict[str, Any]] = Field(None, description="Container MCP configuration JSON override") - @field_validator("name", "description", "version", "mcp_server", mode="before") + @field_validator("name", "description", "mcp_server", mode="before") @classmethod def _strip_publish_optional_text(cls, value: Any): if isinstance(value, str): @@ -1472,18 +1530,19 @@ def _strip_publish_optional_text(cls, value: Any): class CommunityUpdateRequest(BaseModel): """Request model for updating community MCP service""" - community_id: int = Field(..., gt=0, description="Community record ID") + market_id: int = Field(..., gt=0, description="Market record ID") name: Optional[str] = Field(default=None, min_length=1, description="New MCP service name") description: Optional[str] = Field(None, description="MCP service description") tags: List[str] = Field(default_factory=list, description="MCP tags") - version: Optional[str] = Field(None, description="MCP version") registry_json: Optional[Dict[str, Any]] = Field(None, description="Registry metadata JSON") + mcp_server: Optional[str] = Field(None, max_length=500, description="MCP server URL") + transport_type: Optional[str] = Field(None, description="Transport type") config_json: Optional[Dict[str, Any]] = Field( None, description="Container MCP configuration JSON (omit to leave unchanged)", ) - @field_validator("name", "description", "version", mode="before") + @field_validator("name", "description", "mcp_server", "transport_type", mode="before") @classmethod def _strip_text(cls, value: Any): if isinstance(value, str): @@ -1492,6 +1551,11 @@ def _strip_text(cls, value: Any): return value +class CommunityStatusUpdateRequest(BaseModel): + """Request model for changing MCP market listing status (PATCH).""" + status: str = Field(..., description="New status: shared / rejected / not_shared / pending_review") + + class DeleteMcpServiceRequest(BaseModel): """Request model for deleting an MCP service""" mcp_id: int = Field(..., gt=0, description="MCP record ID to delete") diff --git a/backend/consts/tool_labels.py b/backend/consts/tool_labels.py index 2f7228bbe..649e63574 100644 --- a/backend/consts/tool_labels.py +++ b/backend/consts/tool_labels.py @@ -33,6 +33,7 @@ _category_kb = { "dify_search": ["knowledge-base"], "datamate_search": ["knowledge-base"], "idata_search": ["knowledge-base"], "haotian_search": ["knowledge-base"], + "ragflow_search": ["knowledge-base"], "aidp_search": ["knowledge-base"], } _category_multimodal = { @@ -52,3 +53,5 @@ BUILTIN_LABEL_MAP.update(_category_email) BUILTIN_LABEL_MAP.update(_category_memory) BUILTIN_LABEL_MAP.update(_category_terminal) + +SYSTEM_MANAGED_TOOL_NAMES = frozenset({"store_memory", "search_memory"}) diff --git a/backend/database/agent_evaluation_db.py b/backend/database/agent_evaluation_db.py new file mode 100644 index 000000000..0688319cd --- /dev/null +++ b/backend/database/agent_evaluation_db.py @@ -0,0 +1,345 @@ +import logging +from typing import Any, Dict, List, Optional + +from sqlalchemy import case as sql_case, func +from database.client import as_dict, get_db_session +from database.db_models import AgentEvaluation, AgentEvaluationCase, AgentInfo, EvaluationSet, ModelRecord + +logger = logging.getLogger("agent_evaluation_db") + + +def create_agent_evaluation( + tenant_id: str, + agent_id: int, + agent_version_no: int, + evaluation_set_id: int, + total: int, + judge_model_id: Optional[int], + created_by: Optional[str], +) -> Dict[str, Any]: + with get_db_session() as session: + rec = AgentEvaluation( + tenant_id=tenant_id, + agent_id=agent_id, + agent_version_no=agent_version_no, + evaluation_set_id=evaluation_set_id, + status="PENDING", + progress_total=total, + progress_done=0, + judge_model_id=judge_model_id, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + session.flush() + + es_row = ( + session.query(EvaluationSet.name) + .filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + ) + .scalar() + ) + + judge_model_name = None + if judge_model_id is not None: + judge_model_name = ( + session.query(ModelRecord.display_name) + .filter( + ModelRecord.model_id == judge_model_id, + ModelRecord.tenant_id == tenant_id, + ) + .scalar() + ) + + result = as_dict(rec) + result["evaluation_set_name"] = es_row + result["judge_model_name"] = judge_model_name + return result + + +def update_agent_evaluation_status( + agent_evaluation_id: int, + tenant_id: str, + status: str, + updated_by: Optional[str] = None, + error_message: Optional[str] = None, + score_overall: Optional[float] = None, + progress_done: Optional[int] = None, +) -> None: + updates: Dict[str, Any] = {"status": status, "updated_by": updated_by} + if error_message is not None: + updates["error_message"] = error_message + if score_overall is not None: + updates["score_overall"] = score_overall + if progress_done is not None: + updates["progress_done"] = progress_done + + with get_db_session() as session: + session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).update(updates, synchronize_session=False) + + +def get_agent_evaluation(agent_evaluation_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).first() + if not rec: + raise ValueError("agent evaluation not found") + result = as_dict(rec) + + evaluation_set_name = ( + session.query(EvaluationSet.name) + .filter( + EvaluationSet.evaluation_set_id == rec.evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + ) + .scalar() + ) + result["evaluation_set_name"] = evaluation_set_name + + agent_name = ( + session.query(AgentInfo.display_name, AgentInfo.name) + .filter( + AgentInfo.agent_id == rec.agent_id, + AgentInfo.tenant_id == tenant_id, + ) + .order_by(AgentInfo.version_no.desc()) + .first() + ) + if agent_name is not None: + display_name, programmatic_name = agent_name + result["agent_name"] = display_name or programmatic_name or "" + else: + result["agent_name"] = "" + + judge_model_name = None + if rec.judge_model_id is not None: + judge_model_name = ( + session.query(ModelRecord.display_name, ModelRecord.model_name) + .filter( + ModelRecord.model_id == rec.judge_model_id, + ModelRecord.tenant_id == tenant_id, + ) + .first() + ) + if judge_model_name is not None: + judge_display, judge_repo = judge_model_name + judge_model_name = judge_display or judge_repo + result["judge_model_name"] = judge_model_name + return result + + +def list_agent_evaluations_by_agent( + agent_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + # ``case((, 1), else_=0)`` translates to SQL + # ``CASE WHEN THEN 1 ELSE 0 END``, which is summed per row. + # The previous ``func.cast(, Integer)`` produced a single + # constant (the Python-side truthiness of the whole comparison), making + # every pass_count return 0. See CRITICAL-1 in the audit report. + pass_count_expr = func.sum( + sql_case( + (AgentEvaluationCase.pass_status == "pass", 1), + else_=0, + ) + ).label("pass_count") + + q = ( + session.query( + AgentEvaluation, + EvaluationSet.name.label("evaluation_set_name"), + ModelRecord.display_name.label("judge_model_name"), + func.count(AgentEvaluationCase.agent_evaluation_case_id).label("case_count"), + pass_count_expr, + ) + .outerjoin( + EvaluationSet, + (AgentEvaluation.evaluation_set_id == EvaluationSet.evaluation_set_id) + & (AgentEvaluation.tenant_id == EvaluationSet.tenant_id), + ) + .outerjoin( + ModelRecord, + (AgentEvaluation.judge_model_id == ModelRecord.model_id) + & (AgentEvaluation.tenant_id == ModelRecord.tenant_id), + ) + .outerjoin( + AgentEvaluationCase, + AgentEvaluation.agent_evaluation_id == AgentEvaluationCase.agent_evaluation_id, + ) + .filter( + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.agent_id == agent_id, + AgentEvaluation.delete_flag == "N", + ) + .group_by( + AgentEvaluation.agent_evaluation_id, + EvaluationSet.name, + ModelRecord.display_name, + ) + .order_by(AgentEvaluation.create_time.desc()) + .offset(offset) + .limit(limit) + ) + rows = q.all() + results = [] + for eval_row, evaluation_set_name, judge_model_name, case_count, pass_count in rows: + rec = as_dict(eval_row) + rec["evaluation_set_name"] = evaluation_set_name + rec["judge_model_name"] = judge_model_name + rec["case_count"] = case_count or 0 + rec["pass_count"] = pass_count or 0 + rec["fail_count"] = (case_count or 0) - (pass_count or 0) + results.append(rec) + return results + + +def create_agent_evaluation_cases( + tenant_id: str, + agent_evaluation_id: int, + set_cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> int: + with get_db_session() as session: + inserted = 0 + for sc in set_cases: + rec = AgentEvaluationCase( + tenant_id=tenant_id, + agent_evaluation_id=agent_evaluation_id, + evaluation_set_case_id=sc["evaluation_set_case_id"], + inputs=sc["inputs"], + label=sc["label"], + predict=None, + score=None, + reason=None, + status="PENDING", + error_message=None, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + inserted += 1 + session.flush() + return inserted + + +def update_agent_evaluation_case_result( + agent_evaluation_case_id: int, + tenant_id: str, + status: str, + predict: Optional[Dict[str, Any]] = None, + score: Optional[float] = None, + reason: Optional[str] = None, + error_message: Optional[str] = None, + pass_status: Optional[str] = None, + updated_by: Optional[str] = None, +) -> None: + """Update a case result. + + Storage policy: when a case is judged as ``pass`` (either via an explicit + ``pass_status="pass"`` argument or an observed ``score == 1``), the heavy + detail fields (``predict``, ``reason``, ``label.answer``) are cleared to + save space. Only failed cases retain the full detail for debugging. + """ + updates: Dict[str, Any] = {"status": status, "updated_by": updated_by} + + is_pass = (pass_status == "pass") or (score == 1) + + if not is_pass: + if predict is not None: + updates["predict"] = predict + if reason is not None: + updates["reason"] = reason + else: + # Pass case: trim heavy fields regardless of what was passed in. + updates["predict"] = None + updates["reason"] = None + updates["label"] = {"answer": ""} + + if score is not None: + updates["score"] = score + if pass_status is not None: + updates["pass_status"] = pass_status + if error_message is not None: + updates["error_message"] = error_message + + with get_db_session() as session: + rows = session.query(AgentEvaluationCase).filter( + AgentEvaluationCase.agent_evaluation_case_id == agent_evaluation_case_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ).update(updates, synchronize_session=False) + if rows == 0: + logger.warning( + "agent_evaluation_case not updated: id=%s, tenant=%s", + agent_evaluation_case_id, + tenant_id, + ) + + +def list_agent_evaluation_cases( + agent_evaluation_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(AgentEvaluationCase) + .filter( + AgentEvaluationCase.agent_evaluation_id == agent_evaluation_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ) + .order_by(AgentEvaluationCase.agent_evaluation_case_id.asc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_agent_evaluation_case(agent_evaluation_case_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(AgentEvaluationCase).filter( + AgentEvaluationCase.agent_evaluation_case_id == agent_evaluation_case_id, + AgentEvaluationCase.tenant_id == tenant_id, + AgentEvaluationCase.delete_flag == "N", + ).first() + if not rec: + raise ValueError("agent evaluation case not found") + return as_dict(rec) + + +def soft_delete_agent_evaluation( + agent_evaluation_id: int, + tenant_id: str, + deleted_by: str, +) -> None: + """Soft-delete an evaluation run by setting delete_flag='Y'. + + Raises ``ValueError`` when the run is not found or has already been deleted. + """ + with get_db_session() as session: + rows = session.query(AgentEvaluation).filter( + AgentEvaluation.agent_evaluation_id == agent_evaluation_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).update( + {"delete_flag": "Y", "updated_by": deleted_by}, + synchronize_session=False, + ) + if rows == 0: + raise ValueError("agent evaluation not found or already deleted") diff --git a/backend/database/agent_repository_db.py b/backend/database/agent_repository_db.py index fe6813188..0937d69df 100644 --- a/backend/database/agent_repository_db.py +++ b/backend/database/agent_repository_db.py @@ -28,7 +28,6 @@ "display_name", "description", "author", - "category_id", "tags", "tool_count", "version_name", @@ -180,7 +179,6 @@ def list_agent_repository_summaries( *, status: Optional[str] = None, agent_id: Optional[int] = None, - category_id: Optional[int] = None, ) -> List[dict]: """List active repository summaries for a publisher tenant without heavy JSON blobs.""" with get_db_session() as session: @@ -193,7 +191,6 @@ def list_agent_repository_summaries( AgentRepository.display_name, AgentRepository.description, AgentRepository.status, - AgentRepository.category_id, AgentRepository.tags, AgentRepository.tool_count, AgentRepository.version_name, @@ -207,8 +204,6 @@ def list_agent_repository_summaries( query = query.filter(AgentRepository.status == status) if agent_id is not None: query = query.filter(AgentRepository.agent_id == agent_id) - if category_id is not None: - query = query.filter(AgentRepository.category_id == category_id) rows = query.order_by(AgentRepository.agent_repository_id.desc()).all() return [ { @@ -220,7 +215,6 @@ def list_agent_repository_summaries( "display_name": row.display_name, "description": row.description, "status": row.status, - "category_id": row.category_id, "tags": row.tags, "tool_count": row.tool_count, "version_name": row.version_name, @@ -244,7 +238,6 @@ def update_agent_repository_by_id( "description", "author", "submitted_by", - "category_id", "tags", "tool_count", "version_name", diff --git a/backend/database/client.py b/backend/database/client.py index e095c5636..28cc483ae 100644 --- a/backend/database/client.py +++ b/backend/database/client.py @@ -330,8 +330,12 @@ def as_dict(obj): for c in class_mapper(obj.__class__).columns: value = getattr(obj, c.key) # Convert datetime to ISO format string for JSON serialization + # Naive datetime (TIMESTAMP WITHOUT TIME ZONE) is treated as local time. + # Append 'Z' so browsers interpret it as UTC-equivalent local time + # without shifting. if isinstance(value, datetime): - result[c.key] = value.isoformat() + iso = value.isoformat() + result[c.key] = iso if value.tzinfo else iso + "Z" else: result[c.key] = value return result diff --git a/backend/database/community_mcp_db.py b/backend/database/community_mcp_db.py index 92b78a4ed..278eccc98 100644 --- a/backend/database/community_mcp_db.py +++ b/backend/database/community_mcp_db.py @@ -19,7 +19,8 @@ def get_mcp_community_records( ) -> Dict[str, Any]: with get_db_session() as session: query = session.query(McpCommunityRecord).filter( - McpCommunityRecord.delete_flag != "Y" + McpCommunityRecord.delete_flag != "Y", + McpCommunityRecord.review_status == "approved", ) if transport_type: @@ -77,6 +78,7 @@ def get_mcp_community_tag_stats() -> List[Dict[str, Any]]: ) .filter( McpCommunityRecord.delete_flag != "Y", + McpCommunityRecord.review_status == "approved", ) .group_by("tag") .order_by(func.count(McpCommunityRecord.community_id).desc(), "tag") @@ -101,13 +103,15 @@ def create_mcp_community_record(mcp_data: Dict[str, Any], tenant_id: str, user_i return int(new_record.community_id) -def get_mcp_community_record_by_id_and_tenant(community_id: int, tenant_id: str) -> Dict[str, Any] | None: +def get_mcp_community_record_by_id_and_tenant(community_id: int, tenant_id: str | None) -> Dict[str, Any] | None: with get_db_session() as session: - record = session.query(McpCommunityRecord).filter( + query = session.query(McpCommunityRecord).filter( McpCommunityRecord.community_id == community_id, - McpCommunityRecord.tenant_id == tenant_id, McpCommunityRecord.delete_flag != "Y", - ).first() + ) + if tenant_id is not None: + query = query.filter(McpCommunityRecord.tenant_id == tenant_id) + record = query.first() return as_dict(record) if record else None @@ -122,6 +126,10 @@ def update_mcp_community_record_by_id( version: str | None = None, registry_json: Dict[str, Any] | None = None, config_json: Dict[str, Any] | None = None, + mcp_server: str | None = None, + transport_type: str | None = None, + review_status: str | None = None, + review_type: str | None = None, ) -> None: update_fields: Dict[str, Any] = {"updated_by": user_id} @@ -137,6 +145,14 @@ def update_mcp_community_record_by_id( update_fields["registry_json"] = registry_json if config_json is not None: update_fields["config_json"] = config_json + if mcp_server is not None: + update_fields["mcp_server"] = mcp_server + if transport_type is not None: + update_fields["transport_type"] = transport_type + if review_status is not None: + update_fields["review_status"] = review_status + if review_type is not None: + update_fields["review_type"] = review_type with get_db_session() as session: session.query(McpCommunityRecord).filter( @@ -155,14 +171,91 @@ def delete_mcp_community_record_by_id(*, community_id: int, tenant_id: str, user ).update({"delete_flag": "Y", "updated_by": user_id}) -def list_mcp_community_records_by_tenant(tenant_id: str) -> List[Dict[str, Any]]: +def list_mcp_community_records_by_tenant_and_user(tenant_id: str, user_id: str) -> List[Dict[str, Any]]: with get_db_session() as session: rows = session.query(McpCommunityRecord).filter( McpCommunityRecord.tenant_id == tenant_id, + McpCommunityRecord.user_id == user_id, McpCommunityRecord.delete_flag != "Y", ).order_by(McpCommunityRecord.community_id.desc()).all() return [as_dict(row) for row in rows] + +def list_mcp_community_review_records( + *, + tenant_id: str | None, + status: str | None = None, + search: str | None = None, + tag: str | None = None, + transport_type: str | None = None, + cursor: str | None = None, + limit: int = 30, +) -> Dict[str, Any]: + with get_db_session() as session: + query = session.query(McpCommunityRecord).filter( + McpCommunityRecord.delete_flag != "Y", + ) + + if tenant_id is not None: + query = query.filter(McpCommunityRecord.tenant_id == tenant_id) + if status: + query = query.filter(McpCommunityRecord.review_status == status) + if transport_type: + query = query.filter(McpCommunityRecord.transport_type == transport_type) + if tag: + query = query.filter(McpCommunityRecord.tags.any(tag)) + if search: + keyword = f"%{search}%" + query = query.filter( + or_( + McpCommunityRecord.mcp_name.ilike(keyword), + McpCommunityRecord.description.ilike(keyword), + func.array_to_string(McpCommunityRecord.tags, ",").ilike(keyword), + ) + ) + + cursor_id: int | None = None + if cursor: + try: + cursor_id = int(cursor) + except ValueError: + cursor_id = None + if cursor_id is not None: + query = query.filter(McpCommunityRecord.community_id < cursor_id) + + rows: List[McpCommunityRecord] = ( + query.order_by(McpCommunityRecord.community_id.desc()) + .limit(limit + 1) + .all() + ) + has_next = len(rows) > limit + page_rows = rows[:limit] + next_cursor = str(page_rows[-1].community_id) if has_next and page_rows else None + + return { + "count": len(page_rows), + "nextCursor": next_cursor, + "items": [as_dict(row) for row in page_rows], + } + + +def update_mcp_community_review_status( + *, + community_id: int, + tenant_id: str | None, + user_id: str, + review_status: str, +) -> None: + with get_db_session() as session: + query = session.query(McpCommunityRecord).filter( + McpCommunityRecord.community_id == community_id, + McpCommunityRecord.delete_flag != "Y", + ) + if tenant_id is not None: + query = query.filter(McpCommunityRecord.tenant_id == tenant_id) + query.update({"review_status": review_status, "updated_by": user_id}) + + def get_mcp_community_tag_stats_by_tenant(tenant_id: str) -> List[Dict[str, Any]]: with get_db_session() as session: rows = ( diff --git a/backend/database/conversation_db.py b/backend/database/conversation_db.py index a82c00d24..9efd49881 100644 --- a/backend/database/conversation_db.py +++ b/backend/database/conversation_db.py @@ -46,19 +46,22 @@ class ImageRecord(TypedDict): class ConversationHistory(TypedDict): conversation_id: int + agent_id: Optional[int] create_time: int message_records: List[MessageRecord] search_records: List[SearchRecord] image_records: List[ImageRecord] -def create_conversation(conversation_title: str, user_id: Optional[str] = None) -> Dict[str, Any]: +def create_conversation(conversation_title: str, user_id: Optional[str] = None, + agent_id: Optional[int] = None) -> Dict[str, Any]: """ Create a new conversation record Args: conversation_title: Conversation title user_id: Reserved parameter for created_by and updated_by fields + agent_id: Agent used by the latest run in this conversation Returns: Dict[str, Any]: Dictionary containing complete information of the newly created conversation @@ -66,12 +69,15 @@ def create_conversation(conversation_title: str, user_id: Optional[str] = None) with get_db_session() as session: # Prepare data dictionary data = {"conversation_title": conversation_title, "delete_flag": 'N'} + if agent_id is not None: + data["agent_id"] = agent_id if user_id: data = add_creation_tracking(data, user_id) stmt = insert(ConversationRecord).values(**data).returning( ConversationRecord.conversation_id, ConversationRecord.conversation_title, + ConversationRecord.agent_id, (func.extract('epoch', ConversationRecord.create_time) * 1000).label('create_time'), (func.extract('epoch', ConversationRecord.update_time) @@ -84,6 +90,7 @@ def create_conversation(conversation_title: str, user_id: Optional[str] = None) result_dict = { "conversation_id": record.conversation_id, "conversation_title": record.conversation_title, + "agent_id": record.agent_id, "create_time": int(record.create_time), "update_time": int(record.update_time) } @@ -433,6 +440,7 @@ def get_conversation_list(user_id: Optional[str] = None) -> List[Dict[str, Any]] stmt = select( ConversationRecord.conversation_id, ConversationRecord.conversation_title, + ConversationRecord.agent_id, (func.extract('epoch', ConversationRecord.create_time) * 1000).label('create_time'), (func.extract('epoch', ConversationRecord.update_time) @@ -461,6 +469,36 @@ def get_conversation_list(user_id: Optional[str] = None) -> List[Dict[str, Any]] return result +def update_conversation_agent_id(conversation_id: int, agent_id: int, user_id: Optional[str] = None) -> bool: + """ + Update the agent associated with a conversation. + + Args: + conversation_id: Conversation ID (integer) + agent_id: Latest agent ID used by this conversation + user_id: Reserved parameter for updated_by field + + Returns: + bool: Whether the operation was successful + """ + with get_db_session() as session: + conversation_id = int(conversation_id) + update_data = { + "agent_id": int(agent_id), + "update_time": func.current_timestamp() + } + if user_id: + update_data = add_update_tracking(update_data, user_id) + + stmt = update(ConversationRecord).where( + ConversationRecord.conversation_id == conversation_id, + ConversationRecord.delete_flag == 'N' + ).values(update_data) + + result = session.execute(stmt) + return result.rowcount > 0 + + def rename_conversation(conversation_id: int, new_title: str, user_id: Optional[str] = None) -> bool: """ Rename a conversation @@ -678,6 +716,7 @@ def get_conversation_history(conversation_id: int, user_id: Optional[str] = None # First check if conversation exists check_stmt = select( ConversationRecord.conversation_id, + ConversationRecord.agent_id, (func.extract('epoch', ConversationRecord.create_time) * 1000).label('create_time') ).where( @@ -769,6 +808,7 @@ def get_conversation_history(conversation_id: int, user_id: Optional[str] = None return { 'conversation_id': conversation['conversation_id'], + 'agent_id': conversation.get('agent_id'), 'create_time': int(conversation['create_time']), 'message_records': message_list, 'search_records': [as_dict(record) for record in search_records], diff --git a/backend/database/db_models.py b/backend/database/db_models.py index ca2f898e1..b50e801fb 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -13,6 +13,9 @@ # Shared doc strings for primary key columns _PRIMARY_KEY_DOC = "Primary key, auto-increment" _TENANT_ID_DOC = "Tenant ID for multi-tenancy isolation" +_PUBLISHER_TENANT_ID_DOC = "Publisher tenant ID" +_PUBLISHER_USER_ID_DOC = "Publisher user ID" +_MCP_NAME_DOC = "MCP name" # Base class for tables without audit fields @@ -43,6 +46,7 @@ class ConversationRecord(TableBase): conversation_id = Column(Integer, Sequence( "conversation_record_t_conversation_id_seq", schema=SCHEMA), primary_key=True, nullable=False) conversation_title = Column(String(100), doc="Conversation title") + agent_id = Column(Integer, doc="Agent ID used by the latest run in this conversation") class ConversationMessage(TableBase): @@ -624,7 +628,7 @@ class McpRecord(TableBase): primary_key=True, nullable=False, doc="MCP record ID, unique primary key") tenant_id = Column(String(100), doc="Tenant ID") user_id = Column(String(100), doc="User ID") - mcp_name = Column(String(100), doc="MCP name") + mcp_name = Column(String(100), doc=_MCP_NAME_DOC) mcp_server = Column(String(500), doc="MCP server address") status = Column( Boolean, @@ -651,6 +655,7 @@ class McpRecord(TableBase): ) source = Column( String(30), doc="Source type: local/mcp_registry/community") + market_id = Column(Integer, doc="Published market record ID (FK to mcp_market_record_t)") registry_json = Column(JSONB, doc="Full MCP registry server.json snapshot") config_json = Column(JSON, doc="MCP config data") enabled = Column(Boolean, default=True, doc="Enabled") @@ -671,20 +676,53 @@ class McpCommunityRecord(TableBase): nullable=False, doc="Community record ID, unique primary key", ) - tenant_id = Column(String(100), doc="Publisher tenant ID") - user_id = Column(String(100), doc="Publisher user ID") - mcp_name = Column(String(100), doc="MCP name") + tenant_id = Column(String(100), doc=_PUBLISHER_TENANT_ID_DOC) + user_id = Column(String(100), doc=_PUBLISHER_USER_ID_DOC) + mcp_name = Column(String(100), doc=_MCP_NAME_DOC) mcp_server = Column(String(500), doc="MCP server URL") source = Column(String(30), doc="Source type, fixed to community") - version = Column(String(50), doc="MCP version") registry_json = Column(JSONB, doc="Full MCP metadata JSON") transport_type = Column( String(30), doc="Transport type: http/sse/container") config_json = Column(JSON, doc="Public-shareable MCP configuration JSON") + review_status = Column( + String(30), default="pending", doc="Review status: pending/approved/rejected/offline") + review_type = Column( + String(30), default="initial_listing", doc="Review submission type: initial_listing/update") tags = Column(ARRAY(Text), doc="Tags") description = Column(Text, doc="Description") +class McpMarketRecord(TableBase): + """MCP market (community) record — single table covering all listing states.""" + + __tablename__ = "mcp_market_record_t" + __table_args__ = {"schema": SCHEMA} + + market_id = Column( + BigInteger, + Sequence("mcp_market_record_t_market_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc="Market record ID, unique primary key", + ) + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + user_id = Column(String(100), nullable=False, doc="Publisher user ID") + mcp_name = Column(String(100), doc=_MCP_NAME_DOC) + mcp_server = Column(String(500), doc="MCP server URL") + source = Column(String(30), doc="Source type, fixed to community") + registry_json = Column(JSONB, doc="Full MCP metadata JSON") + transport_type = Column(String(30), doc="Transport type: http/sse/container") + config_json = Column(JSON, doc="Public-shareable MCP configuration JSON") + tags = Column(ARRAY(Text), doc="Tags") + description = Column(Text, doc="Description") + download_count = Column(Integer, default=0, doc="Cumulative download/install count") + review_status = Column(String(30), default="not_shared", + doc="Listing status: not_shared / pending_review / rejected / shared") + submitted_by = Column(String(100), doc="Submitter email when listing enters pending_review") + source_mcp_id = Column(Integer, doc="Local MCP record ID that created this market record") + + class UserTenant(TableBase): """ User and tenant relationship table @@ -859,8 +897,8 @@ class AgentRepository(TableBase): agent_repository_id = Column(BigInteger, Sequence("ag_agent_repository_t_agent_repository_id_seq", schema=SCHEMA), primary_key=True, nullable=False, doc="Agent repository listing ID, unique primary key") - publisher_tenant_id = Column(String(100), nullable=False, doc="Publisher tenant ID") - publisher_user_id = Column(String(100), nullable=False, doc="Publisher user ID") + publisher_tenant_id = Column(String(100), nullable=False, doc=_PUBLISHER_TENANT_ID_DOC) + publisher_user_id = Column(String(100), nullable=False, doc=_PUBLISHER_USER_ID_DOC) agent_id = Column(Integer, nullable=False, doc="Root agent ID from ag_tenant_agent_t; upsert key") version_no = Column(Integer, nullable=False, @@ -871,7 +909,6 @@ class AgentRepository(TableBase): description = Column(Text, doc="Root agent description") author = Column(String(100), doc="Agent author") submitted_by = Column(String(100), doc="Submitter email when listing enters pending_review") - category_id = Column(Integer, doc="Optional marketplace category ID") tags = Column(ARRAY(Text), doc="Marketplace tags") tool_count = Column(Integer, doc="Total tool count across all agents in the bundle (display only)") @@ -886,6 +923,32 @@ class AgentRepository(TableBase): doc="Listing status: not_shared (未共享) / pending_review (待审核) / rejected (审核驳回) / shared (已共享)") +class SkillRepository(TableBase): + """ + Skill repository (marketplace) table. Frozen snapshot of a shared skill for installation. + """ + __tablename__ = "ag_skill_repository_t" + __table_args__ = {"schema": SCHEMA} + + skill_repository_id = Column(BigInteger, Sequence("ag_skill_repository_t_skill_repository_id_seq", schema=SCHEMA), + primary_key=True, nullable=False, doc="Skill repository listing ID, unique primary key") + publisher_tenant_id = Column(String(100), nullable=False, doc=_PUBLISHER_TENANT_ID_DOC) + publisher_user_id = Column(String(100), nullable=False, doc=_PUBLISHER_USER_ID_DOC) + skill_id = Column(Integer, nullable=False, doc="Source skill ID from ag_skill_info_t") + name = Column(String(100), nullable=False, doc="Skill name for display and search") + description = Column(Text, doc="Skill description") + source = Column(String(30), doc="Skill source") + submitted_by = Column(String(100), doc="Submitter email when listing enters pending_review") + category_id = Column(Integer, doc="Optional marketplace category ID") + tags = Column(ARRAY(Text), doc="Marketplace tags") + icon = Column(String(100), doc="Marketplace card icon (emoji or URL)") + downloads = Column(Integer, default=0, doc="Marketplace install count for card display") + skill_info_json = Column(JSONB, nullable=False, doc="Frozen skill metadata snapshot") + skill_zip_base64 = Column(Text, nullable=False, doc="Frozen skill ZIP payload encoded as base64") + status = Column(String(30), default="not_shared", + doc="Listing status: not_shared / pending_review / rejected / shared") + + class UserTokenInfo(TableBase): """ User token (AK/SK) information table @@ -1373,5 +1436,164 @@ class A2AArtifact(SimpleTableBase): extensions = Column(JSON, doc="Extension URI list") # Timestamp - create_time = Column(TIMESTAMP( - timezone=False), server_default=func.now(), doc="Artifact creation timestamp") + create_time = Column(TIMESTAMP(timezone=False), server_default=func.now(), doc="Artifact creation timestamp") + + +# ----------------------------------------------------------------------------- +# Agent Evaluation (offline) tables +# ----------------------------------------------------------------------------- +class EvaluationSet(TableBase): + """Evaluation set metadata.""" + + __tablename__ = "evaluation_set_t" + __table_args__ = {"schema": SCHEMA} + + evaluation_set_id = Column( + BigInteger, + Sequence("evaluation_set_t_evaluation_set_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + name = Column(String(255), nullable=False, doc="Evaluation set name") + description = Column(Text, doc="Evaluation set description") + + source_filename = Column(String(255), doc="Original uploaded filename") + case_count = Column(Integer, default=0, doc="Total number of cases") + + __table_args__ = ( + Index("ix_eval_set_tenant_id", "tenant_id"), + Index("ix_eval_set_name", "tenant_id", "name"), + {"schema": SCHEMA}, + ) + + +class EvaluationSetCase(TableBase): + """Evaluation cases belonging to a set.""" + + __tablename__ = "evaluation_set_case_t" + __table_args__ = {"schema": SCHEMA} + + evaluation_set_case_id = Column( + BigInteger, + Sequence("evaluation_set_case_t_evaluation_set_case_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + evaluation_set_id = Column(BigInteger, nullable=False, doc="Evaluation set id") + + case_id = Column(String(128), doc="External case_id from JSONL (optional)") + + inputs = Column(JSONB, nullable=False, doc="Case inputs JSON") + label = Column(JSONB, nullable=False, doc="Case label JSON") + + order_no = Column(Integer, default=0, doc="Case order in the set") + + __table_args__ = ( + Index("ix_eval_set_case_set_id", "evaluation_set_id"), + Index("ix_eval_set_case_tenant_id", "tenant_id"), + {"schema": SCHEMA}, + ) + + +class AgentEvaluation(TableBase): + """An evaluation run for a specific agent and evaluation set.""" + + __tablename__ = "agent_evaluation_t" + __table_args__ = {"schema": SCHEMA} + + agent_evaluation_id = Column( + BigInteger, + Sequence("agent_evaluation_t_agent_evaluation_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + + agent_id = Column(Integer, nullable=False, doc="Agent id") + agent_version_no = Column(Integer, nullable=False, doc="Published agent version_no used for evaluation") + + evaluation_set_id = Column(BigInteger, nullable=False, doc="Evaluation set id") + + status = Column( + String(30), + nullable=False, + default="PENDING", + doc="Run status: PENDING/RUNNING/COMPLETED/FAILED", + ) + + progress_total = Column(Integer, default=0, doc="Total cases") + progress_done = Column(Integer, default=0, doc="Completed cases") + + judge_model_id = Column( + Integer, + doc=( + "Model id used by the judge. Persisted so the background worker can " + "recover it after restart and the frontend can resolve judge_model_name." + ), + ) + + score_overall = Column(Float, doc="Overall score (0-1)") + + error_message = Column(Text, doc="Failure reason") + + __table_args__ = ( + Index("ix_agent_eval_tenant_id", "tenant_id"), + Index("ix_agent_eval_agent_id", "tenant_id", "agent_id"), + Index("ix_agent_eval_set_id", "tenant_id", "evaluation_set_id"), + Index("ix_agent_eval_judge_model_id", "tenant_id", "judge_model_id"), + {"schema": SCHEMA}, + ) + + +class AgentEvaluationCase(TableBase): + """Per-case evaluation details within an evaluation run.""" + + __tablename__ = "agent_evaluation_case_t" + __table_args__ = {"schema": SCHEMA} + + agent_evaluation_case_id = Column( + BigInteger, + Sequence("agent_evaluation_case_t_agent_evaluation_case_id_seq", schema=SCHEMA), + primary_key=True, + nullable=False, + doc=_PRIMARY_KEY_DOC, + ) + + tenant_id = Column(String(100), nullable=False, doc=_TENANT_ID_DOC) + + agent_evaluation_id = Column(BigInteger, nullable=False, doc="Evaluation run id") + evaluation_set_case_id = Column(BigInteger, nullable=False, doc="Evaluation set case id") + + inputs = Column(JSONB, nullable=False, doc="Case inputs snapshot (query only for pass cases)") + label = Column(JSONB, nullable=False, doc="Case label snapshot (cleared to {answer:''} for pass cases)") + predict = Column(JSONB, doc="Predict JSON (answer/raw); NULL for pass cases") + + score = Column(Float, doc="Case score (0-1)") + reason = Column(Text, doc="Judge reason; NULL for pass cases") + pass_status = Column( + String(16), + doc="Judge result: pass / fail. Pass cases have predict/reason/label.answer cleared to save space.", + ) + + status = Column( + String(30), + nullable=False, + default="PENDING", + doc="Case status: PENDING/RUNNING/COMPLETED/FAILED", + ) + error_message = Column(Text, doc="Per-case failure reason") + + __table_args__ = ( + Index("ix_agent_eval_case_eval_id", "agent_evaluation_id"), + Index("ix_agent_eval_case_tenant_id", "tenant_id"), + Index("ix_agent_eval_case_pass_status", "tenant_id", "agent_evaluation_id", "pass_status"), + {"schema": SCHEMA}, + ) diff --git a/backend/database/evaluation_set_db.py b/backend/database/evaluation_set_db.py new file mode 100644 index 000000000..ad2cbc88b --- /dev/null +++ b/backend/database/evaluation_set_db.py @@ -0,0 +1,149 @@ +import json +import logging +from typing import Any, Dict, List, Optional, Tuple + +from database.client import as_dict, filter_property, get_db_session +from database.db_models import EvaluationSet, EvaluationSetCase + +logger = logging.getLogger("evaluation_set_db") + + +def create_evaluation_set( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + created_by: Optional[str], +) -> Dict[str, Any]: + with get_db_session() as session: + rec = EvaluationSet( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + session.flush() + return as_dict(rec) + + +def update_evaluation_set_case_count(evaluation_set_id: int, case_count: int, updated_by: Optional[str] = None) -> None: + with get_db_session() as session: + session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.delete_flag == "N", + ).update({"case_count": case_count, "updated_by": updated_by}, synchronize_session=False) + + +def list_evaluation_sets(tenant_id: str, limit: int = 50, offset: int = 0) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSet) + .filter(EvaluationSet.tenant_id == tenant_id, EvaluationSet.delete_flag == "N") + .order_by(EvaluationSet.update_time.desc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_evaluation_set(evaluation_set_id: int, tenant_id: str) -> Dict[str, Any]: + with get_db_session() as session: + rec = session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + EvaluationSet.delete_flag == "N", + ).first() + if not rec: + raise ValueError("evaluation set not found") + return as_dict(rec) + + +def insert_evaluation_set_cases( + tenant_id: str, + evaluation_set_id: int, + cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> int: + """Insert cases. Each case must have: inputs(dict), label(dict), optional case_id(str). + + Returns inserted count. + """ + with get_db_session() as session: + inserted = 0 + for i, c in enumerate(cases): + rec = EvaluationSetCase( + tenant_id=tenant_id, + evaluation_set_id=evaluation_set_id, + case_id=c.get("case_id"), + inputs=c["inputs"], + label=c["label"], + order_no=int(c.get("order_no", i)), + created_by=created_by, + updated_by=created_by, + delete_flag="N", + ) + session.add(rec) + inserted += 1 + session.flush() + return inserted + + +def list_evaluation_set_cases( + evaluation_set_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSetCase) + .filter( + EvaluationSetCase.evaluation_set_id == evaluation_set_id, + EvaluationSetCase.tenant_id == tenant_id, + EvaluationSetCase.delete_flag == "N", + ) + .order_by(EvaluationSetCase.order_no.asc(), EvaluationSetCase.evaluation_set_case_id.asc()) + .offset(offset) + .limit(limit) + ) + return [as_dict(x) for x in q.all()] + + +def get_evaluation_set_cases_all(evaluation_set_id: int, tenant_id: str) -> List[Dict[str, Any]]: + with get_db_session() as session: + q = ( + session.query(EvaluationSetCase) + .filter( + EvaluationSetCase.evaluation_set_id == evaluation_set_id, + EvaluationSetCase.tenant_id == tenant_id, + EvaluationSetCase.delete_flag == "N", + ) + .order_by(EvaluationSetCase.order_no.asc(), EvaluationSetCase.evaluation_set_case_id.asc()) + ) + return [as_dict(x) for x in q.all()] + + +def soft_delete_evaluation_set( + evaluation_set_id: int, + tenant_id: str, + deleted_by: str, +) -> None: + """Soft-delete an evaluation set by setting delete_flag='Y'. + + Raises ``ValueError`` when the set is not found or has already been deleted. + """ + with get_db_session() as session: + rows = session.query(EvaluationSet).filter( + EvaluationSet.evaluation_set_id == evaluation_set_id, + EvaluationSet.tenant_id == tenant_id, + EvaluationSet.delete_flag == "N", + ).update( + {"delete_flag": "Y", "updated_by": deleted_by}, + synchronize_session=False, + ) + if rows == 0: + raise ValueError("evaluation set not found or already deleted") diff --git a/backend/database/market_mcp_db.py b/backend/database/market_mcp_db.py new file mode 100644 index 000000000..a27b139b1 --- /dev/null +++ b/backend/database/market_mcp_db.py @@ -0,0 +1,303 @@ +import logging +from typing import Any, Dict, List + +from sqlalchemy import func, or_ + +from database.client import as_dict, filter_property, get_db_session +from database.db_models import McpMarketRecord + +logger = logging.getLogger("market_mcp_db") + + +def get_mcp_market_records( + *, + tenant_id: str | None = None, + search: str | None = None, + tag: str | None = None, + transport_type: str | None = None, + cursor: str | None = None, + limit: int = 30, +) -> Dict[str, Any]: + """Cursor-paginated listing of shared (approved) market records scoped to a tenant.""" + with get_db_session() as session: + query = session.query(McpMarketRecord).filter( + McpMarketRecord.delete_flag != "Y", + McpMarketRecord.review_status == "shared", + ) + + if tenant_id: + query = query.filter(McpMarketRecord.tenant_id == tenant_id) + + if transport_type: + query = query.filter(McpMarketRecord.transport_type == transport_type) + + if tag: + query = query.filter(McpMarketRecord.tags.any(tag)) + + if search: + keyword = f"%{search}%" + query = query.filter( + or_( + McpMarketRecord.mcp_name.ilike(keyword), + McpMarketRecord.description.ilike(keyword), + func.array_to_string(McpMarketRecord.tags, ",").ilike(keyword), + ) + ) + + cursor_id: int | None = None + if cursor: + try: + cursor_id = int(cursor) + except ValueError: + cursor_id = None + + if cursor_id is not None: + query = query.filter(McpMarketRecord.market_id < cursor_id) + + rows: List[McpMarketRecord] = ( + query.order_by(McpMarketRecord.market_id.desc()) + .limit(limit + 1) + .all() + ) + + has_next = len(rows) > limit + page_rows = rows[:limit] + + next_cursor = None + if has_next and page_rows: + next_cursor = str(page_rows[-1].market_id) + + return { + "count": len(page_rows), + "nextCursor": next_cursor, + "items": [as_dict(row) for row in page_rows], + } + + +def get_mcp_market_tag_stats() -> List[Dict[str, Any]]: + """Aggregate tag statistics from all shared market records.""" + with get_db_session() as session: + rows = ( + session.query( + func.unnest(McpMarketRecord.tags).label("tag"), + func.count(McpMarketRecord.market_id).label("count"), + ) + .filter( + McpMarketRecord.delete_flag != "Y", + McpMarketRecord.review_status == "shared", + ) + .group_by("tag") + .order_by(func.count(McpMarketRecord.market_id).desc(), "tag") + .all() + ) + return [{"tag": str(row.tag), "count": int(row.count)} for row in rows if row.tag] + + +def create_mcp_market_record(mcp_data: Dict[str, Any], tenant_id: str, user_id: str) -> int: + """Create a new market record. Returns the new market_id.""" + with get_db_session() as session: + mcp_data.update({ + "tenant_id": tenant_id, + "user_id": user_id, + "created_by": user_id, + "updated_by": user_id, + "delete_flag": "N", + "source": "community", + }) + if "review_status" not in mcp_data: + mcp_data["review_status"] = "not_shared" + new_record = McpMarketRecord(**filter_property(mcp_data, McpMarketRecord)) + session.add(new_record) + session.flush() + return int(new_record.market_id) + + +def get_mcp_market_record_by_id(market_id: int) -> Dict[str, Any] | None: + """Fetch a single market record by ID (any status, non-deleted).""" + with get_db_session() as session: + record = session.query(McpMarketRecord).filter( + McpMarketRecord.market_id == market_id, + McpMarketRecord.delete_flag != "Y", + ).first() + return as_dict(record) if record else None + + +def check_mcp_market_name_exists(mcp_name: str) -> bool: + """Check if a shared market record with the given name already exists. + + Matches the partial unique index uq_mcp_market_name_active: + WHERE delete_flag = 'N' AND review_status = 'shared' + """ + with get_db_session() as session: + record = session.query(McpMarketRecord).filter( + McpMarketRecord.mcp_name == mcp_name, + McpMarketRecord.delete_flag != "Y", + McpMarketRecord.review_status == "shared", + ).first() + return record is not None + + +def update_mcp_market_record( + *, + market_id: int, + user_id: str, + mcp_name: str | None = None, + description: str | None = None, + tags: List[str] | None = None, + registry_json: Dict[str, Any] | None = None, + mcp_server: str | None = None, + config_json: Dict[str, Any] | None = None, + transport_type: str | None = None, +) -> None: + """Update editable fields on a market record (does not change status).""" + update_fields: Dict[str, Any] = {"updated_by": user_id} + if mcp_name is not None: + update_fields["mcp_name"] = mcp_name + if description is not None: + update_fields["description"] = description + if tags is not None: + update_fields["tags"] = tags + if registry_json is not None: + update_fields["registry_json"] = registry_json + if mcp_server is not None: + update_fields["mcp_server"] = mcp_server + if config_json is not None: + update_fields["config_json"] = config_json + if transport_type is not None: + update_fields["transport_type"] = transport_type + + with get_db_session() as session: + session.query(McpMarketRecord).filter( + McpMarketRecord.market_id == market_id, + McpMarketRecord.delete_flag != "Y", + ).update(update_fields) + + +def update_mcp_market_status( + *, + market_id: int, + user_id: str, + review_status: str, + submitted_by: str | None = None, +) -> None: + """Atomically update the review_status, optionally recording the submitter.""" + update_fields: Dict[str, Any] = {"updated_by": user_id, "review_status": review_status} + if submitted_by is not None: + update_fields["submitted_by"] = submitted_by + + with get_db_session() as session: + session.query(McpMarketRecord).filter( + McpMarketRecord.market_id == market_id, + McpMarketRecord.delete_flag != "Y", + ).update(update_fields) + + +def delete_mcp_market_record_by_id(*, market_id: int, user_id: str) -> None: + """Soft-delete a market record.""" + with get_db_session() as session: + session.query(McpMarketRecord).filter( + McpMarketRecord.market_id == market_id, + McpMarketRecord.delete_flag != "Y", + ).update({"delete_flag": "Y", "updated_by": user_id}) + + +def list_mcp_market_records_by_tenant_and_user(tenant_id: str, user_id: str) -> List[Dict[str, Any]]: + """List all market records (all statuses) for a specific tenant+user.""" + with get_db_session() as session: + rows = session.query(McpMarketRecord).filter( + McpMarketRecord.tenant_id == tenant_id, + McpMarketRecord.user_id == user_id, + McpMarketRecord.delete_flag != "Y", + ).order_by(McpMarketRecord.market_id.desc()).all() + return [as_dict(row) for row in rows] + + +def list_mcp_market_records_by_status( + *, + tenant_id: str | None = None, + review_status: str | None = None, + search: str | None = None, + tag: str | None = None, + transport_type: str | None = None, + cursor: str | None = None, + limit: int = 30, +) -> Dict[str, Any]: + """Cursor-paginated listing of market records filtered by review status (review queue).""" + with get_db_session() as session: + query = session.query(McpMarketRecord).filter( + McpMarketRecord.delete_flag != "Y", + ) + + if tenant_id is not None: + query = query.filter(McpMarketRecord.tenant_id == tenant_id) + if review_status: + query = query.filter(McpMarketRecord.review_status == review_status) + if transport_type: + query = query.filter(McpMarketRecord.transport_type == transport_type) + if tag: + query = query.filter(McpMarketRecord.tags.any(tag)) + if search: + keyword = f"%{search}%" + query = query.filter( + or_( + McpMarketRecord.mcp_name.ilike(keyword), + McpMarketRecord.description.ilike(keyword), + func.array_to_string(McpMarketRecord.tags, ",").ilike(keyword), + ) + ) + + cursor_id: int | None = None + if cursor: + try: + cursor_id = int(cursor) + except ValueError: + cursor_id = None + if cursor_id is not None: + query = query.filter(McpMarketRecord.market_id < cursor_id) + + rows: List[McpMarketRecord] = ( + query.order_by(McpMarketRecord.market_id.desc()) + .limit(limit + 1) + .all() + ) + has_next = len(rows) > limit + page_rows = rows[:limit] + next_cursor = str(page_rows[-1].market_id) if has_next and page_rows else None + + return { + "count": len(page_rows), + "nextCursor": next_cursor, + "items": [as_dict(row) for row in page_rows], + } + + +def increment_mcp_market_download_count(market_id: int) -> None: + """Atomically increment the download counter on a market record.""" + with get_db_session() as session: + session.query(McpMarketRecord).filter( + McpMarketRecord.market_id == market_id, + McpMarketRecord.delete_flag != "Y", + ).update( + {McpMarketRecord.download_count: McpMarketRecord.download_count + 1}, + synchronize_session=False, + ) + + +def get_mcp_market_tag_stats_by_tenant(tenant_id: str) -> List[Dict[str, Any]]: + """Tag stats scoped to a specific tenant from shared records only.""" + with get_db_session() as session: + rows = ( + session.query( + func.unnest(McpMarketRecord.tags).label("tag"), + func.count(McpMarketRecord.market_id).label("count"), + ) + .filter( + McpMarketRecord.tenant_id == tenant_id, + McpMarketRecord.delete_flag != "Y", + McpMarketRecord.review_status == "shared", + ) + .group_by("tag") + .order_by(func.count(McpMarketRecord.market_id).desc(), "tag") + .all() + ) + return [{"tag": str(row.tag), "count": int(row.count)} for row in rows if row.tag] diff --git a/backend/database/model_management_db.py b/backend/database/model_management_db.py index 1a1a98c8b..64b00b9f3 100644 --- a/backend/database/model_management_db.py +++ b/backend/database/model_management_db.py @@ -182,7 +182,7 @@ def get_model_by_display_name(display_name: str, tenant_id: str, model_type: str tenant_id: """ filters = {'display_name': display_name} - + if model_type in ["multiEmbedding", "multi_embedding"]: filters['model_type'] = "multi_embedding" elif model_type == "embedding": @@ -216,7 +216,7 @@ def get_model_id_by_display_name(display_name: str, tenant_id: str, model_type: Get a model ID by display name Args: - display_name: Model display name + display_name: Model display name tenant_id: tenant_id Returns: @@ -228,7 +228,7 @@ def get_model_id_by_display_name(display_name: str, tenant_id: str, model_type: def get_model_by_model_id(model_id: int, tenant_id: Optional[str] = None) -> Optional[Dict[str, Any]]: """ - Get a model record using native SQLAlchemy query + Get a model record using native SQLAlchemy query. Args: model_id (int): Model ID @@ -269,6 +269,37 @@ def get_model_by_model_id(model_id: int, tenant_id: Optional[str] = None) -> Opt return result_dict +def get_model_by_model_id_ignore_delete(model_id: int, tenant_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """ + Get a model record without filtering by delete_flag. + + Used when checking whether a previously selected model has been deleted + (e.g., for tools that store a selected_model_id in their params). + + Args: + model_id (int): Model ID + tenant_id (Optional[str]): Tenant ID, optional + + Returns: + Optional[Dict[str, Any]]: Model record as a dictionary, or None if not found + """ + with get_db_session() as session: + stmt = select(ModelRecord).where(ModelRecord.model_id == model_id) + if tenant_id: + stmt = stmt.where(ModelRecord.tenant_id == tenant_id) + result = session.scalars(stmt).first() + if result is None: + return None + result_dict = {key: value for key, + value in result.__dict__.items() if not key.startswith('_')} + if result_dict.get("model_type") in ["embedding", "multi_embedding"]: + if result_dict.get("expected_chunk_size") is None: + result_dict["expected_chunk_size"] = DEFAULT_EXPECTED_CHUNK_SIZE + if result_dict.get("maximum_chunk_size") is None: + result_dict["maximum_chunk_size"] = DEFAULT_MAXIMUM_CHUNK_SIZE + return result_dict + + def get_models_by_tenant_factory_type(tenant_id: str, model_factory: str, model_type: str) -> List[Dict[str, Any]]: """ Get all model database records matching tenant_id, model_factory, and model_type. @@ -280,15 +311,44 @@ def get_models_by_tenant_factory_type(tenant_id: str, model_factory: str, model_ return get_model_records(filters, tenant_id) +def get_valid_model_ids(model_ids: List[int], tenant_id: str) -> List[int]: + """ + Filter model IDs to only include those that are not soft-deleted (delete_flag='N'). + + When a model is deleted from model management, its delete_flag is set to 'Y'. + This function ensures that only valid (non-deleted) model IDs are returned, + preserving the order of the input model_ids. + + Args: + model_ids: List of model IDs to filter + tenant_id: Tenant ID for filtering + + Returns: + List[int]: Filtered list of valid model IDs in original order + """ + if not model_ids: + return [] + + with get_db_session() as session: + stmt = select(ModelRecord.model_id).where( + ModelRecord.model_id.in_(model_ids), + ModelRecord.delete_flag == 'N' + ) + valid_ids = session.scalars(stmt).all() + valid_ids_set = set(valid_ids) + + return [mid for mid in model_ids if mid in valid_ids_set] + + def get_model_by_name_factory(model_name: str, model_factory: str, tenant_id: str) -> Optional[Dict[str, Any]]: """ Get a model record by model_name and model_factory for deduplication. - + Args: model_name: Model name (e.g., "deepseek-r1-distill-qwen-14b") model_factory: Model factory (e.g., "ModelEngine") tenant_id: Tenant ID - + Returns: Optional[Dict[str, Any]]: Model record if found, None otherwise """ diff --git a/backend/database/remote_mcp_db.py b/backend/database/remote_mcp_db.py index b08769437..d6223628f 100644 --- a/backend/database/remote_mcp_db.py +++ b/backend/database/remote_mcp_db.py @@ -16,18 +16,11 @@ def create_mcp_record(mcp_data: Dict[str, Any], tenant_id: str, user_id: str): :param user_id: User ID :return: Created MCP record - Note: Only fields defined in the McpRecord model are inserted. - Fields like 'transport_type' and 'version' are not part of McpRecord - and will be ignored. """ - # Filter to only include fields that exist in the model - # McpRecord fields: mcp_id, tenant_id, user_id, mcp_name, mcp_server, status, - # container_id, container_port, authorization_token, source, registry_json, - # config_json, enabled, tags, description, create_time, update_time, created_by, updated_by, delete_flag allowed_fields = { 'mcp_name', 'mcp_server', 'status', 'container_id', 'container_port', - 'authorization_token', 'custom_headers', 'source', 'registry_json', 'config_json', - 'enabled', 'tags', 'description' + 'authorization_token', 'custom_headers', 'source', 'market_id', + 'registry_json', 'config_json', 'enabled', 'tags', 'description' } filtered_data = {k: v for k, v in mcp_data.items() if k in allowed_fields and v is not None} @@ -145,6 +138,7 @@ def update_mcp_record_manage_fields_by_id( authorization_token: str | None, custom_headers: Dict[str, Any] | None, config_json: Dict[str, Any] | None, + market_id: int | None, ) -> None: with get_db_session() as session: session.query(McpRecord).filter( @@ -161,11 +155,41 @@ def update_mcp_record_manage_fields_by_id( "authorization_token": authorization_token, "custom_headers": custom_headers, "config_json": config_json, + "market_id": market_id, "updated_by": user_id, } ) +def update_mcp_record_market_id_by_id( + *, + mcp_id: int, + tenant_id: str, + user_id: str, + market_id: int | None, +) -> None: + with get_db_session() as session: + session.query(McpRecord).filter( + McpRecord.mcp_id == mcp_id, + McpRecord.tenant_id == tenant_id, + McpRecord.delete_flag != 'Y' + ).update({"market_id": market_id, "updated_by": user_id}) + + +def clear_mcp_record_market_id( + *, + tenant_id: str, + user_id: str, + market_id: int, +) -> None: + with get_db_session() as session: + session.query(McpRecord).filter( + McpRecord.tenant_id == tenant_id, + McpRecord.market_id == market_id, + McpRecord.delete_flag != 'Y' + ).update({"market_id": None, "updated_by": user_id}) + + def update_mcp_record_enabled_by_id( *, mcp_id: int, @@ -387,3 +411,22 @@ def get_mcp_record_by_id_and_tenant(mcp_id: int, tenant_id: str) -> Dict[str, An ).first() return as_dict(mcp_record) if mcp_record else None + + +def update_mcp_record_registry_json_by_id( + *, + mcp_id: int, + tenant_id: str, + user_id: str, + registry_json: dict, +) -> None: + """Update the registry_json field on an MCP record (e.g. to persist refreshed tool names).""" + with get_db_session() as session: + session.query(McpRecord).filter( + McpRecord.mcp_id == mcp_id, + McpRecord.tenant_id == tenant_id, + McpRecord.delete_flag != 'Y' + ).update({ + "registry_json": registry_json, + "updated_by": user_id, + }) diff --git a/backend/database/skill_db.py b/backend/database/skill_db.py index 6a3f69069..8f51849bb 100644 --- a/backend/database/skill_db.py +++ b/backend/database/skill_db.py @@ -35,7 +35,8 @@ def create_or_update_skill_by_skill_info(skill_info, tenant_id: str, user_id: st Returns: Created or updated SkillInstance object """ - skill_info_dict = skill_info.__dict__ if hasattr(skill_info, '__dict__') else skill_info + skill_info_dict = skill_info.__dict__ if hasattr( + skill_info, '__dict__') else skill_info skill_info_dict = skill_info_dict.copy() skill_info_dict.setdefault("tenant_id", tenant_id) skill_info_dict.setdefault("user_id", user_id) @@ -178,7 +179,6 @@ def delete_skill_instances_by_tenant(tenant_id: str, user_id: str) -> int: return count - # ============== SkillInfo Repository Functions ============== @@ -190,6 +190,46 @@ def _get_tool_ids(session, skill_id: int) -> List[int]: return [r.tool_id for r in relations] +def _build_skill_update_values( + skill_data: Dict[str, Any], + updated_by: Optional[str], +) -> Dict[str, Any]: + row_values: Dict[str, Any] = {"update_time": datetime.now()} + if updated_by: + row_values["updated_by"] = updated_by + + field_mapping = { + "description": "skill_description", + "content": "skill_content", + "tags": "skill_tags", + "source": "source", + } + for input_field, model_field in field_mapping.items(): + if input_field in skill_data: + row_values[model_field] = skill_data[input_field] + + for field in ("config_schemas", "config_values"): + if field in skill_data: + row_values[field] = _params_value_for_db(skill_data[field]) + return row_values + + +def _replace_skill_tool_relations( + session, + skill_id: int, + tool_ids: List[int], +) -> None: + session.query(SkillToolRelation).filter( + SkillToolRelation.skill_id == skill_id + ).delete() + for tool_id in tool_ids: + session.add(SkillToolRelation( + skill_id=skill_id, + tool_id=tool_id, + create_time=datetime.now(), + )) + + def _to_dict(skill: SkillInfo) -> Dict[str, Any]: """Convert SkillInfo to dict.""" return { @@ -323,8 +363,10 @@ def create_skill(skill_data: Dict[str, Any], tenant_id: str) -> Dict[str, Any]: skill_description=skill_data.get("description", ""), skill_tags=skill_data.get("tags", []), skill_content=skill_data.get("content", ""), - config_schemas=_params_value_for_db(skill_data.get("config_schemas")), - config_values=_params_value_for_db(skill_data.get("config_values")), + config_schemas=_params_value_for_db( + skill_data.get("config_schemas")), + config_values=_params_value_for_db( + skill_data.get("config_values")), source=skill_data.get("source", "custom"), created_by=skill_data.get("created_by"), create_time=datetime.now(), @@ -383,23 +425,7 @@ def update_skill( raise ValueError(f"Skill not found: {skill_name}") skill_id = skill.skill_id - now = datetime.now() - row_values: Dict[str, Any] = {"update_time": now} - if updated_by: - row_values["updated_by"] = updated_by - - if "description" in skill_data: - row_values["skill_description"] = skill_data["description"] - if "content" in skill_data: - row_values["skill_content"] = skill_data["content"] - if "tags" in skill_data: - row_values["skill_tags"] = skill_data["tags"] - if "source" in skill_data: - row_values["source"] = skill_data["source"] - if "config_schemas" in skill_data: - row_values["config_schemas"] = _params_value_for_db(skill_data["config_schemas"]) - if "config_values" in skill_data: - row_values["config_values"] = _params_value_for_db(skill_data["config_values"]) + row_values = _build_skill_update_values(skill_data, updated_by) session.execute( sa_update(SkillInfo) @@ -411,17 +437,11 @@ def update_skill( ) if "tool_ids" in skill_data: - session.query(SkillToolRelation).filter( - SkillToolRelation.skill_id == skill_id - ).delete() - - for tool_id in skill_data["tool_ids"]: - rel = SkillToolRelation( - skill_id=skill_id, - tool_id=tool_id, - create_time=datetime.now() - ) - session.add(rel) + _replace_skill_tool_relations( + session, + skill_id, + skill_data["tool_ids"], + ) session.commit() @@ -441,6 +461,74 @@ def update_skill( return result +def update_skill_by_id( + skill_id: int, + skill_data: Dict[str, Any], + tenant_id: str, + updated_by: Optional[str] = None, +) -> Dict[str, Any]: + """Update an existing skill by ID for a tenant.""" + with get_db_session() as session: + skill = session.query(SkillInfo).filter( + SkillInfo.skill_id == skill_id, + SkillInfo.tenant_id == tenant_id, + SkillInfo.delete_flag != "Y", + ).first() + + if not skill: + raise ValueError(f"Skill not found: {skill_id}") + + row_values = _build_skill_update_values(skill_data, updated_by) + + if "name" in skill_data: + new_name = str(skill_data["name"] or "").strip() + if not new_name: + raise ValueError("Skill name is required") + if new_name != skill.skill_name: + duplicate = session.query(SkillInfo).filter( + SkillInfo.skill_name == new_name, + SkillInfo.tenant_id == tenant_id, + SkillInfo.skill_id != skill_id, + SkillInfo.delete_flag != "Y", + ).first() + if duplicate: + raise ValueError(f"Skill '{new_name}' already exists") + row_values["skill_name"] = new_name + session.execute( + sa_update(SkillInfo) + .where( + SkillInfo.skill_id == skill_id, + SkillInfo.tenant_id == tenant_id, + SkillInfo.delete_flag != "Y", + ) + .values(**row_values) + ) + + if "tool_ids" in skill_data: + _replace_skill_tool_relations( + session, + skill_id, + skill_data["tool_ids"], + ) + + session.commit() + + refreshed = session.query(SkillInfo).filter( + SkillInfo.skill_id == skill_id, + SkillInfo.tenant_id == tenant_id, + SkillInfo.delete_flag != "Y", + ).first() + if not refreshed: + raise ValueError(f"Skill not found after update: {skill_id}") + + result = _to_dict(refreshed) + result["tool_ids"] = skill_data.get( + "tool_ids", + _get_tool_ids(session, skill_id), + ) + return result + + def delete_skill(skill_name: str, tenant_id: str, updated_by: Optional[str] = None) -> bool: """Soft delete a skill for a tenant (mark as deleted). @@ -600,8 +688,10 @@ def upsert_scanned_skills(skills: List[Dict[str, Any]], user_id: str, tenant_id: existing.skill_description = skill_data.get("description", "") existing.skill_tags = skill_data.get("tags", []) existing.skill_content = skill_data.get("content", "") - existing.config_schemas = _params_value_for_db(skill_data.get("config_schemas")) - existing.config_values = _params_value_for_db(skill_data.get("config_values")) + existing.config_schemas = _params_value_for_db( + skill_data.get("config_schemas")) + existing.config_values = _params_value_for_db( + skill_data.get("config_values")) existing.updated_by = user_id else: new_skill = SkillInfo( @@ -610,8 +700,10 @@ def upsert_scanned_skills(skills: List[Dict[str, Any]], user_id: str, tenant_id: skill_description=skill_data.get("description", ""), skill_tags=skill_data.get("tags", []), skill_content=skill_data.get("content", ""), - config_schemas=_params_value_for_db(skill_data.get("config_schemas")), - config_values=_params_value_for_db(skill_data.get("config_values")), + config_schemas=_params_value_for_db( + skill_data.get("config_schemas")), + config_values=_params_value_for_db( + skill_data.get("config_values")), source=skill_data.get("source", "official"), created_by=user_id, updated_by=user_id, diff --git a/backend/database/skill_repository_db.py b/backend/database/skill_repository_db.py new file mode 100644 index 000000000..e99997476 --- /dev/null +++ b/backend/database/skill_repository_db.py @@ -0,0 +1,333 @@ +import math +from typing import Any, Collection, Dict, List, Optional + +from sqlalchemy import func, or_, update + +from consts.agent_repository import STATUS_NOT_SHARED +from database.client import as_dict, filter_property, get_db_session +from database.db_models import SkillRepository + +_UPDATE_ALLOWED_FIELDS = frozenset({ + "name", + "description", + "source", + "submitted_by", + "category_id", + "tags", + "icon", + "downloads", + "skill_info_json", + "skill_zip_base64", + "status", +}) + + +def insert_skill_repository_record( + repository_data: Dict[str, Any], + publisher_tenant_id: str, + publisher_user_id: str, +) -> int: + """Insert a new skill repository listing record.""" + with get_db_session() as session: + payload = { + **repository_data, + "publisher_tenant_id": publisher_tenant_id, + "publisher_user_id": publisher_user_id, + "created_by": publisher_user_id, + "updated_by": publisher_user_id, + "delete_flag": "N", + } + if payload.get("status") is None: + payload["status"] = STATUS_NOT_SHARED + + new_record = SkillRepository( + **filter_property(payload, SkillRepository) + ) + session.add(new_record) + session.flush() + return int(new_record.skill_repository_id) + + +def get_skill_repository_by_id_and_publisher( + repository_id: int, + publisher_tenant_id: str, +) -> Optional[dict]: + """Fetch a repository listing scoped to the publisher tenant.""" + with get_db_session() as session: + record = session.query(SkillRepository).filter( + SkillRepository.skill_repository_id == repository_id, + SkillRepository.publisher_tenant_id == publisher_tenant_id, + SkillRepository.delete_flag != "Y", + ).first() + return as_dict(record) if record else None + + +def get_skill_repository_by_skill_id( + skill_id: int, + *, + publisher_tenant_id: Optional[str] = None, +) -> Optional[dict]: + """Fetch an active repository listing by source skill_id.""" + with get_db_session() as session: + query = session.query(SkillRepository).filter( + SkillRepository.skill_id == skill_id, + SkillRepository.delete_flag != "Y", + ) + if publisher_tenant_id is not None: + query = query.filter( + SkillRepository.publisher_tenant_id == publisher_tenant_id, + ) + record = query.first() + return as_dict(record) if record else None + + +def _apply_skill_repository_filters( + query, + *, + publisher_tenant_id: str, + status: Optional[str], + skill_id: Optional[int], + category_id: Optional[int], + search: Optional[str], +): + query = query.filter( + SkillRepository.delete_flag != "Y", + SkillRepository.publisher_tenant_id == publisher_tenant_id, + ) + if status: + query = query.filter(SkillRepository.status == status) + if skill_id is not None: + query = query.filter(SkillRepository.skill_id == skill_id) + if category_id is not None: + query = query.filter(SkillRepository.category_id == category_id) + + keyword = (search or "").strip() + if keyword: + pattern = f"%{keyword}%" + query = query.filter(or_( + SkillRepository.name.ilike(pattern), + SkillRepository.description.ilike(pattern), + SkillRepository.source.ilike(pattern), + SkillRepository.submitted_by.ilike(pattern), + func.array_to_string(SkillRepository.tags, " ").ilike(pattern), + )) + return query + + +def list_skill_repository_summaries( + publisher_tenant_id: str, + *, + status: Optional[str] = None, + skill_id: Optional[int] = None, + category_id: Optional[int] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + sort_by_update_time: bool = False, +) -> Dict[str, Any]: + """List active repository summaries for a publisher tenant without heavy payloads.""" + safe_page = max(int(page or 1), 1) + safe_page_size = max(int(page_size or 10), 1) + + with get_db_session() as session: + query = session.query( + SkillRepository.skill_repository_id, + SkillRepository.skill_id, + SkillRepository.submitted_by, + SkillRepository.name, + SkillRepository.description, + SkillRepository.source, + SkillRepository.status, + SkillRepository.category_id, + SkillRepository.tags, + SkillRepository.icon, + SkillRepository.downloads, + SkillRepository.create_time, + ) + query = _apply_skill_repository_filters( + query, + publisher_tenant_id=publisher_tenant_id, + status=status, + skill_id=skill_id, + category_id=category_id, + search=search, + ) + + total = query.count() + order_by_fields = ( + (SkillRepository.update_time.desc(), + SkillRepository.skill_repository_id.desc()) + if sort_by_update_time + else (SkillRepository.skill_repository_id.desc(),) + ) + rows = ( + query.order_by(*order_by_fields) + .offset((safe_page - 1) * safe_page_size) + .limit(safe_page_size) + .all() + ) + + items = [ + { + "skill_repository_id": row.skill_repository_id, + "skill_id": row.skill_id, + "submitted_by": row.submitted_by, + "name": row.name, + "description": row.description, + "source": row.source, + "status": row.status, + "category_id": row.category_id, + "tags": row.tags or [], + "icon": row.icon, + "downloads": row.downloads or 0, + "created_at": row.create_time.isoformat() if row.create_time else None, + } + for row in rows + ] + return { + "items": items, + "pagination": { + "page": safe_page, + "page_size": safe_page_size, + "total": total, + "total_pages": math.ceil(total / safe_page_size) if total else 0, + }, + } + + +def update_skill_repository_by_id( + *, + repository_id: int, + publisher_tenant_id: str, + user_id: str, + updates: Dict[str, Any], +) -> int: + """Update a repository listing owned by the publisher tenant. Returns affected row count.""" + update_fields = { + key: value + for key, value in updates.items() + if key in _UPDATE_ALLOWED_FIELDS + } + if not update_fields: + return 0 + + update_fields["updated_by"] = user_id + + with get_db_session() as session: + result = session.execute( + update(SkillRepository) + .where( + SkillRepository.skill_repository_id == repository_id, + SkillRepository.publisher_tenant_id == publisher_tenant_id, + SkillRepository.delete_flag != "Y", + ) + .values(**update_fields) + ) + return int(result.rowcount or 0) + + +def update_skill_repository_status_by_id( + *, + repository_id: int, + status: str, + user_id: str, + filter_publisher_tenant_id: Optional[str] = None, + publisher_tenant_id: Optional[str] = None, + publisher_user_id: Optional[str] = None, + submitted_by: Optional[str] = None, +) -> int: + """Update repository listing status by primary key. Returns affected row count.""" + update_values: Dict[str, Any] = { + "status": status, + "updated_by": user_id, + } + if publisher_tenant_id is not None: + update_values["publisher_tenant_id"] = publisher_tenant_id + if publisher_user_id is not None: + update_values["publisher_user_id"] = publisher_user_id + if submitted_by is not None: + update_values["submitted_by"] = submitted_by + + with get_db_session() as session: + where_clauses = [ + SkillRepository.skill_repository_id == repository_id, + SkillRepository.delete_flag != "Y", + ] + if filter_publisher_tenant_id is not None: + where_clauses.append( + SkillRepository.publisher_tenant_id == filter_publisher_tenant_id + ) + result = session.execute( + update(SkillRepository) + .where(*where_clauses) + .values(**update_values) + ) + return int(result.rowcount or 0) + + +def increment_skill_repository_downloads( + *, + repository_id: int, + user_id: Optional[str] = None, + increment: int = 1, +) -> int: + """Increment install count for a repository listing. Returns affected row count.""" + update_values: Dict[str, Any] = { + "downloads": func.coalesce(SkillRepository.downloads, 0) + increment, + } + if user_id is not None: + update_values["updated_by"] = user_id + + with get_db_session() as session: + result = session.execute( + update(SkillRepository) + .where( + SkillRepository.skill_repository_id == repository_id, + SkillRepository.delete_flag != "Y", + ) + .values(**update_values) + ) + return int(result.rowcount or 0) + + +def list_skill_repository_by_skill_ids( + skill_ids: List[int], + *, + statuses: Collection[str], + publisher_tenant_id: str, +) -> List[dict]: + """List repository rows for the given skills, scoped to publisher tenant and statuses.""" + if not skill_ids: + return [] + + status_list = list(statuses) + with get_db_session() as session: + rows = ( + session.query( + SkillRepository.skill_repository_id, + SkillRepository.skill_id, + SkillRepository.status, + SkillRepository.create_time, + ) + .filter( + SkillRepository.delete_flag != "Y", + SkillRepository.publisher_tenant_id == publisher_tenant_id, + SkillRepository.skill_id.in_(skill_ids), + SkillRepository.status.in_(status_list), + ) + .order_by( + SkillRepository.skill_id, + SkillRepository.create_time.desc(), + ) + .all() + ) + + return [ + { + "skill_repository_id": row.skill_repository_id, + "skill_id": row.skill_id, + "status": row.status, + "create_time": row.create_time, + } + for row in rows + ] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b8f51dd4c..9481cdbc7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "openjiuwen>=0.1.0", "pydantic-settings>=2.0.0", "python-docx>=1.1.0", + "xlrd>=2.0.1", ] [project.optional-dependencies] @@ -38,6 +39,8 @@ data-process = [ "flower>=2.0.1", "nest_asyncio>=1.5.6", "unstructured[csv,docx,pdf,pptx,xlsx,md]==0.18.14", + "openpyxl>=3.1.5", + "xlrd>=2.0.1", "huggingface_hub>=0.30.0,<1.0" ] test = [ diff --git a/backend/services/agent_evaluation_service.py b/backend/services/agent_evaluation_service.py new file mode 100644 index 000000000..49c199623 --- /dev/null +++ b/backend/services/agent_evaluation_service.py @@ -0,0 +1,866 @@ +import asyncio +import io +import json +import logging +from statistics import mean +from typing import Any, Dict, List, Optional, Tuple + +from adapters.exception import JiuwenSDKError, JiuwenSDKUnavailableError + +try: + from adapters.jiuwen_sdk_adapter import JiuwenSDKAdapter +except ModuleNotFoundError: + JiuwenSDKAdapter = None # type: ignore[assignment, misc] +from consts.model import AgentRequest +from database.agent_evaluation_db import ( + create_agent_evaluation, + create_agent_evaluation_cases, + get_agent_evaluation, + list_agent_evaluation_cases, + list_agent_evaluations_by_agent, + soft_delete_agent_evaluation, + update_agent_evaluation_case_result, + update_agent_evaluation_status, +) +from database.evaluation_set_db import get_evaluation_set_cases_all +from services.evaluation_set_service import resolve_latest_published_version_no +from services.agent_service import prepare_agent_run +from utils.thread_utils import pool +from openpyxl import Workbook +from openpyxl.styles import Font, PatternFill, Alignment +import re + +logger = logging.getLogger(__name__) + + +# Log records emitted during SDK invocations may bleed into the ``reason`` +# field as ``"[HH:MM:SS LEVEL logger_name] {...payload...}"``. Extract the +# embedded JSON ``reason`` from those polluted strings so the report shows the +# judge's actual explanation rather than the surrounding log envelope. +_LOG_PREFIX_RE = re.compile( + r"(?:" + # Short form: ``[HH:MM:SS LEVEL logger] `` + r"\[(\d{2}:\d{2}:\d{2}|\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\w+\s+[\w.]+\]" + # Long form: ``YYYY-MM-DD HH:MM:SS | logger_name | trace_id | LEVEL | `` + r"|(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})" + r"(?:\s*\|\s*[\w.]+){1,3}\s*\|?" + r")\s*" +) +_MARKDOWN_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL) + + +def _extract_clean_reason(raw: Any) -> str: + """Best-effort extraction of the judge model's reason text. + + The ``reason`` column persisted by the judge pipeline can take three + shapes depending on which step produced the value: + + 1. The plain judge verdict: ``"pass"`` or ``"fail"``. + 2. The judge's free-form explanation (when the SDK upgrades to return it). + 3. A captured log record of the form + ``"[HH:MM:SS LEVEL llm] {event_id:..., response_content: '```json\\n{result, reason}\\n```', ...}"`` + where the judge JSON is nested inside ``response_content``. + + This helper unwraps case 3 by parsing the outer JSON, pulling + ``response_content``, stripping the markdown code fence, and returning the + inner ``reason`` field. Cases 1 and 2 are returned as-is. + """ + if raw is None: + return "" + text = str(raw).strip() + if not text: + return "" + + # Strip leading "[HH:MM:SS LEVEL logger] " log envelope(s) that may have + # been prepended one or more times by the root-logger StreamHandler when + # the SDK redirected ``response`` into a log call. + stripped = _LOG_PREFIX_RE.sub("", text).strip() + if not stripped: + return text + + # Try the full string first; if it is not parseable JSON, fall back to + # the inner match. This handles both well-formed outer objects and strings + # that have stray characters around a single JSON object. + parsed: Optional[Dict[str, Any]] = None + try: + parsed = json.loads(stripped) + except (ValueError, TypeError): + match = _JSON_OBJECT_RE.search(stripped) + if match: + try: + parsed = json.loads(match.group(0)) + except (ValueError, TypeError): + parsed = None + + if not isinstance(parsed, dict): + # Not a JSON envelope at all — leave the stripped text in place so + # plain "pass"/"fail" and free-form explanations still render. + return stripped + + response_content = parsed.get("response_content") + if isinstance(response_content, str): + # Strip the ```json ... ``` fence the judge model wraps its verdict in. + fence_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", response_content, re.DOTALL) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + if isinstance(inner, dict) and isinstance(inner.get("reason"), str): + return inner["reason"].strip() + except (ValueError, TypeError): + pass + + reason_field = parsed.get("reason") + if isinstance(reason_field, str): + return reason_field.strip() + + return stripped + + +def _iter_log_envelopes(text: str): + """Yield ``(log_prefix, json_payload)`` for every + ``"[HH:MM:SS LEVEL logger] {...}"`` envelope in *text*. + + The SDK's root logger may emit several records per evaluation + (``"OpenAI API response received"``, ``"Before parse response with output + parser"``, ``"Before parse content with parser"``, ...). Some of them carry + the judge's verdict inside their JSON payload, others do not. The caller + picks the envelope that contains ``result`` + ``reason``. + """ + cursor = 0 + while cursor < len(text): + # Skip any whitespace / newlines separating envelopes. + while cursor < len(text) and text[cursor] in " \t\r\n": + cursor += 1 + if cursor >= len(text): + break + match = _LOG_PREFIX_RE.match(text, cursor) + if not match: + break + prefix_end = match.end() + # Find the balanced JSON object that begins at prefix_end. If the + # payload is not valid JSON (e.g. truncated) we still want to advance + # the cursor past the broken prefix so the loop can find the next + # envelope. + depth = 0 + payload_start = -1 + payload_end = -1 + in_string = False + escape = False + for idx in range(prefix_end, len(text)): + ch = text[idx] + if in_string: + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + continue + if ch == "{": + if depth == 0: + payload_start = idx + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0 and payload_start != -1: + payload_end = idx + 1 + break + if payload_start == -1 or payload_end == -1: + # No balanced JSON after this prefix — skip the prefix and keep + # scanning for the next log envelope. + cursor = prefix_end + continue + yield text[cursor:prefix_end], text[payload_start:payload_end] + cursor = payload_end + + +def _reason_from_json_envelope(payload: str) -> Optional[str]: + """Pull the judge's ``reason`` text out of one log-envelope JSON payload. + + Tries the following strategies, in order: + + 1. ``payload["response_content"]`` wrapped in a markdown code fence — the + shape produced by the judge LLM's first ``llm_call_end`` event. + 2. ``payload["response_content"]`` being a plain JSON string. + 3. ``payload["response"]["choices"][0]["message"]["content"]`` — the + OpenAI ``ChatCompletion`` repr captured by the request-side log. + 4. ``payload["reason"]`` at the top level — fallback. + """ + try: + data = json.loads(payload) + except (ValueError, TypeError): + return None + if not isinstance(data, dict): + return None + + response_content = data.get("response_content") + if isinstance(response_content, str): + fence_match = _MARKDOWN_FENCE_RE.search(response_content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + # ``response_content`` may be the raw JSON string itself. + try: + inner = json.loads(response_content) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + + # OpenAI ChatCompletion repr — unwrap the ``message.content`` field. + # The SDK captures the OpenAI response by calling ``repr(chat_completion)`` + # so the ``response`` field stores a Python repr (single-quoted strings, + # ``\n`` literal characters, ...) rather than a JSON document. Use a + # targeted regex to pull the ``content='...'`` blob out of the repr, then + # unwrap the markdown fence the judge model wraps its verdict in. + # The field can live either at the top level (``payload["response"]``) or + # nested under ``payload["metadata"]["response"]`` depending on which SDK + # code path emitted the log record. + response = data.get("response") + if response is None: + metadata = data.get("metadata") + if isinstance(metadata, dict): + response = metadata.get("response") + if isinstance(response, str) and "ChatCompletion" in response: + # Use a non-greedy match that ends at the next ``, refusal`` token. + # The repr is single-quoted and the ``content`` field is followed by + # ``, refusal=`` so this anchor avoids trying to balance quotes in + # the repr string — which is fragile once ``\n`` has been unescaped + # by ``json.loads``. + content_match = re.search( + r"ChatCompletionMessage\(content='(.*?)', refusal=", + response, + re.DOTALL, + ) + if content_match: + # The captured group is the raw repr body; turn the literal + # ``\n`` / ``\"`` / ``\\`` escape sequences back into real + # characters, then look for the markdown code fence inside. + try: + content = content_match.group(1).encode("utf-8").decode("unicode_escape") + except UnicodeDecodeError: + content = content_match.group(1) + if isinstance(content, str): + fence_match = _MARKDOWN_FENCE_RE.search(content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + # The judge verdict may live directly on ``response`` if the + # repr was stored without the ``metadata.response`` wrapping. + try: + inner = json.loads(content) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + elif isinstance(response, str): + # Some SDKs serialise the response as a plain JSON string. + try: + response_obj = json.loads(response) + except (ValueError, TypeError): + response_obj = None + if isinstance(response_obj, dict): + choices = response_obj.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message") + if isinstance(message, dict): + content = message.get("content") + if isinstance(content, str): + fence_match = _MARKDOWN_FENCE_RE.search(content) + if fence_match: + try: + inner = json.loads(fence_match.group(1)) + except (ValueError, TypeError): + inner = None + if isinstance(inner, dict): + reason = inner.get("reason") + if isinstance(reason, str): + return reason.strip() + + top_reason = data.get("reason") + if isinstance(top_reason, str): + return top_reason.strip() + + return None + + +def _extract_clean_reason_v2(raw: Any) -> str: + """Walk every log envelope in *raw* and pull out the judge reason. + + See ``_extract_clean_reason`` for the original entry point; this helper + handles the noisy multi-envelope shape where several SDK log records + (with the verdict buried in the first one) are concatenated. + """ + if raw is None: + return "" + text = str(raw).strip() + if not text: + return "" + + # Fast path: the string is already a single JSON object (no log envelope) + # containing the judge verdict. + standalone = _reason_from_json_envelope(text) + if standalone is not None: + return standalone + + # Slow path: walk every ``[time LEVEL logger] {...}`` envelope and take + # the first one whose payload yields a reason. + for _, payload in _iter_log_envelopes(text): + reason = _reason_from_json_envelope(payload) + if reason is not None: + return reason + + # No JSON envelope produced a reason. Return the input with leading log + # prefixes stripped so plain "pass" / "fail" and free-form explanations + # render without the surrounding noise. + stripped = text + while True: + match = _LOG_PREFIX_RE.match(stripped) + if not match: + break + stripped = stripped[match.end():].lstrip() + return stripped or text + + +def _is_llm_related_error(exc: Exception) -> bool: + """Check if an exception is related to LLM API calls.""" + error_str = str(exc).lower() + llm_keywords = [ + "openai", "api", "llm", "model", "completion", "chat", + "connection", "timeout", "rate limit", "authentication", + "invalid response", "async invoke", "jiuwen", "sdk", + "schedule new futures", "interpreter shutdown", + ] + return any(keyword in error_str for keyword in llm_keywords) + + +def _generate_friendly_error_message(exc: Exception, default_msg: str) -> str: + """Generate a friendly error message for LLM-related errors using another LLM.""" + if not _is_llm_related_error(exc): + return default_msg + + # Only call LLM for LLM-related errors + try: + import os + from openai import AsyncOpenAI + + client = AsyncOpenAI(api_key=os.environ.get("OPENAI_API_KEY")) + error_snippet = str(exc)[:500] + + response = asyncio.run( + client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + { + "role": "system", + "content": ( + "You are a helpful assistant that explains technical errors " + "to end users in simple Chinese. Be concise and actionable. " + "Focus on what the user can do to fix the problem." + ), + }, + { + "role": "user", + "content": ( + f"以下是智能体评估时的错误信息，请用简单的中文解释给用户，并给出建议的操作：\n\n{error_snippet}" + ), + }, + ], + max_tokens=200, + temperature=0.3, + ) + ) + return response.choices[0].message.content or default_msg + except Exception as llm_exc: + logger.warning("Failed to generate friendly error message: %r", llm_exc) + return default_msg + +logger = logging.getLogger("agent_evaluation_service") + + +def _make_background_done_callback( + tenant_id: str, + user_id: str, + agent_evaluation_id: int, +): + """Return a done-callback that marks the run FAILED if the worker raised.""" + + def callback(future): + exc = future.exception() + if exc is not None: + logger.exception( + "Background evaluation run failed (id=%s): %r", + agent_evaluation_id, + exc, + ) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + try: + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="FAILED", + updated_by=user_id, + error_message=friendly_msg, + ) + except Exception as update_exc: + logger.error( + "Failed to write FAILED status for evaluation id=%s: %r", + agent_evaluation_id, + update_exc, + ) + + return callback + + +def _build_case_for_jiuwen( + inputs: Dict[str, Any], + label: Dict[str, Any], +) -> Dict[str, Any]: + # Jiuwen Case(inputs, label) will accept arbitrary keys, but we keep the stable schema + return { + "inputs": { + "query": inputs.get("query", ""), + }, + "label": {"answer": label.get("answer", "")}, + } + + +async def _run_agent_to_final_answer( + agent_id: int, + tenant_id: str, + user_id: str, + query: str, + version_no: int, +) -> str: + """Run agent once and aggregate final answer text.""" + + # Build a single-turn AgentRequest. We do not persist messages for offline eval. + agent_request = AgentRequest( + query=query, + conversation_id=0, + history=None, + minio_files=None, + agent_id=agent_id, + version_no=version_no, + is_debug=True, + ) + + agent_run_info, memory_context = await prepare_agent_run( + agent_request=agent_request, + user_id=user_id, + tenant_id=tenant_id, + allow_memory_search=False, + ) + + # Stream chunks from core agent runner and extract final_answer content. + from nexent.core.agents.run_agent import agent_run + + final_answer_parts: List[str] = [] + async for chunk in agent_run(agent_run_info): + try: + if isinstance(chunk, str): + data = json.loads(chunk) + if isinstance(data, dict) and data.get("type") == "final_answer": + content = data.get("content") + if isinstance(content, str): + final_answer_parts.append(content) + except Exception: + continue + + return "".join(final_answer_parts).strip() + + +def create_agent_evaluation_run_impl( + tenant_id: str, + user_id: str, + agent_id: int, + evaluation_set_id: int, + judge_model_id: int, +) -> Dict[str, Any]: + set_cases = get_evaluation_set_cases_all(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + if not set_cases: + raise ValueError("evaluation set has no cases") + + agent_version_no = resolve_latest_published_version_no(agent_id=agent_id, tenant_id=tenant_id) + + run = create_agent_evaluation( + tenant_id=tenant_id, + agent_id=agent_id, + agent_version_no=agent_version_no, + evaluation_set_id=evaluation_set_id, + total=len(set_cases), + judge_model_id=judge_model_id, + created_by=user_id, + ) + + create_agent_evaluation_cases( + tenant_id=tenant_id, + agent_evaluation_id=run["agent_evaluation_id"], + set_cases=set_cases, + created_by=user_id, + ) + + # Kick off background execution — attach a callback so that any uncaught + # exception inside the worker thread is surfaced as a RUN→FAILED transition + # instead of silently leaving the run stuck at RUNNING forever. + future = pool.submit( + execute_agent_evaluation_run, + tenant_id, + user_id, + run["agent_evaluation_id"], + judge_model_id, + ) + future.add_done_callback( + _make_background_done_callback( + tenant_id, + user_id, + run["agent_evaluation_id"], + ) + ) + + return run + + +def execute_agent_evaluation_run( + tenant_id: str, + user_id: str, + agent_evaluation_id: int, + judge_model_id: Optional[int] = None, +): + """Background execution entry point (sync). + + ``judge_model_id`` is normally supplied by ``submit`` when the run is + first created. If the worker process restarts mid-run and the queued + payload is lost, we fall back to whatever was persisted on the run + record so the run can still recover. + """ + try: + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + ) + + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + agent_id = int(run["agent_id"]) + agent_version_no = int(run["agent_version_no"]) + if judge_model_id is None: + judge_model_id = run.get("judge_model_id") + if judge_model_id is None: + raise ValueError("judge_model_id is required but neither passed in nor persisted on the run") + judge_model_id = int(judge_model_id) + + if JiuwenSDKAdapter is None: + raise JiuwenSDKUnavailableError("Jiuwen SDK adapter is unavailable. Please install optional dependencies for openjiuwen.") + + adapter = JiuwenSDKAdapter(model_id=judge_model_id, tenant_id=tenant_id) + + cases = list_agent_evaluation_cases(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id, limit=100000, offset=0) + scores: List[float] = [] + + for idx, c in enumerate(cases, start=1): + case_id = c["agent_evaluation_case_id"] + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + ) + + inputs = c["inputs"] or {} + label = c["label"] or {} + + query = inputs.get("query", "") + + try: + answer_text = asyncio.run( + _run_agent_to_final_answer( + agent_id=agent_id, + tenant_id=tenant_id, + user_id=user_id, + query=query, + version_no=agent_version_no, + ) + ) + + predict = {"answer": answer_text} + + # Judge with openjiuwen LLM-as-judge metric (binary 1/0) + expected = (label.get("answer") or "").strip() + + score, reason = adapter.evaluate_semantic_consistency( + question=query, + expected_answer=expected, + model_answer=answer_text, + ) + + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="COMPLETED", + predict=predict, + score=score, + pass_status="pass" if score == 1 else "fail", + reason=reason, + updated_by=user_id, + ) + + scores.append(score) + + except Exception as exc: + logger.exception("Evaluation case failed: %r", exc) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + update_agent_evaluation_case_result( + agent_evaluation_case_id=case_id, + tenant_id=tenant_id, + status="FAILED", + pass_status="fail", + error_message=friendly_msg, + updated_by=user_id, + ) + + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="RUNNING", + updated_by=user_id, + progress_done=idx, + ) + + overall = float(mean(scores)) if scores else 0.0 + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="COMPLETED", + updated_by=user_id, + score_overall=overall, + progress_done=len(cases), + ) + + except Exception as exc: + logger.exception("Evaluation run failed: %r", exc) + friendly_msg = _generate_friendly_error_message(exc, str(exc)) + update_agent_evaluation_status( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + status="FAILED", + updated_by=user_id, + error_message=friendly_msg, + ) + + +def get_agent_evaluation_run_impl(agent_evaluation_id: int, tenant_id: str) -> Dict[str, Any]: + return get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + + +def list_agent_evaluations_by_agent_impl( + agent_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_agent_evaluations_by_agent(agent_id=agent_id, tenant_id=tenant_id, limit=limit, offset=offset) + + +def list_agent_evaluation_cases_impl( + agent_evaluation_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_agent_evaluation_cases(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id, limit=limit, offset=offset) + + +def delete_agent_evaluation_run_impl( + agent_evaluation_id: int, + tenant_id: str, + user_id: str, +) -> None: + """Soft-delete an evaluation run. + + Only the creator can delete the run. Tenant admins are handled at the + service layer when permissions are extended. + """ + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + if run.get("created_by") != user_id: + raise ValueError("Only the creator can delete this evaluation run") + soft_delete_agent_evaluation(agent_evaluation_id, tenant_id, user_id) + + +def generate_agent_evaluation_report_impl( + agent_evaluation_id: int, + tenant_id: str, +) -> Tuple[bytes, int]: + """Build the evaluation report workbook. + + Returns ``(excel_bytes, failed_count)``. ``failed_count`` lets the API + layer name the downloaded file ``_all.xlsx`` vs ``_failed.xlsx`` so a + clean run does not download a file whose name suggests failure. + """ + run = get_agent_evaluation(agent_evaluation_id=agent_evaluation_id, tenant_id=tenant_id) + all_cases = list_agent_evaluation_cases( + agent_evaluation_id=agent_evaluation_id, + tenant_id=tenant_id, + limit=100000, + offset=0, + ) + + failed_cases = [ + c for c in all_cases + if c.get("status") == "FAILED" + or c.get("score") == 0 + or c.get("pass_status") == "fail" + ] + pass_count = sum( + 1 for c in all_cases + if c.get("status") != "FAILED" and c.get("score") == 1 + ) + fail_count = len(failed_cases) + total = len(all_cases) + pass_rate = f"{(pass_count / total * 100):.2f}%" if total else "-" + + wb = Workbook() + ws_summary = wb.active + ws_summary.title = "概要" + + header_font = Font(bold=True) + header_fill = PatternFill(start_color="E8F4FD", end_color="E8F4FD", fill_type="solid") + center = Alignment(horizontal="center", vertical="center", wrap_text=True) + # Excel's default cell style is right-aligned for numbers; force every + # value cell to render flush-left so mixed types (numbers, percentages, + # timestamps, error messages) line up consistently in the report. + left_align = Alignment(horizontal="left", vertical="center", wrap_text=True) + + # The three "ID" rows (评估ID / Agent ID / Evaluation Set ID) expose the + # human-readable name (judge model / agent / evaluation set) instead of + # the opaque numeric ID. Falls back to the raw ID when the name lookup + # returned nothing, so the cell is never blank. + judge_model_label = run.get("judge_model_name") or run.get("judge_model_id") or "-" + agent_label = run.get("agent_name") or run.get("agent_id") or "-" + evaluation_set_label = run.get("evaluation_set_name") or run.get("evaluation_set_id") or "-" + + summary_rows = [ + ("测评模型", judge_model_label), + ("智能体名称", agent_label), + ("智能体版本", run.get("agent_version_no", "")), + ("评测集名称", evaluation_set_label), + ("状态", run.get("status", "")), + ("用例总数", total), + ("通过用例数", pass_count), + ("失败用例数", fail_count), + ("通过率", pass_rate), + ("综合得分", f"{run.get('score_overall', 0):.4f}" if run.get('score_overall') is not None else "-"), + ("错误信息", run.get("error_message") or "-"), + ("创建时间", run.get("create_time") or "-"), + ("报告范围", "失败用例"), + ] + + ws_summary.append(["字段", "值"]) + for cell in ws_summary[1]: + cell.font = header_font + cell.fill = header_fill + cell.alignment = center + + # Pre-build the data rows so we can apply the left alignment to every + # value cell in one pass (rows start at index 3 after the header in row 1 + # and the empty default row that ``append`` does not create; openpyxl + # places our header at row 1 and the data rows begin at row 2). + for field, value in summary_rows: + ws_summary.append([field, value]) + + for row_idx in range(2, 2 + len(summary_rows)): + ws_summary.cell(row=row_idx, column=1).alignment = left_align + ws_summary.cell(row=row_idx, column=2).alignment = left_align + + ws_summary.column_dimensions["A"].width = 24 + ws_summary.column_dimensions["B"].width = 60 + + ws_cases = wb.create_sheet("失败用例") + + # All columns in the failed-cases sheet are localized to Chinese to match + # the rest of the report. + case_headers = [ + "用例ID", + "问题", + "期望答案", + "模型答案", + "得分", + "评测理由", + "用例状态", + "错误信息", + ] + ws_cases.append(case_headers) + for cell in ws_cases[1]: + cell.font = header_font + cell.fill = header_fill + cell.alignment = center + + for c in failed_cases: + inputs = c.get("inputs") or {} + label = c.get("label") or {} + predict = c.get("predict") or {} + score = c.get("score") + if score == 0: + score_str = "0.0000" + elif isinstance(score, (int, float)): + score_str = f"{score:.4f}" + elif score is not None: + score_str = str(score) + else: + score_str = "-" + + # error_message is only meaningful when the case is FAILED + # (execution crashed before/during model call). For scored cases + # that simply scored 0 it is usually empty. + error_msg = c.get("error_message") or "" + + ws_cases.append([ + c.get("agent_evaluation_case_id", ""), + inputs.get("query", ""), + label.get("answer", ""), + predict.get("answer", ""), + score_str, + _extract_clean_reason_v2(c.get("reason")), + c.get("status", ""), + error_msg, + ]) + + # Apply left alignment + wrap to every data cell so the row stays readable + # when any column overflows the configured width. + if failed_cases: + last_data_row = 1 + len(failed_cases) + for row in ws_cases.iter_rows( + min_row=2, max_row=last_data_row, min_col=1, max_col=len(case_headers) + ): + for cell in row: + cell.alignment = Alignment(horizontal="left", vertical="top", wrap_text=True) + + widths = {"A": 14, "B": 50, "C": 40, "D": 40, "E": 10, "F": 50, "G": 12, "H": 40} + for col, w in widths.items(): + ws_cases.column_dimensions[col].width = w + + out = io.BytesIO() + wb.save(out) + return out.getvalue(), fail_count diff --git a/backend/services/agent_repository_service.py b/backend/services/agent_repository_service.py index 84099ae99..9ead66438 100644 --- a/backend/services/agent_repository_service.py +++ b/backend/services/agent_repository_service.py @@ -91,7 +91,6 @@ def _to_summary_item( "display_name": record.get("display_name"), "description": record.get("description"), "status": record.get("status"), - "category_id": record.get("category_id"), "tags": record.get("tags") or [], "tool_count": record.get("tool_count") or 0, "version_label": record.get("version_name"), @@ -119,13 +118,11 @@ def _matches_repository_listing_search_filter(record: dict, search: str) -> bool return True name = str(record.get("display_name") or record.get("name") or "").lower() description = str(record.get("description") or "").lower() - author = str(record.get("author") or "").lower() tags = record.get("tags") or [] tag_text = " ".join(str(tag).lower() for tag in tags if isinstance(tag, str)) return ( query in name or query in description - or query in author or query in tag_text ) @@ -135,7 +132,6 @@ def list_agent_repository_listings_impl( *, status: Optional[str] = None, agent_id: Optional[int] = None, - category_id: Optional[int] = None, page: int = 1, page_size: int = 10, search: Optional[str] = None, @@ -150,7 +146,6 @@ def list_agent_repository_listings_impl( publisher_tenant_id=tenant_id, status=status, agent_id=agent_id, - category_id=category_id, ) if search and search.strip(): records = [ @@ -225,10 +220,6 @@ def _validate_card_fields(repository_data: Dict[str, Any]) -> None: f"icon must be at most {_MAX_LISTING_ICON_LENGTH} characters" ) - category_id = repository_data.get("category_id") - if category_id is None or not isinstance(category_id, int): - raise ValueError("category_id is required and must be an integer") - tags = repository_data.get("tags") if tags is None: raise ValueError("tags is required for marketplace listing submission") @@ -728,7 +719,6 @@ def _to_list_item(record: Dict[str, Any]) -> Dict[str, Any]: "description": record.get("description"), "author": record.get("author"), "submitted_by": record.get("submitted_by"), - "category_id": record.get("category_id"), "tags": record.get("tags") or [], "tool_count": record.get("tool_count"), "version_label": record.get("version_name"), @@ -850,7 +840,7 @@ async def _build_repository_data_from_agent( } if card_fields: - for key in ("icon", "downloads", "category_id", "tool_count"): + for key in ("icon", "downloads", "tool_count"): if key in card_fields and card_fields[key] is not None: repository_data[key] = card_fields[key] if "tags" in card_fields and card_fields["tags"] is not None: @@ -873,7 +863,7 @@ async def create_agent_repository_listing_impl( then inserts or updates the marketplace table. When a listing for the same agent version already exists, its status is - updated to pending_review along with icon, tags, and category when provided. + updated to pending_review along with icon and tags when provided. """ if version_no < 0: raise ValueError("version_no must be >= 0") @@ -902,7 +892,7 @@ async def create_agent_repository_listing_impl( else: repository_id = int(existing["agent_repository_id"]) updates: Dict[str, Any] = {"status": STATUS_PENDING_REVIEW} - for key in ("icon", "tags", "category_id", "tool_count"): + for key in ("icon", "tags", "tool_count"): if key in repository_data: updates[key] = repository_data[key] affected = update_agent_repository_by_id( diff --git a/backend/services/agent_service.py b/backend/services/agent_service.py index 2ae91966f..f14c4995f 100644 --- a/backend/services/agent_service.py +++ b/backend/services/agent_service.py @@ -22,7 +22,8 @@ from services.agent_version_service import publish_version_impl from utils.prompt_template_utils import normalize_prompt_generate_template_content from consts.const import MEMORY_SEARCH_START_MSG, MEMORY_SEARCH_DONE_MSG, MEMORY_SEARCH_FAIL_MSG, TOOL_TYPE_MAPPING, \ - LANGUAGE, MESSAGE_ROLE, MODEL_CONFIG_MAPPING, CAN_EDIT_ALL_USER_ROLES, PERMISSION_PRIVATE, STREAM_STATUS_EVENT + LANGUAGE, MESSAGE_ROLE, MODEL_CONFIG_MAPPING, CAN_EDIT_ALL_USER_ROLES, PERMISSION_PRIVATE, STREAM_STATUS_EVENT, \ + DEFAULT_EN_TITLE, DEFAULT_ZH_TITLE, RUNTIME_CANCEL_POLL_INTERVAL_SECONDS from consts.exceptions import AppException, MemoryPreparationException, SkillDuplicateError from consts.error_code import ErrorCode from consts.agent_unavailable_reasons import AgentUnavailableReason @@ -61,7 +62,7 @@ clear_agent_new_mark ) from database import a2a_agent_db -from database.model_management_db import get_model_by_model_id, get_model_id_by_display_name +from database.model_management_db import get_model_by_model_id, get_model_by_model_id_ignore_delete, get_model_id_by_display_name, get_valid_model_ids from database.remote_mcp_db import get_mcp_server_by_name_and_tenant from database.tool_db import ( check_tool_is_available, @@ -88,6 +89,8 @@ ) from utils.str_utils import convert_list_to_string, convert_string_to_list from services.conversation_management_service import ( + create_new_conversation, + generate_conversation_title_service, get_latest_assistant_message, get_last_unit_for_message, save_conversation_user, @@ -96,6 +99,7 @@ save_source_image, save_source_search, save_skill_files_to_conversation, + update_conversation_agent_id_service, update_message_content, update_message_status, update_unit_content, @@ -103,6 +107,7 @@ ) from services.memory_config_service import build_memory_context from services.streaming_channel import streaming_channel_manager +from services.runtime_state_service import runtime_state_service from utils.auth_utils import get_current_user_info, get_user_language from utils.config_utils import tenant_config_manager from utils.memory_utils import build_memory_config @@ -129,6 +134,34 @@ async def _cleanup_channel_later(conversation_id: int, user_id: str, delay: floa await streaming_channel_manager.remove_channel(conversation_id, user_id) +async def _poll_runtime_cancel_signal(conversation_id: int, user_id: str, stop_event) -> None: + """Mirror Redis cancel signal into the local agent stop_event.""" + while not stop_event.is_set(): + if await runtime_state_service.is_cancelled_async(user_id=user_id, conversation_id=conversation_id): + stop_event.set() + logger.info( + "Runtime cancel signal received, user_id=%s, conversation_id=%s", + user_id, + conversation_id, + ) + return + await asyncio.sleep(RUNTIME_CANCEL_POLL_INTERVAL_SECONDS) + + +async def _cancel_task_on_runtime_signal(conversation_id: int, user_id: str, task: asyncio.Task) -> None: + """Cancel a local asyncio task when another Pod writes the runtime cancel signal.""" + while not task.done(): + if await runtime_state_service.is_cancelled_async(user_id=user_id, conversation_id=conversation_id): + task.cancel() + logger.info( + "Runtime cancel signal cancelled task, user_id=%s, conversation_id=%s", + user_id, + conversation_id, + ) + return + await asyncio.sleep(RUNTIME_CANCEL_POLL_INTERVAL_SECONDS) + + def _extract_json_objects_from_text(text: str) -> list[dict]: """Extract all JSON objects embedded in a text blob.""" if not text: @@ -930,6 +963,14 @@ async def _stream_agent_chunks( user_id=user_id ) + cancel_poll_task = asyncio.create_task( + _poll_runtime_cancel_signal( + conversation_id=agent_request.conversation_id, + user_id=user_id, + stop_event=agent_run_info.stop_event, + ) + ) + # In resume mode, emit a status event first if is_resume_mode: await channel.publish(STREAM_STATUS_EVENT) @@ -1192,7 +1233,8 @@ async def _stream_agent_chunks( except Exception: logger.exception("Failed to mark last unit as completed") - terminal_status = "completed" if stream_completed_normally else "failed" + was_stopped = getattr(agent_run_info, "stop_event", None) and agent_run_info.stop_event.is_set() + terminal_status = "stopped" if was_stopped else "completed" if stream_completed_normally else "failed" try: update_message_status( streaming_message_id, @@ -1202,12 +1244,17 @@ async def _stream_agent_chunks( except Exception: logger.exception("Failed to mark assistant message as %s", terminal_status) + if not cancel_poll_task.done(): + cancel_poll_task.cancel() + + was_stopped = getattr(agent_run_info, "stop_event", None) and agent_run_info.stop_event.is_set() + terminal_status = 'stopped' if was_stopped else 'completed' if stream_completed_normally else 'failed' + agent_run_manager.unregister_agent_run( - agent_request.conversation_id, user_id) + agent_request.conversation_id, user_id, status=terminal_status) # Mark channel as completed and schedule cleanup if channel is not None: - terminal_status = 'completed' if stream_completed_normally else 'failed' await streaming_channel_manager.complete_channel( conversation_id=agent_request.conversation_id, user_id=user_id, @@ -1373,6 +1420,22 @@ async def get_agent_info_impl(agent_id: int, tenant_id: str, version_no: int = 0 try: tool_info = search_tools_for_sub_agent( agent_id=agent_id, tenant_id=tenant_id) + # Check if selected_model_id in tool params points to a deleted model + for tool in tool_info: + unavailable_reasons: List[str] = [] + params = tool.get("params") or [] + if isinstance(params, list): + for param_def in params: + if not isinstance(param_def, dict): + continue + if param_def.get("name") == "selected_model_id": + selected_model_id = param_def.get("default") + if selected_model_id is not None: + model_record = get_model_by_model_id_ignore_delete(selected_model_id, tenant_id) + if model_record is not None and model_record.get("delete_flag") == "Y": + unavailable_reasons.append(AgentUnavailableReason.MCP_MODEL_UNAVAILABLE) + break + tool["unavailable_reasons"] = unavailable_reasons agent_info["tools"] = tool_info except Exception as e: logger.error(f"Failed to get agent tools: {str(e)}") @@ -1409,19 +1472,22 @@ async def get_agent_info_impl(agent_id: int, tenant_id: str, version_no: int = 0 agent_info["external_sub_agent_id_list"] = [] # Get model names from model_ids array - model_ids = agent_info.get("model_ids") + # Filter out deleted models (delete_flag='Y' in model_record_t) + model_ids = agent_info.get("model_ids") or [] + valid_model_ids = get_valid_model_ids(model_ids, tenant_id) + agent_info["model_ids"] = valid_model_ids + model_names: List[str] = [] - if model_ids and len(model_ids) > 0: - for mid in model_ids: - model_info = get_model_by_model_id(mid) - if model_info: - display_name = model_info.get("display_name") - if display_name: - model_names.append(display_name) + for mid in valid_model_ids: + model_info = get_model_by_model_id(mid) + if model_info: + display_name = model_info.get("display_name") + if display_name: + model_names.append(display_name) agent_info["model_names"] = model_names - # Always derive model_name from model_ids so the API contract is consistent. - if model_ids and len(model_ids) > 0: - first_model_info = get_model_by_model_id(model_ids[0]) + # Always derive model_name from valid_model_ids so the API contract is consistent. + if valid_model_ids: + first_model_info = get_model_by_model_id(valid_model_ids[0]) agent_info["model_name"] = first_model_info.get( "display_name", None) if first_model_info is not None else None else: @@ -2219,9 +2285,9 @@ async def import_agent_by_agent_id( enabled=True, params=tool.params)) # check the validity of the agent parameters - if import_agent_info.max_steps <= 0 or import_agent_info.max_steps > 30: + if import_agent_info.max_steps <= 0: raise ValueError( - f"Invalid max steps: {import_agent_info.max_steps}. max steps must be greater than 0 and less than 30.") + f"Invalid max steps: {import_agent_info.max_steps}. max steps must be greater than 0.") if not import_agent_info.name.isidentifier(): raise ValueError( f"Invalid agent name: {import_agent_info.name}. agent name must be a valid python variable name.") @@ -2380,6 +2446,11 @@ async def list_all_agent_info_impl(tenant_id: str, user_id: str) -> list[dict]: if not is_creator and (len(user_group_ids.intersection(agent_group_ids)) == 0 or ingroup_permission == PERMISSION_PRIVATE): continue + # Filter out deleted models (delete_flag='Y' in model_record_t) + raw_model_ids = agent.get("model_ids") or [] + valid_model_ids = get_valid_model_ids(raw_model_ids, tenant_id) + agent["model_ids"] = valid_model_ids + # Use shared availability check function _, unavailable_reasons = check_agent_availability( agent_id=agent["agent_id"], @@ -2576,6 +2647,22 @@ def check_agent_availability( if not all(tool_statuses): unavailable_reasons.append(AgentUnavailableReason.TOOL_UNAVAILABLE) + # Check if any tool has a selected_model_id pointing to a deleted model + for tool in tool_info: + params = tool.get("params") or [] + if isinstance(params, list): + for param_def in params: + if not isinstance(param_def, dict): + continue + if param_def.get("name") == "selected_model_id": + selected_model_id = param_def.get("default") + if selected_model_id is not None: + model_record = get_model_by_model_id_ignore_delete( + selected_model_id, tenant_id) + if model_record is not None and model_record.get("delete_flag") == "Y": + unavailable_reasons.append(AgentUnavailableReason.TOOL_UNAVAILABLE) + break + # Check model availability model_reasons = _collect_model_availability_reasons( agent=agent_info, @@ -2703,6 +2790,11 @@ async def generate_stream_with_memory( preprocess_manager.register_preprocess_task( task_id, conversation_id, current_task ) + cancel_poll_task = ( + asyncio.create_task(_cancel_task_on_runtime_signal(conversation_id, user_id, current_task)) + if current_task + else None + ) # Helper to emit memory_search token def _memory_token(message_text: str) -> str: @@ -2801,6 +2893,8 @@ def _memory_token(message_text: str) -> str: yield _safe_agent_stream_error_chunk() return finally: + if cancel_poll_task and not cancel_poll_task.done(): + cancel_poll_task.cancel() # Always unregister preprocess task preprocess_manager.unregister_preprocess_task(task_id) @@ -2932,6 +3026,43 @@ async def run_agent_stream( tenant_id=tenant_id, ) + # Auto-create conversation when conversation_id is not provided. + # Skip in debug mode: debug runs are ephemeral and must not persist + # conversations, titles, or messages to the user's history. + is_new_conversation = False + if agent_request.is_debug: + logger.info( + "Skipping conversation auto-create: is_debug=True (conversation_id=%s)", + agent_request.conversation_id, + ) + elif agent_request.conversation_id is None: + default_title = DEFAULT_EN_TITLE if language == LANGUAGE["EN"] else DEFAULT_ZH_TITLE + conversation_data = create_new_conversation( + title=default_title, + user_id=resolved_user_id, + agent_id=agent_request.agent_id, + ) + agent_request.conversation_id = conversation_data["conversation_id"] + is_new_conversation = True + logger.info( + "Auto-created conversation_id=%s for user=%s (new conversation)", + agent_request.conversation_id, + resolved_user_id, + ) + + if ( + not agent_request.is_debug + and not resume + and not is_new_conversation + and agent_request.conversation_id is not None + and agent_request.agent_id is not None + ): + update_conversation_agent_id_service( + conversation_id=agent_request.conversation_id, + agent_id=agent_request.agent_id, + user_id=resolved_user_id, + ) + # Resume mode: check for existing streaming message if resume: resume_info = _detect_resume_position( @@ -2954,8 +3085,13 @@ async def run_agent_stream( user_id=resolved_user_id, conversation_id=agent_request.conversation_id ) + run_state = await runtime_state_service.get_run_state_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + ) + is_remote_running = run_state.get("status") == "running" - if existing_run_info is None: + if existing_run_info is None and not is_remote_running: # Agent has finished while frontend was disconnected # Update message status to completed if it's still streaming try: @@ -2979,8 +3115,82 @@ async def run_agent_stream( conversation_id=agent_request.conversation_id, user_id=resolved_user_id ) + last_unit_index = resume_info["resume_from_unit_index"] - 1 + + def _resume_status_chunk(replay_chunk_count: int) -> str: + payload = { + 'status': 'resumed', + 'last_unit_index': last_unit_index, + 'replay_chunk_count': replay_chunk_count, + } + return f"data: {json.dumps(payload)}\n\n" + + def _resume_completed_chunk(status: str = "completed") -> str: + payload = { + 'status': status, + 'last_unit_index': last_unit_index, + } + return f"data: {json.dumps(payload)}\n\n" if channel is None: + if runtime_state_service.enabled and is_remote_running: + async def redis_channel_stream(): + replay_events = await runtime_state_service.read_stream_events_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + ) + replay_chunk_count = len(replay_events) + + yield STREAM_STATUS_EVENT + yield _resume_status_chunk(replay_chunk_count) + + last_event_id = "0-0" + for event_id, chunk in replay_events: + last_event_id = event_id + if chunk: + yield chunk + + while True: + events = await runtime_state_service.wait_for_stream_events_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + last_id=last_event_id, + ) + for event_id, chunk in events: + last_event_id = event_id + if chunk: + yield chunk + + stream_status = await runtime_state_service.get_stream_status_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + ) + latest_run_state = await runtime_state_service.get_run_state_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + ) + if stream_status.get("status") or latest_run_state.get("status") in { + "completed", + "failed", + "stopped", + }: + break + + terminal_status = stream_status.get("status") or latest_run_state.get("status") or "completed" + yield STREAM_STATUS_EVENT + yield _resume_completed_chunk(terminal_status) + + return StreamingResponse( + redis_channel_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Stream-Status": "resumed", + "X-Last-Unit-Index": str(resume_info['resume_from_unit_index']), + }, + ) + # No channel exists, agent might be in a different state return JSONResponse( status_code=HTTPStatus.OK, @@ -2997,7 +3207,7 @@ async def channel_stream(): # Emit status event first with chunk count for skip tracking yield STREAM_STATUS_EVENT - yield f'data: {{"status": "resumed", "last_unit_index": {resume_info["resume_from_unit_index"] - 1}, "replay_chunk_count": {replay_chunk_count}}}\n\n' + yield _resume_status_chunk(replay_chunk_count) # Use subscribe_with_history(0) to replay ALL chunks from the buffer # This ensures no chunks are lost even if frontend disconnected during streaming @@ -3007,7 +3217,7 @@ async def channel_stream(): # Mark as complete when channel ends yield STREAM_STATUS_EVENT - yield f'data: {{"status": "completed", "last_unit_index": {resume_info["resume_from_unit_index"] - 1}}}\n\n' + yield _resume_completed_chunk() return StreamingResponse( channel_stream(), @@ -3021,6 +3231,11 @@ async def channel_stream(): ) # Normal mode: start new stream + await runtime_state_service.reset_stream_async( + user_id=resolved_user_id, + conversation_id=agent_request.conversation_id, + ) + if not agent_request.is_debug and not skip_user_save: save_messages( agent_request, @@ -3078,6 +3293,10 @@ async def channel_stream(): async def stream_with_agent_context(): try: + # Emit conversation_created event for new conversations + if is_new_conversation: + yield f'data: {{"type": "conversation_created", "content": {{"conversation_id": {agent_request.conversation_id}}}}}\n\n' + with agent_monitoring_context(agent_metadata): async for data_chunk in stream_gen: yield data_chunk @@ -3088,6 +3307,23 @@ async def stream_with_agent_context(): exc_info=True, ) yield _safe_agent_stream_error_chunk() + finally: + # Auto-generate title for new conversations after stream completes + if is_new_conversation: + try: + await generate_conversation_title_service( + conversation_id=agent_request.conversation_id, + question=agent_request.query, + user_id=resolved_user_id, + tenant_id=resolved_tenant_id, + language=language, + ) + except Exception as title_exc: + logger.warning( + "Failed to auto-generate title for conversation_id=%s: %r", + agent_request.conversation_id, + title_exc, + ) return StreamingResponse( stream_with_agent_context(), diff --git a/backend/services/agent_version_service.py b/backend/services/agent_version_service.py index 0398ce3f5..1bc91d333 100644 --- a/backend/services/agent_version_service.py +++ b/backend/services/agent_version_service.py @@ -31,7 +31,7 @@ STATUS_DISABLED, STATUS_ARCHIVED, ) -from database.model_management_db import get_model_by_model_id +from database.model_management_db import get_model_by_model_id, get_valid_model_ids from utils.str_utils import convert_string_to_list from consts.agent_unavailable_reasons import AgentUnavailableReason @@ -899,6 +899,11 @@ async def list_published_agents_impl( # Add current version info agent_info['current_version_no'] = current_version_no + # Filter out deleted models (delete_flag='Y' in model_record_t) + raw_model_ids = agent_info.get("model_ids") or [] + valid_model_ids = get_valid_model_ids(raw_model_ids, tenant_id) + agent_info["model_ids"] = valid_model_ids + # Check agent availability using the shared function _, unavailable_reasons = check_agent_availability( agent_id=agent_id, diff --git a/backend/services/asset_owner_visibility.py b/backend/services/asset_owner_visibility.py index 24cb697b2..dae2f5553 100644 --- a/backend/services/asset_owner_visibility.py +++ b/backend/services/asset_owner_visibility.py @@ -17,6 +17,12 @@ ASSET_OWNER_RESOURCES_ROUTE = "/asset-owner-resources" +OWNER_MANAGE_ROUTE = "/owner-manage" + +_ASSET_OWNER_FEATURE_NAV_ROUTES = frozenset({ + ASSET_OWNER_RESOURCES_ROUTE, # legacy + OWNER_MANAGE_ROUTE, # current SU page +}) def is_asset_owner_enabled() -> bool: @@ -33,10 +39,10 @@ def require_asset_owner_enabled() -> None: def filter_accessible_routes_for_asset_owner_feature( accessible_routes: List[str], ) -> List[str]: - """Remove asset-owner nav route when the ASSET_OWNER feature flag is disabled.""" + """Remove asset-owner nav routes when the ASSET_OWNER feature flag is disabled.""" if ENABLE_ASSET_OWNER_ROLE: return accessible_routes - return [r for r in accessible_routes if r != ASSET_OWNER_RESOURCES_ROUTE] + return [r for r in accessible_routes if r not in _ASSET_OWNER_FEATURE_NAV_ROUTES] def can_view_skill(caller_tenant_id: Optional[str], skill_tenant_id: Optional[str]) -> bool: diff --git a/backend/services/conversation_management_service.py b/backend/services/conversation_management_service.py index 64482416c..772d8d31a 100644 --- a/backend/services/conversation_management_service.py +++ b/backend/services/conversation_management_service.py @@ -28,6 +28,7 @@ get_source_searches_by_conversation, get_source_searches_by_message, rename_conversation, + update_conversation_agent_id, update_conversation_message_content, update_conversation_message_status, update_message_minio_files, @@ -290,19 +291,20 @@ def update_conversation_title(conversation_id: int, title: str, user_id: str = N return success -def create_new_conversation(title: str, user_id: str) -> Dict[str, Any]: +def create_new_conversation(title: str, user_id: str, agent_id: Optional[int] = None) -> Dict[str, Any]: """ Create a new conversation Args: title: Conversation title user_id: User ID + agent_id: Agent used by the latest run in this conversation Returns: Dict containing conversation data """ try: - conversation_data = create_conversation(title, user_id) + conversation_data = create_conversation(title, user_id, agent_id=agent_id) return conversation_data except Exception as e: logging.error(f"Failed to create conversation: {str(e)}") @@ -324,6 +326,20 @@ def get_conversation_list_service(user_id: str) -> List[Dict[str, Any]]: raise Exception(str(e)) +def update_conversation_agent_id_service(conversation_id: int, agent_id: int, user_id: str) -> bool: + """ + Update the latest agent associated with a conversation. + """ + try: + success = update_conversation_agent_id(conversation_id, agent_id, user_id) + if not success: + raise Exception(f"Conversation {conversation_id} does not exist or has been deleted") + return True + except Exception as e: + logging.error(f"Failed to update conversation agent: {str(e)}") + raise Exception(str(e)) + + def rename_conversation_service(conversation_id: int, name: str, user_id: str) -> bool: """ Rename a conversation @@ -560,6 +576,7 @@ def get_conversation_history_service(conversation_id: int, user_id: str) -> List formatted_history = { # Convert to string 'conversation_id': str(history_data['conversation_id']), + 'agent_id': history_data.get('agent_id'), 'create_time': history_data['create_time'], 'message': messages } diff --git a/backend/services/evaluation_set_service.py b/backend/services/evaluation_set_service.py new file mode 100644 index 000000000..888ea4e47 --- /dev/null +++ b/backend/services/evaluation_set_service.py @@ -0,0 +1,193 @@ +import json +import logging +import uuid +from typing import Any, Dict, List, Optional, Tuple + +from consts.model import AgentRequest +from database.agent_version_db import query_version_list +from database.client import get_db_session +from database.db_models import AgentEvaluation +from database.evaluation_set_db import ( + create_evaluation_set, + get_evaluation_set, + get_evaluation_set_cases_all, + insert_evaluation_set_cases, + list_evaluation_set_cases, + list_evaluation_sets, + soft_delete_evaluation_set, + update_evaluation_set_case_count, +) + +logger = logging.getLogger("evaluation_set_service") + + +def _validate_single_turn_case(obj: Dict[str, Any]) -> Dict[str, Any]: + if not isinstance(obj, dict): + raise ValueError("case must be an object") + + inputs = obj.get("inputs") + label = obj.get("label") + + if not isinstance(inputs, dict): + raise ValueError("inputs must be an object") + if not isinstance(label, dict): + raise ValueError("label must be an object") + + query = inputs.get("query") + if not isinstance(query, str) or not query.strip(): + raise ValueError("inputs.query must be a non-empty string") + + context = inputs.get("context") + if context is not None and not isinstance(context, str): + raise ValueError("inputs.context must be a string when provided") + + answer = label.get("answer") + if not isinstance(answer, str) or not answer.strip(): + raise ValueError("label.answer must be a non-empty string") + + case_id = obj.get("case_id") + if case_id is not None and not isinstance(case_id, str): + raise ValueError("case_id must be a string when provided") + + return { + "case_id": case_id, + "inputs": {"query": query, **({"context": context} if context is not None else {})}, + "label": {"answer": answer}, + } + + +def parse_jsonl_cases(jsonl_text: str) -> List[Dict[str, Any]]: + cases: List[Dict[str, Any]] = [] + for idx, line in enumerate((jsonl_text or "").splitlines(), start=1): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except Exception as exc: + raise ValueError(f"Invalid JSON at line {idx}: {exc}") from exc + + normalized = _validate_single_turn_case(obj) + normalized["order_no"] = len(cases) + cases.append(normalized) + + if not cases: + raise ValueError("JSONL contains no cases") + + return cases + + +def create_evaluation_set_from_cases( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + cases: List[Dict[str, Any]], + created_by: Optional[str], +) -> Dict[str, Any]: + if not cases: + raise ValueError("cases is empty") + + meta = create_evaluation_set( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + created_by=created_by, + ) + + inserted = insert_evaluation_set_cases( + tenant_id=tenant_id, + evaluation_set_id=meta["evaluation_set_id"], + cases=cases, + created_by=created_by, + ) + + update_evaluation_set_case_count(meta["evaluation_set_id"], inserted, updated_by=created_by) + meta["case_count"] = inserted + return meta + + +def create_evaluation_set_from_jsonl( + tenant_id: str, + name: str, + description: Optional[str], + source_filename: Optional[str], + jsonl_text: str, + created_by: Optional[str], +) -> Dict[str, Any]: + cases = parse_jsonl_cases(jsonl_text) + return create_evaluation_set_from_cases( + tenant_id=tenant_id, + name=name, + description=description, + source_filename=source_filename, + cases=cases, + created_by=created_by, + ) + + +def list_evaluation_sets_impl(tenant_id: str, limit: int = 50, offset: int = 0) -> List[Dict[str, Any]]: + return list_evaluation_sets(tenant_id=tenant_id, limit=limit, offset=offset) + + +def get_evaluation_set_impl(evaluation_set_id: int, tenant_id: str) -> Dict[str, Any]: + return get_evaluation_set(evaluation_set_id=evaluation_set_id, tenant_id=tenant_id) + + +def list_evaluation_set_cases_impl( + evaluation_set_id: int, + tenant_id: str, + limit: int = 50, + offset: int = 0, +) -> List[Dict[str, Any]]: + return list_evaluation_set_cases( + evaluation_set_id=evaluation_set_id, + tenant_id=tenant_id, + limit=limit, + offset=offset, + ) + + +def resolve_latest_published_version_no(agent_id: int, tenant_id: str) -> int: + """Return latest published version_no for the agent. + + Raises ValueError if no published version exists. + """ + versions = query_version_list(agent_id, tenant_id) + if not versions: + raise ValueError("agent has no published versions") + # query_version_list returns latest first in existing code usage + latest = versions[0].get("version_no") + if latest is None: + raise ValueError("failed to resolve latest published version") + return int(latest) + + +def count_active_runs_using_set(evaluation_set_id: int, tenant_id: str) -> int: + """Return the number of active (non-soft-deleted) evaluation runs referencing the set.""" + with get_db_session() as session: + return session.query(AgentEvaluation).filter( + AgentEvaluation.evaluation_set_id == evaluation_set_id, + AgentEvaluation.tenant_id == tenant_id, + AgentEvaluation.delete_flag == "N", + ).count() + + +def delete_evaluation_set_impl( + evaluation_set_id: int, + tenant_id: str, + user_id: str, +) -> None: + """Soft-delete an evaluation set. + + Blocked when any active evaluation run still references the set, so historical + runs never lose their context. Use ``count_active_runs_using_set`` first if + the caller wants to surface the referenced count to the user before deletion. + """ + referenced = count_active_runs_using_set(evaluation_set_id, tenant_id) + if referenced > 0: + raise ValueError( + f"evaluation set is referenced by {referenced} evaluation run(s); cannot delete" + ) + soft_delete_evaluation_set(evaluation_set_id, tenant_id, user_id) diff --git a/backend/services/mcp_management_service.py b/backend/services/mcp_management_service.py index a62de250a..e67d265cc 100644 --- a/backend/services/mcp_management_service.py +++ b/backend/services/mcp_management_service.py @@ -1,29 +1,188 @@ import logging from datetime import datetime -from typing import Any, Dict, List +from typing import Any, Dict, FrozenSet, List, Tuple from urllib.parse import urlencode import aiohttp from consts.exceptions import ( MCPConnectionError, + McpNameConflictError, McpNotFoundError, McpValidationError, + UnauthorizedError, ) -from database.community_mcp_db import ( - create_mcp_community_record, - delete_mcp_community_record_by_id, - get_mcp_community_record_by_id_and_tenant, - get_mcp_community_records, - get_mcp_community_tag_stats, - list_mcp_community_records_by_tenant, - update_mcp_community_record_by_id, +from consts.mcp_market import ( + STATUS_NOT_SHARED, + STATUS_PENDING_REVIEW, + STATUS_REJECTED, + STATUS_SHARED, + VALID_MARKET_STATUSES, ) -from database.remote_mcp_db import get_mcp_record_by_id_and_tenant +from database.market_mcp_db import ( + check_mcp_market_name_exists, + create_mcp_market_record, + delete_mcp_market_record_by_id, + get_mcp_market_record_by_id, + get_mcp_market_records, + get_mcp_market_tag_stats_by_tenant, + increment_mcp_market_download_count, + list_mcp_market_records_by_status, + list_mcp_market_records_by_tenant_and_user, + update_mcp_market_record, + update_mcp_market_status, +) +from database.remote_mcp_db import ( + clear_mcp_record_market_id, + get_mcp_record_by_id_and_tenant, + update_mcp_record_market_id_by_id, +) +from database.user_tenant_db import get_user_tenant_by_user_id logger = logging.getLogger("mcp_management_service") MCP_REGISTRY_BASE_URL = "https://registry.modelcontextprotocol.io/v0.1/servers" +ADMIN_ROLES = {"ADMIN", "SUPER_ADMIN", "SU"} +SUPER_ADMIN_ROLES = {"SUPER_ADMIN", "SU"} + +# --------------------------------------------------------------------------- +# State machine transitions (following Agent Repository pattern) +# --------------------------------------------------------------------------- + +_SU_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_ADMIN_REVIEW_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), +}) + +_PUBLISHER_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), + (STATUS_PENDING_REVIEW, STATUS_NOT_SHARED), + (STATUS_REJECTED, STATUS_NOT_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_SUBMIT_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), +}) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _get_mcp_review_admin_scope(user_id: str, tenant_id: str) -> str | None: + """Return tenant scope for review queries. None means no scope (SU sees all).""" + user_tenant = get_user_tenant_by_user_id(user_id) + user_role = (user_tenant or {}).get("user_role", "").upper() + if user_role not in ADMIN_ROLES: + raise UnauthorizedError("Only administrators can review MCP submissions") + if user_role in SUPER_ADMIN_ROLES: + return None + return tenant_id + + +def _resolve_user_email(user_id: str) -> str | None: + """Resolve user_id to email for submitted_by tracking.""" + user_tenant = get_user_tenant_by_user_id(user_id) or {} + email = str(user_tenant.get("user_email") or "").strip() + return email or None + + +def _resolve_author_display_name(user_id: str | None) -> str | None: + if not user_id: + return None + return _resolve_user_email(user_id) + + +def _to_community_card(row: Dict[str, Any]) -> Dict[str, Any]: + raw_status = row.get("review_status") or "not_shared" + status_map: Dict[str, str] = { + STATUS_NOT_SHARED: "offline", + STATUS_PENDING_REVIEW: "pending", + STATUS_SHARED: "approved", + STATUS_REJECTED: "rejected", + } + return { + "communityId": row.get("market_id"), + "marketId": row.get("market_id"), + "reviewId": row.get("market_id"), + "sourceMcpId": row.get("source_mcp_id"), + "name": row.get("mcp_name"), + "description": row.get("description"), + "status": "active" if raw_status == STATUS_SHARED else "inactive", + "createdAt": row.get("create_time"), + "updatedAt": row.get("update_time"), + "source": "community", + "transportType": row.get("transport_type"), + "serverUrl": row.get("mcp_server"), + "configJson": row.get("config_json") if isinstance(row.get("config_json"), dict) else None, + "registryJson": row.get("registry_json") if isinstance(row.get("registry_json"), dict) else None, + "tags": row.get("tags") or [], + "reviewStatus": status_map.get(raw_status, raw_status), + "reviewType": "initial_listing", + "installCount": row.get("download_count") or 0, + "authorDisplayName": _resolve_author_display_name(row.get("user_id")), + } + + +def _get_user_role(user_id: str) -> str: + user_tenant = get_user_tenant_by_user_id(user_id) or {} + return str(user_tenant.get("user_role", "")).upper() + + +def _validate_market_status_transition( + *, + user_role: str, + current_status: str, + new_status: str, + record: Dict[str, Any], + user_id: str, + tenant_id: str, +) -> str | None: + """Validate role, ownership, and allowed status transition. + + Returns submitter email if this is a submit action (not_shared/pending_review + or rejected/pending_review), otherwise None. + """ + transition = (current_status, new_status) + + if user_role == "SU": + if transition not in _SU_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + return None + + if user_role in ("ADMIN", "DEV"): + if record.get("tenant_id") != tenant_id: + raise UnauthorizedError("Not authorized to update this marketplace listing") + if user_role == "DEV" and record.get("user_id") != user_id: + raise UnauthorizedError("Not authorized to update this marketplace listing") + + # ADMIN can also act as reviewer + if user_role == "ADMIN" and transition in _ADMIN_REVIEW_TRANSITIONS: + return None + + if transition not in _PUBLISHER_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + + if transition in _SUBMIT_TRANSITIONS: + return _resolve_user_email(user_id) + return None + + raise UnauthorizedError( + f"User role {user_role} not authorized to update marketplace listing" + ) # --------------------------------------------------------------------------- @@ -32,67 +191,31 @@ async def list_community_mcp_services( *, + tenant_id: str, search: str | None = None, tag: str | None = None, transport_type: str | None = None, cursor: str | None = None, limit: int = 30, ) -> Dict[str, Any]: - """List public community MCP services. - - Args: - search: Search keyword - tag: Filter by tag - transport_type: Filter by transport (url or container) - cursor: Pagination cursor - limit: Items per page - - Returns: - Dictionary with count, nextCursor, and items - """ - db_result = get_mcp_community_records( + """List shared (approved) community MCP services scoped to a tenant.""" + db_result = get_mcp_market_records( + tenant_id=tenant_id, search=search, tag=tag, transport_type=transport_type, cursor=cursor, limit=limit, ) - - raw_items = db_result.get("items", []) - items = [] - for item in raw_items: - items.append({ - "communityId": item.get("community_id"), - "name": item.get("mcp_name"), - "version": item.get("version"), - "description": item.get("description"), - "status": "active", - "createdAt": item.get("create_time"), - "updatedAt": item.get("update_time"), - "source": "community", - "transportType": item.get("transport_type"), - "serverUrl": item.get("mcp_server"), - "configJson": item.get("config_json") if isinstance(item.get("config_json"), dict) else None, - "registryJson": item.get("registry_json") if isinstance(item.get("registry_json"), dict) else None, - "tags": item.get("tags") or [], - }) return { - "count": len(items), + "count": db_result.get("count", 0), "nextCursor": db_result.get("nextCursor"), - "items": items, + "items": [_to_community_card(item) for item in db_result.get("items", [])], } -def list_community_mcp_tag_stats() -> List[Dict[str, Any]]: - """Get community MCP tag statistics. - - Args: - tenant_id: Tenant ID - - Returns: - List of tag statistics - """ - return get_mcp_community_tag_stats() +def list_community_mcp_tag_stats(tenant_id: str) -> List[Dict[str, Any]]: + return get_mcp_market_tag_stats_by_tenant(tenant_id=tenant_id) async def publish_community_mcp_service( @@ -102,187 +225,281 @@ async def publish_community_mcp_service( mcp_id: int, name: str | None = None, description: str | None = None, - version: str | None = None, tags: List[str] | None = None, mcp_server: str | None = None, config_json: Dict[str, Any] | None = None, ) -> int: - """Publish a local MCP service to the community. - - Optional ``name`` / ``description`` / ``version`` / ``tags`` / ``mcp_server`` / - ``config_json`` override the values copied from the local MCP row when creating - the community record. Omit an optional field (``None``) to keep the local MCP - value for that field. - - Args: - tenant_id: Tenant ID - user_id: User ID - mcp_id: MCP record ID to publish - name: Optional community display name override - description: Optional description override - version: Optional version override - tags: Optional tags override - mcp_server: Optional remote MCP URL override - config_json: Optional container config override - - Returns: - Community record ID - - Raises: - McpNotFoundError: If MCP record is not found + """Submit a local MCP service for review. + + Creates a market record with status=pending_review directly (no separate review table). + Returns the new market_id. """ source_record = get_mcp_record_by_id_and_tenant(mcp_id=mcp_id, tenant_id=tenant_id) if not source_record: raise McpNotFoundError("MCP record not found") - source_registry_json = source_record.get("registry_json") if isinstance(source_record.get("registry_json"), dict) else None - source_config_json = source_record.get("config_json") if isinstance(source_record.get("config_json"), dict) else None + source_registry_json = ( + source_record.get("registry_json") + if isinstance(source_record.get("registry_json"), dict) + else None + ) + source_config_json = ( + source_record.get("config_json") + if isinstance(source_record.get("config_json"), dict) + else None + ) final_name = name if name is not None else source_record.get("mcp_name") final_description = description if description is not None else source_record.get("description") - final_version = version if version is not None else source_record.get("version") final_tags = tags if tags is not None else source_record.get("tags") - final_mcp_server = ( - mcp_server if mcp_server is not None else source_record.get("mcp_server") - ) - final_config_json = ( - config_json if isinstance(config_json, dict) else source_config_json - ) + final_mcp_server = mcp_server if mcp_server is not None else source_record.get("mcp_server") + final_config_json = config_json if isinstance(config_json, dict) else source_config_json - # Remote MCP table may omit transport_type; community list still needs it for filters. community_transport_type = "container" if final_config_json is not None else "url" - community_id = create_mcp_community_record( + # Check name uniqueness among shared records only + if check_mcp_market_name_exists(final_name): + raise McpNameConflictError(f"MCP name '{final_name}' already exists in the community market") + + market_id = create_mcp_market_record( mcp_data={ "mcp_name": final_name, + "source_mcp_id": mcp_id, "mcp_server": final_mcp_server, - "version": final_version, "registry_json": source_registry_json, "transport_type": source_record.get("transport_type") or community_transport_type, "config_json": final_config_json, + "review_status": STATUS_PENDING_REVIEW, + "submitted_by": _resolve_user_email(user_id), "tags": final_tags, "description": final_description, }, tenant_id=tenant_id, user_id=user_id, ) - return community_id + return market_id async def update_community_mcp_service( *, tenant_id: str, user_id: str, - community_id: int, + market_id: int, name: str | None, description: str | None, tags: List[str] | None, - version: str | None, registry_json: Dict[str, Any] | None, + mcp_server: str | None = None, + config_json: Dict[str, Any] | None = None, + transport_type: str | None = None, ) -> None: - """Update a community MCP service. - - Args: - tenant_id: Tenant ID - user_id: User ID - community_id: Community record ID - name: New MCP service name - description: MCP service description - tags: MCP tags - version: MCP version - registry_json: Registry metadata JSON - - Raises: - McpNotFoundError: If community MCP record is not found - """ - current = get_mcp_community_record_by_id_and_tenant(community_id=community_id, tenant_id=tenant_id) + """Update a published market MCP and set it back to pending_review for re-approval.""" + current = get_mcp_market_record_by_id(market_id=market_id) if not current: - raise McpNotFoundError("Community MCP record not found") - - existing_config_json = current.get("config_json") if isinstance(current.get("config_json"), dict) else None - next_registry_json = registry_json if isinstance(registry_json, dict) else current.get("registry_json") - next_config_json = existing_config_json if isinstance(existing_config_json, dict) else None + raise McpNotFoundError("Market MCP record not found") - update_mcp_community_record_by_id( - community_id=community_id, - tenant_id=tenant_id, + existing_config_json = ( + current.get("config_json") if isinstance(current.get("config_json"), dict) else None + ) + next_registry_json = ( + registry_json if isinstance(registry_json, dict) else (current.get("registry_json") or {}) + ) + next_config_json = config_json if isinstance(config_json, dict) else existing_config_json + next_transport_type = transport_type + if next_transport_type is None and isinstance(config_json, dict): + next_transport_type = "container" + if next_transport_type is None and mcp_server is not None: + next_transport_type = "url" + + # Check name uniqueness if name is changing + if name is not None and name != current.get("mcp_name") and check_mcp_market_name_exists(name): + raise McpNameConflictError(f"MCP name '{name}' already exists in the community market") + + # Update fields + update_mcp_market_record( + market_id=market_id, user_id=user_id, - name=name, + mcp_name=name, description=description, tags=tags, - version=version, registry_json=next_registry_json, + mcp_server=mcp_server, config_json=next_config_json, + transport_type=next_transport_type, ) + # Set back to pending_review for re-approval + update_mcp_market_status( + market_id=market_id, + user_id=user_id, + review_status=STATUS_PENDING_REVIEW, + submitted_by=_resolve_user_email(user_id), + ) -async def delete_community_mcp_service( + +async def change_mcp_market_status( *, tenant_id: str, user_id: str, - community_id: int, + market_id: int, + new_status: str, ) -> None: - """Delete a community MCP service. - - Args: - tenant_id: Tenant ID - user_id: User ID - community_id: Community record ID + """Unified status change endpoint. Validates state machine transitions. - Raises: - McpNotFoundError: If community MCP record is not found + Supports all allowed transitions: submit, approve, reject, withdraw, unshare. """ - current = get_mcp_community_record_by_id_and_tenant(community_id=community_id, tenant_id=tenant_id) + if new_status not in VALID_MARKET_STATUSES: + raise ValueError( + f"Invalid status '{new_status}'; must be one of: " + f"{', '.join(sorted(VALID_MARKET_STATUSES))}" + ) + + current = get_mcp_market_record_by_id(market_id=market_id) if not current: - raise McpNotFoundError("Community MCP record not found") - delete_mcp_community_record_by_id( - community_id=community_id, + raise McpNotFoundError("Market MCP record not found") + + user_role = _get_user_role(user_id) + current_status = current.get("review_status", STATUS_NOT_SHARED) + + submitted_by = _validate_market_status_transition( + user_role=user_role, + current_status=current_status, + new_status=new_status, + record=current, + user_id=user_id, tenant_id=tenant_id, + ) + + update_mcp_market_status( + market_id=market_id, user_id=user_id, + review_status=new_status, + submitted_by=submitted_by, ) + # When approving for the first time, link the source MCP record + if new_status == STATUS_SHARED and current_status == STATUS_PENDING_REVIEW: + source_mcp_id = current.get("source_mcp_id") + if source_mcp_id is not None: + update_mcp_record_market_id_by_id( + mcp_id=source_mcp_id, + tenant_id=current.get("tenant_id"), + user_id=current.get("user_id"), + market_id=market_id, + ) -async def list_my_community_mcp_services( + +async def list_community_mcp_review_services( *, tenant_id: str, + user_id: str, + status: str | None = None, + search: str | None = None, + tag: str | None = None, + transport_type: str | None = None, + cursor: str | None = None, + limit: int = 30, ) -> Dict[str, Any]: - """List MCP services published by the current user to the community. + """List market records awaiting review (for the review center).""" + review_tenant_id = _get_mcp_review_admin_scope(user_id, tenant_id) + review_status = status or STATUS_PENDING_REVIEW + db_result = list_mcp_market_records_by_status( + tenant_id=review_tenant_id, + review_status=review_status, + search=search, + tag=tag, + transport_type=transport_type, + cursor=cursor, + limit=limit, + ) + return { + "count": db_result.get("count", 0), + "nextCursor": db_result.get("nextCursor"), + "items": [_to_community_card(item) for item in db_result.get("items", [])], + } - Args: - tenant_id: Tenant ID - Returns: - Dictionary with count and items - """ - rows = list_mcp_community_records_by_tenant(tenant_id=tenant_id) - items = [] - for row in rows: - items.append({ - "communityId": row.get("community_id"), - "name": row.get("mcp_name"), - "version": row.get("version"), - "description": row.get("description"), - "status": "active", - "createdAt": row.get("create_time"), - "updatedAt": row.get("update_time"), - "source": "community", - "transportType": row.get("transport_type"), - "serverUrl": row.get("mcp_server"), - "configJson": row.get("config_json") if isinstance(row.get("config_json"), dict) else None, - "registryJson": row.get("registry_json") if isinstance(row.get("registry_json"), dict) else None, - "tags": row.get("tags") or [], - }) +async def delete_community_mcp_service( + *, + tenant_id: str, + user_id: str, + market_id: int, +) -> None: + """Soft-delete a market MCP service and clear FK references.""" + current = get_mcp_market_record_by_id(market_id=market_id) + if not current: + raise McpNotFoundError("Market MCP record not found") + + delete_mcp_market_record_by_id(market_id=market_id, user_id=user_id) + clear_mcp_record_market_id( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + ) + + +async def list_my_community_mcp_services( + *, + tenant_id: str, + user_id: str, +) -> Dict[str, Any]: + """List MCP services (all statuses) published by the current user.""" + market_rows = list_mcp_market_records_by_tenant_and_user( + tenant_id=tenant_id, + user_id=user_id, + ) + items = [_to_community_card(row) for row in market_rows] return { "count": len(items), "items": items, } +# --------------------------------------------------------------------------- +# Legacy convenience wrappers (for backward compat / internal use) +# --------------------------------------------------------------------------- + +async def approve_community_mcp_service( + *, + tenant_id: str, + user_id: str, + market_id: int, +) -> None: + """Approve: pending_review -> shared.""" + user_role = _get_user_role(user_id) + if user_role not in ADMIN_ROLES: + raise UnauthorizedError("Only administrators can approve MCP submissions") + await change_mcp_market_status( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + new_status=STATUS_SHARED, + ) + + +async def reject_community_mcp_service( + *, + tenant_id: str, + user_id: str, + market_id: int, +) -> None: + """Reject: pending_review -> rejected.""" + user_role = _get_user_role(user_id) + if user_role not in ADMIN_ROLES: + raise UnauthorizedError("Only administrators can reject MCP submissions") + await change_mcp_market_status( + tenant_id=tenant_id, + user_id=user_id, + market_id=market_id, + new_status=STATUS_REJECTED, + ) + + # --------------------------------------------------------------------------- # Registry Functions # --------------------------------------------------------------------------- -async def list_registry_mcp_services( +async def _list_official_registry_mcp_services( *, search: str | None = None, include_deleted: bool = False, @@ -291,19 +508,7 @@ async def list_registry_mcp_services( cursor: str | None = None, limit: int = 30, ) -> Dict[str, Any]: - """List MCP services from the official MCP Registry. - - Args: - search: Search keyword - include_deleted: Include deleted records - updated_since: Filter by update time - version: Filter by version - cursor: Pagination cursor - limit: Items per page - - Returns: - Dictionary with servers and metadata - """ + """List MCP services from the official MCP Registry.""" params: Dict[str, Any] = {"limit": limit} if search: params["search"] = search @@ -332,3 +537,23 @@ async def list_registry_mcp_services( "servers": raw_servers if isinstance(raw_servers, list) else [], "metadata": metadata, } + + +async def list_registry_mcp_services( + *, + search: str | None = None, + include_deleted: bool = False, + updated_since: str | None = None, + version: str | None = None, + cursor: str | None = None, + limit: int = 30, +) -> Dict[str, Any]: + """List MCP services from the official registry.""" + return await _list_official_registry_mcp_services( + search=search, + include_deleted=include_deleted, + updated_since=updated_since, + version=version, + cursor=cursor, + limit=limit, + ) diff --git a/backend/services/memory_config_service.py b/backend/services/memory_config_service.py index 9841caf8d..40cdc5b0e 100644 --- a/backend/services/memory_config_service.py +++ b/backend/services/memory_config_service.py @@ -94,6 +94,17 @@ def _update_single_config(user_id: str, config_key: str, config_value: str) -> b f"update_config_by_id failed, user_id={user_id}, key={config_key}, value={config_value}" ) return False + + # Handle duplicate records: delete all except the first one + if len(record_list) > 1: + logger.warning( + f"Found {len(record_list)} duplicate records for user_id={user_id}, key={config_key}. Cleaning up..." + ) + for duplicate in record_list[1:]: + dup_id = duplicate["config_id"] + delete_result = delete_config_by_config_id(dup_id, updated_by=user_id) + if not delete_result: + logger.error(f"Failed to delete duplicate config_id={dup_id}") return True diff --git a/backend/services/northbound_service.py b/backend/services/northbound_service.py index 4589eb4ba..46c60eae8 100644 --- a/backend/services/northbound_service.py +++ b/backend/services/northbound_service.py @@ -11,7 +11,12 @@ from fastapi.responses import StreamingResponse -from consts.const import ASSET_OWNER_TENANT_ID +from consts.const import ( + ASSET_OWNER_TENANT_ID, + NORTHBOUND_IDEMPOTENCY_TTL_SECONDS, + NORTHBOUND_RATE_LIMIT_ENABLED, + NORTHBOUND_RATE_LIMIT_PER_MINUTE, +) from consts.exceptions import ( LimitExceededError, UnauthorizedError, @@ -25,6 +30,7 @@ stop_agent_tasks, get_agent_id_by_name ) +from services.runtime_state_service import runtime_state_service from services.agent_version_service import list_published_agents_impl from services.conversation_management_service import ( save_conversation_user, @@ -234,10 +240,8 @@ def _normalize_northbound_attachments( # In-memory idempotency and rate limit placeholders # ----------------------------- _IDEMPOTENCY_RUNNING: Dict[str, float] = {} -_IDEMPOTENCY_TTL_SECONDS_DEFAULT = 10 * 60 _IDEMPOTENCY_LOCK = asyncio.Lock() -_RATE_LIMIT_PER_MINUTE = 120 # simple default quota per tenant per minute _RATE_STATE: Dict[str, Dict[str, int]] = {} _RATE_LOCK = asyncio.Lock() @@ -252,10 +256,21 @@ def _minute_bucket(ts: Optional[float] = None) -> str: async def idempotency_start(key: str, ttl_seconds: Optional[int] = None) -> None: + ttl = ttl_seconds or NORTHBOUND_IDEMPOTENCY_TTL_SECONDS + if runtime_state_service.enabled: + try: + acquired = await runtime_state_service.acquire_idempotency_async(key, ttl) + except Exception: + logger.exception("Northbound idempotency Redis operation failed") + raise LimitExceededError("Idempotency service is unavailable. Please try again later.") + if not acquired: + raise LimitExceededError("Duplicate request is still running, please wait.") + return + async with _IDEMPOTENCY_LOCK: # purge expired now = _now_seconds() - expired = [k for k, v in _IDEMPOTENCY_RUNNING.items() if now - v > (ttl_seconds or _IDEMPOTENCY_TTL_SECONDS_DEFAULT)] + expired = [k for k, v in _IDEMPOTENCY_RUNNING.items() if now - v > ttl] for k in expired: _IDEMPOTENCY_RUNNING.pop(k, None) if key in _IDEMPOTENCY_RUNNING: @@ -264,6 +279,13 @@ async def idempotency_start(key: str, ttl_seconds: Optional[int] = None) -> None async def idempotency_end(key: str) -> None: + if runtime_state_service.enabled: + try: + await runtime_state_service.release_idempotency_async(key) + except Exception as exc: + logger.warning("Northbound idempotency release failed: %s", exc) + return + async with _IDEMPOTENCY_LOCK: _IDEMPOTENCY_RUNNING.pop(key, None) @@ -274,11 +296,27 @@ async def _release_idempotency_after_delay(key: str, seconds: int = 3) -> None: async def check_and_consume_rate_limit(tenant_id: str) -> None: + if not NORTHBOUND_RATE_LIMIT_ENABLED: + return + + if runtime_state_service.enabled: + try: + await runtime_state_service.consume_rate_limit_async( + tenant_id=tenant_id, + limit_per_minute=NORTHBOUND_RATE_LIMIT_PER_MINUTE, + ) + return + except ValueError: + raise LimitExceededError("Query rate exceeded limit. Please try again later") + except Exception: + logger.exception("Northbound rate limit Redis operation failed") + raise LimitExceededError("Rate limit service is unavailable. Please try again later.") + bucket = _minute_bucket() async with _RATE_LOCK: state = _RATE_STATE.setdefault(tenant_id, {}) count = state.get(bucket, 0) - if count >= _RATE_LIMIT_PER_MINUTE: + if count >= NORTHBOUND_RATE_LIMIT_PER_MINUTE: raise LimitExceededError("Query rate exceeded limit. Please try again later") state[bucket] = count + 1 # cleanup old buckets, keep only current @@ -371,8 +409,8 @@ async def start_streaming_chat( except Exception as e: raise Exception(f"Failed to persist user message: {str(e)}") - except LimitExceededError as _: - raise LimitExceededError("Query rate exceeded limit. Please try again later.") + except LimitExceededError as exc: + raise LimitExceededError(str(exc)) except UnauthorizedError as _: raise UnauthorizedError("Cannot authenticate.") except Exception as e: diff --git a/backend/services/oauth_service.py b/backend/services/oauth_service.py index fe2aa0c42..31a70273e 100644 --- a/backend/services/oauth_service.py +++ b/backend/services/oauth_service.py @@ -17,6 +17,7 @@ ASSET_OWNER_TENANT_ID, DEFAULT_TENANT_ID, OAUTH_CALLBACK_BASE_URL, + OAUTH_LOGIN_MODE, OAUTH_SSL_VERIFY, OAUTH_CA_BUNDLE, SUPABASE_JWT_SECRET, @@ -88,6 +89,12 @@ def get_supported_providers() -> set: return set(get_all_provider_definitions().keys()) +def _get_oauth_login_mode() -> str: + if OAUTH_LOGIN_MODE in {"button", "force", "disabled"}: + return OAUTH_LOGIN_MODE + return "disabled" + + def get_enabled_providers() -> List[Dict[str, str]]: providers = [] for name, definition in get_all_provider_definitions().items(): @@ -103,6 +110,31 @@ def get_enabled_providers() -> List[Dict[str, str]]: return providers +def get_oauth_config() -> Dict[str, Any]: + mode = _get_oauth_login_mode() + providers = get_enabled_providers() + auto_login_provider = None + + if mode == "force": + if len(providers) == 1: + auto_login_provider = providers[0]["name"] + else: + logger.warning( + "OAuth auto login requires exactly one enabled provider; found %s", + len(providers), + ) + + if not providers: + mode = "disabled" + + return { + "enabled": bool(providers), + "login_mode": mode, + "auto_login_provider": auto_login_provider, + "providers": providers, + } + + def get_authorize_url(provider: str, link_user_id: str = "") -> str: try: definition = get_provider_definition(provider) diff --git a/backend/services/ragflow_service.py b/backend/services/ragflow_service.py new file mode 100644 index 000000000..335bcd7f4 --- /dev/null +++ b/backend/services/ragflow_service.py @@ -0,0 +1,124 @@ +import json +import logging +from typing import Any, Dict + +import httpx + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from nexent.utils.http_client_manager import http_client_manager + +logger = logging.getLogger("ragflow_service") + + +def _validate_ragflow_config(ragflow_api_base: str, api_key: str) -> None: + """Validate RAGFlow API configuration parameters. + + Raises AppException if any parameter is invalid. + """ + if not ragflow_api_base or not isinstance(ragflow_api_base, str): + raise AppException( + ErrorCode.RAGFLOW_CONFIG_INVALID, + "RAGFlow API URL is required and must be a non-empty string" + ) + + if not api_key or not isinstance(api_key, str): + raise AppException( + ErrorCode.RAGFLOW_CONFIG_INVALID, + "RAGFlow API key is required and must be a non-empty string" + ) + + +def _format_dataset_item(ds: Dict[str, Any]) -> Dict[str, Any]: + """Format a single RAGFlow dataset into the standard response shape.""" + return { + "id": str(ds.get("id", "")), + "name": ds.get("name", ""), + "description": ds.get("description", ""), + "doc_count": ds.get("doc_num", 0) or ds.get("document_count", 0), + "chunk_count": ds.get("chunk_num", 0) or ds.get("chunk_count", 0), + "create_time": str(ds.get("create_time", "")) or str(ds.get("create_date", "")), + "update_time": str(ds.get("update_time", "")) or str(ds.get("update_date", "")), + } + + +def fetch_ragflow_datasets_impl( + ragflow_api_base: str, + api_key: str, +) -> Dict[str, Any]: + """ + Fetch datasets from RAGFlow API. + + Args: + ragflow_api_base: RAGFlow API base URL (e.g., 'http://localhost:9380') + api_key: RAGFlow API key + + Returns: + Dictionary containing datasets: + { + "data": [ + { + "id": "dataset_id", + "name": "Dataset Name", + "description": "...", + "doc_count": 10, + "chunk_count": 100, + "create_time": "...", + "update_time": "...", + } + ] + } + """ + _validate_ragflow_config(ragflow_api_base, api_key) + + api_base = ragflow_api_base.rstrip("/") + url = f"{api_base}/api/v1/datasets" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + logger.info(f"Fetching RAGFlow datasets from: {url}") + + try: + client = http_client_manager.get_sync_client( + base_url=api_base, + timeout=10.0, + verify_ssl=True + ) + response = client.get(url, headers=headers) + response.raise_for_status() + + result = response.json() + + if result.get("code") != 0: + raise AppException( + ErrorCode.RAGFLOW_SERVICE_ERROR, + f"RAGFlow API returned error code {result.get('code')}: " + f"{result.get('message', 'Unknown error')}" + ) + + datasets = result.get("data", []) + return {"data": [_format_dataset_item(ds) for ds in datasets]} + + except httpx.RequestError as e: + logger.exception("RAGFlow API request failed") + raise AppException(ErrorCode.RAGFLOW_CONNECTION_ERROR, + f"RAGFlow API request failed: {str(e)}") + except httpx.HTTPStatusError as e: + logger.exception( + f"RAGFlow API HTTP error: status_code={e.response.status_code}" + ) + if e.response.status_code == 401: + raise AppException(ErrorCode.RAGFLOW_AUTH_ERROR, + f"RAGFlow authentication failed: {str(e)}") + if e.response.status_code == 403: + raise AppException(ErrorCode.RAGFLOW_AUTH_ERROR, + f"RAGFlow access forbidden: {str(e)}") + raise AppException(ErrorCode.RAGFLOW_SERVICE_ERROR, + f"RAGFlow API HTTP error {e.response.status_code}: {str(e)}") + except json.JSONDecodeError as e: + logger.exception("Failed to parse RAGFlow API response") + raise AppException(ErrorCode.RAGFLOW_RESPONSE_ERROR, + f"Failed to parse RAGFlow API response: {str(e)}") diff --git a/backend/services/remote_mcp_service.py b/backend/services/remote_mcp_service.py index 7e77a9c43..be67e9826 100644 --- a/backend/services/remote_mcp_service.py +++ b/backend/services/remote_mcp_service.py @@ -29,6 +29,7 @@ update_mcp_record_enabled_by_id, update_mcp_record_container_fields_by_id, update_mcp_record_status_by_id, + update_mcp_record_registry_json_by_id, delete_mcp_record_by_id, get_mcp_authorization_token_by_name_and_url, get_mcp_record_by_id_and_tenant, @@ -46,15 +47,30 @@ # --------------------------------------------------------------------------- async def mcp_server_health(remote_mcp_server: str, authorization_token: str | None = None, custom_headers: dict | None = None) -> bool: - """Check if an MCP server is healthy and reachable.""" - try: - url_stripped = remote_mcp_server.strip() - headers = {} - if authorization_token: - headers["Authorization"] = authorization_token - if custom_headers: - headers.update(custom_headers) + """Check if an MCP server is healthy and reachable via MCP protocol. + + Returns True if the server is reachable and responds to tool listing. + Raises MCPConnectionError if the server is unreachable or does not support MCP. + """ + url_stripped = remote_mcp_server.strip() + headers = {} + if authorization_token: + headers["Authorization"] = authorization_token + if custom_headers: + headers.update(custom_headers) + + tool_names = await _mcp_protocol_health_check(url_stripped, headers) + if not tool_names: + raise MCPConnectionError("MCP server is unreachable or does not support MCP protocol") + return True + + +async def _mcp_protocol_health_check(url_stripped: str, headers: dict) -> list[str]: + """Try to establish an MCP protocol-level connection and return tool names. + Returns a list of tool names on success, or an empty list on failure. + """ + try: if url_stripped.endswith("/sse"): transport = SSETransport( url=url_stripped, @@ -68,7 +84,6 @@ async def mcp_server_health(remote_mcp_server: str, authorization_token: str | N httpx_client_factory=create_httpx_client ) else: - # Default to StreamableHttpTransport for unrecognized formats transport = StreamableHttpTransport( url=url_stripped, headers=headers, @@ -77,14 +92,73 @@ async def mcp_server_health(remote_mcp_server: str, authorization_token: str | N client = Client(transport=transport) async with client: - connected = client.is_connected() - return connected + # Verify the server can actually serve tools. + # This exercises API key validation and end-to-end connectivity, + # unlike is_connected() which only checks the initialize handshake. + tools_result = await asyncio.wait_for(client.list_tools(), timeout=10) + return [t.name for t in tools_result] if tools_result else [] except BaseException as e: - logger.error(f"Remote MCP server health check failed: {e}", exc_info=True) - error_message = str(e).strip() or repr(e) - if isinstance(e, (asyncio.TimeoutError, TimeoutError)) or "timeout" in error_message.lower(): - raise MCPConnectionError("MCP_HEALTH_TIMEOUT") - raise MCPConnectionError(error_message) + logger.debug(f"MCP protocol health check failed: {e}") + return [] + + +async def _mcp_protocol_connect(url_stripped: str, headers: dict) -> bool: + """Lightweight MCP connectivity check: establish an MCP initialize handshake only. + + Uses fastmcp.Client in an async context manager. The ``async with client:`` + block performs the MCP initialize handshake. After that, + ``client.is_connected()`` returns True if the handshake succeeded. + + This is significantly faster than _mcp_protocol_health_check() which + additionally calls list_tools(). + """ + try: + if url_stripped.endswith("/sse"): + transport = SSETransport( + url=url_stripped, + headers=headers, + httpx_client_factory=create_httpx_client, + ) + elif url_stripped.endswith("/mcp"): + transport = StreamableHttpTransport( + url=url_stripped, + headers=headers, + httpx_client_factory=create_httpx_client, + ) + else: + transport = StreamableHttpTransport( + url=url_stripped, + headers=headers, + httpx_client_factory=create_httpx_client, + ) + + client = Client(transport=transport) + async with client: + return client.is_connected() + except Exception as e: + logger.debug(f"MCP protocol connect handshake failed: {e}") + return False + + +async def test_mcp_connection( + server_url: str, + authorization_token: str | None = None, + custom_headers: dict | None = None, +) -> bool: + """Test connectivity to an MCP server using a lightweight initialize handshake. + + Returns True if the MCP initialize handshake succeeded, False otherwise. + Does NOT call list_tools(), making it faster and lighter than + mcp_server_health(). + """ + url_stripped = server_url.strip() + headers = {} + if authorization_token: + headers["Authorization"] = authorization_token + if custom_headers: + headers.update(custom_headers) + + return await _mcp_protocol_connect(url_stripped, headers) # --------------------------------------------------------------------------- @@ -204,7 +278,14 @@ async def add_remote_mcp_server_list( logger.error(f"MCP name already exists: {remote_mcp_server_name}") raise MCPNameIllegal("MCP name already exists") - if not await mcp_server_health(remote_mcp_server=remote_mcp_server, authorization_token=authorization_token, custom_headers=custom_headers): + headers = {} + if authorization_token: + headers["Authorization"] = authorization_token + if custom_headers: + headers.update(custom_headers) + + tool_names = await _mcp_protocol_health_check(remote_mcp_server.strip(), headers) + if not tool_names: raise MCPConnectionError("MCP connection failed") insert_mcp_data = { @@ -216,10 +297,42 @@ async def add_remote_mcp_server_list( "custom_headers": custom_headers, "source": source, "container_port": container_port, + "registry_json": {"_toolNames": tool_names}, } create_mcp_record(mcp_data=insert_mcp_data, tenant_id=tenant_id, user_id=user_id) +def _build_mcp_headers( + authorization_token: str | None, + custom_headers: dict | None, +) -> dict: + headers = {} + if authorization_token: + headers["Authorization"] = authorization_token + if custom_headers: + headers.update(custom_headers) + return headers + + +async def _check_mcp_connectivity( + server_url: str, + headers: dict, + is_container: bool, + name: str, +) -> list[str] | None: + tool_names = await _mcp_protocol_health_check(server_url.strip(), headers) + if not tool_names: + if is_container: + logger.warning( + "Container MCP service %s is not reachable yet, " + "tool count will be unavailable until the next refresh", + name, + ) + return None + raise MCPConnectionError("MCP server is unreachable or does not support MCP protocol") + return tool_names + + async def add_mcp_service( *, tenant_id: str, @@ -233,6 +346,8 @@ async def add_mcp_service( custom_headers: dict | None = None, container_config: dict | None, registry_json: dict | None, + config_json: dict | None = None, + market_id: int | None = None, enabled: bool = False, container_id: str | None = None, container_port: int | None = None, @@ -251,6 +366,8 @@ async def add_mcp_service( custom_headers: Custom HTTP headers container_config: Container configuration registry_json: Registry metadata JSON + config_json: MCP configuration JSON (e.g. OpenAPI spec for API-type MCP) + market_id: Linked market record ID enabled: Whether the MCP is enabled container_id: Docker container ID container_port: Container port @@ -258,16 +375,20 @@ async def add_mcp_service( status: bool | None = None normalized_container_id = container_id if isinstance(container_id, str) and container_id else None is_container = container_id is not None or container_config is not None - config_json = container_config if is_container and isinstance(container_config, dict) else None + resolved_config_json = container_config if is_container and isinstance(container_config, dict) else config_json - if enabled: - if check_mcp_name_exists(mcp_name=name, tenant_id=tenant_id): - logger.error(f"MCP name already exists: {name}") - raise MCPNameIllegal("MCP name already exists") + if check_mcp_name_exists(mcp_name=name, tenant_id=tenant_id): + logger.error(f"MCP name already exists: {name}") + raise MCPNameIllegal("MCP name already exists") - if not await mcp_server_health(remote_mcp_server=server_url, authorization_token=authorization_token, custom_headers=custom_headers): - raise MCPConnectionError("MCP connection failed") + resolved_registry_json = registry_json or {} + if server_url: + headers = _build_mcp_headers(authorization_token, custom_headers) + tool_names = await _check_mcp_connectivity(server_url, headers, is_container, name) + if tool_names: + resolved_registry_json["_toolNames"] = tool_names + if enabled: status = True create_mcp_record( @@ -280,11 +401,12 @@ async def add_mcp_service( "authorization_token": authorization_token, "custom_headers": custom_headers, "source": source, - "registry_json": registry_json, + "registry_json": resolved_registry_json, + "market_id": market_id, "enabled": enabled, "tags": tags, "description": description, - "config_json": config_json, + "config_json": resolved_config_json, }, tenant_id=tenant_id, user_id=user_id, @@ -301,6 +423,8 @@ async def add_container_mcp_service( tags: list | None, authorization_token: str | None, registry_json: dict | None, + version: str | None, + market_id: int | None, port: int, mcp_config: MCPConfigRequest, ) -> dict: @@ -315,6 +439,8 @@ async def add_container_mcp_service( tags: MCP tags authorization_token: Authorization token registry_json: Registry metadata JSON + version: MCP version + community_id: Linked community record ID port: Host port for the container mcp_config: MCP server configuration @@ -385,6 +511,8 @@ async def add_container_mcp_service( authorization_token=auth_token, container_config=container_config, registry_json=registry_json, + version=version, + market_id=market_id, enabled=True, container_id=container_info.get("container_id"), container_port=container_info.get("host_port"), @@ -458,7 +586,9 @@ def update_mcp_service( server_url: str, authorization_token: str | None, custom_headers: dict | None, + config_json: dict | None, tags: list | None, + market_id: int | None, ) -> None: """Update an MCP service record by ID. @@ -471,7 +601,9 @@ def update_mcp_service( server_url: New MCP server URL authorization_token: Authorization token custom_headers: Custom HTTP headers + config_json: MCP configuration JSON tags: MCP tags + market_id: Linked market record ID Raises: McpNotFoundError: If MCP record is not found @@ -480,10 +612,10 @@ def update_mcp_service( if not current_record: raise McpNotFoundError("MCP record not found") - is_container = _is_container_record(current_record) - config_json = None - if is_container: - config_json = current_record.get("config_json") if isinstance(current_record.get("config_json"), dict) else None + current_config_json = current_record.get("config_json") if isinstance(current_record.get("config_json"), dict) else None + next_config_json = config_json if config_json is not None else current_config_json + + next_market_id = market_id if market_id is not None else current_record.get("market_id") update_mcp_record_manage_fields_by_id( mcp_id=mcp_id, @@ -495,8 +627,9 @@ def update_mcp_service( source=(current_record.get("source") or "local"), authorization_token=authorization_token, custom_headers=custom_headers, - config_json=config_json, + config_json=next_config_json, tags=tags, + market_id=next_market_id, ) @@ -808,6 +941,8 @@ async def get_remote_mcp_server_list( "container_port": record.get("container_port"), "registry_json": record.get("registry_json"), "config_json": record.get("config_json"), + "market_id": record.get("market_id"), + "is_listed_in_repository": record.get("market_id") is not None, "container_status": container_status, } if is_need_auth: @@ -1046,6 +1181,60 @@ async def list_mcp_service_tools_by_id(*, tenant_id: str, mcp_id: int) -> list[d return [tool.__dict__ for tool in tools_info] +async def refresh_mcp_service_tool_count( + *, + tenant_id: str, + user_id: str, + mcp_id: int, +) -> list[str]: + """Connect to the MCP server, fetch tool names, and persist them to the record. + + Args: + tenant_id: Tenant ID + user_id: User ID + mcp_id: MCP record ID + + Returns: + List of tool names + + Raises: + McpNotFoundError: If MCP record is not found + McpValidationError: If MCP record has no server URL + MCPConnectionError: If MCP connection fails + """ + record = get_mcp_record_by_id_and_tenant(mcp_id=mcp_id, tenant_id=tenant_id) + if not record: + raise McpNotFoundError("MCP record not found") + + server_url = record.get("mcp_server") + if not server_url: + raise McpValidationError("MCP record has no server URL to connect to") + + authorization_token = record.get("authorization_token") + custom_headers = record.get("custom_headers") + + headers = {} + if authorization_token: + headers["Authorization"] = authorization_token + if custom_headers: + headers.update(custom_headers) + + tool_names = await _mcp_protocol_health_check(server_url, headers) + if not tool_names: + raise MCPConnectionError("MCP server is unreachable or does not support MCP protocol") + + registry_json = record.get("registry_json") or {} + registry_json["_toolNames"] = tool_names + + update_mcp_record_registry_json_by_id( + mcp_id=mcp_id, + tenant_id=tenant_id, + user_id=user_id, + registry_json=registry_json, + ) + return tool_names + + # --------------------------------------------------------------------------- # Image Upload Functions # --------------------------------------------------------------------------- diff --git a/backend/services/runtime_state_service.py b/backend/services/runtime_state_service.py new file mode 100644 index 000000000..49fea9744 --- /dev/null +++ b/backend/services/runtime_state_service.py @@ -0,0 +1,319 @@ +import asyncio +import hashlib +import logging +import socket +import time +from typing import Any, Dict, List, Optional, Tuple + +try: + import redis +except ImportError: + redis = None + +from consts.const import ( + RUNTIME_CANCEL_TTL_SECONDS, + RUNTIME_COMPLETED_TTL_SECONDS, + RUNTIME_RUN_TTL_SECONDS, + RUNTIME_STATE_REDIS_URL, + RUNTIME_STREAM_MAX_LEN, + RUNTIME_STREAM_TTL_SECONDS, +) + +logger = logging.getLogger(__name__) + + +class RuntimeStateService: + """Redis-backed short-lived state used by multi-replica runtime services.""" + + def __init__(self): + self._client: Optional[Any] = None + self._pod_name = socket.gethostname() + + @property + def enabled(self) -> bool: + return bool(RUNTIME_STATE_REDIS_URL) + + @property + def client(self) -> Any: + if not RUNTIME_STATE_REDIS_URL: + raise ValueError("RUNTIME_STATE_REDIS_URL or REDIS_URL environment variable is not set") + if redis is None: + raise ValueError("redis package is not installed") + if self._client is None: + self._client = redis.from_url( + RUNTIME_STATE_REDIS_URL, + socket_timeout=5, + socket_connect_timeout=5, + decode_responses=True, + ) + return self._client + + def _run_key(self, user_id: str, conversation_id: int) -> str: + return f"runtime:run:{user_id}:{conversation_id}" + + def _cancel_key(self, user_id: str, conversation_id: int) -> str: + return f"runtime:cancel:{user_id}:{conversation_id}" + + def _stream_key(self, user_id: str, conversation_id: int) -> str: + return f"runtime:stream:{user_id}:{conversation_id}" + + def _stream_done_key(self, user_id: str, conversation_id: int) -> str: + return f"runtime:stream:done:{user_id}:{conversation_id}" + + def _idempotency_key(self, key: str) -> str: + digest = hashlib.sha256(key.encode("utf-8")).hexdigest() + return f"northbound:idempotency:{digest}" + + def _rate_key(self, tenant_id: str, minute_bucket: str) -> str: + return f"northbound:rate:{tenant_id}:{minute_bucket}" + + def _expire_completed_runtime_keys(self, user_id: str, conversation_id: int) -> None: + ttl = max(1, RUNTIME_COMPLETED_TTL_SECONDS) + for key in ( + self._run_key(user_id, conversation_id), + self._cancel_key(user_id, conversation_id), + self._stream_key(user_id, conversation_id), + self._stream_done_key(user_id, conversation_id), + ): + self.client.expire(key, ttl) + + def reset_stream(self, user_id: str, conversation_id: int) -> None: + if not self.enabled: + return + try: + self.client.delete( + self._stream_key(user_id, conversation_id), + self._stream_done_key(user_id, conversation_id), + ) + except Exception as exc: + logger.warning("Failed to reset runtime stream state: %s", exc) + + async def reset_stream_async(self, user_id: str, conversation_id: int) -> None: + await asyncio.to_thread(self.reset_stream, user_id, conversation_id) + + def register_run(self, user_id: str, conversation_id: int, message_id: Optional[int] = None) -> None: + if not self.enabled: + return + try: + now = str(int(time.time())) + payload = { + "owner_pod": self._pod_name, + "status": "running", + "started_at": now, + "updated_at": now, + } + if message_id is not None: + payload["message_id"] = str(message_id) + key = self._run_key(user_id, conversation_id) + self.client.hset(key, mapping=payload) + self.client.expire(key, RUNTIME_RUN_TTL_SECONDS) + self.client.delete(self._cancel_key(user_id, conversation_id)) + except Exception as exc: + logger.warning("Failed to register runtime run state: %s", exc) + + def mark_run_finished(self, user_id: str, conversation_id: int, status: str) -> None: + if not self.enabled: + return + try: + key = self._run_key(user_id, conversation_id) + self.client.hset(key, mapping={ + "status": status, + "updated_at": str(int(time.time())), + }) + self._expire_completed_runtime_keys(user_id, conversation_id) + except Exception as exc: + logger.warning("Failed to mark runtime run state as finished: %s", exc) + + def get_run_state(self, user_id: str, conversation_id: int) -> Dict[str, str]: + if not self.enabled: + return {} + try: + return self.client.hgetall(self._run_key(user_id, conversation_id)) or {} + except Exception as exc: + logger.warning("Failed to get runtime run state: %s", exc) + return {} + + async def get_run_state_async(self, user_id: str, conversation_id: int) -> Dict[str, str]: + return await asyncio.to_thread(self.get_run_state, user_id, conversation_id) + + def set_cancel_signal(self, user_id: str, conversation_id: int) -> bool: + if not self.enabled: + return False + try: + self.client.setex(self._cancel_key(user_id, conversation_id), RUNTIME_CANCEL_TTL_SECONDS, self._pod_name) + return True + except Exception as exc: + logger.warning("Failed to set runtime cancel signal: %s", exc) + return False + + def is_cancelled(self, user_id: str, conversation_id: int) -> bool: + if not self.enabled: + return False + try: + return bool(self.client.get(self._cancel_key(user_id, conversation_id))) + except Exception as exc: + logger.warning("Failed to read runtime cancel signal: %s", exc) + return False + + async def is_cancelled_async(self, user_id: str, conversation_id: int) -> bool: + return await asyncio.to_thread(self.is_cancelled, user_id, conversation_id) + + def append_stream_event(self, user_id: str, conversation_id: int, chunk: str) -> Optional[str]: + if not self.enabled: + return None + try: + stream_key = self._stream_key(user_id, conversation_id) + event_id = self.client.xadd( + stream_key, + {"chunk": chunk}, + maxlen=RUNTIME_STREAM_MAX_LEN, + approximate=True, + ) + self.client.expire(stream_key, RUNTIME_STREAM_TTL_SECONDS) + return event_id + except Exception as exc: + logger.warning("Failed to append runtime stream event: %s", exc) + return None + + async def append_stream_event_async(self, user_id: str, conversation_id: int, chunk: str) -> Optional[str]: + return await asyncio.to_thread(self.append_stream_event, user_id, conversation_id, chunk) + + def mark_stream_completed( + self, + user_id: str, + conversation_id: int, + status: str, + error: Optional[str] = None, + ) -> None: + if not self.enabled: + return + try: + payload = { + "status": status, + "updated_at": str(int(time.time())), + } + if error: + payload["error"] = error + done_key = self._stream_done_key(user_id, conversation_id) + self.client.hset(done_key, mapping=payload) + self._expire_completed_runtime_keys(user_id, conversation_id) + except Exception as exc: + logger.warning("Failed to mark runtime stream completed: %s", exc) + + async def mark_stream_completed_async( + self, + user_id: str, + conversation_id: int, + status: str, + error: Optional[str] = None, + ) -> None: + await asyncio.to_thread(self.mark_stream_completed, user_id, conversation_id, status, error) + + def get_stream_status(self, user_id: str, conversation_id: int) -> Dict[str, str]: + if not self.enabled: + return {} + try: + return self.client.hgetall(self._stream_done_key(user_id, conversation_id)) or {} + except Exception as exc: + logger.warning("Failed to get runtime stream status: %s", exc) + return {} + + async def get_stream_status_async(self, user_id: str, conversation_id: int) -> Dict[str, str]: + return await asyncio.to_thread(self.get_stream_status, user_id, conversation_id) + + def read_stream_events( + self, + user_id: str, + conversation_id: int, + after_id: Optional[str] = None, + ) -> List[Tuple[str, str]]: + if not self.enabled: + return [] + try: + min_id = "-" if after_id is None else f"({after_id}" + events = self.client.xrange(self._stream_key(user_id, conversation_id), min=min_id) + return [(event_id, values.get("chunk", "")) for event_id, values in events] + except Exception as exc: + logger.warning("Failed to read runtime stream events: %s", exc) + return [] + + async def read_stream_events_async( + self, + user_id: str, + conversation_id: int, + after_id: Optional[str] = None, + ) -> List[Tuple[str, str]]: + return await asyncio.to_thread(self.read_stream_events, user_id, conversation_id, after_id) + + def wait_for_stream_events( + self, + user_id: str, + conversation_id: int, + last_id: str, + block_ms: int = 1000, + count: int = 100, + ) -> List[Tuple[str, str]]: + if not self.enabled: + return [] + try: + response = self.client.xread( + {self._stream_key(user_id, conversation_id): last_id}, + count=count, + block=block_ms, + ) + if not response: + return [] + _, events = response[0] + return [(event_id, values.get("chunk", "")) for event_id, values in events] + except Exception as exc: + logger.warning("Failed to wait for runtime stream events: %s", exc) + return [] + + async def wait_for_stream_events_async( + self, + user_id: str, + conversation_id: int, + last_id: str, + block_ms: int = 1000, + count: int = 100, + ) -> List[Tuple[str, str]]: + return await asyncio.to_thread( + self.wait_for_stream_events, + user_id, + conversation_id, + last_id, + block_ms, + count, + ) + + def acquire_idempotency(self, key: str, ttl_seconds: int) -> bool: + redis_key = self._idempotency_key(key) + acquired = self.client.set(redis_key, self._pod_name, nx=True, ex=ttl_seconds) + return bool(acquired) + + async def acquire_idempotency_async(self, key: str, ttl_seconds: int) -> bool: + return await asyncio.to_thread(self.acquire_idempotency, key, ttl_seconds) + + def release_idempotency(self, key: str) -> None: + self.client.delete(self._idempotency_key(key)) + + async def release_idempotency_async(self, key: str) -> None: + await asyncio.to_thread(self.release_idempotency, key) + + def consume_rate_limit(self, tenant_id: str, limit_per_minute: int) -> int: + minute_bucket = str(int(time.time() // 60)) + key = self._rate_key(tenant_id, minute_bucket) + pipe = self.client.pipeline() + pipe.incr(key) + pipe.expire(key, 120) + count, _ = pipe.execute() + count = int(count) + if count > limit_per_minute: + raise ValueError("rate limit exceeded") + return count + + async def consume_rate_limit_async(self, tenant_id: str, limit_per_minute: int) -> int: + return await asyncio.to_thread(self.consume_rate_limit, tenant_id, limit_per_minute) + + +runtime_state_service = RuntimeStateService() diff --git a/backend/services/skill_repository_service.py b/backend/services/skill_repository_service.py new file mode 100644 index 000000000..28f181c75 --- /dev/null +++ b/backend/services/skill_repository_service.py @@ -0,0 +1,884 @@ +import base64 +import logging +import math +import re +from typing import Any, Dict, FrozenSet, List, Optional, Tuple + +from consts.agent_repository import ( + OWNERSHIP_ALL, + OWNERSHIP_CREATED, + OWNERSHIP_OTHERS, + STATUS_NOT_SHARED, + STATUS_PENDING_REVIEW, + STATUS_REJECTED, + STATUS_SHARED, + VALID_OWNERSHIP_FILTERS, + VALID_REPOSITORY_STATUSES, +) +from consts.const import CAN_EDIT_ALL_USER_ROLES, PERMISSION_EDIT, PERMISSION_READ +from consts.exceptions import ForbiddenError, SkillDuplicateError, SkillException +from database.skill_repository_db import ( + get_skill_repository_by_id_and_publisher, + get_skill_repository_by_skill_id, + increment_skill_repository_downloads, + insert_skill_repository_record, + list_skill_repository_by_skill_ids, + list_skill_repository_summaries, + update_skill_repository_by_id, + update_skill_repository_status_by_id, +) +from database.skill_db import get_skill_by_name +from database.user_tenant_db import get_user_tenant_by_user_id +from services.skill_service import SkillService + +logger = logging.getLogger("skill_repository_service") +_REPOSITORY_LISTING_NOT_FOUND = "Repository listing not found" + +_MY_SKILL_REPOSITORY_STATUSES = frozenset({ + STATUS_SHARED, + STATUS_PENDING_REVIEW, + STATUS_REJECTED, +}) + +_SU_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_PUBLISHER_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), + (STATUS_PENDING_REVIEW, STATUS_NOT_SHARED), + (STATUS_REJECTED, STATUS_NOT_SHARED), + (STATUS_SHARED, STATUS_NOT_SHARED), +}) + +_PUBLISHER_RESUBMIT_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_NOT_SHARED, STATUS_PENDING_REVIEW), + (STATUS_REJECTED, STATUS_PENDING_REVIEW), +}) + +_ADMIN_REVIEW_STATUS_TRANSITIONS: FrozenSet[Tuple[str, str]] = frozenset({ + (STATUS_PENDING_REVIEW, STATUS_REJECTED), + (STATUS_PENDING_REVIEW, STATUS_SHARED), +}) + +_MAX_LISTING_TAGS = 5 +_MAX_LISTING_TAG_LENGTH = 20 +_MAX_LISTING_ICON_LENGTH = 32 +_MAX_COPY_NAME_LENGTH = 100 +_UPDATE_SNAPSHOT_FIELDS = ( + "name", + "description", + "source", + "submitted_by", + "category_id", + "tags", + "icon", + "downloads", + "skill_info_json", + "skill_zip_base64", + "status", +) + + +def _serialize_created_at(create_time: Any) -> Optional[str]: + """Serialize DB create_time to an ISO string for API consumers.""" + if create_time is None: + return None + if hasattr(create_time, "isoformat"): + return create_time.isoformat() + return str(create_time) + + +def _to_summary_item(record: Dict[str, Any]) -> Dict[str, Any]: + """Map a DB record to a lightweight skill marketplace summary item.""" + return { + "id": record.get("skill_repository_id"), + "skill_repository_id": record.get("skill_repository_id"), + "skill_id": record.get("skill_id"), + "submitted_by": record.get("submitted_by"), + "name": record.get("name"), + "description": record.get("description"), + "source": record.get("source"), + "status": record.get("status"), + "category_id": record.get("category_id"), + "tags": record.get("tags") or [], + "icon": record.get("icon"), + "downloads": record.get("downloads") or 0, + "created_at": record.get("created_at") or _serialize_created_at(record.get("create_time")), + "updated_at": record.get("updated_at") or _serialize_created_at(record.get("update_time")), + } + + +def _to_detail_item( + record: Dict[str, Any], + *, + is_updated: Optional[bool] = None, +) -> Dict[str, Any]: + """Map a DB record to a skill marketplace detail payload.""" + snapshot = _as_dict(record.get("skill_info_json")) + detail = { + "skill_repository_id": record.get("skill_repository_id"), + "skill_id": record.get("skill_id"), + "name": record.get("name"), + "description": record.get("description"), + "source": record.get("source"), + "submitted_by": record.get("submitted_by"), + "icon": record.get("icon"), + "status": record.get("status"), + "category_id": record.get("category_id"), + "tags": record.get("tags") or _as_list(snapshot.get("tags")), + "downloads": record.get("downloads") or 0, + "created_at": _serialize_created_at(record.get("create_time")), + "updated_at": _serialize_created_at(record.get("update_time")), + "content": snapshot.get("content"), + "config_schemas": _as_dict(snapshot.get("config_schemas")), + "config_values": _as_dict(snapshot.get("config_values")), + "tool_ids": _as_list(snapshot.get("tool_ids")), + } + if is_updated is not None: + detail["is_updated"] = is_updated + return detail + + +def _as_list(value: Any) -> List[Any]: + """Return list values safely for JSON snapshot fields.""" + return value if isinstance(value, list) else [] + + +def _as_dict(value: Any) -> Dict[str, Any]: + """Return dict values safely for JSON snapshot fields.""" + return value if isinstance(value, dict) else {} + + +def _to_repository_info_item(record: Dict[str, Any]) -> Dict[str, Any]: + """Map a repository DB row to a my-skills repository_info entry.""" + return { + "skill_repository_id": record.get("skill_repository_id"), + "status": record.get("status"), + "create_time": _serialize_created_at(record.get("create_time")), + } + + +def _matches_ownership(skill: Dict[str, Any], user_id: str, ownership_filter: str) -> bool: + """Return whether a skill belongs to the requested ownership bucket.""" + created_by = skill.get("created_by") + if ownership_filter in (OWNERSHIP_ALL, OWNERSHIP_CREATED): + return created_by == user_id + if ownership_filter == OWNERSHIP_OTHERS: + return False + return created_by == user_id + + +def _matches_search(skill: Dict[str, Any], search: Optional[str]) -> bool: + """Match mine-tab search against skill display fields and tags.""" + keyword = (search or "").strip().lower() + if not keyword: + return True + + haystack = [ + skill.get("name"), + skill.get("description"), + skill.get("source"), + skill.get("created_by"), + ] + haystack.extend(_as_list(skill.get("tags"))) + return any(keyword in str(value or "").lower() for value in haystack) + + +def _count_skills_by_ownership(skills: List[Dict[str, Any]], user_id: str) -> Dict[str, int]: + """Count editable skills in each ownership bucket.""" + created = sum(1 for skill in skills if skill.get("created_by") == user_id) + return { + OWNERSHIP_ALL: created, + OWNERSHIP_CREATED: created, + OWNERSHIP_OTHERS: 0, + } + + +def _paginate_mine_skills_with_optional_padding( + filtered_skills: List[Dict[str, Any]], + page: int, + page_size: int, + include_padding: bool, +) -> Tuple[List[Dict[str, Any]], int]: + """Paginate mine skills with an optional create-skill placeholder at virtual index 0.""" + skill_count = len(filtered_skills) + total = skill_count + 1 if include_padding else skill_count + if total == 0: + return [], 0 + + offset = (page - 1) * page_size + paged_entries: List[Dict[str, Any]] = [] + for slot in range(offset, min(offset + page_size, total)): + if include_padding and slot == 0: + paged_entries.append({"new_skill_padding": True}) + else: + skill_index = slot - 1 if include_padding else slot + paged_entries.append(filtered_skills[skill_index]) + return paged_entries, total + + +def _get_user_role(user_id: str) -> str: + """Resolve user role from user_tenant_t; default to USER when unset.""" + user_tenant = get_user_tenant_by_user_id(user_id) + if not user_tenant: + return "USER" + return str(user_tenant.get("user_role") or "USER") + + +def _resolve_mine_skill_permission( + *, + skill: Dict[str, Any], + user_id: str, + user_role: str, +) -> str: + """Resolve list-item permission for skill repository mine view.""" + if user_role in CAN_EDIT_ALL_USER_ROLES: + return PERMISSION_EDIT + return PERMISSION_EDIT if skill.get("created_by") == user_id else PERMISSION_READ + + +def _resolve_submitter_email(user_id: str) -> Optional[str]: + """Resolve submitter email from user_tenant_t for pending_review listings.""" + user_tenant = get_user_tenant_by_user_id(user_id) or {} + email = str(user_tenant.get("user_email") or "").strip() + return email or None + + +def _validate_create_listing_permission( + *, + user_id: str, + skill_info: Dict[str, Any], +) -> None: + """Only ADMIN, or DEV who created the skill, may share to marketplace.""" + user_role = _get_user_role(user_id) + if user_role == "ADMIN": + return + if user_role == "DEV" and skill_info.get("created_by") == user_id: + return + raise ForbiddenError( + f"User role {user_role} not authorized to create repository listing" + ) + + +def _normalize_listing_tags(tags: Any) -> List[str]: + """Trim, deduplicate, and validate marketplace listing tags.""" + if tags is None: + return [] + if not isinstance(tags, list): + raise ValueError("tags must be a list of strings") + + normalized: List[str] = [] + seen: set[str] = set() + for raw_tag in tags: + if not isinstance(raw_tag, str): + raise ValueError("tags must be a list of strings") + tag = raw_tag.strip() + if not tag: + continue + if len(tag) > _MAX_LISTING_TAG_LENGTH: + raise ValueError( + f"Each tag must be at most {_MAX_LISTING_TAG_LENGTH} characters" + ) + if tag in seen: + continue + seen.add(tag) + normalized.append(tag) + + if len(normalized) > _MAX_LISTING_TAGS: + raise ValueError(f"tags must contain at most {_MAX_LISTING_TAGS} items") + return normalized + + +def _validate_card_fields(repository_data: Dict[str, Any]) -> None: + """Validate marketplace card fields required for listing submission.""" + icon = repository_data.get("icon") or "skill" + if not icon or not isinstance(icon, str) or not icon.strip(): + raise ValueError("icon is required and must be a non-empty string") + if len(icon.strip()) > _MAX_LISTING_ICON_LENGTH: + raise ValueError( + f"icon must be at most {_MAX_LISTING_ICON_LENGTH} characters" + ) + repository_data["icon"] = icon.strip() + + category_id = repository_data.get("category_id") + if category_id is not None and not isinstance(category_id, int): + raise ValueError("category_id must be an integer") + + repository_data["tags"] = _normalize_listing_tags(repository_data.get("tags")) + + +def _build_skill_info_json(skill_info: Dict[str, Any]) -> Dict[str, Any]: + """Build frozen metadata snapshot for a skill repository listing.""" + return { + "skill_id": skill_info.get("skill_id"), + "name": skill_info.get("name"), + "description": skill_info.get("description"), + "tags": skill_info.get("tags") or [], + "content": skill_info.get("content") or "", + "config_schemas": skill_info.get("config_schemas"), + "config_values": skill_info.get("config_values"), + "source": skill_info.get("source"), + "tool_ids": skill_info.get("tool_ids") or [], + "created_by": skill_info.get("created_by"), + } + + +def _export_skill_zip_base64( + *, + skill_name: str, + tenant_id: str, +) -> str: + """Export a skill ZIP payload as base64 for frozen repository installation.""" + service = SkillService(tenant_id=tenant_id) + exports = service.export_skills_by_names([skill_name], tenant_id=tenant_id) + for item in exports: + if item.get("skill_name") == skill_name and item.get("skill_zip_base64"): + return item["skill_zip_base64"] + raise ValueError(f"Failed to export skill ZIP for repository listing: {skill_name}") + + +def _build_repository_data_from_skill( + skill_id: int, + tenant_id: str, + user_id: str, + *, + card_fields: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Build a repository upsert payload from the current skill snapshot.""" + service = SkillService(tenant_id=tenant_id) + skill_info = service.get_skill_by_id(skill_id, tenant_id=tenant_id) + if not skill_info: + raise ValueError("Skill not found") + + _validate_create_listing_permission(user_id=user_id, skill_info=skill_info) + + skill_name = str(skill_info.get("name") or "").strip() + if not skill_name: + raise ValueError("Skill name is required") + + repository_data: Dict[str, Any] = { + "skill_id": skill_id, + "name": skill_name, + "description": skill_info.get("description"), + "source": skill_info.get("source"), + "submitted_by": _resolve_submitter_email(user_id), + "icon": "skill", + "tags": skill_info.get("tags") or [], + "skill_info_json": _build_skill_info_json(skill_info), + "skill_zip_base64": _export_skill_zip_base64( + skill_name=skill_name, + tenant_id=tenant_id, + ), + "status": STATUS_PENDING_REVIEW, + } + + if card_fields: + for key in ("icon", "downloads", "category_id"): + if key in card_fields and card_fields[key] is not None: + repository_data[key] = card_fields[key] + if "tags" in card_fields and card_fields["tags"] is not None: + repository_data["tags"] = card_fields["tags"] + + return repository_data + + +def _validate_create_payload(repository_data: Dict[str, Any]) -> None: + """Validate required fields before inserting a repository listing.""" + required_fields = ( + "skill_id", + "name", + "skill_info_json", + "skill_zip_base64", + ) + missing = [ + field for field in required_fields + if field not in repository_data or repository_data[field] is None + ] + if missing: + raise ValueError(f"Missing required repository fields: {', '.join(missing)}") + if not repository_data.get("name"): + raise ValueError("name must be a non-empty string") + if not isinstance(repository_data.get("skill_info_json"), dict): + raise ValueError("skill_info_json must be a JSON object") + if not isinstance(repository_data.get("skill_zip_base64"), str): + raise ValueError("skill_zip_base64 must be a string") + + _validate_card_fields(repository_data) + + +def create_skill_repository_listing_impl( + skill_id: int, + tenant_id: str, + user_id: str, + *, + card_fields: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Create or update a repository listing from the current skill snapshot.""" + repository_data = _build_repository_data_from_skill( + skill_id, + tenant_id, + user_id, + card_fields=card_fields, + ) + _validate_create_payload(repository_data) + + existing = get_skill_repository_by_skill_id( + skill_id, + publisher_tenant_id=tenant_id, + ) + if not existing: + repository_id = insert_skill_repository_record( + repository_data=repository_data, + publisher_tenant_id=tenant_id, + publisher_user_id=user_id, + ) + is_updated = False + else: + repository_id = int(existing["skill_repository_id"]) + updates = { + key: repository_data[key] + for key in _UPDATE_SNAPSHOT_FIELDS + if key in repository_data + } + affected = update_skill_repository_by_id( + repository_id=repository_id, + publisher_tenant_id=tenant_id, + user_id=user_id, + updates=updates, + ) + if affected == 0: + raise ValueError("Failed to update repository listing") + is_updated = True + + record = get_skill_repository_by_id_and_publisher( + repository_id, + tenant_id, + ) + if not record: + raise ValueError("Failed to load repository listing after write") + return _to_detail_item(record, is_updated=is_updated) + + +def _validate_su_status_transition( + transition: Tuple[str, str], + current_status: str, + new_status: str, +) -> None: + if transition not in _SU_STATUS_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + + +def _validate_publisher_status_transition( + *, + user_role: str, + transition: Tuple[str, str], + current_status: str, + new_status: str, + record: Dict[str, Any], + user_id: str, + tenant_id: str, +) -> Optional[Dict[str, str]]: + if record.get("publisher_tenant_id") != tenant_id: + raise ForbiddenError("Not authorized to update this repository listing") + if user_role == "DEV" and record.get("publisher_user_id") != user_id: + raise ForbiddenError("Not authorized to update this repository listing") + if user_role == "ADMIN" and transition in _ADMIN_REVIEW_STATUS_TRANSITIONS: + return None + if transition not in _PUBLISHER_STATUS_TRANSITIONS: + raise ValueError( + f"Invalid status transition from '{current_status}' to '{new_status}'" + ) + if transition in _PUBLISHER_RESUBMIT_TRANSITIONS: + return { + "publisher_tenant_id": tenant_id, + "publisher_user_id": user_id, + } + return None + + +def _validate_repository_status_transition( + *, + user_role: str, + current_status: str, + new_status: str, + record: Dict[str, Any], + user_id: str, + tenant_id: str, +) -> Optional[Dict[str, str]]: + """Validate role, ownership, and allowed status transition.""" + transition = (current_status, new_status) + + if user_role == "SU": + _validate_su_status_transition(transition, current_status, new_status) + return None + + if user_role in ("ADMIN", "DEV"): + return _validate_publisher_status_transition( + user_role=user_role, + transition=transition, + current_status=current_status, + new_status=new_status, + record=record, + user_id=user_id, + tenant_id=tenant_id, + ) + + raise ForbiddenError( + f"User role {user_role} not authorized to update repository status" + ) + + +def update_skill_repository_status_impl( + *, + skill_repository_id: int, + status: str, + user_id: str, + tenant_id: str, +) -> Dict[str, Any]: + """Update a skill repository listing status by primary key.""" + if status not in VALID_REPOSITORY_STATUSES: + raise ValueError( + f"Invalid status '{status}'; must be one of: " + f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" + ) + + record = get_skill_repository_by_id_and_publisher( + skill_repository_id, + tenant_id, + ) + if not record: + raise ValueError(_REPOSITORY_LISTING_NOT_FOUND) + + current_status = record.get("status") + publisher_updates: Optional[Dict[str, str]] = None + submitted_by: Optional[str] = None + if current_status != status: + user_role = _get_user_role(user_id) + publisher_updates = _validate_repository_status_transition( + user_role=user_role, + current_status=current_status, + new_status=status, + record=record, + user_id=user_id, + tenant_id=tenant_id, + ) + if status == STATUS_PENDING_REVIEW: + submitted_by = _resolve_submitter_email(user_id) + + rows_affected = update_skill_repository_status_by_id( + repository_id=skill_repository_id, + status=status, + user_id=user_id, + filter_publisher_tenant_id=tenant_id, + publisher_tenant_id=( + publisher_updates["publisher_tenant_id"] + if publisher_updates + else None + ), + publisher_user_id=( + publisher_updates["publisher_user_id"] + if publisher_updates + else None + ), + submitted_by=submitted_by, + ) + if rows_affected == 0: + raise ValueError(_REPOSITORY_LISTING_NOT_FOUND) + + updated = get_skill_repository_by_id_and_publisher( + skill_repository_id, + tenant_id, + ) + if not updated: + raise ValueError("Failed to load repository listing after update") + return _to_summary_item(updated) + + +def _extract_duplicate_skill_name(error_message: str) -> Optional[str]: + """Extract duplicate skill name from existing SkillException messages.""" + match = re.search(r"Skill '([^']+)' already exists", error_message) + if match: + return match.group(1) + return None + + +def _truncate_copy_base_name(base_name: str, suffix: str) -> str: + """Trim a copied skill base name so the final name fits the database limit.""" + max_base_length = max(_MAX_COPY_NAME_LENGTH - len(suffix), 1) + if len(base_name) <= max_base_length: + return base_name + return base_name[:max_base_length].rstrip() or base_name[:max_base_length] + + +def _generate_available_copy_skill_name( + *, + base_name: str, + tenant_id: str, +) -> str: + """Generate an available skill name for repository copy within the tenant.""" + normalized_base = (base_name or "Skill").strip() or "Skill" + if not get_skill_by_name(normalized_base, tenant_id): + return normalized_base + + index = 1 + while True: + suffix = " 副本" if index == 1 else f" 副本 {index}" + candidate = f"{_truncate_copy_base_name(normalized_base, suffix)}{suffix}" + if not get_skill_by_name(candidate, tenant_id): + return candidate + index += 1 + + +def install_skill_from_repository_impl( + *, + skill_repository_id: int, + tenant_id: str, + user_id: str, + target_name: Optional[str] = None, +) -> Dict[str, Any]: + """Install a shared skill repository listing into the current tenant.""" + record = get_skill_repository_by_id_and_publisher( + skill_repository_id, + tenant_id, + ) + if not record: + raise ValueError(_REPOSITORY_LISTING_NOT_FOUND) + if record.get("status") != STATUS_SHARED: + raise ValueError("Repository listing is not available for install") + + skill_zip_base64 = record.get("skill_zip_base64") + if not isinstance(skill_zip_base64, str) or not skill_zip_base64.strip(): + raise ValueError("Repository listing has no skill ZIP payload") + + try: + zip_bytes = base64.b64decode(skill_zip_base64, validate=True) + except Exception as exc: + raise ValueError("Repository listing has invalid skill ZIP payload") from exc + + copy_skill_name = str(target_name or "").strip() + if not copy_skill_name: + copy_skill_name = _generate_available_copy_skill_name( + base_name=str(record.get("name") or "").strip(), + tenant_id=tenant_id, + ) + if not copy_skill_name: + raise ValueError("Skill name is required") + + try: + created_skill = SkillService(tenant_id=tenant_id).create_skill_from_zip_bytes( + zip_bytes=zip_bytes, + skill_name=copy_skill_name, + source="repository", + user_id=user_id, + tenant_id=tenant_id, + ) + except SkillException as exc: + message = str(exc) + if "already exists" in message.lower(): + duplicate_name = _extract_duplicate_skill_name(message) or copy_skill_name + raise SkillDuplicateError([duplicate_name]) from exc + raise + + affected = increment_skill_repository_downloads( + repository_id=skill_repository_id, + user_id=user_id, + ) + if affected == 0: + logger.warning( + "Failed to increment skill repository downloads after install " + "(skill_repository_id=%s)", + skill_repository_id, + ) + + return { + "skill_id": created_skill.get("skill_id"), + "name": created_skill.get("name"), + "description": created_skill.get("description"), + "source": created_skill.get("source"), + "tags": created_skill.get("tags") or [], + } + + +def _list_repository_info_by_skill_id( + paged_skills: List[Dict[str, Any]], + tenant_id: str, +) -> Dict[int, List[Dict[str, Any]]]: + skill_ids = [ + int(skill["skill_id"]) + for skill in paged_skills + if skill.get("skill_id") is not None + ] + if not skill_ids: + return {} + + repository_by_skill_id: Dict[int, List[Dict[str, Any]]] = {} + repository_records = list_skill_repository_by_skill_ids( + skill_ids, + statuses=_MY_SKILL_REPOSITORY_STATUSES, + publisher_tenant_id=tenant_id, + ) + for record in repository_records: + skill_id = record.get("skill_id") + if skill_id is None: + continue + repository_by_skill_id.setdefault(int(skill_id), []).append( + _to_repository_info_item(record) + ) + return repository_by_skill_id + + +def _to_mine_skill_item( + skill: Dict[str, Any], + *, + user_id: str, + user_role: str, + repository_by_skill_id: Dict[int, List[Dict[str, Any]]], +) -> Dict[str, Any]: + if skill.get("new_skill_padding"): + return {"new_skill_padding": True} + + skill_id = skill.get("skill_id") + repository_info = ( + repository_by_skill_id.get(int(skill_id), []) + if skill_id is not None + else [] + ) + return { + "skill_id": skill_id, + "name": skill.get("name"), + "description": skill.get("description"), + "source": skill.get("source"), + "tags": skill.get("tags") or [], + "created_by": skill.get("created_by"), + "created_at": skill.get("create_time"), + "updated_at": skill.get("update_time"), + "permission": _resolve_mine_skill_permission( + skill=skill, + user_id=user_id, + user_role=user_role, + ), + "repository_info": repository_info, + } + + +def list_my_editable_skills_impl( + tenant_id: str, + user_id: str, + ownership: str = OWNERSHIP_ALL, + *, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + new_skill_padding: bool = False, +) -> Dict[str, Any]: + """List editable skills for the current user with repository listing info.""" + normalized_ownership = (ownership or OWNERSHIP_ALL).strip().lower() + if normalized_ownership not in VALID_OWNERSHIP_FILTERS: + raise ValueError( + f"Invalid ownership filter: {ownership}. " + f"Allowed values: {', '.join(sorted(VALID_OWNERSHIP_FILTERS))}." + ) + + safe_page = max(int(page or 1), 1) + safe_page_size = max(int(page_size or 10), 1) + + user_role = _get_user_role(user_id) + skills = SkillService(tenant_id=tenant_id).list_skills(tenant_id=tenant_id) + counts = _count_skills_by_ownership(skills, user_id) + + filtered_skills = [ + skill for skill in skills + if _matches_ownership(skill, user_id, normalized_ownership) + and _matches_search(skill, search) + ] + include_padding = ( + new_skill_padding + and normalized_ownership == OWNERSHIP_ALL + and not (search and search.strip()) + ) + paged_skills, total = _paginate_mine_skills_with_optional_padding( + filtered_skills, + page=safe_page, + page_size=safe_page_size, + include_padding=include_padding, + ) + + repository_by_skill_id = _list_repository_info_by_skill_id( + paged_skills, + tenant_id, + ) + items = [ + _to_mine_skill_item( + skill, + user_id=user_id, + user_role=user_role, + repository_by_skill_id=repository_by_skill_id, + ) + for skill in paged_skills + ] + + return { + "items": items, + "counts": counts, + "pagination": { + "page": safe_page, + "page_size": safe_page_size, + "total": total, + "total_pages": math.ceil(total / safe_page_size) if total else 0, + }, + } + + +def list_skill_repository_listings_impl( + tenant_id: str, + *, + status: Optional[str] = None, + skill_id: Optional[int] = None, + category_id: Optional[int] = None, + page: int = 1, + page_size: int = 10, + search: Optional[str] = None, + sort_by_update_time: bool = False, +) -> Dict[str, Any]: + """List skill repository listings for the caller tenant with optional filters.""" + if status is not None and status not in VALID_REPOSITORY_STATUSES: + raise ValueError( + f"Invalid status '{status}'; must be one of: " + f"{', '.join(sorted(VALID_REPOSITORY_STATUSES))}" + ) + + result = list_skill_repository_summaries( + publisher_tenant_id=tenant_id, + status=status, + skill_id=skill_id, + category_id=category_id, + page=page, + page_size=page_size, + search=search, + sort_by_update_time=sort_by_update_time, + ) + return { + "items": [_to_summary_item(record) for record in result.get("items", [])], + "pagination": result.get("pagination"), + } + + +def get_skill_repository_listing_detail_impl( + skill_repository_id: int, + tenant_id: str, +) -> Dict[str, Any]: + """Load a skill repository listing and return a frozen detail payload for the UI.""" + record = get_skill_repository_by_id_and_publisher( + skill_repository_id, + tenant_id, + ) + if not record: + raise ValueError(_REPOSITORY_LISTING_NOT_FOUND) + + return _to_detail_item(record) diff --git a/backend/services/skill_service.py b/backend/services/skill_service.py index f5b7d1c7c..e32cc532a 100644 --- a/backend/services/skill_service.py +++ b/backend/services/skill_service.py @@ -22,7 +22,7 @@ from nexent.core.utils.observer import MessageObserver from nexent.core.agents.agent_model import ModelConfig from consts.const import CONTAINER_SKILLS_PATH, OFFICIAL_SKILLS_ZIP_PATH, ROOT_DIR -from consts.exceptions import SkillException +from consts.exceptions import ForbiddenError, SkillException from database import skill_db from agents.skill_creation_agent import create_skill_from_request from utils.prompt_template_utils import get_skill_creation_simple_prompt_template @@ -760,12 +760,61 @@ def _get_skill_inputs_from_zip( def _local_skill_config_yaml_path(skill_name: str, local_skills_dir: str) -> str: """Absolute path to //config/config.yaml.""" - return os.path.join(local_skills_dir, skill_name, "config", "config.yaml") + return _resolve_local_skill_path( + local_skills_dir, + skill_name, + "config", + "config.yaml", + ) def _local_skill_schema_yaml_path(skill_name: str, local_skills_dir: str) -> str: """Absolute path to //config/schema.yaml.""" - return os.path.join(local_skills_dir, skill_name, "config", "schema.yaml") + return _resolve_local_skill_path( + local_skills_dir, + skill_name, + "config", + "schema.yaml", + ) + + +def _resolve_local_skill_path( + local_skills_dir: str, + skill_name: str, + *parts: str, +) -> str: + """Resolve a path below the configured skills root and one skill directory.""" + name = str(skill_name or "").strip() + if ( + not name + or name in {".", ".."} + or "/" in name + or "\\" in name + or "\x00" in name + or os.path.basename(name) != name + ): + raise SkillException("Invalid skill name for local file access") + + allowed_root = os.path.realpath(CONTAINER_SKILLS_PATH) + local_root = os.path.realpath(local_skills_dir) + if ( + local_root != allowed_root + and not local_root.startswith(allowed_root + os.sep) + ): + raise SkillException("Unsafe local skills directory") + + candidate = os.path.realpath(os.path.join(local_root, name, *parts)) + if ( + candidate != allowed_root + and not candidate.startswith(allowed_root + os.sep) + ): + raise SkillException("Unsafe local skill path") + if ( + candidate != local_root + and not candidate.startswith(local_root + os.sep) + ): + raise SkillException("Unsafe local skill path") + return candidate def _write_skill_params_to_local_config_yaml( @@ -778,9 +827,9 @@ def _write_skill_params_to_local_config_yaml( if not local_skills_dir: return - config_dir = os.path.join(local_skills_dir, skill_name, "config") - os.makedirs(config_dir, exist_ok=True) path = _local_skill_config_yaml_path(skill_name, local_skills_dir) + config_dir = os.path.dirname(path) + os.makedirs(config_dir, exist_ok=True) text = params_dict_to_roundtrip_yaml_text(params) with open(path, "w", encoding="utf-8") as f: f.write(text) @@ -822,7 +871,12 @@ def __init__(self, skill_manager: Optional[SkillManager] = None, tenant_id: Opti def _resolve_local_skills_dir_for_overlay(self) -> Optional[str]: """Directory where skill folders live: ``SKILLS_PATH``, else ``ROOT_DIR/skills`` if present.""" - d = self.skill_manager.local_skills_dir or CONTAINER_SKILLS_PATH + manager_dir = getattr(self.skill_manager, "local_skills_dir", None) + d = ( + manager_dir + if isinstance(manager_dir, str) + else CONTAINER_SKILLS_PATH + ) if d: return str(d).rstrip(os.sep) or None if ROOT_DIR: @@ -970,8 +1024,10 @@ def create_skill( # Check if skill directory already exists locally resolved = self._resolve_local_skills_dir_for_overlay() - if resolved and os.path.exists(os.path.join(resolved, skill_name)): - raise SkillException(f"Skill '{skill_name}' already exists locally") + if resolved: + local_skill_dir = _resolve_local_skill_path(resolved, skill_name) + if os.path.exists(local_skill_dir): + raise SkillException(f"Skill '{skill_name}' already exists locally") # Set created_by and updated_by if user_id is provided if user_id: @@ -1013,7 +1069,7 @@ def create_skill_from_file( file_content: Union[bytes, str, io.BytesIO], skill_name: Optional[str] = None, file_type: str = "auto", - source: str = "自定义", + source: str = "custom", tenant_id: Optional[str] = None, user_id: Optional[str] = None ) -> Dict[str, Any]: @@ -1027,7 +1083,7 @@ def create_skill_from_file( file_content: File content as bytes, string, or BytesIO skill_name: Optional skill name (extracted from ZIP if not provided) file_type: File type hint - "md", "zip", or "auto" (detect) - source: Source identifier for the skill (e.g., "自定义", "官方", "导入") + source: Source identifier for the skill (e.g., "custom", "official", "repository") tenant_id: Tenant ID for skill isolation. Uses instance tenant_id if not provided. user_id: User ID of the creator @@ -1058,7 +1114,7 @@ def _create_skill_from_md( self, content_bytes: bytes, skill_name: Optional[str] = None, - source: str = "自定义", + source: str = "custom", user_id: Optional[str] = None, tenant_id: Optional[str] = None ) -> Dict[str, Any]: @@ -1073,6 +1129,9 @@ def _create_skill_from_md( name = skill_name or skill_data.get("name") if not name: raise SkillException("Skill name is required") + local_dir = self._resolve_local_skills_dir_for_overlay() + if local_dir: + _resolve_local_skill_path(local_dir, name) # Check if skill already exists in database existing = skill_db.get_skill_by_name(name, tenant_id) @@ -1113,7 +1172,7 @@ def _create_skill_from_zip( self, zip_bytes: bytes, skill_name: Optional[str] = None, - source: str = "自定义", + source: str = "custom", user_id: Optional[str] = None, tenant_id: Optional[str] = None ) -> Dict[str, Any]: @@ -1168,6 +1227,9 @@ def _create_skill_from_zip( name = skill_name or detected_skill_name if not name: raise SkillException("Skill name is required") + local_dir = self._resolve_local_skills_dir_for_overlay() + if local_dir: + _resolve_local_skill_path(local_dir, name) # Check if skill already exists in database existing = skill_db.get_skill_by_name(name, tenant_id) @@ -1615,6 +1677,110 @@ def update_skill( logger.error(f"Error updating skill {skill_name}: {e}") raise SkillException(f"Failed to update skill: {str(e)}") from e + def update_skill_by_id( + self, + skill_id: int, + skill_data: Dict[str, Any], + tenant_id: Optional[str] = None, + user_id: Optional[str] = None + ) -> Dict[str, Any]: + """Update an existing skill by ID for a tenant.""" + effective_tenant_id = tenant_id or self.tenant_id + if not effective_tenant_id: + raise SkillException("tenant_id is required") + try: + existing = skill_db.get_skill_by_id(skill_id, effective_tenant_id) + if not existing: + raise SkillException(f"Skill not found: {skill_id}") + if not user_id or existing.get("created_by") != user_id: + raise ForbiddenError("Not authorized to update this skill") + + local_dir = self._resolve_local_skills_dir_for_overlay() + if local_dir and "name" in skill_data: + _resolve_local_skill_path( + local_dir, + str(skill_data["name"] or ""), + ) + + result = skill_db.update_skill_by_id( + skill_id, + skill_data, + effective_tenant_id, + updated_by=user_id or None, + ) + + if not local_dir: + return self._enrich_configs_from_yaml(result) + + persisted_skill_id = int(existing["skill_id"]) + skill_id_directory = _resolve_local_skill_path( + local_dir, + f"skill_{persisted_skill_id}", + ) + local_skill_name = ( + f"skill_{persisted_skill_id}" + if os.path.isdir(skill_id_directory) + else str(result.get("name") or existing.get("name") or "") + ) + if not local_skill_name: + return self._enrich_configs_from_yaml(result) + + if local_dir and "config_values" in skill_data: + try: + raw_config_values = skill_data["config_values"] + if raw_config_values is None: + _remove_local_skill_config_yaml(local_skill_name, local_dir) + else: + _write_skill_params_to_local_config_yaml( + local_skill_name, + _params_dict_to_storable(raw_config_values), + local_dir, + ) + except Exception as exc: + logger.warning( + "Local config/config.yaml sync failed after skill ID update for %s: %s", + skill_id, + exc, + ) + + if not local_dir: + return self._enrich_configs_from_yaml(result) + + try: + allowed_tools = skill_db.get_tool_names_by_skill_name( + str(existing.get("name") or ""), + effective_tenant_id, + ) + local_skill_dict = { + "name": local_skill_name, + "description": skill_data.get("description", existing.get("description", "")), + "content": skill_data.get("content", existing.get("content", "")), + "tags": skill_data.get("tags", existing.get("tags", [])), + "allowed-tools": allowed_tools, + "files": skill_data.get("files", []), + } + self.skill_manager.save_skill(local_skill_dict) + previous_name = str(existing.get("name") or "").strip() + if ( + local_skill_name != f"skill_{skill_id}" + and previous_name + and previous_name != local_skill_name + ): + self.skill_manager.delete_skill(previous_name) + except Exception as exc: + logger.warning( + "Local SKILL.md sync failed after DB update for skill ID %s: %s", + skill_id, + exc, + ) + + return self._enrich_configs_from_yaml(result) + except (ForbiddenError, SkillException): + raise + except Exception as e: + logger.exception("Error updating skill by ID %s", skill_id) + raise SkillException(f"Failed to update skill: {str(e)}") from e + def delete_skill( self, skill_name: str, @@ -2078,7 +2244,6 @@ def create_skill_from_zip_bytes( result = skill_db.create_skill(skill_dict, tenant_id) self.skill_manager.save_skill(skill_dict) - self._upload_zip_files(zip_bytes, name, detected_skill_name) return self._enrich_configs_from_yaml(result) diff --git a/backend/services/streaming_channel.py b/backend/services/streaming_channel.py index 2f62ee33c..ad3f20ab3 100644 --- a/backend/services/streaming_channel.py +++ b/backend/services/streaming_channel.py @@ -10,6 +10,8 @@ import logging from typing import Dict, Optional, AsyncIterator, List +from services.runtime_state_service import runtime_state_service + logger = logging.getLogger(__name__) # Default history buffer size (kept for backward compatibility with callers). @@ -85,6 +87,12 @@ async def publish(self, chunk: str): async with self._lock: self._history_buffer.append(chunk) + await runtime_state_service.append_stream_event_async( + user_id=self.user_id, + conversation_id=self.conversation_id, + chunk=chunk, + ) + # Wake up waiting subscribers immediately self._data_event.set() @@ -276,6 +284,11 @@ async def complete_channel( channel = self.get_channel(conversation_id, user_id) if channel: channel.complete(status) + await runtime_state_service.mark_stream_completed_async( + user_id=user_id, + conversation_id=conversation_id, + status=status, + ) async def remove_channel(self, conversation_id: int, user_id: str): """Remove a channel from the manager.""" diff --git a/backend/services/tool_configuration_service.py b/backend/services/tool_configuration_service.py index 32460fcee..e61273cb1 100644 --- a/backend/services/tool_configuration_service.py +++ b/backend/services/tool_configuration_service.py @@ -15,6 +15,7 @@ from consts.const import DATA_PROCESS_SERVICE, LOCAL_MCP_SERVER, MCP_MANAGEMENT_API from consts.exceptions import MCPConnectionError, NotFoundException, ToolExecutionException from consts.model import ToolInstanceInfoRequest, ToolInfo, ToolSourceEnum, ToolValidateRequest +from consts.tool_labels import SYSTEM_MANAGED_TOOL_NAMES from database.outer_api_tool_db import ( upsert_openapi_service, query_openapi_services_by_tenant, @@ -482,7 +483,10 @@ async def update_tool_list(tenant_id: str, user_id: str): mcp_tools = await get_all_mcp_tools(tenant_id) except Exception as e: logger.error(f"failed to get all mcp tools, detail: {e}") - raise MCPConnectionError(f"failed to get all mcp tools, detail: {e}") + # Don't block local/langchain tool update when MCP is unavailable. + # MCP tools will be marked as is_available=False in the DB, which + # is the correct state when the MCP server is unreachable. + mcp_tools = [] update_tool_table_from_scan_tool_list(tenant_id=tenant_id, user_id=user_id, @@ -506,6 +510,9 @@ async def list_all_tools(tenant_id: str, labels: Optional[List[str]] = None): for tool in tools_info: tool_name = tool.get("name") + if tool_name in SYSTEM_MANAGED_TOOL_NAMES: + continue + # Always use SDK inputs for local tools to stay in sync with current tool code is_local = tool.get("source") == "local" sdk_info = local_tool_descriptions.get(tool_name) if is_local else None @@ -808,6 +815,7 @@ def _validate_local_tool( rerank = instantiation_params.get("rerank", False) rerank_model_name = instantiation_params.get("rerank_model_name", "") rerank_model = None + if rerank and rerank_model_name: rerank_model = get_rerank_model(tenant_id=tenant_id, model_name=rerank_model_name) @@ -816,6 +824,12 @@ def _validate_local_tool( 'rerank_model': rerank_model, } tool_instance = tool_class(**params) + elif tool_name == "ragflow_search": + # RAGFlowSearchTool does not accept rerank/rerank_model_name params + # RAGFlow handles reranking internally via its API + filtered_params = {k: v for k, v in instantiation_params.items() + if k not in ["rerank_model", "rerank", "rerank_model_name"]} + tool_instance = tool_class(**filtered_params) elif tool_name in ("haotian_search", "aidp_search"): # Haotian and AIDP share the same instantiation shape: drop the # backend-only rerank keys and explicitly set observer=None diff --git a/backend/utils/auth_utils.py b/backend/utils/auth_utils.py index 4ade6f211..648069ae9 100644 --- a/backend/utils/auth_utils.py +++ b/backend/utils/auth_utils.py @@ -314,15 +314,18 @@ def get_jwt_expiry_seconds(token: str) -> int: if DEBUG_JWT_EXPIRE_SECONDS > 0: return DEBUG_JWT_EXPIRE_SECONDS - # Decode JWT token (without signature verification, only parse content) - decoded = jwt.decode(jwt_token, options={"verify_signature": False}) + # Decode JWT token with signature verification. Expiration validation is + # disabled intentionally because callers need the original exp/iat span. + decoded = _decode_jwt_token_for_expiry(jwt_token) # Extract expiration time and issued time from JWT claims - exp = decoded.get("exp", 0) - iat = decoded.get("iat", 0) + exp = int(decoded["exp"]) + iat = int(decoded["iat"]) # Calculate validity period (seconds) expiry_seconds = exp - iat + if expiry_seconds <= 0: + raise ValueError("JWT exp must be greater than iat") return expiry_seconds except Exception as e: @@ -348,6 +351,24 @@ def calculate_expires_at(token: Optional[str] = None) -> int: return int((datetime.now() + timedelta(seconds=expiry_seconds)).timestamp()) +def _decode_jwt_token_for_expiry(token: str) -> dict: + """ + Decode JWT claims for session timing after verifying the token signature. + + Expiration validation is intentionally disabled so callers can compute the + original token lifetime even when the token is already expired. + """ + if not SUPABASE_JWT_SECRET: + raise UnauthorizedError("JWT verification is not configured") + + return jwt.decode( + token, + SUPABASE_JWT_SECRET, + algorithms=["HS256"], + options={"verify_exp": False, "verify_aud": False}, + ) + + def _decode_jwt_token(authorization: str) -> dict: """ Extract user ID from JWT token after verifying signature and expiration. diff --git a/backend/utils/context_utils.py b/backend/utils/context_utils.py index 4ddaa6d63..690eaf21e 100644 --- a/backend/utils/context_utils.py +++ b/backend/utils/context_utils.py @@ -181,8 +181,11 @@ def _format_skills_description( lines.append("") lines.append("3. **遵循技能指南**：技能内容注入后，严格按其中的步骤执行。不要跳过技能指南中的步骤，也不要用自行编写的代码替代技能定义的流程。") lines.append("") - lines.append("4. **执行技能脚本**：如果技能指南中引用了附加脚本（形如 ``），使用以下格式调用：") - lines.append(" 代码：") + lines.append("4. **执行技能脚本**：技能中引用的脚本（参考文档、脚本声明）可通过以下任一形式表达，**功能上完全等同**，模型必须把它们都识别为路径声明：") + lines.append(" - XML 标签形式：``、``") + lines.append(" - 单个反引号包裹：`` `scripts/analyze.py` ``、`` `reference/api_doc` ``") + lines.append(" - 三重反引号代码块：`` ```scripts/analyze.py``` ``（当代码块内仅有单行路径时）") + lines.append(" 调用 `run_skill_script` 时，`script_path` **始终相对于技能根目录**解析（平台行为，不是当前工作目录），常见形式如下：") lines.append(" ") lines.append(" result = run_skill_script(\"skill_name\", \"script_path\")") lines.append(" print(result)") @@ -193,13 +196,20 @@ def _format_skills_description( lines.append(" result = run_skill_script(\"skill_name\", \"script_path\", \"--param1 value1 --flag\")") lines.append(" print(result)") lines.append(" ") - lines.append(" 注意：只执行技能指南中明确声明的脚本路径，绝不自行构造脚本路径。") + lines.append(" 注意：") + lines.append(" - 只执行技能指南中明确声明的脚本路径，绝不自行构造脚本路径。") + lines.append(" - 不要把脚本当作当前工作目录（CWD）的相对路径处理；也不要使用绝对路径。") + lines.append(" - 当脚本不存在时，返回的错误信息中会列出该技能根目录下的可用脚本，请据此修正路径。") lines.append("") lines.append("5. **整合输出**：根据技能指南要求的输出格式，结合脚本执行结果生成最终回答。") lines.append("") - lines.append("6. **引用场景处理**：当技能内容中出现引用标记或需要引用其他文件时，需要识别并再次调用 read_skill_md：") - lines.append(" - **引用模板识别**：注意技能内容中形如 `` 或自然语言式的引用声明（如\"详见 examples.md\"、\"请参考 reference/api_doc\"）") - lines.append(" - **自动补全**：发现引用后，尝试读取被引用的文件获取更多信息") + lines.append("6. **引用场景处理**：技能中的引用既可以通过 XML 标签表达，也可以通过下列 Markdown 语法表达，**功能上完全等同**，必须都能识别：") + lines.append(" - **引用模板识别**：") + lines.append(" - XML 形式：``") + lines.append(" - 单个反引号形式：`` `examples.md` ``、`` `reference/api_doc` ``") + lines.append(" - 三重反引号代码块形式：`` ```examples.md``` ``（当代码块内仅有单行路径时）") + lines.append(" - 自然语言式的引用声明（如\"详见 examples.md\"、\"请参考 reference/api_doc\"）") + lines.append(" - **自动补全**：发现引用后，按需调用 `read_skill_md(\"skill_name\", [\"<路径>\"])` 读取被引用的文件，**不要一次性全部读取**，应基于当前任务判断哪些文件确实必要。") lines.append(" - **示例**：") lines.append(" ") lines.append(" # 技能内容提示\"请参考 examples.md 获取详细示例\"") @@ -235,7 +245,11 @@ def _format_skills_description( lines.append("") lines.append("3. **Follow Skill Guide**: After skill content is injected, strictly follow its steps. Do not skip steps or replace with your own code.") lines.append("") - lines.append("4. **Execute Skill Script**: If the skill guide references additional scripts (like ``), call:") + lines.append("4. **Execute Skill Script**: Skill-internal references (for both documentation and scripts) may be declared with **any of the following equivalent forms** - treat them all the same way: ") + lines.append(" - XML tags: ``, ``") + lines.append(" - Single inline backticks: `` `scripts/analyze.py` ``, `` `reference/api_doc` ``") + lines.append(" - Triple-backtick fenced code blocks: `` ```scripts/analyze.py``` `` (only when the block body is a single path line)") + lines.append(" When calling `run_skill_script`, the `script_path` is **always resolved relative to the skill's root directory** (this is the platform behaviour, not the agent's CWD). Common forms:") lines.append(" ") lines.append(" result = run_skill_script(\"skill_name\", \"script_path\")") lines.append(" print(result)") @@ -246,13 +260,17 @@ def _format_skills_description( lines.append(" result = run_skill_script(\"skill_name\", \"script_path\", \"--param1 value1 --flag\")") lines.append(" print(result)") lines.append(" ") - lines.append(" Note: Only execute script paths explicitly declared in the skill guide. Never construct paths yourself.") + lines.append(" Note: Only execute script paths explicitly declared in the skill guide. Never construct paths yourself. Do not treat the script as relative to the current working directory (CWD), and never pass absolute paths. When the requested script cannot be found, the error returned by `run_skill_script` lists the scripts that *do* exist under the skill root - use it to correct the path.") lines.append("") lines.append("5. **Integrate Output**: Generate the final answer based on the skill guide's output format and script execution results.") lines.append("") - lines.append("6. **Handle References**: When the skill content has reference markers or needs to reference other files, identify and call read_skill_md again:") - lines.append(" - **Reference template recognition**: Look for patterns like `` or natural-language references (\"see examples.md\", \"refer to reference/api_doc\")") - lines.append(" - **Auto-complete**: After discovering a reference, try reading the referenced file for more info") + lines.append("6. **Handle References**: Skill-internal references can be expressed using XML tags, markdown forms, or natural-language hints. All three are **functionally equivalent** and must be recognised: ") + lines.append(" - **Reference patterns to recognise**:") + lines.append(" - XML tag form: ``") + lines.append(" - Single inline backtick form: `` `examples.md` ``, `` `reference/api_doc` ``") + lines.append(" - Triple-backtick fenced block form: `` ```examples.md``` `` (only when the block body is a single path line)") + lines.append(" - Natural-language references (\"see examples.md\", \"refer to reference/api_doc\")") + lines.append(" - **Auto-complete**: After discovering a reference, call `read_skill_md(\"skill_name\", [\"\"])` only for the files you actually need. Do **not** load every referenced file blindly - decide based on the current task which references matter.") lines.append(" - **Example**:") lines.append(" ") lines.append(" # Skill content says \"see examples.md for detailed examples\"") @@ -487,7 +505,11 @@ def _format_skills_usage_requirements( lines.append("### 技能使用要求") lines.append("1. **技能优先**：如果用户请求匹配了某个技能的 description，必须先调用 `read_skill_md()` 加载技能指南，再按指南执行。不得跳过技能自行编写代码解决。") lines.append("2. **忠实执行**：读取技能内容后，严格按技能指南中的步骤操作。不要自行修改流程、跳过步骤或用通用代码替代技能定义的流程。") - lines.append("3. **脚本调用规范**：只使用 `run_skill_script` 工具执行技能指南中明确要求的脚本。传入的 `skill_name` 和 `script_path` 必须与技能指南中的声明完全一致，不要自行拼接或猜测路径。如果需要附加参数，将参数以命令行字符串形式传递给`run_skill_script`。") + lines.append("3. **脚本调用规范**：") + lines.append(" - 路径声明识别：技能指南中的脚本路径既可以以 XML 标签（``）声明，也允许以等价的 Markdown 形式（`` `scripts/foo.py` `` 单反引号，或 `` ```scripts/foo.py``` `` 三重反引号代码块）声明。模型必须把这些形式都识别为脚本路径。") + lines.append(" - 路径解析：`run_skill_script` 的 `script_path` 参数**始终相对于技能根目录**解析，平台不会基于当前工作目录或绝对路径查找。请直接复用技能指南中的声明字符串，不要自行拼接或猜测路径。") + lines.append(" - 参数传递：如果需要附加参数，将参数以命令行字符串形式传递给 `run_skill_script`。") + lines.append(" - 错误回退：脚本不存在时，`run_skill_script` 返回的错误信息会列出当前技能根目录下可用的脚本路径，请据此修正。") lines.append("4. **失败回退**：如果 `read_skill_md` 返回错误或 `run_skill_script` 执行失败，向用户说明情况，并尝试用通用推理模式提供替代方案。") lines.append("5. **技能组合**：如果一个任务需要多个技能配合，按逻辑依赖顺序依次加载和执行，前一个技能的输出可作为后一个技能的输入。") else: @@ -496,7 +518,11 @@ def _format_skills_usage_requirements( lines.append("### Skill Usage Requirements") lines.append("1. **Skill Priority**: If a user request matches a skill's description, you must first call `read_skill_md()` to load the skill guide, then execute per the guide. Do not skip skills and write your own code.") lines.append("2. **Faithful Execution**: After reading skill content, strictly follow the skill guide's steps. Do not modify the flow, skip steps, or replace with generic code.") - lines.append("3. **Script Calling Specification**: Only use `run_skill_script` to execute scripts explicitly required in the skill guide. The `skill_name` and `script_path` must match the skill guide's declaration exactly. Do not construct or guess paths. For extra params, pass them as a command-line string to `run_skill_script`.") + lines.append("3. **Script Calling Specification**:") + lines.append(" - **Path declaration recognition**: A script path inside the skill guide may be declared using XML tags (``) OR via the equivalent markdown forms - single inline backticks like `` `scripts/foo.py` ``, or triple-backtick fenced blocks like `` ```scripts/foo.py``` ``. Treat all three as the same kind of declaration.") + lines.append(" - **Path resolution**: The `script_path` argument of `run_skill_script` is **always resolved relative to the skill's root directory**. The platform will not look in the current working directory and will not follow absolute paths. Pass the path verbatim from the skill guide - never construct or guess a path.") + lines.append(" - **Parameter passing**: For extra parameters, pass them as a command-line string to `run_skill_script`.") + lines.append(" - **Error fallback**: When the script cannot be located, the error returned by `run_skill_script` lists the scripts that *do* exist under the skill root - use it to correct the path.") lines.append("4. **Failure Fallback**: If `read_skill_md` returns an error or `run_skill_script` fails, explain to the user and try to provide an alternative via general reasoning mode.") lines.append("5. **Skill Combination**: If a task needs multiple skills, load and execute in logical dependency order. The output of one skill can be input to the next.") diff --git a/backend/utils/evaluation_set_excel_utils.py b/backend/utils/evaluation_set_excel_utils.py new file mode 100644 index 000000000..c9f7de4b6 --- /dev/null +++ b/backend/utils/evaluation_set_excel_utils.py @@ -0,0 +1,214 @@ +import io +from typing import List, Optional, Dict, Any + +import xlrd +from openpyxl import Workbook, load_workbook +from openpyxl.styles import Font, PatternFill + + +REQUIRED_HEADERS = ["query", "answer"] +OPTIONAL_HEADERS = ["case_id"] +ALL_HEADERS = REQUIRED_HEADERS + OPTIONAL_HEADERS + + +def _normalize_header(v: Any) -> str: + if v is None: + return "" + return str(v).strip().lower() + + +def build_evaluation_set_excel_template_bytes() -> bytes: + """Build a downloadable XLSX template. + + Column order puts required fields first. + """ + wb = Workbook() + ws = wb.active + ws.title = "evaluation_cases" + + headers = [ + "序号", + "问题*", + "答案*", + ] + + ws.append(headers) + ws.freeze_panes = "A2" + + # Styling + bold = Font(bold=True) + required_fill = PatternFill(start_color="FFF7E6", end_color="FFF7E6", fill_type="solid") + + for col_idx, title in enumerate(headers, start=1): + cell = ws.cell(row=1, column=col_idx) + cell.font = bold + if title.endswith("*"): + cell.fill = required_fill + + # Column widths + ws.column_dimensions["A"].width = 12 # 序号 (case_id) + ws.column_dimensions["B"].width = 50 # 问题 (query) + ws.column_dimensions["C"].width = 50 # 答案 (answer) + + # Example rows + ws.append([ + "c1", + "1+1等于几？", + "2", + ]) + ws.append([ + "c2", + "中国首都是哪里？", + "北京", + ]) + + out = io.BytesIO() + wb.save(out) + return out.getvalue() + + +def parse_evaluation_cases_from_excel(filename: str, raw: bytes) -> List[Dict[str, Any]]: + """Parse evaluation cases from .xlsx or .xls. + + Expected headers (case-insensitive): query, answer, case_id (or aliases). + A trailing '*' in header is allowed (e.g. query*). + Chinese aliases are also recognized so the template round-trip works: + 序号 / case_id / case_id -> case_id + 问题 / query / query -> query + 答案 / answer / answer -> answer + + Returns normalized case dicts compatible with insert_evaluation_set_cases. + """ + + HEADER_ALIASES = { + "query": "query", + "问题": "query", + "answer": "answer", + "答案": "answer", + "case_id": "case_id", + "序号": "case_id", + "caseid": "case_id", + "id": "case_id", + } + + def _canonical_header(v: Any) -> Optional[str]: + key = _normalize_header(v).rstrip("*") + if not key: + return None + return HEADER_ALIASES.get(key) + + lower_name = (filename or "").lower() + if lower_name.endswith(".xlsx"): + wb = load_workbook(io.BytesIO(raw), read_only=True, data_only=True) + ws = wb.active + rows = ws.iter_rows(values_only=True) + header_row = next(rows, None) + if not header_row: + raise ValueError("Excel contains no header row") + + header_map: Dict[str, int] = {} + for idx, v in enumerate(header_row): + canon = _canonical_header(v) + if canon: + header_map[canon] = idx + + for h in REQUIRED_HEADERS: + if h not in header_map: + raise ValueError(f"Missing required column: {h}") + + cases: List[Dict[str, Any]] = [] + for excel_row_idx, row in enumerate(rows, start=2): + if row is None: + continue + + def get_col(col: str) -> Optional[str]: + if col not in header_map: + return None + v = row[header_map[col]] if header_map[col] < len(row) else None + if v is None: + return None + s = str(v).strip() + return s if s != "" else None + + query = get_col("query") + answer = get_col("answer") + case_id = get_col("case_id") + + # Skip fully empty rows + if not any([query, answer, case_id]): + continue + + if not query: + raise ValueError(f"Row {excel_row_idx}: 问题 is required") + if not answer: + raise ValueError(f"Row {excel_row_idx}: 答案 is required") + + normalized: Dict[str, Any] = { + "case_id": case_id, + "inputs": {"query": query}, + "label": {"answer": answer}, + "order_no": len(cases), + } + cases.append(normalized) + + if not cases: + raise ValueError("Excel contains no cases") + + return cases + + if lower_name.endswith(".xls"): + book = xlrd.open_workbook(file_contents=raw) + sheet = book.sheet_by_index(0) + if sheet.nrows < 1: + raise ValueError("Excel contains no header row") + + header_row = sheet.row_values(0) + header_map: Dict[str, int] = {} + for idx, v in enumerate(header_row): + canon = _canonical_header(v) + if canon: + header_map[canon] = idx + + for h in REQUIRED_HEADERS: + if h not in header_map: + raise ValueError(f"Missing required column: {h}") + + cases: List[Dict[str, Any]] = [] + for r in range(1, sheet.nrows): + excel_row_idx = r + 1 + + def get_cell(col: str) -> Optional[str]: + if col not in header_map: + return None + v = sheet.cell_value(r, header_map[col]) + if v is None: + return None + s = str(v).strip() + return s if s != "" else None + + query = get_cell("query") + answer = get_cell("answer") + case_id = get_cell("case_id") + + if not any([query, answer, case_id]): + continue + + if not query: + raise ValueError(f"Row {excel_row_idx}: 问题 is required") + if not answer: + raise ValueError(f"Row {excel_row_idx}: 答案 is required") + + normalized: Dict[str, Any] = { + "case_id": case_id, + "inputs": {"query": query}, + "label": {"answer": answer}, + "order_no": len(cases), + } + cases.append(normalized) + + if not cases: + raise ValueError("Excel contains no cases") + + return cases + + raise ValueError("Unsupported file type. Please upload .xlsx or .xls") diff --git a/backend/utils/skill_params_utils.py b/backend/utils/skill_params_utils.py index 404e16ccb..f40a218c4 100644 --- a/backend/utils/skill_params_utils.py +++ b/backend/utils/skill_params_utils.py @@ -16,7 +16,7 @@ def split_string_inline_comment(s: str) -> Tuple[str, Optional[str]]: idx = s.find(" # ") if idx == -1: return s, None - return s[:idx].rstrip(), s[idx + 3 :].strip() or None + return s[:idx].rstrip(), s[idx + 3:].strip() or None def strip_params_comments_for_db(obj: Any) -> Any: diff --git a/build.sh b/build.sh new file mode 100644 index 000000000..6b80b8938 --- /dev/null +++ b/build.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_COMMON="$SCRIPT_DIR/deploy/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi +[ -n "${DEPLOYMENT_LANGUAGE:-}" ] || DEPLOYMENT_LANGUAGE="en" + +usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + cat <<'USAGE' +用法： + bash build.sh [镜像构建选项] + bash build.sh --package [离线包构建选项] + +镜像构建会转发到：deploy/images/build.sh +离线包构建会转发到：deploy/offline/build_offline_package.sh + +常用示例： + bash build.sh --main --version latest --dry-run + bash build.sh --package --version latest --target docker --dry-run + +更多选项： + bash deploy/images/build.sh --help + bash deploy/offline/build_offline_package.sh --help +USAGE + return + fi + + cat <<'USAGE' +Usage: + bash build.sh [image build options] + bash build.sh --package [offline package options] + +Image builds are forwarded to: deploy/images/build.sh +Offline package builds are forwarded to: deploy/offline/build_offline_package.sh + +Examples: + bash build.sh --main --version latest --dry-run + bash build.sh --package --version latest --target docker --dry-run + +More options: + bash deploy/images/build.sh --help + bash deploy/offline/build_offline_package.sh --help +USAGE +} + +case "${1:-}" in + --help|-h) + usage + exit 0 + ;; +esac + +if [ "${1:-}" = "--package" ]; then + shift + exec bash "$SCRIPT_DIR/deploy/offline/build_offline_package.sh" "$@" +fi + +exec bash "$SCRIPT_DIR/deploy/images/build.sh" "$@" diff --git a/deploy.sh b/deploy.sh index a5a013f2b..aad8dae84 100755 --- a/deploy.sh +++ b/deploy.sh @@ -3,19 +3,81 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_COMMON="$SCRIPT_DIR/deploy/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi +[ -n "${DEPLOYMENT_LANGUAGE:-}" ] || DEPLOYMENT_LANGUAGE="en" +DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE="" usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + cat <<'USAGE' +用法： + bash deploy.sh [--load-images] [--push-images] [--image-registry-prefix PREFIX] [--config|--defaults] docker [Docker 部署选项] + bash deploy.sh [--load-images] [--push-images] [--image-registry-prefix PREFIX] [--config|--defaults] k8s [K8s 部署选项] + +USAGE + if [ "$DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE" = "defaults" ]; then + cat <<'USAGE' +此离线包入口默认复用保存配置或内置默认值部署，不进入交互界面。 +添加 --config 可进入交互式部署配置界面。 +实现：deploy/deploy.sh + +USAGE + else + cat <<'USAGE' +此根入口只转发到目标专用部署脚本。 +实现：deploy/deploy.sh + +USAGE + fi + cat <<'USAGE' +选项： + --load-images 部署前从 ./images 加载 Docker 镜像 tar 文件。 + 默认关闭。 + --push-images 部署前调用 push-images.sh 推送镜像。 + --image-registry-prefix PREFIX + 镜像仓库前缀，例如 registry.example.com/nexent。 + 使用 --push-images 且未传入时会交互询问。 + --config 进入交互式部署配置界面。 + --defaults 复用保存配置或内置默认值，跳过交互界面。 +USAGE + return + fi + cat <<'USAGE' Usage: - bash deploy.sh [--load-images] docker [docker deploy options] - bash deploy.sh [--load-images] k8s [k8s deploy options] + bash deploy.sh [--load-images] [--push-images] [--image-registry-prefix PREFIX] [--config|--defaults] docker [docker deploy options] + bash deploy.sh [--load-images] [--push-images] [--image-registry-prefix PREFIX] [--config|--defaults] k8s [k8s deploy options] +USAGE + if [ "$DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE" = "defaults" ]; then + cat <<'USAGE' +This offline entrypoint deploys with saved configuration or built-in defaults by default. +Add --config to open the interactive deployment configuration. +Implementation: deploy/deploy.sh + +USAGE + else + cat <<'USAGE' This root entrypoint only forwards to the target-specific deploy script. Implementation: deploy/deploy.sh +USAGE + fi + cat <<'USAGE' Options: --load-images Load Docker image tar files from ./images before deploying. Defaults to off. + --push-images Run push-images.sh before deploying. + --image-registry-prefix PREFIX + Image registry prefix, e.g. registry.example.com/nexent. + Prompts when --push-images is used and no prefix is provided. + --config Open the interactive deployment configuration. + --defaults Use saved configuration or built-in defaults and skip TUI. USAGE } @@ -25,6 +87,9 @@ if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ] || [ $# -eq 0 ]; then fi LOAD_IMAGES="false" +PUSH_IMAGES="false" +IMAGE_REGISTRY_PREFIX="${IMAGE_REGISTRY_PREFIX:-}" +DEPLOY_CONFIG_MODE="$DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE" FORWARD_ARGS=() while [ $# -gt 0 ]; do @@ -33,6 +98,30 @@ while [ $# -gt 0 ]; do LOAD_IMAGES="true" shift ;; + --push-images) + PUSH_IMAGES="true" + shift + ;; + --image-registry-prefix|--registry-prefix|--image-registry) + if [ $# -lt 2 ]; then + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：$1 需要一个值" >&2 + else + echo "Error: $1 requires a value" >&2 + fi + exit 1 + fi + IMAGE_REGISTRY_PREFIX="$2" + shift 2 + ;; + --config) + DEPLOY_CONFIG_MODE="tui" + shift + ;; + --defaults) + DEPLOY_CONFIG_MODE="defaults" + shift + ;; *) FORWARD_ARGS+=("$1") shift @@ -40,18 +129,94 @@ while [ $# -gt 0 ]; do esac done +normalize_image_registry_prefix() { + if declare -F deployment_normalize_image_registry_prefix_value >/dev/null 2>&1; then + deployment_normalize_image_registry_prefix_value "$1" + return 0 + fi + + local prefix="$1" + prefix="${prefix#"${prefix%%[![:space:]]*}"}" + prefix="${prefix%"${prefix##*[![:space:]]}"}" + prefix="${prefix#http://}" + prefix="${prefix#https://}" + while [[ "$prefix" == */ ]]; do + prefix="${prefix%/}" + done + printf '%s' "$prefix" +} + +require_image_registry_prefix() { + if [ -z "$IMAGE_REGISTRY_PREFIX" ]; then + if [ -t 0 ]; then + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + read -r -p "请输入镜像仓库前缀（例如 registry.example.com/nexent）: " IMAGE_REGISTRY_PREFIX + else + read -r -p "Enter image registry prefix (e.g. registry.example.com/nexent): " IMAGE_REGISTRY_PREFIX + fi + else + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：--push-images 需要 --image-registry-prefix，或设置 IMAGE_REGISTRY_PREFIX。" >&2 + else + echo "Error: --push-images requires --image-registry-prefix or IMAGE_REGISTRY_PREFIX." >&2 + fi + exit 1 + fi + fi + + IMAGE_REGISTRY_PREFIX="$(normalize_image_registry_prefix "$IMAGE_REGISTRY_PREFIX")" + if [ -z "$IMAGE_REGISTRY_PREFIX" ]; then + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：镜像仓库前缀不能为空。" >&2 + else + echo "Error: image registry prefix cannot be empty." >&2 + fi + exit 1 + fi +} + if [ "${#FORWARD_ARGS[@]}" -eq 0 ]; then usage exit 0 fi -if [ "$LOAD_IMAGES" = "true" ]; then +if [ "$LOAD_IMAGES" = "true" ] && [ "$PUSH_IMAGES" != "true" ]; then LOAD_SCRIPT="$SCRIPT_DIR/load-images.sh" if [ ! -f "$LOAD_SCRIPT" ]; then - echo "Error: --load-images requires $LOAD_SCRIPT" >&2 + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：--load-images 需要 $LOAD_SCRIPT" >&2 + else + echo "Error: --load-images requires $LOAD_SCRIPT" >&2 + fi exit 1 fi bash "$LOAD_SCRIPT" fi +if [ "$PUSH_IMAGES" = "true" ]; then + PUSH_SCRIPT="$SCRIPT_DIR/push-images.sh" + if [ ! -f "$PUSH_SCRIPT" ]; then + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：--push-images 需要 $PUSH_SCRIPT" >&2 + else + echo "Error: --push-images requires $PUSH_SCRIPT" >&2 + fi + exit 1 + fi + + require_image_registry_prefix + bash "$PUSH_SCRIPT" --image-registry-prefix "$IMAGE_REGISTRY_PREFIX" --load-images +fi + +if [ -n "$IMAGE_REGISTRY_PREFIX" ]; then + IMAGE_REGISTRY_PREFIX="$(normalize_image_registry_prefix "$IMAGE_REGISTRY_PREFIX")" + if [ -n "$IMAGE_REGISTRY_PREFIX" ]; then + FORWARD_ARGS+=("--image-registry-prefix" "$IMAGE_REGISTRY_PREFIX") + fi +fi + +if [ -n "$DEPLOY_CONFIG_MODE" ]; then + NEXENT_DEPLOY_CONFIG_MODE="$DEPLOY_CONFIG_MODE" exec bash "$SCRIPT_DIR/deploy/deploy.sh" "${FORWARD_ARGS[@]}" +fi + exec bash "$SCRIPT_DIR/deploy/deploy.sh" "${FORWARD_ARGS[@]}" diff --git a/deploy/common/common.sh b/deploy/common/common.sh index 47d9097e1..b9649e77a 100755 --- a/deploy/common/common.sh +++ b/deploy/common/common.sh @@ -9,25 +9,26 @@ DEPLOYMENT_COMPONENTS_DEFAULT="infrastructure,application,data-process,supabase" DEPLOYMENT_PORT_POLICY_DEFAULT="development" DEPLOYMENT_IMAGE_SOURCE_DEFAULT="general" DEPLOYMENT_REGISTRY_PROFILE_DEFAULT="general" +DEPLOYMENT_IMAGE_REGISTRY_PREFIX_DEFAULT="" DEPLOYMENT_MONITORING_PROVIDER_DEFAULT="otlp" DEPLOYMENT_COMPONENTS="" DEPLOYMENT_PORT_POLICY="" DEPLOYMENT_IMAGE_SOURCE="" DEPLOYMENT_REGISTRY_PROFILE="" +DEPLOYMENT_IMAGE_REGISTRY_PREFIX="" DEPLOYMENT_APP_VERSION="" DEPLOYMENT_MONITORING_PROVIDER="" -DEPLOYMENT_CONFIG_PATH="" DEPLOYMENT_USE_LOCAL_CONFIG="false" DEPLOYMENT_RECONFIGURE="false" DEPLOYMENT_LOCAL_CONFIG_PATH="" DEPLOYMENT_SELECTED_DOCKER_SERVICES="" DEPLOYMENT_SELECTED_HELM_CHARTS="" DEPLOYMENT_LOADED_SCHEMA_VERSION="" -DEPLOYMENT_LOADED_APP_VERSION="" DEPLOYMENT_CONFIG_FILE_LOADED="false" DEPLOYMENT_DOCKER_PORTS="" DEPLOYMENT_ROOT_ENV="" +DEPLOYMENT_LANGUAGE="${DEPLOYMENT_LANGUAGE:-}" deployment_component_list="infrastructure application data-process supabase terminal monitoring" deployment_port_policy_list="development production" @@ -35,6 +36,229 @@ deployment_image_source_list="general mainland local-latest" deployment_registry_profile_list="general mainland" deployment_monitoring_provider_list="otlp phoenix langfuse langsmith grafana zipkin" +deployment_locale_value_is_zh() { + local value="$1" + value="${value%%:*}" + value="${value%%.*}" + value="${value//-/_}" + value="$(printf '%s' "$value" | LC_ALL=C tr '[:upper:]' '[:lower:]')" + + case "$value" in + zh|zh_*|cn|chinese) + return 0 + ;; + esac + return 1 +} + +deployment_detect_language() { + local explicit="${DEPLOYMENT_LANG:-}" + explicit="$(printf '%s' "$explicit" | LC_ALL=C tr '[:upper:]' '[:lower:]')" + case "$explicit" in + zh|zh_*|zh-*|cn|chinese) + printf 'zh\n' + return 0 + ;; + en|en_*|en-*|c|posix) + printf 'en\n' + return 0 + ;; + esac + + if deployment_locale_value_is_zh "${LC_ALL:-}"; then + printf 'zh\n' + elif deployment_locale_value_is_zh "${LC_MESSAGES:-}"; then + printf 'zh\n' + elif deployment_locale_value_is_zh "${LANGUAGE:-}"; then + printf 'zh\n' + elif deployment_locale_value_is_zh "${LANG:-}"; then + printf 'zh\n' + else + printf 'en\n' + fi +} + +deployment_init_language() { + if [ -z "${DEPLOYMENT_LANGUAGE:-}" ]; then + DEPLOYMENT_LANGUAGE="$(deployment_detect_language)" + fi + + case "$DEPLOYMENT_LANGUAGE" in + zh|zh_*|zh-*|cn|chinese) + DEPLOYMENT_LANGUAGE="zh" + ;; + *) + DEPLOYMENT_LANGUAGE="en" + ;; + esac + export -n DEPLOYMENT_LANGUAGE 2>/dev/null || true +} + +deployment_language() { + printf '%s\n' "${DEPLOYMENT_LANGUAGE:-en}" +} + +deployment_init_language + +deployment_i18n_format() { + local lang="$1" + local key="$2" + + if [ "$lang" = "zh" ]; then + case "$key" in + password.validation) printf '密码至少 8 位，并且包含大写字母、小写字母和数字。' ;; + env.created_from_docker) printf '✅ 已从 docker/.env 创建 deploy/env/.env' ;; + env.created_from_example) printf '✅ 已从 deploy/env/.env.example 创建 deploy/env/.env' ;; + env.root_missing) printf '未找到 deploy/env/.env，且没有可用的 docker/.env 或 deploy/env/.env.example 模板' ;; + validation.local_config_schema) printf '%s' '本地配置 schemaVersion %s 与 %s 不兼容。请使用 --reconfigure 重新配置。' ;; + validation.unknown_component) printf '%s' '未知部署组件：%s' ;; + validation.unsupported_port_policy) printf '%s' '不支持的端口策略：%s。可用值：development 或 production。' ;; + validation.unsupported_image_source) printf '%s' '不支持的镜像源：%s。可用值：general、mainland 或 local-latest。' ;; + validation.unsupported_registry_profile) printf '%s' '不支持的 registry profile：%s' ;; + validation.unsupported_image_registry_prefix) printf '%s' '不支持的镜像仓库前缀：%s。请使用 registry.example.com/project 格式，不要包含空格。' ;; + validation.unsupported_monitoring_provider) printf '%s' '不支持的监控 provider：%s' ;; + tui.cancelled) printf '已取消部署配置。' ;; + tui.components.title) printf '选择部署组件' ;; + tui.components.subtitle) printf '选择要安装的服务组。infrastructure 为必选项，不能禁用。' ;; + tui.components.help) printf '使用 Up/Down 或 j/k 移动，空格切换，Enter 确认，q 退出。' ;; + tui.component.infrastructure) printf '必需核心依赖：Elasticsearch、PostgreSQL、Redis、MinIO' ;; + tui.component.application) printf 'Nexent 应用服务：config、runtime、MCP、northbound API、web UI' ;; + tui.component.data_process) printf '后台文件解析、索引和知识处理 Worker' ;; + tui.component.supabase) printf '用户、租户、登录、邀请和权限服务' ;; + tui.component.terminal) printf '终端工具使用的 OpenSSH 容器' ;; + tui.component.monitoring) printf 'OpenTelemetry Collector 和可选链路追踪看板' ;; + tui.monitoring.title) printf '选择监控 provider' ;; + tui.monitoring.subtitle) printf '仅在选择 monitoring 组件时使用。' ;; + tui.monitoring.description) printf 'Provider 决定 OpenTelemetry traces 的存储和查看位置。' ;; + tui.radio.help) printf '使用 Up/Down 或 j/k 移动，Enter 确认，b/Backspace 返回，q 退出。' ;; + tui.monitoring.otlp) printf '仅 Collector；用于转发到外部 OTLP 后端' ;; + tui.monitoring.phoenix) printf '本地 Phoenix UI，用于查看 LLM traces 和 span' ;; + tui.monitoring.langfuse) printf '本地自托管 Langfuse；生产环境请替换默认密钥' ;; + tui.monitoring.langsmith) printf '转发 traces 到托管 LangSmith；需要 LANGSMITH_API_KEY' ;; + tui.monitoring.grafana) printf '本地 Grafana + Tempo traces 看板' ;; + tui.monitoring.zipkin) printf '本地 Zipkin trace 浏览 UI' ;; + tui.port.title) printf '选择端口策略' ;; + tui.port.subtitle) printf '控制哪些服务端口暴露到主机或集群节点。' ;; + tui.port.description) printf '本地调试选择 development；更小外部暴露面选择 production。' ;; + tui.port.development) printf '暴露 Web 和调试/内部服务端口，便于本地排查' ;; + tui.port.production) printf '只暴露生产入口端口，内部服务保持私有' ;; + tui.image.title) printf '选择镜像源' ;; + tui.image.description) printf '每个选项展示将使用的后端镜像 tag 示例。' ;; + image_build.detail.main) printf '后端 API 服务' ;; + image_build.detail.web) printf 'Next.js 前端' ;; + image_build.detail.data_process) printf '文档解析和向量化 Worker' ;; + image_build.detail.mcp) printf 'MCP 代理镜像' ;; + image_build.detail.terminal) printf 'OpenSSH 终端工具镜像' ;; + image_build.detail.docs) printf 'VitePress 文档站点' ;; + local_config.found) printf '%s' '发现已有部署配置：%s' ;; + local_config.choose) printf '请选择如何处理已保存的部署选项：' ;; + local_config.use) printf ' 1) 使用本地配置 - 跳过菜单，复用已保存的组件、端口策略、镜像源、镜像仓库前缀和监控 provider。' ;; + local_config.reconfigure) printf ' 2) 重新配置 - 将已保存的值作为默认值，并显示菜单供修改。' ;; + local_config.reconfigure_hint) printf ' 启用/禁用监控、切换 provider 或调整部署范围时请选择此项。' ;; + prompt.choose_1_2) printf '请选择 [1/2]（默认：1）：' ;; + summary.components) printf '%s' '部署组件：%s' ;; + summary.port_policy) printf '%s' '端口策略：%s' ;; + summary.image_source) printf '%s' '镜像源：%s' ;; + summary.image_registry_prefix) printf '%s' '镜像仓库前缀：%s' ;; + summary.monitoring_provider) printf '%s' '监控 provider：%s' ;; + summary.docker_services) printf '%s' 'Docker 服务：%s' ;; + summary.docker_ports) printf '%s' 'Docker 暴露端口：%s' ;; + summary.helm_charts) printf '%s' 'Helm charts：%s' ;; + *) return 1 ;; + esac + else + case "$key" in + password.validation) printf 'Password must be at least 8 characters and include uppercase letters, lowercase letters, and numbers.' ;; + env.created_from_docker) printf '✅ Created deploy/env/.env from docker/.env' ;; + env.created_from_example) printf '✅ Created deploy/env/.env from deploy/env/.env.example' ;; + env.root_missing) printf 'deploy/env/.env not found and no docker/.env or deploy/env/.env.example template is available' ;; + validation.local_config_schema) printf '%s' 'Local config schemaVersion %s is incompatible with %s. Re-run with --reconfigure.' ;; + validation.unknown_component) printf '%s' 'Unknown deployment component: %s' ;; + validation.unsupported_port_policy) printf '%s' 'Unsupported port policy: %s. Use development or production.' ;; + validation.unsupported_image_source) printf '%s' 'Unsupported image source: %s. Use general, mainland, or local-latest.' ;; + validation.unsupported_registry_profile) printf '%s' 'Unsupported registry profile: %s' ;; + validation.unsupported_image_registry_prefix) printf '%s' 'Unsupported image registry prefix: %s. Use registry.example.com/project format without spaces.' ;; + validation.unsupported_monitoring_provider) printf '%s' 'Unsupported monitoring provider: %s' ;; + tui.cancelled) printf 'Deployment configuration cancelled.' ;; + tui.components.title) printf 'Select deployment components' ;; + tui.components.subtitle) printf 'Choose which service groups to install. infrastructure is required and cannot be disabled.' ;; + tui.components.help) printf 'Use Up/Down or j/k to move, Space to toggle, Enter to confirm, q to quit.' ;; + tui.component.infrastructure) printf 'required core dependencies: Elasticsearch, PostgreSQL, Redis, MinIO' ;; + tui.component.application) printf 'Nexent app services: config, runtime, MCP, northbound API, web UI' ;; + tui.component.data_process) printf 'background file parsing, indexing, and knowledge processing workers' ;; + tui.component.supabase) printf 'user, tenant, login, invitation, and permission services' ;; + tui.component.terminal) printf 'OpenSSH container used by the terminal tool' ;; + tui.component.monitoring) printf 'OpenTelemetry collector and optional tracing dashboard' ;; + tui.monitoring.title) printf 'Select monitoring provider' ;; + tui.monitoring.subtitle) printf 'This is used only when the monitoring component is selected.' ;; + tui.monitoring.description) printf 'Provider controls where OpenTelemetry traces are stored and viewed.' ;; + tui.radio.help) printf 'Use Up/Down or j/k to move, Enter to confirm, b/Backspace to go back, q to quit.' ;; + tui.monitoring.otlp) printf 'collector only; use this when forwarding to an external OTLP backend' ;; + tui.monitoring.phoenix) printf 'local Phoenix UI for LLM traces and span inspection' ;; + tui.monitoring.langfuse) printf 'local self-hosted Langfuse stack; replace default secrets for production' ;; + tui.monitoring.langsmith) printf 'forward traces to hosted LangSmith; requires LANGSMITH_API_KEY' ;; + tui.monitoring.grafana) printf 'local Grafana + Tempo dashboard for traces' ;; + tui.monitoring.zipkin) printf 'local Zipkin UI for trace browsing' ;; + tui.port.title) printf 'Select port policy' ;; + tui.port.subtitle) printf 'This controls which service ports are exposed on the host or cluster node.' ;; + tui.port.description) printf 'Choose development for local debugging; choose production for a smaller external surface.' ;; + tui.port.development) printf 'publish web plus debug/internal service ports for local troubleshooting' ;; + tui.port.production) printf 'publish only production entry ports; keep internal services private' ;; + tui.image.title) printf 'Select image source' ;; + tui.image.description) printf 'Each option shows the backend image tag pattern that will be used.' ;; + image_build.detail.main) printf 'backend API service' ;; + image_build.detail.web) printf 'Next.js frontend' ;; + image_build.detail.data_process) printf 'document parsing and vectorization worker' ;; + image_build.detail.mcp) printf 'MCP proxy image' ;; + image_build.detail.terminal) printf 'OpenSSH terminal tool image' ;; + image_build.detail.docs) printf 'VitePress documentation site' ;; + local_config.found) printf '%s' 'Existing deployment config found: %s' ;; + local_config.choose) printf 'Choose how to handle saved deployment options:' ;; + local_config.use) printf ' 1) Use local config - skip the menus and reuse the saved components, port policy, image source, image registry prefix, and monitoring provider.' ;; + local_config.reconfigure) printf ' 2) Reconfigure - load the saved values as defaults, then show the menus so you can change them.' ;; + local_config.reconfigure_hint) printf ' Choose this option when enabling or disabling monitoring, switching providers, or changing deployment scope.' ;; + prompt.choose_1_2) printf 'Choose [1/2] (default: 1): ' ;; + summary.components) printf '%s' 'Deployment components: %s' ;; + summary.port_policy) printf '%s' 'Port policy: %s' ;; + summary.image_source) printf '%s' 'Image source: %s' ;; + summary.image_registry_prefix) printf '%s' 'Image registry prefix: %s' ;; + summary.monitoring_provider) printf '%s' 'Monitoring provider: %s' ;; + summary.docker_services) printf '%s' 'Docker services: %s' ;; + summary.docker_ports) printf '%s' 'Docker published ports: %s' ;; + summary.helm_charts) printf '%s' 'Helm charts: %s' ;; + *) return 1 ;; + esac + fi +} + +deployment_i18n() { + local key="$1" + shift || true + local lang + local format + lang="${DEPLOYMENT_LANGUAGE:-en}" + format="$(deployment_i18n_format "$lang" "$key" || true)" + if [ -z "$format" ]; then + printf '%s\n' "$key" + return 0 + fi + printf "$format\n" "$@" +} + +deployment_prompt() { + local key="$1" + shift || true + local lang + local format + lang="${DEPLOYMENT_LANGUAGE:-en}" + format="$(deployment_i18n_format "$lang" "$key" || true)" + if [ -z "$format" ]; then + printf '%s' "$key" + return 0 + fi + printf "$format" "$@" +} + deployment_log() { printf '%s\n' "$*" } @@ -82,7 +306,7 @@ deployment_validate_password() { } deployment_password_validation_message() { - printf '%s\n' "Password must be at least 8 characters and include uppercase letters, lowercase letters, and numbers." + deployment_i18n password.validation } deployment_ensure_root_env() { @@ -103,17 +327,17 @@ deployment_ensure_root_env() { if [ -f "$docker_env" ]; then cp "$docker_env" "$root_env" - deployment_log "✅ Created deploy/env/.env from docker/.env" + deployment_log "$(deployment_i18n env.created_from_docker)" return 0 fi if [ -f "$root_example" ]; then cp "$root_example" "$root_env" - deployment_log "✅ Created deploy/env/.env from deploy/env/.env.example" + deployment_log "$(deployment_i18n env.created_from_example)" return 0 fi - deployment_error "deploy/env/.env not found and no docker/.env or deploy/env/.env.example template is available" + deployment_error "$(deployment_i18n env.root_missing)" return 1 } @@ -129,6 +353,85 @@ deployment_source_root_env() { set +a } +deployment_env_dir() { + if [ -n "${DEPLOYMENT_ROOT_ENV:-}" ]; then + dirname "$DEPLOYMENT_ROOT_ENV" + return 0 + fi + + local common_dir + common_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + printf '%s\n' "$(cd "$common_dir/../env" && pwd)" +} + +deployment_monitoring_env_example_file() { + printf '%s/monitoring.env.example\n' "$(deployment_env_dir)" +} + +deployment_monitoring_env_file() { + printf '%s/monitoring.env\n' "$(deployment_env_dir)" +} + +deployment_legacy_monitoring_env_file() { + printf '%s/../docker/assets/monitoring/monitoring.env\n' "$(deployment_env_dir)" +} + +deployment_source_env_file() { + local env_file="$1" + [ -f "$env_file" ] || return 0 + + set -a + # shellcheck source=/dev/null + source "$env_file" + set +a +} + +deployment_env_values_payload() { + local env_file="${DEPLOYMENT_ROOT_ENV:-}" + local monitoring_env_file + local files=() + + if [ -n "$env_file" ] && [ -f "$env_file" ]; then + files+=("$env_file") + else + deployment_warn "deploy/env/.env is not available; environment rollout checksum will use available env files only." + fi + + monitoring_env_file="$(deployment_monitoring_env_file)" + if [ -f "$monitoring_env_file" ]; then + files+=("$monitoring_env_file") + fi + + [ "${#files[@]}" -gt 0 ] || return 0 + + awk ' + /^[[:space:]]*($|#)/ { next } + { + line = $0 + sub(/\r$/, "", line) + sub(/^[[:space:]]*/, "", line) + sub(/^export[[:space:]]+/, "", line) + if (line !~ /^[A-Za-z_][A-Za-z0-9_]*=/) { + next + } + key = line + sub(/=.*/, "", key) + value = line + sub(/^[^=]*=/, "", value) + values[key] = key "=" value + } + END { + for (key in values) { + print values[key] + } + } + ' "${files[@]}" | LC_ALL=C sort -t '=' -k1,1 +} + +deployment_env_values_checksum() { + deployment_sha256_string "$(deployment_env_values_payload)" +} + deployment_update_env_var_file() { local env_file="$1" local key="$2" @@ -175,6 +478,183 @@ deployment_get_env_var_file() { printf '%s' "$value" } +deployment_sync_env_defaults() { + local example_file="$1" + local env_file="$2" + local line key value env_value + + if [ ! -f "$example_file" ]; then + deployment_error "Monitoring env example not found: $example_file" + return 1 + fi + + mkdir -p "$(dirname "$env_file")" + touch "$env_file" + + while IFS= read -r line || [ -n "$line" ]; do + line="${line%$'\r'}" + line="$(deployment_trim "$line")" + case "$line" in + ""|\#*) + continue + ;; + esac + [[ "$line" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || continue + + key="${line%%=*}" + if grep -q "^${key}=" "$env_file"; then + continue + fi + + value="${line#*=}" + env_value="$(printenv "$key" 2>/dev/null || true)" + if [ -n "$env_value" ]; then + value="$env_value" + fi + deployment_update_env_var_file "$env_file" "$key" "$value" + done < "$example_file" +} + +deployment_monitoring_env_value() { + local key="$1" + local default_value="${2:-}" + local env_file + local value + + env_file="$(deployment_monitoring_env_file)" + if value="$(deployment_get_env_var_file "$env_file" "$key" 2>/dev/null)"; then + printf '%s' "$value" + return 0 + fi + if [ "${!key+x}" = "x" ]; then + printf '%s' "${!key}" + return 0 + fi + printf '%s' "$default_value" +} + +deployment_update_monitoring_env_var() { + local key="$1" + local value="$2" + deployment_update_env_var_file "$(deployment_monitoring_env_file)" "$key" "$value" +} + +deployment_monitoring_collector_config_file() { + local target="${1:-docker}" + local provider="${2:-${MONITORING_PROVIDER:-$DEPLOYMENT_MONITORING_PROVIDER}}" + local config_name + + case "$provider" in + phoenix) + config_name="otel-collector-phoenix-config.yml" + ;; + langfuse) + config_name="otel-collector-langfuse-config.yml" + ;; + langsmith) + config_name="otel-collector-langsmith-config.yml" + ;; + grafana) + config_name="otel-collector-grafana-config.yml" + ;; + zipkin) + config_name="otel-collector-zipkin-config.yml" + ;; + otlp|*) + config_name="otel-collector-config.yml" + ;; + esac + + case "$target" in + docker) + printf '../assets/monitoring/%s' "$config_name" + ;; + k8s|helm) + printf '%s' "$config_name" + ;; + *) + printf '%s' "$config_name" + ;; + esac +} + +deployment_prepare_monitoring_env() { + local target="${1:-docker}" + local env_file + local example_file + local legacy_file + local telemetry_enabled + local dashboard_url + local existing_dashboard_url + local provider + local collector_config_file + local otlp_endpoint + local langfuse_public_key + local langfuse_secret_key + local langfuse_auth_header + local langsmith_api_key + + env_file="$(deployment_monitoring_env_file)" + example_file="$(deployment_monitoring_env_example_file)" + legacy_file="$(deployment_legacy_monitoring_env_file)" + + if [ ! -f "$env_file" ] && [ -f "$legacy_file" ]; then + mkdir -p "$(dirname "$env_file")" + cp "$legacy_file" "$env_file" + deployment_log "✅ Migrated monitoring.env to $env_file" + fi + + deployment_sync_env_defaults "$example_file" "$env_file" || return 1 + deployment_source_env_file "$env_file" + + telemetry_enabled="$(deployment_monitoring_enabled)" + provider="$DEPLOYMENT_MONITORING_PROVIDER" + + deployment_update_monitoring_env_var "ENABLE_TELEMETRY" "$telemetry_enabled" + deployment_update_monitoring_env_var "MONITORING_PROVIDER" "$provider" + + deployment_source_env_file "$env_file" + existing_dashboard_url="$(deployment_get_env_var_file "$env_file" "MONITORING_DASHBOARD_URL" 2>/dev/null || true)" + if [ "$telemetry_enabled" != "true" ]; then + deployment_update_monitoring_env_var "MONITORING_DASHBOARD_URL" "" + elif [ -z "$existing_dashboard_url" ]; then + dashboard_url="$(deployment_monitoring_dashboard_url "$target")" + deployment_update_monitoring_env_var "MONITORING_DASHBOARD_URL" "$dashboard_url" + fi + + case "$target" in + k8s|helm) + otlp_endpoint="http://nexent-otel-collector:4318" + ;; + docker|*) + otlp_endpoint="http://otel-collector:4318" + ;; + esac + deployment_update_monitoring_env_var "OTEL_EXPORTER_OTLP_ENDPOINT" "$otlp_endpoint" + deployment_update_monitoring_env_var "OTEL_EXPORTER_OTLP_PROTOCOL" "http" + + collector_config_file="$(deployment_monitoring_collector_config_file "$target" "$provider")" + deployment_update_monitoring_env_var "OTEL_COLLECTOR_CONFIG_FILE" "$collector_config_file" + + if [ "$provider" = "langfuse" ]; then + langfuse_public_key="$(deployment_monitoring_env_value "LANGFUSE_INIT_PROJECT_PUBLIC_KEY" "pk-lf-nexent-local")" + langfuse_secret_key="$(deployment_monitoring_env_value "LANGFUSE_INIT_PROJECT_SECRET_KEY" "sk-lf-nexent-local")" + langfuse_auth_header="Basic $(printf "%s:%s" "$langfuse_public_key" "$langfuse_secret_key" | base64 | tr -d '\n')" + deployment_update_monitoring_env_var "LANGFUSE_OTLP_AUTH_HEADER" "$langfuse_auth_header" + fi + + if [ "$provider" = "langsmith" ]; then + langsmith_api_key="$(deployment_monitoring_env_value "LANGSMITH_API_KEY" "")" + deployment_update_monitoring_env_var "LANGSMITH_API_KEY" "$langsmith_api_key" + deployment_update_monitoring_env_var "LANGSMITH_PROJECT" "$(deployment_monitoring_env_value "LANGSMITH_PROJECT" "nexent")" + deployment_update_monitoring_env_var "LANGSMITH_OTLP_TRACES_ENDPOINT" "$(deployment_monitoring_env_value "LANGSMITH_OTLP_TRACES_ENDPOINT" "https://api.smith.langchain.com/otel/v1/traces")" + fi + + deployment_source_env_file "$env_file" + DEPLOYMENT_MONITORING_PROVIDER="${MONITORING_PROVIDER:-$DEPLOYMENT_MONITORING_PROVIDER}" + export DEPLOYMENT_MONITORING_PROVIDER +} + deployment_sha256_string() { if command -v sha256sum >/dev/null 2>&1; then printf '%s' "$1" | sha256sum | awk '{print $1}' @@ -224,20 +704,20 @@ deployment_init_defaults() { DEPLOYMENT_PORT_POLICY="$DEPLOYMENT_PORT_POLICY_DEFAULT" DEPLOYMENT_IMAGE_SOURCE="$DEPLOYMENT_IMAGE_SOURCE_DEFAULT" DEPLOYMENT_REGISTRY_PROFILE="$DEPLOYMENT_REGISTRY_PROFILE_DEFAULT" + DEPLOYMENT_IMAGE_REGISTRY_PREFIX="$DEPLOYMENT_IMAGE_REGISTRY_PREFIX_DEFAULT" DEPLOYMENT_APP_VERSION="${APP_VERSION:-latest}" DEPLOYMENT_MONITORING_PROVIDER="$DEPLOYMENT_MONITORING_PROVIDER_DEFAULT" - DEPLOYMENT_CONFIG_PATH="" DEPLOYMENT_USE_LOCAL_CONFIG="false" DEPLOYMENT_RECONFIGURE="false" DEPLOYMENT_ROTATE_SECRETS="false" DEPLOYMENT_REFRESH_ES_KEY="false" DEPLOYMENT_LOCAL_CONFIG_PATH="$(deployment_default_local_config_path)" DEPLOYMENT_LOADED_SCHEMA_VERSION="" - DEPLOYMENT_LOADED_APP_VERSION="" DEPLOYMENT_CONFIG_FILE_LOADED="false" DEPLOYMENT_CONFIG_VALUES_LOADED="false" DEPLOYMENT_DOCKER_PORTS="" unset DEPLOYMENT_COMPONENTS_EXPLICIT DEPLOYMENT_PORT_POLICY_EXPLICIT DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT + unset DEPLOYMENT_IMAGE_REGISTRY_PREFIX_EXPLICIT unset DEPLOYMENT_MONITORING_PROVIDER_EXPLICIT DEPLOYMENT_IMAGE_SOURCE_EXPLICIT DEPLOYMENT_APP_VERSION_EXPLICIT } @@ -260,6 +740,10 @@ deployment_parse_common_args() { DEPLOYMENT_REGISTRY_PROFILE="$2" shift 2 ;; + --image-registry-prefix|--registry-prefix|--image-registry) + DEPLOYMENT_IMAGE_REGISTRY_PREFIX="$2" + shift 2 + ;; --app-version|--version) DEPLOYMENT_APP_VERSION="$2" shift 2 @@ -273,9 +757,15 @@ deployment_parse_common_args() { shift ;; --reconfigure) + NEXENT_DEPLOY_CONFIG_MODE="tui" DEPLOYMENT_RECONFIGURE="true" shift ;; + --defaults) + NEXENT_DEPLOY_CONFIG_MODE="defaults" + DEPLOYMENT_RECONFIGURE="false" + shift + ;; --rotate-secrets) DEPLOYMENT_ROTATE_SECRETS="true" shift @@ -285,8 +775,9 @@ deployment_parse_common_args() { shift ;; --config) - DEPLOYMENT_CONFIG_PATH="$2" - shift 2 + NEXENT_DEPLOY_CONFIG_MODE="tui" + DEPLOYMENT_RECONFIGURE="true" + shift ;; --local-config) DEPLOYMENT_LOCAL_CONFIG_PATH="$2" @@ -352,9 +843,8 @@ deployment_load_config_file() { DEPLOYMENT_REGISTRY_PROFILE="$value" loaded_config_value="true" ;; - appVersion) - DEPLOYMENT_APP_VERSION="$value" - [ "$load_mode" = "apply" ] && DEPLOYMENT_LOADED_APP_VERSION="$value" + imageRegistryPrefix) + DEPLOYMENT_IMAGE_REGISTRY_PREFIX="$value" loaded_config_value="true" ;; monitoringProvider) @@ -442,6 +932,40 @@ deployment_normalize_image_source() { esac } +deployment_normalize_image_registry_prefix_value() { + local prefix="$1" + prefix="$(deployment_trim "$prefix")" + prefix="${prefix#http://}" + prefix="${prefix#https://}" + while [[ "$prefix" == */ ]]; do + prefix="${prefix%/}" + done + printf '%s' "$prefix" +} + +deployment_normalize_image_registry_prefix() { + DEPLOYMENT_IMAGE_REGISTRY_PREFIX="$(deployment_normalize_image_registry_prefix_value "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX")" +} + +deployment_add_image_registry_prefix() { + local image="$1" + local prefix="${2:-$DEPLOYMENT_IMAGE_REGISTRY_PREFIX}" + prefix="$(deployment_normalize_image_registry_prefix_value "$prefix")" + + if [ -z "$prefix" ]; then + printf '%s' "$image" + return 0 + fi + case "$image" in + "$prefix"/*) + printf '%s' "$image" + ;; + *) + printf '%s/%s' "$prefix" "$image" + ;; + esac +} + deployment_ensure_required_components() { local source_components="$DEPLOYMENT_COMPONENTS" local normalized="" @@ -475,7 +999,7 @@ deployment_is_valid_value() { deployment_validate() { if [ -n "$DEPLOYMENT_LOADED_SCHEMA_VERSION" ] && [ "$DEPLOYMENT_LOADED_SCHEMA_VERSION" != "$DEPLOYMENT_SCHEMA_VERSION" ]; then - deployment_error "Local config schemaVersion $DEPLOYMENT_LOADED_SCHEMA_VERSION is incompatible with $DEPLOYMENT_SCHEMA_VERSION. Re-run with --reconfigure." + deployment_error "$(deployment_i18n validation.local_config_schema "$DEPLOYMENT_LOADED_SCHEMA_VERSION" "$DEPLOYMENT_SCHEMA_VERSION")" return 1 fi local old_ifs="$IFS" @@ -485,7 +1009,7 @@ deployment_validate() { component="$(deployment_trim "$component")" IFS="$old_ifs" deployment_is_valid_value "$component" $deployment_component_list || { - deployment_error "Unknown deployment component: $component" + deployment_error "$(deployment_i18n validation.unknown_component "$component")" return 1 } IFS=',' @@ -493,19 +1017,23 @@ deployment_validate() { IFS="$old_ifs" deployment_is_valid_value "$DEPLOYMENT_PORT_POLICY" $deployment_port_policy_list || { - deployment_error "Unsupported port policy: $DEPLOYMENT_PORT_POLICY. Use development or production." + deployment_error "$(deployment_i18n validation.unsupported_port_policy "$DEPLOYMENT_PORT_POLICY")" return 1 } deployment_is_valid_value "$DEPLOYMENT_IMAGE_SOURCE" $deployment_image_source_list || { - deployment_error "Unsupported image source: $DEPLOYMENT_IMAGE_SOURCE. Use general, mainland, or local-latest." + deployment_error "$(deployment_i18n validation.unsupported_image_source "$DEPLOYMENT_IMAGE_SOURCE")" return 1 } deployment_is_valid_value "$DEPLOYMENT_REGISTRY_PROFILE" $deployment_registry_profile_list || { - deployment_error "Unsupported registry profile: $DEPLOYMENT_REGISTRY_PROFILE" + deployment_error "$(deployment_i18n validation.unsupported_registry_profile "$DEPLOYMENT_REGISTRY_PROFILE")" return 1 } + if [[ "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" == *[[:space:]]* ]]; then + deployment_error "$(deployment_i18n validation.unsupported_image_registry_prefix "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX")" + return 1 + fi deployment_is_valid_value "$DEPLOYMENT_MONITORING_PROVIDER" $deployment_monitoring_provider_list || { - deployment_error "Unsupported monitoring provider: $DEPLOYMENT_MONITORING_PROVIDER" + deployment_error "$(deployment_i18n validation.unsupported_monitoring_provider "$DEPLOYMENT_MONITORING_PROVIDER")" return 1 } } @@ -513,7 +1041,7 @@ deployment_validate() { deployment_tui_cancel() { printf '\033[?25h' printf '\033[2J\033[H' - deployment_warn "Deployment configuration cancelled." + deployment_warn "$(deployment_i18n tui.cancelled)" return 130 } @@ -539,12 +1067,12 @@ deployment_tui_multiselect_components() { local components=(infrastructure application data-process supabase terminal monitoring) local details=( - "required core dependencies: Elasticsearch, PostgreSQL, Redis, MinIO" - "Nexent app services: config, runtime, MCP, northbound API, web UI" - "background file parsing, indexing, and knowledge processing workers" - "user, tenant, login, invitation, and permission services" - "OpenSSH container used by the terminal tool" - "OpenTelemetry collector and optional tracing dashboard" + "$(deployment_i18n tui.component.infrastructure)" + "$(deployment_i18n tui.component.application)" + "$(deployment_i18n tui.component.data_process)" + "$(deployment_i18n tui.component.supabase)" + "$(deployment_i18n tui.component.terminal)" + "$(deployment_i18n tui.component.monitoring)" ) local selected=(0 0 0 0 0 0) local cursor=0 @@ -558,9 +1086,9 @@ deployment_tui_multiselect_components() { deployment_tui_render_components() { printf '\033[2J\033[H' - printf 'Select deployment components\n' - printf 'Choose which service groups to install. infrastructure is required and cannot be disabled.\n' - printf 'Use Up/Down or j/k to move, Space to toggle, Enter to confirm, q to quit.\n\n' + printf '%s\n' "$(deployment_i18n tui.components.title)" + printf '%s\n' "$(deployment_i18n tui.components.subtitle)" + printf '%s\n\n' "$(deployment_i18n tui.components.help)" local row marker check for row in "${!components[@]}"; do marker=" " @@ -635,12 +1163,12 @@ deployment_tui_select_monitoring_provider() { local providers=(otlp phoenix langfuse langsmith grafana zipkin) local details=( - "collector only; use this when forwarding to an external OTLP backend" - "local Phoenix UI for LLM traces and span inspection" - "local self-hosted Langfuse stack; replace default secrets for production" - "forward traces to hosted LangSmith; requires LANGSMITH_API_KEY" - "local Grafana + Tempo dashboard for traces" - "local Zipkin UI for trace browsing" + "$(deployment_i18n tui.monitoring.otlp)" + "$(deployment_i18n tui.monitoring.phoenix)" + "$(deployment_i18n tui.monitoring.langfuse)" + "$(deployment_i18n tui.monitoring.langsmith)" + "$(deployment_i18n tui.monitoring.grafana)" + "$(deployment_i18n tui.monitoring.zipkin)" ) local cursor=0 local i key key_tail @@ -654,10 +1182,10 @@ deployment_tui_select_monitoring_provider() { deployment_tui_render_monitoring_provider() { printf '\033[2J\033[H' - printf 'Select monitoring provider\n' - printf 'This is used only when the monitoring component is selected.\n' - printf 'Provider controls where OpenTelemetry traces are stored and viewed.\n' - printf 'Use Up/Down or j/k to move, Enter to confirm, b/Backspace to go back, q to quit.\n\n' + printf '%s\n' "$(deployment_i18n tui.monitoring.title)" + printf '%s\n' "$(deployment_i18n tui.monitoring.subtitle)" + printf '%s\n' "$(deployment_i18n tui.monitoring.description)" + printf '%s\n\n' "$(deployment_i18n tui.radio.help)" local row marker radio for row in "${!providers[@]}"; do marker=" " @@ -714,8 +1242,8 @@ deployment_tui_select_port_policy() { local policies=(development production) local details=( - "publish web plus debug/internal service ports for local troubleshooting" - "publish only production entry ports; keep internal services private" + "$(deployment_i18n tui.port.development)" + "$(deployment_i18n tui.port.production)" ) local cursor=0 local i key key_tail @@ -729,10 +1257,10 @@ deployment_tui_select_port_policy() { deployment_tui_render_port_policy() { printf '\033[2J\033[H' - printf 'Select port policy\n' - printf 'This controls which service ports are exposed on the host or cluster node.\n' - printf 'Choose development for local debugging; choose production for a smaller external surface.\n' - printf 'Use Up/Down or j/k to move, Enter to confirm, b/Backspace to go back, q to quit.\n\n' + printf '%s\n' "$(deployment_i18n tui.port.title)" + printf '%s\n' "$(deployment_i18n tui.port.subtitle)" + printf '%s\n' "$(deployment_i18n tui.port.description)" + printf '%s\n\n' "$(deployment_i18n tui.radio.help)" local row marker radio for row in "${!policies[@]}"; do marker=" " @@ -782,6 +1310,23 @@ deployment_tui_select_port_policy() { printf '\033[2J\033[H' } +deployment_image_source_example_tag() { + local source="$1" + local version="${DEPLOYMENT_APP_VERSION:-${APP_VERSION:-latest}}" + local prefix="nexent" + + case "$source" in + mainland) + prefix="ccr.ccs.tencentyun.com/nexent-hub" + ;; + local-latest) + version="latest" + ;; + esac + + printf '%s/nexent:%s' "$prefix" "$version" +} + deployment_tui_select_image_source() { [ -t 0 ] || return 0 [ -n "${DEPLOYMENT_IMAGE_SOURCE_EXPLICIT:-}" ] && return 0 @@ -789,9 +1334,9 @@ deployment_tui_select_image_source() { local sources=(general mainland local-latest) local details=( - "pull images from standard public registries" - "pull from mainland China mirrors for better access in mainland networks" - "use locally built Nexent :latest images and avoid pulling app images" + "$(deployment_image_source_example_tag general)" + "$(deployment_image_source_example_tag mainland)" + "$(deployment_image_source_example_tag local-latest)" ) local cursor=0 local i key key_tail @@ -805,10 +1350,9 @@ deployment_tui_select_image_source() { deployment_tui_render_image_source() { printf '\033[2J\033[H' - printf 'Select image source\n' - printf 'This controls where deployment images come from.\n' - printf 'Use local-latest only after building Nexent images locally.\n' - printf 'Use Up/Down or j/k to move, Enter to confirm, b/Backspace to go back, q to quit.\n\n' + printf '%s\n' "$(deployment_i18n tui.image.title)" + printf '%s\n' "$(deployment_i18n tui.image.description)" + printf '%s\n\n' "$(deployment_i18n tui.radio.help)" local row marker radio for row in "${!sources[@]}"; do marker=" " @@ -911,6 +1455,16 @@ deployment_tui_previous_step() { deployment_run_tui_configuration() { local step=0 local result=0 + local config_mode="${NEXENT_DEPLOY_CONFIG_MODE:-}" + + if [ "$config_mode" = "defaults" ]; then + return 0 + fi + + if { [ "$config_mode" = "tui" ] || [ "$DEPLOYMENT_RECONFIGURE" = "true" ]; } && [ ! -t 0 ]; then + deployment_error "Interactive deployment configuration requires a TTY." + return 1 + fi if ! deployment_tui_step_should_run "$step"; then step="$(deployment_tui_next_step "$step")" @@ -959,26 +1513,37 @@ deployment_run_tui_configuration() { } deployment_maybe_select_local_config() { + local config_mode="${NEXENT_DEPLOY_CONFIG_MODE:-}" + + case "$config_mode" in + ""|defaults|tui) + ;; + *) + deployment_error "Unsupported NEXENT_DEPLOY_CONFIG_MODE: $config_mode. Use defaults or tui." + return 1 + ;; + esac + [ -f "$DEPLOYMENT_LOCAL_CONFIG_PATH" ] || return 0 - if [ "$DEPLOYMENT_RECONFIGURE" = "true" ]; then - deployment_load_config_file "$DEPLOYMENT_LOCAL_CONFIG_PATH" defaults || return 1 + if [ "$config_mode" = "defaults" ] || [ "$DEPLOYMENT_USE_LOCAL_CONFIG" = "true" ]; then + deployment_load_config_file "$DEPLOYMENT_LOCAL_CONFIG_PATH" || return 1 return 0 fi - if [ "$DEPLOYMENT_USE_LOCAL_CONFIG" = "true" ]; then - DEPLOYMENT_CONFIG_PATH="$DEPLOYMENT_LOCAL_CONFIG_PATH" + if [ "$config_mode" = "tui" ] || [ "$DEPLOYMENT_RECONFIGURE" = "true" ]; then + deployment_load_config_file "$DEPLOYMENT_LOCAL_CONFIG_PATH" defaults || return 1 return 0 fi [ -t 0 ] || return 0 - deployment_log "Existing deployment config found: $DEPLOYMENT_LOCAL_CONFIG_PATH" - deployment_log "Choose how to handle saved deployment options:" - deployment_log " 1) Use local config - skip the menus and reuse the saved components, port policy, image source, and monitoring provider." - deployment_log " 2) Reconfigure - load the saved values as defaults, then show the menus so you can change them." - deployment_log " Choose this option when enabling or disabling monitoring, switching providers, or changing deployment scope." + deployment_log "$(deployment_i18n local_config.found "$DEPLOYMENT_LOCAL_CONFIG_PATH")" + deployment_log "$(deployment_i18n local_config.choose)" + deployment_log "$(deployment_i18n local_config.use)" + deployment_log "$(deployment_i18n local_config.reconfigure)" + deployment_log "$(deployment_i18n local_config.reconfigure_hint)" local input - read -r -p "Choose [1/2] (default: 1): " input + read -r -p "$(deployment_prompt prompt.choose_1_2)" input if [ "${input:-1}" = "1" ]; then - DEPLOYMENT_CONFIG_PATH="$DEPLOYMENT_LOCAL_CONFIG_PATH" + deployment_load_config_file "$DEPLOYMENT_LOCAL_CONFIG_PATH" || return 1 else DEPLOYMENT_RECONFIGURE="true" deployment_load_config_file "$DEPLOYMENT_LOCAL_CONFIG_PATH" defaults || return 1 @@ -1067,8 +1632,34 @@ deployment_image_tag() { printf '%s' "${image##*:}" } +deployment_image_rollout_checksum() { + local image="$1" + local image_id="" + + if command -v docker >/dev/null 2>&1; then + image_id="$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null || true)" + if [ -n "$image_id" ]; then + deployment_sha256_string "local-image=${image}|id=${image_id}" + return 0 + fi + deployment_warn "Local image not found: $image; same-tag image changes cannot be detected from this host." + else + deployment_warn "Docker is not available; same-tag image changes cannot be detected for $image." + fi + + deployment_sha256_string "image-ref=${image}" +} + +deployment_render_image_rollout_checksums() { + printf ' backendImage: "%s"\n' "$(deployment_image_rollout_checksum "$NEXENT_IMAGE")" + printf ' webImage: "%s"\n' "$(deployment_image_rollout_checksum "$NEXENT_WEB_IMAGE")" + printf ' dataProcessImage: "%s"\n' "$(deployment_image_rollout_checksum "$NEXENT_DATA_PROCESS_IMAGE")" + printf ' sshImage: "%s"\n' "$(deployment_image_rollout_checksum "$OPENSSH_SERVER_IMAGE")" +} + deployment_apply_image_source() { local version="${DEPLOYMENT_APP_VERSION:-latest}" + local image_var if [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ]; then export NEXENT_IMAGE="nexent/nexent:latest" @@ -1090,6 +1681,47 @@ deployment_apply_image_source() { export SUPABASE_KONG="${SUPABASE_KONG:-kong:2.8.1}" export SUPABASE_GOTRUE="${SUPABASE_GOTRUE:-supabase/gotrue:v2.170.0}" export SUPABASE_DB="${SUPABASE_DB:-supabase/postgres:15.8.1.060}" + + export OTEL_COLLECTOR_IMAGE="${OTEL_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.151.0}" + export PHOENIX_IMAGE="${PHOENIX_IMAGE:-arizephoenix/phoenix:15}" + export TEMPO_IMAGE="${TEMPO_IMAGE:-grafana/tempo:2.10.5}" + export GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:12.4}" + export ZIPKIN_IMAGE="${ZIPKIN_IMAGE:-openzipkin/zipkin:latest}" + export LANGFUSE_WORKER_IMAGE="${LANGFUSE_WORKER_IMAGE:-docker.io/langfuse/langfuse-worker:3}" + export LANGFUSE_WEB_IMAGE="${LANGFUSE_WEB_IMAGE:-docker.io/langfuse/langfuse:3}" + export CLICKHOUSE_IMAGE="${CLICKHOUSE_IMAGE:-docker.io/clickhouse/clickhouse-server:26.3-alpine}" + export LANGFUSE_MINIO_IMAGE="${LANGFUSE_MINIO_IMAGE:-docker.io/minio/minio:RELEASE.2023-12-20T01-00-02Z}" + export LANGFUSE_REDIS_IMAGE="${LANGFUSE_REDIS_IMAGE:-docker.io/redis:alpine}" + export LANGFUSE_POSTGRES_IMAGE="${LANGFUSE_POSTGRES_IMAGE:-docker.io/postgres:15-alpine}" + + if [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ]; then + for image_var in \ + NEXENT_IMAGE \ + NEXENT_WEB_IMAGE \ + NEXENT_DATA_PROCESS_IMAGE \ + NEXENT_MCP_DOCKER_IMAGE \ + ELASTICSEARCH_IMAGE \ + POSTGRESQL_IMAGE \ + REDIS_IMAGE \ + MINIO_IMAGE \ + OPENSSH_SERVER_IMAGE \ + SUPABASE_KONG \ + SUPABASE_GOTRUE \ + SUPABASE_DB \ + OTEL_COLLECTOR_IMAGE \ + PHOENIX_IMAGE \ + TEMPO_IMAGE \ + GRAFANA_IMAGE \ + ZIPKIN_IMAGE \ + LANGFUSE_WORKER_IMAGE \ + LANGFUSE_WEB_IMAGE \ + CLICKHOUSE_IMAGE \ + LANGFUSE_MINIO_IMAGE \ + LANGFUSE_REDIS_IMAGE \ + LANGFUSE_POSTGRES_IMAGE; do + export "$image_var=$(deployment_add_image_registry_prefix "${!image_var}")" + done + fi } deployment_monitoring_enabled() { @@ -1122,16 +1754,16 @@ deployment_monitoring_dashboard_url() { printf 'http://localhost:%s' "${ZIPKIN_PORT:-9411}" ;; k8s:phoenix|helm:phoenix) - printf 'http://localhost:30006' + printf 'http://localhost:%s' "${K8S_PHOENIX_NODE_PORT:-30006}" ;; k8s:langfuse|helm:langfuse) - printf 'http://localhost:30001' + printf 'http://localhost:%s' "${K8S_LANGFUSE_NODE_PORT:-30001}" ;; k8s:grafana|helm:grafana) - printf 'http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1' + printf 'http://localhost:%s/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1' "${K8S_GRAFANA_NODE_PORT:-30002}" ;; k8s:zipkin|helm:zipkin) - printf 'http://localhost:30011' + printf 'http://localhost:%s' "${K8S_ZIPKIN_NODE_PORT:-30011}" ;; *:langsmith) printf 'https://smith.langchain.com/' @@ -1144,8 +1776,12 @@ deployment_monitoring_dashboard_url() { deployment_render_docker_env() { local output_file="$1" + local compose_image_registry_prefix="" + [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && compose_image_registry_prefix="${DEPLOYMENT_IMAGE_REGISTRY_PREFIX}/" mkdir -p "$(dirname "$output_file")" { + printf 'DEPLOYMENT_IMAGE_REGISTRY_PREFIX="%s"\n' "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" + printf 'NEXENT_IMAGE_REGISTRY_PREFIX="%s"\n' "$compose_image_registry_prefix" printf 'NEXENT_IMAGE="%s"\n' "$NEXENT_IMAGE" printf 'NEXENT_WEB_IMAGE="%s"\n' "$NEXENT_WEB_IMAGE" printf 'NEXENT_DATA_PROCESS_IMAGE="%s"\n' "$NEXENT_DATA_PROCESS_IMAGE" @@ -1158,9 +1794,17 @@ deployment_render_docker_env() { printf 'SUPABASE_KONG="%s"\n' "$SUPABASE_KONG" printf 'SUPABASE_GOTRUE="%s"\n' "$SUPABASE_GOTRUE" printf 'SUPABASE_DB="%s"\n' "$SUPABASE_DB" - printf 'ENABLE_TELEMETRY="%s"\n' "$(deployment_monitoring_enabled)" - printf 'MONITORING_PROVIDER="%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" - printf 'MONITORING_DASHBOARD_URL="%s"\n' "$(deployment_monitoring_dashboard_url docker)" + printf 'OTEL_COLLECTOR_IMAGE="%s"\n' "$OTEL_COLLECTOR_IMAGE" + printf 'PHOENIX_IMAGE="%s"\n' "$PHOENIX_IMAGE" + printf 'TEMPO_IMAGE="%s"\n' "$TEMPO_IMAGE" + printf 'GRAFANA_IMAGE="%s"\n' "$GRAFANA_IMAGE" + printf 'ZIPKIN_IMAGE="%s"\n' "$ZIPKIN_IMAGE" + printf 'LANGFUSE_WORKER_IMAGE="%s"\n' "$LANGFUSE_WORKER_IMAGE" + printf 'LANGFUSE_WEB_IMAGE="%s"\n' "$LANGFUSE_WEB_IMAGE" + printf 'CLICKHOUSE_IMAGE="%s"\n' "$CLICKHOUSE_IMAGE" + printf 'LANGFUSE_MINIO_IMAGE="%s"\n' "$LANGFUSE_MINIO_IMAGE" + printf 'LANGFUSE_REDIS_IMAGE="%s"\n' "$LANGFUSE_REDIS_IMAGE" + printf 'LANGFUSE_POSTGRES_IMAGE="%s"\n' "$LANGFUSE_POSTGRES_IMAGE" } > "$output_file" } @@ -1177,7 +1821,7 @@ deployment_render_component_values() { deployment_render_image_values() { local local_pull_policy="IfNotPresent" - [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ] && local_pull_policy="Never" + [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ] && [ -z "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && local_pull_policy="Never" printf 'nexent-config:\n' printf ' images:\n backend:\n repository: "%s"\n tag: "%s"\n pullPolicy: "%s"\n' "$(deployment_image_repo "$NEXENT_IMAGE")" "$(deployment_image_tag "$NEXENT_IMAGE")" "$local_pull_policy" @@ -1259,7 +1903,7 @@ deployment_render_helm_chart_values() { local local_pull_policy="IfNotPresent" local northbound_type="NodePort" local internal_type="ClusterIP" - [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ] && local_pull_policy="Never" + [ "$DEPLOYMENT_IMAGE_SOURCE" = "local-latest" ] && [ -z "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && local_pull_policy="Never" if [ "$DEPLOYMENT_PORT_POLICY" = "development" ]; then internal_type="NodePort" fi @@ -1314,6 +1958,7 @@ deployment_render_helm_chart_values() { printf 'nexent-supabase-auth:\n' printf ' enabled: %s\n' "$(deployment_chart_enabled supabase)" printf ' image:\n repository: "%s"\n tag: "%s"\n pullPolicy: "IfNotPresent"\n' "$(deployment_image_repo "$SUPABASE_GOTRUE")" "$(deployment_image_tag "$SUPABASE_GOTRUE")" + printf ' initImage:\n repository: "%s"\n tag: "%s"\n pullPolicy: "IfNotPresent"\n' "$(deployment_image_repo "$POSTGRESQL_IMAGE")" "$(deployment_image_tag "$POSTGRESQL_IMAGE")" printf ' service:\n type: "%s"\n nodePort: 30999\n' "$internal_type" printf 'nexent-supabase-db:\n' printf ' enabled: %s\n' "$(deployment_chart_enabled supabase)" @@ -1323,6 +1968,160 @@ deployment_render_helm_chart_values() { printf ' images:\n mcp:\n repository: "%s"\n tag: "%s"\n pullPolicy: "%s"\n' "$(deployment_image_repo "$NEXENT_MCP_DOCKER_IMAGE")" "$(deployment_image_tag "$NEXENT_MCP_DOCKER_IMAGE")" "$local_pull_policy" } +deployment_yaml_quote() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '"%s"' "$value" +} + +deployment_render_helm_monitoring_global_values() { + local enabled + enabled="$(deployment_monitoring_enabled)" + + printf ' monitoring:\n' + printf ' enabled: %s\n' "$enabled" + printf ' provider: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_PROVIDER "$DEPLOYMENT_MONITORING_PROVIDER")")" + printf ' dashboardUrl: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_DASHBOARD_URL "$(deployment_monitoring_dashboard_url k8s)")")" + printf ' projectName: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_PROJECT_NAME "nexent")")" + printf ' serviceName: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_SERVICE_NAME "nexent-backend")")" + printf ' otlpEndpoint: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_ENDPOINT "http://nexent-otel-collector:4318")")" + printf ' otlpTracesEndpoint: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "")")" + printf ' otlpMetricsEndpoint: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_METRICS_ENDPOINT "")")" + printf ' otlpProtocol: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_PROTOCOL "http")")" + printf ' otlpHeaders: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_HEADERS "")")" + printf ' otlpAuthorization: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_AUTHORIZATION "")")" + printf ' otlpApiKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_X_API_KEY "")")" + printf ' otlpLangfuseIngestionVersion: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION "")")" + printf ' langsmithApiKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_API_KEY "")")" + printf ' langsmithProject: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_PROJECT "nexent")")" + printf ' langsmithOtlpTracesEndpoint: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_OTLP_TRACES_ENDPOINT "https://api.smith.langchain.com/otel/v1/traces")")" + printf ' otlpMetricsEnabled: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_EXPORTER_OTLP_METRICS_ENABLED "true")")" + printf ' instrumentRequests: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_INSTRUMENT_REQUESTS "false")")" + printf ' fastapiIncludedUrls: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_FASTAPI_INCLUDED_URLS "/agent/run")")" + printf ' fastapiExcludedUrls: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_FASTAPI_EXCLUDED_URLS "")")" + printf ' fastapiExcludeSpans: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_FASTAPI_EXCLUDE_SPANS "receive,send")")" + printf ' telemetrySampleRate: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value TELEMETRY_SAMPLE_RATE "1.0")")" + printf ' traceContentMode: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_TRACE_CONTENT_MODE "full")")" + printf ' traceMaxChars: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_TRACE_MAX_CHARS "4000")")" + printf ' traceMaxItems: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_TRACE_MAX_ITEMS "20")")" +} + +deployment_render_monitoring_image_value() { + local key="$1" + local repository="$2" + local tag="$3" + local image + image="$(deployment_add_image_registry_prefix "${repository}:${tag}")" + + printf ' %s:\n' "$key" + printf ' repository: %s\n' "$(deployment_yaml_quote "$(deployment_image_repo "$image")")" + printf ' tag: %s\n' "$(deployment_yaml_quote "$(deployment_image_tag "$image")")" +} + +deployment_render_helm_monitoring_chart_values() { + local enabled + local langfuse_nextauth_url + local otel_collector_tag + local phoenix_tag + local tempo_tag + local grafana_tag + local zipkin_tag + local langfuse_tag + local clickhouse_tag + local minio_tag + local redis_tag + local postgres_tag + enabled="$(deployment_monitoring_enabled)" + langfuse_nextauth_url="$(deployment_monitoring_env_value K8S_LANGFUSE_NEXTAUTH_URL "http://localhost:${K8S_LANGFUSE_NODE_PORT:-30001}")" + otel_collector_tag="$(deployment_monitoring_env_value OTEL_COLLECTOR_VERSION "0.151.0")" + phoenix_tag="$(deployment_monitoring_env_value PHOENIX_VERSION "15")" + tempo_tag="$(deployment_monitoring_env_value TEMPO_VERSION "2.10.5")" + grafana_tag="$(deployment_monitoring_env_value GRAFANA_VERSION "12.4")" + zipkin_tag="$(deployment_monitoring_env_value ZIPKIN_VERSION "latest")" + langfuse_tag="$(deployment_monitoring_env_value LANGFUSE_VERSION "3")" + clickhouse_tag="$(deployment_monitoring_env_value LANGFUSE_CLICKHOUSE_VERSION "26.3-alpine")" + minio_tag="$(deployment_monitoring_env_value LANGFUSE_MINIO_VERSION "RELEASE.2023-12-20T01-00-02Z")" + redis_tag="$(deployment_monitoring_env_value LANGFUSE_REDIS_VERSION "alpine")" + postgres_tag="$(deployment_monitoring_env_value LANGFUSE_POSTGRES_VERSION "15-alpine")" + + printf 'nexent-monitoring:\n' + printf ' enabled: %s\n' "$enabled" + printf ' provider: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value MONITORING_PROVIDER "$DEPLOYMENT_MONITORING_PROVIDER")")" + printf ' images:\n' + deployment_render_monitoring_image_value otelCollector otel/opentelemetry-collector-contrib "$otel_collector_tag" + deployment_render_monitoring_image_value phoenix arizephoenix/phoenix "$phoenix_tag" + deployment_render_monitoring_image_value tempo grafana/tempo "$tempo_tag" + deployment_render_monitoring_image_value grafana grafana/grafana "$grafana_tag" + deployment_render_monitoring_image_value zipkin openzipkin/zipkin "$zipkin_tag" + deployment_render_monitoring_image_value langfuseWeb docker.io/langfuse/langfuse "$langfuse_tag" + deployment_render_monitoring_image_value langfuseWorker docker.io/langfuse/langfuse-worker "$langfuse_tag" + deployment_render_monitoring_image_value clickhouse docker.io/clickhouse/clickhouse-server "$clickhouse_tag" + deployment_render_monitoring_image_value minio docker.io/minio/minio "$minio_tag" + deployment_render_monitoring_image_value redis docker.io/redis "$redis_tag" + deployment_render_monitoring_image_value postgres docker.io/postgres "$postgres_tag" + printf ' collector:\n' + printf ' configFile: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value OTEL_COLLECTOR_CONFIG_FILE "$(deployment_monitoring_collector_config_file k8s "$DEPLOYMENT_MONITORING_PROVIDER")")")" + printf ' service:\n' + printf ' grpcPort: %s\n' "$(deployment_monitoring_env_value OTEL_COLLECTOR_GRPC_PORT "4317")" + printf ' httpPort: %s\n' "$(deployment_monitoring_env_value OTEL_COLLECTOR_HTTP_PORT "4318")" + printf ' env:\n' + printf ' langsmithApiKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_API_KEY "")")" + printf ' langsmithProject: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_PROJECT "nexent")")" + printf ' langsmithOtlpTracesEndpoint: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGSMITH_OTLP_TRACES_ENDPOINT "https://api.smith.langchain.com/otel/v1/traces")")" + printf ' langfuseOtlpAuthHeader: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_OTLP_AUTH_HEADER "")")" + printf ' phoenix:\n' + printf ' service:\n' + printf ' port: %s\n' "$(deployment_monitoring_env_value PHOENIX_PORT "6006")" + printf ' nodePort: %s\n' "$(deployment_monitoring_env_value K8S_PHOENIX_NODE_PORT "30006")" + printf ' grafana:\n' + printf ' adminUser: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value GRAFANA_ADMIN_USER "admin")")" + printf ' adminPassword: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value GRAFANA_ADMIN_PASSWORD "nexent@4321")")" + printf ' defaultLanguage: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value GRAFANA_DEFAULT_LANGUAGE "zh-Hans")")" + printf ' service:\n' + printf ' port: %s\n' "$(deployment_monitoring_env_value GRAFANA_PORT "3002")" + printf ' nodePort: %s\n' "$(deployment_monitoring_env_value K8S_GRAFANA_NODE_PORT "30002")" + printf ' tempo:\n' + printf ' service:\n' + printf ' port: %s\n' "$(deployment_monitoring_env_value TEMPO_PORT "3200")" + printf ' zipkin:\n' + printf ' service:\n' + printf ' port: %s\n' "$(deployment_monitoring_env_value ZIPKIN_PORT "9411")" + printf ' nodePort: %s\n' "$(deployment_monitoring_env_value K8S_ZIPKIN_NODE_PORT "30011")" + printf ' langfuse:\n' + printf ' nextauthUrl: %s\n' "$(deployment_yaml_quote "$langfuse_nextauth_url")" + printf ' nextauthSecret: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_NEXTAUTH_SECRET "nexent-langfuse-secret")")" + printf ' salt: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_SALT "nexent-langfuse-salt")")" + printf ' encryptionKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_ENCRYPTION_KEY "0000000000000000000000000000000000000000000000000000000000000000")")" + printf ' telemetryEnabled: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_TELEMETRY_ENABLED "false")")" + printf ' enableExperimentalFeatures: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES "false")")" + printf ' init:\n' + printf ' orgId: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_ORG_ID "nexent")")" + printf ' orgName: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_ORG_NAME "Nexent")")" + printf ' projectId: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_PROJECT_ID "nexent")")" + printf ' projectName: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_PROJECT_NAME "Nexent")")" + printf ' projectPublicKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_PROJECT_PUBLIC_KEY "pk-lf-nexent-local")")" + printf ' projectSecretKey: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_PROJECT_SECRET_KEY "sk-lf-nexent-local")")" + printf ' userEmail: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_USER_EMAIL "admin@nexent.com")")" + printf ' userName: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_USER_NAME "admin")")" + printf ' userPassword: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_INIT_USER_PASSWORD "nexent@4321")")" + printf ' service:\n' + printf ' nodePort: %s\n' "$(deployment_monitoring_env_value K8S_LANGFUSE_NODE_PORT "30001")" + printf ' postgres:\n' + printf ' user: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_POSTGRES_USER "postgres")")" + printf ' password: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_POSTGRES_PASSWORD "nexent@4321")")" + printf ' database: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_POSTGRES_DB "postgres")")" + printf ' clickhouse:\n' + printf ' user: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_CLICKHOUSE_USER "clickhouse")")" + printf ' password: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_CLICKHOUSE_PASSWORD "clickhouse")")" + printf ' minio:\n' + printf ' rootUser: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_MINIO_ROOT_USER "minio")")" + printf ' rootPassword: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_MINIO_ROOT_PASSWORD "miniosecret")")" + printf ' bucket: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_S3_BUCKET "langfuse")")" + printf ' redis:\n' + printf ' auth: %s\n' "$(deployment_yaml_quote "$(deployment_monitoring_env_value LANGFUSE_REDIS_AUTH "myredissecret")")" +} + deployment_render_helm_values() { local output_file="$1" mkdir -p "$(dirname "$output_file")" @@ -1338,21 +2137,9 @@ deployment_render_helm_values() { fi printf ' portPolicy: "%s"\n' "$DEPLOYMENT_PORT_POLICY" printf ' imageSource: "%s"\n' "$DEPLOYMENT_IMAGE_SOURCE" - printf ' monitoring:\n' - if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then - printf ' enabled: true\n' - else - printf ' enabled: false\n' - fi - printf ' provider: "%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" - printf ' dashboardUrl: "%s"\n' "$(deployment_monitoring_dashboard_url k8s)" - printf 'nexent-monitoring:\n' - if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then - printf ' enabled: true\n' - else - printf ' enabled: false\n' - fi - printf ' provider: "%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" + printf ' imageRegistryPrefix: "%s"\n' "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" + deployment_render_helm_monitoring_global_values + deployment_render_helm_monitoring_chart_values deployment_render_helm_chart_values } > "$output_file" } @@ -1362,7 +2149,6 @@ deployment_persist_local_config() { mkdir -p "$(dirname "$output_file")" { printf 'schemaVersion: "%s"\n' "$DEPLOYMENT_SCHEMA_VERSION" - printf 'appVersion: "%s"\n' "$DEPLOYMENT_APP_VERSION" printf 'components:\n' local old_ifs="$IFS" IFS=',' @@ -1374,6 +2160,7 @@ deployment_persist_local_config() { IFS="$old_ifs" printf 'portPolicy: "%s"\n' "$DEPLOYMENT_PORT_POLICY" printf 'imageSource: "%s"\n' "$DEPLOYMENT_IMAGE_SOURCE" + printf 'imageRegistryPrefix: "%s"\n' "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" printf 'monitoringProvider: "%s"\n' "$DEPLOYMENT_MONITORING_PROVIDER" } > "$output_file" } @@ -1381,29 +2168,34 @@ deployment_persist_local_config() { deployment_print_summary() { local target="${1:-all}" - deployment_log "Deployment components: $DEPLOYMENT_COMPONENTS" - deployment_log "Port policy: $DEPLOYMENT_PORT_POLICY" - deployment_log "Image source: $DEPLOYMENT_IMAGE_SOURCE" + deployment_log "$(deployment_i18n summary.components "$DEPLOYMENT_COMPONENTS")" + deployment_log "$(deployment_i18n summary.port_policy "$DEPLOYMENT_PORT_POLICY")" + deployment_log "$(deployment_i18n summary.image_source "$DEPLOYMENT_IMAGE_SOURCE")" + if [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ]; then + deployment_log "$(deployment_i18n summary.image_registry_prefix "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX")" + fi if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then - deployment_log "Monitoring provider: $DEPLOYMENT_MONITORING_PROVIDER" + deployment_log "$(deployment_i18n summary.monitoring_provider "$DEPLOYMENT_MONITORING_PROVIDER")" fi case "$target" in docker) - deployment_log "Docker services: $DEPLOYMENT_SELECTED_DOCKER_SERVICES" - deployment_log "Docker published ports: $DEPLOYMENT_DOCKER_PORTS" + deployment_log "$(deployment_i18n summary.docker_services "$DEPLOYMENT_SELECTED_DOCKER_SERVICES")" + deployment_log "$(deployment_i18n summary.docker_ports "$DEPLOYMENT_DOCKER_PORTS")" ;; k8s|helm) - deployment_log "Helm charts: $DEPLOYMENT_SELECTED_HELM_CHARTS" + deployment_log "$(deployment_i18n summary.helm_charts "$DEPLOYMENT_SELECTED_HELM_CHARTS")" ;; *) - deployment_log "Docker services: $DEPLOYMENT_SELECTED_DOCKER_SERVICES" - deployment_log "Helm charts: $DEPLOYMENT_SELECTED_HELM_CHARTS" - deployment_log "Docker published ports: $DEPLOYMENT_DOCKER_PORTS" + deployment_log "$(deployment_i18n summary.docker_services "$DEPLOYMENT_SELECTED_DOCKER_SERVICES")" + deployment_log "$(deployment_i18n summary.helm_charts "$DEPLOYMENT_SELECTED_HELM_CHARTS")" + deployment_log "$(deployment_i18n summary.docker_ports "$DEPLOYMENT_DOCKER_PORTS")" ;; esac } deployment_prepare_config() { + local NEXENT_DEPLOY_CONFIG_MODE="${NEXENT_DEPLOY_CONFIG_MODE:-}" + deployment_init_defaults local raw_args=("$@") @@ -1414,8 +2206,12 @@ deployment_prepare_config() { --port-policy) DEPLOYMENT_PORT_POLICY_EXPLICIT="true" ;; --image-source) DEPLOYMENT_IMAGE_SOURCE_EXPLICIT="true" ;; --registry-profile) DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT="true" ;; + --image-registry-prefix|--registry-prefix|--image-registry) DEPLOYMENT_IMAGE_REGISTRY_PREFIX_EXPLICIT="true" ;; --app-version|--version) DEPLOYMENT_APP_VERSION_EXPLICIT="true" ;; --monitoring-provider) DEPLOYMENT_MONITORING_PROVIDER_EXPLICIT="true" ;; + --config) DEPLOYMENT_RECONFIGURE="true" ;; + --reconfigure) DEPLOYMENT_RECONFIGURE="true" ;; + --defaults) DEPLOYMENT_RECONFIGURE="false" ;; --rotate-secrets) DEPLOYMENT_ROTATE_SECRETS="true" ;; --refresh-es-key) DEPLOYMENT_REFRESH_ES_KEY="true" ;; esac @@ -1425,10 +2221,7 @@ deployment_prepare_config() { if [ -n "${DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT:-}" ] && [ -z "${DEPLOYMENT_IMAGE_SOURCE_EXPLICIT:-}" ]; then DEPLOYMENT_IMAGE_SOURCE="$DEPLOYMENT_REGISTRY_PROFILE" fi - deployment_maybe_select_local_config - if [ -n "$DEPLOYMENT_CONFIG_PATH" ] && [ "$DEPLOYMENT_RECONFIGURE" != "true" ]; then - deployment_load_config_file "$DEPLOYMENT_CONFIG_PATH" || return 1 - fi + deployment_maybe_select_local_config || return 1 deployment_apply_legacy_inputs deployment_parse_common_args "$@" if [ -n "${DEPLOYMENT_REGISTRY_PROFILE_EXPLICIT:-}" ] && [ -z "${DEPLOYMENT_IMAGE_SOURCE_EXPLICIT:-}" ]; then @@ -1439,6 +2232,7 @@ deployment_prepare_config() { deployment_run_tui_configuration || tui_result=$? [ "$tui_result" -eq 0 ] || return "$tui_result" deployment_normalize_image_source || return 1 + deployment_normalize_image_registry_prefix deployment_validate || return 1 deployment_compute_selection } diff --git a/deploy/common/config.example.yaml b/deploy/common/config.example.yaml index 2b5262edb..dcf497011 100644 --- a/deploy/common/config.example.yaml +++ b/deploy/common/config.example.yaml @@ -1,5 +1,4 @@ schemaVersion: "1" -appVersion: "latest" # Default deployment: shared infrastructure plus Nexent core application. components: @@ -21,7 +20,6 @@ monitoringProvider: "otlp" # Local development image example: # imageSource: "local-latest" -# appVersion: "latest" # Mainland China registry mirror example: # imageSource: "mainland" diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 6e4478984..dc5dfe286 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -3,19 +3,53 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_COMMON="$SCRIPT_DIR/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi usage() { + if [ "${DEPLOYMENT_LANGUAGE:-en}" = "zh" ]; then + cat <<'USAGE' +用法： + bash deploy.sh [--config|--defaults] docker [Docker 部署选项] + bash deploy.sh [--config|--defaults] k8s [K8s 部署选项] + +Docker 实现：deploy/docker/deploy.sh +K8s 实现： deploy/k8s/deploy.sh +选项： + --config 进入交互式部署配置界面。 + --defaults 复用保存配置或内置默认值，跳过交互界面。 +USAGE + return + fi + cat <<'USAGE' Usage: - bash deploy.sh docker [docker deploy options] - bash deploy.sh k8s [k8s deploy options] + bash deploy.sh [--config|--defaults] docker [docker deploy options] + bash deploy.sh [--config|--defaults] k8s [k8s deploy options] Docker implementation: deploy/docker/deploy.sh K8s implementation: deploy/k8s/deploy.sh +Options: + --config Open the interactive deployment configuration. + --defaults Use saved configuration or built-in defaults and skip TUI. USAGE } case "${1:-}" in + --config) + export NEXENT_DEPLOY_CONFIG_MODE="tui" + shift + exec bash "$0" "$@" + ;; + --defaults) + export NEXENT_DEPLOY_CONFIG_MODE="defaults" + shift + exec bash "$0" "$@" + ;; docker) shift exec bash "$SCRIPT_DIR/docker/deploy.sh" "$@" @@ -28,7 +62,11 @@ case "${1:-}" in usage ;; *) - echo "Unknown deploy target: $1" >&2 + if [ "${DEPLOYMENT_LANGUAGE:-en}" = "zh" ]; then + echo "未知部署目标：$1" >&2 + else + echo "Unknown deploy target: $1" >&2 + fi usage >&2 exit 1 ;; diff --git a/deploy/docker/compose/docker-compose-monitoring.yml b/deploy/docker/compose/docker-compose-monitoring.yml index cd6805a2a..34722918a 100644 --- a/deploy/docker/compose/docker-compose-monitoring.yml +++ b/deploy/docker/compose/docker-compose-monitoring.yml @@ -2,7 +2,7 @@ name: monitor services: otel-collector: - image: otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION:-0.151.0} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}otel/opentelemetry-collector-contrib:${OTEL_COLLECTOR_VERSION:-0.151.0} container_name: nexent-otel-collector command: ["--config=/etc/otel-collector-config.yml"] environment: @@ -20,7 +20,7 @@ services: restart: unless-stopped phoenix: - image: arizephoenix/phoenix:${PHOENIX_VERSION:-15} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}arizephoenix/phoenix:${PHOENIX_VERSION:-15} container_name: nexent-phoenix profiles: ["phoenix"] environment: @@ -35,7 +35,7 @@ services: restart: unless-stopped tempo: - image: grafana/tempo:${TEMPO_VERSION:-2.10.5} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}grafana/tempo:${TEMPO_VERSION:-2.10.5} container_name: nexent-tempo profiles: ["grafana"] command: ["--config.file=/etc/tempo.yml"] @@ -49,12 +49,12 @@ services: restart: unless-stopped grafana: - image: grafana/grafana:${GRAFANA_VERSION:-12.4} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}grafana/grafana:${GRAFANA_VERSION:-12.4} container_name: nexent-grafana profiles: ["grafana"] environment: GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-nexent-grafana-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-nexent@4321} GF_USERS_ALLOW_SIGN_UP: "false" GF_USERS_DEFAULT_LANGUAGE: ${GRAFANA_DEFAULT_LANGUAGE:-zh-Hans} GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false" @@ -71,7 +71,7 @@ services: restart: unless-stopped zipkin: - image: openzipkin/zipkin:${ZIPKIN_VERSION:-latest} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}openzipkin/zipkin:${ZIPKIN_VERSION:-latest} container_name: nexent-zipkin profiles: ["zipkin"] ports: @@ -81,7 +81,7 @@ services: restart: unless-stopped langfuse-worker: - image: docker.io/langfuse/langfuse-worker:${LANGFUSE_VERSION:-3} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/langfuse/langfuse-worker:${LANGFUSE_VERSION:-3} container_name: nexent-langfuse-worker profiles: ["langfuse"] restart: unless-stopped @@ -97,7 +97,7 @@ services: environment: &langfuse-env NEXTAUTH_URL: ${LANGFUSE_NEXTAUTH_URL:-http://localhost:3001} NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET:-nexent-langfuse-secret} - DATABASE_URL: postgresql://${LANGFUSE_POSTGRES_USER:-postgres}:${LANGFUSE_POSTGRES_PASSWORD:-postgres}@langfuse-postgres:5432/${LANGFUSE_POSTGRES_DB:-postgres} + DATABASE_URL: postgresql://${LANGFUSE_POSTGRES_USER:-postgres}:${LANGFUSE_POSTGRES_PASSWORD:-nexent@4321}@langfuse-postgres:5432/${LANGFUSE_POSTGRES_DB:-postgres} SALT: ${LANGFUSE_SALT:-nexent-langfuse-salt} ENCRYPTION_KEY: ${LANGFUSE_ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} TELEMETRY_ENABLED: ${LANGFUSE_TELEMETRY_ENABLED:-false} @@ -139,7 +139,7 @@ services: - nexent langfuse-web: - image: docker.io/langfuse/langfuse:${LANGFUSE_VERSION:-3} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/langfuse/langfuse:${LANGFUSE_VERSION:-3} container_name: nexent-langfuse-web profiles: ["langfuse"] restart: unless-stopped @@ -148,20 +148,20 @@ services: <<: *langfuse-env LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-nexent} LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-Nexent} - LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-nexent-local} - LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-Nexent Local} + LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-nexent} + LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-Nexent} LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-pk-lf-nexent-local} LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-sk-lf-nexent-local} - LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-admin@nexent.local} - LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-Nexent Admin} - LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-nexent-langfuse-admin} + LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-admin@nexent.com} + LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-admin} + LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-nexent@4321} ports: - "${LANGFUSE_PORT:-3001}:3000" networks: - nexent langfuse-clickhouse: - image: docker.io/clickhouse/clickhouse-server:${LANGFUSE_CLICKHOUSE_VERSION:-26.3-alpine} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/clickhouse/clickhouse-server:${LANGFUSE_CLICKHOUSE_VERSION:-26.3-alpine} container_name: nexent-langfuse-clickhouse profiles: ["langfuse"] restart: unless-stopped @@ -186,7 +186,7 @@ services: - nexent langfuse-minio: - image: docker.io/minio/minio:${LANGFUSE_MINIO_VERSION:-RELEASE.2023-12-20T01-00-02Z} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/minio/minio:${LANGFUSE_MINIO_VERSION:-RELEASE.2023-12-20T01-00-02Z} container_name: nexent-langfuse-minio profiles: ["langfuse"] restart: unless-stopped @@ -210,7 +210,7 @@ services: - nexent langfuse-redis: - image: docker.io/redis:${LANGFUSE_REDIS_VERSION:-alpine} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/redis:${LANGFUSE_REDIS_VERSION:-alpine} container_name: nexent-langfuse-redis profiles: ["langfuse"] restart: unless-stopped @@ -230,13 +230,13 @@ services: - nexent langfuse-postgres: - image: docker.io/postgres:${LANGFUSE_POSTGRES_VERSION:-15-alpine} + image: ${NEXENT_IMAGE_REGISTRY_PREFIX:-}docker.io/postgres:${LANGFUSE_POSTGRES_VERSION:-15-alpine} container_name: nexent-langfuse-postgres profiles: ["langfuse"] restart: unless-stopped environment: POSTGRES_USER: ${LANGFUSE_POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${LANGFUSE_POSTGRES_PASSWORD:-postgres} + POSTGRES_PASSWORD: ${LANGFUSE_POSTGRES_PASSWORD:-nexent@4321} POSTGRES_DB: ${LANGFUSE_POSTGRES_DB:-postgres} TZ: UTC PGTZ: UTC diff --git a/deploy/docker/compose/docker-compose.prod.yml b/deploy/docker/compose/docker-compose.prod.yml index 77555e73e..5dbe32a57 100644 --- a/deploy/docker/compose/docker-compose.prod.yml +++ b/deploy/docker/compose/docker-compose.prod.yml @@ -86,6 +86,7 @@ services: UMASK: 0022 env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: @@ -116,6 +117,7 @@ services: UMASK: 0022 env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: diff --git a/deploy/docker/compose/docker-compose.yml b/deploy/docker/compose/docker-compose.yml index 1efa1da67..a5f5acd88 100644 --- a/deploy/docker/compose/docker-compose.yml +++ b/deploy/docker/compose/docker-compose.yml @@ -99,6 +99,7 @@ services: UMASK: 0022 env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: @@ -131,6 +132,7 @@ services: UMASK: 0022 env_file: - ../../env/.env + - ../../env/monitoring.env user: root depends_on: nexent-elasticsearch: diff --git a/deploy/docker/deploy.sh b/deploy/docker/deploy.sh index 3ffab8db3..dda77c0bb 100755 --- a/deploy/docker/deploy.sh +++ b/deploy/docker/deploy.sh @@ -20,6 +20,7 @@ ORIGINAL_ARGS=("$@") ROOT_ENV_FILE="$DEPLOY_ROOT/env/.env" COMPOSE_DIR="$SCRIPT_DIR/compose" DOCKER_ASSETS_DIR="$SCRIPT_DIR/assets" +MONITORING_ENV_FILE="$DEPLOY_ROOT/env/monitoring.env" SQL_DIR="$DEPLOY_ROOT/sql" if [ -f "$DEPLOYMENT_COMMON" ]; then @@ -57,28 +58,65 @@ ROOT_DIR_PARAM="" # Suppress the orphan warning export COMPOSE_IGNORE_ORPHANS=True +print_docker_deploy_usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "用法：$0 [选项]" + echo "" + echo "部署选项：" + echo " --components LIST 要部署的组件列表" + echo " --port-policy POLICY development 或 production" + echo " --image-source SOURCE general、mainland 或 local-latest" + echo " --registry-profile NAME 兼容旧参数，映射为 general/mainland 镜像源" + echo " --image-registry-prefix P 镜像仓库前缀，例如 registry.example.com/nexent" + echo " --monitoring-provider NAME 选中 monitoring 组件时使用的监控 provider" + echo " --version VERSION 指定应用版本（未设置时自动检测）" + echo " --defaults 复用保存配置或内置默认值并跳过交互界面" + echo " --use-local-config 复用保存的本地部署配置并跳过交互界面" + echo " --reconfigure 使用保存配置作为默认值并进入交互式配置界面" + echo " --rotate-secrets 强制轮换部署密钥" + echo " --refresh-es-key 强制重新创建 ELASTICSEARCH_API_KEY" + echo " --config 进入交互式部署配置界面" + echo " --root-dir PATH Docker 数据和运行文件根目录" + echo " --help, -h 显示帮助信息" + echo "" + echo "卸载：bash uninstall.sh" + return + fi + + echo "Usage: $0 [options]" + echo "" + echo "Deploy options:" + echo " --components LIST Components to deploy" + echo " --port-policy POLICY development or production" + echo " --image-source SOURCE general, mainland, or local-latest" + echo " --registry-profile NAME Legacy alias for image source general/mainland" + echo " --image-registry-prefix P Image registry prefix, e.g. registry.example.com/nexent" + echo " --monitoring-provider NAME Monitoring provider when monitoring is selected" + echo " --version VERSION Specify app version (auto-detected if not set)" + echo " --defaults Use saved config or built-in defaults and skip TUI" + echo " --use-local-config Reuse saved local deployment config and skip TUI" + echo " --reconfigure Open TUI using saved local config as defaults" + echo " --rotate-secrets Force rotation of deployment secrets" + echo " --refresh-es-key Force recreation of ELASTICSEARCH_API_KEY" + echo " --config Open the interactive deployment configuration" + echo " --root-dir PATH Root directory for Docker data and runtime files" + echo " --help, -h Show this help message" + echo "" + echo "Uninstall: bash uninstall.sh" +} + while [[ $# -gt 0 ]]; do case "$1" in delete|delete-all|--delete-volumes|--remove-volumes|--keep-volumes) - echo "❌ Docker uninstall has moved to uninstall.sh. Use: bash uninstall.sh" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ Docker 卸载已迁移到 uninstall.sh。请使用：bash uninstall.sh" + else + echo "❌ Docker uninstall has moved to uninstall.sh. Use: bash uninstall.sh" + fi exit 1 ;; --help|-h) - echo "Usage: $0 [options]" - echo "" - echo "Deploy options:" - echo " --components LIST" - echo " --port-policy development|production" - echo " --image-source general|mainland|local-latest" - echo " --version VERSION" - echo " --use-local-config" - echo " --reconfigure" - echo " --rotate-secrets" - echo " --refresh-es-key" - echo " --config PATH" - echo " --root-dir PATH" - echo "" - echo "Uninstall: bash uninstall.sh" + print_docker_deploy_usage exit 0 ;; --mode) @@ -656,10 +694,107 @@ disable_dashboard() { update_env_var "DISABLE_CELERY_FLOWER" "true" } +docker_monitoring_active_services() { + printf '%s\n' otel-collector + + case "$DEPLOYMENT_MONITORING_PROVIDER" in + phoenix) + printf '%s\n' phoenix + ;; + grafana) + printf '%s\n' tempo grafana + ;; + zipkin) + printf '%s\n' zipkin + ;; + langfuse) + printf '%s\n' langfuse-worker langfuse-web langfuse-clickhouse langfuse-minio langfuse-redis langfuse-postgres + ;; + esac +} + +docker_monitoring_profile_services() { + printf '%s\n' \ + phoenix \ + tempo \ + grafana \ + zipkin \ + langfuse-worker \ + langfuse-web \ + langfuse-clickhouse \ + langfuse-minio \ + langfuse-redis \ + langfuse-postgres +} + +docker_monitoring_container_names() { + printf '%s\n' \ + nexent-otel-collector \ + nexent-phoenix \ + nexent-tempo \ + nexent-grafana \ + nexent-zipkin \ + nexent-langfuse-worker \ + nexent-langfuse-web \ + nexent-langfuse-clickhouse \ + nexent-langfuse-minio \ + nexent-langfuse-redis \ + nexent-langfuse-postgres +} + +docker_monitoring_containers_exist() { + command -v docker >/dev/null 2>&1 || return 1 + + local container + while IFS= read -r container; do + [ -n "$container" ] || continue + if docker container inspect "$container" >/dev/null 2>&1; then + return 0 + fi + done < <(docker_monitoring_container_names) + + return 1 +} + +stop_monitoring_services() { + local all_profile_args=(--profile phoenix --profile grafana --profile zipkin --profile langfuse) + + [ -f "$COMPOSE_DIR/docker-compose-monitoring.yml" ] || return 0 + docker_monitoring_containers_exist || return 0 + + echo "🔭 Stopping monitoring services..." + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" --env-file "$MONITORING_ENV_FILE" "${all_profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" down --remove-orphans; then + echo " ❌ ERROR Failed to stop monitoring services" + return 1 + fi +} + +cleanup_stale_monitoring_services() { + local all_profile_args=(--profile phoenix --profile grafana --profile zipkin --profile langfuse) + local active_services + local service + local stale_services=() + + active_services="$(docker_monitoring_active_services)" + while IFS= read -r service; do + [ -n "$service" ] || continue + if ! printf '%s\n' "$active_services" | grep -Fxq "$service"; then + stale_services+=("$service") + fi + done < <(docker_monitoring_profile_services) + + [ "${#stale_services[@]}" -gt 0 ] || return 0 + + echo " 🧹 Removing stale monitoring provider services..." + ${docker_compose_command} --env-file "$ROOT_ENV_FILE" --env-file "$MONITORING_ENV_FILE" "${all_profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" stop "${stale_services[@]}" >/dev/null 2>&1 || true + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" --env-file "$MONITORING_ENV_FILE" "${all_profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" rm -f "${stale_services[@]}"; then + echo " ❌ ERROR Failed to remove stale monitoring services" + return 1 + fi +} + sync_monitoring_env_vars() { - update_env_var "ENABLE_TELEMETRY" "$(deployment_monitoring_enabled)" - update_env_var "MONITORING_PROVIDER" "$DEPLOYMENT_MONITORING_PROVIDER" - update_env_var "MONITORING_DASHBOARD_URL" "$(deployment_monitoring_dashboard_url docker)" + deployment_prepare_monitoring_env docker || return 1 } pull_mcp_image() { @@ -976,13 +1111,21 @@ deploy_infrastructure() { } deploy_monitoring() { - deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring" || return 0 + if ! deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then + stop_monitoring_services || return 1 + return 0 + fi if [ ! -f "$COMPOSE_DIR/docker-compose-monitoring.yml" ]; then echo " ❌ ERROR Monitoring compose file not found: $COMPOSE_DIR/docker-compose-monitoring.yml" return 1 fi + if [ "$DEPLOYMENT_MONITORING_PROVIDER" = "langsmith" ] && [ -z "${LANGSMITH_API_KEY:-}" ]; then + echo " ❌ ERROR LANGSMITH_API_KEY is required when --monitoring-provider langsmith is selected" + return 1 + fi + local profile_args=() case "$DEPLOYMENT_MONITORING_PROVIDER" in phoenix|grafana|zipkin|langfuse) @@ -991,7 +1134,8 @@ deploy_monitoring() { esac echo "🔭 Starting monitoring services..." - if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" "${profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" up -d; then + cleanup_stale_monitoring_services || return 1 + if ! ${docker_compose_command} --env-file "$ROOT_ENV_FILE" --env-file "$MONITORING_ENV_FILE" "${profile_args[@]}" -f "$COMPOSE_DIR/docker-compose-monitoring.yml" up -d; then echo " ❌ ERROR Failed to start monitoring services" return 1 fi @@ -1428,54 +1572,137 @@ choose_image_env() { main_deploy() { # Main deployment function - echo "🚀 Nexent Deployment Script 🚀" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🚀 Nexent 部署脚本 🚀" + else + echo "🚀 Nexent Deployment Script 🚀" + fi echo "" echo "--------------------------------" echo "" APP_VERSION="$(get_app_version)" if [ -z "$APP_VERSION" ]; then - echo "❌ Failed to get app version, please check VERSION or backend/consts/const.py" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 获取应用版本失败，请检查 VERSION 或 backend/consts/const.py" + else + echo "❌ Failed to get app version, please check VERSION or backend/consts/const.py" + fi exit 1 fi - echo "🌐 App version: $APP_VERSION" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🌐 应用版本：$APP_VERSION" + else + echo "🌐 App version: $APP_VERSION" + fi # Select deployment components, port policy and image source via shared config. - apply_deployment_common_config || { echo "❌ Deployment configuration failed"; exit 1; } + apply_deployment_common_config || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 部署配置失败" + else + echo "❌ Deployment configuration failed" + fi + exit 1 + } deployment_persist_local_config # Check only the ports published by the selected deployment configuration. check_deployment_ports - configure_root_dir_from_env || { echo "❌ ROOT_DIR configuration failed"; exit 1; } + configure_root_dir_from_env || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ ROOT_DIR 配置失败" + else + echo "❌ ROOT_DIR configuration failed" + fi + exit 1 + } # Set NEXENT_MCP_DOCKER_IMAGE in .env file if [ -n "${NEXENT_MCP_DOCKER_IMAGE:-}" ]; then update_env_var "NEXENT_MCP_DOCKER_IMAGE" "${NEXENT_MCP_DOCKER_IMAGE}" - echo "🔧 NEXENT_MCP_DOCKER_IMAGE set to: ${NEXENT_MCP_DOCKER_IMAGE}" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🔧 NEXENT_MCP_DOCKER_IMAGE 已设置为：${NEXENT_MCP_DOCKER_IMAGE}" + else + echo "🔧 NEXENT_MCP_DOCKER_IMAGE set to: ${NEXENT_MCP_DOCKER_IMAGE}" + fi else - echo "⚠️ NEXENT_MCP_DOCKER_IMAGE not found in environment, will use default from code" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "⚠️ 环境中未找到 NEXENT_MCP_DOCKER_IMAGE，将使用代码默认值" + else + echo "⚠️ NEXENT_MCP_DOCKER_IMAGE not found in environment, will use default from code" + fi fi # Add permission - prepare_directory_and_data || { echo "❌ Permission setup failed"; exit 1; } - update_sql_files_checksum || { echo "ERROR SQL checksum update failed"; exit 1; } - generate_minio_ak_sk || { echo "❌ MinIO key generation failed"; exit 1; } + prepare_directory_and_data || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 权限设置失败" + else + echo "❌ Permission setup failed" + fi + exit 1 + } + update_sql_files_checksum || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "ERROR SQL checksum 更新失败" + else + echo "ERROR SQL checksum update failed" + fi + exit 1 + } + generate_minio_ak_sk || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ MinIO key 生成失败" + else + echo "❌ MinIO key generation failed" + fi + exit 1 + } # Generate Supabase secrets - generate_supabase_keys || { echo "❌ Supabase secrets generation failed"; exit 1; } + generate_supabase_keys || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ Supabase secrets 生成失败" + else + echo "❌ Supabase secrets generation failed" + fi + exit 1 + } # Deploy infrastructure services - deploy_infrastructure || { echo "❌ Infrastructure deployment failed"; exit 1; } + deploy_infrastructure || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 基础设施部署失败" + else + echo "❌ Infrastructure deployment failed" + fi + exit 1 + } - deploy_monitoring || { echo "❌ Monitoring deployment failed"; exit 1; } + deploy_monitoring || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 监控部署失败" + else + echo "❌ Monitoring deployment failed" + fi + exit 1 + } stop_unselected_data_process_service # Generate Elasticsearch API key - generate_elasticsearch_api_key || { echo "❌ Elasticsearch API key generation failed"; exit 1; } + generate_elasticsearch_api_key || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ Elasticsearch API key 生成失败" + else + echo "❌ Elasticsearch API key generation failed" + fi + exit 1 + } echo "" echo "--------------------------------" @@ -1483,17 +1710,38 @@ main_deploy() { # Special handling for infrastructure mode if [ "$DEPLOYMENT_MODE" = "infrastructure" ]; then - generate_env_for_infrastructure || { echo "❌ Environment generation failed"; exit 1; } + generate_env_for_infrastructure || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 环境变量生成失败" + else + echo "❌ Environment generation failed" + fi + exit 1 + } # Create default super admin user (only for full version) if [ "$DEPLOYMENT_VERSION" = "full" ]; then - create_default_super_admin_user || { echo "❌ Default super admin user creation failed"; exit 1; } + create_default_super_admin_user || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 默认超级管理员创建失败" + else + echo "❌ Default super admin user creation failed" + fi + exit 1 + } fi - echo "🎉 Infrastructure deployment completed successfully!" - echo " You can now start the core services manually using dev containers" - echo " Environment file available at: $ROOT_ENV_FILE" - echo "💡 Use 'source deploy/env/.env' from the project root to load environment variables" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🎉 基础设施部署完成！" + echo " 现在可以使用 dev containers 手动启动核心服务" + echo " 环境变量文件：$ROOT_ENV_FILE" + echo "💡 在项目根目录执行 'source deploy/env/.env' 可加载环境变量" + else + echo "🎉 Infrastructure deployment completed successfully!" + echo " You can now start the core services manually using dev containers" + echo " Environment file available at: $ROOT_ENV_FILE" + echo "💡 Use 'source deploy/env/.env' from the project root to load environment variables" + fi # Pull MCP image for later use pull_mcp_image @@ -1504,16 +1752,34 @@ main_deploy() { fi # Start core services - deploy_core_services || { echo "❌ Core services deployment failed"; exit 1; } + deploy_core_services || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 核心服务部署失败" + else + echo "❌ Core services deployment failed" + fi + exit 1 + } - echo " ✅ Core services started successfully" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo " ✅ 核心服务启动成功" + else + echo " ✅ Core services started successfully" + fi echo "" echo "--------------------------------" echo "" # Create default super admin user if [ "$DEPLOYMENT_VERSION" = "full" ]; then - create_default_super_admin_user || { echo "❌ Default super admin user creation failed"; exit 1; } + create_default_super_admin_user || { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 默认超级管理员创建失败" + else + echo "❌ Default super admin user creation failed" + fi + exit 1 + } fi persist_deploy_options @@ -1522,14 +1788,23 @@ main_deploy() { # Pull MCP image for later use pull_mcp_image - echo "🎉 Deployment completed successfully!" - echo "🌐 You can now access the application at http://localhost:3000" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🎉 部署完成！" + echo "🌐 现在可以访问应用：http://localhost:3000" + else + echo "🎉 Deployment completed successfully!" + echo "🌐 You can now access the application at http://localhost:3000" + fi } # get docker compose version version_info=$(get_compose_version) if [[ $version_info == "unknown" ]]; then - echo "Error: Docker Compose not found or version detection failed" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：未找到 Docker Compose 或版本检测失败" + else + echo "Error: Docker Compose not found or version detection failed" + fi exit 1 fi @@ -1541,27 +1816,47 @@ version_number=$(echo "$version_info" | awk '{print $2}') docker_compose_command="" case $version_type in "v1") - echo "Detected Docker Compose V1, version: $version_number" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "检测到 Docker Compose V1，版本：$version_number" + else + echo "Detected Docker Compose V1, version: $version_number" + fi # The version 1.28.0 is the minimum requirement in Docker Compose v1 for default interpolation syntax. if [[ $version_number < "1.28.0" ]]; then - echo "Warning: V1 version is too old, consider upgrading to V2" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "警告：V1 版本过旧，建议升级到 V2" + else + echo "Warning: V1 version is too old, consider upgrading to V2" + fi exit 1 fi docker_compose_command="docker-compose" ;; "v2") - echo "Detected Docker Compose V2, version: $version_number" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "检测到 Docker Compose V2，版本：$version_number" + else + echo "Detected Docker Compose V2, version: $version_number" + fi docker_compose_command="docker compose" ;; *) - echo "Error: Unknown docker compose version type." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：未知 Docker Compose 版本类型。" + else + echo "Error: Unknown docker compose version type." + fi exit 1 ;; esac # Execute main deployment with error handling if ! main_deploy; then - echo "❌ Deployment failed. Please check the error messages above and try again." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 部署失败。请检查上面的错误信息后重试。" + else + echo "❌ Deployment failed. Please check the error messages above and try again." + fi exit 1 fi diff --git a/deploy/docker/start-monitoring.sh b/deploy/docker/start-monitoring.sh deleted file mode 100755 index ecdc244ed..000000000 --- a/deploy/docker/start-monitoring.sh +++ /dev/null @@ -1,546 +0,0 @@ -#!/bin/bash - -# Nexent LLM Performance Monitoring Setup Script -# This script starts the OpenTelemetry Collector alone, or with a local -# Phoenix/Langfuse/Grafana/Zipkin observability backend, or forwards to -# online LangSmith. - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MONITORING_DIR="$SCRIPT_DIR/assets/monitoring" -COMPOSE_FILE="$SCRIPT_DIR/compose/docker-compose-monitoring.yml" - -SUPPORTED_STACKS="otlp, collector, phoenix, langfuse, langsmith, grafana, zipkin" - -usage() { - cat < - $(basename "$0") [stack] - $(basename "$0") [stack] - $(basename "$0") [stack] - -Stacks are mutually exclusive. Starting one stack removes containers from the -other monitoring stacks while preserving their data volumes. - -Stacks: - otlp Start OpenTelemetry Collector only. This is the default. - collector Alias for otlp. - phoenix Start Collector and local Arize Phoenix. - langfuse Start Collector and local Langfuse self-host stack. - langsmith Start Collector and forward traces to online LangSmith. - grafana Start Collector, Grafana, and Tempo. - zipkin Start Collector and local Zipkin. - -Actions: - start/up Start the selected stack and stop containers from other stacks. - stop/down Stop and remove containers for the selected stack. Data is kept. - uninstall Stop and remove containers and data volumes for the selected stack. - -Set MONITORING_PROVIDER in monitoring/monitoring.env to change the default stack. -EOF -} - -ACTION="start" -STACK_ARG="" - -set_stack_arg() { - local value="$1" - if [ -n "$STACK_ARG" ] && [ "$STACK_ARG" != "$value" ]; then - echo "❌ Error: multiple monitoring stacks specified: '$STACK_ARG' and '$value'." - usage - exit 1 - fi - STACK_ARG="$value" -} - -while [ $# -gt 0 ]; do - case "$1" in - --stack) - if [ $# -lt 2 ]; then - echo "❌ Error: --stack requires a value." - usage - exit 1 - fi - set_stack_arg "$2" - shift 2 - ;; - --stop|--down) - ACTION="stop" - shift - ;; - --uninstall|--remove) - ACTION="uninstall" - shift - ;; - start|up) - ACTION="start" - shift - ;; - stop|down) - ACTION="stop" - shift - ;; - uninstall|remove) - ACTION="uninstall" - shift - ;; - -h|--help) - usage - exit 0 - ;; - otlp|collector|phoenix|langfuse|langsmith|grafana|zipkin) - set_stack_arg "$1" - shift - ;; - *) - echo "❌ Error: unknown argument '$1'." - usage - exit 1 - ;; - esac -done - -normalize_stack() { - case "$1" in - ""|otlp|collector) - echo "collector" - ;; - phoenix|langfuse|langsmith|grafana|zipkin) - echo "$1" - ;; - *) - echo "❌ Error: unsupported monitoring provider '$1'. Supported: $SUPPORTED_STACKS." >&2 - exit 1 - ;; - esac -} - -if [ -n "$STACK_ARG" ]; then - normalize_stack "$STACK_ARG" > /dev/null -fi - -remove_containers() { - if [ "$#" -eq 0 ]; then - return - fi - - local existing=() - local container - for container in "$@"; do - if docker ps -a --format '{{.Names}}' | grep -qx "$container"; then - existing+=("$container") - fi - done - - if [ "${#existing[@]}" -gt 0 ]; then - docker rm -f "${existing[@]}" > /dev/null - echo "🧹 Removed containers: ${existing[*]}" - fi -} - -remove_volumes() { - if [ "$#" -eq 0 ]; then - return - fi - - local existing=() - local volume - for volume in "$@"; do - if docker volume ls --format '{{.Name}}' | grep -qx "$volume"; then - existing+=("$volume") - fi - done - - if [ "${#existing[@]}" -gt 0 ]; then - docker volume rm "${existing[@]}" > /dev/null - echo "🧹 Removed volumes: ${existing[*]}" - fi -} - -stack_containers() { - case "$1" in - collector|langsmith) - echo "nexent-otel-collector" - ;; - phoenix) - echo "nexent-otel-collector nexent-phoenix" - ;; - langfuse) - echo "nexent-otel-collector nexent-langfuse-worker nexent-langfuse-web nexent-langfuse-clickhouse nexent-langfuse-minio nexent-langfuse-redis nexent-langfuse-postgres" - ;; - grafana) - echo "nexent-otel-collector nexent-grafana nexent-tempo" - ;; - zipkin) - echo "nexent-otel-collector nexent-zipkin" - ;; - esac -} - -stack_data_volumes() { - case "$1" in - phoenix) - echo "monitor_phoenix-data" - ;; - langfuse) - echo "monitor_langfuse-postgres-data monitor_langfuse-clickhouse-data monitor_langfuse-clickhouse-logs monitor_langfuse-minio-data monitor_langfuse-redis-data" - ;; - grafana) - echo "monitor_grafana-data monitor_tempo-data" - ;; - collector|langsmith|zipkin) - echo "" - ;; - esac -} - -all_backend_containers() { - echo "nexent-phoenix nexent-langfuse-worker nexent-langfuse-web nexent-langfuse-clickhouse nexent-langfuse-minio nexent-langfuse-redis nexent-langfuse-postgres nexent-grafana nexent-tempo nexent-zipkin" -} - -incompatible_containers() { - local stack="$1" - local containers - containers="$(all_backend_containers)" - case "$stack" in - phoenix) - echo "$containers" | sed 's/nexent-phoenix//g' - ;; - langfuse) - echo "$containers" | sed 's/nexent-langfuse-worker//g; s/nexent-langfuse-web//g; s/nexent-langfuse-clickhouse//g; s/nexent-langfuse-minio//g; s/nexent-langfuse-redis//g; s/nexent-langfuse-postgres//g' - ;; - grafana) - echo "$containers" | sed 's/nexent-grafana//g; s/nexent-tempo//g' - ;; - zipkin) - echo "$containers" | sed 's/nexent-zipkin//g' - ;; - collector|langsmith) - echo "$containers" - ;; - esac -} - -configure_stack() { - MONITORING_PROVIDER="${STACK_ARG:-${MONITORING_PROVIDER:-otlp}}" - LOCAL_STACK="$(normalize_stack "$MONITORING_PROVIDER")" - - case "$LOCAL_STACK" in - collector) - BACKEND_MONITORING_PROVIDER="otlp" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-config.yml}" - COMPOSE_PROFILES=() - ;; - phoenix) - BACKEND_MONITORING_PROVIDER="phoenix" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-phoenix-config.yml}" - COMPOSE_PROFILES=(--profile phoenix) - ;; - langfuse) - BACKEND_MONITORING_PROVIDER="langfuse" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-langfuse-config.yml}" - COMPOSE_PROFILES=(--profile langfuse) - LANGFUSE_INIT_PROJECT_PUBLIC_KEY="${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-pk-lf-nexent-local}" - LANGFUSE_INIT_PROJECT_SECRET_KEY="${LANGFUSE_INIT_PROJECT_SECRET_KEY:-sk-lf-nexent-local}" - if [ -z "${LANGFUSE_OTLP_AUTH_HEADER:-}" ]; then - LANGFUSE_OTLP_AUTH_HEADER="Basic $(printf "%s:%s" "$LANGFUSE_INIT_PROJECT_PUBLIC_KEY" "$LANGFUSE_INIT_PROJECT_SECRET_KEY" | base64 | tr -d '\n')" - fi - export LANGFUSE_OTLP_AUTH_HEADER - ;; - langsmith) - BACKEND_MONITORING_PROVIDER="langsmith" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-langsmith-config.yml}" - COMPOSE_PROFILES=() - LANGSMITH_OTLP_TRACES_ENDPOINT="${LANGSMITH_OTLP_TRACES_ENDPOINT:-https://api.smith.langchain.com/otel/v1/traces}" - LANGSMITH_PROJECT="${LANGSMITH_PROJECT:-nexent}" - if [ "$ACTION" = "start" ] && [ -z "${LANGSMITH_API_KEY:-}" ]; then - echo "❌ Error: LANGSMITH_API_KEY is required for the langsmith stack." - echo " Set it in $MONITORING_DIR/monitoring.env or export it before running this script." - exit 1 - fi - export LANGSMITH_API_KEY LANGSMITH_PROJECT LANGSMITH_OTLP_TRACES_ENDPOINT - ;; - grafana) - BACKEND_MONITORING_PROVIDER="grafana" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-grafana-config.yml}" - COMPOSE_PROFILES=(--profile grafana) - ;; - zipkin) - BACKEND_MONITORING_PROVIDER="zipkin" - OTEL_COLLECTOR_CONFIG_FILE="${OTEL_COLLECTOR_CONFIG_FILE:-../assets/monitoring/otel-collector-zipkin-config.yml}" - COMPOSE_PROFILES=(--profile zipkin) - ;; - esac - export OTEL_COLLECTOR_CONFIG_FILE -} - -dashboard_url() { - case "$LOCAL_STACK" in - phoenix) - echo "http://localhost:${PHOENIX_PORT:-6006}" - ;; - langfuse) - echo "http://localhost:${LANGFUSE_PORT:-3001}" - ;; - langsmith) - echo "https://smith.langchain.com/" - ;; - grafana) - echo "http://localhost:${GRAFANA_PORT:-3002}/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1" - ;; - zipkin) - echo "http://localhost:${ZIPKIN_PORT:-9411}" - ;; - collector) - echo "" - ;; - esac -} - -print_access_hints() { - local dashboard - dashboard="$(dashboard_url)" - - echo "" - echo "📊 Access your monitoring tools:" - echo " • OTLP HTTP receiver: http://localhost:${OTEL_COLLECTOR_HTTP_PORT:-4318}" - echo " • OTLP gRPC receiver: localhost:${OTEL_COLLECTOR_GRPC_PORT:-4317}" - echo " • Docker backend endpoint: http://otel-collector:4318" - - case "$LOCAL_STACK" in - phoenix) - echo " • Phoenix UI: $dashboard" - echo " • Phoenix direct gRPC ingest: localhost:${PHOENIX_GRPC_HOST_PORT:-4319}" - ;; - langfuse) - echo " • Langfuse UI: $dashboard" - echo " • Langfuse admin: ${LANGFUSE_INIT_USER_EMAIL:-admin@nexent.com} / ${LANGFUSE_INIT_USER_PASSWORD:-nexent@4321}" - echo " • Langfuse project keys: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-pk-lf-nexent-local} / ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-sk-lf-nexent-local}" - echo " • MinIO API: http://localhost:${LANGFUSE_MINIO_API_PORT:-9092}" - echo " • MinIO console: http://localhost:${LANGFUSE_MINIO_CONSOLE_PORT:-9093}" - ;; - langsmith) - echo " • LangSmith UI: $dashboard" - echo " • LangSmith project: ${LANGSMITH_PROJECT:-nexent}" - echo " • LangSmith OTLP traces endpoint: ${LANGSMITH_OTLP_TRACES_ENDPOINT:-https://api.smith.langchain.com/otel/v1/traces}" - echo " • No local LangSmith UI is started; open the hosted UI and select the project above." - ;; - grafana) - echo " • Grafana dashboard: $dashboard" - echo " • Grafana home: http://localhost:${GRAFANA_PORT:-3002}" - echo " • Grafana admin: ${GRAFANA_ADMIN_USER:-admin} / ${GRAFANA_ADMIN_PASSWORD:-nexent@4321}" - echo " • Tempo API: http://localhost:${TEMPO_PORT:-3200}" - ;; - zipkin) - echo " • Zipkin UI: $dashboard" - ;; - collector) - echo " • Collector-only mode has no monitoring UI." - echo " • View Collector logs: docker logs -f nexent-otel-collector" - echo " • Configure Phoenix, Langfuse, LangSmith, Grafana/Tempo, Zipkin, or another OTLP backend when you need a UI." - ;; - esac - - echo "" - echo "🔗 Frontend monitoring entry:" - if [ -n "$dashboard" ]; then - echo " Set MONITORING_DASHBOARD_URL=$dashboard" - else - echo " Leave MONITORING_DASHBOARD_URL empty to hide the monitoring entry." - fi -} - -print_backend_hints() { - echo "" - echo "🔧 To enable monitoring in your Nexent backend:" - echo " 1. Set ENABLE_TELEMETRY=true in the project deploy/env/.env" - echo " 2. Set MONITORING_PROVIDER=$BACKEND_MONITORING_PROVIDER in the project deploy/env/.env" - echo " 3. Set OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 for Docker services" - echo " or http://localhost:${OTEL_COLLECTOR_HTTP_PORT:-4318} for a backend running on the host" - echo " 4. Set MONITORING_DASHBOARD_URL as shown above when a UI is available" - echo " 5. Install performance dependencies:" - echo " uv sync --extra performance" - echo " 6. Restart your Nexent backend service" -} - -print_uninstall_hints() { - echo "" - echo "🛑 Stop or uninstall this monitoring stack:" - echo " • Stop containers and keep data:" - echo " $(basename "$0") stop $LOCAL_STACK" - echo " • Remove containers and this stack's data volumes:" - echo " $(basename "$0") uninstall $LOCAL_STACK" - echo "" - echo " Stacks are mutually exclusive; do not run multiple monitoring providers in parallel." -} - -load_env_for_start() { - if [ ! -f "$MONITORING_DIR/monitoring.env" ]; then - echo "📋 Creating monitoring.env from example..." - cp "$MONITORING_DIR/monitoring.env.example" "$MONITORING_DIR/monitoring.env" - echo "⚠️ Please review and update $MONITORING_DIR/monitoring.env as needed" - fi - - set -a - # shellcheck disable=SC1091 - . "$MONITORING_DIR/monitoring.env" - set +a -} - -load_env_if_present() { - if [ -f "$MONITORING_DIR/monitoring.env" ]; then - set -a - # shellcheck disable=SC1091 - . "$MONITORING_DIR/monitoring.env" - set +a - fi -} - -resolve_compose_cmd() { - if docker compose version > /dev/null 2>&1; then - COMPOSE_CMD=(docker compose) - elif command -v docker-compose > /dev/null 2>&1; then - COMPOSE_CMD=(docker-compose) - else - echo "❌ Error: Docker Compose is not installed." - exit 1 - fi -} - -check_service() { - local name=$1 - local url=$2 - local port=$3 - - if curl -s --max-time 5 --connect-timeout 3 "$url" > /dev/null 2>&1; then - echo "✅ $name is running at http://localhost:$port" - return 0 - else - echo "⚠️ $name may not be ready yet (will start in background)" - return 1 - fi -} - -check_stack_health() { - echo "🔍 Checking service health..." - check_service "OpenTelemetry Collector HTTP receiver" "http://localhost:${OTEL_COLLECTOR_HTTP_PORT:-4318}" "${OTEL_COLLECTOR_HTTP_PORT:-4318}" || true - - case "$LOCAL_STACK" in - phoenix) - check_service "Phoenix UI" "http://localhost:${PHOENIX_PORT:-6006}" "${PHOENIX_PORT:-6006}" || true - ;; - langfuse) - check_service "Langfuse UI" "http://localhost:${LANGFUSE_PORT:-3001}" "${LANGFUSE_PORT:-3001}" || true - ;; - langsmith) - echo "✅ LangSmith forwarding is configured for project: ${LANGSMITH_PROJECT:-nexent}" - ;; - grafana) - check_service "Grafana" "http://localhost:${GRAFANA_PORT:-3002}/api/health" "${GRAFANA_PORT:-3002}" || true - check_service "Tempo API" "http://localhost:${TEMPO_PORT:-3200}/ready" "${TEMPO_PORT:-3200}" || true - ;; - zipkin) - check_service "Zipkin UI" "http://localhost:${ZIPKIN_PORT:-9411}" "${ZIPKIN_PORT:-9411}" || true - ;; - esac -} - -start_stack() { - echo "🚀 Starting Nexent LLM Performance Monitoring Setup..." - - if ! docker info > /dev/null 2>&1; then - echo "❌ Error: Docker is not running. Please start Docker first." - exit 1 - fi - - resolve_compose_cmd - - if ! docker network ls --format '{{.Name}}' | grep -qx nexent_network; then - echo "🔗 Creating nexent_network..." - docker network create nexent_network - else - echo "✅ nexent_network already exists" - fi - - load_env_for_start - configure_stack - - local incompatible - incompatible="$(incompatible_containers "$LOCAL_STACK")" - if [ -n "$incompatible" ]; then - # shellcheck disable=SC2086 - remove_containers $incompatible - fi - - echo "🐳 Starting monitoring services with provider: $MONITORING_PROVIDER" - echo " Selected stack: $LOCAL_STACK" - "${COMPOSE_CMD[@]}" -f "$COMPOSE_FILE" --env-file "$MONITORING_DIR/monitoring.env" "${COMPOSE_PROFILES[@]}" up -d --remove-orphans - - echo "⏳ Waiting for services to start..." - sleep 10 - check_stack_health - - echo "" - echo "🎉 Monitoring setup complete!" - print_access_hints - print_backend_hints - echo "" - echo "🔎 Key Trace Data to Inspect:" - echo " • Agent span hierarchy" - echo " • LLM generation spans" - echo " • Retriever and memory spans" - echo " • Tool call spans" - echo " • Error events" - print_uninstall_hints -} - -stop_or_uninstall_stack() { - local remove_data="$1" - - if ! docker info > /dev/null 2>&1; then - echo "❌ Error: Docker is not running. Please start Docker first." - exit 1 - fi - - load_env_if_present - configure_stack - - local containers - containers="$(stack_containers "$LOCAL_STACK")" - echo "🛑 Removing monitoring containers for stack: $LOCAL_STACK" - # shellcheck disable=SC2086 - remove_containers $containers - - if [ "$remove_data" = "true" ]; then - local volumes - volumes="$(stack_data_volumes "$LOCAL_STACK")" - if [ -n "$volumes" ]; then - echo "🧹 Removing data volumes for stack: $LOCAL_STACK" - # shellcheck disable=SC2086 - remove_volumes $volumes - else - echo "ℹ️ Stack '$LOCAL_STACK' has no dedicated local data volumes." - fi - echo "✅ Monitoring stack '$LOCAL_STACK' has been uninstalled." - else - echo "✅ Monitoring stack '$LOCAL_STACK' has been stopped. Data volumes were kept." - fi - - echo "" - echo "ℹ️ The shared Docker network 'nexent_network' is kept because it is also used by Nexent services." -} - -case "$ACTION" in - start) - start_stack - ;; - stop) - stop_or_uninstall_stack false - ;; - uninstall) - stop_or_uninstall_stack true - ;; -esac diff --git a/deploy/docker/uninstall.sh b/deploy/docker/uninstall.sh index f4dad85d8..3d9dc2d76 100755 --- a/deploy/docker/uninstall.sh +++ b/deploy/docker/uninstall.sh @@ -9,13 +9,40 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DEPLOY_ROOT="$PROJECT_ROOT/deploy" +DEPLOYMENT_COMMON="$DEPLOY_ROOT/common/common.sh" ROOT_ENV_FILE="$PROJECT_ROOT/deploy/env/.env" COMPOSE_DIR="$SCRIPT_DIR/compose" +MONITORING_ENV_FILE="$PROJECT_ROOT/deploy/env/monitoring.env" cd "$SCRIPT_DIR" +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi + DELETE_VOLUMES="" print_usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "用法：$0 [delete-all] [选项]" + echo "" + echo "卸载 Nexent Docker 部署。" + echo "" + echo "选项：" + echo " --delete-volumes true|false 控制是否删除持久化数据" + echo " --remove-volumes 等同于 --delete-volumes true" + echo " --keep-volumes 等同于 --delete-volumes false" + echo " --help, -h 显示帮助信息" + echo "" + echo "示例：" + echo " bash uninstall.sh" + echo " bash uninstall.sh --delete-volumes false" + echo " bash uninstall.sh --delete-volumes true" + echo " bash uninstall.sh delete-all" + return + fi + echo "Usage: $0 [delete-all] [options]" echo "" echo "Uninstall Docker deployment for Nexent." @@ -45,7 +72,11 @@ parse_bool_option() { true|TRUE|True|yes|YES|Yes|y|Y|1) return 0 ;; false|FALSE|False|no|NO|No|n|N|0) return 1 ;; *) - echo "❌ Invalid boolean value: $value. Use true or false." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 无效布尔值：$value。请使用 true 或 false。" + else + echo "❌ Invalid boolean value: $value. Use true or false." + fi exit 1 ;; esac @@ -74,7 +105,11 @@ while [[ $# -gt 0 ]]; do exit 0 ;; *) - echo "❌ Unknown option: $1" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 未知选项：$1" + else + echo "❌ Unknown option: $1" + fi print_usage exit 1 ;; @@ -157,10 +192,19 @@ resolve_delete_volumes() { [ -t 0 ] || return 1 echo "" - echo "🧹 Delete Docker volumes and Nexent data directories?" - echo " This removes persistent data under ROOT_DIR, including elasticsearch, postgresql, redis, minio, scripts, and supabase volumes." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🧹 是否删除 Docker volumes 和 Nexent 数据目录？" + echo " 这会删除 ROOT_DIR 下的持久化数据，包括 elasticsearch、postgresql、redis、minio、scripts 和 supabase volumes。" + else + echo "🧹 Delete Docker volumes and Nexent data directories?" + echo " This removes persistent data under ROOT_DIR, including elasticsearch, postgresql, redis, minio, scripts, and supabase volumes." + fi local answer - read -r -p " Delete data volumes? [y/N]: " answer + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + read -r -p " 删除数据 volumes？[y/N]：" answer + else + read -r -p " Delete data volumes? [y/N]: " answer + fi answer="$(sanitize_input "$answer")" [[ "$answer" =~ ^[Yy]$ ]] } @@ -189,6 +233,69 @@ remove_docker_named_volumes() { fi } +monitoring_container_names() { + printf '%s\n' \ + nexent-otel-collector \ + nexent-phoenix \ + nexent-langfuse-worker \ + nexent-langfuse-web \ + nexent-langfuse-clickhouse \ + nexent-langfuse-minio \ + nexent-langfuse-redis \ + nexent-langfuse-postgres \ + nexent-grafana \ + nexent-tempo \ + nexent-zipkin +} + +monitoring_volume_names() { + printf '%s\n' \ + monitor_phoenix-data \ + monitor_langfuse-postgres-data \ + monitor_langfuse-clickhouse-data \ + monitor_langfuse-clickhouse-logs \ + monitor_langfuse-minio-data \ + monitor_langfuse-redis-data \ + monitor_grafana-data \ + monitor_tempo-data +} + +remove_docker_containers_by_name() { + command -v docker >/dev/null 2>&1 || return 0 + + local containers_to_remove=() + local container + while IFS= read -r container; do + [ -n "$container" ] || continue + if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$container"; then + containers_to_remove+=("$container") + fi + done + + if [ "${#containers_to_remove[@]}" -gt 0 ]; then + echo "🧹 Removing Docker containers: ${containers_to_remove[*]}" + docker rm -f "${containers_to_remove[@]}" >/dev/null 2>&1 || true + fi +} + +remove_docker_volumes_by_name() { + command -v docker >/dev/null 2>&1 || return 0 + + local volumes_to_remove=() + local volume + while IFS= read -r volume; do + [ -n "$volume" ] || continue + if docker volume ls --format '{{.Name}}' 2>/dev/null | grep -qx "$volume"; then + volumes_to_remove+=("$volume") + fi + done + + if [ "${#volumes_to_remove[@]}" -gt 0 ]; then + echo "🧹 Removing Docker volumes: ${volumes_to_remove[*]}" + docker volume rm -f "${volumes_to_remove[@]}" >/dev/null 2>&1 || true + fi +} + docker_compose_down_file() { local compose_file="$1" local use_project_name="$2" @@ -204,6 +311,9 @@ docker_compose_down_file() { if [ -f "$ROOT_ENV_FILE" ]; then env_file_args=(--env-file "$ROOT_ENV_FILE") fi + if [ "$(basename "$compose_file")" = "docker-compose-monitoring.yml" ] && [ -f "$MONITORING_ENV_FILE" ]; then + env_file_args+=(--env-file "$MONITORING_ENV_FILE") + fi if [ "$use_project_name" = "true" ]; then $docker_compose_command "${env_file_args[@]}" -p nexent -f "$compose_file" down --remove-orphans "${volume_args[@]}" || true @@ -218,7 +328,11 @@ remove_nexent_data_dirs() { root_dir="${root_dir%/}" if [ -z "$root_dir" ] || [ "$root_dir" = "/" ]; then - echo "❌ Refusing to remove unsafe ROOT_DIR: ${root_dir:-}" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "❌ 拒绝删除不安全的 ROOT_DIR：${root_dir:-}" + else + echo "❌ Refusing to remove unsafe ROOT_DIR: ${root_dir:-}" + fi return 1 fi @@ -237,7 +351,11 @@ remove_nexent_data_dirs() { local dir for dir in "${dirs[@]}"; do if [ -e "$dir" ]; then - echo "🧹 Removing data directory: $dir" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🧹 正在删除数据目录：$dir" + else + echo "🧹 Removing data directory: $dir" + fi rm -rf "$dir" fi done @@ -251,14 +369,30 @@ main() { resolve_compose_command - echo "🛑 Stopping and removing Docker deployment..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "🛑 正在停止并删除 Docker 部署..." + else + echo "🛑 Stopping and removing Docker deployment..." + fi if [ "$remove_volumes" = "true" ]; then - echo "⚠️ Data volumes will be deleted." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "⚠️ 数据 volumes 将被删除。" + else + echo "⚠️ Data volumes will be deleted." + fi else - echo "ℹ️ Data volumes will be preserved." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "ℹ️ 数据 volumes 将被保留。" + else + echo "ℹ️ Data volumes will be preserved." + fi fi docker_compose_down_file "$COMPOSE_DIR/docker-compose-monitoring.yml" false "$remove_volumes" + remove_docker_containers_by_name < <(monitoring_container_names) + if [ "$remove_volumes" = "true" ]; then + remove_docker_volumes_by_name < <(monitoring_volume_names) + fi docker_compose_down_file "$COMPOSE_DIR/docker-compose-supabase.prod.yml" true "$remove_volumes" docker_compose_down_file "$COMPOSE_DIR/docker-compose-supabase.yml" true "$remove_volumes" docker_compose_down_file "$COMPOSE_DIR/docker-compose.prod.yml" true "$remove_volumes" @@ -269,7 +403,11 @@ main() { remove_nexent_data_dirs fi - echo "✅ Docker deployment removed." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "✅ Docker 部署已删除。" + else + echo "✅ Docker deployment removed." + fi } main diff --git a/deploy/env/.env.example b/deploy/env/.env.example index 2d95c8972..08761efa8 100644 --- a/deploy/env/.env.example +++ b/deploy/env/.env.example @@ -160,45 +160,10 @@ WORKER_CONCURRENCY=4 # Skills Configuration SKILLS_PATH=/mnt/nexent-data/skills -# Telemetry and Monitoring Configuration (OTLP Protocol) -# Enable OpenTelemetry monitoring for agent observability -ENABLE_TELEMETRY=false -# Provider profile: otlp, phoenix, langfuse, langsmith, grafana, zipkin -MONITORING_PROVIDER=otlp -MONITORING_PROJECT_NAME=nexent -# Browser-accessible monitoring UI URL. Leave empty to hide the frontend entry. -MONITORING_DASHBOARD_URL= -# Trace payload capture mode: -# summary: bounded preview + type/size/count metadata; metrics: metadata only; full: full preview capped by max chars. -# MAX_CHARS limits preview length; MAX_ITEMS limits dict/list preview items. -MONITORING_TRACE_CONTENT_MODE=full -MONITORING_TRACE_MAX_CHARS=4000 -MONITORING_TRACE_MAX_ITEMS=20 -# Service name for identifying traces in observability platforms -OTEL_SERVICE_NAME=nexent-backend -OTEL_EXPORTER_OTLP_ENDPOINT=http://nexent-otel-collector:4318 -# Optional signal-specific endpoints. Leave empty unless the backend requires them. -OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= -OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= -# Protocol: "http" or "grpc" -OTEL_EXPORTER_OTLP_PROTOCOL=http - -# Authentication headers (format: key1=value1,key2=value2) -# Prefer platform-specific variables when using the Collector. -OTEL_EXPORTER_OTLP_HEADERS= -OTEL_EXPORTER_OTLP_AUTHORIZATION= -OTEL_EXPORTER_OTLP_X_API_KEY= -OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION= -OTEL_EXPORTER_OTLP_METRICS_ENABLED=true -MONITORING_INSTRUMENT_REQUESTS=false -# FastAPI endpoint monitoring filters. Values are comma-separated regex patterns. -# Excluded URLs are always skipped. If included URLs is empty, all non-excluded endpoints are monitored. -# If included URLs is non-empty, only matching endpoints are monitored. -MONITORING_FASTAPI_INCLUDED_URLS= -MONITORING_FASTAPI_EXCLUDED_URLS= -MONITORING_FASTAPI_EXCLUDE_SPANS=receive,send - -TELEMETRY_SAMPLE_RATE=1.0 +# Public base URL used to generate conversation share links in the web UI. +# Leave empty to use the current browser origin. Set this when users access Nexent +# through a public domain or reverse proxy that differs from the internal address. +# SHARE_BASE_URL=https://your-domain.com # Market Backend Address MARKET_BACKEND=http://60.204.251.153:8010 @@ -223,6 +188,11 @@ WECHAT_OAUTH_APP_SECRET= OAUTH_SSL_VERIFY=true OAUTH_CA_BUNDLE= OAUTH_CALLBACK_BASE_URL=http://localhost:30000 +# Supported values: +# - disabled: hide OAuth login entries and disable automatic OAuth redirects. +# - button: show configured OAuth providers as optional login entries. +# - force: automatically redirect when exactly one OAuth provider is configured. +OAUTH_LOGIN_MODE=button # Asset owner role (opt-in; default false). Set true to enable ASSET_OWNER. ENABLE_ASSET_OWNER_ROLE=false diff --git a/deploy/docker/assets/monitoring/monitoring.env.example b/deploy/env/monitoring.env.example similarity index 63% rename from deploy/docker/assets/monitoring/monitoring.env.example rename to deploy/env/monitoring.env.example index 17f75a3c9..17d9281cd 100644 --- a/deploy/docker/assets/monitoring/monitoring.env.example +++ b/deploy/env/monitoring.env.example @@ -1,22 +1,46 @@ -# Monitoring stack selector for ./start-monitoring.sh. -# Supported values: otlp, collector, phoenix, langfuse, langsmith, grafana, zipkin. +# Monitoring provider selected by deploy/docker/deploy.sh or Helm values. +# Supported values: otlp, phoenix, langfuse, langsmith, grafana, zipkin. +ENABLE_TELEMETRY=false MONITORING_PROVIDER=otlp +MONITORING_DASHBOARD_URL= +MONITORING_PROJECT_NAME=nexent +OTEL_SERVICE_NAME=nexent-backend +OTEL_EXPORTER_OTLP_ENDPOINT= +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= +OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= +OTEL_EXPORTER_OTLP_PROTOCOL=http +OTEL_EXPORTER_OTLP_HEADERS= +OTEL_EXPORTER_OTLP_AUTHORIZATION= +OTEL_EXPORTER_OTLP_X_API_KEY= +OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION= +OTEL_EXPORTER_OTLP_METRICS_ENABLED=true +MONITORING_INSTRUMENT_REQUESTS=false +MONITORING_FASTAPI_INCLUDED_URLS=/agent/run +MONITORING_FASTAPI_EXCLUDED_URLS= +MONITORING_FASTAPI_EXCLUDE_SPANS=receive,send +TELEMETRY_SAMPLE_RATE=1.0 +MONITORING_TRACE_CONTENT_MODE=full +MONITORING_TRACE_MAX_CHARS=4000 +MONITORING_TRACE_MAX_ITEMS=20 OTEL_COLLECTOR_GRPC_PORT=4317 OTEL_COLLECTOR_HTTP_PORT=4318 OTEL_COLLECTOR_CONFIG_FILE= OTEL_COLLECTOR_VERSION=0.151.0 -# Local Phoenix stack. Used by: ./start-monitoring.sh --stack phoenix +# Local Phoenix stack. PHOENIX_VERSION=15 PHOENIX_PORT=6006 PHOENIX_GRPC_HOST_PORT=4319 +K8S_PHOENIX_NODE_PORT=30006 -# Local Langfuse stack. Used by: ./start-monitoring.sh --stack langfuse +# Local Langfuse stack. # Defaults are for local development only. Replace secrets before production use. LANGFUSE_VERSION=3 LANGFUSE_PORT=3001 LANGFUSE_NEXTAUTH_URL=http://localhost:3001 +K8S_LANGFUSE_NEXTAUTH_URL=http://localhost:30001 +K8S_LANGFUSE_NODE_PORT=30001 LANGFUSE_NEXTAUTH_SECRET=nexent-langfuse-secret LANGFUSE_SALT=nexent-langfuse-salt LANGFUSE_ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 @@ -40,6 +64,7 @@ LANGFUSE_POSTGRES_PORT=5440 LANGFUSE_CLICKHOUSE_VERSION=26.3-alpine LANGFUSE_CLICKHOUSE_USER=clickhouse LANGFUSE_CLICKHOUSE_PASSWORD=clickhouse +LANGFUSE_CLICKHOUSE_CLUSTER_ENABLED=false LANGFUSE_CLICKHOUSE_HTTP_PORT=8124 LANGFUSE_CLICKHOUSE_NATIVE_PORT=9002 LANGFUSE_MINIO_VERSION=RELEASE.2023-12-20T01-00-02Z @@ -52,21 +77,23 @@ LANGFUSE_REDIS_AUTH=myredissecret LANGFUSE_REDIS_VERSION=alpine LANGFUSE_REDIS_PORT=6380 -# Online LangSmith forwarding. Used by: ./start-monitoring.sh --stack langsmith +# Online LangSmith forwarding. # LangSmith currently ingests OTLP traces. Metrics remain in the Collector debug pipeline. LANGSMITH_API_KEY= LANGSMITH_PROJECT=nexent LANGSMITH_OTLP_TRACES_ENDPOINT=https://api.smith.langchain.com/otel/v1/traces -# Local Grafana stack. Used by: ./start-monitoring.sh --stack grafana +# Local Grafana stack. GRAFANA_VERSION=12.4 GRAFANA_PORT=3002 +K8S_GRAFANA_NODE_PORT=30002 GRAFANA_ADMIN_USER=admin GRAFANA_ADMIN_PASSWORD=nexent@4321 GRAFANA_DEFAULT_LANGUAGE=zh-Hans TEMPO_VERSION=2.10.5 TEMPO_PORT=3200 -# Local Zipkin stack. Used by: ./start-monitoring.sh --stack zipkin +# Local Zipkin stack. ZIPKIN_VERSION=latest ZIPKIN_PORT=9411 +K8S_ZIPKIN_NODE_PORT=30011 diff --git a/deploy/images/build.sh b/deploy/images/build.sh index 8a7459910..25ad6ae60 100755 --- a/deploy/images/build.sh +++ b/deploy/images/build.sh @@ -19,10 +19,13 @@ COMPONENTS="" PLATFORM="" VERSION="$(deployment_read_version)" REGISTRY="general" +REPO_PREFIX="nexent" +MAINLAND_REPO_PREFIX="ccr.ccs.tencentyun.com/nexent-hub" DEPENDENCY_VARIANT="cpu" TERMINAL_VARIANT="slim" PUSH=false LOAD=false +NO_CACHE=false DRY_RUN=false INTERACTIVE=false ARGS_COUNT=$# @@ -33,6 +36,37 @@ if [ "$ARGS_COUNT" -eq 0 ] && [ -t 0 ]; then fi usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + cat <<'USAGE' +用法：deploy/images/build.sh [选项] + +选项： + --images LIST 逗号分隔的镜像列表：all,main,web,data-process,mcp,terminal,docs + --image IMAGE 兼容参数，等同于只传一个 --images + --all 构建全部镜像 + --main 构建 nexent/nexent + --web 构建 nexent/nexent-web + --data-process 构建 nexent/nexent-data-process + --mcp 构建 nexent/nexent-mcp + --terminal 构建 nexent/nexent-ubuntu-terminal + --docs 构建 nexent/nexent-docs + --components LIST 将部署组件兼容映射为镜像列表 + --platform linux/amd64|linux/arm64|linux/amd64,linux/arm64 + --version VERSION 镜像 tag，例如 v2.2.1 或 latest。默认读取根目录 VERSION + --registry general|mainland + --dependency-variant cpu|gpu + data-process 依赖变体，默认 cpu + --terminal-variant slim|conda + terminal 镜像变体，默认 slim + --push + --load + --no-cache + --dry-run + --interactive 交互式选择镜像、版本和镜像源 +USAGE + return + fi + cat <<'USAGE' Usage: deploy/images/build.sh [options] @@ -56,6 +90,7 @@ Options: terminal image variant. Defaults to slim. --push --load + --no-cache --dry-run --interactive Prompt for images, version, and registry. USAGE @@ -80,10 +115,19 @@ while [ $# -gt 0 ]; do --terminal-variant) TERMINAL_VARIANT="$2"; shift 2 ;; --push) PUSH=true; shift ;; --load) LOAD=true; shift ;; + --no-cache) NO_CACHE=true; shift ;; --dry-run) DRY_RUN=true; shift ;; --interactive) INTERACTIVE=true; shift ;; --help|-h) usage; exit 0 ;; - *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "未知选项：$1" >&2 + else + echo "Unknown option: $1" >&2 + fi + usage >&2 + exit 1 + ;; esac done @@ -127,7 +171,11 @@ select_images_from_csv() { add_image_if_missing "$normalized" ;; *) - echo "Unsupported image: $normalized" >&2 + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的镜像：$normalized" >&2 + else + echo "Unsupported image: $normalized" >&2 + fi exit 1 ;; esac @@ -140,12 +188,12 @@ image_tui_multiselect() { local images=(main web data-process mcp terminal docs) local details=( - "backend API service" - "Next.js frontend" - "document parsing and vectorization worker" - "MCP proxy image" - "OpenSSH terminal tool image" - "VitePress documentation site" + "$(deployment_i18n image_build.detail.main)" + "$(deployment_i18n image_build.detail.web)" + "$(deployment_i18n image_build.detail.data_process)" + "$(deployment_i18n image_build.detail.mcp)" + "$(deployment_i18n image_build.detail.terminal)" + "$(deployment_i18n image_build.detail.docs)" ) local selected=(1 1 0 0 0 0) local cursor=0 @@ -153,8 +201,13 @@ image_tui_multiselect() { image_tui_render() { printf '\033[2J\033[H' - printf 'Select images to build\n' - printf 'Use Up/Down or j/k to move, Space to toggle, Enter to confirm, q to quit.\n\n' + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + printf '选择要构建的镜像\n' + printf '使用 Up/Down 或 j/k 移动，空格切换，Enter 确认，q 退出。\n\n' + else + printf 'Select images to build\n' + printf 'Use Up/Down or j/k to move, Space to toggle, Enter to confirm, q to quit.\n\n' + fi local row marker check for row in "${!images[@]}"; do marker=" " @@ -207,7 +260,11 @@ image_tui_multiselect() { q|Q) printf '\033[?25h' printf '\033[2J\033[H' - echo "Image build configuration cancelled." >&2 + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "已取消镜像构建配置。" >&2 + else + echo "Image build configuration cancelled." >&2 + fi return 130 ;; esac @@ -220,36 +277,77 @@ run_interactive_configuration() { local root_version root_version="$(deployment_read_version)" - echo "Nexent image build configuration" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Nexent 镜像构建配置" + else + echo "Nexent image build configuration" + fi echo "" if [ -z "$IMAGES" ] && [ "${#REQUESTED_IMAGES[@]}" -eq 0 ] && [ -z "$COMPONENTS" ] && [ "$IMAGE" = "all" ]; then if [ -t 0 ]; then image_tui_multiselect || return $? else - echo "Images:" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "镜像：" + else + echo "Images:" + fi echo " main, web, data-process, mcp, terminal, docs" - IMAGES="$(prompt_choice "Enter images (default: main,web): " "main,web")" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + IMAGES="$(prompt_choice "请输入镜像（默认：main,web）：" "main,web")" + else + IMAGES="$(prompt_choice "Enter images (default: main,web): " "main,web")" + fi fi fi - echo "Image version:" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "镜像版本：" + else + echo "Image version:" + fi echo " 1) latest" - echo " 2) Root VERSION ($root_version)" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo " 2) 根目录 VERSION ($root_version)" + else + echo " 2) Root VERSION ($root_version)" + fi local version_choice - version_choice="$(prompt_choice "Choose version [1/2] (default: 1): " "1")" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + version_choice="$(prompt_choice "选择版本 [1/2]（默认：1）：" "1")" + else + version_choice="$(prompt_choice "Choose version [1/2] (default: 1): " "1")" + fi case "$version_choice" in 1|latest|"") VERSION="latest" ;; 2|root|version|VERSION) VERSION="$root_version" ;; - *) echo "Unsupported version choice: $version_choice" >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的版本选择：$version_choice" >&2 + else + echo "Unsupported version choice: $version_choice" >&2 + fi + exit 1 + ;; esac echo "" - echo "Image registry:" - echo " 1) general (nexent/*)" - echo " 2) mainland (ccr.ccs.tencentyun.com/nexent-hub/*)" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "镜像源：" + echo " 1) general（公共镜像源）" + echo " 2) mainland（中国大陆镜像源）" + else + echo "Image source:" + echo " 1) general (public image sources)" + echo " 2) mainland (mainland China image sources and build mirrors)" + fi local registry_choice - registry_choice="$(prompt_choice "Choose registry [1/2] (default: 1): " "1")" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + registry_choice="$(prompt_choice "选择镜像源 [1/2]（默认：1）：" "1")" + else + registry_choice="$(prompt_choice "Choose image source [1/2] (default: 1): " "1")" + fi case "$registry_choice" in 2|mainland) REGISTRY="mainland" ;; 1|general|"") REGISTRY="general" ;; @@ -264,26 +362,45 @@ fi case "$REGISTRY" in general) - REPO_PREFIX="nexent" PY_MIRROR_ARGS=() WEB_MIRROR_ARGS=() ;; mainland) - REPO_PREFIX="ccr.ccs.tencentyun.com/nexent-hub" PY_MIRROR_ARGS=(--build-arg MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple --build-arg APT_MIRROR=tsinghua) WEB_MIRROR_ARGS=(--build-arg MIRROR=https://registry.npmmirror.com --build-arg APK_MIRROR=tsinghua) ;; - *) echo "Unsupported registry: $REGISTRY" >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的 registry：$REGISTRY" >&2 + else + echo "Unsupported registry: $REGISTRY" >&2 + fi + exit 1 + ;; esac case "$DEPENDENCY_VARIANT" in cpu|gpu) ;; - *) echo "Unsupported data-process dependency variant: $DEPENDENCY_VARIANT" >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的 data-process 依赖变体：$DEPENDENCY_VARIANT" >&2 + else + echo "Unsupported data-process dependency variant: $DEPENDENCY_VARIANT" >&2 + fi + exit 1 + ;; esac case "$TERMINAL_VARIANT" in slim|conda) ;; - *) echo "Unsupported terminal variant: $TERMINAL_VARIANT" >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的 terminal 变体：$TERMINAL_VARIANT" >&2 + else + echo "Unsupported terminal variant: $TERMINAL_VARIANT" >&2 + fi + exit 1 + ;; esac run_cmd() { @@ -360,10 +477,16 @@ build_one() { local dockerfile="$2" shift 2 local tag="$REPO_PREFIX/$name:$VERSION" + if [ "$PUSH" = true ] && [ "$REGISTRY" = "mainland" ]; then + tag="$MAINLAND_REPO_PREFIX/$name:$VERSION" + fi local cmd=(docker buildx build) if [ -n "$PLATFORM" ]; then cmd+=(--platform "$PLATFORM") fi + if [ "$NO_CACHE" = true ] || { [ "$REGISTRY" = "mainland" ] && [ "$name" = "nexent-web" ]; }; then + cmd+=(--no-cache) + fi cmd+=(-t "$tag" -f "$dockerfile") if [ "$PUSH" = true ]; then cmd+=(--push) @@ -393,7 +516,14 @@ build_selected_image() { [ "$TERMINAL_VARIANT" = "conda" ] && image_name="nexent-ubuntu-terminal-conda" build_one "$image_name" "$DOCKERFILE_DIR/terminal/Dockerfile" --build-arg TERMINAL_VARIANT="$TERMINAL_VARIANT" ;; - *) echo "Unsupported image: $1" >&2; exit 1 ;; + *) + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "不支持的镜像：$1" >&2 + else + echo "Unsupported image: $1" >&2 + fi + exit 1 + ;; esac } @@ -421,7 +551,11 @@ select_images_from_components() { add_image_if_missing terminal ;; *) - echo "Unsupported component for image build: $normalized" >&2 + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "镜像构建不支持该组件：$normalized" >&2 + else + echo "Unsupported component for image build: $normalized" >&2 + fi exit 1 ;; esac @@ -450,7 +584,11 @@ else fi if [ "${#SELECTED_IMAGES[@]}" -eq 0 ]; then - echo "No Nexent images selected for build." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "未选择任何 Nexent 镜像进行构建。" + else + echo "No Nexent images selected for build." + fi exit 0 fi diff --git a/deploy/k8s/deploy.sh b/deploy/k8s/deploy.sh index 15dd35540..a335df933 100755 --- a/deploy/k8s/deploy.sh +++ b/deploy/k8s/deploy.sh @@ -68,14 +68,25 @@ case "${1:-}" in shift ;; delete|delete-all|clean) - echo "K8s uninstall and cleanup have moved to uninstall.sh." - echo "Use: bash uninstall.sh ${1}" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "K8s 卸载和清理已迁移到 uninstall.sh。" + echo "请使用：bash uninstall.sh ${1}" + else + echo "K8s uninstall and cleanup have moved to uninstall.sh." + echo "Use: bash uninstall.sh ${1}" + fi exit 1 ;; *) - echo "Unknown command: $1" - echo "Usage: $0 [apply] [options]" - echo "Uninstall: bash uninstall.sh" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "未知命令：$1" + echo "用法：$0 [apply] [选项]" + echo "卸载：bash uninstall.sh" + else + echo "Unknown command: $1" + echo "Usage: $0 [apply] [options]" + echo "Uninstall: bash uninstall.sh" + fi exit 1 ;; esac @@ -89,26 +100,32 @@ while [[ $# -gt 0 ]]; do case "$1" in --is-mainland) IS_MAINLAND="$2" + K8S_IS_MAINLAND_EXPLICIT="true" shift 2 ;; --version) APP_VERSION="$2" + K8S_APP_VERSION_EXPLICIT="true" shift 2 ;; --deployment-version) DEPLOYMENT_VERSION="$2" + K8S_DEPLOYMENT_VERSION_EXPLICIT="true" shift 2 ;; --persistence-mode) PERSISTENCE_MODE="$2" + K8S_PERSISTENCE_MODE_EXPLICIT="true" shift 2 ;; --storage-class|--storageclass|--storage-class-name|--sc) STORAGE_CLASS_NAME="$2" + K8S_STORAGE_CLASS_NAME_EXPLICIT="true" shift 2 ;; --local-path) LOCAL_PATH="$2" + K8S_LOCAL_PATH_EXPLICIT="true" shift 2 ;; --local-node-name) @@ -117,6 +134,7 @@ while [[ $# -gt 0 ]]; do ;; --existing-claim-prefix) EXISTING_CLAIM_PREFIX="$2" + K8S_EXISTING_CLAIM_PREFIX_EXPLICIT="true" shift 2 ;; --wait-timeout) @@ -133,7 +151,9 @@ while [[ $# -gt 0 ]]; do done cd "$SCRIPT_DIR" -deployment_source_root_env "$PROJECT_ROOT" "$PROJECT_ROOT/docker" || exit 1 +if [ "$COMMAND" != "help" ]; then + deployment_source_root_env "$PROJECT_ROOT" "$PROJECT_ROOT/docker" || exit 1 +fi # Helper function to sanitize input (remove Windows CR) sanitize_input() { @@ -142,6 +162,8 @@ sanitize_input() { } apply_deployment_common_config() { + load_deploy_options + if [ -z "$APP_VERSION" ]; then APP_VERSION=$(get_app_version) fi @@ -172,6 +194,7 @@ apply_deployment_common_config() { esac deployment_apply_image_source + deployment_prepare_monitoring_env k8s || return 1 deployment_render_helm_values "$GENERATED_VALUES" render_k8s_runtime_config_values "$GENERATED_RUNTIME_VALUES" render_persistence_values "$GENERATED_PERSISTENCE_VALUES" @@ -201,7 +224,7 @@ render_one_persistence_values() { printf ' mode: "%s"\n' "$PERSISTENCE_MODE" printf ' storageClassName: "%s"\n' "$storage_class" printf ' accessModes:\n' - printf ' - ReadWriteOnce\n' + printf ' - ReadWriteMany\n' printf ' localPath: "%s/%s"\n' "$LOCAL_PATH" "$component" printf ' existingClaim: "%s"\n' "$(persistence_existing_claim "$component")" printf ' storage:\n' @@ -222,7 +245,7 @@ render_monitoring_persistence_values() { printf ' mode: "%s"\n' "$PERSISTENCE_MODE" printf ' storageClassName: "%s"\n' "$storage_class" printf ' accessModes:\n' - printf ' - ReadWriteOnce\n' + printf ' - ReadWriteMany\n' printf ' localPath: "%s"\n' "$LOCAL_PATH" printf ' existingClaimPrefix: "%s"\n' "$EXISTING_CLAIM_PREFIX" } >> "$output_file" @@ -240,7 +263,7 @@ render_shared_storage_persistence_values() { printf ' mode: "%s"\n' "$PERSISTENCE_MODE" printf ' storageClassName: "%s"\n' "$storage_class" printf ' accessModes:\n' - printf ' - ReadWriteOnce\n' + printf ' - ReadWriteMany\n' printf ' workspace:\n' printf ' size: "10Gi"\n' printf ' localPath: "/var/lib/nexent"\n' @@ -472,31 +495,6 @@ render_k8s_runtime_config_values() { printf ' queues: %s\n' "$(yaml_quote "$(env_or_default QUEUES "process_q,forward_q")")" printf ' workerName: %s\n' "$(yaml_quote "$(env_or_default WORKER_NAME "")")" printf ' workerConcurrency: %s\n' "$(yaml_quote "$(env_or_default WORKER_CONCURRENCY "4")")" - echo " telemetry:" - printf ' enabled: %s\n' "$(yaml_quote "$(env_or_default ENABLE_TELEMETRY "false")")" - printf ' provider: %s\n' "$(yaml_quote "$(env_or_default MONITORING_PROVIDER "otlp")")" - printf ' projectName: %s\n' "$(yaml_quote "$(env_or_default MONITORING_PROJECT_NAME "")")" - printf ' serviceName: %s\n' "$(yaml_quote "$(env_or_default OTEL_SERVICE_NAME "nexent-backend")")" - printf ' otlpEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_ENDPOINT "http://nexent-otel-collector:4318")")" - printf ' otlpTracesEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_TRACES_ENDPOINT "")")" - printf ' otlpMetricsEndpoint: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_METRICS_ENDPOINT "")")" - printf ' otlpProtocol: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_PROTOCOL "http")")" - printf ' otlpHeaders: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_HEADERS "")")" - printf ' otlpAuthorization: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_AUTHORIZATION "")")" - printf ' otlpApiKey: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_X_API_KEY "")")" - printf ' otlpLangfuseIngestionVersion: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_LANGFUSE_INGESTION_VERSION "")")" - printf ' langsmithApiKey: %s\n' "$(yaml_quote "$(env_or_default LANGSMITH_API_KEY "")")" - printf ' langsmithProject: %s\n' "$(yaml_quote "$(env_or_default LANGSMITH_PROJECT "")")" - printf ' otlpMetricsEnabled: %s\n' "$(yaml_quote "$(env_or_default OTEL_EXPORTER_OTLP_METRICS_ENABLED "true")")" - printf ' instrumentRequests: %s\n' "$(yaml_quote "$(env_or_default MONITORING_INSTRUMENT_REQUESTS "false")")" - printf ' fastapiIncludedUrls: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_INCLUDED_URLS "")")" - printf ' fastapiExcludedUrls: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_EXCLUDED_URLS "")")" - printf ' fastapiExcludeSpans: %s\n' "$(yaml_quote "$(env_or_default MONITORING_FASTAPI_EXCLUDE_SPANS "receive,send")")" - printf ' dashboardUrl: %s\n' "$(yaml_quote "$(env_or_default MONITORING_DASHBOARD_URL "")")" - printf ' telemetrySampleRate: %s\n' "$(yaml_quote "$(env_or_default TELEMETRY_SAMPLE_RATE "1.0")")" - printf ' traceContentMode: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_CONTENT_MODE "full")")" - printf ' traceMaxChars: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_MAX_CHARS "4000")")" - printf ' traceMaxItems: %s\n' "$(yaml_quote "$(env_or_default MONITORING_TRACE_MAX_ITEMS "20")")" echo " oauth:" printf ' githubClientId: %s\n' "$(yaml_quote "$(env_or_default GITHUB_OAUTH_CLIENT_ID "")")" printf ' githubClientSecret: %s\n' "$(yaml_quote "$(env_or_default GITHUB_OAUTH_CLIENT_SECRET "")")" @@ -509,6 +507,7 @@ render_k8s_runtime_config_values() { printf ' sslVerify: %s\n' "$(yaml_quote "$(env_or_default OAUTH_SSL_VERIFY "true")")" printf ' caBundle: %s\n' "$(yaml_quote "$(env_or_default OAUTH_CA_BUNDLE "")")" printf ' callbackBaseUrl: %s\n' "$(yaml_quote "$(env_or_default OAUTH_CALLBACK_BASE_URL "http://localhost:30000")")" + printf ' loginMode: %s\n' "$(yaml_quote "$(env_or_default OAUTH_LOGIN_MODE "button")")" echo " cas:" printf ' enabled: %s\n' "$(yaml_quote "$(env_or_default CAS_ENABLED "false")")" printf ' serverUrl: %s\n' "$(yaml_quote "$(env_or_default CAS_SERVER_URL "")")" @@ -554,17 +553,92 @@ get_app_version() { # Persist deployment options to file persist_deploy_options() { + deployment_persist_local_config "$DEPLOY_OPTIONS_FILE" { - echo "APP_VERSION=\"${APP_VERSION}\"" - echo "IS_MAINLAND=\"${IS_MAINLAND_SAVED}\"" - echo "DEPLOYMENT_VERSION=\"${VERSION_CHOICE_SAVED}\"" - } > "$DEPLOY_OPTIONS_FILE" + printf 'k8s:\n' + printf ' appVersion: %s\n' "$(yaml_quote "$APP_VERSION")" + printf ' isMainland: %s\n' "$(yaml_quote "$IS_MAINLAND_SAVED")" + printf ' deploymentVersion: %s\n' "$(yaml_quote "$VERSION_CHOICE_SAVED")" + printf ' persistenceMode: %s\n' "$(yaml_quote "$PERSISTENCE_MODE")" + printf ' storageClassName: %s\n' "$(yaml_quote "$STORAGE_CLASS_NAME")" + printf ' localPath: %s\n' "$(yaml_quote "$LOCAL_PATH")" + printf ' existingClaimPrefix: %s\n' "$(yaml_quote "$EXISTING_CLAIM_PREFIX")" + } >> "$DEPLOY_OPTIONS_FILE" +} + +deploy_options_unquote() { + local value + value="$(deployment_trim "$1")" + value="${value%$'\r'}" + value="${value%%#*}" + value="$(deployment_trim "$value")" + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + printf '%s' "$value" +} + +apply_loaded_deploy_option() { + local key="$1" + local value="$2" + case "$key" in + APP_VERSION|appVersion) + [ -n "${K8S_APP_VERSION_EXPLICIT:-}" ] || APP_VERSION="$value" + ;; + IS_MAINLAND|isMainland) + [ -n "${K8S_IS_MAINLAND_EXPLICIT:-}" ] || IS_MAINLAND="$value" + ;; + DEPLOYMENT_VERSION|deploymentVersion) + [ -n "${K8S_DEPLOYMENT_VERSION_EXPLICIT:-}" ] || DEPLOYMENT_VERSION="$value" + ;; + PERSISTENCE_MODE|persistenceMode) + [ -n "${K8S_PERSISTENCE_MODE_EXPLICIT:-}" ] || PERSISTENCE_MODE="$value" + ;; + STORAGE_CLASS_NAME|storageClassName) + [ -n "${K8S_STORAGE_CLASS_NAME_EXPLICIT:-}" ] || STORAGE_CLASS_NAME="$value" + ;; + LOCAL_PATH|localPath) + [ -n "${K8S_LOCAL_PATH_EXPLICIT:-}" ] || LOCAL_PATH="$value" + ;; + EXISTING_CLAIM_PREFIX|existingClaimPrefix) + [ -n "${K8S_EXISTING_CLAIM_PREFIX_EXPLICIT:-}" ] || EXISTING_CLAIM_PREFIX="$value" + ;; + esac } # Load deployment options from file if exists load_deploy_options() { + local line trimmed key value in_k8s_section if [ -f "$DEPLOY_OPTIONS_FILE" ]; then - source "$DEPLOY_OPTIONS_FILE" + in_k8s_section="false" + while IFS= read -r line || [ -n "$line" ]; do + trimmed="$(deployment_trim "${line%%#*}")" + [ -z "$trimmed" ] && continue + + if [[ "$trimmed" =~ ^([A-Z_][A-Z0-9_]*)=(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="$(deploy_options_unquote "${BASH_REMATCH[2]}")" + apply_loaded_deploy_option "$key" "$value" + continue + fi + + if [[ "$trimmed" =~ ^k8s:[[:space:]]*$ ]]; then + in_k8s_section="true" + continue + fi + + if [[ "$line" =~ ^[A-Za-z][A-Za-z0-9_]*:[[:space:]]* ]]; then + in_k8s_section="false" + continue + fi + + if [ "$in_k8s_section" = "true" ] && [[ "$line" =~ ^[[:space:]]+([A-Za-z][A-Za-z0-9_]*):[[:space:]]*(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="$(deploy_options_unquote "${BASH_REMATCH[2]}")" + apply_loaded_deploy_option "$key" "$value" + fi + done < "$DEPLOY_OPTIONS_FILE" fi } @@ -622,6 +696,7 @@ update_values_yaml() { echo "" deployment_apply_image_source + deployment_prepare_monitoring_env k8s || exit 1 deployment_render_helm_values "$GENERATED_VALUES" render_k8s_runtime_config_values "$GENERATED_RUNTIME_VALUES" render_persistence_values "$GENERATED_PERSISTENCE_VALUES" @@ -889,32 +964,24 @@ pull_mcp_image() { render_runtime_secret_values() { local gotrue_db_url - local runtime_config_hash - local backend_checksum - local minio_checksum - local supabase_checksum - local web_checksum - local ssh_checksum + local env_checksum local sql_checksum + local supabase_postgres_password + local supabase_secret_checksum - gotrue_db_url="$(env_or_default GOTRUE_DB_DATABASE_URL "postgres://supabase_auth_admin:$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")@$(env_or_default SUPABASE_POSTGRES_HOST "nexent-supabase-db"):$(env_or_default SUPABASE_POSTGRES_PORT "5436")/$(env_or_default SUPABASE_POSTGRES_DB "supabase")?search_path=auth&sslmode=disable")" - runtime_config_hash="$(deployment_sha256_file "$GENERATED_RUNTIME_VALUES")" + supabase_postgres_password="$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")" + gotrue_db_url="$(env_or_default GOTRUE_DB_DATABASE_URL "postgres://supabase_auth_admin:${supabase_postgres_password}@$(env_or_default SUPABASE_POSTGRES_HOST "nexent-supabase-db"):$(env_or_default SUPABASE_POSTGRES_PORT "5436")/$(env_or_default SUPABASE_POSTGRES_DB "supabase")?search_path=auth&sslmode=disable")" + env_checksum="$(deployment_env_values_checksum)" sql_checksum="$(sql_files_checksum)" - backend_checksum="$(deployment_sha256_string "runtime=${runtime_config_hash}|sql=${sql_checksum}|elastic=$(env_or_default ELASTICSEARCH_API_KEY "")|postgres=$(env_or_default NEXENT_POSTGRES_PASSWORD "nexent@4321")|minio=${MINIO_ACCESS_KEY}:${MINIO_SECRET_KEY}")" - minio_checksum="$(deployment_sha256_string "root=$(env_or_default MINIO_ROOT_USER "nexent"):$(env_or_default MINIO_ROOT_PASSWORD "nexent@4321")|client=${MINIO_ACCESS_KEY}:${MINIO_SECRET_KEY}")" - supabase_checksum="$(deployment_sha256_string "jwt=${JWT_SECRET:-}|base=${SECRET_KEY_BASE:-}|vault=${VAULT_ENC_KEY:-}|anon=${SUPABASE_ANON_KEY:-}|service=${SUPABASE_SERVICE_ROLE_KEY:-}|pg=$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")|db=${gotrue_db_url}")" - web_checksum="$(deployment_sha256_string "market=$(env_or_default MARKET_BACKEND "http://60.204.251.153:8010")|model=$(env_or_default MODEL_ENGINE_ENABLED "false")")" - ssh_checksum="$(deployment_sha256_string "ssh=$(env_or_default SSH_USERNAME "nexent"):$(env_or_default SSH_PASSWORD "nexent@2025")")" + supabase_secret_checksum="$(deployment_sha256_string "jwt=${JWT_SECRET:-}|secretKeyBase=${SECRET_KEY_BASE:-}|vault=${VAULT_ENC_KEY:-}|anon=${SUPABASE_ANON_KEY:-}|service=${SUPABASE_SERVICE_ROLE_KEY:-}|postgres=${supabase_postgres_password}|gotrue=${gotrue_db_url}")" { echo "global:" echo " rolloutChecksums:" - printf ' backend: %s\n' "$(yaml_quote "$backend_checksum")" - printf ' minio: %s\n' "$(yaml_quote "$minio_checksum")" - printf ' supabase: %s\n' "$(yaml_quote "$supabase_checksum")" - printf ' web: %s\n' "$(yaml_quote "$web_checksum")" - printf ' ssh: %s\n' "$(yaml_quote "$ssh_checksum")" + printf ' env: %s\n' "$(yaml_quote "$env_checksum")" printf ' sql: %s\n' "$(yaml_quote "$sql_checksum")" + printf ' supabaseSecret: %s\n' "$(yaml_quote "$supabase_secret_checksum")" + deployment_render_image_rollout_checksums echo "nexent-common:" echo " secrets:" printf ' elasticPassword: %s\n' "$(yaml_quote "$(env_or_default ELASTIC_PASSWORD "nexent@2025")")" @@ -935,18 +1002,22 @@ render_runtime_secret_values() { printf ' vaultEncKey: %s\n' "$(yaml_quote "$VAULT_ENC_KEY")" printf ' anonKey: %s\n' "$(yaml_quote "$SUPABASE_ANON_KEY")" printf ' serviceRoleKey: %s\n' "$(yaml_quote "$SUPABASE_SERVICE_ROLE_KEY")" - printf ' postgresPassword: %s\n' "$(yaml_quote "$(env_or_default SUPABASE_POSTGRES_PASSWORD "Huawei123")")" + printf ' postgresPassword: %s\n' "$(yaml_quote "$supabase_postgres_password")" printf ' gotrueDbUrl: %s\n' "$(yaml_quote "$gotrue_db_url")" fi } > "$GENERATED_SECRETS_VALUES" } apply() { - echo "Deploying Nexent using Helm..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在使用 Helm 部署 Nexent..." + else + echo "Deploying Nexent using Helm..." + fi # Step 1: Select deployment components, port policy and image source. apply_deployment_common_config - deployment_persist_local_config + persist_deploy_options # Step 2: Render generated values with image tags from selected environment update_values_yaml @@ -996,24 +1067,45 @@ apply() { # Step 5: Configure Terminal tool (OpenSSH) only when selected. if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "terminal"; then ENABLE_OPENSSH="true" - echo "Terminal tool will be enabled." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "将启用终端工具。" + else + echo "Terminal tool will be enabled." + fi # Ask for SSH credentials echo "" - echo "SSH credentials configuration:" - read -p "SSH Username (default: nexent): " ssh_username + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "SSH 凭据配置：" + read -p "SSH 用户名（默认：nexent）：" ssh_username + else + echo "SSH credentials configuration:" + read -p "SSH Username (default: nexent): " ssh_username + fi SSH_USERNAME="${ssh_username:-nexent}" - read -s -p "SSH Password (default: nexent@2025): " ssh_password + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + read -s -p "SSH 密码（默认：nexent@2025）：" ssh_password + else + read -s -p "SSH Password (default: nexent@2025): " ssh_password + fi echo "" SSH_PASSWORD="${ssh_password:-nexent@2025}" else ENABLE_OPENSSH="false" - echo "Terminal tool disabled." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "终端工具已禁用。" + else + echo "Terminal tool disabled." + fi fi echo "" # Step 6: Clean up stale PVs - echo "Checking for stale PersistentVolumes..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在检查残留 PersistentVolumes..." + else + echo "Checking for stale PersistentVolumes..." + fi for pv in nexent-workspace-pv nexent-skills-pv nexent-elasticsearch-pv nexent-postgresql-pv nexent-redis-pv nexent-minio-pv; do pv_status=$(kubectl get pv $pv -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") if [ "$pv_status" = "Released" ]; then @@ -1036,7 +1128,11 @@ apply() { # Step 7: Deploy using Helm ensure_namespace recreate_legacy_nexent_secret_for_helm_management - echo "Deploying Helm chart..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在部署 Helm chart..." + else + echo "Deploying Helm chart..." + fi helm_upgrade_release # Step 9: Wait for Elasticsearch to be ready and initialize API key @@ -1046,15 +1142,27 @@ apply() { echo "==========================================" local deploy_success=true - echo "Waiting for Elasticsearch deployment to be ready..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在等待 Elasticsearch deployment 就绪..." + else + echo "Waiting for Elasticsearch deployment to be ready..." + fi sleep 5 if wait_for_deployment_ready "nexent-elasticsearch"; then - echo "Elasticsearch deployment is ready." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Elasticsearch deployment 已就绪。" + else + echo "Elasticsearch deployment is ready." + fi # Initialize Elasticsearch API key only when it is missing, invalid, or explicitly refreshed. INIT_ES_SCRIPT="$SCRIPT_DIR/init-elasticsearch.sh" if [ -f "$INIT_ES_SCRIPT" ]; then - echo "Running Elasticsearch initialization script..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在运行 Elasticsearch 初始化脚本..." + else + echo "Running Elasticsearch initialization script..." + fi local es_key_before local es_key_after local es_key_output_file @@ -1067,11 +1175,19 @@ apply() { es_key_after="$es_key_before" fi rm -f "$es_key_output_file" - echo "Elasticsearch API key initialized successfully." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Elasticsearch API key 初始化成功。" + else + echo "Elasticsearch API key initialized successfully." + fi if [ "$es_key_before" != "$es_key_after" ]; then echo "" - echo "ELASTICSEARCH_API_KEY updated; refreshing Helm values and rolling affected backend services..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "ELASTICSEARCH_API_KEY 已更新；正在刷新 Helm values 并滚动受影响的后端服务..." + else + echo "ELASTICSEARCH_API_KEY updated; refreshing Helm values and rolling affected backend services..." + fi ELASTICSEARCH_API_KEY="$es_key_after" render_runtime_secret_values helm_upgrade_release @@ -1080,7 +1196,11 @@ apply() { deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "data-process" && backend_services="$backend_services data-process" echo "" - echo "Waiting for backend services to be ready..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在等待后端服务就绪..." + else + echo "Waiting for backend services to be ready..." + fi sleep 5 for svc in $backend_services; do echo " Waiting for nexent-$svc..." @@ -1092,26 +1212,46 @@ apply() { fi done else - echo "ELASTICSEARCH_API_KEY unchanged; backend rollout is not needed." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "ELASTICSEARCH_API_KEY 未变化；无需滚动后端服务。" + else + echo "ELASTICSEARCH_API_KEY unchanged; backend rollout is not needed." + fi fi else rm -f "$es_key_output_file" - echo "Error: Elasticsearch initialization script failed." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：Elasticsearch 初始化脚本执行失败。" + else + echo "Error: Elasticsearch initialization script failed." + fi deploy_success=false fi else - echo "Error: init-elasticsearch.sh not found at $INIT_ES_SCRIPT" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：未找到 init-elasticsearch.sh：$INIT_ES_SCRIPT" + else + echo "Error: init-elasticsearch.sh not found at $INIT_ES_SCRIPT" + fi deploy_success=false fi else - echo "Error: nexent-elasticsearch did not become ready within ${K8S_WAIT_TIMEOUT_SECONDS}s." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：nexent-elasticsearch 未能在 ${K8S_WAIT_TIMEOUT_SECONDS}s 内就绪。" + else + echo "Error: nexent-elasticsearch did not become ready within ${K8S_WAIT_TIMEOUT_SECONDS}s." + fi deploy_success=false fi if [ "$deploy_success" = false ]; then echo "" echo "==========================================" - echo " Deployment Failed!" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo " 部署失败！" + else + echo " Deployment Failed!" + fi echo "==========================================" exit 1 fi @@ -1125,30 +1265,79 @@ apply() { echo " Super Admin User Creation" echo "==========================================" if bash "$CREATE_SUADMIN_SCRIPT"; then - echo "Super admin user creation completed." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "超级管理员创建完成。" + else + echo "Super admin user creation completed." + fi else - echo "Warning: Super admin user creation failed, but continuing deployment." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "警告：超级管理员创建失败，但部署将继续。" + else + echo "Warning: Super admin user creation failed, but continuing deployment." + fi fi else - echo "Warning: create-suadmin.sh not found at $CREATE_SUADMIN_SCRIPT" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "警告：未找到 create-suadmin.sh：$CREATE_SUADMIN_SCRIPT" + else + echo "Warning: create-suadmin.sh not found at $CREATE_SUADMIN_SCRIPT" + fi fi fi # Save deployment options for future use persist_deploy_options - deployment_persist_local_config # Step 11: Pull MCP image after persisting deployment options pull_mcp_image - echo "Deployment completed successfully!" - echo "Access the application at: http://localhost:30000" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "部署完成！" + echo "应用访问地址：http://localhost:30000" + else + echo "Deployment completed successfully!" + echo "Access the application at: http://localhost:30000" + fi if [ "$ENABLE_OPENSSH" = "true" ]; then - echo "SSH Terminal at: localhost:30022" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "SSH Terminal 地址：localhost:30022" + else + echo "SSH Terminal at: localhost:30022" + fi fi } print_usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "用法：$0 [apply] [选项]" + echo "" + echo "使用 Helm 部署 Nexent K8s 资源。" + echo "" + echo "选项：" + echo " --components LIST 要部署的组件" + echo " --port-policy POLICY development 或 production" + echo " --image-source SOURCE general、mainland 或 local-latest" + echo " --image-registry-prefix P 镜像仓库前缀，例如 registry.example.com/nexent" + echo " --is-mainland Y|N 兼容旧参数，映射为 mainland/general 镜像源" + echo " --version VERSION 指定应用版本（未设置时自动从 const.py 检测）" + echo " --defaults 复用保存配置或内置默认值并跳过交互界面" + echo " --deployment-version VER 兼容旧部署版本：speed 或 full" + echo " --persistence-mode MODE local、dynamic 或 existing" + echo " --storage-class NAME 用于 PV/PVC 绑定的 StorageClass（别名：--storageclass、--storage-class-name、--sc）" + echo " --local-path PATH 本地 PV 基础路径" + echo " --local-node-name NAME 已废弃；local 模式使用 hostPath，不需要 nodeAffinity" + echo " --existing-claim-prefix P 现有 PVC 前缀，渲染为 P-" + echo " --wait-timeout SECONDS Kubernetes 部署等待超时（默认：600）" + echo " --rotate-secrets 强制轮换部署密钥" + echo " --refresh-es-key 强制重新创建 ELASTICSEARCH_API_KEY" + echo " --config 进入交互式部署配置界面" + echo " --help, -h 显示帮助信息" + echo "" + echo "卸载：bash uninstall.sh" + return + fi + echo "Usage: $0 [apply] [options]" echo "" echo "Deploy Nexent K8s resources using Helm." @@ -1157,8 +1346,10 @@ print_usage() { echo " --components LIST Components to deploy" echo " --port-policy POLICY development or production" echo " --image-source SOURCE general, mainland, or local-latest" + echo " --image-registry-prefix P Image registry prefix, e.g. registry.example.com/nexent" echo " --is-mainland Y|N Legacy alias for image source mainland/general" echo " --version VERSION Specify app version (auto-detected from const.py if not set)" + echo " --defaults Use saved config or built-in defaults and skip TUI" echo " --deployment-version VER Legacy deployment version: speed or full" echo " --persistence-mode MODE local, dynamic, or existing" echo " --storage-class NAME StorageClass for PV/PVC binding (aliases: --storageclass, --storage-class-name, --sc)" @@ -1168,6 +1359,7 @@ print_usage() { echo " --wait-timeout SECONDS Kubernetes deployment wait timeout (default: 600)" echo " --rotate-secrets Force rotation of deployment secrets" echo " --refresh-es-key Force recreation of ELASTICSEARCH_API_KEY" + echo " --config Open the interactive deployment configuration" echo " --help, -h Show this help message" echo "" echo "Uninstall: bash uninstall.sh" diff --git a/deploy/k8s/helm/nexent/README.md b/deploy/k8s/helm/nexent/README.md index 3b46c1cfb..72e7997bc 100644 --- a/deploy/k8s/helm/nexent/README.md +++ b/deploy/k8s/helm/nexent/README.md @@ -83,8 +83,8 @@ When `--persistence-mode local` is used, Nexent renders static PVs with `hostPat | `--registry-profile` | Legacy registry profile option | `general` or `mainland`; maps to `--image-source` | | `--monitoring-provider` | Provider when `monitoring` is selected | `otlp`, `phoenix`, `langfuse`, `langsmith`, `grafana`, `zipkin` | | `--use-local-config` | Reuse saved local deployment config | Flag | -| `--reconfigure` | Ignore saved local config and run full configuration | Flag | -| `--config` | Deployment config path | YAML file | +| `--reconfigure` | Run interactive configuration using saved local config as defaults | Flag | +| `--config` | Open the interactive deployment configuration | Flag | | `--is-mainland` | Legacy network location option | `Y` maps to `--image-source mainland`; `N` maps to `general` | | `--version` | Application version | Version tag (auto-detected from `backend/consts/const.py` if not set) | | `--deployment-version` | Legacy deployment version | `speed` maps to `infrastructure,application`; `full` adds `supabase` | diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml index 0f1a4a5a3..e70602aaf 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml @@ -25,7 +25,7 @@ data: MCP_MANAGEMENT_API: {{ .Values.config.services.mcpManagementServer | quote }} DATA_PROCESS_SERVICE: {{ .Values.config.services.dataProcessService | quote }} NORTHBOUND_API_SERVER: {{ .Values.config.services.northboundServer | quote }} - + # Service URLs (external) NORTHBOUND_EXTERNAL_URL: {{ .Values.config.services.northboundExternalUrl | quote }} @@ -43,6 +43,16 @@ data: # Redis Config REDIS_URL: {{ .Values.config.redis.url | quote }} REDIS_BACKEND_URL: {{ .Values.config.redis.backendUrl | quote }} + RUNTIME_STATE_REDIS_URL: {{ default .Values.config.redis.url .Values.config.runtimeState.redisUrl | quote }} + RUNTIME_STREAM_TTL_SECONDS: {{ .Values.config.runtimeState.streamTtlSeconds | quote }} + RUNTIME_STREAM_MAX_LEN: {{ .Values.config.runtimeState.streamMaxLen | quote }} + RUNTIME_RUN_TTL_SECONDS: {{ .Values.config.runtimeState.runTtlSeconds | quote }} + RUNTIME_CANCEL_TTL_SECONDS: {{ .Values.config.runtimeState.cancelTtlSeconds | quote }} + RUNTIME_COMPLETED_TTL_SECONDS: {{ .Values.config.runtimeState.completedTtlSeconds | quote }} + RUNTIME_CANCEL_POLL_INTERVAL_SECONDS: {{ .Values.config.runtimeState.cancelPollIntervalSeconds | quote }} + NORTHBOUND_IDEMPOTENCY_TTL_SECONDS: {{ .Values.config.northbound.idempotencyTtlSeconds | quote }} + NORTHBOUND_RATE_LIMIT_ENABLED: {{ .Values.config.northbound.rateLimitEnabled | quote }} + NORTHBOUND_RATE_LIMIT_PER_MINUTE: {{ .Values.config.northbound.rateLimitPerMinute | quote }} # Model Engine Config MODEL_ENGINE_ENABLED: {{ .Values.config.modelEngine.enabled | quote }} @@ -166,6 +176,7 @@ data: OAUTH_SSL_VERIFY: {{ .Values.config.oauth.sslVerify | quote }} OAUTH_CA_BUNDLE: {{ .Values.config.oauth.caBundle | quote }} OAUTH_CALLBACK_BASE_URL: {{ .Values.config.oauth.callbackBaseUrl | quote }} + OAUTH_LOGIN_MODE: {{ .Values.config.oauth.loginMode | quote }} # ===== CAS SSO Configuration ===== CAS_ENABLED: {{ .Values.config.cas.enabled | quote }} diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml index 560dd8b45..a54358e1f 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml @@ -2,7 +2,7 @@ {{- $shared := default dict $global.sharedStorage }} {{- $mode := default "local" $shared.mode }} {{- $storageClassName := default "" $shared.storageClassName }} -{{- $accessModes := default (list "ReadWriteOnce") $shared.accessModes }} +{{- $accessModes := default (list "ReadWriteMany") $shared.accessModes }} {{- $workspace := default dict $shared.workspace }} {{- $workspaceSize := default "10Gi" $workspace.size }} {{- $workspaceLocalPath := default "/var/lib/nexent" $workspace.localPath }} diff --git a/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml index 26bdafc22..5882585de 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-common/values.yaml @@ -38,6 +38,18 @@ config: url: "redis://nexent-redis:6379/0" backendUrl: "redis://nexent-redis:6379/1" port: "6379" + runtimeState: + redisUrl: "" + streamTtlSeconds: "86400" + streamMaxLen: "10000" + runTtlSeconds: "86400" + cancelTtlSeconds: "86400" + completedTtlSeconds: "300" + cancelPollIntervalSeconds: "1.0" + northbound: + idempotencyTtlSeconds: "600" + rateLimitEnabled: "true" + rateLimitPerMinute: "120" minio: endpoint: "http://nexent-minio:9000" region: "cn-north-1" @@ -147,6 +159,8 @@ config: sslVerify: "true" caBundle: "" callbackBaseUrl: "http://localhost:30000" + # Supported values: disabled, button, force. + loginMode: "button" cas: enabled: "false" serverUrl: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml index c31aa74bc..a393ce86e 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-config/templates/deployment.yaml @@ -14,6 +14,8 @@ metadata: "helm.sh/hook-weight": "20" spec: replicas: {{ .Values.replicaCount }} + strategy: +{{ toYaml .Values.strategy | indent 4 }} selector: matchLabels: app: nexent-config @@ -22,7 +24,8 @@ spec: labels: app: nexent-config annotations: - checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-backend-image: {{ dig "rolloutChecksums" "backendImage" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: serviceAccountName: {{ .Values.serviceAccount.name }} diff --git a/deploy/k8s/helm/nexent/charts/nexent-config/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-config/values.yaml index 90ea85e8c..b98d139f1 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-config/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-config/values.yaml @@ -1,5 +1,11 @@ replicaCount: 1 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + images: backend: repository: nexent/nexent diff --git a/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml index 9637bd281..d7ad4fe2a 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-data-process/templates/deployment.yaml @@ -21,7 +21,8 @@ spec: labels: app: nexent-data-process annotations: - checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-data-process-image: {{ dig "rolloutChecksums" "dataProcessImage" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: containers: diff --git a/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml index 620f7f7ad..99b057980 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-elasticsearch/values.yaml @@ -20,7 +20,7 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: "/var/lib/nexent-data/nexent-elasticsearch" existingClaim: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml index defa5f869..19d73023b 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-mcp/templates/deployment.yaml @@ -21,7 +21,8 @@ spec: labels: app: nexent-mcp annotations: - checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-backend-image: {{ dig "rolloutChecksums" "backendImage" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: containers: diff --git a/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml index 101cf726c..ec648db26 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-minio/templates/deployment.yaml @@ -17,7 +17,7 @@ spec: labels: app: nexent-minio annotations: - checksum/nexent-minio: {{ dig "rolloutChecksums" "minio" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} spec: containers: - name: minio diff --git a/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml index a8ee99381..69700debe 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-minio/values.yaml @@ -20,7 +20,7 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: "/var/lib/nexent-data/nexent-minio" existingClaim: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl index dd7c0fa26..bd109c006 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/templates/_helpers.tpl @@ -54,7 +54,7 @@ {{- $mode := default "local" $root.Values.persistence.mode -}} {{- $storageClassName := default "" $root.Values.persistence.storageClassName -}} {{- $localPath := default "/var/lib/nexent-data" $root.Values.persistence.localPath -}} -{{- $accessModes := default (list "ReadWriteOnce") $root.Values.persistence.accessModes -}} +{{- $accessModes := default (list "ReadWriteMany") $root.Values.persistence.accessModes -}} {{- if and $root.Values.enabled $root.Values.persistence.enabled -}} {{- if eq $mode "local" }} apiVersion: v1 diff --git a/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml index 76cf76862..4ef831ef7 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-monitoring/values.yaml @@ -3,8 +3,8 @@ enabled: true global: namespace: nexent -# Matches docker/start-monitoring.sh stack names: -# otlp, collector, phoenix, langfuse, langsmith, grafana, zipkin. +# Matches deployment monitoring provider names: +# otlp, phoenix, langfuse, langsmith, grafana, zipkin. provider: otlp images: @@ -87,7 +87,7 @@ phoenix: grafana: enabled: false adminUser: admin - adminPassword: nexent-grafana-admin + adminPassword: nexent@4321 defaultLanguage: zh-Hans service: type: NodePort @@ -137,7 +137,7 @@ langfuse: nodePort: 30001 postgres: user: postgres - password: postgres + password: nexent@4321 database: postgres storage: size: 10Gi @@ -163,6 +163,6 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: /var/lib/nexent-data existingClaimPrefix: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml index d2a49039e..0ca8ebb51 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-northbound/templates/deployment.yaml @@ -14,6 +14,8 @@ metadata: "helm.sh/hook-weight": "20" spec: replicas: {{ .Values.replicaCount }} + strategy: +{{ toYaml .Values.strategy | indent 4 }} selector: matchLabels: app: nexent-northbound @@ -22,7 +24,8 @@ spec: labels: app: nexent-northbound annotations: - checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-backend-image: {{ dig "rolloutChecksums" "backendImage" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: containers: diff --git a/deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml index 600728432..c384cc72e 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-northbound/values.yaml @@ -1,5 +1,11 @@ replicaCount: 1 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + images: backend: repository: nexent/nexent diff --git a/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml index 4921c832d..0dd1771f6 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-openssh/templates/deployment.yaml @@ -18,7 +18,8 @@ spec: labels: app: nexent-openssh-server annotations: - checksum/nexent-ssh: {{ dig "rolloutChecksums" "ssh" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-ssh-image: {{ dig "rolloutChecksums" "sshImage" "" .Values.global | quote }} spec: containers: - name: openssh-server diff --git a/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml index eeb6b2e38..694fad1e5 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-postgresql/values.yaml @@ -20,7 +20,7 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: "/var/lib/nexent-data/nexent-postgresql" existingClaim: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml index 3c94070b4..3b7ceadfa 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-redis/values.yaml @@ -20,6 +20,6 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: "/var/lib/nexent-data/nexent-redis" existingClaim: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml index 411d04500..0d29634f5 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-runtime/templates/deployment.yaml @@ -14,6 +14,8 @@ metadata: "helm.sh/hook-weight": "20" spec: replicas: {{ .Values.replicaCount }} + strategy: +{{ toYaml .Values.strategy | indent 4 }} selector: matchLabels: app: nexent-runtime @@ -22,7 +24,8 @@ spec: labels: app: nexent-runtime annotations: - checksum/nexent-backend: {{ dig "rolloutChecksums" "backend" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-backend-image: {{ dig "rolloutChecksums" "backendImage" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: containers: diff --git a/deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml index b593d3e66..c713ce15c 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-runtime/values.yaml @@ -1,5 +1,11 @@ replicaCount: 1 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + images: backend: repository: nexent/nexent diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml index 46ec3c137..3373b3e2e 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/templates/deployment.yaml @@ -19,12 +19,12 @@ spec: labels: app: nexent-supabase-auth annotations: - checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} + checksum/nexent-supabase-secret: {{ dig "rolloutChecksums" "supabaseSecret" "" .Values.global | quote }} spec: initContainers: - name: init-db - image: postgres:15-alpine - imagePullPolicy: IfNotPresent + image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" + imagePullPolicy: {{ .Values.initImage.pullPolicy }} env: - name: DB_HOST value: {{ .Values.config.postgresHost | quote }} diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml index da15ffbeb..807907515 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-auth/values.yaml @@ -5,6 +5,11 @@ image: tag: v2.170.0 pullPolicy: IfNotPresent +initImage: + repository: postgres + tag: 15-alpine + pullPolicy: IfNotPresent + resources: requests: memory: 512Mi diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml index 2d8f7acfc..00039abdf 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/templates/deployment.yaml @@ -21,7 +21,7 @@ spec: labels: app: nexent-supabase-db annotations: - checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} + checksum/nexent-supabase-secret: {{ dig "rolloutChecksums" "supabaseSecret" "" .Values.global | quote }} checksum/nexent-sql: {{ dig "rolloutChecksums" "sql" "" .Values.global | quote }} spec: initContainers: @@ -42,6 +42,8 @@ spec: cp /custom-supabase-sql/_supabase.sql /initdb.d/migrations/97-_supabase.sql cp /custom-supabase-sql/pooler.sql /initdb.d/migrations/99-pooler.sql + chmod 755 -R /initdb.d/ + echo "Initialization scripts are ready" volumeMounts: - mountPath: /custom-supabase-sql diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml index fc61e6c93..c5a156683 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-db/values.yaml @@ -20,7 +20,7 @@ persistence: mode: local storageClassName: nexent-local accessModes: - - ReadWriteOnce + - ReadWriteMany localPath: "/var/lib/nexent-data/nexent-supabase-db" existingClaim: "" diff --git a/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml index 296b74656..6d9dc0aa9 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-supabase-kong/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: labels: app: nexent-supabase-kong annotations: - checksum/nexent-supabase: {{ dig "rolloutChecksums" "supabase" "" .Values.global | quote }} + checksum/nexent-supabase-secret: {{ dig "rolloutChecksums" "supabaseSecret" "" .Values.global | quote }} spec: containers: - name: kong diff --git a/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml index 729fdfbd0..af07956e5 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-web/templates/deployment.yaml @@ -9,6 +9,8 @@ metadata: "helm.sh/hook-weight": "20" spec: replicas: {{ .Values.replicaCount }} + strategy: +{{ toYaml .Values.strategy | indent 4 }} selector: matchLabels: app: nexent-web @@ -17,7 +19,8 @@ spec: labels: app: nexent-web annotations: - checksum/nexent-web: {{ dig "rolloutChecksums" "web" "" .Values.global | quote }} + checksum/nexent-env: {{ dig "rolloutChecksums" "env" "" .Values.global | quote }} + checksum/nexent-web-image: {{ dig "rolloutChecksums" "webImage" "" .Values.global | quote }} spec: containers: - name: nexent-web @@ -41,6 +44,12 @@ spec: value: {{ .Values.global.deploymentVersion | quote }} - name: MARKET_BACKEND value: {{ .Values.config.marketBackend | quote }} + - name: PROXY_TIMEOUT_MS + value: {{ .Values.config.proxyTimeoutMs | quote }} + - name: PROXY_WS_TIMEOUT_MS + value: {{ .Values.config.proxyWsTimeoutMs | quote }} + - name: SSE_PROXY_TIMEOUT_MS + value: {{ .Values.config.sseProxyTimeoutMs | quote }} - name: MODEL_ENGINE_ENABLED value: {{ .Values.config.modelEngine.enabled | quote }} resources: diff --git a/deploy/k8s/helm/nexent/charts/nexent-web/values.yaml b/deploy/k8s/helm/nexent/charts/nexent-web/values.yaml index 74337791c..acffefb84 100644 --- a/deploy/k8s/helm/nexent/charts/nexent-web/values.yaml +++ b/deploy/k8s/helm/nexent/charts/nexent-web/values.yaml @@ -1,5 +1,11 @@ replicaCount: 1 +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + images: web: repository: nexent/nexent-web @@ -17,6 +23,9 @@ resources: config: marketBackend: "http://60.204.251.153:8010" + proxyTimeoutMs: "600000" + proxyWsTimeoutMs: "600000" + sseProxyTimeoutMs: "600000" modelEngine: enabled: "false" diff --git a/deploy/k8s/helm/nexent/values.yaml b/deploy/k8s/helm/nexent/values.yaml index bda678f7b..c454daea4 100644 --- a/deploy/k8s/helm/nexent/values.yaml +++ b/deploy/k8s/helm/nexent/values.yaml @@ -6,7 +6,7 @@ global: mode: "local" storageClassName: "nexent-local" accessModes: - - ReadWriteOnce + - ReadWriteMany workspace: size: "10Gi" localPath: "/var/lib/nexent" @@ -90,14 +90,18 @@ nexent-minio: enabled: true nexent-config: enabled: true + replicaCount: 1 nexent-runtime: enabled: true + replicaCount: 1 nexent-mcp: enabled: true nexent-northbound: enabled: true + replicaCount: 1 nexent-web: enabled: true + replicaCount: 1 nexent-data-process: enabled: true nexent-supabase-kong: diff --git a/deploy/k8s/uninstall.sh b/deploy/k8s/uninstall.sh index 1ee6f249a..140874e78 100755 --- a/deploy/k8s/uninstall.sh +++ b/deploy/k8s/uninstall.sh @@ -9,8 +9,15 @@ fi set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DEPLOYMENT_COMMON="$PROJECT_ROOT/deploy/common/common.sh" cd "$SCRIPT_DIR" +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi + NAMESPACE="nexent" RELEASE_NAME="nexent" DELETE_DATA="" @@ -20,6 +27,45 @@ LOCAL_DATA_DELETED="false" COMMAND="uninstall" print_usage() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "用法：$0 [delete|delete-all|clean] [选项]" + echo "" + echo "卸载 Nexent K8s 资源。" + echo "" + echo "命令：" + echo " delete 卸载 Helm release 并删除 namespace" + echo " delete-all 卸载 Helm release、删除 namespace，并删除本地数据" + echo " clean 仅清理 Helm release 状态" + echo "" + echo "选项：" + echo " --delete-data true|false 兼容选项；Helm 会删除托管的 PV/PVC 资源" + echo " --delete-volumes true|false 等同于 --delete-data" + echo " --remove-volumes 等同于 --delete-data true" + echo " --keep-volumes 等同于 --delete-data false" + echo " --delete-local-data true|false 控制是否删除本地 PV 数据" + echo " --remove-local-data 等同于 --delete-local-data true" + echo " --keep-local-data 等同于 --delete-local-data false" + echo " --delete-namespace true|false 控制是否删除 namespace" + echo " --remove-namespace 等同于 --delete-namespace true" + echo " --keep-namespace 等同于 --delete-namespace false" + echo " --namespace NAME Kubernetes namespace（默认：nexent）" + echo " --release NAME Helm release 名称（默认：nexent）" + echo " --help, -h 显示帮助信息" + echo "" + echo "示例：" + echo " bash uninstall.sh" + echo " bash uninstall.sh --delete-data false" + echo " bash uninstall.sh --delete-data true" + echo " bash uninstall.sh --delete-local-data true" + echo " bash uninstall.sh --keep-local-data" + echo " bash uninstall.sh --keep-namespace" + echo " bash uninstall.sh --delete-namespace true" + echo " bash uninstall.sh delete-all" + echo " bash uninstall.sh delete-all --keep-local-data" + echo " bash uninstall.sh clean" + return + fi + echo "Usage: $0 [delete|delete-all|clean] [options]" echo "" echo "Uninstall Nexent K8s resources." @@ -69,7 +115,11 @@ parse_bool_option() { true|TRUE|True|yes|YES|Yes|y|Y|1) return 0 ;; false|FALSE|False|no|NO|No|n|N|0) return 1 ;; *) - echo "Invalid boolean value: $value. Use true or false." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "无效布尔值：$value。请使用 true 或 false。" + else + echo "Invalid boolean value: $value. Use true or false." + fi exit 1 ;; esac @@ -143,7 +193,11 @@ while [[ $# -gt 0 ]]; do exit 0 ;; *) - echo "Unknown option: $1" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "未知选项：$1" + else + echo "Unknown option: $1" + fi print_usage exit 1 ;; @@ -151,12 +205,20 @@ while [[ $# -gt 0 ]]; do done clean_helm_state() { - echo "Cleaning Helm release state..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在清理 Helm release 状态..." + else + echo "Cleaning Helm release state..." + fi helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" --no-hooks 2>/dev/null || true kubectl delete secret -n "$NAMESPACE" -l "owner=helm" --ignore-not-found=true 2>/dev/null || true kubectl delete secret -n "$NAMESPACE" --field-selector type=helm.sh/release.v1 --ignore-not-found=true 2>/dev/null || true kubectl delete secret -n "$NAMESPACE" -l "name=$RELEASE_NAME" --ignore-not-found=true 2>/dev/null || true - echo "Helm state cleaned." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Helm 状态已清理。" + else + echo "Helm state cleaned." + fi } helm_uninstall_release() { @@ -169,7 +231,11 @@ helm_uninstall_release() { local status=$? [ -z "$output" ] || printf '%s\n' "$output" if printf '%s\n' "$output" | grep -qi 'not found'; then - echo "Helm release '$RELEASE_NAME' is already absent; continuing cleanup." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Helm release '$RELEASE_NAME' 已不存在；继续清理。" + else + echo "Helm release '$RELEASE_NAME' is already absent; continuing cleanup." + fi return 0 fi @@ -177,7 +243,11 @@ helm_uninstall_release() { } delete_namespace_after_uninstall() { - echo "Deleting namespace..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在删除 namespace..." + else + echo "Deleting namespace..." + fi kubectl delete namespace "$NAMESPACE" --ignore-not-found=true || true } @@ -190,9 +260,17 @@ resolve_delete_namespace() { [ -t 0 ] || return 1 echo "" - echo "Delete Kubernetes namespace '$NAMESPACE'?" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "是否删除 Kubernetes namespace '$NAMESPACE'？" + else + echo "Delete Kubernetes namespace '$NAMESPACE'?" + fi local answer - read -r -p "Delete namespace? [y/N]: " answer + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + read -r -p "删除 namespace？[y/N]：" answer + else + read -r -p "Delete namespace? [y/N]: " answer + fi answer="$(sanitize_input "$answer")" [[ "$answer" =~ ^[Yy]$ ]] } @@ -201,7 +279,11 @@ maybe_delete_namespace_after_uninstall() { if resolve_delete_namespace; then delete_namespace_after_uninstall else - echo "Namespace '$NAMESPACE' preserved." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Namespace '$NAMESPACE' 已保留。" + else + echo "Namespace '$NAMESPACE' preserved." + fi fi } @@ -233,27 +315,47 @@ resolve_delete_local_data() { [ -t 0 ] || return 1 echo "" - echo "Delete local PV data under /var/lib/nexent and /var/lib/nexent-data?" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "是否删除 /var/lib/nexent 和 /var/lib/nexent-data 下的本地 PV 数据？" + else + echo "Delete local PV data under /var/lib/nexent and /var/lib/nexent-data?" + fi local answer - read -r -p "Delete local volume data? [y/N]: " answer + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + read -r -p "删除本地 volume 数据？[y/N]：" answer + else + read -r -p "Delete local volume data? [y/N]: " answer + fi answer="$(sanitize_input "$answer")" [[ "$answer" =~ ^[Yy]$ ]] } delete_local_volume_data() { - echo "Deleting local PV data..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在删除本地 PV 数据..." + else + echo "Deleting local PV data..." + fi local path while IFS= read -r path; do case "$path" in /var/lib/nexent|/var/lib/nexent-data/skills|/var/lib/nexent-data/nexent-*) if [ -e "$path" ]; then - echo "Removing $path" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在删除 $path" + else + echo "Removing $path" + fi rm -rf -- "$path" fi ;; *) - echo "Refusing to remove unsafe path: $path" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "拒绝删除不安全路径：$path" + else + echo "Refusing to remove unsafe path: $path" + fi return 1 ;; esac @@ -265,7 +367,11 @@ maybe_delete_local_volume_data() { if resolve_delete_local_data; then delete_local_volume_data else - echo "Local PV data preserved." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "本地 PV 数据已保留。" + else + echo "Local PV data preserved." + fi fi } @@ -274,39 +380,108 @@ cleanup_leftover_data_process_resources() { return 0 fi - echo "Cleaning up leftover nexent-data-process resources..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在清理残留的 nexent-data-process 资源..." + else + echo "Cleaning up leftover nexent-data-process resources..." + fi kubectl delete deployment nexent-data-process -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true kubectl delete service nexent-data-process -n "$NAMESPACE" --ignore-not-found=true 2>/dev/null || true kubectl delete rs,pod -n "$NAMESPACE" -l app=nexent-data-process --ignore-not-found=true 2>/dev/null || true } +cleanup_leftover_monitoring_resources() { + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + return 0 + fi + + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在清理残留的 monitoring 资源..." + else + echo "Cleaning up leftover monitoring resources..." + fi + local app + for app in \ + nexent-otel-collector \ + nexent-phoenix \ + nexent-tempo \ + nexent-grafana \ + nexent-zipkin \ + nexent-langfuse-postgres \ + nexent-langfuse-clickhouse \ + nexent-langfuse-minio \ + nexent-langfuse-redis \ + nexent-langfuse-web \ + nexent-langfuse-worker + do + kubectl delete deployment,service,configmap,rs,pod -n "$NAMESPACE" -l app="$app" --ignore-not-found=true 2>/dev/null || true + done +} + +cleanup_leftover_nexent_resources() { + cleanup_leftover_data_process_resources + cleanup_leftover_monitoring_resources +} + uninstall_preserve_data() { - echo "Uninstalling Helm release..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在卸载 Helm release..." + else + echo "Uninstalling Helm release..." + fi if ! helm_uninstall_release; then - echo "Helm uninstall failed; continuing best-effort cleanup of nexent-data-process." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Helm 卸载失败；继续尽力清理已知 Nexent 资源。" + else + echo "Helm uninstall failed; continuing best-effort cleanup of known Nexent resources." + fi fi - cleanup_leftover_data_process_resources + cleanup_leftover_nexent_resources maybe_delete_local_volume_data maybe_delete_namespace_after_uninstall - echo "Cleanup completed. Helm-managed resources were removed." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "清理完成。Helm 托管资源已删除。" + else + echo "Cleanup completed. Helm-managed resources were removed." + fi if [ "$LOCAL_DATA_DELETED" = "true" ]; then - echo "Re-run './deploy.sh' to redeploy with fresh local data." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "重新运行 './deploy.sh' 可使用全新的本地数据部署。" + else + echo "Re-run './deploy.sh' to redeploy with fresh local data." + fi else - echo "Re-run './deploy.sh' to redeploy with existing data." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "重新运行 './deploy.sh' 可使用现有数据部署。" + else + echo "Re-run './deploy.sh' to redeploy with existing data." + fi fi } delete_all_data() { - echo "Deleting Helm release..." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "正在删除 Helm release..." + else + echo "Deleting Helm release..." + fi if ! helm_uninstall_release; then - echo "Helm uninstall failed. Namespace was not deleted." - cleanup_leftover_data_process_resources + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "Helm 卸载失败。Namespace 未删除。" + else + echo "Helm uninstall failed. Namespace was not deleted." + fi + cleanup_leftover_nexent_resources return 1 fi - cleanup_leftover_data_process_resources + cleanup_leftover_nexent_resources maybe_delete_local_volume_data maybe_delete_namespace_after_uninstall - echo "Cleanup completed. Helm-managed PV/PVC resources were deleted with the release." + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "清理完成。Helm 托管的 PV/PVC 资源已随 release 删除。" + else + echo "Cleanup completed. Helm-managed PV/PVC resources were deleted with the release." + fi } case "$COMMAND" in diff --git a/deploy/offline/build_offline_package.sh b/deploy/offline/build_offline_package.sh index 8c1184ca1..afae3ead6 100755 --- a/deploy/offline/build_offline_package.sh +++ b/deploy/offline/build_offline_package.sh @@ -38,6 +38,41 @@ if [ -f "$VERSION_HELPER" ]; then fi show_help() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "用法：$0 [选项]" + echo "" + echo "构建 Nexent 离线部署包" + echo "" + echo "选项：" + echo " --version VERSION Nexent 镜像版本（例如 v1.0.0 或 latest）" + echo " 默认：$DEFAULT_VERSION" + echo " --platform PLATFORM 目标平台（amd64 或 arm64）" + echo " 默认：$DEFAULT_PLATFORM" + echo " --output-dir DIR 离线包输出目录" + echo " 默认：$DEFAULT_OUTPUT_DIR" + echo " --include-source BOOL 是否包含源码（true 或 false）" + echo " 默认：$DEFAULT_INCLUDE_SOURCE" + echo " --target TARGET docker、k8s 或 all" + echo " 默认：$DEFAULT_TARGET" + echo " --compress BOOL 构建后是否创建 zip 压缩包（true 或 false）" + echo " 默认：$DEFAULT_COMPRESS" + echo " --components LIST 用于镜像选择的部署组件" + echo " --image-source SOURCE general、mainland 或 local-latest" + echo " --registry-profile NAME 兼容旧参数，映射到 --image-source general|mainland" + echo " --image-registry-prefix PREFIX" + echo " 使用指定镜像仓库前缀拉取和打包镜像" + echo " --defaults 复用保存配置或内置默认值并跳过交互界面" + echo " --config 进入交互式部署配置界面" + echo " --dry-run 只展示执行计划，不执行实际操作" + echo " --help 显示帮助信息" + echo "" + echo "示例：" + echo " $0 --version v1.0.0 --platform arm64" + echo " $0 --version latest --platform amd64 --include-source false" + echo " $0 --dry-run # 只展示执行计划" + return + fi + echo "Usage: $0 [OPTIONS]" echo "" echo "Build offline deployment package for Nexent" @@ -58,7 +93,10 @@ show_help() { echo " --components LIST Deployment components for image selection" echo " --image-source SOURCE general, mainland, or local-latest" echo " --registry-profile NAME Legacy alias for --image-source general|mainland" - echo " --config FILE Deployment config with components and image source" + echo " --image-registry-prefix PREFIX" + echo " Pull and package images with this registry prefix" + echo " --defaults Use saved config or built-in defaults and skip TUI" + echo " --config Open the interactive deployment configuration" echo " --dry-run Show execution plan without actual operations" echo " --help Show this help message" echo "" @@ -101,11 +139,11 @@ parse_args() { DRY_RUN="true" shift ;; - --components|--image-source|--registry-profile|--app-version|--monitoring-provider|--port-policy|--config|--local-config) + --components|--image-source|--registry-profile|--image-registry-prefix|--registry-prefix|--image-registry|--app-version|--monitoring-provider|--port-policy|--local-config) COMMON_ARGS+=("$1" "$2") shift 2 ;; - --use-local-config|--reconfigure) + --defaults|--config|--use-local-config|--reconfigure) COMMON_ARGS+=("$1") shift ;; @@ -114,7 +152,11 @@ parse_args() { exit 0 ;; *) - echo "Unknown option: $1" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "未知选项：$1" + else + echo "Unknown option: $1" + fi show_help exit 1 ;; @@ -133,15 +175,27 @@ parse_args() { COMPRESS="${COMPRESS:-$DEFAULT_COMPRESS}" if [[ "$PLATFORM" != "amd64" && "$PLATFORM" != "arm64" ]]; then - echo "Error: Platform must be 'amd64' or 'arm64'" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：Platform 必须是 'amd64' 或 'arm64'" + else + echo "Error: Platform must be 'amd64' or 'arm64'" + fi exit 1 fi if [[ "$TARGET" != "docker" && "$TARGET" != "k8s" && "$TARGET" != "all" ]]; then - echo "Error: Target must be 'docker', 'k8s', or 'all'" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：Target 必须是 'docker'、'k8s' 或 'all'" + else + echo "Error: Target must be 'docker', 'k8s', or 'all'" + fi exit 1 fi if [[ "$COMPRESS" != "true" && "$COMPRESS" != "false" ]]; then - echo "Error: Compress must be 'true' or 'false'" + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "错误：Compress 必须是 'true' 或 'false'" + else + echo "Error: Compress must be 'true' or 'false'" + fi exit 1 fi } @@ -163,6 +217,26 @@ prepare_deployment_image_config() { } show_dry_run_plan() { + if [ "$DEPLOYMENT_LANGUAGE" = "zh" ]; then + echo "=== DRY RUN 模式 ===" + echo "版本：$VERSION" + echo "平台：$PLATFORM" + echo "输出目录：$OUTPUT_DIR" + echo "包含源码：$INCLUDE_SOURCE" + echo "目标：$TARGET" + echo "压缩：$COMPRESS" + echo "组件：$DEPLOYMENT_COMPONENTS" + echo "镜像源：$DEPLOYMENT_IMAGE_SOURCE" + [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && echo "镜像仓库前缀：$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" + echo "" + echo "将拉取的镜像：" + get_nexent_images + get_third_party_images + echo "" + echo "不会执行实际操作。" + exit 0 + fi + echo "=== DRY RUN MODE ===" echo "Version: $VERSION" echo "Platform: $PLATFORM" @@ -172,6 +246,7 @@ show_dry_run_plan() { echo "Compress: $COMPRESS" echo "Components: $DEPLOYMENT_COMPONENTS" echo "Image source: $DEPLOYMENT_IMAGE_SOURCE" + [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && echo "Image registry prefix: $DEPLOYMENT_IMAGE_REGISTRY_PREFIX" echo "" echo "Images to pull:" get_nexent_images @@ -191,6 +266,12 @@ get_nexent_images() { } get_third_party_images() { + local image + echo_image_ref() { + deployment_add_image_registry_prefix "$1" + echo "" + } + if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "infrastructure"; then echo "$ELASTICSEARCH_IMAGE" echo "$POSTGRESQL_IMAGE" @@ -203,21 +284,24 @@ get_third_party_images() { echo "$SUPABASE_DB" fi if deployment_csv_contains "$DEPLOYMENT_COMPONENTS" "monitoring"; then - echo "otel/opentelemetry-collector-contrib:0.151.0" + echo_image_ref "otel/opentelemetry-collector-contrib:0.151.0" case "$DEPLOYMENT_MONITORING_PROVIDER" in - phoenix) echo "arizephoenix/phoenix:15" ;; + phoenix) echo_image_ref "arizephoenix/phoenix:15" ;; grafana) - echo "grafana/tempo:2.10.5" - echo "grafana/grafana:12.4" + echo_image_ref "grafana/tempo:2.10.5" + echo_image_ref "grafana/grafana:12.4" ;; - zipkin) echo "openzipkin/zipkin:latest" ;; + zipkin) echo_image_ref "openzipkin/zipkin:latest" ;; langfuse) - echo "docker.io/langfuse/langfuse-worker:3" - echo "docker.io/langfuse/langfuse:3" - echo "docker.io/clickhouse/clickhouse-server:26.3-alpine" - echo "docker.io/minio/minio:RELEASE.2023-12-20T01-00-02Z" - echo "docker.io/redis:alpine" - echo "docker.io/postgres:15-alpine" + for image in \ + "docker.io/langfuse/langfuse-worker:3" \ + "docker.io/langfuse/langfuse:3" \ + "docker.io/clickhouse/clickhouse-server:26.3-alpine" \ + "docker.io/minio/minio:RELEASE.2023-12-20T01-00-02Z" \ + "docker.io/redis:alpine" \ + "docker.io/postgres:15-alpine"; do + echo_image_ref "$image" + done ;; esac fi @@ -243,11 +327,6 @@ should_skip_pull() { return 0 fi - if uses_latest_tag "$image"; then - echo "Skipping pull for latest image; expecting local image: $image" - return 0 - fi - return 1 } @@ -445,6 +524,9 @@ copy_source_code() { done <<< "$git_files" echo "✅ Git-managed source code copied to: $source_dir" + if [ -f "$source_dir/VERSION" ]; then + printf '%s\n' "$VERSION" > "$source_dir/VERSION" + fi local total_size total_size=$(du -sh "$source_dir" | cut -f1) @@ -453,38 +535,61 @@ copy_source_code() { return 0 } -create_load_script() { +copy_load_script() { local load_script="$OUTPUT_DIR/load-images.sh" + local template_script="$SCRIPT_DIR/load-images.sh" echo "" echo "========================================" - echo "Creating load-images.sh script..." + echo "Copying load-images.sh script..." echo "========================================" - cat > "$load_script" << 'LOADSCRIPT' -#!/bin/bash + if [ ! -f "$template_script" ]; then + echo "❌ load-images.sh template not found: $template_script" + return 1 + fi -set -e + cp "$template_script" "$load_script" + chmod +x "$load_script" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IMAGES_DIR="$SCRIPT_DIR/images" + echo "✅ Created: $load_script" +} + +copy_push_script() { + local push_script="$OUTPUT_DIR/push-images.sh" + local template_script="$SCRIPT_DIR/push-images.sh" -echo "Loading Docker images from $IMAGES_DIR..." + echo "" + echo "========================================" + echo "Copying push-images.sh script..." + echo "========================================" -for tar_file in "$IMAGES_DIR"/*.tar; do - if [[ -f "$tar_file" ]]; then - echo "Loading: $tar_file" - docker load -i "$tar_file" + if [ ! -f "$template_script" ]; then + echo "❌ push-images.sh template not found: $template_script" + return 1 fi -done -echo "" -echo "✅ All images loaded successfully" -LOADSCRIPT + cp "$template_script" "$push_script" + chmod +x "$push_script" - chmod +x "$load_script" + echo "✅ Created: $push_script" +} - echo "✅ Created: $load_script" +create_offline_deploy_entrypoint() { + local deploy_script="$OUTPUT_DIR/deploy.sh" + + if [ ! -f "$deploy_script" ]; then + echo "❌ deploy.sh not found in offline package: $deploy_script" + return 1 + fi + if ! grep -q '^DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE=""$' "$deploy_script"; then + echo "❌ deploy.sh does not contain the offline mode marker: $deploy_script" + return 1 + fi + + local tmp_script="${deploy_script}.tmp" + sed 's/^DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE=""$/DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE="defaults"/' "$deploy_script" > "$tmp_script" + mv "$tmp_script" "$deploy_script" } copy_deployment_bundle() { @@ -495,7 +600,7 @@ copy_deployment_bundle() { cp "$PROJECT_ROOT/deploy.sh" "$OUTPUT_DIR/deploy.sh" cp "$PROJECT_ROOT/uninstall.sh" "$OUTPUT_DIR/uninstall.sh" - cp "$PROJECT_ROOT/VERSION" "$OUTPUT_DIR/VERSION" + printf '%s\n' "$VERSION" > "$OUTPUT_DIR/VERSION" if command -v rsync >/dev/null 2>&1; then rsync -a \ @@ -503,6 +608,7 @@ copy_deployment_bundle() { --exclude='deploy.options' \ --exclude='env/.env' \ --exclude='env/.env.bak' \ + --exclude='env/monitoring.env' \ --exclude='docker/.env.generated' \ --exclude='k8s/helm/nexent/generated-values.yaml' \ --exclude='k8s/helm/nexent/generated-runtime-values.yaml' \ @@ -514,15 +620,16 @@ copy_deployment_bundle() { find "$OUTPUT_DIR" -name '.DS_Store' -type f -delete 2>/dev/null || true fi - rm -f "$OUTPUT_DIR/deploy/env/.env" "$OUTPUT_DIR/deploy/env/.env.bak" "$OUTPUT_DIR/deploy/docker/.env.generated" "$OUTPUT_DIR/deploy/docker/deploy.options" "$OUTPUT_DIR/deploy/k8s/deploy.options" + rm -f "$OUTPUT_DIR/deploy/env/.env" "$OUTPUT_DIR/deploy/env/.env.bak" "$OUTPUT_DIR/deploy/env/monitoring.env" "$OUTPUT_DIR/deploy/docker/.env.generated" "$OUTPUT_DIR/deploy/docker/deploy.options" "$OUTPUT_DIR/deploy/k8s/deploy.options" rm -f "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-runtime-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-secrets-values.yaml" "$OUTPUT_DIR/deploy/k8s/helm/nexent/generated-persistence-values.yaml" case "$TARGET" in docker) rm -rf "$OUTPUT_DIR/deploy/k8s" ;; k8s) rm -rf "$OUTPUT_DIR/deploy/docker" ;; esac + create_offline_deploy_entrypoint find "$OUTPUT_DIR" -name '.git' -type d -prune -exec rm -rf {} + 2>/dev/null || true - chmod +x "$OUTPUT_DIR/deploy.sh" "$OUTPUT_DIR/uninstall.sh" "$OUTPUT_DIR/load-images.sh" 2>/dev/null || true + chmod +x "$OUTPUT_DIR/deploy.sh" "$OUTPUT_DIR/uninstall.sh" "$OUTPUT_DIR/load-images.sh" "$OUTPUT_DIR/push-images.sh" 2>/dev/null || true find "$OUTPUT_DIR/deploy" -type f -name '*.sh' -exec chmod +x {} \; 2>/dev/null || true echo "✅ Deployment bundle copied" @@ -543,6 +650,7 @@ create_manifest() { echo "target: \"$TARGET\"" echo "components: \"$DEPLOYMENT_COMPONENTS\"" echo "imageSource: \"$DEPLOYMENT_IMAGE_SOURCE\"" + echo "imageRegistryPrefix: \"$DEPLOYMENT_IMAGE_REGISTRY_PREFIX\"" echo "images:" while IFS= read -r image; do [ -n "$image" ] && echo " - \"$image\"" @@ -635,6 +743,7 @@ main() { echo "Compress: $COMPRESS" echo "Components: $DEPLOYMENT_COMPONENTS" echo "Image source: $DEPLOYMENT_IMAGE_SOURCE" + [ -n "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" ] && echo "Image registry prefix: $DEPLOYMENT_IMAGE_REGISTRY_PREFIX" echo "========================================" rm -rf "$OUTPUT_DIR" @@ -655,11 +764,16 @@ main() { exit 1 } - create_load_script || { + copy_load_script || { echo "❌ Load script creation failed, aborting" exit 1 } + copy_push_script || { + echo "❌ Push script creation failed, aborting" + exit 1 + } + copy_deployment_bundle || { echo "❌ Deployment bundle copy failed, aborting" exit 1 diff --git a/deploy/offline/load-images.sh b/deploy/offline/load-images.sh new file mode 100755 index 000000000..d8045c122 --- /dev/null +++ b/deploy/offline/load-images.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGES_DIR="$SCRIPT_DIR/images" + +echo "Loading Docker images from $IMAGES_DIR..." + +for tar_file in "$IMAGES_DIR"/*.tar; do + if [[ -f "$tar_file" ]]; then + echo "Loading: $tar_file" + docker load -i "$tar_file" + fi +done + +echo "" +echo "✅ All images loaded successfully" diff --git a/deploy/offline/push-images.sh b/deploy/offline/push-images.sh new file mode 100755 index 000000000..9567490b0 --- /dev/null +++ b/deploy/offline/push-images.sh @@ -0,0 +1,223 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MANIFEST_FILE="$SCRIPT_DIR/manifest.yaml" +LOAD_SCRIPT="$SCRIPT_DIR/load-images.sh" + +IMAGE_REGISTRY_PREFIX="${IMAGE_REGISTRY_PREFIX:-}" +REGISTRY_USERNAME="${REGISTRY_USERNAME:-}" +REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}" +LOAD_IMAGES="false" +PROMPT_FOR_MISSING="true" +OUTPUT_ENV_FILE="" + +usage() { + cat <<'USAGE' +Usage: + bash push-images.sh [--image-registry-prefix registry.example.com/nexent] [options] + +Options: + --image-registry-prefix PREFIX Registry prefix used for pushed image tags. Prompts when omitted. + --registry-username USER Username for docker login. Prompts when omitted. + --registry-password PASSWORD Password for docker login. REGISTRY_PASSWORD is also supported. + --registry-password-stdin Read docker login password from standard input. + --load-images Load images from ./images before pushing. + --no-prompt Fail instead of prompting for missing values. + --output-env-file FILE Write selected image registry prefix to FILE. + --help Show this help message. +USAGE +} + +normalize_image_registry_prefix() { + local prefix="$1" + prefix="${prefix#"${prefix%%[![:space:]]*}"}" + prefix="${prefix%"${prefix##*[![:space:]]}"}" + prefix="${prefix#http://}" + prefix="${prefix#https://}" + while [[ "$prefix" == */ ]]; do + prefix="${prefix%/}" + done + printf '%s' "$prefix" +} + +prefixed_image_ref() { + local image="$1" + local prefix="$2" + if [ -z "$prefix" ]; then + printf '%s' "$image" + return 0 + fi + case "$image" in + "$prefix"/*) printf '%s' "$image" ;; + *) printf '%s/%s' "$prefix" "$image" ;; + esac +} + +registry_host_from_prefix() { + local prefix="$1" + printf '%s' "${prefix%%/*}" +} + +while [ $# -gt 0 ]; do + case "$1" in + --image-registry-prefix|--registry-prefix|--image-registry) + if [ $# -lt 2 ]; then + echo "Error: $1 requires a value." >&2 + exit 1 + fi + IMAGE_REGISTRY_PREFIX="$2" + shift 2 + ;; + --registry-username) + if [ $# -lt 2 ]; then + echo "Error: $1 requires a value." >&2 + exit 1 + fi + REGISTRY_USERNAME="$2" + shift 2 + ;; + --registry-password) + if [ $# -lt 2 ]; then + echo "Error: $1 requires a value." >&2 + exit 1 + fi + REGISTRY_PASSWORD="$2" + shift 2 + ;; + --registry-password-stdin) + REGISTRY_PASSWORD="$(cat)" + shift + ;; + --load-images) + LOAD_IMAGES="true" + shift + ;; + --no-prompt) + PROMPT_FOR_MISSING="false" + shift + ;; + --output-env-file) + if [ $# -lt 2 ]; then + echo "Error: $1 requires a value." >&2 + exit 1 + fi + OUTPUT_ENV_FILE="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [ -z "$IMAGE_REGISTRY_PREFIX" ]; then + if [ -t 0 ]; then + if [ "$PROMPT_FOR_MISSING" = "true" ]; then + read -r -p "Enter image registry prefix (e.g. registry.example.com/nexent): " IMAGE_REGISTRY_PREFIX + else + echo "Error: --image-registry-prefix is required." >&2 + exit 1 + fi + else + echo "Error: --image-registry-prefix is required." >&2 + exit 1 + fi +fi + +IMAGE_REGISTRY_PREFIX="$(normalize_image_registry_prefix "$IMAGE_REGISTRY_PREFIX")" +if [ -z "$IMAGE_REGISTRY_PREFIX" ]; then + echo "Error: image registry prefix cannot be empty." >&2 + exit 1 +fi + +if [ -z "$REGISTRY_USERNAME" ]; then + if [ -t 0 ]; then + if [ "$PROMPT_FOR_MISSING" = "true" ]; then + read -r -p "Registry username: " REGISTRY_USERNAME + else + echo "Error: --registry-username is required when pushing images." >&2 + exit 1 + fi + else + echo "Error: --registry-username is required when pushing images." >&2 + exit 1 + fi +fi + +if [ -z "$REGISTRY_USERNAME" ]; then + echo "Error: registry username cannot be empty." >&2 + exit 1 +fi + +if [ -z "$REGISTRY_PASSWORD" ]; then + if [ -t 0 ]; then + if [ "$PROMPT_FOR_MISSING" = "true" ]; then + read -r -s -p "Registry password: " REGISTRY_PASSWORD + echo "" + else + echo "Error: registry password is required when pushing images." >&2 + exit 1 + fi + else + echo "Error: registry password is required when pushing images." >&2 + exit 1 + fi +fi + +if [ -z "$REGISTRY_PASSWORD" ]; then + echo "Error: registry password cannot be empty." >&2 + exit 1 +fi + +if [ ! -f "$MANIFEST_FILE" ]; then + echo "Error: manifest not found: $MANIFEST_FILE" >&2 + exit 1 +fi + +registry_host="$(registry_host_from_prefix "$IMAGE_REGISTRY_PREFIX")" +printf '%s' "$REGISTRY_PASSWORD" | docker login "$registry_host" --username "$REGISTRY_USERNAME" --password-stdin + +if [ -n "$OUTPUT_ENV_FILE" ]; then + mkdir -p "$(dirname "$OUTPUT_ENV_FILE")" + printf 'IMAGE_REGISTRY_PREFIX=%q\n' "$IMAGE_REGISTRY_PREFIX" > "$OUTPUT_ENV_FILE" +fi + +if [ "$LOAD_IMAGES" = "true" ]; then + if [ ! -f "$LOAD_SCRIPT" ]; then + echo "Error: --load-images requires $LOAD_SCRIPT" >&2 + exit 1 + fi + bash "$LOAD_SCRIPT" +fi + +echo "Pushing images with prefix: $IMAGE_REGISTRY_PREFIX" + +awk -F'"' '/^[[:space:]]*-[[:space:]]*"/ { print $2 }' "$MANIFEST_FILE" | while IFS= read -r image; do + [ -n "$image" ] || continue + target_image="$(prefixed_image_ref "$image" "$IMAGE_REGISTRY_PREFIX")" + + if ! docker image inspect "$image" >/dev/null 2>&1; then + echo "Error: image is not loaded locally: $image" >&2 + echo "Run bash load-images.sh first, or use --load-images." >&2 + exit 1 + fi + + if [ "$target_image" != "$image" ]; then + echo "Tagging: $image -> $target_image" + docker tag "$image" "$target_image" + fi + + echo "Pushing: $target_image" + docker push "$target_image" +done + +echo "" +echo "✅ All images pushed successfully" diff --git a/deploy/sql/init.sql b/deploy/sql/init.sql index 1e9f83e97..f62970a66 100644 --- a/deploy/sql/init.sql +++ b/deploy/sql/init.sql @@ -444,65 +444,3 @@ EXECUTE FUNCTION update_ag_tool_instance_update_time(); -- Add comment to the trigger COMMENT ON TRIGGER update_ag_tool_instance_update_time_trigger ON nexent.ag_tool_instance_t IS 'Trigger to call update_ag_tool_instance_update_time function before each update on ag_tool_instance_t table'; -CREATE TABLE IF NOT EXISTS nexent.conversation_share_t ( - share_id integer NOT NULL PRIMARY KEY, - share_token varchar(64) NOT NULL UNIQUE, - conversation_id integer NOT NULL, - tenant_id varchar(100), - title varchar(200), - mode varchar(30) DEFAULT 'selected', - selected_message_ids jsonb, - snapshot_json jsonb NOT NULL, - status varchar(30) DEFAULT 'active', - expire_time timestamp without time zone, - create_time timestamp without time zone DEFAULT now(), - update_time timestamp without time zone DEFAULT now(), - created_by varchar(100), - updated_by varchar(100), - delete_flag varchar(1) DEFAULT 'N' -); - -CREATE SEQUENCE IF NOT EXISTS nexent.conversation_share_t_share_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE nexent.conversation_share_t_share_id_seq OWNED BY nexent.conversation_share_t.share_id; -ALTER TABLE ONLY nexent.conversation_share_t ALTER COLUMN share_id SET DEFAULT nextval('nexent.conversation_share_t_share_id_seq'::regclass); - -CREATE INDEX IF NOT EXISTS idx_conversation_share_token ON nexent.conversation_share_t (share_token); -CREATE INDEX IF NOT EXISTS idx_conversation_share_conversation_id ON nexent.conversation_share_t (conversation_id); - -CREATE TABLE IF NOT EXISTS nexent.conversation_share_asset_t ( - share_asset_id integer NOT NULL PRIMARY KEY, - asset_id varchar(64) NOT NULL UNIQUE, - share_token varchar(64) NOT NULL, - object_name varchar(1000) NOT NULL, - filename varchar(500), - content_type varchar(200), - size bigint, - source_kind varchar(50), - metadata_json jsonb, - create_time timestamp without time zone DEFAULT now(), - update_time timestamp without time zone DEFAULT now(), - created_by varchar(100), - updated_by varchar(100), - delete_flag varchar(1) DEFAULT 'N' -); - -CREATE SEQUENCE IF NOT EXISTS nexent.conversation_share_asset_t_share_asset_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE nexent.conversation_share_asset_t_share_asset_id_seq OWNED BY nexent.conversation_share_asset_t.share_asset_id; -ALTER TABLE ONLY nexent.conversation_share_asset_t ALTER COLUMN share_asset_id SET DEFAULT nextval('nexent.conversation_share_asset_t_share_asset_id_seq'::regclass); - -CREATE INDEX IF NOT EXISTS idx_conversation_share_asset_token ON nexent.conversation_share_asset_t (share_token); -CREATE INDEX IF NOT EXISTS idx_conversation_share_asset_id ON nexent.conversation_share_asset_t (asset_id); diff --git a/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql b/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql index 8dcba06ba..91db40618 100644 --- a/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql +++ b/deploy/sql/migrations/v2.2.2_0622_update_left_nav_menu.sql @@ -25,7 +25,8 @@ ADD COLUMN IF NOT EXISTS parent_key VARCHAR(50); -- SU Menus (root level) INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES (1001, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), -(1002, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-manage'); +(1002, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-manage'), +(1003, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/owner-manage'); -- ADMIN Menus (root level) INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES @@ -86,13 +87,12 @@ INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_ (1411, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/mcp-space', '/resource-space'), (1412, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/skill-space', '/resource-space'); --- ASSET_OWNER Menus (root level) +-- ASSET_OWNER Menus (root level; /owner-manage is SU-only, see v2.3.0_0713_move_owner_manage_to_su.sql) INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype) VALUES (1501, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/'), (1502, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/chat'), (1503, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/agent-dev'), -(1504, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'), -(1505, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/owner-manage'); +(1504, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/resource-space'); INSERT INTO nexent.role_permission_t (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) VALUES (1506, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/models', '/agent-dev'), (1507, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/knowledges', '/agent-dev'), diff --git a/deploy/sql/migrations/v2.2_merged_migrations.sql b/deploy/sql/migrations/v2.2_merged_migrations.sql index 2c134da51..c1bd4b5be 100644 --- a/deploy/sql/migrations/v2.2_merged_migrations.sql +++ b/deploy/sql/migrations/v2.2_merged_migrations.sql @@ -346,7 +346,6 @@ CREATE TABLE IF NOT EXISTS nexent.ag_agent_repository_t ( description TEXT, author VARCHAR(100), submitted_by VARCHAR(100), - category_id INTEGER, tags TEXT[], tool_count INTEGER, icon VARCHAR(100), @@ -408,7 +407,6 @@ COMMENT ON COLUMN nexent.ag_agent_repository_t.display_name IS 'Root agent displ COMMENT ON COLUMN nexent.ag_agent_repository_t.description IS 'Root agent description'; COMMENT ON COLUMN nexent.ag_agent_repository_t.author IS 'Agent author'; COMMENT ON COLUMN nexent.ag_agent_repository_t.submitted_by IS 'Submitter email when listing enters pending_review'; -COMMENT ON COLUMN nexent.ag_agent_repository_t.category_id IS 'Optional marketplace category ID'; COMMENT ON COLUMN nexent.ag_agent_repository_t.tags IS 'Marketplace tags'; COMMENT ON COLUMN nexent.ag_agent_repository_t.tool_count IS 'Total tool count across all agents in the bundle (display only)'; COMMENT ON COLUMN nexent.ag_agent_repository_t.version_name IS 'Repository entry version name for display (from ag_tenant_agent_version_t)'; diff --git a/deploy/sql/migrations/v2.3.0_0624_add_labels_to_ag_tool_info.sql b/deploy/sql/migrations/v2.3.0_0624_add_labels_to_ag_tool_info.sql index 4c1e60723..f1e54084d 100644 --- a/deploy/sql/migrations/v2.3.0_0624_add_labels_to_ag_tool_info.sql +++ b/deploy/sql/migrations/v2.3.0_0624_add_labels_to_ag_tool_info.sql @@ -18,6 +18,7 @@ WITH label_map AS ( "search_memory": "search", "knowledge_base_search": "search", "dify_search": "knowledge-base", "datamate_search": "knowledge-base", "idata_search": "knowledge-base", "haotian_search": "knowledge-base", + "ragflow_search": "knowledge-base", "aidp_search": "knowledge-base", "analyze_image": "multimodal", "analyze_audio": "multimodal", "analyze_video": "multimodal", "analyze_text_file": "multimodal", diff --git a/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql b/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql new file mode 100644 index 000000000..3ec4e1dbf --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0628_add_agent_evaluation_full.sql @@ -0,0 +1,220 @@ +-- ============================================================================= +-- Agent evaluation (offline) - full bundle +-- ============================================================================= +-- Version: v2.3.0 +-- Date: 2026-06-30 +-- Description: Single-file bundle for the v2.3.0 agent evaluation feature. +-- Combines what were originally three separate migration drafts (0628, 0630 +-- pass_status, 0630 route grant) before any of them had been applied to +-- any environment. Use this file on fresh installs. +-- +-- Idempotency: every DDL statement in this file is safe to run multiple times. +-- - CREATE TABLE IF NOT EXISTS +-- - ALTER TABLE ... ADD COLUMN IF NOT EXISTS +-- - CREATE INDEX IF NOT EXISTS +-- - COMMENT ON (overwrites previous value, no-op if identical) +-- - INSERT ... ON CONFLICT (role_permission_id) DO NOTHING +-- +-- Sections: +-- 1. evaluation_set_t / evaluation_set_case_t / agent_evaluation_t / +-- agent_evaluation_case_t (incl. judge_model_id on agent_evaluation_t) +-- 2. pass_status column on agent_evaluation_case_t + composite index +-- 3. LEFT_NAV_MENU '/space' grant for roles that have /agent-space +-- +-- Design decisions (see PR review 2026-06-30): +-- * No standalone (tenant_id) index on any table. Every case-level and +-- run-level read is scoped by PK or by a foreign key into a parent that +-- itself is already tenant-scoped at the application layer. A bare +-- (tenant_id) index has no real query plan and only inflates write cost. +-- * No (tenant_id, judge_model_id) index. judge_model_id is read alongside +-- the row via PK; "list runs by judge model" is not a supported query. +-- * No (tenant_id, evaluation_set_id) on evaluation_set_case_t. The set +-- itself is tenant-scoped at the app layer, and the existing +-- (evaluation_set_id) index already covers set-case listing. +-- * ix_agent_eval_case_pass_status is (agent_evaluation_id, pass_status) +-- rather than (tenant_id, agent_evaluation_id, pass_status): case-level +-- reads never filter on tenant_id directly, and dropping the leading +-- tenant_id column keeps the most common "list failed cases for run X" +-- query on a single composite index. +-- * Section 3 INSERT must include parent_key (see 0622 menu migration) so +-- a future renderer that joins on parent_key does not leave this batch +-- as orphans. /space is a first-level entry for the route guard only, +-- so parent_key is NULL. +-- ============================================================================= + +SET search_path TO nexent; + +BEGIN; + + +-- ----------------------------------------------------------------------------- +-- Section 1: Evaluation set & evaluation run tables +-- ----------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS nexent.evaluation_set_t ( + evaluation_set_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + name VARCHAR(255) NOT NULL, + description TEXT, + + source_filename VARCHAR(255), + case_count INTEGER DEFAULT 0, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_eval_set_name ON nexent.evaluation_set_t(tenant_id, name); + +COMMENT ON TABLE nexent.evaluation_set_t IS 'Offline evaluation sets (JSONL single-turn cases).'; +COMMENT ON COLUMN nexent.evaluation_set_t.tenant_id IS 'Tenant ID for multi-tenancy isolation'; +COMMENT ON COLUMN nexent.evaluation_set_t.source_filename IS 'Original uploaded filename'; +COMMENT ON COLUMN nexent.evaluation_set_t.case_count IS 'Total number of cases'; + + +CREATE TABLE IF NOT EXISTS nexent.evaluation_set_case_t ( + evaluation_set_case_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + evaluation_set_id BIGINT NOT NULL, + + case_id VARCHAR(128), + inputs JSONB NOT NULL, + label JSONB NOT NULL, + order_no INTEGER DEFAULT 0, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_eval_set_case_set_id ON nexent.evaluation_set_case_t(evaluation_set_id); + +COMMENT ON TABLE nexent.evaluation_set_case_t IS 'Cases within evaluation sets.'; +COMMENT ON COLUMN nexent.evaluation_set_case_t.inputs IS 'Case inputs JSON: {query: string, context?: string}'; +COMMENT ON COLUMN nexent.evaluation_set_case_t.label IS 'Case label JSON: {answer: string}'; + + +CREATE TABLE IF NOT EXISTS nexent.agent_evaluation_t ( + agent_evaluation_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + agent_id INTEGER NOT NULL, + agent_version_no INTEGER NOT NULL, + + evaluation_set_id BIGINT NOT NULL, + + status VARCHAR(30) NOT NULL DEFAULT 'PENDING', + + progress_total INTEGER DEFAULT 0, + progress_done INTEGER DEFAULT 0, + + score_overall DOUBLE PRECISION, + error_message TEXT, + + judge_model_id INTEGER, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_agent_eval_agent_id ON nexent.agent_evaluation_t(tenant_id, agent_id); +CREATE INDEX IF NOT EXISTS ix_agent_eval_set_id ON nexent.agent_evaluation_t(tenant_id, evaluation_set_id); + +COMMENT ON TABLE nexent.agent_evaluation_t IS 'Offline evaluation runs for an agent.'; +COMMENT ON COLUMN nexent.agent_evaluation_t.status IS 'Run status: PENDING/RUNNING/COMPLETED/FAILED'; +COMMENT ON COLUMN nexent.agent_evaluation_t.judge_model_id IS + 'Model id used by the judge. Persisted so the background worker can recover it after restart and so the frontend can display judge_model_name.'; + + +CREATE TABLE IF NOT EXISTS nexent.agent_evaluation_case_t ( + agent_evaluation_case_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(100) NOT NULL, + + agent_evaluation_id BIGINT NOT NULL, + evaluation_set_case_id BIGINT NOT NULL, + + inputs JSONB NOT NULL, + label JSONB NOT NULL, + predict JSONB, + + score DOUBLE PRECISION, + reason TEXT, + + status VARCHAR(30) NOT NULL DEFAULT 'PENDING', + error_message TEXT, + + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +CREATE INDEX IF NOT EXISTS ix_agent_eval_case_eval_id ON nexent.agent_evaluation_case_t(agent_evaluation_id); + +COMMENT ON TABLE nexent.agent_evaluation_case_t IS 'Per-case evaluation results.'; +COMMENT ON COLUMN nexent.agent_evaluation_case_t.predict IS 'Predict JSON: {answer: string, raw?: any}'; +COMMENT ON COLUMN nexent.agent_evaluation_case_t.status IS 'Case status: PENDING/RUNNING/COMPLETED/FAILED'; + + +-- ----------------------------------------------------------------------------- +-- Section 2: pass_status on agent_evaluation_case_t +-- ----------------------------------------------------------------------------- +-- Stores the binary judge result ("pass" / "fail") for each case. +-- Enables fast filtering for failed-case reports and storage optimization: +-- passed cases have predict/reason/label.answer cleared to save space, +-- while only failed cases retain full detail. + +ALTER TABLE nexent.agent_evaluation_case_t +ADD COLUMN IF NOT EXISTS pass_status VARCHAR(16); + +COMMENT ON COLUMN nexent.agent_evaluation_case_t.pass_status IS + 'Judge result per case: pass / fail. pass cases have predict/reason/label.answer cleared to save space.'; + +-- Composite index to support failed-case listing and "only failed" reports. +-- Scoped by (agent_evaluation_id, pass_status) only; tenant_id is enforced +-- at the application layer via the parent run's tenant. +CREATE INDEX IF NOT EXISTS ix_agent_eval_case_pass_status + ON nexent.agent_evaluation_case_t (agent_evaluation_id, pass_status); + + +-- ----------------------------------------------------------------------------- +-- Section 3: Grant /space LEFT_NAV_MENU so the evaluation page is reachable +-- ----------------------------------------------------------------------------- +-- The agent evaluation page lives at the route prefix /space/agents/{id}/evaluate. +-- The previous menu migration (v2.2.2_0622_update_left_nav_menu.sql) removed +-- the legacy /space entry when it refactored the menu structure. As a result +-- the frontend route guard (which uses accessibleRoutes prefix matching) blocks +-- any user from entering the evaluation page with "no access permission". +-- +-- This section adds LEFT_NAV_MENU = '/space' for every role that already has +-- access to the resource-space (i.e. /agent-space). This entry is NOT rendered +-- in the side navigation (SideNavigation uses exact-match against ROUTE_CONFIG) +-- but it IS picked up by the backend as part of accessibleRoutes, so the route +-- guard will allow /space/agents/{id}/evaluate and its sub-paths. + +-- Roles that already have /resource-space (and thus /agent-space) get /space. +-- Mirrors v2.2.2_0622_update_left_nav_menu.sql IDs (16xx range) to keep the +-- scheme consistent. parent_key is NULL: /space is a top-level entry used +-- only by the backend route guard (prefix match on accessibleRoutes) and +-- is not rendered by SideNavigation. +INSERT INTO nexent.role_permission_t + (role_permission_id, user_role, permission_category, permission_type, permission_subtype, parent_key) +VALUES + (1600, 'SU', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1601, 'ADMIN', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1602, 'DEV', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1603, 'SPEED', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL), + (1604, 'ASSET_OWNER', 'VISIBILITY', 'LEFT_NAV_MENU', '/space', NULL) +ON CONFLICT (role_permission_id) DO NOTHING; + + +COMMIT; diff --git a/deploy/sql/migrations/v2.3.0_0629_add_skill_repository_table.sql b/deploy/sql/migrations/v2.3.0_0629_add_skill_repository_table.sql new file mode 100644 index 000000000..43f7fd8a4 --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0629_add_skill_repository_table.sql @@ -0,0 +1,99 @@ +-- Migration: Add ag_skill_repository_t table +-- Date: 2026-06-29 +-- Description: Skill marketplace repository for frozen installable skill snapshots. + +SET search_path TO nexent; + +CREATE SEQUENCE IF NOT EXISTS nexent.ag_skill_repository_t_skill_repository_id_seq; + +CREATE TABLE IF NOT EXISTS nexent.ag_skill_repository_t ( + skill_repository_id BIGINT NOT NULL DEFAULT nextval('nexent.ag_skill_repository_t_skill_repository_id_seq'), + publisher_tenant_id VARCHAR(100) NOT NULL, + publisher_user_id VARCHAR(100) NOT NULL, + skill_id INTEGER NOT NULL, + name VARCHAR(100) NOT NULL, + description TEXT, + source VARCHAR(30), + submitted_by VARCHAR(100), + category_id INTEGER, + tags TEXT[], + icon VARCHAR(100), + downloads INTEGER DEFAULT 0, + skill_info_json JSONB NOT NULL, + skill_zip_base64 TEXT NOT NULL, + status VARCHAR(30) DEFAULT 'not_shared', + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N', + CONSTRAINT ag_skill_repository_t_pkey PRIMARY KEY (skill_repository_id) +); + +ALTER SEQUENCE nexent.ag_skill_repository_t_skill_repository_id_seq + OWNED BY nexent.ag_skill_repository_t.skill_repository_id; + +ALTER TABLE nexent.ag_skill_repository_t OWNER TO root; + +ALTER TABLE nexent.ag_skill_repository_t + ADD COLUMN IF NOT EXISTS submitted_by VARCHAR(100), + ADD COLUMN IF NOT EXISTS icon VARCHAR(100), + ADD COLUMN IF NOT EXISTS downloads INTEGER DEFAULT 0, + ADD COLUMN IF NOT EXISTS skill_zip_base64 TEXT; + +COMMENT ON TABLE nexent.ag_skill_repository_t IS 'Skill marketplace repository for frozen installable skill snapshots'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.skill_repository_id IS 'Skill repository listing ID, unique primary key'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.publisher_tenant_id IS 'Publisher tenant ID'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.publisher_user_id IS 'Publisher user ID'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.skill_id IS 'Source skill ID from ag_skill_info_t; unique when active (delete_flag = N)'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.name IS 'Skill name for display and search'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.description IS 'Skill description'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.source IS 'Skill source'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.submitted_by IS 'Submitter email when listing enters pending_review'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.category_id IS 'Optional marketplace category ID'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.tags IS 'Marketplace tags'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.icon IS 'Marketplace card icon (emoji or URL)'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.downloads IS 'Marketplace install count for card display'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.skill_info_json IS 'Frozen skill metadata snapshot'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.skill_zip_base64 IS 'Frozen skill ZIP payload encoded as base64'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.status IS 'Listing status: not_shared / pending_review / rejected / shared'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.create_time IS 'Creation time'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.update_time IS 'Update time'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.created_by IS 'Creator ID'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.updated_by IS 'Updater ID'; +COMMENT ON COLUMN nexent.ag_skill_repository_t.delete_flag IS 'Soft delete flag: Y/N'; + +CREATE UNIQUE INDEX IF NOT EXISTS uq_skill_repository_skill_active + ON nexent.ag_skill_repository_t (skill_id) + WHERE delete_flag = 'N'; + +CREATE INDEX IF NOT EXISTS idx_skill_repository_publisher_delete + ON nexent.ag_skill_repository_t (publisher_tenant_id, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_skill_repository_status_delete + ON nexent.ag_skill_repository_t (status, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_skill_repository_name_delete + ON nexent.ag_skill_repository_t (name, delete_flag); + +CREATE INDEX IF NOT EXISTS idx_skill_repository_tags_gin + ON nexent.ag_skill_repository_t USING GIN (tags); + +CREATE OR REPLACE FUNCTION update_ag_skill_repository_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION update_ag_skill_repository_update_time() IS 'Auto-update update_time for ag_skill_repository_t'; + +DROP TRIGGER IF EXISTS update_ag_skill_repository_update_time_trigger ON nexent.ag_skill_repository_t; +CREATE TRIGGER update_ag_skill_repository_update_time_trigger +BEFORE UPDATE ON nexent.ag_skill_repository_t +FOR EACH ROW +EXECUTE FUNCTION update_ag_skill_repository_update_time(); + +COMMENT ON TRIGGER update_ag_skill_repository_update_time_trigger +ON nexent.ag_skill_repository_t IS 'Trigger to maintain update_time'; diff --git a/deploy/sql/migrations/v2.3.0_0709_add_conversation_agent_id.sql b/deploy/sql/migrations/v2.3.0_0709_add_conversation_agent_id.sql new file mode 100644 index 000000000..8f59d7544 --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0709_add_conversation_agent_id.sql @@ -0,0 +1,6 @@ +-- Store the latest agent used by each conversation so history selection can restore agent context. +ALTER TABLE nexent.conversation_record_t + ADD COLUMN IF NOT EXISTS agent_id INTEGER; + +COMMENT ON COLUMN nexent.conversation_record_t.agent_id + IS 'Agent ID used by the latest run in this conversation'; diff --git a/deploy/sql/migrations/v2.3.0_0709_add_mcp_market_tables.sql b/deploy/sql/migrations/v2.3.0_0709_add_mcp_market_tables.sql new file mode 100644 index 000000000..93ad7cedf --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0709_add_mcp_market_tables.sql @@ -0,0 +1,152 @@ +-- Migration: Add MCP market tables (v2.4.0 single-table design) +-- Date: 2026-07-09 +-- Description: Create mcp_market_record_t (single-table with inline review status), +-- add market_id to mcp_record_t, add review_status/review_type to +-- mcp_community_record_t. + +SET search_path TO nexent; + +BEGIN; + +-- ============================================================================ +-- 1) Extend mcp_record_t for market integration (idempotent) +-- ============================================================================ +ALTER TABLE IF EXISTS nexent.mcp_record_t + ADD COLUMN IF NOT EXISTS market_id INTEGER; + +COMMENT ON COLUMN nexent.mcp_record_t.market_id IS 'Published market record ID (FK to mcp_market_record_t)'; + +-- ============================================================================ +-- 2) Extend mcp_community_record_t for review workflow (idempotent) +-- ============================================================================ +ALTER TABLE IF EXISTS nexent.mcp_community_record_t + ADD COLUMN IF NOT EXISTS review_status VARCHAR(30) DEFAULT 'pending', + ADD COLUMN IF NOT EXISTS review_type VARCHAR(30) DEFAULT 'initial_listing'; + +COMMENT ON COLUMN nexent.mcp_community_record_t.review_status IS 'Review status: pending/approved/rejected/offline'; +COMMENT ON COLUMN nexent.mcp_community_record_t.review_type IS 'Review submission type: initial_listing/update'; + +-- ============================================================================ +-- 3) Create mcp_market_record_t (single-table design) +-- ============================================================================ + +CREATE SEQUENCE IF NOT EXISTS nexent.mcp_market_record_t_market_id_seq + OWNED BY nexent.mcp_market_record_t.market_id; + +CREATE TABLE IF NOT EXISTS nexent.mcp_market_record_t ( + market_id BIGINT NOT NULL DEFAULT nextval('nexent.mcp_market_record_t_market_id_seq'), + tenant_id VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + mcp_name VARCHAR(100) NOT NULL, + mcp_server VARCHAR(500) NOT NULL, + source VARCHAR(30) DEFAULT 'community', + registry_json JSONB, + transport_type VARCHAR(30), + config_json JSON, + tags TEXT[], + description TEXT, + download_count INTEGER DEFAULT 0, + review_status VARCHAR(30) DEFAULT 'not_shared', + submitted_by VARCHAR(100), + source_mcp_id INTEGER, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE nexent.mcp_market_record_t OWNER TO root; + +COMMENT ON TABLE nexent.mcp_market_record_t IS 'MCP market (community) records — single table covering all listing states'; +COMMENT ON COLUMN nexent.mcp_market_record_t.market_id IS 'Market record ID, unique primary key'; +COMMENT ON COLUMN nexent.mcp_market_record_t.tenant_id IS 'Publisher tenant ID'; +COMMENT ON COLUMN nexent.mcp_market_record_t.user_id IS 'Publisher user ID'; +COMMENT ON COLUMN nexent.mcp_market_record_t.mcp_name IS 'MCP name'; +COMMENT ON COLUMN nexent.mcp_market_record_t.mcp_server IS 'MCP server URL'; +COMMENT ON COLUMN nexent.mcp_market_record_t.source IS 'Source type, fixed to community'; +COMMENT ON COLUMN nexent.mcp_market_record_t.registry_json IS 'Full MCP metadata JSON'; +COMMENT ON COLUMN nexent.mcp_market_record_t.transport_type IS 'Transport type: http/sse/container'; +COMMENT ON COLUMN nexent.mcp_market_record_t.config_json IS 'Public-shareable MCP configuration JSON'; +COMMENT ON COLUMN nexent.mcp_market_record_t.tags IS 'Tags'; +COMMENT ON COLUMN nexent.mcp_market_record_t.description IS 'Description'; +COMMENT ON COLUMN nexent.mcp_market_record_t.download_count IS 'Cumulative download/install count'; +COMMENT ON COLUMN nexent.mcp_market_record_t.review_status IS 'Listing status: not_shared/pending_review/shared/rejected'; +COMMENT ON COLUMN nexent.mcp_market_record_t.submitted_by IS 'Email of the user who submitted for review'; +COMMENT ON COLUMN nexent.mcp_market_record_t.source_mcp_id IS 'Source mcp_record_t ID that was published to the market'; + +-- Indexes +CREATE UNIQUE INDEX IF NOT EXISTS uq_mcp_market_name_active + ON nexent.mcp_market_record_t (mcp_name) + WHERE delete_flag = 'N' AND review_status = 'shared'; + +CREATE INDEX IF NOT EXISTS idx_mcp_market_tenant_delete + ON nexent.mcp_market_record_t (tenant_id, delete_flag); +CREATE INDEX IF NOT EXISTS idx_mcp_market_status_delete + ON nexent.mcp_market_record_t (review_status, delete_flag); +CREATE INDEX IF NOT EXISTS idx_mcp_market_tags_gin + ON nexent.mcp_market_record_t USING GIN (tags); + +-- Trigger: auto-update update_time +CREATE OR REPLACE FUNCTION update_mcp_market_record_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION update_mcp_market_record_update_time() IS 'Auto-update update_time for mcp_market_record_t'; + +DROP TRIGGER IF EXISTS update_mcp_market_record_update_time_trigger ON nexent.mcp_market_record_t; +CREATE TRIGGER update_mcp_market_record_update_time_trigger +BEFORE UPDATE ON nexent.mcp_market_record_t +FOR EACH ROW +EXECUTE FUNCTION update_mcp_market_record_update_time(); + +COMMENT ON TRIGGER update_mcp_market_record_update_time_trigger ON nexent.mcp_market_record_t IS 'Trigger to maintain update_time'; + +-- ============================================================================ +-- 4) Backfill: migrate old community records to the market table +-- Old mcp_community_record_t had no review workflow — published immediately. +-- Set their review_status to approved, copy to mcp_market_record_t as 'shared', +-- then link mcp_record_t rows to the newly created market records. +-- ============================================================================ + +-- Mark old community records as approved (they were published without review) +UPDATE nexent.mcp_community_record_t +SET review_status = 'approved' +WHERE (review_status IS NULL OR review_status = 'pending') + AND delete_flag != 'Y'; + +-- Backfill: copy old community records into the market table (idempotent) +WITH inserted AS ( + INSERT INTO nexent.mcp_market_record_t ( + tenant_id, user_id, mcp_name, mcp_server, source, + registry_json, transport_type, config_json, tags, description, + download_count, create_time, update_time, created_by, updated_by, delete_flag, + review_status + ) + SELECT + c.tenant_id, c.user_id, c.mcp_name, c.mcp_server, c.source, + c.registry_json, c.transport_type, c.config_json, c.tags, c.description, + 0, c.create_time, c.update_time, c.created_by, c.updated_by, c.delete_flag, + 'shared' + FROM nexent.mcp_community_record_t c + WHERE c.delete_flag != 'Y' + AND c.review_status = 'approved' + AND NOT EXISTS ( + SELECT 1 FROM nexent.mcp_market_record_t m + WHERE m.tenant_id = c.tenant_id + AND m.mcp_name = c.mcp_name + ) + RETURNING market_id, tenant_id, mcp_name +) +UPDATE nexent.mcp_record_t AS mr +SET market_id = ins.market_id +FROM inserted AS ins +WHERE mr.tenant_id = ins.tenant_id + AND mr.mcp_name = ins.mcp_name + AND mr.market_id IS NULL; + +COMMIT; diff --git a/deploy/sql/migrations/v2.3.0_0713_move_owner_manage_to_su.sql b/deploy/sql/migrations/v2.3.0_0713_move_owner_manage_to_su.sql new file mode 100644 index 000000000..2173d7d0e --- /dev/null +++ b/deploy/sql/migrations/v2.3.0_0713_move_owner_manage_to_su.sql @@ -0,0 +1,45 @@ +-- ============================================================ +-- Move /owner-manage left-nav from ASSET_OWNER to SU +-- Migration Date: 2026-07-13 +-- ============================================================ +-- ASSET_OWNER no longer sees the asset-admin resource management page. +-- SU gains /owner-manage (id 1003) alongside existing / and /resource-manage. +-- ============================================================ + +BEGIN; + +-- Remove ASSET_OWNER access to /owner-manage +DELETE FROM nexent.role_permission_t +WHERE role_permission_id = 1505 + OR ( + user_role = 'ASSET_OWNER' + AND permission_category = 'VISIBILITY' + AND permission_type = 'LEFT_NAV_MENU' + AND permission_subtype = '/owner-manage' + ); + +-- Grant SU access to /owner-manage (idempotent) +DELETE FROM nexent.role_permission_t +WHERE role_permission_id = 1003 + OR ( + user_role = 'SU' + AND permission_category = 'VISIBILITY' + AND permission_type = 'LEFT_NAV_MENU' + AND permission_subtype = '/owner-manage' + ); + +INSERT INTO nexent.role_permission_t ( + role_permission_id, + user_role, + permission_category, + permission_type, + permission_subtype +) VALUES ( + 1003, + 'SU', + 'VISIBILITY', + 'LEFT_NAV_MENU', + '/owner-manage' +); + +COMMIT; diff --git a/deploy/tests/test_build_offline_package.sh b/deploy/tests/test_build_offline_package.sh index c89391b33..e4673b58d 100755 --- a/deploy/tests/test_build_offline_package.sh +++ b/deploy/tests/test_build_offline_package.sh @@ -7,6 +7,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" TMP_DIR="${TMPDIR:-/tmp}/nexent-offline-package-test-$$" BIN_DIR="$TMP_DIR/bin" OUT_DIR="$TMP_DIR/out" +export DEPLOYMENT_LANG=en mkdir -p "$BIN_DIR" "$OUT_DIR" trap 'rm -rf "$TMP_DIR"' EXIT @@ -40,6 +41,15 @@ case "$1" in [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" exit 0 ;; + login) + password="$(cat)" + [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s password=%s\n' "$*" "$password" >> "$FAKE_DOCKER_LOG" + exit 0 + ;; + load|tag|push) + [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" + exit 0 + ;; save) [ -n "${FAKE_DOCKER_LOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" out="" @@ -69,25 +79,39 @@ assert_common_package_files() { [ ! -f "$package_dir/install.sh" ] || fail "install.sh should not be packaged" [ ! -f "$package_dir/offline-install.sh" ] || fail "offline-install.sh should not be packaged" [ -f "$package_dir/load-images.sh" ] || fail "load-images.sh should be packaged" + [ -f "$package_dir/push-images.sh" ] || fail "push-images.sh should be packaged" [ -f "$package_dir/manifest.yaml" ] || fail "manifest.yaml should be packaged" [ -f "$package_dir/checksums.txt" ] || fail "checksums.txt should be packaged" [ -f "$package_dir/deploy/deploy.sh" ] || fail "deploy/deploy.sh should be packaged" [ -f "$package_dir/deploy/uninstall.sh" ] || fail "deploy/uninstall.sh should be packaged" [ -f "$package_dir/VERSION" ] || fail "root VERSION should be packaged" [ -f "$package_dir/deploy/env/.env.example" ] || fail "deploy/env/.env.example should be packaged" + [ -f "$package_dir/deploy/env/monitoring.env.example" ] || fail "deploy/env/monitoring.env.example should be packaged" [ -f "$package_dir/deploy/sql/init.sql" ] || fail "deploy/sql/init.sql should be packaged" [ -d "$package_dir/deploy/sql/migrations" ] || fail "deploy/sql/migrations should be packaged" [ -d "$package_dir/deploy/sql/supabase" ] || fail "deploy/sql/supabase should be packaged" [ -f "$package_dir/deploy/sql/supabase/webhooks.sql" ] || fail "deploy/sql/supabase/webhooks.sql should be packaged" [ ! -f "$package_dir/.env" ] || fail "root .env should not be packaged" [ ! -f "$package_dir/deploy/env/.env" ] || fail "deploy/env/.env should not be packaged" + [ ! -f "$package_dir/deploy/env/monitoring.env" ] || fail "generated deploy/env/monitoring.env should not be packaged" [ ! -f "$package_dir/deploy/docker/.env" ] || fail "deploy/docker/.env should not be packaged" [ ! -f "$package_dir/deploy/docker/.env.generated" ] || fail "deploy/docker/.env.generated should not be packaged" + if [ -d "$package_dir/deploy/docker" ]; then + [ ! -f "$package_dir/deploy/docker/assets/monitoring/monitoring.env" ] || fail "generated monitoring.env should not be packaged" + [ ! -f "$package_dir/deploy/docker/assets/monitoring/monitoring.env.example" ] || fail "monitoring.env.example should live under deploy/env" + fi [ ! -f "$package_dir/deploy/docker/deploy.options" ] || fail "deploy/docker/deploy.options should not be packaged" } create_fake_docker +WORKFLOW_CONTENT="$(cat "$PROJECT_ROOT/.github/workflows/build-offline-package.yml")" +echo "$WORKFLOW_CONTENT" | grep -q 'SOURCE_SUFFIX="-with-source"' || fail "offline package workflow should append with-source when source is included" +echo "$WORKFLOW_CONTENT" | grep -q 'package-name=nexent-${VERSION}-${PLATFORM}${SOURCE_SUFFIX}' || fail "offline package workflow package name should include source suffix" +echo "$WORKFLOW_CONTENT" | grep -q -- '--compress false' || fail "offline package workflow should let GitHub create the final artifact zip" +echo "$WORKFLOW_CONTENT" | grep -q 'path: ./offline-output' || fail "offline package workflow should upload package contents, not an inner zip" +! echo "$WORKFLOW_CONTENT" | grep -q 'path: .*package-name.*\\.zip' || fail "offline package workflow should not upload a pre-compressed zip" + for target in docker k8s all; do package_dir="$OUT_DIR/$target" PATH="$BIN_DIR:$PATH" \ @@ -101,6 +125,7 @@ for target in docker k8s all; do --output-dir "$package_dir" >/tmp/nexent-offline-package-${target}.log assert_common_package_files "$package_dir" + [ "$(cat "$package_dir/VERSION")" = "v2.2.0" ] || fail "root VERSION should match requested package version for target $target" [ -f "$OUT_DIR/nexent-offline-${target}-amd64-v2.2.0.zip" ] || fail "zip package should be created for target $target" grep -q "target: \"$target\"" "$package_dir/manifest.yaml" || fail "manifest should record target $target" grep -q "nexent/nexent:v2.2.0" "$package_dir/manifest.yaml" || fail "manifest should include Nexent image" @@ -129,9 +154,15 @@ cat > "$deploy_wrapper_dir/load-images.sh" <<'SH' printf 'load-images\n' >> "$DEPLOY_WRAPPER_LOG" SH chmod +x "$deploy_wrapper_dir/load-images.sh" +cat > "$deploy_wrapper_dir/push-images.sh" <<'SH' +#!/usr/bin/env bash +args=("$@") +printf 'push:%s:%s\n' "${REGISTRY_PASSWORD:-}" "${args[*]}" >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$deploy_wrapper_dir/push-images.sh" cat > "$deploy_wrapper_dir/deploy/deploy.sh" <<'SH' #!/usr/bin/env bash -printf 'deploy:%s\n' "$*" >> "$DEPLOY_WRAPPER_LOG" +printf 'deploy:%s:%s\n' "${NEXENT_DEPLOY_CONFIG_MODE:-}" "$*" >> "$DEPLOY_WRAPPER_LOG" SH chmod +x "$deploy_wrapper_dir/deploy/deploy.sh" @@ -140,19 +171,50 @@ DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" do if grep -q '^load-images$' "$deploy_wrapper_log"; then fail "deploy.sh should not load images by default" fi -grep -q '^deploy:docker --foo bar$' "$deploy_wrapper_log" || fail "deploy.sh should forward args without --load-images" +grep -q '^deploy::docker --foo bar$' "$deploy_wrapper_log" || fail "deploy.sh should forward args without --load-images" : > "$deploy_wrapper_log" DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" --load-images docker --foo bar first_line="$(sed -n '1p' "$deploy_wrapper_log")" second_line="$(sed -n '2p' "$deploy_wrapper_log")" [ "$first_line" = "load-images" ] || fail "deploy.sh --load-images should load images before deploy" -[ "$second_line" = "deploy:docker --foo bar" ] || fail "deploy.sh --load-images should strip only the wrapper flag" +[ "$second_line" = "deploy::docker --foo bar" ] || fail "deploy.sh --load-images should strip only the wrapper flag" + +: > "$deploy_wrapper_log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" --defaults docker --foo bar +grep -q '^deploy:defaults:docker --foo bar$' "$deploy_wrapper_log" || fail "deploy.sh --defaults before target should enable defaults mode" + +: > "$deploy_wrapper_log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" bash "$deploy_wrapper_dir/deploy.sh" docker --defaults --foo bar +grep -q '^deploy:defaults:docker --foo bar$' "$deploy_wrapper_log" || fail "deploy.sh --defaults after target should enable defaults mode and consume the flag" + +: > "$deploy_wrapper_log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" REGISTRY_USERNAME=user REGISTRY_PASSWORD=secret bash "$deploy_wrapper_dir/deploy.sh" --push-images --image-registry-prefix registry.local/nexent docker --foo bar +first_line="$(sed -n '1p' "$deploy_wrapper_log")" +second_line="$(sed -n '2p' "$deploy_wrapper_log")" +[[ "$first_line" == "push:secret:--image-registry-prefix registry.local/nexent --load-images" ]] || fail "deploy.sh --push-images should delegate push args to push-images.sh" +[ "$second_line" = "deploy::docker --foo bar --image-registry-prefix registry.local/nexent" ] || fail "deploy.sh --push-images should forward image registry prefix to deploy config" +: > "$deploy_wrapper_log" +DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" REGISTRY_USERNAME=user REGISTRY_PASSWORD=secret bash "$deploy_wrapper_dir/deploy.sh" --load-images --push-images --image-registry-prefix registry.local/nexent docker --foo bar +first_line="$(sed -n '1p' "$deploy_wrapper_log")" +second_line="$(sed -n '2p' "$deploy_wrapper_log")" +[[ "$first_line" == "push:secret:--image-registry-prefix registry.local/nexent --load-images" ]] || fail "deploy.sh --load-images --push-images should not load before push login" +[ "$second_line" = "deploy::docker --foo bar --image-registry-prefix registry.local/nexent" ] || fail "deploy.sh --load-images --push-images should forward deploy args" + +if DEPLOY_WRAPPER_LOG="$deploy_wrapper_log" REGISTRY_USERNAME=user REGISTRY_PASSWORD=secret bash "$deploy_wrapper_dir/deploy.sh" --push-images docker --foo bar >/tmp/nexent-deploy-wrapper-missing-prefix.log 2>&1; then + fail "deploy.sh --push-images should require image registry prefix in non-interactive mode" +fi +grep -q -- '--image-registry-prefix' /tmp/nexent-deploy-wrapper-missing-prefix.log || fail "deploy.sh missing prefix error should be explicit" latest_package_dir="$OUT_DIR/latest" latest_pull_log="$TMP_DIR/latest-docker.log" : > "$latest_pull_log" +output="$(bash "$PROJECT_ROOT/build.sh" --package --version v2.2.0 --platform amd64 --components infrastructure,application --image-source general --target docker --dry-run)" +echo "$output" | grep -q "=== DRY RUN MODE ===" || fail "build.sh --package should forward to offline package builder" +echo "$output" | grep -q "Target: docker" || fail "build.sh --package should forward package arguments" +echo "$output" | grep -q "nexent/nexent:v2.2.0" || fail "build.sh --package should render package image plan" + PATH="$BIN_DIR:$PATH" FAKE_DOCKER_LOG="$latest_pull_log" \ bash "$PROJECT_ROOT/deploy/offline/build_offline_package.sh" \ --version latest \ @@ -164,11 +226,93 @@ PATH="$BIN_DIR:$PATH" FAKE_DOCKER_LOG="$latest_pull_log" \ --output-dir "$latest_package_dir" >/tmp/nexent-offline-package-latest.log assert_common_package_files "$latest_package_dir" +[ "$(cat "$latest_package_dir/VERSION")" = "latest" ] || fail "root VERSION should match requested latest package version" +grep -q '^DEPLOY_WRAPPER_DEFAULT_CONFIG_MODE="defaults"$' "$latest_package_dir/deploy.sh" || fail "offline deploy.sh should reuse the root entrypoint with defaults mode enabled" +offline_help="$(DEPLOYMENT_LANG=en bash "$latest_package_dir/deploy.sh" --help)" +echo "$offline_help" | grep -q "deploys with saved configuration or built-in defaults" || fail "offline deploy help should explain default non-interactive mode" + +push_log="$TMP_DIR/push-images.log" +: > "$push_log" +PATH="$BIN_DIR:$PATH" \ + FAKE_DOCKER_LOG="$push_log" \ + FAKE_DOCKER_LOCAL_IMAGES="nexent/nexent:latest,nexent/nexent-web:latest,nexent/nexent-mcp:latest,docker.elastic.co/elasticsearch/elasticsearch:8.17.4,postgres:15-alpine,redis:alpine,quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z" \ + REGISTRY_PASSWORD=secret \ + bash "$latest_package_dir/push-images.sh" \ + --image-registry-prefix https://registry.local/nexent/ \ + --registry-username user \ + --output-env-file "$TMP_DIR/push-images.env" >/tmp/nexent-offline-package-push.log +grep -q '^login registry.local --username user --password-stdin password=secret$' "$push_log" || fail "push-images.sh should login to registry host with password stdin" +grep -q '^IMAGE_REGISTRY_PREFIX=registry.local/nexent$' "$TMP_DIR/push-images.env" || fail "push-images.sh should write selected registry prefix to output env file" +grep -q '^tag nexent/nexent:latest registry.local/nexent/nexent/nexent:latest$' "$push_log" || fail "push-images.sh should tag Nexent image with registry prefix" +grep -q '^push registry.local/nexent/nexent/nexent:latest$' "$push_log" || fail "push-images.sh should push prefixed Nexent image" +grep -q '^tag docker.elastic.co/elasticsearch/elasticsearch:8.17.4 registry.local/nexent/docker.elastic.co/elasticsearch/elasticsearch:8.17.4$' "$push_log" || fail "push-images.sh should tag third-party image with registry prefix" +: > "$push_log" +PATH="$BIN_DIR:$PATH" \ + FAKE_DOCKER_LOG="$push_log" \ + FAKE_DOCKER_LOCAL_IMAGES="nexent/nexent:latest,nexent/nexent-web:latest,nexent/nexent-mcp:latest,docker.elastic.co/elasticsearch/elasticsearch:8.17.4,postgres:15-alpine,redis:alpine,quay.io/minio/minio:RELEASE.2023-12-20T01-00-02Z" \ + REGISTRY_PASSWORD=secret \ + bash "$latest_package_dir/push-images.sh" \ + --load-images \ + --image-registry-prefix registry.local/nexent \ + --registry-username user >/tmp/nexent-offline-package-push-load.log +first_line="$(sed -n '1p' "$push_log")" +second_line="$(sed -n '2p' "$push_log")" +[ "$first_line" = "login registry.local --username user --password-stdin password=secret" ] || fail "push-images.sh should docker login before loading images" +[[ "$second_line" == load\ -i\ * ]] || fail "push-images.sh should load images only after docker login" +if PATH="$BIN_DIR:$PATH" REGISTRY_PASSWORD=secret bash "$latest_package_dir/push-images.sh" --image-registry-prefix registry.local/nexent >/tmp/nexent-offline-package-push-missing-user.log 2>&1; then + fail "push-images.sh should require registry username in non-interactive mode" +fi +grep -q -- '--registry-username is required' /tmp/nexent-offline-package-push-missing-user.log || fail "push-images.sh missing username error should be explicit" + +cat > "$latest_package_dir/load-images.sh" <<'SH' +#!/usr/bin/env bash +printf 'load-images\n' >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$latest_package_dir/load-images.sh" +cat > "$latest_package_dir/push-images.sh" <<'SH' +#!/usr/bin/env bash +args=("$@") +printf 'push:%s:%s\n' "${REGISTRY_PASSWORD:-}" "${args[*]}" >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$latest_package_dir/push-images.sh" +cat > "$latest_package_dir/deploy/deploy.sh" <<'SH' +#!/usr/bin/env bash +printf 'deploy:%s:%s\n' "${NEXENT_DEPLOY_CONFIG_MODE:-}" "$*" >> "$DEPLOY_WRAPPER_LOG" +SH +chmod +x "$latest_package_dir/deploy/deploy.sh" + +offline_deploy_log="$TMP_DIR/offline-deploy-wrapper.log" +: > "$offline_deploy_log" +DEPLOY_WRAPPER_LOG="$offline_deploy_log" bash "$latest_package_dir/deploy.sh" docker --foo bar +grep -q '^deploy:defaults:docker --foo bar$' "$offline_deploy_log" || fail "offline deploy.sh should default to non-interactive defaults mode" + +: > "$offline_deploy_log" +DEPLOY_WRAPPER_LOG="$offline_deploy_log" bash "$latest_package_dir/deploy.sh" docker --config --foo bar +grep -q '^deploy:tui:docker --foo bar$' "$offline_deploy_log" || fail "offline deploy.sh --config should enable TUI mode and consume the flag" + +: > "$offline_deploy_log" +DEPLOY_WRAPPER_LOG="$offline_deploy_log" bash "$latest_package_dir/deploy.sh" docker --defaults --foo bar +grep -q '^deploy:defaults:docker --foo bar$' "$offline_deploy_log" || fail "offline deploy.sh --defaults should preserve defaults mode and consume the flag" + +: > "$offline_deploy_log" +DEPLOY_WRAPPER_LOG="$offline_deploy_log" bash "$latest_package_dir/deploy.sh" --load-images docker --foo bar +first_line="$(sed -n '1p' "$offline_deploy_log")" +second_line="$(sed -n '2p' "$offline_deploy_log")" +[ "$first_line" = "load-images" ] || fail "offline deploy.sh --load-images should load images before deploy" +[ "$second_line" = "deploy:defaults:docker --foo bar" ] || fail "offline deploy.sh --load-images should preserve defaults mode" + +: > "$offline_deploy_log" +DEPLOY_WRAPPER_LOG="$offline_deploy_log" REGISTRY_USERNAME=user REGISTRY_PASSWORD=secret bash "$latest_package_dir/deploy.sh" --push-images --image-registry-prefix registry.local/nexent docker --foo bar +first_line="$(sed -n '1p' "$offline_deploy_log")" +second_line="$(sed -n '2p' "$offline_deploy_log")" +[[ "$first_line" == "push:secret:--image-registry-prefix registry.local/nexent --load-images" ]] || fail "offline deploy.sh --push-images should push before deploy" +[ "$second_line" = "deploy:defaults:docker --foo bar --image-registry-prefix registry.local/nexent" ] || fail "offline deploy.sh --push-images should preserve defaults mode and forward registry prefix" + [ -f "$OUT_DIR/nexent-offline-docker-amd64-latest.zip" ] || fail "zip package should be created for latest package" grep -q "nexent/nexent:latest" "$latest_package_dir/manifest.yaml" || fail "manifest should include local latest Nexent image" -! grep -q '^pull .*nexent/nexent:latest$' "$latest_pull_log" || fail "latest Nexent image should not be pulled" -! grep -q '^pull .*nexent/nexent-web:latest$' "$latest_pull_log" || fail "latest Nexent web image should not be pulled" -! grep -q '^pull .*nexent/nexent-mcp:latest$' "$latest_pull_log" || fail "latest Nexent MCP image should not be pulled" +grep -q '^pull .*nexent/nexent:latest$' "$latest_pull_log" || fail "latest Nexent image should be pulled" +grep -q '^pull .*nexent/nexent-web:latest$' "$latest_pull_log" || fail "latest Nexent web image should be pulled" +grep -q '^pull .*nexent/nexent-mcp:latest$' "$latest_pull_log" || fail "latest Nexent MCP image should be pulled" grep -q '^pull .*docker.elastic.co/elasticsearch/elasticsearch:8.17.4$' "$latest_pull_log" || fail "non-latest infrastructure images should still be pulled" local_package_dir="$OUT_DIR/local-existing/package" @@ -188,6 +332,7 @@ PATH="$BIN_DIR:$PATH" \ --output-dir "$local_package_dir" >/tmp/nexent-offline-package-local-existing.log assert_common_package_files "$local_package_dir" +[ "$(cat "$local_package_dir/VERSION")" = "v2.2.0" ] || fail "root VERSION should match requested package version" [ -f "$OUT_DIR/local-existing/nexent-offline-docker-amd64-v2.2.0.zip" ] || fail "zip package should be created for local existing package" ! grep -q '^pull .*nexent/nexent:v2.2.0$' "$local_pull_log" || fail "existing local Nexent image should not be pulled" ! grep -q '^pull .*docker.elastic.co/elasticsearch/elasticsearch:8.17.4$' "$local_pull_log" || fail "existing local infrastructure image should not be pulled" diff --git a/deploy/tests/test_common.sh b/deploy/tests/test_common.sh index 6975f06e5..90dd0d53c 100755 --- a/deploy/tests/test_common.sh +++ b/deploy/tests/test_common.sh @@ -11,6 +11,9 @@ source "$SCRIPT_DIR/../common/version.sh" TMP_DIR="${TMPDIR:-/tmp}/nexent-deployment-test-$$" mkdir -p "$TMP_DIR" trap 'rm -rf "$TMP_DIR"' EXIT +DEPLOYMENT_ROOT_ENV="$TMP_DIR/root.env" +: > "$DEPLOYMENT_ROOT_ENV" +export DEPLOYMENT_LANG=en assert_eq() { local expected="$1" @@ -24,6 +27,17 @@ assert_eq() { fi } +assert_not_eq() { + local first="$1" + local second="$2" + local message="$3" + if [ "$first" = "$second" ]; then + echo "FAIL: $message" + echo " both: $first" + exit 1 + fi +} + assert_contains() { local haystack="$1" local needle="$2" @@ -36,6 +50,18 @@ assert_contains() { fi } +assert_not_contains() { + local haystack="$1" + local needle="$2" + local message="$3" + if [[ "$haystack" == *"$needle"* ]]; then + echo "FAIL: $message" + echo " unexpected: $needle" + echo " in: $haystack" + exit 1 + fi +} + assert_success() { local message="$1" shift @@ -61,6 +87,51 @@ write_full_config() { } > "$file" } +assert_eq "en" "$(DEPLOYMENT_LANG="" LC_ALL="" LC_MESSAGES="" LANGUAGE="" LANG="en_US.UTF-8" deployment_detect_language)" "English locale should select English" +assert_eq "zh" "$(DEPLOYMENT_LANG="" LC_ALL="" LC_MESSAGES="" LANGUAGE="" LANG="zh_CN.UTF-8" deployment_detect_language)" "Chinese LANG should select Chinese" +assert_eq "zh" "$(DEPLOYMENT_LANG="" LC_ALL="zh_CN.UTF-8" LC_MESSAGES="" LANGUAGE="" LANG="en_US.UTF-8" deployment_detect_language)" "LC_ALL should take priority over LANG" +assert_eq "en" "$(DEPLOYMENT_LANG="en" LC_ALL="" LC_MESSAGES="" LANGUAGE="" LANG="zh_CN.UTF-8" deployment_detect_language)" "DEPLOYMENT_LANG should force English" +assert_eq "zh" "$(DEPLOYMENT_LANG=zh bash -c 'source deploy/common/common.sh; printf "%s" "$DEPLOYMENT_LANGUAGE"')" "language initialization should cache Chinese" +assert_eq "en" "$(DEPLOYMENT_LANG=en LANG="zh_CN.UTF-8" bash -c 'source deploy/common/common.sh; printf "%s" "$DEPLOYMENT_LANGUAGE"')" "language initialization should cache forced English" +assert_eq "后端 API 服务" "$(DEPLOYMENT_LANGUAGE=zh deployment_i18n image_build.detail.main)" "image build TUI details should support Chinese" +assert_eq "backend API service" "$(DEPLOYMENT_LANGUAGE=en deployment_i18n image_build.detail.main)" "image build TUI details should preserve English" + +ZH_DEPLOY_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../deploy.sh" --help)" +assert_contains "$ZH_DEPLOY_HELP" "用法：" "deploy wrapper help should follow Chinese locale" +assert_contains "$ZH_DEPLOY_HELP" "--defaults" "deploy wrapper help should document defaults mode" +EN_DEPLOY_HELP="$(DEPLOYMENT_LANG=en LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../deploy.sh" --help)" +assert_contains "$EN_DEPLOY_HELP" "Usage:" "DEPLOYMENT_LANG should force English deploy wrapper help" +assert_contains "$EN_DEPLOY_HELP" "--defaults" "English deploy wrapper help should document defaults mode" + +ZH_ROOT_DEPLOY_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../../deploy.sh" --help)" +assert_contains "$ZH_ROOT_DEPLOY_HELP" "用法：" "root deploy help should follow Chinese locale" +assert_contains "$ZH_ROOT_DEPLOY_HELP" "此根入口只转发到目标专用部署脚本。" "root deploy help should describe forwarding in Chinese" +assert_contains "$ZH_ROOT_DEPLOY_HELP" "--defaults" "root deploy help should document defaults mode" + +ZH_UNINSTALL_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../uninstall.sh" --help)" +assert_contains "$ZH_UNINSTALL_HELP" "用法：" "uninstall wrapper help should follow Chinese locale" + +ZH_ROOT_UNINSTALL_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../../uninstall.sh" --help)" +assert_contains "$ZH_ROOT_UNINSTALL_HELP" "用法：" "root uninstall help should follow Chinese locale" +assert_contains "$ZH_ROOT_UNINSTALL_HELP" "此根入口只转发到目标专用卸载脚本。" "root uninstall help should describe forwarding in Chinese" + +ZH_IMAGE_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../images/build.sh" --help)" +assert_contains "$ZH_IMAGE_HELP" "用法：deploy/images/build.sh" "image build help should follow Chinese locale" + +ZH_ROOT_BUILD_HELP="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../../build.sh" --help)" +assert_contains "$ZH_ROOT_BUILD_HELP" "用法：" "root build help should follow Chinese locale" +assert_contains "$ZH_ROOT_BUILD_HELP" "bash build.sh --package" "root build help should document package mode" + +ZH_OFFLINE_DRY_RUN="$(DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../offline/build_offline_package.sh" --version v2.2.0 --platform amd64 --components infrastructure,application --image-source general --target docker --dry-run)" +assert_contains "$ZH_OFFLINE_DRY_RUN" "=== DRY RUN 模式 ===" "offline dry-run should follow Chinese locale" +assert_contains "$ZH_OFFLINE_DRY_RUN" "目标：docker" "offline dry-run target label should follow Chinese locale" + +if DEPLOYMENT_LANG="" LANG="zh_CN.UTF-8" bash "$SCRIPT_DIR/../images/build.sh" --unknown >/tmp/nexent-image-build-zh-invalid.log 2>&1; then + echo "FAIL: unknown image build option should fail" + exit 1 +fi +assert_contains "$(cat /tmp/nexent-image-build-zh-invalid.log)" "未知选项：--unknown" "invalid image build option should follow Chinese locale" + APP_VERSION="latest" deployment_prepare_config --app-version latest assert_eq "infrastructure,application,data-process,supabase" "$DEPLOYMENT_COMPONENTS" "default components should include data-process and supabase" @@ -71,6 +142,11 @@ assert_eq "infrastructure,application" "$DEPLOYMENT_COMPONENTS" "components shou assert_eq "production" "$DEPLOYMENT_PORT_POLICY" "port policy should come from CLI" assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "image source should come from CLI" assert_contains "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" "nexent-web" "application services should include web" +DOCKER_SUMMARY="$(deployment_print_summary docker)" +assert_contains "$DOCKER_SUMMARY" "Deployment components: infrastructure,application" "docker summary should include selected components" +assert_contains "$DOCKER_SUMMARY" "Port policy: production" "docker summary should include selected port policy" +assert_contains "$DOCKER_SUMMARY" "Image source: general" "docker summary should include selected image source" +assert_contains "$DOCKER_SUMMARY" "Docker services: " "docker summary should include selected services" if [[ "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" == *"nexent-data-process"* ]]; then echo "FAIL: application should not include data-process" exit 1 @@ -85,6 +161,27 @@ PRODUCTION_HELM_CONTENT="$(cat "$PRODUCTION_HELM_VALUES")" assert_contains "$PRODUCTION_HELM_CONTENT" $'services:\n northbound:\n type: "NodePort"\n nodePort: 30013' "production k8s should expose northbound as NodePort" assert_contains "$PRODUCTION_HELM_CONTENT" $'services:\n web:\n type: "NodePort"\n nodePort: 30000' "production k8s should expose web as NodePort" +unset NEXENT_IMAGE NEXENT_WEB_IMAGE NEXENT_DATA_PROCESS_IMAGE NEXENT_MCP_DOCKER_IMAGE +unset ELASTICSEARCH_IMAGE POSTGRESQL_IMAGE REDIS_IMAGE MINIO_IMAGE OPENSSH_SERVER_IMAGE +unset SUPABASE_KONG SUPABASE_GOTRUE SUPABASE_DB +deployment_prepare_config --components infrastructure,application --port-policy development --image-source local-latest --image-registry-prefix https://registry.local/nexent/ --app-version latest +assert_eq "registry.local/nexent" "$DEPLOYMENT_IMAGE_REGISTRY_PREFIX" "image registry prefix should be normalized" +deployment_apply_image_source +assert_eq "registry.local/nexent/nexent/nexent:latest" "$NEXENT_IMAGE" "image registry prefix should be applied to local latest backend image" +PREFIXED_DOCKER_ENV="$TMP_DIR/prefixed-docker.env" +deployment_render_docker_env "$PREFIXED_DOCKER_ENV" +assert_contains "$(cat "$PREFIXED_DOCKER_ENV")" 'NEXENT_IMAGE_REGISTRY_PREFIX="registry.local/nexent/"' "docker env should expose registry prefix for monitoring compose images" +PREFIXED_HELM_VALUES="$TMP_DIR/prefixed-generated-values.yaml" +deployment_render_helm_values "$PREFIXED_HELM_VALUES" +PREFIXED_HELM_CONTENT="$(cat "$PREFIXED_HELM_VALUES")" +assert_contains "$PREFIXED_HELM_CONTENT" 'imageRegistryPrefix: "registry.local/nexent"' "helm values should record registry prefix" +assert_contains "$PREFIXED_HELM_CONTENT" 'repository: "registry.local/nexent/nexent/nexent"' "helm values should use prefixed app image repositories" +assert_contains "$PREFIXED_HELM_CONTENT" $' initImage:\n repository: "registry.local/nexent/postgres"\n tag: "15-alpine"\n pullPolicy: "IfNotPresent"' "helm values should use prefixed supabase auth init image" +assert_contains "$PREFIXED_HELM_CONTENT" 'pullPolicy: "IfNotPresent"' "prefixed local-latest images should be pulled from the registry" +unset NEXENT_IMAGE NEXENT_WEB_IMAGE NEXENT_DATA_PROCESS_IMAGE NEXENT_MCP_DOCKER_IMAGE +unset ELASTICSEARCH_IMAGE POSTGRESQL_IMAGE REDIS_IMAGE MINIO_IMAGE OPENSSH_SERVER_IMAGE +unset SUPABASE_KONG SUPABASE_GOTRUE SUPABASE_DB + deployment_prepare_config --components supabase --port-policy development --app-version latest assert_eq "infrastructure,supabase" "$DEPLOYMENT_COMPONENTS" "only infrastructure should be required and added" if [[ "$DEPLOYMENT_SELECTED_DOCKER_SERVICES" == *"nexent-web"* ]]; then @@ -95,6 +192,11 @@ fi deployment_prepare_config --components infrastructure,application --port-policy development --registry-profile mainland --app-version latest assert_eq "mainland" "$DEPLOYMENT_IMAGE_SOURCE" "legacy registry profile should map to mainland image source" +DEPLOYMENT_APP_VERSION="v1.2.3" +assert_eq "nexent/nexent:v1.2.3" "$(deployment_image_source_example_tag general)" "general image source description should list backend image tag" +assert_eq "ccr.ccs.tencentyun.com/nexent-hub/nexent:v1.2.3" "$(deployment_image_source_example_tag mainland)" "mainland image source description should list mirrored backend image tag" +assert_eq "nexent/nexent:latest" "$(deployment_image_source_example_tag local-latest)" "local-latest image source description should list local backend image tag" + if deployment_prepare_config --components infrastructure,application --port-policy development --image-source pinned --app-version latest 2>/dev/null; then echo "FAIL: pinned image source should be rejected" exit 1 @@ -111,11 +213,52 @@ unset DEPLOYMENT_VERSION DEPLOYMENT_MODE IS_MAINLAND FULL_CONFIG="$TMP_DIR/full.yaml" write_full_config "$FULL_CONFIG" -deployment_prepare_config --config "$FULL_CONFIG" + +APP_VERSION="v2.2.2" +NEXENT_DEPLOY_CONFIG_MODE=defaults deployment_prepare_config --local-config "$FULL_CONFIG" +assert_eq "v2.2.2" "$DEPLOYMENT_APP_VERSION" "local config appVersion should not override current VERSION" +assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "local config should still load saved components while ignoring appVersion" +NEXENT_DEPLOY_CONFIG_MODE=defaults deployment_prepare_config --local-config "$FULL_CONFIG" --version v2.2.3 +assert_eq "v2.2.3" "$DEPLOYMENT_APP_VERSION" "explicit --version should override current VERSION" +APP_VERSION="latest" + +NEXENT_DEPLOY_CONFIG_MODE=defaults deployment_prepare_config --local-config "$FULL_CONFIG" --app-version latest deployment_apply_image_source assert_eq "nexent/nexent:latest" "$NEXENT_IMAGE" "local-latest image should be applied" assert_contains "$DEPLOYMENT_SELECTED_HELM_CHARTS" "nexent-data-process" "data-process chart should be selected" +NEXENT_DEPLOY_CONFIG_MODE=defaults deployment_prepare_config --local-config "$TMP_DIR/missing.yaml" --app-version latest +assert_eq "infrastructure,application,data-process,supabase" "$DEPLOYMENT_COMPONENTS" "defaults mode should use built-in defaults when local config is absent" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "defaults mode should use built-in image source when local config is absent" + +deployment_prepare_config --defaults --local-config "$TMP_DIR/missing.yaml" --app-version latest +assert_eq "infrastructure,application,data-process,supabase" "$DEPLOYMENT_COMPONENTS" "--defaults should use built-in defaults when local config is absent" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "--defaults should use built-in image source when local config is absent" + +deployment_prepare_config --local-config "$FULL_CONFIG" --defaults --image-source general --app-version latest +assert_eq "true" "$DEPLOYMENT_CONFIG_FILE_LOADED" "--defaults should load saved local config" +assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "--defaults should include saved components" +assert_eq "development" "$DEPLOYMENT_PORT_POLICY" "--defaults should include saved port policy" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "explicit CLI image source should override --defaults local config" + +if deployment_prepare_config --config "$FULL_CONFIG" --app-version latest 2>"$TMP_DIR/config-tui-error.log"; then + echo "FAIL: --config should request interactive TUI and fail without a TTY" + exit 1 +fi +assert_contains "$(cat "$TMP_DIR/config-tui-error.log")" "Interactive deployment configuration requires a TTY." "--config should no longer load a config file path" +if deployment_prepare_config --defaults --config --app-version latest 2>"$TMP_DIR/defaults-config-tui-error.log"; then + echo "FAIL: --config should override earlier --defaults and require a TTY" + exit 1 +fi +assert_contains "$(cat "$TMP_DIR/defaults-config-tui-error.log")" "Interactive deployment configuration requires a TTY." "--config should override earlier --defaults" +deployment_prepare_config --config --defaults --local-config "$FULL_CONFIG" --app-version latest +assert_eq "true" "$DEPLOYMENT_CONFIG_FILE_LOADED" "--defaults should override earlier --config when it appears later" +if NEXENT_DEPLOY_CONFIG_MODE=tui deployment_prepare_config --app-version latest 2>"$TMP_DIR/mode-tui-error.log"; then + echo "FAIL: NEXENT_DEPLOY_CONFIG_MODE=tui should fail without a TTY" + exit 1 +fi +assert_contains "$(cat "$TMP_DIR/mode-tui-error.log")" "Interactive deployment configuration requires a TTY." "tui mode should require a TTY" + DEPLOYMENT_VERSION="speed" DEPLOYMENT_MODE="production" IS_MAINLAND="Y" @@ -128,16 +271,248 @@ unset DEPLOYMENT_VERSION DEPLOYMENT_MODE IS_MAINLAND LOCAL_HELM_VALUES="$TMP_DIR/local-generated-values.yaml" deployment_render_helm_values "$LOCAL_HELM_VALUES" -assert_contains "$(sed -n '1,90p' "$LOCAL_HELM_VALUES")" "repository: \"nexent/nexent\"" "local-latest should render mcp chart with backend image" -assert_contains "$(sed -n '1,90p' "$LOCAL_HELM_VALUES")" "pullPolicy: \"Never\"" "local-latest should render mcp chart with local pull policy" -assert_contains "$(sed -n '140,180p' "$LOCAL_HELM_VALUES")" "repository: \"nexent/nexent-mcp\"" "local-latest should keep common mcp docker image" +LOCAL_HELM_CONTENT="$(cat "$LOCAL_HELM_VALUES")" +assert_contains "$LOCAL_HELM_CONTENT" "repository: \"nexent/nexent\"" "local-latest should render mcp chart with backend image" +assert_contains "$LOCAL_HELM_CONTENT" "pullPolicy: \"Never\"" "local-latest should render mcp chart with local pull policy" +assert_contains "$LOCAL_HELM_CONTENT" "repository: \"nexent/nexent-mcp\"" "local-latest should keep common mcp docker image" + +FAKE_DOCKER_DIR="$TMP_DIR/fake-docker" +FAKE_DOCKER_LOG="$TMP_DIR/fake-docker.log" +mkdir -p "$FAKE_DOCKER_DIR" +cat > "$FAKE_DOCKER_DIR/docker" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$FAKE_DOCKER_LOG" + +if [ "${1:-}" = "image" ] && [ "${2:-}" = "inspect" ]; then + shift 2 + image="" + while [ "$#" -gt 0 ]; do + case "$1" in + --format) + shift 2 + ;; + *) + image="$1" + shift + ;; + esac + done + [ "$image" = "missing/nexent:latest" ] && exit 1 + printf '%s\n' "${FAKE_DOCKER_IMAGE_ID:-sha256:fake-image-id}" + exit 0 +fi + +echo "unexpected docker command: $*" >&2 +exit 2 +SH +chmod +x "$FAKE_DOCKER_DIR/docker" + +( + PATH="$FAKE_DOCKER_DIR:$PATH" + export PATH FAKE_DOCKER_LOG + FAKE_DOCKER_IMAGE_ID="sha256:first-local-image" + export FAKE_DOCKER_IMAGE_ID + + first_image_checksum="$(deployment_image_rollout_checksum "nexent/nexent:latest")" + first_rollout_checksums="$( + NEXENT_IMAGE="nexent/nexent:latest" \ + NEXENT_WEB_IMAGE="nexent/nexent-web:latest" \ + NEXENT_DATA_PROCESS_IMAGE="nexent/nexent-data-process:latest" \ + OPENSSH_SERVER_IMAGE="nexent/nexent-ubuntu-terminal:latest" \ + deployment_render_image_rollout_checksums + )" + + FAKE_DOCKER_IMAGE_ID="sha256:second-local-image" + export FAKE_DOCKER_IMAGE_ID + second_image_checksum="$(deployment_image_rollout_checksum "nexent/nexent:latest")" + second_rollout_checksums="$( + NEXENT_IMAGE="nexent/nexent:latest" \ + NEXENT_WEB_IMAGE="nexent/nexent-web:latest" \ + NEXENT_DATA_PROCESS_IMAGE="nexent/nexent-data-process:latest" \ + OPENSSH_SERVER_IMAGE="nexent/nexent-ubuntu-terminal:latest" \ + deployment_render_image_rollout_checksums + )" + + assert_not_eq "$first_image_checksum" "$second_image_checksum" "local image ID changes should change image rollout checksum" + assert_not_eq "$first_rollout_checksums" "$second_rollout_checksums" "rendered image rollout checksums should change when local image IDs change" + assert_contains "$first_rollout_checksums" "backendImage:" "image rollout checksums should include backend image" + assert_contains "$first_rollout_checksums" "webImage:" "image rollout checksums should include web image" + assert_contains "$first_rollout_checksums" "dataProcessImage:" "image rollout checksums should include data-process image" + assert_contains "$first_rollout_checksums" "sshImage:" "image rollout checksums should include ssh image" + + missing_checksum_a="$(deployment_image_rollout_checksum "missing/nexent:latest" 2>/dev/null)" + FAKE_DOCKER_IMAGE_ID="sha256:third-local-image" + export FAKE_DOCKER_IMAGE_ID + missing_checksum_b="$(deployment_image_rollout_checksum "missing/nexent:latest" 2>/dev/null)" + assert_eq "$missing_checksum_a" "$missing_checksum_b" "missing local image should use stable image reference fallback" +) + +assert_contains "$(cat "$FAKE_DOCKER_LOG")" "image inspect --format {{.Id}} nexent/nexent:latest" "image fingerprint should inspect local Docker image" +if grep -Eq '(^| )(pull|manifest|buildx|imagetools)( |$)' "$FAKE_DOCKER_LOG"; then + echo "FAIL: image fingerprint should not use remote Docker commands" + cat "$FAKE_DOCKER_LOG" + exit 1 +fi + +K8S_CHART_DIR="$SCRIPT_DIR/../k8s/helm/nexent" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-config/templates/deployment.yaml")" "checksum/nexent-backend-image" "config deployment should include backend image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-config/templates/deployment.yaml")" "checksum/nexent-env" "config deployment should include env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-config/templates/deployment.yaml")" "checksum/nexent-backend:" "config deployment should not keep removed backend rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-runtime/templates/deployment.yaml")" "checksum/nexent-backend-image" "runtime deployment should include backend image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-runtime/templates/deployment.yaml")" "checksum/nexent-env" "runtime deployment should include env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-runtime/templates/deployment.yaml")" "checksum/nexent-backend:" "runtime deployment should not keep removed backend rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-mcp/templates/deployment.yaml")" "checksum/nexent-backend-image" "mcp deployment should include backend image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-mcp/templates/deployment.yaml")" "checksum/nexent-env" "mcp deployment should include env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-mcp/templates/deployment.yaml")" "checksum/nexent-backend:" "mcp deployment should not keep removed backend rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-northbound/templates/deployment.yaml")" "checksum/nexent-backend-image" "northbound deployment should include backend image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-northbound/templates/deployment.yaml")" "checksum/nexent-env" "northbound deployment should include env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-northbound/templates/deployment.yaml")" "checksum/nexent-backend:" "northbound deployment should not keep removed backend rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-web/templates/deployment.yaml")" "checksum/nexent-web-image" "web deployment should include web image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-web/templates/deployment.yaml")" "checksum/nexent-env" "web deployment should include env rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-data-process/templates/deployment.yaml")" "checksum/nexent-data-process-image" "data-process deployment should include data-process image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-data-process/templates/deployment.yaml")" "checksum/nexent-env" "data-process deployment should include env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-data-process/templates/deployment.yaml")" "checksum/nexent-backend:" "data-process deployment should not keep removed backend rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-openssh/templates/deployment.yaml")" "checksum/nexent-ssh-image" "openssh deployment should include ssh image rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-openssh/templates/deployment.yaml")" "checksum/nexent-env" "openssh deployment should include env rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-minio/templates/deployment.yaml")" "checksum/nexent-env" "minio deployment should include env rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-auth/templates/deployment.yaml")" "checksum/nexent-supabase-secret" "supabase auth deployment should include supabase secret rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-auth/templates/deployment.yaml")" ".Values.initImage.repository" "supabase auth init container should use configurable image repository" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-auth/templates/deployment.yaml")" "image: postgres:15-alpine" "supabase auth init container should not hardcode postgres image" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-auth/templates/deployment.yaml")" "checksum/nexent-env" "supabase auth deployment should not use full env rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-db/templates/deployment.yaml")" "checksum/nexent-supabase-secret" "supabase db deployment should include supabase secret rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-db/templates/deployment.yaml")" "checksum/nexent-sql" "supabase db deployment should keep SQL rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-db/templates/deployment.yaml")" "checksum/nexent-env" "supabase db deployment should not use full env rollout annotation" +assert_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-kong/templates/deployment.yaml")" "checksum/nexent-supabase-secret" "supabase kong deployment should include supabase secret rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-supabase-kong/templates/deployment.yaml")" "checksum/nexent-env" "supabase kong deployment should not use full env rollout annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-web/templates/deployment.yaml")" "checksum/nexent-web:" "web deployment should not keep component-named env checksum annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-openssh/templates/deployment.yaml")" "checksum/nexent-ssh:" "openssh deployment should not keep component-named env checksum annotation" +assert_not_contains "$(cat "$K8S_CHART_DIR/charts/nexent-minio/templates/deployment.yaml")" "checksum/nexent-minio" "minio deployment should not keep component-named env checksum annotation" + +ENV_CHECKSUM_A="$TMP_DIR/env-checksum-a.env" +cat > "$ENV_CHECKSUM_A" <<'ENV' +# ignored comment + +BETA=two +ALPHA=one +export GAMMA="three" +not an assignment +ENV +DEPLOYMENT_ROOT_ENV="$ENV_CHECKSUM_A" +ENV_CHECKSUM_PAYLOAD_A="$(deployment_env_values_payload)" +ENV_CHECKSUM_A_VALUE="$(deployment_env_values_checksum)" +assert_eq $'ALPHA=one\nBETA=two\nGAMMA="three"' "$ENV_CHECKSUM_PAYLOAD_A" "env checksum payload should normalize valid assignments by key" + +ENV_CHECKSUM_B="$TMP_DIR/env-checksum-b.env" +cat > "$ENV_CHECKSUM_B" <<'ENV' + +export GAMMA="three" +# another ignored comment +ALPHA=one +BETA=two +ENV +DEPLOYMENT_ROOT_ENV="$ENV_CHECKSUM_B" +ENV_CHECKSUM_B_VALUE="$(deployment_env_values_checksum)" +assert_eq "$ENV_CHECKSUM_A_VALUE" "$ENV_CHECKSUM_B_VALUE" "env checksum should ignore comments, blank lines, and assignment order" + +ENV_CHECKSUM_C="$TMP_DIR/env-checksum-c.env" +cat > "$ENV_CHECKSUM_C" <<'ENV' +ALPHA=one +BETA=changed +GAMMA="three" +ENV +DEPLOYMENT_ROOT_ENV="$ENV_CHECKSUM_C" +ENV_CHECKSUM_C_VALUE="$(deployment_env_values_checksum)" +assert_not_eq "$ENV_CHECKSUM_A_VALUE" "$ENV_CHECKSUM_C_VALUE" "env checksum should change when any valid env value changes" + +MONITORING_ROOT_ENV="$TMP_DIR/monitoring-root.env" +MONITORING_EXAMPLE_TMP="$TMP_DIR/monitoring.env.example" +MONITORING_ENV_TMP="$TMP_DIR/monitoring.env" +cp "$SCRIPT_DIR/../env/monitoring.env.example" "$MONITORING_EXAMPLE_TMP" +cat > "$MONITORING_ROOT_ENV" <<'ENV' +LANGSMITH_API_KEY=ls-root-fallback +MONITORING_PROVIDER=root-provider +ENV +DEPLOYMENT_ROOT_ENV="$MONITORING_ROOT_ENV" +deployment_source_env_file "$MONITORING_ROOT_ENV" +deployment_prepare_config --components infrastructure,application,monitoring --monitoring-provider langsmith --app-version latest +deployment_prepare_monitoring_env k8s +assert_eq "true" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "ENABLE_TELEMETRY")" "monitoring.env should record selected monitoring enablement" +assert_eq "langsmith" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "MONITORING_PROVIDER")" "monitoring.env should record selected provider" +assert_eq "https://smith.langchain.com/" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL")" "monitoring.env should record K8s dashboard URL" +assert_eq "http://nexent-otel-collector:4318" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "OTEL_EXPORTER_OTLP_ENDPOINT")" "monitoring.env should record K8s OTLP endpoint" +assert_eq "otel-collector-langsmith-config.yml" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "OTEL_COLLECTOR_CONFIG_FILE")" "monitoring.env should record K8s collector config file" +assert_eq "ls-root-fallback" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "LANGSMITH_API_KEY")" "monitoring.env should migrate LangSmith key from root .env when missing" +MONITORING_PAYLOAD="$(deployment_env_values_payload)" +assert_contains "$MONITORING_PAYLOAD" 'MONITORING_PROVIDER="langsmith"' "monitoring.env should override duplicate root monitoring keys in checksum payload" +assert_not_contains "$MONITORING_PAYLOAD" "MONITORING_PROVIDER=root-provider" "root monitoring provider should not win over monitoring.env" +MONITORING_CHECKSUM_A="$(deployment_env_values_checksum)" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "MONITORING_TRACE_MAX_CHARS" "1234" +MONITORING_CHECKSUM_B="$(deployment_env_values_checksum)" +assert_not_eq "$MONITORING_CHECKSUM_A" "$MONITORING_CHECKSUM_B" "env checksum should change when monitoring.env changes" +MONITORING_HELM_VALUES="$TMP_DIR/monitoring-generated-values.yaml" +deployment_render_helm_values "$MONITORING_HELM_VALUES" +MONITORING_HELM_CONTENT="$(cat "$MONITORING_HELM_VALUES")" +assert_contains "$MONITORING_HELM_CONTENT" 'provider: "langsmith"' "Helm values should render monitoring provider from monitoring.env" +assert_contains "$MONITORING_HELM_CONTENT" 'langsmithApiKey: "ls-root-fallback"' "Helm values should pass LangSmith key to monitoring collector values" +assert_contains "$MONITORING_HELM_CONTENT" 'configFile: "otel-collector-langsmith-config.yml"' "Helm values should pass collector config from monitoring.env" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "LANGFUSE_INIT_PROJECT_PUBLIC_KEY" "pk-test" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "LANGFUSE_INIT_PROJECT_SECRET_KEY" "sk-test" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "LANGFUSE_OTLP_AUTH_HEADER" "Basic stale" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL" "" +deployment_prepare_config --components infrastructure,application,monitoring --monitoring-provider langfuse --app-version latest +deployment_prepare_monitoring_env docker +EXPECTED_LANGFUSE_AUTH_HEADER="Basic $(printf "%s:%s" "pk-test" "sk-test" | base64 | tr -d '\n')" +assert_eq "$EXPECTED_LANGFUSE_AUTH_HEADER" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "LANGFUSE_OTLP_AUTH_HEADER")" "monitoring.env should refresh derived Langfuse OTLP auth header" +assert_eq "../assets/monitoring/otel-collector-langfuse-config.yml" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "OTEL_COLLECTOR_CONFIG_FILE")" "monitoring.env should record Docker collector config file" +assert_eq "http://localhost:3001" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL")" "empty monitoring dashboard URL should use the selected provider default" +deployment_update_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL" "https://monitor.example.com/grafana" +deployment_prepare_config --components infrastructure,application,monitoring --monitoring-provider grafana --app-version latest +deployment_prepare_monitoring_env docker +assert_eq "https://monitor.example.com/grafana" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL")" "explicit monitoring dashboard URL should be preserved" +deployment_prepare_config --components infrastructure,application --monitoring-provider grafana --app-version latest +deployment_prepare_monitoring_env docker +assert_eq "" "$(deployment_get_env_var_file "$MONITORING_ENV_TMP" "MONITORING_DASHBOARD_URL")" "disabled monitoring should clear dashboard URL" +while IFS='=' read -r key _; do + [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue + unset "$key" +done < "$MONITORING_EXAMPLE_TMP" +unset DEPLOYMENT_MONITORING_PROVIDER + +DEPLOYMENT_ROOT_ENV="$TMP_DIR/missing-env-file.env" +MISSING_ENV_CHECKSUM_A="$(deployment_env_values_checksum 2>/dev/null)" +MISSING_ENV_CHECKSUM_B="$(deployment_env_values_checksum 2>/dev/null)" +assert_eq "$MISSING_ENV_CHECKSUM_A" "$MISSING_ENV_CHECKSUM_B" "missing env file should use a stable empty checksum fallback" + +K8S_DEPLOY_CHECKSUM_BLOCK="$(awk '/render_runtime_secret_values\(\) {/,/deployment_render_image_rollout_checksums/' "$SCRIPT_DIR/../k8s/deploy.sh")" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'env_checksum="$(deployment_env_values_checksum)"' "k8s deploy should compute rollout checksum from root and monitoring env files" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "printf ' env: %s\\n'" "k8s deploy should render the combined env checksum under the env name" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "supabase_secret_checksum" "k8s deploy should compute a dedicated Supabase secret rollout checksum" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "printf ' supabaseSecret: %s\\n'" "k8s deploy should render the Supabase secret checksum" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'JWT_SECRET:-' "supabase secret checksum should include JWT secret" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'SECRET_KEY_BASE:-' "supabase secret checksum should include secret key base" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'VAULT_ENC_KEY:-' "supabase secret checksum should include vault encryption key" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'SUPABASE_ANON_KEY:-' "supabase secret checksum should include anon key" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'SUPABASE_SERVICE_ROLE_KEY:-' "supabase secret checksum should include service role key" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'supabase_postgres_password' "supabase secret checksum should include Supabase postgres password" +assert_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'gotrue_db_url' "supabase secret checksum should include gotrue DB URL" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "printf ' backend:" "k8s deploy should not render the removed backend rollout checksum" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "backend_checksum" "k8s deploy should not compute the removed backend rollout checksum" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'ELASTICSEARCH_API_KEY' "k8s deploy should not enumerate backend env variables in rollout checksum" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'DASHBOARD_USERNAME' "supabase secret checksum should not include direct Supabase env values" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" 'SITE_URL' "supabase secret checksum should not include direct Supabase env values" +assert_not_contains "$K8S_DEPLOY_CHECKSUM_BLOCK" "printf ' web:" "k8s deploy should not render component-named env checksum keys" DEPLOYMENT_VERSION="speed" -deployment_prepare_config --local-config "$FULL_CONFIG" --reconfigure --image-source general --app-version latest -assert_eq "false" "$DEPLOYMENT_CONFIG_FILE_LOADED" "reconfigure should use local config as defaults without skipping configuration" -assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "reconfigure defaults should include saved components" -assert_eq "development" "$DEPLOYMENT_PORT_POLICY" "reconfigure defaults should include saved port policy" -assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "explicit image source should override reconfigure defaults" +if deployment_prepare_config --local-config "$FULL_CONFIG" --reconfigure --image-source general --app-version latest 2>"$TMP_DIR/reconfigure-tui-error.log"; then + echo "FAIL: --reconfigure should request interactive TUI and fail without a TTY" + exit 1 +fi +assert_contains "$(cat "$TMP_DIR/reconfigure-tui-error.log")" "Interactive deployment configuration requires a TTY." "--reconfigure should require a TTY" +NEXENT_DEPLOY_CONFIG_MODE=defaults deployment_prepare_config --local-config "$FULL_CONFIG" --image-source general --app-version latest +assert_eq "true" "$DEPLOYMENT_CONFIG_FILE_LOADED" "defaults mode should load local config without entering TUI" +assert_contains "$DEPLOYMENT_COMPONENTS" "data-process" "defaults mode should include saved components" +assert_eq "development" "$DEPLOYMENT_PORT_POLICY" "defaults mode should include saved port policy" +assert_eq "general" "$DEPLOYMENT_IMAGE_SOURCE" "explicit image source should override defaults mode local config" unset DEPLOYMENT_VERSION HELM_VALUES="$TMP_DIR/generated-values.yaml" @@ -149,12 +524,144 @@ assert_contains "$(sed -n '1,260p' "$HELM_VALUES")" "enabled: true" "selected ch DOCKER_ENV="$TMP_DIR/.env.generated" deployment_render_docker_env "$DOCKER_ENV" assert_contains "$(sed -n '1,120p' "$DOCKER_ENV")" "NEXENT_IMAGE=" "docker generated env should contain image variables" +assert_not_contains "$(sed -n '1,120p' "$DOCKER_ENV")" "ENABLE_TELEMETRY=" "docker generated env should not contain monitoring enablement" +assert_not_contains "$(sed -n '1,120p' "$DOCKER_ENV")" "MONITORING_PROVIDER=" "docker generated env should not contain monitoring provider" +assert_not_contains "$(sed -n '1,120p' "$DOCKER_ENV")" "MONITORING_DASHBOARD_URL=" "docker generated env should not contain monitoring dashboard URL" if grep -Eq '^DEPLOYMENT_(SCHEMA_VERSION|COMPONENTS|PORT_POLICY|IMAGE_SOURCE|REGISTRY_PROFILE|APP_VERSION|MONITORING_PROVIDER|SELECTED_DOCKER_SERVICES|DOCKER_PORTS)=' "$DOCKER_ENV"; then echo "FAIL: docker generated env should not contain persisted deployment decisions" exit 1 fi +COMMON_MONITORING_CONFIG_BLOCK="$(awk '/deployment_monitoring_env_example_file\(\)/,/^deployment_sha256_string\(\)/' "$SCRIPT_DIR/../common/common.sh")" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'deployment_prepare_monitoring_env()' "common deploy should prepare monitoring.env" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'deployment_sync_env_defaults "$example_file" "$env_file"' "common deploy should sync monitoring.env defaults" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'deployment_source_env_file "$env_file"' "common deploy should source generated monitoring.env" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'deployment_update_monitoring_env_var "OTEL_EXPORTER_OTLP_ENDPOINT" "$otlp_endpoint"' "common deploy should persist target OTLP endpoint in monitoring.env" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'deployment_update_monitoring_env_var "OTEL_COLLECTOR_CONFIG_FILE" "$collector_config_file"' "common deploy should persist monitoring collector config in monitoring.env" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'otel-collector-phoenix-config.yml' "common deploy should map phoenix to its collector config" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'otel-collector-langfuse-config.yml' "common deploy should map langfuse to its collector config" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'otel-collector-langsmith-config.yml' "common deploy should map langsmith to its collector config" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'otel-collector-grafana-config.yml' "common deploy should map grafana to its collector config" +assert_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'otel-collector-zipkin-config.yml' "common deploy should map zipkin to its collector config" +assert_not_contains "$COMMON_MONITORING_CONFIG_BLOCK" 'update_env_var "OTEL_EXPORTER_OTLP_ENDPOINT"' "common monitoring prep should not write monitoring keys to deploy/env/.env" + +DOCKER_MONITORING_CONFIG_BLOCK="$(awk '/docker_monitoring_active_services\(\)/,/^pull_mcp_image\(\)/' "$SCRIPT_DIR/../docker/deploy.sh")" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'deployment_prepare_monitoring_env docker' "docker deploy should use shared monitoring.env preparation" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'docker_monitoring_active_services()' "docker deploy should define active monitoring services" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'docker_monitoring_profile_services()' "docker deploy should enumerate monitoring profile services" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'docker_monitoring_container_names()' "docker deploy should enumerate monitoring containers for cleanup" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'stop_monitoring_services()' "docker deploy should define full monitoring stop cleanup" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'cleanup_stale_monitoring_services()' "docker deploy should define stale monitoring cleanup" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'langfuse-worker langfuse-web langfuse-clickhouse langfuse-minio langfuse-redis langfuse-postgres' "docker deploy should treat Langfuse services as one provider bundle" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'down --remove-orphans' "docker deploy should stop monitoring services when monitoring is disabled" +assert_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'rm -f "${stale_services[@]}"' "docker deploy should remove stale monitoring containers without deleting volumes" +assert_not_contains "$DOCKER_MONITORING_CONFIG_BLOCK" 'update_env_var "OTEL_EXPORTER_OTLP_ENDPOINT"' "docker deploy should not write monitoring keys to deploy/env/.env" +assert_not_contains "$DOCKER_MONITORING_CONFIG_BLOCK" "LANGFUSE_OLTP_AUTH_HEADER" "docker deploy should not use the misspelled Langfuse OLTP auth header alias" + +if [ -e "$SCRIPT_DIR/../docker/start-monitoring.sh" ]; then + echo "FAIL: standalone monitoring script should be removed" + exit 1 +fi +DOCKER_COMPOSE_FILE="$SCRIPT_DIR/../docker/compose/docker-compose.yml" +DOCKER_PROD_COMPOSE_FILE="$SCRIPT_DIR/../docker/compose/docker-compose.prod.yml" +DOCKER_DEV_COMPOSE_FILE="$SCRIPT_DIR/../docker/compose/docker-compose.dev.yml" +for compose_file in "$DOCKER_COMPOSE_FILE" "$DOCKER_PROD_COMPOSE_FILE"; do + assert_contains "$(awk '/^ nexent-config:/,/^ nexent-runtime:/' "$compose_file")" $'env_file:\n - ../../env/.env\n - ../../env/monitoring.env' "docker config service should load monitoring.env after root .env" + assert_contains "$(awk '/^ nexent-runtime:/,/^ nexent-mcp:/' "$compose_file")" $'env_file:\n - ../../env/.env\n - ../../env/monitoring.env' "docker runtime service should load monitoring.env after root .env" + assert_not_contains "$(awk '/^ nexent-mcp:/,/^ nexent-northbound:/' "$compose_file")" "monitoring.env" "docker mcp service should not receive monitoring.env" + assert_not_contains "$(awk '/^ nexent-northbound:/,/^ nexent-web:/' "$compose_file")" "monitoring.env" "docker northbound service should not receive monitoring.env" + assert_not_contains "$(awk '/^ nexent-data-process:/,/^ redis:/' "$compose_file")" "monitoring.env" "docker data-process service should not receive monitoring.env" +done +assert_not_contains "$(cat "$DOCKER_DEV_COMPOSE_FILE")" "monitoring.env" "docker dev data-process compose should not receive monitoring.env" +assert_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" 'LANGFUSE_OTLP_AUTH_HEADER: ${LANGFUSE_OTLP_AUTH_HEADER:-}' "docker monitoring compose should pass Langfuse OTLP auth header to the collector" +assert_not_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" "LANGFUSE_OLTP_AUTH_HEADER" "docker monitoring compose should not pass the misspelled Langfuse auth header alias" +assert_contains "$(cat "$SCRIPT_DIR/../k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml")" "LANGFUSE_OTLP_AUTH_HEADER" "k8s collector should pass Langfuse OTLP auth header" +assert_not_contains "$(cat "$SCRIPT_DIR/../k8s/helm/nexent/charts/nexent-monitoring/templates/otel-collector.yaml")" "LANGFUSE_OLTP_AUTH_HEADER" "k8s collector should not pass the misspelled Langfuse auth header alias" + +DOCKER_DEPLOY_MONITORING_BLOCK="$(awk '/deploy_monitoring\(\)/,/^configure_root_dir_from_env\(\)/' "$SCRIPT_DIR/../docker/deploy.sh")" +assert_contains "$DOCKER_DEPLOY_MONITORING_BLOCK" 'stop_monitoring_services || return 1' "docker deploy should stop monitoring services when the component is disabled" +assert_contains "$DOCKER_DEPLOY_MONITORING_BLOCK" 'LANGSMITH_API_KEY is required' "docker deploy should fail fast when LangSmith API key is missing" +assert_contains "$DOCKER_DEPLOY_MONITORING_BLOCK" 'profile_args+=(--profile "$DEPLOYMENT_MONITORING_PROVIDER")' "docker deploy should keep provider-specific compose profiles" +assert_contains "$DOCKER_DEPLOY_MONITORING_BLOCK" 'cleanup_stale_monitoring_services || return 1' "docker deploy should remove stale monitoring provider services before starting selected provider" +assert_contains "$DOCKER_DEPLOY_MONITORING_BLOCK" '--env-file "$ROOT_ENV_FILE" --env-file "$MONITORING_ENV_FILE"' "docker deploy should load generated monitoring.env for monitoring compose" + +DOCKER_UNINSTALL_CONTENT="$(cat "$SCRIPT_DIR/../docker/uninstall.sh")" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'MONITORING_ENV_FILE="$PROJECT_ROOT/deploy/env/monitoring.env"' "docker uninstall should know the generated monitoring env file" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'env_file_args+=(--env-file "$MONITORING_ENV_FILE")' "docker uninstall should load generated monitoring.env when downing monitoring compose" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'docker_compose_down_file "$COMPOSE_DIR/docker-compose-monitoring.yml" false "$remove_volumes"' "docker uninstall should down the monitoring compose file" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'remove_docker_containers_by_name < <(monitoring_container_names)' "docker uninstall should remove monitoring containers by name as a fallback" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'nexent-otel-collector' "docker uninstall should include the collector container in fallback cleanup" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'nexent-langfuse-web' "docker uninstall should include Langfuse containers in fallback cleanup" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'nexent-grafana' "docker uninstall should include Grafana containers in fallback cleanup" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'nexent-zipkin' "docker uninstall should include Zipkin containers in fallback cleanup" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'monitor_langfuse-postgres-data' "docker uninstall should include monitoring volumes in delete-all cleanup" +assert_contains "$DOCKER_UNINSTALL_CONTENT" 'remove_docker_volumes_by_name < <(monitoring_volume_names)' "docker uninstall should remove monitoring volumes when delete-volumes is enabled" + +K8S_UNINSTALL_CONTENT="$(cat "$SCRIPT_DIR/../k8s/uninstall.sh")" +assert_contains "$K8S_UNINSTALL_CONTENT" "cleanup_leftover_monitoring_resources()" "k8s uninstall should define monitoring fallback cleanup" +assert_contains "$K8S_UNINSTALL_CONTENT" "deployment,service,configmap,rs,pod" "k8s monitoring fallback should remove workloads and service resources" +assert_contains "$K8S_UNINSTALL_CONTENT" "nexent-otel-collector" "k8s monitoring fallback should include the collector" +assert_contains "$K8S_UNINSTALL_CONTENT" "nexent-langfuse-web" "k8s monitoring fallback should include Langfuse" +assert_contains "$K8S_UNINSTALL_CONTENT" "nexent-grafana" "k8s monitoring fallback should include Grafana" +assert_contains "$K8S_UNINSTALL_CONTENT" "nexent-zipkin" "k8s monitoring fallback should include Zipkin" +assert_contains "$K8S_UNINSTALL_CONTENT" "cleanup_leftover_nexent_resources" "k8s uninstall should use shared leftover cleanup" + +MONITORING_EXAMPLE_FILE="$SCRIPT_DIR/../env/monitoring.env.example" +MONITORING_COMPOSE_FILE="$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml" +MONITORING_COMPOSE_DEFAULTS="$TMP_DIR/docker-compose-monitoring-defaults.txt" +awk ' + { + line = $0 + while (match(line, /\$\{[A-Za-z_][A-Za-z0-9_]*:-[^}]*\}/)) { + expr = substr(line, RSTART + 2, RLENGTH - 3) + key = expr + sub(/:-.*/, "", key) + value = expr + sub(/^[^:]*:-/, "", value) + print key "=" value + line = substr(line, RSTART + RLENGTH) + } + } +' "$MONITORING_COMPOSE_FILE" | sort -u > "$MONITORING_COMPOSE_DEFAULTS" +while IFS='=' read -r key compose_default; do + [ -n "$key" ] || continue + case "$key" in + OTEL_COLLECTOR_CONFIG_FILE) + continue + ;; + esac + if example_default="$(deployment_get_env_var_file "$MONITORING_EXAMPLE_FILE" "$key" 2>/dev/null)"; then + assert_eq "$example_default" "$compose_default" "docker compose fallback for $key should match monitoring.env.example" + fi +done < "$MONITORING_COMPOSE_DEFAULTS" + +if [ -f "$SCRIPT_DIR/../docker/assets/monitoring/monitoring.env.example" ]; then + echo "FAIL: monitoring.env.example should live under deploy/env" + exit 1 +fi +if [ -f "$SCRIPT_DIR/../docker/assets/monitoring/monitoring.env" ]; then + echo "FAIL: monitoring.env should live under deploy/env" + exit 1 +fi +ROOT_ENV_EXAMPLE_CONTENT="$(cat "$SCRIPT_DIR/../env/.env.example")" +assert_not_contains "$ROOT_ENV_EXAMPLE_CONTENT" "ENABLE_TELEMETRY=" "root .env.example should not contain monitoring enablement" +assert_not_contains "$ROOT_ENV_EXAMPLE_CONTENT" "MONITORING_PROVIDER=" "root .env.example should not contain monitoring provider" +assert_not_contains "$ROOT_ENV_EXAMPLE_CONTENT" "OTEL_EXPORTER_OTLP_ENDPOINT=" "root .env.example should not contain OTLP endpoint" +assert_not_contains "$ROOT_ENV_EXAMPLE_CONTENT" "TELEMETRY_SAMPLE_RATE=" "root .env.example should not contain telemetry sampling" +assert_contains "$(cat "$MONITORING_EXAMPLE_FILE")" "GRAFANA_ADMIN_PASSWORD=nexent@4321" "docker monitoring defaults should define Grafana admin password" +assert_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" 'GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-nexent@4321}' "docker compose Grafana password fallback should match monitoring.env.example" +assert_contains "$(cat "$SCRIPT_DIR/../k8s/helm/nexent/charts/nexent-monitoring/values.yaml")" "adminPassword: nexent@4321" "k8s monitoring Grafana password should match docker monitoring default" +assert_contains "$(cat "$MONITORING_EXAMPLE_FILE")" "LANGFUSE_POSTGRES_PASSWORD=nexent@4321" "docker monitoring defaults should define Langfuse postgres password" +assert_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" 'LANGFUSE_POSTGRES_PASSWORD:-nexent@4321' "docker compose Langfuse postgres password fallback should match monitoring.env.example" +assert_contains "$(cat "$SCRIPT_DIR/../k8s/helm/nexent/charts/nexent-monitoring/values.yaml")" "password: nexent@4321" "k8s monitoring Langfuse postgres password should match docker monitoring default" +assert_contains "$(cat "$MONITORING_EXAMPLE_FILE")" "LANGFUSE_INIT_USER_PASSWORD=nexent@4321" "docker monitoring defaults should define Langfuse init user password" +assert_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" 'LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-nexent@4321}' "docker compose Langfuse init user password fallback should match monitoring.env.example" +assert_contains "$(cat "$SCRIPT_DIR/../k8s/helm/nexent/charts/nexent-monitoring/values.yaml")" "userPassword: nexent@4321" "k8s monitoring Langfuse init user password should match docker monitoring default" +assert_contains "$(cat "$MONITORING_EXAMPLE_FILE")" "LANGFUSE_CLICKHOUSE_CLUSTER_ENABLED=false" "docker monitoring defaults should include all compose Langfuse clickhouse settings" +assert_contains "$(cat "$SCRIPT_DIR/../docker/compose/docker-compose-monitoring.yml")" 'CLICKHOUSE_CLUSTER_ENABLED: ${LANGFUSE_CLICKHOUSE_CLUSTER_ENABLED:-false}' "docker compose Langfuse clickhouse cluster fallback should match monitoring.env.example" + LOCAL_CONFIG="$TMP_DIR/local-config.yaml" +DEPLOYMENT_IMAGE_REGISTRY_PREFIX="registry.local/nexent" deployment_persist_local_config "$LOCAL_CONFIG" if grep -Eq 'PASSWORD|TOKEN|JWT|SECRET|KEY' "$LOCAL_CONFIG"; then echo "FAIL: persisted local config should not contain secret-looking fields" @@ -164,6 +671,32 @@ if grep -q 'registryProfile' "$LOCAL_CONFIG"; then echo "FAIL: persisted local config should not contain registryProfile" exit 1 fi +if grep -q 'appVersion' "$LOCAL_CONFIG"; then + echo "FAIL: persisted local config should not contain appVersion" + exit 1 +fi +assert_contains "$(cat "$LOCAL_CONFIG")" 'imageRegistryPrefix: "registry.local/nexent"' "persisted local config should include image registry prefix" + +DEPLOY_OPTIONS_FILE="$TMP_DIR/deploy.options" +deployment_init_defaults +assert_eq "$TMP_DIR/deploy.options" "$DEPLOYMENT_LOCAL_CONFIG_PATH" "default local config should use deploy.options" +unset DEPLOY_OPTIONS_FILE + +K8S_DEPLOY_OPTIONS_BLOCK="$(awk '/persist_deploy_options\(\) {/,/^}/' "$SCRIPT_DIR/../k8s/deploy.sh")" +K8S_DEPLOY_HEADER="$(sed -n '1,60p' "$SCRIPT_DIR/../k8s/deploy.sh")" +assert_contains "$K8S_DEPLOY_HEADER" 'DEPLOY_OPTIONS_FILE="$SCRIPT_DIR/deploy.options"' "k8s deploy config should use deploy.options" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" 'deployment_persist_local_config "$DEPLOY_OPTIONS_FILE"' "k8s deploy options should include shared local config" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" "printf 'k8s:\\n'" "k8s deploy options should include a k8s section" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" "persistenceMode:" "k8s deploy options should persist persistence mode" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" "storageClassName:" "k8s deploy options should persist storage class" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" "localPath:" "k8s deploy options should persist local path" +assert_contains "$K8S_DEPLOY_OPTIONS_BLOCK" "existingClaimPrefix:" "k8s deploy options should persist existing claim prefix" +assert_not_contains "$K8S_DEPLOY_OPTIONS_BLOCK" 'LOCAL_NODE_NAME=' "k8s deploy options should not persist deprecated local node name" +assert_not_contains "$K8S_DEPLOY_OPTIONS_BLOCK" 'K8S_WAIT_TIMEOUT_SECONDS=' "k8s deploy options should not persist one-time wait timeout" +if echo "$K8S_DEPLOY_OPTIONS_BLOCK" | grep -Eq 'PASSWORD|TOKEN|JWT|SECRET|KEY'; then + echo "FAIL: k8s deploy options should not persist secret-looking fields" + exit 1 +fi assert_success "b should be treated as TUI back key" deployment_tui_is_back_key "b" assert_success "Backspace should be treated as TUI back key" deployment_tui_is_back_key $'\177' diff --git a/deploy/tests/test_images_build.sh b/deploy/tests/test_images_build.sh index eb1310867..94e81d7a6 100755 --- a/deploy/tests/test_images_build.sh +++ b/deploy/tests/test_images_build.sh @@ -5,6 +5,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" BUILD_SCRIPT="$PROJECT_ROOT/deploy/images/build.sh" +ROOT_BUILD_SCRIPT="$PROJECT_ROOT/build.sh" +export DEPLOYMENT_LANG=en fail() { echo "FAIL: $*" @@ -47,10 +49,31 @@ output="$(bash "$BUILD_SCRIPT" --main --version latest --platform linux/amd64 -- assert_contains "$output" "--platform linux/amd64" "explicit platform should be forwarded" assert_contains "$output" "nexent/nexent:latest" "explicit platform build should still build selected image" +output="$(bash "$ROOT_BUILD_SCRIPT" --web --version latest --dry-run)" +assert_contains "$output" "nexent/nexent-web:latest" "root image build entrypoint should forward to deploy/images/build.sh" +assert_not_contains "$output" "nexent/nexent:latest" "root image build entrypoint should preserve selected image arguments" + +output="$(bash "$BUILD_SCRIPT" --main --version latest --no-cache --dry-run)" +assert_contains "$output" "--no-cache" "explicit no-cache option should be forwarded" +assert_contains "$output" "nexent/nexent:latest" "explicit no-cache build should still build selected image" + +output="$(bash "$BUILD_SCRIPT" --web --version v9.9.9 --registry mainland --dry-run)" +assert_contains "$output" "--no-cache" "mainland web build should avoid stale Docker cache" +assert_contains "$output" "nexent/nexent-web:v9.9.9" "mainland web build without push should keep local Nexent tag" + +output="$(bash "$BUILD_SCRIPT" --web --version v9.9.9 --registry mainland --push --dry-run)" +assert_contains "$output" "--no-cache" "mainland web push should avoid stale Docker cache" +assert_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent-web:v9.9.9" "mainland web push should use CCS tag" + output="$(bash "$BUILD_SCRIPT" --terminal --version v9.9.9 --registry mainland --dry-run)" -assert_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:v9.9.9" "terminal option should build terminal image with selected version" +assert_contains "$output" "nexent/nexent-ubuntu-terminal:v9.9.9" "mainland build without push should keep local Nexent tag" +assert_not_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:v9.9.9" "mainland build without push should not use CCS tag" assert_not_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent:v9.9.9" "main image should not be built for terminal-only option" +output="$(bash "$BUILD_SCRIPT" --terminal --version v9.9.9 --registry mainland --push --dry-run)" +assert_contains "$output" "ccr.ccs.tencentyun.com/nexent-hub/nexent-ubuntu-terminal:v9.9.9" "mainland push should use CCS tag" +assert_not_contains "$output" "nexent/nexent-ubuntu-terminal:v9.9.9" "mainland push should not use local Nexent tag" + output="$(bash "$BUILD_SCRIPT" --web --docs --version v8.8.8 --registry general --dry-run)" assert_contains "$output" "nexent/nexent-web:v8.8.8" "web option should build web image" assert_contains "$output" "nexent/nexent-docs:v8.8.8" "docs option should build docs image" diff --git a/deploy/uninstall.sh b/deploy/uninstall.sh index 01632236c..ce21e4b3c 100755 --- a/deploy/uninstall.sh +++ b/deploy/uninstall.sh @@ -3,8 +3,26 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_COMMON="$SCRIPT_DIR/common/common.sh" + +if [ -f "$DEPLOYMENT_COMMON" ]; then + # shellcheck source=/dev/null + source "$DEPLOYMENT_COMMON" +fi usage() { + if [ "${DEPLOYMENT_LANGUAGE:-en}" = "zh" ]; then + cat <<'USAGE' +用法： + bash uninstall.sh docker [Docker 卸载选项] + bash uninstall.sh k8s [K8s 卸载选项] + +Docker 实现：deploy/docker/uninstall.sh +K8s 实现： deploy/k8s/uninstall.sh +USAGE + return + fi + cat <<'USAGE' Usage: bash uninstall.sh docker [docker uninstall options] @@ -28,7 +46,11 @@ case "${1:-}" in usage ;; *) - echo "Unknown uninstall target: $1" >&2 + if [ "${DEPLOYMENT_LANGUAGE:-en}" = "zh" ]; then + echo "未知卸载目标：$1" >&2 + else + echo "Unknown uninstall target: $1" >&2 + fi usage >&2 exit 1 ;; diff --git a/doc/docs/en/deployment/docker-build.md b/doc/docs/en/deployment/docker-build.md index a69856606..a2bbe92cd 100644 --- a/doc/docs/en/deployment/docker-build.md +++ b/doc/docs/en/deployment/docker-build.md @@ -4,10 +4,13 @@ Recommended unified build entry: ```bash # Run interactive selection, similar to the deploy scripts +bash build.sh + +# Equivalent direct image builder bash deploy/images/build.sh # Build selected images with a fixed version tag -bash deploy/images/build.sh \ +bash build.sh \ --images main,web,mcp,data-process,terminal \ --version v2.2.1 \ --registry general \ @@ -15,7 +18,7 @@ bash deploy/images/build.sh \ --push # Build the same image set as latest -bash deploy/images/build.sh \ +bash build.sh \ --images main,web,mcp,data-process \ --version latest \ --registry general \ @@ -23,12 +26,15 @@ bash deploy/images/build.sh \ --load # Build one or more explicit images when needed -bash deploy/images/build.sh --web --docs --version v2.2.1 --dry-run +bash build.sh --web --docs --version v2.2.1 --dry-run + +# Build without Docker cache +bash build.sh --web --version v2.2.1 --no-cache ``` -When run in a terminal without arguments, `deploy/images/build.sh` prompts for images, image version (`latest` or root `VERSION`), and registry. The interactive defaults are images `main,web` and version `latest`. Use `--interactive` to force the same prompts. +The root `build.sh` forwards image builds to `deploy/images/build.sh`. Use `bash build.sh --package ...` to forward to the offline package builder. When run in a terminal without arguments, `build.sh` prompts for images, image version (`latest` or root `VERSION`), and image source. The interactive defaults are images `main,web` and version `latest`. Use `--interactive` to force the same prompts. -`--platform` is command-line only. Omit it to build for the local architecture. +`--platform` and `--no-cache` are command-line only. Omit `--platform` to build for the local architecture. The `mainland` web build also uses `--no-cache` automatically to avoid stale frontend dependency caches. Variant options: - `--dependency-variant cpu|gpu` controls data-process dependencies and defaults to `cpu`. `gpu` builds GPU/CUDA dependencies and uses the `-gpu` image-name suffix. @@ -157,9 +163,10 @@ docker builder prune -f && docker system prune -f ### 🏷️ Tagging Strategy -Each image is pushed to two repositories: -- `nexent/*` - Main public image repository -- `ccr.ccs.tencentyun.com/nexent-hub/*` - Tencent Cloud image repository (China region acceleration) +Repository selection depends on `--registry` and `--push`: +- `--registry general` builds or pushes `nexent/*`. +- `--registry mainland --push` pushes to `ccr.ccs.tencentyun.com/nexent-hub/*` for mainland China acceleration. +- `--registry mainland` without `--push` still builds local `nexent/*` tags while using mainland build mirrors. All images include: - `nexent/nexent` - Main application backend service @@ -237,7 +244,7 @@ bash deploy.sh docker --image-source local-latest After building local `latest` images, package them with the offline builder: ```bash -bash deploy/offline/build_offline_package.sh \ +bash build.sh --package \ --target docker \ --version latest \ --platform amd64 \ @@ -256,3 +263,5 @@ bash deploy.sh --load-images docker \ --components infrastructure,application,data-process,supabase \ --image-source local-latest ``` + +To push the packaged images to an internal registry during offline deployment, replace `--load-images` with `--push-images --image-registry-prefix registry.example.com/nexent`. If the prefix is omitted, the wrapper prompts for it before `push-images.sh` asks for the registry username and password. The deployment config will use the same registry prefix for Docker Compose image references. diff --git a/doc/docs/en/quick-start/installation.md b/doc/docs/en/quick-start/installation.md index f62721b77..5cc574e71 100644 --- a/doc/docs/en/quick-start/installation.md +++ b/doc/docs/en/quick-start/installation.md @@ -53,6 +53,9 @@ After running the command, the script opens Bash TUI menus for deployment option You can also pass options directly: ```bash +# Use saved deploy.options or built-in defaults without opening the TUI +bash deploy.sh docker --defaults + # Default component set, development port policy, standard image source bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development --image-source general @@ -66,7 +69,7 @@ bash deploy.sh docker --image-source mainland bash deploy.sh docker --image-source local-latest ``` -After a successful deployment, non-sensitive choices are saved to `deploy/docker/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. +After a successful deployment, non-sensitive choices are saved to `deploy/docker/deploy.options`. `--defaults` reuses that file when it exists, otherwise it uses built-in defaults. The next interactive deployment can reuse the local config or run a full reconfiguration. #### ⚠️ Important Notes @@ -187,15 +190,23 @@ bash deploy/offline/build_offline_package.sh \ --output-dir offline-package ``` -The package directory contains `images/*.tar`, `load-images.sh`, `deploy.sh`, `uninstall.sh`, `manifest.yaml`, `checksums.txt`, `deploy/env/.env.example`, and `deploy/sql`. It does not include local `deploy/env/.env` or `deploy.options`. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. +The package directory contains `images/*.tar`, `load-images.sh`, `push-images.sh`, `deploy.sh`, `uninstall.sh`, `manifest.yaml`, `checksums.txt`, `deploy/env/.env.example`, `deploy/env/monitoring.env.example`, and `deploy/sql`. It does not include local `deploy/env/.env`, `deploy/env/monitoring.env`, or `deploy.options`. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. -On the target host, keep the deployment options consistent with the package manifest: +On the target host, the package root `deploy.sh` uses saved `deploy.options` when present, otherwise built-in defaults, and does not open the TUI by default. Add `--config` to open the interactive configuration UI. If the package was built with a custom version, component set, port policy, or image source, pass the same options during deployment or use `--config` to select them interactively: ```bash cd offline-package bash deploy.sh --load-images docker ``` +To push packaged images to an internal registry and deploy with that prefix: + +```bash +bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent docker +``` + +When `--push-images` is used without a prefix, `deploy.sh` prompts for the image registry prefix first. `push-images.sh` then prompts for the registry username and password before pushing. + ## 🔌 Port Mapping | Service | Internal Port | External Port | Description | @@ -217,7 +228,7 @@ For complete port mapping details, see our [Dev Container Guide](../deployment/d ### Monitoring Configuration -Select the `monitoring` component in the deployment script UI to enable OpenTelemetry monitoring. The script synchronizes `ENABLE_TELEMETRY`, `MONITORING_PROVIDER`, and `MONITORING_DASHBOARD_URL` in `deploy/env/.env`, then starts the matching observability services from `deploy/docker/compose/docker-compose-monitoring.yml`. +Select the `monitoring` component in the deployment script UI to enable OpenTelemetry monitoring. The script synchronizes `ENABLE_TELEMETRY`, `MONITORING_PROVIDER`, `MONITORING_DASHBOARD_URL`, OTLP endpoints, and provider defaults in `deploy/env/monitoring.env`, then starts the matching observability services from `deploy/docker/compose/docker-compose-monitoring.yml`. The frontend monitoring entry is visible in speed mode when a dashboard URL is configured; in standard mode, only the super administrator can see it. ```bash cd nexent @@ -240,7 +251,7 @@ Supported providers: To change ports, image versions, or local Langfuse bootstrap credentials, copy and edit the monitoring environment file first: ```bash -cp deploy/docker/assets/monitoring/monitoring.env.example deploy/docker/assets/monitoring/monitoring.env +cp deploy/env/monitoring.env.example deploy/env/monitoring.env ``` Common variables: @@ -253,7 +264,7 @@ Common variables: | `LANGFUSE_INIT_USER_EMAIL` / `LANGFUSE_INIT_USER_PASSWORD` | Local Langfuse bootstrap admin | | `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` | Local Grafana admin | -Before choosing the `langsmith` provider, configure `LANGSMITH_API_KEY` in `deploy/docker/assets/monitoring/monitoring.env`. If you only need to connect to an existing external Collector, adjust the OTLP target in `deploy/env/.env`: +Before choosing the `langsmith` provider, configure `LANGSMITH_API_KEY` in `deploy/env/monitoring.env`. If you only need to connect to an existing external Collector, adjust the OTLP target in `deploy/env/monitoring.env`: ```bash ENABLE_TELEMETRY=true @@ -302,6 +313,11 @@ WECHAT_OAUTH_APP_SECRET= # TLS verification when contacting OAuth providers OAUTH_SSL_VERIFY=true OAUTH_CA_BUNDLE= + +# disabled: hide OAuth login entries and disable automatic redirects +# button: show configured OAuth providers as login buttons +# force: redirect automatically when exactly one provider is configured +OAUTH_LOGIN_MODE=button ``` Provider enablement rules: @@ -315,6 +331,8 @@ Provider enablement rules: For local Docker, a GitHub callback example is `http://localhost:3000/api/user/oauth/callback?provider=github`. In production, use a public HTTPS domain such as `https://nexent.example.com/api/user/oauth/callback?provider=github` and register the exact same URL in the OAuth provider console. +`OAUTH_LOGIN_MODE` supports `disabled`, `button`, and `force`, and defaults to `button`. In `force` mode, unauthenticated users are redirected when exactly one provider is enabled. OAuth is disabled when no provider is available, while multiple providers fall back to login buttons. CAS `force` mode takes precedence when both are configured. + ### CAS Login Configuration CAS SSO does not require the `supabase` component. Set `CAS_CALLBACK_BASE_URL` to the browser-accessible Nexent Web URL without a trailing `/`. `CAS_SERVER_URL` is the CAS Server root URL and should also not include a trailing `/`. diff --git a/doc/docs/en/quick-start/kubernetes-installation.md b/doc/docs/en/quick-start/kubernetes-installation.md index 61218b3de..af8db06b4 100644 --- a/doc/docs/en/quick-start/kubernetes-installation.md +++ b/doc/docs/en/quick-start/kubernetes-installation.md @@ -59,6 +59,8 @@ After running the command, the script opens Bash TUI menus for configuration. Us Kubernetes uses the same `deploy/env/.env` file as Docker. Existing `deploy/env/.env` is kept as-is. If it does not exist, the deploy scripts first reuse `docker/.env`, then fall back to `deploy/env/.env.example`. +Use `bash deploy.sh k8s --defaults` to skip the TUI and deploy with saved `deploy.options` or built-in defaults. + After a successful deployment, non-sensitive choices are saved to `deploy/k8s/deploy.options`. The next interactive deployment can reuse the local config or run a full reconfiguration. ### ⚠️ Important Notes @@ -200,14 +202,22 @@ bash deploy/offline/build_offline_package.sh \ --output-dir offline-package ``` -The package includes image tar files, `load-images.sh`, root deploy/uninstall entrypoints, Kubernetes Helm assets, SQL files, `manifest.yaml`, and `checksums.txt`. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. On a single-node Docker-backed cluster, you can load and deploy directly: +The package includes image tar files, `load-images.sh`, `push-images.sh`, root deploy/uninstall entrypoints, Kubernetes Helm assets, SQL files, `deploy/env/.env.example`, `deploy/env/monitoring.env.example`, `manifest.yaml`, and `checksums.txt`. It does not include local `deploy/env/.env`, `deploy/env/monitoring.env`, or generated Helm values. With `--compress true`, a `nexent-offline---.zip` archive is created next to the output directory. + +On the target host, the package root `deploy.sh` uses saved `deploy.options` when present, otherwise built-in defaults, and does not open the TUI by default. Add `--config` to open the interactive configuration UI. If the package was built with a custom version, component set, port policy, or image source, pass the same options during deployment or use `--config` to select them interactively. On a single-node Docker-backed cluster, you can load and deploy directly: ```bash cd offline-package bash deploy.sh --load-images k8s ``` -For multi-node clusters, load the images on every node that may run Nexent Pods, or push the loaded images to an internal registry and deploy with matching image settings. +For multi-node clusters, load the images on every node that may run Nexent Pods, or push the packaged images to an internal registry and deploy with matching image settings: + +```bash +bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent k8s +``` + +When `--push-images` is used without a prefix, `deploy.sh` prompts for the image registry prefix first. `push-images.sh` then prompts for the registry username and password before pushing. ## 🔧 Deployment Commands @@ -250,7 +260,7 @@ bash uninstall.sh k8s delete-all --keep-local-data ### Monitoring Configuration -Kubernetes deployments enable monitoring through the `monitoring` component in the deployment script UI. The deployment script renders runtime Helm values for `global.monitoring.enabled`, `global.monitoring.provider`, and `global.monitoring.dashboardUrl`, and enables the `nexent-monitoring` subchart. +Kubernetes deployments enable monitoring through the `monitoring` component in the deployment script UI. The deployment script synchronizes provider settings in `deploy/env/monitoring.env`, renders runtime Helm values for `global.monitoring.*` and `nexent-monitoring.*`, and enables the `nexent-monitoring` subchart. ```bash cd nexent @@ -270,21 +280,29 @@ Supported providers: | `grafana` | Local Grafana + Tempo | `http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1` | | `zipkin` | Local Zipkin | `http://localhost:30011` | -Before choosing the `langsmith` provider, configure `global.monitoring.langsmithApiKey` and `global.monitoring.langsmithProject` in `deploy/deploy/k8s/helm/nexent/values.yaml`. To change local Grafana, Langfuse, or dashboard ports, adjust the values file first, then re-run the deployment script, choose to reconfigure, and manually select `monitoring`. +Before choosing the `langsmith` provider, configure `LANGSMITH_API_KEY` and optionally `LANGSMITH_PROJECT` in `deploy/env/monitoring.env`. To change local Grafana, Langfuse, or dashboard ports, adjust the related `K8S_*_NODE_PORT` or service variables in `deploy/env/monitoring.env`, then re-run the deployment script, choose to reconfigure, and manually select `monitoring`. -Common Helm values: +Common generated Helm values: | Value | Description | |-------|-------------| | `global.monitoring.enabled` | Enables OpenTelemetry export in the Nexent backend | | `global.monitoring.provider` | Backend provider label: `otlp`, `phoenix`, `langfuse`, `langsmith`, `grafana`, `zipkin` | | `global.monitoring.otlpEndpoint` | Backend OTLP HTTP endpoint, default `http://nexent-otel-collector:4318` | -| `global.monitoring.dashboardUrl` | Frontend monitoring entry URL; leave empty to hide the entry | +| `global.monitoring.dashboardUrl` | Frontend monitoring entry URL; leave empty to hide the entry. Visible in speed mode; in standard mode only the super administrator can see it | | `global.monitoring.traceContentMode` | Trace content capture mode: `summary`, `metrics`, or `full` | | `nexent-monitoring..service.nodePort` | NodePort override for provider dashboards | | `nexent-monitoring.langfuse.init.*` | Local Langfuse bootstrap organization, project, and admin account | | `nexent-monitoring.grafana.adminUser` / `adminPassword` | Local Grafana admin credentials | +Common `deploy/env/monitoring.env` variables: + +| Variable | Description | +|----------|-------------| +| `LANGSMITH_API_KEY` / `LANGSMITH_PROJECT` | LangSmith forwarding configuration | +| `K8S_PHOENIX_NODE_PORT` / `K8S_LANGFUSE_NODE_PORT` / `K8S_GRAFANA_NODE_PORT` / `K8S_ZIPKIN_NODE_PORT` | NodePort overrides for local dashboards | +| `K8S_LANGFUSE_NEXTAUTH_URL` | Browser-accessible Langfuse URL used by the K8s Langfuse stack | + Check monitoring status: ```bash @@ -313,7 +331,8 @@ helm upgrade --install nexent nexent \ --set nexent-supabase-db.enabled=true \ --set nexent-common.config.oauth.callbackBaseUrl=https://nexent.example.com \ --set nexent-common.config.oauth.githubClientId=your_github_client_id \ - --set nexent-common.config.oauth.githubClientSecret=your_github_client_secret + --set nexent-common.config.oauth.githubClientSecret=your_github_client_secret \ + --set nexent-common.config.oauth.loginMode=force ``` Configurable OAuth values: @@ -331,6 +350,9 @@ Configurable OAuth values: | `nexent-common.config.oauth.wechatClientSecret` | `WECHAT_OAUTH_APP_SECRET` | WeChat App Secret | | `nexent-common.config.oauth.sslVerify` | `OAUTH_SSL_VERIFY` | Whether to verify provider TLS certificates | | `nexent-common.config.oauth.caBundle` | `OAUTH_CA_BUNDLE` | Custom CA bundle path | +| `nexent-common.config.oauth.loginMode` | `OAUTH_LOGIN_MODE` | `disabled`, `button`, or `force` | + +`loginMode` defaults to `button`. In `force` mode, OAuth is disabled when no provider is available, while multiple providers fall back to login buttons. CAS `force` mode takes precedence over OAuth auto login. Provider callback URLs: diff --git a/doc/docs/en/sdk/monitoring.md b/doc/docs/en/sdk/monitoring.md index 2c90180c6..bb4dabd5d 100644 --- a/doc/docs/en/sdk/monitoring.md +++ b/doc/docs/en/sdk/monitoring.md @@ -15,20 +15,8 @@ NexentAgent ──► OpenTelemetry SDK ──► OTLP Collector ──► Arize ## Quick Start ```bash -cd deploy/docker -[ -f ../../.env ] || cp ../../.env.example ../../.env -cp assets/monitoring/monitoring.env.example assets/monitoring/monitoring.env - -vim ../../.env -ENABLE_TELEMETRY=true -MONITORING_PROVIDER=otlp -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http - -vim assets/monitoring/monitoring.env -MONITORING_PROVIDER=otlp - -./start-monitoring.sh --stack collector +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider otlp ``` ## AI Observability Platforms @@ -89,15 +77,16 @@ LangSmith supports online OTLP trace ingestion through the OpenTelemetry endpoin **Collector forwarding:** ```bash -cd deploy/docker -vim assets/monitoring/monitoring.env +cd deploy +[ -f env/monitoring.env ] || cp env/monitoring.env.example env/monitoring.env +vim env/monitoring.env MONITORING_PROVIDER=langsmith LANGSMITH_API_KEY=lsv2_xxx LANGSMITH_PROJECT=nexent LANGSMITH_OTLP_TRACES_ENDPOINT=https://api.smith.langchain.com/otel/v1/traces -./start-monitoring.sh --stack langsmith +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langsmith ``` Nexent backend configuration when it sends OTLP to the Collector: @@ -123,7 +112,7 @@ OTEL_EXPORTER_OTLP_PROTOCOL=http MONITORING_DASHBOARD_URL=http://localhost:9411 ``` -Set `MONITORING_DASHBOARD_URL` to the browser-accessible monitoring UI URL. The backend returns this value to the frontend top bar without deriving a provider-specific path. +Set `MONITORING_DASHBOARD_URL` in `deploy/env/monitoring.env` to the browser-accessible monitoring UI URL. The backend returns this value to the frontend top bar without deriving a provider-specific path. In speed mode, the top-bar entry is visible when the URL is configured; in standard mode, only the super administrator can see it. ```bash MONITORING_DASHBOARD_URL=http://localhost:6006 @@ -138,9 +127,9 @@ MONITORING_DASHBOARD_URL=http://localhost:9411 |----------|---------|-------------| | `ENABLE_TELEMETRY` | `false` | Enable/disable monitoring | | `MONITORING_PROVIDER` | `otlp` | Provider profile: `otlp`, `phoenix`, `langfuse`, `langsmith`, `grafana`, `zipkin` | -| `MONITORING_DASHBOARD_URL` | (empty) | Browser-accessible monitoring UI URL used by the frontend top bar | +| `MONITORING_DASHBOARD_URL` | (empty) | Browser-accessible monitoring UI URL used by the frontend top bar; visible in speed mode and to super administrators in standard mode | | `MONITORING_PROJECT_NAME` | `nexent` | Observability platform project name | -| `MONITORING_TRACE_CONTENT_MODE` | `summary` | Trace payload mode: `summary` records bounded previews plus metadata, `metrics` records only structure/size metadata, `full` keeps full payloads subject to `MONITORING_TRACE_MAX_CHARS` | +| `MONITORING_TRACE_CONTENT_MODE` | `full` | Trace payload mode: `summary` records bounded previews plus metadata, `metrics` records only structure/size metadata, `full` keeps full payloads subject to `MONITORING_TRACE_MAX_CHARS` | | `MONITORING_TRACE_MAX_CHARS` | `4000` | Maximum characters for each payload preview written to trace attributes | | `MONITORING_TRACE_MAX_ITEMS` | `20` | Maximum dict keys/list items included in payload previews | | `OTEL_SERVICE_NAME` | `nexent-backend` | Service identifier | @@ -310,7 +299,7 @@ All monitoring methods work without errors when disabled - decorators pass throu ### No data appearing -1. Check `ENABLE_TELEMETRY=true` in `.env` +1. Check `ENABLE_TELEMETRY=true` in `deploy/env/monitoring.env` 2. Verify OTLP endpoint is reachable 3. Check authentication headers are correct diff --git a/doc/docs/zh/deployment/docker-build.md b/doc/docs/zh/deployment/docker-build.md index a389aabd4..6f2539e17 100644 --- a/doc/docs/zh/deployment/docker-build.md +++ b/doc/docs/zh/deployment/docker-build.md @@ -8,10 +8,10 @@ ```bash # 类似部署脚本，进入交互式选择 -bash deploy/images/build.sh +bash build.sh # 按镜像构建指定版本 -bash deploy/images/build.sh \ +bash build.sh \ --images main,web,mcp,data-process,terminal \ --version v2.2.1 \ --registry general \ @@ -19,7 +19,7 @@ bash deploy/images/build.sh \ --push # 按同一镜像集合构建 latest 镜像 -bash deploy/images/build.sh \ +bash build.sh \ --images main,web,mcp,data-process \ --version latest \ --registry general \ @@ -27,12 +27,15 @@ bash deploy/images/build.sh \ --load # 需要时也可以只构建一个或多个指定镜像 -bash deploy/images/build.sh --web --docs --version v2.2.1 --dry-run +bash build.sh --web --docs --version v2.2.1 --dry-run + +# 跳过 Docker 构建缓存 +bash build.sh --web --version v2.2.1 --no-cache ``` -在终端无参数运行 `deploy/images/build.sh` 时，会依次选择镜像、镜像版本（`latest` 或根 `VERSION`）和镜像源。交互式默认选择 `main,web` 和 `latest`。也可以用 `--interactive` 强制进入同样的选择流程。 +根目录 `build.sh` 会把镜像构建转发到 `deploy/images/build.sh`。使用 `bash build.sh --package ...` 可以转发到离线包构建脚本。在终端无参数运行 `build.sh` 时，会依次选择镜像、镜像版本（`latest` 或根 `VERSION`）和镜像源。交互式默认选择 `main,web` 和 `latest`。也可以用 `--interactive` 强制进入同样的选择流程。 -`--platform` 仅支持命令行传入。不传时不会添加 `--platform` 参数，默认按本地架构构建。 +`--platform` 和 `--no-cache` 仅支持命令行传入。不传 `--platform` 时不会添加该参数，默认按本地架构构建。`mainland` 的 web 镜像构建也会自动使用 `--no-cache`，避免前端依赖缓存过期。 变体选项： - `--dependency-variant cpu|gpu` 控制数据处理依赖，默认 `cpu`。`gpu` 会构建带 GPU/CUDA 依赖的镜像，并使用 `-gpu` 镜像名后缀。 @@ -154,9 +157,10 @@ docker build --progress=plain -t nexent/nexent-ubuntu-terminal-conda -f deploy/i ## 🏷️ 标签策略 -每个镜像都会推送到两个仓库： -- `nexent/*` - 主要的公共镜像仓库 -- `ccr.ccs.tencentyun.com/nexent-hub/*` - 腾讯云镜像仓库（中国地区加速） +镜像仓库由 `--registry` 和 `--push` 决定： +- `--registry general` 构建或推送 `nexent/*`。 +- `--registry mainland --push` 推送到 `ccr.ccs.tencentyun.com/nexent-hub/*`，用于中国大陆加速。 +- `--registry mainland` 但不带 `--push` 时，仍构建本地 `nexent/*` tag，同时使用大陆构建镜像源。 所有镜像包括： - `nexent/nexent` - 主应用后端服务 @@ -219,7 +223,7 @@ bash deploy.sh docker --image-source local-latest 构建本地 `latest` 镜像后，可以使用离线打包脚本把镜像和部署资源打包： ```bash -bash deploy/offline/build_offline_package.sh \ +bash build.sh --package \ --target docker \ --version latest \ --platform amd64 \ @@ -238,3 +242,5 @@ bash deploy.sh --load-images docker \ --components infrastructure,application,data-process,supabase \ --image-source local-latest ``` + +如果离线部署时需要推送到内部镜像仓库，可将 `--load-images` 替换为 `--push-images --image-registry-prefix registry.example.com/nexent`。如果省略前缀，入口脚本会先询问镜像仓库前缀，随后 `push-images.sh` 询问仓库账号和密码。部署配置会使用同一个镜像仓库前缀生成 Docker Compose 镜像引用。 diff --git a/doc/docs/zh/deployment/kubernetes-multi-replica-design.md b/doc/docs/zh/deployment/kubernetes-multi-replica-design.md new file mode 100644 index 000000000..de49c1287 --- /dev/null +++ b/doc/docs/zh/deployment/kubernetes-multi-replica-design.md @@ -0,0 +1,1029 @@ +# Kubernetes 多副本改造设计 + +## 1. 背景与目标 + +当前 Kubernetes Helm 部署可以把大部分服务的 `replicaCount` 调大，但项目并不等同于无状态服务。应用层存在进程内运行状态、流式连接状态、后台调度状态和动态 MCP 注册状态；基础设施层也使用单实例 PVC。直接扩容会出现流式恢复失败、停止任务失效、MCP 工具列表不一致、后台任务重复执行、PVC 无法跨节点挂载等问题。 + +本设计文档的目标是定义“业务应用层多副本”的分阶段改造方案。第一阶段落地四个应用服务的手工多副本： + +- `nexent-web` +- `nexent-config` +- `nexent-runtime` +- `nexent-northbound` + +第一阶段明确不改 `nexent-mcp`、`nexent-data-process` 的多副本能力；这些组件继续按单副本运行，后续阶段再单独设计。 + +第一阶段不要求内置 Postgres、Redis、Elasticsearch、MinIO、Supabase DB 做集群高可用。这些组件仍保持单实例，或由生产环境替换为外部托管服务。 + +第一阶段的“扩缩容”仅指通过 Helm `replicaCount` 手工调整副本数，不启用 autoscaling/HPA。 + +## 2. 当前状态问题清单 + +### 2.1 Helm 与存储 + +当前默认值： + +- `deploy/k8s/helm/nexent/values.yaml` + - `global.sharedStorage.mode: local` + - `global.sharedStorage.accessModes: ReadWriteOnce` + - `workspace.localPath: /var/lib/nexent` + - `skills.localPath: /var/lib/nexent-data/skills` +- `deploy/k8s/helm/nexent/charts/nexent-common/templates/shared-storage.yaml` + - local 模式会创建 `hostPath` PV。 + +业务服务挂载共享卷： + +- `nexent-config`: `/mnt/nexent`, `/mnt/nexent-data/skills` +- `nexent-runtime`: `/mnt/nexent`, `/mnt/nexent-data/skills` +- `nexent-northbound`: `/mnt/nexent`, `/mnt/nexent-data/skills` +- `nexent-mcp`: `/mnt/nexent` +- `nexent-data-process`: `/mnt/nexent` + +问题： + +- `ReadWriteOnce` 通常不能跨节点被多个 Pod 同时读写。 +- `hostPath` 强绑定节点，Pod 调度到其他节点后看不到相同数据。 +- 技能目录和工作区被多个业务服务读写，不能按普通临时目录处理。 + +### 2.2 Runtime 流式对话状态 + +相关代码： + +- `backend/services/streaming_channel.py` +- `backend/agents/agent_run_manager.py` +- `backend/agents/preprocess_manager.py` +- `backend/services/agent_service.py` + +当前状态： + +- `StreamingChannelManager` 是进程内 singleton，`_channels` 保存 SSE 历史、订阅者和完成状态。 +- `AgentRunManager` 是进程内 singleton，保存 `agent_runs`、`stop_event`、conversation 级 `ContextManager`。 +- `PreprocessManager` 是进程内 singleton，保存预处理任务和取消句柄。 + +问题： + +- 用户发起 `/api/agent/run` 后，如果重连请求落到另一个 `nexent-runtime` Pod，另一个 Pod 找不到原来的 channel。 +- `/api/agent/stop/{conversation_id}` 落到非 owner Pod 时，找不到原来的 `stop_event`。 +- 正在运行的 agent 状态只存在本地内存，Pod 重启后无法恢复运行控制。 +- conversation 级 `ContextManager` 如果参与正确性，跨 Pod 会产生上下文不一致；如果只是优化，可以允许 miss 后重建。 + +### 2.3 MCP 动态工具状态 + +相关代码： + +- `backend/mcp_service.py` +- `backend/services/tool_configuration_service.py` +- `backend/apps/tool_config_app.py` + +当前状态： + +- `mcp_service.py` 中 `_openapi_mcp_services` 是进程内 dict。 +- `refresh_openapi_services_by_tenant` 会清理并重新 mount 当前进程中的 FastMCP 服务。 +- config 服务通过 `MCP_MANAGEMENT_API` 调用 `nexent-mcp:5015` 刷新 MCP 服务。 + +问题： + +- 多个 `nexent-mcp` Pod 后，Service 只会把刷新请求转发到其中一个 Pod。 +- 被刷新 Pod 的工具列表正确，其他 Pod 仍是旧状态。 +- FastMCP mount 状态无法仅靠 DB 查询自动保持一致。 + +### 2.4 Data Process 与后台调度 + +相关代码： + +- `backend/data_process_service.py` +- `backend/data_process/app.py` +- `backend/data_process/worker.py` +- `backend/services/auto_summary_scheduler.py` + +当前状态： + +- `data_process_service.py` 在一个 Pod 中启动 Redis 连接检查、Ray、Celery workers、Flower 和 FastAPI。 +- `service_processes` 保存本地启动的 worker、Flower、Ray 状态。 +- `auto_summary_scheduler` 在进程内启动线程，`_in_flight` 只做单进程去重。 + +问题： + +- 多个 data-process Pod 会各自启动 Ray 和 worker，需要重新设计队列消费和 Ray 拓扑。 +- auto-summary 在多 Pod 下会重复扫描并处理相同知识库。 +- Flower 和 Ray dashboard 不适合作为每个副本都暴露的应用端口。 + +### 2.5 Northbound 幂等与限流 + +相关代码： + +- `backend/services/northbound_service.py` + +当前状态： + +- `_IDEMPOTENCY_RUNNING` 是进程内 dict。 +- `_RATE_STATE` 是进程内 dict。 + +问题： + +- 同一 `Idempotency-Key` 的并发请求打到不同 Pod 时会同时执行。 +- 租户级每分钟限流在每个 Pod 单独计数，实际总额度会随副本数线性放大。 + +### 2.6 基础设施组件 + +当前 Helm chart 中以下组件是带 PVC 的单实例 Deployment： + +- `nexent-postgresql` +- `nexent-redis` +- `nexent-elasticsearch` +- `nexent-minio` +- `nexent-supabase-db` + +问题： + +- Elasticsearch 使用 `discovery.type=single-node`，不能通过 `replicaCount` 变成集群。 +- Postgres、Redis、MinIO、Supabase DB 没有 StatefulSet、主从、Sentinel、Operator 或分布式部署配置。 +- 这些组件第一阶段必须明确为单实例状态依赖，不参与横向扩容。 + +## 3. 目标架构 + +### 3.1 状态分层 + +改造后状态分为四类： + +| 状态类型 | 例子 | 第一阶段承载位置 | +| --- | --- | --- | +| 持久业务状态 | 用户、租户、智能体、消息、MCP 配置 | Postgres | +| 对象与文件状态 | 上传文件、预览缓存、技能 zip 或产物 | MinIO 或 RWX PVC | +| 短生命周期运行状态 | SSE 事件、运行任务 owner、取消信号、幂等键、限流计数 | Redis | +| 本地优化缓存 | HTTP client、模型实例、可重建 ContextManager | Pod 内存 | + +原则： + +- 影响正确性的运行状态必须外部化。 +- 本地内存只能保存可丢弃、可重建、不会影响跨 Pod 语义的缓存。 +- 第一阶段 `web`、`config`、`runtime`、`northbound` 不能依赖 sticky session 才正确；未来其它应用服务扩容时也遵循同一原则。 +- Sticky session 可以保留为临时兼容策略，但不是设计前提。 + +### 3.2 应用层多副本边界 + +第一阶段允许多副本： + +- `nexent-web` +- `nexent-config` +- `nexent-runtime` +- `nexent-northbound` + +第一阶段保持单副本： + +- `nexent-mcp` +- `nexent-data-process` +- `nexent-postgresql` +- `nexent-redis` +- `nexent-elasticsearch` +- `nexent-minio` +- `nexent-supabase-db` +- `nexent-openssh` + +`nexent-web` 和 `nexent-config` 纳入第一阶段，但只做无状态化确认、启动/迁移并发保护、代理超时和 Helm 手工多副本配置。`nexent-mcp` 的动态工具一致性、`nexent-data-process` 的 scheduler/worker/Ray 拆分均放到后续阶段。 + +## 4. 详细修改点 + +### 4.1 Helm values 与模板 + +#### 4.1.1 新增应用服务手工多副本字段 + +第一阶段不启用 autoscaling/HPA，也不引入 PDB、topology spread、affinity、nodeSelector、tolerations 等调度增强项。`web`、`config`、`runtime` 和 `northbound` chart 只保留最小手工多副本配置： + +- `replicaCount` +- `strategy` + +涉及 chart： + +- `deploy/k8s/helm/nexent/charts/nexent-web` +- `deploy/k8s/helm/nexent/charts/nexent-config` +- `deploy/k8s/helm/nexent/charts/nexent-runtime` +- `deploy/k8s/helm/nexent/charts/nexent-northbound` + +修改模板： + +- `templates/deployment.yaml` + +最小配置示例： + +```yaml +replicaCount: 2 + +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` + +验收要求： + +- 默认仍然 `replicaCount: 1`，避免破坏现有安装。 +- 生产示例 values 中仅 `nexent-web`、`nexent-config`、`nexent-runtime` 和 `nexent-northbound` 配置 `replicaCount: 2`。 +- `web`、`config`、`runtime` 和 `northbound` Deployment 渲染出上述 RollingUpdate 策略。 +- 第一阶段不渲染 `HorizontalPodAutoscaler`，也不新增 `autoscaling.*` values；自动扩缩容作为二期能力单独设计。 +- 第一阶段不新增 PDB、topology spread、affinity、nodeSelector、tolerations；这些调度增强项如有需要放到后续生产增强阶段。 + +#### 4.1.2 共享存储生产模式 + +确认并补充 `global.sharedStorage.mode` 的生产使用说明： + +- `local`: 当前默认，单节点/开发使用，保留 `hostPath`。 +- `dynamic`: 使用 StorageClass 动态创建 PVC。 +- `existing`: 使用用户提供的 PVC。 + +第一阶段要求： + +- `config`、`runtime` 和 `northbound` 多副本生产示例必须使用 `existing` 或 `dynamic`。 +- 如果 `config`、`runtime` 或 `northbound` 副本数大于 1 且共享 PVC access mode 不是 `ReadWriteMany`，安装文档必须给出明显 warning；Helm 模板阻断可作为后续增强。 +- `local + ReadWriteOnce` 文档标注为不支持跨节点多副本；如果只在单节点开发环境验证，可以保留当前默认值。 + +需要修改： + +- `deploy/k8s/helm/nexent/values.yaml` +- `deploy/k8s/helm/nexent/README.md` +- `doc/docs/zh/quick-start/kubernetes-installation.md` + +#### 4.1.3 内置状态组件防误扩 + +第一阶段不改造以下状态组件的多副本能力： + +- `nexent-postgresql` +- `nexent-redis` +- `nexent-elasticsearch` +- `nexent-minio` +- `nexent-supabase-db` + +要求： + +- values 和安装文档中标注这些组件不应通过 `replicaCount > 1` 扩容。 +- 第一阶段不新增 StatefulSet、Sentinel、Operator 或集群模板。 +- 生产 HA 推荐外接托管服务或后续阶段专门改造。 + +### 4.2 Runtime 分布式运行状态 + +#### 4.2.1 新增 Redis 运行状态服务 + +新增模块： + +- `backend/services/runtime_state_service.py` + +职责： + +- 管理 agent run owner。 +- 管理取消信号。 +- 管理 SSE event buffer。 +- 管理 channel completion 状态。 +- 提供统一 TTL 清理。 + +建议 Redis key： + +| Key | 类型 | 用途 | TTL | +| --- | --- | --- | --- | +| `runtime:run:{user_id}:{conversation_id}` | hash | run owner、status、message_id、started_at、updated_at | 运行中 24h；结束后 5min | +| `runtime:cancel:{user_id}:{conversation_id}` | string | stop 请求设置的取消信号 | 运行中 24h；结束后 5min | +| `runtime:stream:{user_id}:{conversation_id}` | stream 或 list | SSE 事件历史 | 运行中 24h；结束后 5min | +| `runtime:stream:done:{user_id}:{conversation_id}` | hash | 完成状态、错误信息、最后 event id | 结束后 5min | + +Redis Stream 方案： + +- `XADD runtime:stream:{key} * event ` +- `XRANGE` 用于恢复历史。 +- `XTRIM` 或 TTL 控制内存。 +- 每个 SSE chunk 保存原始 `data: ...\n\n` 或结构化 JSON。 + +List 方案也可行： + +- `RPUSH` 保存事件。 +- `LRANGE` 恢复事件。 +- 使用递增 index 作为 event id。 + +推荐 Redis Stream，因为天然有 event id，适合 `Last-Event-ID`。 + +#### 4.2.2 改造 StreamingChannelManager + +现状： + +- `backend/services/streaming_channel.py` 中 `_channels` 是进程内 dict。 + +改造： + +- 保留 `StreamingChannel` API 形状，内部 publish 同时写 Redis。 +- 本地 subscriber 仍可通过 asyncio event 低延迟接收当前 Pod 的事件。 +- 跨 Pod resume 不依赖本地 channel，而是从 Redis 读取历史事件。 +- 本地 channel 只作为实时 fan-out 优化，不作为事实来源。 + +新增接口： + +- `append_stream_event(user_id, conversation_id, chunk) -> event_id` +- `read_stream_events(user_id, conversation_id, after_event_id=None) -> list` +- `mark_stream_completed(user_id, conversation_id, status, error=None)` +- `get_stream_status(user_id, conversation_id)` + +兼容策略： + +- 现有 `resume=true` 继续支持。 +- 可新增可选 header `Last-Event-ID`，没有 header 时按当前 `resume_from_unit_index` 逻辑兜底。 +- 如果 Redis 缓冲过期，则退化为 DB 已持久化消息恢复，并返回已完成/过期状态。 + +#### 4.2.3 改造 AgentRunManager + +现状： + +- `agent_runs` 和 `stop_event` 在本地内存。 + +改造： + +- 注册 run 时写入 Redis： + - `owner_pod`: Pod 名，可从环境变量 `HOSTNAME` 获取。 + - `status`: `running` + - `message_id` + - `started_at` + - `updated_at` +- agent 主循环周期性检查 Redis cancel key。 +- stop 请求只需要设置 Redis cancel key，不要求打到 owner Pod。 +- owner Pod 检测到 cancel 后触发本地 `stop_event`。 + +代码修改： + +- `backend/agents/agent_run_manager.py` +- `backend/services/agent_service.py` + +关键点： + +- 本地 `agent_runs` 仍保留，用于 owner Pod 快速停止。 +- `stop_agent_run` 先尝试本地停止，再写 Redis cancel key。 +- agent run finally 中删除或标记 Redis run 状态。 +- Pod 异常退出时，run key 依赖 TTL 自动过期；消息状态由现有 DB 持久化兜底。 + +#### 4.2.4 改造 PreprocessManager + +现状： + +- 预处理任务只在本地保存 task id 和 asyncio task。 + +改造： + +- 对需要跨 Pod 停止的预处理任务写 Redis cancel key。 +- 预处理任务执行过程中定期检查 cancel key。 +- `stop_preprocess_tasks` 写 Redis cancel key，并尝试本地取消。 + +涉及代码： + +- `backend/agents/preprocess_manager.py` +- 使用 preprocess manager 的调用点。 + +验收： + +- 在 Pod A 发起包含预处理的 agent run。 +- `/agent/stop` 打到 Pod B。 +- Pod A 中预处理和 agent run 均停止。 + +#### 4.2.5 Conversation ContextManager 策略 + +现状： + +- `AgentRunManager._conversation_context_managers` 保存 conversation 级 `ContextManager`。 + +第一阶段策略： + +- 将它定义为本地优化缓存。 +- 如果当前 Pod 没有缓存，则从 DB conversation history 重建。 +- 不要求跨 Pod 共享 ContextManager 内部压缩缓存。 + +风险： + +- 多 Pod 下同一 conversation 的连续多轮可能落到不同 Pod，压缩缓存命中率下降。 +- 如果有代码依赖 ContextManager 保存未持久化信息，需要在改造前确认并移除该依赖。 + +### 4.3 后续阶段：MCP 多副本一致性 + +本节不属于第一阶段交付。第一阶段 `nexent-mcp` 保持单副本；下面内容仅作为后续阶段设计预留。 + +#### 4.3.1 新增 MCP 启动初始化 + +现状： + +- 动态 OpenAPI MCP 服务只在 refresh 请求到达当前 Pod 时加载。 + +改造： + +- `nexent-mcp` 启动时读取 DB 中 enabled OpenAPI MCP 服务并注册。 +- 启动初始化失败时应记录错误并继续启动 local MCP，避免整个服务不可用。 + +涉及代码： + +- `backend/mcp_service.py` +- `database/outer_api_tool_db.py` + +新增函数： + +- `load_all_openapi_services_on_startup()` +- 或复用 `refresh_openapi_services_by_tenant`，但需要支持所有 tenant。 + +如果数据库当前只提供按 tenant 查询，需要新增 DB 查询函数： + +- `query_all_available_openapi_services()` + +#### 4.3.2 Redis Pub/Sub 广播刷新 + +新增模块： + +- `backend/services/mcp_refresh_bus.py` + +Redis channel： + +- `mcp:refresh` + +消息结构： + +```json +{ + "event": "refresh_all", + "tenant_id": "xxx", + "service_name": null, + "source_pod": "nexent-config-xxx", + "timestamp": 1234567890 +} +``` + +事件类型： + +- `refresh_all`: 刷新某 tenant 的全部 OpenAPI MCP 服务。 +- `refresh_one`: 刷新某 tenant 的某个 service。 +- `delete_one`: 删除某 tenant 的某个 service。 + +改造流程： + +1. config 服务写 DB 成功。 +2. config 服务发布 Redis Pub/Sub 消息。 +3. 所有 mcp Pod 后台订阅。 +4. 每个 mcp Pod 调用本地 refresh 函数更新自己的 FastMCP mount。 + +涉及代码： + +- `backend/services/tool_configuration_service.py` +- `backend/apps/tool_config_app.py` +- `backend/mcp_service.py` + +降级： + +- 如果 Redis Pub/Sub 发布失败，保留当前直接调用 `MCP_MANAGEMENT_API` 的逻辑作为 best effort。 +- 提供管理接口用于手动触发所有 Pod 刷新，或让 mcp Pod 周期性自检 DB 版本。 + +#### 4.3.3 MCP 配置版本号 + +为避免 Pub/Sub 消息丢失导致长期不一致，建议增加版本检查。 + +方案： + +- 在 DB 或 Redis 保存 `mcp:version:{tenant_id}`。 +- 每次 OpenAPI MCP 配置变更后 `INCR`。 +- 每个 MCP Pod 维护本地 `loaded_version`。 +- 后台每 30-60 秒检查一次版本，发现版本变化就刷新。 + +后续阶段可选： + +- 如果要压缩工作量，可以先只做 Pub/Sub。 +- 生产可靠性建议同时做版本检查。 + +### 4.4 Northbound 幂等与限流 + +#### 4.4.1 幂等键外部化 + +现状： + +- `_IDEMPOTENCY_RUNNING` 是本地 dict。 + +改造： + +- 使用 Redis `SET key value NX EX ttl`。 +- key 格式：`northbound:idempotency:{tenant_id}:{idempotency_key}`。 +- value 保存 request id、pod name、created_at。 +- 请求结束后延迟释放，保持当前 3 秒窗口语义。 + +涉及代码： + +- `backend/services/northbound_service.py` + +异常策略： + +- Redis 不可用时，为了避免重复执行，建议返回 503 或 429，而不是回退到本地 dict。 +- 如果业务更偏可用性，可以通过配置 `NORTHBOUND_IDEMPOTENCY_FAIL_OPEN=true` 允许降级。 + +#### 4.4.2 限流外部化 + +现状： + +- `_RATE_STATE` 是本地 dict。 + +改造： + +- 使用 Redis 计数器： + - `INCR northbound:rate:{tenant_id}:{minute_bucket}` + - 第一次设置 `EXPIRE 120` +- 超过 `_RATE_LIMIT_PER_MINUTE` 返回 `LimitExceededError`。 + +新增配置： + +- `NORTHBOUND_RATE_LIMIT_PER_MINUTE` +- `NORTHBOUND_RATE_LIMIT_ENABLED` + +注意： + +- 后续如果需要更平滑限流，可改为 token bucket 或 sliding window。 +- 第一阶段保持当前 minute bucket 语义。 + +### 4.5 后续阶段：Auto Summary Scheduler 分布式锁 + +本节不属于第一阶段交付。第一阶段 `nexent-data-process` 保持单副本，不启用多个 scheduler 实例。 + +现状： + +- `_in_flight` 只在单进程内避免重复。 + +改造： + +- 每个知识库执行前抢 Redis 锁： + - `SET auto_summary:lock:{index_name} pod_name NX EX lock_ttl` +- 抢锁成功才执行。 +- 执行完成后校验 value 是当前 pod 再释放。 + +涉及代码： + +- `backend/services/auto_summary_scheduler.py` + +锁 TTL： + +- 默认 2 小时，或根据最大摘要任务时间配置。 +- 新增配置 `AUTO_SUMMARY_LOCK_TTL_SECONDS`。 + +调度模式： + +- 后续阶段可以允许多个 Pod 都启动 scheduler，但通过分布式锁保证单个 KB 不重复处理。 +- 更推荐在 Helm 中让 scheduler 拆成独立 deployment，再结合单副本、leader election 或分布式锁运行。 + +### 4.6 后续阶段：Data Process 拆分预留 + +第一阶段不做完整 data-process 横扩，但文档和代码应避免阻塞后续拆分。 + +二期目标形态： + +- `nexent-data-process-api`: 只提供 HTTP API。 +- `nexent-data-process-worker`: Celery worker，可按队列和资源扩容。 +- `nexent-ray-head`: Ray head，StatefulSet 或独立 Deployment。 +- `nexent-ray-worker`: Ray worker，支持手工副本数或 KubeRay；HPA 不进入第一阶段。 +- `nexent-flower`: 可选单副本监控。 +- `nexent-auto-summary-scheduler`: 单副本或 leader election。 + +第一阶段需要避免： + +- 在应用层多副本设计中承诺 `nexent-data-process.replicaCount > 1` 可用。 +- 把 Flower/Ray dashboard 多副本暴露成生产入口。 + +### 4.7 Web 与 Config 多副本 + +#### 4.7.1 Web 代理层 + +现状： + +- `frontend/server.js` 将 `/api/agent/run`、`/api/agent/stop`、`/api/conversation/`、`/api/share/`、`/api/memory/`、`/api/file/storage` 等代理到 runtime。 +- `/api/voice/` WebSocket upgrade 代理到 runtime。 +- 其他 `/api` 代理到 config。 +- `nexent-web` 不挂载共享 PVC，主要状态来自浏览器 cookie、后端服务和对象存储。 + +第一阶段改造： + +- 为 `nexent-web` Deployment 增加最小 RollingUpdate 策略。 +- 确认 Node proxy 对 SSE 不 buffer。 +- 增加代理超时配置，避免长时间 agent run 被 web 层中断。 +- Web Pod 重启或滚动更新导致 SSE 断开时，前端应能重新连接；恢复语义由 runtime Redis stream/DB 兜底。 +- Web 不保存关键服务端状态，不要求 sticky session。 + +建议配置： + +- `PROXY_TIMEOUT_MS` +- `PROXY_WS_TIMEOUT_MS` +- `SSE_PROXY_TIMEOUT_MS` + +Ingress： + +- 不要求 sticky session。 +- 需要为 SSE/WebSocket 配置合适 timeout： + - nginx: `proxy-read-timeout`, `proxy-send-timeout` + - 其他 ingress controller 按实际配置。 + +#### 4.7.2 Config 服务 + +现状： + +- `nexent-config` 挂载 `/mnt/nexent` 和 `/mnt/nexent-data/skills`。 +- Deployment 中 `NEXENT_SQL_STARTUP_MODE` 当前为 `migrate`，多副本启动时可能多个 Pod 同时执行数据库迁移。 +- config 服务写入 Postgres、MinIO 或共享 PVC，并通过 `MCP_MANAGEMENT_API` 通知 `nexent-mcp` 刷新工具。 +- 第一阶段 `nexent-mcp` 仍是单副本，因此 config 多副本不会引入 MCP 多 Pod 工具列表不一致问题。 + +第一阶段改造： + +- 为 `nexent-config` Deployment 增加最小 RollingUpdate 策略。 +- config Pod 不保存影响正确性的进程内状态；如发现本地缓存，只能作为可重建缓存。 +- 数据库迁移必须避免多 Pod 并发执行。推荐方案是把迁移拆到 Helm hook Job 或独立 migration Job，config Deployment 启动时不再执行迁移。 +- 如果短期不拆 migration Job，则需要在迁移入口加数据库级互斥锁，例如 Postgres advisory lock，保证同一时间只有一个 Pod 执行迁移，其它 Pod 等待或跳过。 +- config 多副本写共享文件时要求 `nexent-workspace` 和 `nexent-skills` 使用 RWX 或对象存储化路径；`local + ReadWriteOnce` 不支持跨节点生产多副本。 +- 保持现有 `MCP_MANAGEMENT_API` 调用方式，因为第一阶段 mcp 是单副本。MCP 多副本广播刷新放到后续阶段。 + +验收： + +- 两个 config Pod 同时启动时，数据库迁移不会并发冲突。 +- config API 的创建、更新、删除、查询操作在多 Pod 下读取一致。 +- OpenAPI MCP 配置变更仍能刷新单副本 `nexent-mcp`。 +- 滚动更新期间 config 普通 API 可用，不能因为某个 Pod 正在迁移导致整体不可用。 + +### 4.8 配置与环境变量 + +后端新增环境变量必须集中到 `backend/consts/const.py`。Web 代理层环境变量归属前端 Helm chart，不放入后端 `const.py`。 + +建议新增： + +- `RUNTIME_STATE_REDIS_URL` + - 默认复用 `REDIS_URL` +- `RUNTIME_STREAM_TTL_SECONDS` +- `RUNTIME_RUN_TTL_SECONDS` +- `RUNTIME_CANCEL_TTL_SECONDS` +- `RUNTIME_COMPLETED_TTL_SECONDS` +- `NORTHBOUND_IDEMPOTENCY_TTL_SECONDS` +- `NORTHBOUND_RATE_LIMIT_ENABLED` +- `NORTHBOUND_RATE_LIMIT_PER_MINUTE` + +Web chart 建议新增或确认： + +- `PROXY_TIMEOUT_MS` +- `PROXY_WS_TIMEOUT_MS` +- `SSE_PROXY_TIMEOUT_MS` + +Config 需要确认： + +- `NEXENT_SQL_STARTUP_MODE` + - 如果迁移拆成独立 Job，config Deployment 中应关闭启动迁移。 + - 如果继续在 Pod 启动时迁移，迁移逻辑必须加数据库级互斥锁。 + +Helm ConfigMap 需要同步： + +- `deploy/k8s/helm/nexent/charts/nexent-common/templates/configmap.yaml` +- `deploy/k8s/helm/nexent/charts/nexent-common/values.yaml` +- `deploy/k8s/helm/nexent/generated-*.yaml` 生成逻辑如有涉及也需更新。 + +### 4.9 性能影响与优化边界 + +第一阶段不是零成本扩容。`runtime` 和 `northbound` 会引入 Redis 读写，`web` 和 `config` 会增加代理层、数据库连接数和启动并发控制压力。 + +Web 主要影响： + +- 多个 Web Pod 会增加到 runtime/config Service 的并发代理连接数。 +- SSE/WebSocket 长连接会占用 Web Pod 连接和内存资源。 +- Web Pod 滚动更新会断开该 Pod 上的长连接，依赖前端重连和 runtime resume 保证体验。 + +Config 主要影响： + +- 多个 config Pod 会增加 Postgres、MinIO 和共享存储的并发访问。 +- 如果启动迁移使用数据库锁，Pod 启动时间可能变长。 +- 如果数据库连接池按 Pod 固定大小配置，总连接数会随副本数增长，需要校验 Postgres `max_connections`。 + +Runtime 主要影响： + +- SSE 每个 chunk 需要追加到 Redis stream/list，Redis QPS 会随 agent 输出 token/chunk 数增加。 +- resume 需要从 Redis 读取历史事件，历史越长读取越多。 +- stop/cancel 需要 owner Pod 周期性检查 Redis cancel key，检查间隔越短停止越快，但 Redis 读压力越高。 +- 本地 channel 仍保留为同 Pod 实时 fan-out 优化，正常不断线场景不应全部依赖 Redis 轮询。 + +Northbound 主要影响： + +- 每个带 `Idempotency-Key` 的请求至少增加一次 Redis `SET NX EX`。 +- 每个受限流控制的请求至少增加一次 Redis `INCR/EXPIRE`。 +- Redis 不可用时应优先 fail closed，避免重复执行和限流失效；这会牺牲部分可用性。 + +优化要求： + +- SSE buffer 必须设置 TTL 和最大长度，避免长对话撑爆 Redis 内存。 +- agent run 完成、失败或停止后，主动把 runtime Redis key 的 TTL 缩短到 `RUNTIME_COMPLETED_TTL_SECONDS`。 +- cancel 检查使用合理间隔，例如 0.5-2 秒，不做高频忙轮询。 +- Northbound Redis key 必须设置 TTL，避免幂等键和限流桶长期残留。 +- Config 数据库连接池要按副本数重新核算，避免副本数翻倍后压满 Postgres。 +- Web SSE/WebSocket timeout 要大于典型 agent run 时间，避免代理层提前断开。 +- 压测必须覆盖 Redis QPS、内存占用、P95/P99 延迟和滚动更新期间错误率。 + +## 5. 兼容性与迁移策略 + +### 5.1 默认行为 + +- 默认单副本行为保持不变。 +- 第一阶段 Helm 不引入 HPA/autoscaling，所有副本数通过 `replicaCount` 手工控制。 + +### 5.2 上线步骤 + +推荐顺序： + +1. 发布 web/config Helm 策略、config 迁移并发保护、runtime/northbound Redis 状态改造代码，但 `replicaCount` 仍保持 1。 +2. 单副本下验证 Web 代理、config CRUD、config migration、agent run、SSE resume、stop、preprocess cancel、northbound 幂等、northbound 限流。 +3. 确认生产环境 Redis 可用，且 config/runtime/northbound 使用的共享存储不是跨节点 `local + ReadWriteOnce`。 +4. 开启 `nexent-web` 多副本，设置 `replicaCount: 2` 和 RollingUpdate 策略，验证登录、页面访问、SSE/WebSocket 代理。 +5. 开启 `nexent-config` 多副本，设置 `replicaCount: 2` 和 RollingUpdate 策略，验证迁移互斥、配置写入、单副本 MCP refresh。 +6. 开启 `nexent-northbound` 多副本，设置 `replicaCount: 2` 和 RollingUpdate 策略。 +7. 开启 `nexent-runtime` 多副本，设置 `replicaCount: 2` 和 RollingUpdate 策略。 +8. 进行滚动更新、Pod 删除、断连重连和跨 Pod stop 故障演练。 + +### 5.3 回滚策略 + +- Helm 保留 `replicaCount: 1` 快速回滚方式。 +- Redis 中新增运行状态均设置 TTL，不需要数据库迁移清理。 +- Web 如果代理层出现长连接异常，可以先回滚为单副本，不影响后端状态。 +- Config 如果迁移锁或共享存储写入异常，可以先回滚为单副本，并暂停新的配置写入操作。 +- Runtime 如果 Redis 状态异常，可以停止多副本，保留 DB 中已持久化消息。 +- Northbound 如果 Redis 幂等或限流异常，可以先回滚为单副本，避免重复执行或限流放大。 + +## 6. 工作量评估 + +第一阶段包含 `nexent-web`、`nexent-config`、`nexent-runtime` 和 `nexent-northbound` 多副本，预估 20-31 人日。 + +| 模块 | 工作内容 | 预估 | +| --- | --- | --- | +| Helm 多副本模板 | web/config/runtime/northbound 手工 replicaCount、RollingUpdate、生产存储限制说明 | 2-3 人日 | +| Web 多副本 | 代理超时、SSE/WebSocket 不 buffer、滚动更新断连恢复验证 | 1-2 人日 | +| Config 多副本 | 启动迁移互斥、共享存储写入确认、MCP 单副本刷新兼容 | 3-5 人日 | +| Runtime 分布式状态 | Redis stream、run owner、cancel、resume、stop、TTL | 8-12 人日 | +| Northbound 分布式控制 | Redis 幂等、限流、异常降级策略 | 2-3 人日 | +| 测试与文档 | 单测、集成测试、k8s 验证、压测、升级文档 | 4-6 人日 | + +第二阶段额外工作量： + +| 模块 | 工作内容 | 预估 | +| --- | --- | --- | +| MCP 一致性 | 启动加载、Pub/Sub 广播、版本检查、管理接口兼容 | 4-6 人日 | +| Auto Summary Scheduler | scheduler 独立部署、分布式锁或 leader election | 2-4 人日 | +| Data Process 横向扩展 | API/worker/Ray/Flower/scheduler 拆分 | 15-30 人日 | +| 内置状态组件 HA | Postgres、Redis、ES、MinIO、Supabase DB 集群化 | 20-40 人日 | +| 生产观测与容量模型 | 指标、告警、压测模型、容量建议 | 5-10 人日 | + +## 7. 测试计划 + +### 7.1 单元测试 + +新增或修改测试： + +- `test/frontend` 或前端现有代理测试：覆盖 SSE/WebSocket proxy timeout 和不 buffer 行为。 +- `test/backend/apps/test_config_app.py` +- `test/backend/services/test_tool_configuration_service.py` +- `test/backend/services/test_runtime_state_service.py` +- `test/backend/services/test_streaming_channel_distributed.py` +- `test/backend/agents/test_agent_run_manager.py` +- `test/backend/services/test_northbound_service.py` + +重点场景： + +- Redis 写入失败、读取失败、TTL 过期。 +- Web 代理长连接不被提前关闭。 +- config 启动迁移并发时只有一个执行者。 +- config API 多 Pod 下读写 DB 后结果一致。 +- SSE 事件追加和按 event id 恢复。 +- cancel key 被设置后 agent loop 能退出。 +- 幂等 key 并发抢占只有一个成功。 +- 租户级限流在多 Pod 下仍按全局额度生效。 + +### 7.2 集成测试 + +本地或 CI 中用 fake Redis/miniredis 或真实 Redis： + +- Web 代理到 runtime/config 的 API 正常转发。 +- Web 代理 SSE 断开后可重连。 +- config 多实例同时启动不会并发执行迁移。 +- config 写入 OpenAPI MCP 配置后仍能刷新单副本 mcp。 +- agent run 正常完成。 +- agent run 中断后 resume。 +- stop 请求不在 owner 进程内也能停止。 +- northbound 同一 idempotency key 并发请求只执行一次。 +- northbound 限流请求打到不同 Pod 时仍按同一租户额度计数。 + +### 7.3 Kubernetes 验证 + +部署矩阵： + +| 服务 | 副本数 | +| --- | --- | +| nexent-web | 2 | +| nexent-config | 2 | +| nexent-runtime | 2-3 | +| nexent-northbound | 2 | +| nexent-mcp | 1 | +| nexent-data-process | 1 | +| 状态组件 | 1 或外部托管 | + +验证场景： + +1. 两个 web Pod 下登录、页面访问、普通 API、SSE 和 WebSocket 代理正常。 +2. 删除一个 web Pod，确认浏览器重连后 agent 流式对话可恢复或得到明确完成状态。 +3. 两个 config Pod 同时启动或滚动更新，确认数据库迁移不并发冲突。 +4. config API 创建、更新、删除智能体配置、模型配置、工具配置后，另一个 Pod 立即读取一致结果。 +5. config 写入 OpenAPI MCP 配置后，确认单副本 `nexent-mcp` 刷新成功。 +6. 发起长 agent SSE，删除处理该请求的 runtime Pod，确认前端能通过 DB/Redis 得到明确恢复结果。 +7. 发起长 agent SSE，断开浏览器连接，再重新连接到不同 runtime Pod，确认继续接收或正确提示已完成。 +8. 发起 agent run 后，将 stop 请求强制打到另一个 runtime Pod，确认运行停止。 +9. northbound 同一 idempotency key 并发请求打到不同 Pod，确认只有一个执行。 +10. northbound 同一租户请求打到不同 Pod，确认限流总额度不随副本数放大。 +11. 手工调大或调小 web/config/runtime/northbound `replicaCount` 后，Web 登录、智能体配置、知识库查询、文件预览仍可用。 + +### 7.4 压测与容量 + +基础压测指标： + +- agent run 并发数。 +- SSE event 写 Redis 的 QPS 和内存占用。 +- Redis stream/list TTL 清理效果。 +- web Pod 长连接数、代理延迟、断线重连成功率。 +- config Pod DB 连接数、迁移锁等待时间、配置写入延迟。 +- runtime Pod CPU/内存。 +- northbound 限流准确性。 +- northbound Redis 操作延迟和错误率。 + +建议验收门槛： + +- stop 请求跨 Pod 生效小于 2 秒。 +- SSE resume 在 Redis 状态未过期时成功率 100%。 +- web 滚动更新期间长连接可重连，普通页面/API 可用。 +- config 滚动更新期间普通 API 可用，迁移不会并发失败。 +- northbound 幂等键并发抢占成功率符合预期，重复执行次数为 0。 +- 滚动更新期间普通 API 错误率不超过预设阈值。 + +## 8. 风险与待确认项 + +### 8.1 Redis 成为关键依赖 + +runtime/northbound 多副本后，Redis 不再只是 Celery broker/cache，也承载运行控制状态。 + +需要确认： + +- 生产环境是否使用托管 Redis 或 Redis HA。 +- Redis 持久化和内存淘汰策略。 +- Redis 不可用时 runtime/northbound 的失败策略。 + +建议： + +- 第一阶段文档要求生产多副本环境必须提供可靠 Redis。 +- Redis 单实例只适合开发或非关键部署。 +- Redis 自身多副本不在第一阶段 Helm 内置改造范围内。生产推荐三种方式： + - 托管 Redis/云 Redis，应用侧仍使用单个稳定连接地址。 + - Redis Sentinel，一主多从加自动故障转移，应用侧需要确认 Redis client 支持 Sentinel 地址。 + - Redis Cluster，用于更大容量和分片场景，应用侧需要确认 key 设计、client 和部署网络均支持 cluster mode。 +- 如果继续使用 chart 内置单实例 Redis，则只能视为 runtime/northbound 多副本的功能验证环境，不能视为高可用生产形态。 + +### 8.2 技能目录共享语义 + +技能相关代码仍会读写 `SKILLS_PATH`。 + +需要确认： + +- 生产是否提供 RWX PVC。 +- 是否计划将技能文件完全迁移到 MinIO/DB。 + +第一阶段建议： + +- config/runtime/northbound 多副本要求 `nexent-skills` 使用 RWX。 +- 后续单独设计“技能文件对象存储化”。 + +### 8.3 Config 启动迁移并发 + +`nexent-config` 当前通过 `NEXENT_SQL_STARTUP_MODE=migrate` 在 Pod 启动时执行迁移。多副本后，如果两个 Pod 同时启动或滚动更新，可能同时执行 SQL migration。 + +第一阶段建议： + +- 优先把迁移拆为独立 Job，在 config Deployment 启动前完成。 +- 如果继续保留 Pod 启动迁移，必须使用 Postgres advisory lock 或等效数据库锁。 +- 迁移失败时 config Pod 不应进入 Ready，避免服务接入到不完整 schema。 +- 滚动更新时必须验证两个 config Pod 不会因为等待迁移锁而全部不可用。 + +### 8.4 Agent 运行恢复语义 + +Pod 被删除时，正在运行的 agent 无法真正从中间继续推理，只能： + +- 从 DB 返回已持久化部分。 +- 标记为 failed/stopped。 +- 由用户重新发起。 + +第一阶段目标是“连接恢复和停止控制跨 Pod 可用”，不是“Pod 崩溃后 agent 计算继续执行”。 + +### 8.5 Data Process 完整横扩 + +当前 data-process 结构把 API、worker、Ray、Flower、scheduler 放在同一进程树中。 + +第一阶段不承诺： + +- 多个 data-process Pod 同时启动 Ray 后能正确协作。 +- Flower/Ray dashboard 多副本访问一致。 +- Celery worker 和 Ray actor 池按资源自动弹性扩缩。 + +这些应进入第二阶段。 + +## 9. 分阶段实施计划 + +### Milestone 1: 基础设施与配置准备 + +内容： + +- 为 `nexent-web`、`nexent-config`、`nexent-runtime` 和 `nexent-northbound` 新增 Helm 最小多副本字段。 +- 为 web/config/runtime/northbound Deployment 增加 RollingUpdate 策略配置。 +- 新增四个服务的多副本文档和 values 示例。 +- 明确内置状态组件、mcp、data-process 第一阶段保持单副本。 +- 新增 `backend/services/runtime_state_service.py` 基础封装。 + +验收： + +- 单副本行为不变。 +- web/config/runtime/northbound 可渲染多副本 manifests。 +- 生产示例只包含这四个服务的 `replicaCount: 2` 和 RollingUpdate 策略。 + +预估：3-4 人日。 + +### Milestone 2: Web 与 Config 多副本 + +内容： + +- Web proxy SSE/WebSocket timeout 和不 buffer 确认。 +- Web 滚动更新断线重连验证。 +- Config 启动迁移拆 Job 或加数据库锁。 +- Config 多 Pod 下配置读写一致性验证。 +- Config 到单副本 MCP 的 refresh 兼容验证。 + +验收： + +- web 2 副本下页面、普通 API、SSE、WebSocket 正常。 +- config 2 副本下迁移不并发冲突，配置读写一致。 + +预估：4-7 人日。 + +### Milestone 3: Runtime 多副本 + +内容： + +- SSE event 写 Redis。 +- resume 从 Redis/DB 恢复。 +- agent run owner 和 cancel key 外部化。 +- stop 跨 Pod 生效。 +- preprocess cancel 外部化。 + +验收： + +- runtime 2-3 副本下，run/resume/stop 通过集成测试。 + +预估：8-12 人日。 + +### Milestone 4: Northbound 多副本 + +内容： + +- northbound Redis 幂等和限流。 +- Redis 不可用时的失败策略。 +- 多 Pod 并发幂等和全局限流测试。 + +验收： + +- northbound 2 副本下幂等和限流准确。 + +预估：2-3 人日。 + +### Milestone 5: Kubernetes 验证、压测与文档 + +内容： + +- web/config/runtime/northbound k8s 多副本验证。 +- 压测和容量记录。 +- 更新安装/升级文档。 + +验收： + +- web/config/runtime/northbound 多副本部署完成全链路 smoke test。 +- 滚动更新期间核心功能可用。 + +预估：5-8 人日。 + +## 10. 最终交付物 + +代码交付： + +- Web proxy timeout 和长连接兼容配置。 +- Config migration 并发保护或独立 migration Job。 +- Redis 运行状态服务。 +- Runtime SSE/resume/stop 分布式化。 +- Northbound 分布式幂等与限流。 +- Web/config/runtime/northbound Helm 手工多副本 values 示例和 RollingUpdate 策略配置。 + +文档交付： + +- Kubernetes 多副本设计文档。 +- Kubernetes 安装文档中的 web/config/runtime/northbound 多副本限制说明。 +- Kubernetes 升级文档中的 web/config/runtime/northbound 多副本升级步骤。 +- 运维 runbook：如何扩容、回滚、排查 Redis 状态。 + +测试交付： + +- 后端单元测试。 +- Redis 集成测试。 +- k8s 多副本 smoke test 清单。 +- 压测结果和容量建议。 diff --git a/doc/docs/zh/quick-start/installation.md b/doc/docs/zh/quick-start/installation.md index 2cb14c400..2416ed948 100644 --- a/doc/docs/zh/quick-start/installation.md +++ b/doc/docs/zh/quick-start/installation.md @@ -53,6 +53,9 @@ bash deploy.sh docker 您也可以通过参数跳过交互： ```bash +# 使用已保存的 deploy.options 或内置默认值，不进入 TUI +bash deploy.sh docker --defaults + # 默认组件组合，development 端口策略，标准镜像源 bash deploy.sh docker --components infrastructure,application,data-process,supabase --port-policy development --image-source general @@ -66,7 +69,7 @@ bash deploy.sh docker --image-source mainland bash deploy.sh docker --image-source local-latest ``` -部署成功后，非敏感部署选项会保存到 `deploy/docker/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 +部署成功后，非敏感部署选项会保存到 `deploy/docker/deploy.options`。`--defaults` 会优先复用该文件；文件不存在时使用内置默认值。下次交互部署时可选择复用本地配置或重新全量配置。 #### ⚠️ 重要提示 @@ -183,15 +186,23 @@ bash deploy/offline/build_offline_package.sh \ --output-dir offline-package ``` -包目录会包含 `images/*.tar`、`load-images.sh`、`deploy.sh`、`uninstall.sh`、`manifest.yaml`、`checksums.txt`、`deploy/env/.env.example` 和 `deploy/sql`，不会包含本地 `deploy/env/.env` 或 `deploy.options`。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。 +包目录会包含 `images/*.tar`、`load-images.sh`、`push-images.sh`、`deploy.sh`、`uninstall.sh`、`manifest.yaml`、`checksums.txt`、`deploy/env/.env.example`、`deploy/env/monitoring.env.example` 和 `deploy/sql`，不会包含本地 `deploy/env/.env`、`deploy/env/monitoring.env` 或 `deploy.options`。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。 -在目标机器上部署时，请保持部署参数与 `manifest.yaml` 中的版本、组件和镜像源一致： +在目标机器上部署时，包根目录的 `deploy.sh` 会优先复用已保存的 `deploy.options`，否则使用内置默认值，默认不进入 TUI。添加 `--config` 可进入交互式配置界面。如果离线包构建时使用了自定义版本、组件、端口策略或镜像源，请在部署时传入相同选项，或使用 `--config` 交互选择： ```bash cd offline-package bash deploy.sh --load-images docker ``` +如果需要先推送到内部镜像仓库并使用该前缀部署： + +```bash +bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent docker +``` + +启用 `--push-images` 且未传前缀时，`deploy.sh` 会先询问镜像仓库前缀；随后 `push-images.sh` 在推送前询问仓库账号和密码。 + ## 🔌 端口映射 | 服务 | 内部端口 | 外部端口 | 描述 | @@ -213,7 +224,7 @@ bash deploy.sh --load-images docker ### 监控配置 -部署时在脚本交互界面中选择 `monitoring` 组件即可启用 OpenTelemetry 监控。脚本会同步更新 `deploy/env/.env` 中的 `ENABLE_TELEMETRY`、`MONITORING_PROVIDER` 和 `MONITORING_DASHBOARD_URL`，并启动 `deploy/docker/compose/docker-compose-monitoring.yml` 中对应的观测组件。 +部署时在脚本交互界面中选择 `monitoring` 组件即可启用 OpenTelemetry 监控。脚本会在 `deploy/env/monitoring.env` 中同步更新 `ENABLE_TELEMETRY`、`MONITORING_PROVIDER`、`MONITORING_DASHBOARD_URL`、OTLP endpoint 和 provider 默认值，并启动 `deploy/docker/compose/docker-compose-monitoring.yml` 中对应的观测组件。前端监控入口在 speed 模式下配置 dashboard URL 后可见；标准模式下仅超级管理员可见。 ```bash cd nexent @@ -236,7 +247,7 @@ bash deploy.sh docker 如需调整端口、镜像版本或 Langfuse 初始账号，请先复制并编辑监控环境变量： ```bash -cp deploy/docker/assets/monitoring/monitoring.env.example deploy/docker/assets/monitoring/monitoring.env +cp deploy/env/monitoring.env.example deploy/env/monitoring.env ``` 常用变量： @@ -249,7 +260,7 @@ cp deploy/docker/assets/monitoring/monitoring.env.example deploy/docker/assets/m | `LANGFUSE_INIT_USER_EMAIL` / `LANGFUSE_INIT_USER_PASSWORD` | 本地 Langfuse 初始管理员账号 | | `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` | 本地 Grafana 管理员账号 | -选择 `langsmith` provider 前，请先在 `deploy/docker/assets/monitoring/monitoring.env` 中配置 `LANGSMITH_API_KEY`。如果只需要连接已有外部 Collector，也可以在 `deploy/env/.env` 中调整 OTLP 目标地址： +选择 `langsmith` provider 前，请先在 `deploy/env/monitoring.env` 中配置 `LANGSMITH_API_KEY`。如果只需要连接已有外部 Collector，也可以在 `deploy/env/monitoring.env` 中调整 OTLP 目标地址： ```bash ENABLE_TELEMETRY=true @@ -298,6 +309,11 @@ WECHAT_OAUTH_APP_SECRET= # 访问 OAuth provider 时的 TLS 校验 OAUTH_SSL_VERIFY=true OAUTH_CA_BUNDLE= + +# disabled: 隐藏 OAuth 登录入口并禁用自动跳转 +# button: 显示已配置的 OAuth 登录按钮 +# force: 恰好配置一个 Provider 时，未登录用户自动跳转 +OAUTH_LOGIN_MODE=button ``` Provider 启用规则： @@ -311,6 +327,8 @@ Provider 启用规则： 本地默认回调示例为 `http://localhost:3000/api/user/oauth/callback?provider=github`。生产环境应改为公网 HTTPS 域名，例如 `https://nexent.example.com/api/user/oauth/callback?provider=github`，并在 OAuth provider 控制台中登记相同地址。 +`OAUTH_LOGIN_MODE` 支持 `disabled`、`button`、`force`，默认为 `button`。`force` 模式下，恰好有一个 Provider 满足启用条件时，未登录访问 Nexent 会直接跳转到该 Provider；没有可用 Provider 时禁用 OAuth，多个 Provider 时回退到登录按钮。若同时配置了 CAS `force` 模式，CAS 优先。 + ### CAS 登录配置 CAS SSO 不依赖 `supabase`。启用 CAS 时，请将 `CAS_CALLBACK_BASE_URL` 设置为浏览器可访问的 Nexent Web 地址，且不要带结尾 `/`。`CAS_SERVER_URL` 是 CAS Server 根地址，也不要带结尾 `/`。 diff --git a/doc/docs/zh/quick-start/kubernetes-installation.md b/doc/docs/zh/quick-start/kubernetes-installation.md index 10e3180f7..c8f86a4b8 100644 --- a/doc/docs/zh/quick-start/kubernetes-installation.md +++ b/doc/docs/zh/quick-start/kubernetes-installation.md @@ -59,6 +59,8 @@ bash deploy.sh k8s Kubernetes 使用与 Docker 相同的 `deploy/env/.env`。已有 `deploy/env/.env` 会原样保留；如果不存在，部署脚本会优先复用 `docker/.env`，再回退到 `deploy/env/.env.example`。 +使用 `bash deploy.sh k8s --defaults` 可跳过 TUI，并复用已保存的 `deploy.options` 或内置默认值。 + 部署成功后，非敏感部署选项会保存到 `deploy/k8s/deploy.options`。下次交互部署时可选择复用本地配置或重新全量配置。 ### ⚠️ 重要提示 @@ -200,14 +202,22 @@ bash deploy/offline/build_offline_package.sh \ --output-dir offline-package ``` -包内包含镜像 tar、`load-images.sh`、根目录部署/卸载入口、Kubernetes Helm 资源、SQL 文件、`manifest.yaml` 和 `checksums.txt`。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。如果是单节点、Docker 作为容器运行时的集群，可以直接加载并部署： +包内包含镜像 tar、`load-images.sh`、`push-images.sh`、根目录部署/卸载入口、Kubernetes Helm 资源、SQL 文件、`deploy/env/.env.example`、`deploy/env/monitoring.env.example`、`manifest.yaml` 和 `checksums.txt`，不会包含本地 `deploy/env/.env`、`deploy/env/monitoring.env` 或生成的 Helm values。使用 `--compress true` 时，会在输出目录的父目录生成 `nexent-offline---.zip`。 + +在目标机器上部署时，包根目录的 `deploy.sh` 会优先复用已保存的 `deploy.options`，否则使用内置默认值，默认不进入 TUI。添加 `--config` 可进入交互式配置界面。如果离线包构建时使用了自定义版本、组件、端口策略或镜像源，请在部署时传入相同选项，或使用 `--config` 交互选择。如果是单节点、Docker 作为容器运行时的集群，可以直接加载并部署： ```bash cd offline-package bash deploy.sh --load-images k8s ``` -多节点集群需要在每个可能运行 Nexent Pod 的节点上加载镜像，或将镜像推送到集群可访问的内部镜像仓库，再使用匹配的镜像参数部署。 +多节点集群需要在每个可能运行 Nexent Pod 的节点上加载镜像，或将镜像推送到集群可访问的内部镜像仓库，再使用匹配的镜像参数部署： + +```bash +bash deploy.sh --push-images --image-registry-prefix registry.example.com/nexent k8s +``` + +启用 `--push-images` 且未传前缀时，`deploy.sh` 会先询问镜像仓库前缀；随后 `push-images.sh` 在推送前询问仓库账号和密码。 ## 🔧 部署命令 @@ -253,7 +263,7 @@ bash uninstall.sh k8s delete-all --keep-local-data ### 监控配置 -Kubernetes 部署通过脚本交互界面中的 `monitoring` 组件启用监控。部署脚本会生成运行时 Helm values，设置 `global.monitoring.enabled`、`global.monitoring.provider`、`global.monitoring.dashboardUrl`，并启用 `nexent-monitoring` 子 Chart。 +Kubernetes 部署通过脚本交互界面中的 `monitoring` 组件启用监控。部署脚本会在 `deploy/env/monitoring.env` 中同步 provider 配置，生成 `global.monitoring.*` 和 `nexent-monitoring.*` 运行时 Helm values，并启用 `nexent-monitoring` 子 Chart。 ```bash cd nexent @@ -273,21 +283,29 @@ bash deploy.sh k8s | `grafana` | 本地 Grafana + Tempo | `http://localhost:30002/d/nexent-llm-agent/nexent-agent-trace-monitoring?orgId=1` | | `zipkin` | 本地 Zipkin | `http://localhost:30011` | -选择 `langsmith` provider 前，请先在 `deploy/deploy/k8s/helm/nexent/values.yaml` 中配置 `global.monitoring.langsmithApiKey` 和 `global.monitoring.langsmithProject`。如需修改本地 Grafana、Langfuse 或各 Dashboard 的端口，也建议先在 values 文件中调整，再通过部署脚本重新配置并手动选择 `monitoring`。 +选择 `langsmith` provider 前，请先在 `deploy/env/monitoring.env` 中配置 `LANGSMITH_API_KEY`，必要时配置 `LANGSMITH_PROJECT`。如需修改本地 Grafana、Langfuse 或各 Dashboard 的端口，请调整 `deploy/env/monitoring.env` 中对应的 `K8S_*_NODE_PORT` 或服务变量，再通过部署脚本重新配置并手动选择 `monitoring`。 -常用 Helm values： +常用生成的 Helm values： | Values | 说明 | |--------|------| | `global.monitoring.enabled` | 是否让 Nexent 后端开启 OpenTelemetry 上报 | | `global.monitoring.provider` | 后端 provider 标识：`otlp`、`phoenix`、`langfuse`、`langsmith`、`grafana`、`zipkin` | | `global.monitoring.otlpEndpoint` | 后端 OTLP HTTP 上报地址，默认 `http://nexent-otel-collector:4318` | -| `global.monitoring.dashboardUrl` | 前端监控入口地址，留空则隐藏入口 | +| `global.monitoring.dashboardUrl` | 前端监控入口地址，留空则隐藏入口；speed 模式下可见，标准模式下仅超级管理员可见 | | `global.monitoring.traceContentMode` | Trace 内容采集模式：`summary`、`metrics`、`full` | | `nexent-monitoring..service.nodePort` | 调整各 Dashboard 的 NodePort | | `nexent-monitoring.langfuse.init.*` | 本地 Langfuse 初始组织、项目和管理员账号 | | `nexent-monitoring.grafana.adminUser` / `adminPassword` | 本地 Grafana 管理员账号 | +常用 `deploy/env/monitoring.env` 变量： + +| 变量 | 说明 | +|------|------| +| `LANGSMITH_API_KEY` / `LANGSMITH_PROJECT` | LangSmith 转发配置 | +| `K8S_PHOENIX_NODE_PORT` / `K8S_LANGFUSE_NODE_PORT` / `K8S_GRAFANA_NODE_PORT` / `K8S_ZIPKIN_NODE_PORT` | 本地 Dashboard 的 NodePort 覆盖值 | +| `K8S_LANGFUSE_NEXTAUTH_URL` | K8s Langfuse 栈使用的浏览器可访问地址 | + 查看监控组件状态： ```bash @@ -316,7 +334,8 @@ helm upgrade --install nexent nexent \ --set nexent-supabase-db.enabled=true \ --set nexent-common.config.oauth.callbackBaseUrl=https://nexent.example.com \ --set nexent-common.config.oauth.githubClientId=your_github_client_id \ - --set nexent-common.config.oauth.githubClientSecret=your_github_client_secret + --set nexent-common.config.oauth.githubClientSecret=your_github_client_secret \ + --set nexent-common.config.oauth.loginMode=force ``` 可配置的 OAuth values： @@ -334,6 +353,9 @@ helm upgrade --install nexent nexent \ | `nexent-common.config.oauth.wechatClientSecret` | `WECHAT_OAUTH_APP_SECRET` | WeChat App Secret | | `nexent-common.config.oauth.sslVerify` | `OAUTH_SSL_VERIFY` | 访问 OAuth provider 时是否校验证书 | | `nexent-common.config.oauth.caBundle` | `OAUTH_CA_BUNDLE` | 自定义 CA bundle 路径 | +| `nexent-common.config.oauth.loginMode` | `OAUTH_LOGIN_MODE` | `disabled`、`button` 或 `force` | + +`loginMode` 默认为 `button`。`force` 模式下，没有可用 Provider 时禁用 OAuth，同时启用多个 Provider 时回退到登录按钮；CAS `force` 模式优先于 OAuth 自动登录。 Provider 回调地址： diff --git a/doc/docs/zh/sdk/monitoring.md b/doc/docs/zh/sdk/monitoring.md index da2f9e365..f67cf2e55 100644 --- a/doc/docs/zh/sdk/monitoring.md +++ b/doc/docs/zh/sdk/monitoring.md @@ -15,36 +15,24 @@ NexentAgent ──► OpenTelemetry SDK ──► OTLP Collector ──► Arize ## 快速启动 ```bash -cd deploy/docker -[ -f .env ] || cp .env.example .env -cp monitoring/monitoring.env.example monitoring/monitoring.env - -vim .env -ENABLE_TELEMETRY=true -MONITORING_PROVIDER=otlp -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http - -vim monitoring/monitoring.env -MONITORING_PROVIDER=otlp - -./start-monitoring.sh --stack collector +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider otlp ``` ## 本地化部署形态 -`docker/start-monitoring.sh` 支持多种形态，均以 OpenTelemetry Collector 作为统一入口。业务服务只需要把 OTLP 发到 Collector，不需要感知后端平台差异。 +Docker 部署脚本通过 `--monitoring-provider` 支持多种形态，均以 OpenTelemetry Collector 作为统一入口。业务服务只需要把 OTLP 发到 Collector，不需要感知后端平台差异。 | 形态 | 命令 | 包含服务 | 适用场景 | |------|------|----------|----------| -| `collector` | `./start-monitoring.sh --stack collector` | OpenTelemetry Collector | 只验证埋点、或转发到外部云端平台 | -| `phoenix` | `./start-monitoring.sh --stack phoenix` | Collector + Phoenix | 本地 trace 调试、OpenInference 属性查看、实验分析 | -| `langfuse` | `./start-monitoring.sh --stack langfuse` | Collector + Langfuse Web/Worker + Postgres + ClickHouse + MinIO + Redis | 本地完整 LLMOps 体验、会话/用户/反馈/成本分析 | -| `langsmith` | `./start-monitoring.sh --stack langsmith` | OpenTelemetry Collector | 转发 traces 到在线 LangSmith 平台 | -| `grafana` | `./start-monitoring.sh --stack grafana` | Collector + Grafana + Tempo | 本地 Tempo trace 查询 | -| `zipkin` | `./start-monitoring.sh --stack zipkin` | Collector + Zipkin | 本地 trace 查询 | +| `otlp` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider otlp` | OpenTelemetry Collector | 只验证埋点、或转发到外部云端平台 | +| `phoenix` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider phoenix` | Collector + Phoenix | 本地 trace 调试、OpenInference 属性查看、实验分析 | +| `langfuse` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langfuse` | Collector + Langfuse Web/Worker + Postgres + ClickHouse + MinIO + Redis | 本地完整 LLMOps 体验、会话/用户/反馈/成本分析 | +| `langsmith` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langsmith` | OpenTelemetry Collector | 转发 traces 到在线 LangSmith 平台 | +| `grafana` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider grafana` | Collector + Grafana + Tempo | 本地 Tempo trace 查询 | +| `zipkin` | `bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider zipkin` | Collector + Zipkin | 本地 trace 查询 | -也可以在 `deploy/docker/assets/monitoring/monitoring.env` 中设置默认形态： +也可以在 `deploy/env/monitoring.env` 中设置默认形态： ```bash MONITORING_PROVIDER=phoenix @@ -55,8 +43,8 @@ MONITORING_PROVIDER=phoenix Phoenix 本地部署使用 `arizephoenix/phoenix` 镜像，默认 UI 端口为 `6006`，gRPC OTLP 端口映射为 `4319`，数据持久化到 Docker volume `phoenix-data`。 ```bash -cd deploy/docker -./start-monitoring.sh --stack phoenix +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider phoenix ``` 访问地址： @@ -81,32 +69,33 @@ OTEL_EXPORTER_OTLP_METRICS_ENABLED=false Langfuse 本地部署使用 v3 架构：Web、Worker、Postgres、ClickHouse、MinIO、Redis。默认 UI 端口为 `3001`，初始化项目和 API Key 来自 `monitoring.env`。 ```bash -cd deploy/docker -./start-monitoring.sh --stack langfuse +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langfuse ``` 访问地址： - Langfuse UI：`http://localhost:3001` -- 默认管理员：`admin@nexent.local` / `nexent-langfuse-admin` +- 默认管理员：`admin@nexent.com` / `nexent@4321` - 默认项目 Key：`pk-lf-nexent-local` / `sk-lf-nexent-local` -启动脚本会在 `LANGFUSE_OTLP_AUTH_HEADER` 为空时自动生成 `Basic base64(public_key:secret_key)`，并让 Collector 将 trace 转发到 `http://langfuse-web:3000/api/public/otel`。本地默认密钥只适合开发验证，生产部署必须替换 `LANGFUSE_NEXTAUTH_SECRET`、`LANGFUSE_SALT`、`LANGFUSE_ENCRYPTION_KEY`、数据库密码和对象存储密钥。 +部署脚本会在 `LANGFUSE_OTLP_AUTH_HEADER` 为空时自动生成 `Basic base64(public_key:secret_key)`，并让 Collector 将 trace 转发到 `http://langfuse-web:3000/api/public/otel`。本地默认密钥只适合开发验证，生产部署必须替换 `LANGFUSE_NEXTAUTH_SECRET`、`LANGFUSE_SALT`、`LANGFUSE_ENCRYPTION_KEY`、数据库密码和对象存储密钥。 ### 在线 LangSmith LangSmith 支持通过在线 OTLP endpoint 摄取 traces。Nexent 可以先把 OTLP 发到本地 Collector，再由 Collector 转发到 LangSmith，业务服务无需直接保存 LangSmith API Key。 ```bash -cd deploy/docker -vim monitoring/monitoring.env +cd deploy +[ -f env/monitoring.env ] || cp env/monitoring.env.example env/monitoring.env +vim env/monitoring.env MONITORING_PROVIDER=langsmith LANGSMITH_API_KEY=lsv2_xxx LANGSMITH_PROJECT=nexent LANGSMITH_OTLP_TRACES_ENDPOINT=https://api.smith.langchain.com/otel/v1/traces -./start-monitoring.sh --stack langsmith +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langsmith ``` 后端在 Docker 网络内运行时： @@ -126,11 +115,11 @@ LangSmith 当前配置只转发 traces，OTLP metrics 会留在 Collector debug Grafana 本地部署使用 Grafana Tempo 存储 traces，并启用 Tempo `metrics-generator` 的 `local-blocks` processor 支持 Grafana trace breakdown 中的 TraceQL metrics 查询。Collector 接收 Nexent 后端的 OTLP traces/metrics，其中 traces 通过 OTLP gRPC 转发到 Tempo；OTLP metrics 只进入 Collector debug pipeline，不提供独立指标存储或指标 dashboard。 ```bash -cd deploy/docker -./start-monitoring.sh --stack grafana +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider grafana ``` -后端 `.env` 使用 `MONITORING_DASHBOARD_URL` 控制前端顶栏监控入口： +`deploy/env/monitoring.env` 中的 `MONITORING_DASHBOARD_URL` 控制前端顶栏监控入口。speed 模式下配置 URL 后即可显示；标准模式下只有超级管理员可见。 ```bash ENABLE_TELEMETRY=true @@ -142,7 +131,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 访问地址： - Grafana UI：`http://localhost:3002` -- 默认管理员：`admin` / `nexent-grafana-admin` +- 默认管理员：`admin` / `nexent@4321` - Tempo API：`http://localhost:3200` Grafana 会自动预置 Tempo datasource，并加载 `Nexent Agent Trace Monitoring` dashboard。Trace 查询入口在 Grafana Explore 中选择 `Tempo` datasource，示例 TraceQL 为 `{ resource.service.name = "nexent-backend" }`。 @@ -152,11 +141,11 @@ Grafana 会自动预置 Tempo datasource，并加载 `Nexent Agent Trace Monitor Zipkin 本地部署使用 `openzipkin/zipkin` 镜像。Collector 接收 Nexent 后端的 OTLP traces/metrics，其中 traces 转发到 Zipkin v2 spans endpoint；OTLP metrics 当前只进入 Collector debug pipeline。 ```bash -cd deploy/docker -./start-monitoring.sh --stack zipkin +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider zipkin ``` -后端 `.env`： +`deploy/env/monitoring.env`： ```bash ENABLE_TELEMETRY=true @@ -228,9 +217,9 @@ echo -n "$LANGFUSE_PUBLIC_KEY:$LANGFUSE_SECRET_KEY" | base64 |------|--------|------| | `ENABLE_TELEMETRY` | `false` | 启用/禁用监控 | | `MONITORING_PROVIDER` | `otlp` | 平台配置和本地部署形态：`otlp`、`phoenix`、`langfuse`、`langsmith`、`grafana`、`zipkin` | -| `MONITORING_DASHBOARD_URL` | （空） | 前端顶栏监控入口跳转 URL，需配置为浏览器可访问地址 | +| `MONITORING_DASHBOARD_URL` | （空） | 前端顶栏监控入口跳转 URL，需配置为浏览器可访问地址；speed 模式下可见，标准模式下仅超级管理员可见 | | `MONITORING_PROJECT_NAME` | `nexent` | 监控平台项目名 | -| `MONITORING_TRACE_CONTENT_MODE` | `summary` | Trace payload 记录模式：`summary` 写入有界预览和结构元数据，`metrics` 只写结构/大小元数据，`full` 在 `MONITORING_TRACE_MAX_CHARS` 限制内保留完整 payload | +| `MONITORING_TRACE_CONTENT_MODE` | `full` | Trace payload 记录模式：`summary` 写入有界预览和结构元数据，`metrics` 只写结构/大小元数据，`full` 在 `MONITORING_TRACE_MAX_CHARS` 限制内保留完整 payload | | `MONITORING_TRACE_MAX_CHARS` | `4000` | 每个 payload 预览最多写入的字符数 | | `MONITORING_TRACE_MAX_ITEMS` | `20` | dict/list 预览最多写入的 key 或 item 数 | | `OTEL_SERVICE_NAME` | `nexent-backend` | 服务标识 | @@ -259,7 +248,7 @@ echo -n "$LANGFUSE_PUBLIC_KEY:$LANGFUSE_SECRET_KEY" | base64 | `GRAFANA_VERSION` | `12.4` | 本地 Grafana 镜像版本 | | `GRAFANA_PORT` | `3002` | 本地 Grafana UI 端口 | | `GRAFANA_ADMIN_USER` | `admin` | 本地 Grafana 管理员用户名 | -| `GRAFANA_ADMIN_PASSWORD` | `nexent-grafana-admin` | 本地 Grafana 管理员密码 | +| `GRAFANA_ADMIN_PASSWORD` | `nexent@4321` | 本地 Grafana 管理员密码 | | `GRAFANA_DEFAULT_LANGUAGE` | `zh-Hans` | 本地 Grafana 默认界面语言 | | `TEMPO_VERSION` | `2.10.5` | 本地 Tempo 镜像版本，避免浮动 tag 带来的配置兼容性漂移 | | `TEMPO_PORT` | `3200` | 本地 Tempo HTTP API 端口 | @@ -456,7 +445,7 @@ pip install nexent[performance] # 包含 OTLP 支持 ### 数据未显示 -1. 检查 `.env` 中 `ENABLE_TELEMETRY=true` +1. 检查 `deploy/env/monitoring.env` 中 `ENABLE_TELEMETRY=true` 2. 验证 OTLP 端点可访问 3. 检查认证头配置正确 diff --git a/doc/docs/zh/sdk/opentelemetry-design.md b/doc/docs/zh/sdk/opentelemetry-design.md index 46093c633..9b4de05eb 100644 --- a/doc/docs/zh/sdk/opentelemetry-design.md +++ b/doc/docs/zh/sdk/opentelemetry-design.md @@ -242,7 +242,7 @@ flowchart LR |------|--------|------| | `ENABLE_TELEMETRY` | `false` | 监控总开关 | | `MONITORING_PROVIDER` | `otlp` | 监控 provider 和部署形态：`otlp`、`phoenix`、`langfuse`、`langsmith`、`grafana`、`zipkin` | -| `MONITORING_DASHBOARD_URL` | 空 | 前端顶栏监控入口跳转 URL，后端只读取并透传该值 | +| `MONITORING_DASHBOARD_URL` | 空 | 前端顶栏监控入口跳转 URL，后端只读取并透传该值；speed 模式下可见，标准模式下仅超级管理员可见 | | `MONITORING_PROJECT_NAME` | `nexent` | 平台项目名 | | `OTEL_SERVICE_NAME` | `nexent-backend` | OpenTelemetry service name | | `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318` | OTLP base endpoint | @@ -300,7 +300,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http ``` -前端顶栏监控入口不再根据 provider 在代码中映射 UI 端口和路径。后端读取 `MONITORING_DASHBOARD_URL` 并通过 `/monitoring/status` 返回给前端；该值为空时前端不显示监控入口。因此本地 Grafana 形态需要在后端 `.env` 中设置： +前端顶栏监控入口不再根据 provider 在代码中映射 UI 端口和路径。后端读取 `MONITORING_DASHBOARD_URL` 并通过 `/monitoring/status` 返回给前端；该值为空时前端不显示监控入口。speed 模式下配置 URL 后即可显示，标准模式下只有超级管理员可见。因此本地 Grafana 形态需要在 `deploy/env/monitoring.env` 中设置： ```bash MONITORING_PROVIDER=grafana @@ -362,11 +362,11 @@ Zipkin 当前本地形态只转发 traces；metrics 进入 Collector debug pipel ## 本地化部署设计 -本地化部署通过 `docker/start-monitoring.sh` 选择形态。所有形态都保留 OpenTelemetry Collector 作为入口，Nexent 后端统一上报到 `http://otel-collector:4318` 或宿主机的 `http://localhost:4318`，平台差异只体现在 Collector exporter 和本地服务组合上。 +本地化部署通过 Docker 部署脚本的 `--monitoring-provider` 选择形态。所有形态都保留 OpenTelemetry Collector 作为入口，Nexent 后端统一上报到 `http://otel-collector:4318` 或宿主机的 `http://localhost:4318`，平台差异只体现在 Collector exporter 和本地服务组合上。 | 形态 | Collector 配置 | 本地服务 | 数据去向 | 说明 | |------|----------------|----------|----------|------| -| `otlp` | `otel-collector-config.yml` | Collector | debug exporter | 最小形态，用于验证 span/metric 是否产生，或手动改配置转发到云端平台；`collector` 仅作为启动脚本兼容别名 | +| `otlp` | `otel-collector-config.yml` | Collector | debug exporter | 最小形态，用于验证 span/metric 是否产生，或手动改配置转发到云端平台 | | `phoenix` | `otel-collector-phoenix-config.yml` | Collector + Phoenix | `http://phoenix:6006/v1/traces` | Phoenix 容器同时提供 UI 和 OTLP HTTP/gRPC trace collector，适合本地 trace debug | | `langfuse` | `otel-collector-langfuse-config.yml` | Collector + Langfuse Web/Worker + Postgres + ClickHouse + MinIO + Redis | `http://langfuse-web:3000/api/public/otel/v1/traces` | Langfuse v3 依赖多组件，适合完整 LLMOps 能力验证 | | `langsmith` | `otel-collector-langsmith-config.yml` | Collector | `https://api.smith.langchain.com/otel/v1/traces` | 在线 LangSmith trace 分析；API Key 只配置在 Collector 环境 | @@ -376,13 +376,13 @@ Zipkin 当前本地形态只转发 traces；metrics 进入 Collector debug pipel 启动命令： ```bash -cd deploy/docker -./start-monitoring.sh --stack otlp -./start-monitoring.sh --stack phoenix -./start-monitoring.sh --stack langfuse -./start-monitoring.sh --stack langsmith -./start-monitoring.sh --stack grafana -./start-monitoring.sh --stack zipkin +cd deploy +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider otlp +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider phoenix +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langfuse +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider langsmith +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider grafana +bash deploy.sh docker --components infrastructure,monitoring --monitoring-provider zipkin ``` 部署脚本职责： diff --git a/docker/init.sql b/docker/init.sql new file mode 100644 index 000000000..e69de29bb diff --git a/frontend/app/[locale]/agent-space/components/AgentRepositoryCard.tsx b/frontend/app/[locale]/agent-space/components/AgentRepositoryCard.tsx index 4dd96c155..1ea42459a 100644 --- a/frontend/app/[locale]/agent-space/components/AgentRepositoryCard.tsx +++ b/frontend/app/[locale]/agent-space/components/AgentRepositoryCard.tsx @@ -4,12 +4,10 @@ import type { MenuProps } from "antd"; import { Button, Card, Dropdown } from "antd"; import { Bot, Copy, Download, Eye, MoreHorizontal, PackageX } from "lucide-react"; import { useTranslation } from "react-i18next"; -import { getAgentRepositoryTagLabel } from "@/lib/agentRepositoryLabels"; import type { AgentRepositoryListingItem } from "@/types/agentRepository"; interface AgentRepositoryCardProps { listing: AgentRepositoryListingItem; - categoryName?: string | null; showAdminMenu?: boolean; isTakingDown?: boolean; onCopyClick?: (listing: AgentRepositoryListingItem) => void; @@ -19,7 +17,6 @@ interface AgentRepositoryCardProps { export function AgentRepositoryCard({ listing, - categoryName, showAdminMenu = false, isTakingDown = false, onCopyClick, @@ -31,9 +28,6 @@ export function AgentRepositoryCard({ const title = listing.display_name?.trim() || listing.name?.trim() || t("agentRepository.card.untitled"); const author = listing.author?.trim(); - const category = - categoryName?.trim() || t("agentRepository.review.unknownCategory"); - const subtitle = author ? `${author} · ${category}` : category; const tags = listing.tags?.filter((tag) => tag.trim()) ?? []; const toolCount = listing.tool_count ?? 0; const versionText = listing.version_label; @@ -79,9 +73,11 @@ export function AgentRepositoryCard({

{title}

-

- {subtitle} -

+ {author ? ( +

+ {author} +

+ ) : null} {showMenu ? ( @@ -109,7 +105,7 @@ export function AgentRepositoryCard({ key={tag} className="rounded-md bg-slate-100 px-2 py-0.5 text-xs font-medium text-slate-700 dark:bg-slate-800 dark:text-slate-200" > - {getAgentRepositoryTagLabel(tag, t)} + {tag} ))} {toolCount > 0 ? ( diff --git a/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx b/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx index e840ad052..89dffcbc6 100644 --- a/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx +++ b/frontend/app/[locale]/agent-space/components/AgentRepositoryCopyDialog.tsx @@ -167,7 +167,7 @@ export function AgentRepositoryCopyDialog({ title={t("agentRepository.copy.title", { name: listingTitle })} centered width={520} - destroyOnClose + destroyOnHidden footer={
diff --git a/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx b/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx index 98bc649de..3a76312a7 100644 --- a/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx +++ b/frontend/app/[locale]/agent-space/components/AgentRepositoryDetailModal.tsx @@ -8,6 +8,8 @@ import { Clock, Cpu, Download, + Tag as TagIcon, + User, Wrench, XCircle, } from "lucide-react"; @@ -133,25 +135,40 @@ function AgentRepositoryDetailMeta({ const { t } = useTranslation("common"); return ( -
- {detail.model_name ? ( +
+
+ {detail.model_name ? ( + + + {detail.model_name} + + ) : null} + {detail.version_label ? ( + + + {detail.version_label} + + ) : null} - - {detail.model_name} - - ) : null} - {detail.version_label ? {detail.version_label} : null} - - - {t("agentRepository.detail.downloads", { - count: downloads.toLocaleString(), - })} - - {createdAtText ? ( - - - {createdAtText} + + {t("agentRepository.detail.downloads", { + count: downloads.toLocaleString(), + })} + {createdAtText ? ( + + + {createdAtText} + + ) : null} +
+ {detail.author?.trim() ? ( +
+ + + {t("agentRepository.detail.author", { author: detail.author })} + +
) : null}
); @@ -299,7 +316,7 @@ export function AgentRepositoryDetailModal({ footer={null} width={720} centered - destroyOnClose + destroyOnHidden title={null} className="agent-repository-detail-modal" styles={{ body: { padding: 0 } }} diff --git a/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx b/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx index b4964e820..00d3b3f4e 100644 --- a/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx +++ b/frontend/app/[locale]/agent-space/components/MineAgentsView.tsx @@ -2,11 +2,13 @@ import { useState } from "react"; import { useParams, useRouter } from "next/navigation"; -import { useQueryClient } from "@tanstack/react-query"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; import { App, Button, Empty, Input, Spin } from "antd"; import { ChevronLeft, ChevronRight, Plus, Search, Upload } from "lucide-react"; import { useTranslation } from "react-i18next"; import AgentImportWizard from "@/components/agent/AgentImportWizard"; +import { useConfirmModal } from "@/hooks/useConfirmModal"; +import { deleteAgent } from "@/services/agentConfigService"; import { AGENTS_LIST_QUERY_KEY, invalidateAgentRepositoryCaches, @@ -81,6 +83,7 @@ export function MineAgentsView({ }: MineAgentsViewProps) { const { t } = useTranslation("common"); const { message } = App.useApp(); + const { confirm } = useConfirmModal(); const router = useRouter(); const queryClient = useQueryClient(); const params = useParams<{ locale: string }>(); @@ -103,11 +106,14 @@ export function MineAgentsView({ const createListingMutation = useCreateAgentRepositoryListing(); const updateStatusMutation = useUpdateAgentRepositoryStatus(); + const deleteAgentMutation = useMutation({ + mutationFn: (agentId: number) => deleteAgent(agentId), + }); const normalizedQuery = searchQuery.trim().toLowerCase(); const handleCreateAgent = () => { - router.push(`/${locale}/agents?create=true`); + router.push(`/${locale}/agents?create=true&from=agent-space&tab=mine`); }; const handleImportAgent = async () => { @@ -133,7 +139,46 @@ export function MineAgentsView({ if (permission === "READ_ONLY") { return; } - router.push(`/${locale}/agents?agent_id=${agentId}`); + router.push( + `/${locale}/agents?agent_id=${agentId}&from=agent-space&tab=mine` + ); + }; + + const handleDeleteAgent = (agent: MyEditableAgentItem) => { + const name = agent.name?.trim() || t("agentRepository.card.untitled"); + confirm({ + title: t("businessLogic.config.modal.deleteTitle"), + content: t("businessLogic.config.modal.deleteContent", { name }), + onOk: async () => { + try { + const result = await deleteAgentMutation.mutateAsync(agent.agent_id); + if (!result.success) { + throw new Error(result.message || "delete failed"); + } + message.success( + t("businessLogic.config.error.agentDeleteSuccess", { name }) + ); + await Promise.all([ + invalidateAgentRepositoryCaches(queryClient), + queryClient.invalidateQueries({ + queryKey: [AGENTS_LIST_QUERY_KEY], + }), + ]); + } catch (error) { + log.error("Failed to delete agent:", error); + message.error(t("businessLogic.config.error.agentDeleteFailed")); + throw error; + } + }, + }); + }; + + const handleEvaluate = (agent: MyEditableAgentItem) => { + const versionNo = agent.current_version_no ?? 0; + if (versionNo <= 0) { + return; + } + router.push(`/${locale}/space/agents/${agent.agent_id}/evaluate?back_tab=mine`); }; const closeReviewModal = () => { @@ -353,10 +398,16 @@ export function MineAgentsView({ } onApplyListing={() => handleApplyListing(agent)} onViewReview={(mode) => handleViewReview(agent, mode)} + onDelete={() => handleDeleteAgent(agent)} + onEvaluate={() => handleEvaluate(agent)} isApplying={ applyingAgentId === agent.agent_id && createListingMutation.isPending } + isDeleting={ + deleteAgentMutation.isPending && + deleteAgentMutation.variables === agent.agent_id + } />
) diff --git a/frontend/app/[locale]/agent-space/components/MineApplyListingModal.tsx b/frontend/app/[locale]/agent-space/components/MineApplyListingModal.tsx index 70a178251..7bc8156eb 100644 --- a/frontend/app/[locale]/agent-space/components/MineApplyListingModal.tsx +++ b/frontend/app/[locale]/agent-space/components/MineApplyListingModal.tsx @@ -1,19 +1,16 @@ "use client"; -import { useEffect, useMemo, useState } from "react"; -import { App, Button, Modal, Select, Spin } from "antd"; -import { Share2 } from "lucide-react"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { App, Button, Dropdown, Input, Modal, Select, Spin } from "antd"; +import { ChevronDown, Share2 } from "lucide-react"; import { useTranslation } from "react-i18next"; import { - AGENT_REPOSITORY_CATEGORIES, AGENT_REPOSITORY_ICONS, AGENT_REPOSITORY_PRESET_TAGS, } from "@/const/agentRepository"; import { useAgentRepositoryListings } from "@/hooks/agentRepository/useAgentRepositoryListings"; -import { - getAgentRepositoryCategoryLabel, - getAgentRepositoryTagLabel, -} from "@/lib/agentRepositoryLabels"; +import { getAgentRepositoryTagLabel, resolveAgentRepositoryTagForSubmit } from "@/lib/agentRepositoryLabels"; +import { isSingleSimpleEmoji } from "@/lib/agentRepositoryIcon"; import { buildApplyListingFormPrefill, pickApplyListingPrefillSource, @@ -25,6 +22,7 @@ import type { const MAX_TAGS = 5; const MAX_TAG_LENGTH = 20; +const MAX_ICON_LENGTH = 32; interface MineApplyListingModalProps { open: boolean; @@ -45,17 +43,12 @@ export function MineApplyListingModal({ const { message } = App.useApp(); const icons = AGENT_REPOSITORY_ICONS; - const categories = AGENT_REPOSITORY_CATEGORIES; const presetTags = AGENT_REPOSITORY_PRESET_TAGS; - const allowedCategoryIds = useMemo( - () => categories.map((category) => category.id), - [categories] - ); const [selectedIcon, setSelectedIcon] = useState(null); - const [selectedCategoryId, setSelectedCategoryId] = useState( - null - ); + const [iconInput, setIconInput] = useState(""); + const [iconError, setIconError] = useState(null); + const [presetDropdownOpen, setPresetDropdownOpen] = useState(false); const [selectedTags, setSelectedTags] = useState([]); const agentId = agent?.agent_id; @@ -79,17 +72,46 @@ export function MineApplyListingModal({ [presetTags, t] ); + const invalidIconMessage = t( + "agentRepository.mine.applyModal.validation.iconInvalid" + ); + + const applyIconInputFromValue = useCallback( + (value: string, showErrorWhenInvalid = true) => { + setIconInput(value); + + const trimmedValue = value.trim(); + if (!trimmedValue) { + setSelectedIcon(null); + setIconError(null); + return; + } + + if (isSingleSimpleEmoji(trimmedValue)) { + setSelectedIcon(trimmedValue); + setIconError(null); + return; + } + + setSelectedIcon(null); + setIconError(showErrorWhenInvalid ? invalidIconMessage : null); + }, + [invalidIconMessage] + ); + + const clearIconState = useCallback(() => { + setIconInput(""); + setSelectedIcon(null); + setIconError(null); + }, []); + useEffect(() => { if (!open) { return; } - const defaultIcon = icons[0] ?? null; - const defaultCategoryId = categories[0]?.id ?? null; - if (!agent || !isListingsSuccess) { - setSelectedIcon(defaultIcon); - setSelectedCategoryId(defaultCategoryId); + clearIconState(); setSelectedTags([]); return; } @@ -99,29 +121,30 @@ export function MineApplyListingModal({ agent.version_label ); const prefill = buildApplyListingFormPrefill(source, { - allowedIcons: icons, - allowedCategoryIds, maxTags: MAX_TAGS, }); if (!prefill) { - setSelectedIcon(defaultIcon); - setSelectedCategoryId(defaultCategoryId); + clearIconState(); setSelectedTags([]); return; } - setSelectedIcon(prefill.icon ?? defaultIcon); - setSelectedCategoryId(prefill.categoryId ?? defaultCategoryId); + const trimmedIcon = prefill.icon?.trim(); + if (trimmedIcon && isSingleSimpleEmoji(trimmedIcon)) { + applyIconInputFromValue(trimmedIcon, false); + } else { + clearIconState(); + } + setSelectedTags(prefill.tags); }, [ open, agent, isListingsSuccess, listingsData, - icons, - categories, - allowedCategoryIds, + clearIconState, + applyIconInputFromValue, ]); const title = @@ -141,17 +164,44 @@ export function MineApplyListingModal({ return normalized; }; + const handlePresetIconClick = (icon: string) => { + applyIconInputFromValue(icon, false); + setPresetDropdownOpen(false); + }; + + const presetDropdown = ( +
+
+ {icons.map((icon) => ( + + ))} +
+
+ ); + const handleSubmit = () => { - if (!selectedIcon) { - message.warning(t("agentRepository.mine.applyModal.validation.icon")); + if (iconInput.trim() && !isSingleSimpleEmoji(iconInput)) { + setIconError(invalidIconMessage); + message.warning(invalidIconMessage); return; } - if (selectedCategoryId == null) { - message.warning(t("agentRepository.mine.applyModal.validation.category")); + + if (!selectedIcon) { + message.warning(t("agentRepository.mine.applyModal.validation.icon")); return; } - const tags = normalizeTags(selectedTags); + const tags = normalizeTags(selectedTags).map((tag) => + resolveAgentRepositoryTagForSubmit(tag, t) + ); if (tags.length === 0) { message.warning(t("agentRepository.mine.applyModal.validation.tags")); return; @@ -175,7 +225,6 @@ export function MineApplyListingModal({ onSubmit({ icon: selectedIcon, - category_id: selectedCategoryId, tags, }); }; @@ -213,45 +262,47 @@ export function MineApplyListingModal({

{t("agentRepository.mine.applyModal.icon")}

-
- {icons.map((icon) => { - const isSelected = selectedIcon === icon; - return ( + + applyIconInputFromValue(event.target.value) + } + maxLength={MAX_ICON_LENGTH} + status={iconError ? "error" : undefined} + className="w-[5.25rem] shrink-0 text-2xl" + styles={{ + input: { + paddingInline: 2, + textAlign: "center", + }, + }} + suffix={ + presetDropdown} + > - ); - })} -
- - -
-

- {t("agentRepository.mine.applyModal.category")} -

- setNewServerUrl(e.target.value)} style={{ flex: 3 }} disabled={actionsLocked || addingServer} + autoComplete="off" />
setNewServerAuthorizationToken(e.target.value)} disabled={actionsLocked || addingServer} style={{ flex: 1 }} + autoComplete="new-password" /> - )} - - - {/* Chat messages area */} -
- {chatMessages.length === 0 && ( -
- {t("skillManagement.form.chatPlaceholder")} -
- )} - {chatMessages.map((msg) => ( -
-
- {msg.role === "assistant" && msg.id === currentAssistantIdRef.current && isThinkingVisible ? ( -
- - {thinkingDescription && ( - - {thinkingDescription} - - )} -
- ) : msg.role === "assistant" ? ( -
- -
- ) : ( -
{msg.content}
- )} -
-
- ))} -
- - {/* Chat input area */} -
- -