diff --git a/DO_OPENAPI_COMMIT_SHA.txt b/DO_OPENAPI_COMMIT_SHA.txt index 1c7ce0cf..7ffb1d26 100644 --- a/DO_OPENAPI_COMMIT_SHA.txt +++ b/DO_OPENAPI_COMMIT_SHA.txt @@ -1 +1 @@ -3d7b823 +7e5b46e diff --git a/src/pydo/aio/operations/_operations.py b/src/pydo/aio/operations/_operations.py index 1ba3c2d4..120e722d 100644 --- a/src/pydo/aio/operations/_operations.py +++ b/src/pydo/aio/operations/_operations.py @@ -369,6 +369,7 @@ build_genai_update_knowledge_base_data_source_request, build_genai_update_knowledge_base_request, build_genai_update_model_api_key_request, + build_genai_update_model_evaluation_run_request, build_genai_update_model_router_request, build_genai_update_openai_api_key_request, build_genai_update_workspace_request, @@ -208123,8 +208124,14 @@ async def list_agents( "user_id": "str", # Optional. Id of user that created the agent. "uuid": "str", # Optional. Unique agent id. - "version_hash": "str" # Optional. The latest version of the + "version_hash": "str", # Optional. The latest version of the agent. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool to retrieve content from public web + pages. + "web_search_enabled": bool # Optional. Whether this agent + can use the built-in web_search tool to search the public web for current + information. } ], "links": { @@ -208306,6 +208313,10 @@ async def create_agent( "str" # Optional. Agent tag to organize related resources. ], "thinking_token_budget": 0, # Optional. + "web_fetch_enabled": bool, # Optional. Whether the agent can use the + built-in web_fetch tool to retrieve content from public web pages. + "web_search_enabled": bool, # Optional. Whether the agent can use the + built-in web_search tool to search the public web for current information. "workspace_uuid": "str" # Optional. Identifier for the workspace. } @@ -209336,6 +209347,10 @@ async def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -209412,14 +209427,14 @@ async def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -209428,9 +209443,10 @@ async def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -210579,6 +210595,10 @@ async def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -210655,14 +210675,14 @@ async def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -210671,9 +210691,10 @@ async def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -210835,6 +210856,10 @@ async def create_agent( "str" # Optional. Agent tag to organize related resources. ], "thinking_token_budget": 0, # Optional. + "web_fetch_enabled": bool, # Optional. Whether the agent can use the + built-in web_fetch tool to retrieve content from public web pages. + "web_search_enabled": bool, # Optional. Whether the agent can use the + built-in web_search tool to search the public web for current information. "workspace_uuid": "str" # Optional. Identifier for the workspace. } @@ -211865,6 +211890,10 @@ async def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -211941,14 +211970,14 @@ async def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -211957,9 +211986,10 @@ async def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -214141,6 +214171,10 @@ async def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -214217,14 +214251,14 @@ async def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -214233,9 +214267,10 @@ async def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -215387,6 +215422,10 @@ async def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -215463,14 +215502,14 @@ async def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -215479,9 +215518,10 @@ async def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -216643,6 +216683,10 @@ async def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -216719,14 +216763,14 @@ async def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -216735,9 +216779,10 @@ async def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -218002,6 +218047,10 @@ async def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -218078,14 +218127,14 @@ async def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -218094,9 +218143,10 @@ async def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -219251,6 +219301,10 @@ async def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -219327,14 +219381,14 @@ async def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -219343,9 +219397,10 @@ async def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -220511,6 +220566,10 @@ async def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -220587,14 +220646,14 @@ async def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -220603,9 +220662,10 @@ async def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -221845,6 +221905,10 @@ async def detach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -221921,14 +221985,14 @@ async def detach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -221937,9 +222001,10 @@ async def detach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -223180,6 +223245,10 @@ async def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -223256,14 +223325,14 @@ async def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -223272,9 +223341,10 @@ async def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -224426,6 +224496,10 @@ async def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -224502,14 +224576,14 @@ async def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -224518,9 +224592,10 @@ async def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -225678,6 +225753,10 @@ async def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -225754,14 +225833,14 @@ async def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -225770,9 +225849,10 @@ async def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -227010,6 +227090,10 @@ async def detach_agent_guardrail( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -227086,14 +227170,14 @@ async def detach_agent_guardrail( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -227102,9 +227186,10 @@ async def detach_agent_guardrail( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -228322,6 +228407,10 @@ async def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -228398,14 +228487,14 @@ async def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -228414,9 +228503,10 @@ async def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -229637,6 +229727,10 @@ async def attach_knowledge_base( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -229713,14 +229807,14 @@ async def attach_knowledge_base( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -229729,9 +229823,10 @@ async def attach_knowledge_base( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -230953,6 +231048,10 @@ async def detach_knowledge_base( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -231029,14 +231128,14 @@ async def detach_knowledge_base( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -231045,9 +231144,10 @@ async def detach_knowledge_base( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -232914,6 +233014,10 @@ async def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -232990,14 +233094,14 @@ async def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -233006,9 +233110,10 @@ async def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -233281,7 +233386,13 @@ async def update_agent( "top_p": 0.0, # Optional. Defines the cumulative probability threshold for word selection, specified as a number between 0 and 1. Higher values allow for more diverse outputs, while lower values ensure focused and coherent responses. - "uuid": "str" # Optional. Unique agent id. + "uuid": "str", # Optional. Unique agent id. + "web_fetch_enabled": bool, # Optional. Optional. Set to true to let the + agent use the built-in web_fetch tool to retrieve content from public web pages, + or false to disable it. + "web_search_enabled": bool # Optional. Optional. Set to true to let the + agent use the built-in web_search tool to search the public web for current + information, or false to disable it. } # response body for status code(s): 200 @@ -234311,6 +234422,10 @@ async def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -234387,14 +234502,14 @@ async def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -234403,9 +234518,10 @@ async def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -235557,6 +235673,10 @@ async def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -235633,14 +235753,14 @@ async def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -235649,9 +235769,10 @@ async def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -235839,7 +235960,13 @@ async def update_agent( "top_p": 0.0, # Optional. Defines the cumulative probability threshold for word selection, specified as a number between 0 and 1. Higher values allow for more diverse outputs, while lower values ensure focused and coherent responses. - "uuid": "str" # Optional. Unique agent id. + "uuid": "str", # Optional. Unique agent id. + "web_fetch_enabled": bool, # Optional. Optional. Set to true to let the + agent use the built-in web_fetch tool to retrieve content from public web pages, + or false to disable it. + "web_search_enabled": bool # Optional. Optional. Set to true to let the + agent use the built-in web_search tool to search the public web for current + information, or false to disable it. } # response body for status code(s): 200 @@ -236869,6 +236996,10 @@ async def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -236945,14 +237076,14 @@ async def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -236961,9 +237092,10 @@ async def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -238196,6 +238328,10 @@ async def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -238272,14 +238408,14 @@ async def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -238288,9 +238424,10 @@ async def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -239569,6 +239706,10 @@ async def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Child agents. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -239646,16 +239787,14 @@ async def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -239664,10 +239803,10 @@ async def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -240919,6 +241058,10 @@ async def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -240995,14 +241138,14 @@ async def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -241011,9 +241154,10 @@ async def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -242165,6 +242309,10 @@ async def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -242241,14 +242389,14 @@ async def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -242257,9 +242405,10 @@ async def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -243417,6 +243566,10 @@ async def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -243493,14 +243646,14 @@ async def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -243509,9 +243662,10 @@ async def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -246370,6 +246524,10 @@ async def list_agents_by_anthropic_key( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -246447,16 +246605,14 @@ async def list_agents_by_anthropic_key( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -246465,10 +246621,10 @@ async def list_agents_by_anthropic_key( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -246751,8 +246907,8 @@ async def list_custom_models( when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the - custom model import or processing failed. + "error_message": "str", # Optional. User-facing reason the + most recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., @@ -246930,6 +247086,8 @@ async def import_custom_model( # JSON input template you can fill out and use as your body input. body = { + "accept_hf_token_storage": bool, # Optional. Whether the caller accepts + storage of their HuggingFace token for gated model access. "accept_terms_and_conditions": bool, # Optional. Whether the caller accepts the terms and conditions for importing this model. "description": "str", # Optional. Description of the model. @@ -247016,8 +247174,8 @@ async def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -247172,8 +247330,8 @@ async def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -247269,6 +247427,8 @@ async def import_custom_model( # JSON input template you can fill out and use as your body input. body = { + "accept_hf_token_storage": bool, # Optional. Whether the caller accepts + storage of their HuggingFace token for gated model access. "accept_terms_and_conditions": bool, # Optional. Whether the caller accepts the terms and conditions for importing this model. "description": "str", # Optional. Description of the model. @@ -247355,8 +247515,8 @@ async def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -247579,8 +247739,8 @@ async def get_custom_model(self, uuid: str, **kwargs: Any) -> JSON: "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -247871,14 +248031,16 @@ async def update_custom_model_metadata( body = { "description": "str", # Optional. "input_modalities": [ - "str" # Optional. Input modalities supported (e.g., text, image). + "str" # Optional. Optional new input modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "license": "str", # Optional. License under which the model is distributed. + "license": "str", # Optional. "name": "str", # Optional. "output_modalities": [ - "str" # Optional. Output modalities supported (e.g., text, image). + "str" # Optional. Optional new output modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "parameters": "str", # Optional. Number of parameters in the model. + "parameters": "str", # Optional. "tags": { "tags": [ "str" # Optional. List of tag strings. @@ -247924,8 +248086,8 @@ async def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -248058,8 +248220,8 @@ async def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -248151,14 +248313,16 @@ async def update_custom_model_metadata( body = { "description": "str", # Optional. "input_modalities": [ - "str" # Optional. Input modalities supported (e.g., text, image). + "str" # Optional. Optional new input modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "license": "str", # Optional. License under which the model is distributed. + "license": "str", # Optional. "name": "str", # Optional. "output_modalities": [ - "str" # Optional. Output modalities supported (e.g., text, image). + "str" # Optional. Optional new output modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "parameters": "str", # Optional. Number of parameters in the model. + "parameters": "str", # Optional. "tags": { "tags": [ "str" # Optional. List of tag strings. @@ -248204,8 +248368,8 @@ async def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -249275,12 +249439,12 @@ async def list_evaluation_metrics(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # Optional. - Timestamp when the custom metric was created. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. - When set, the custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; ignored on - create/update requests. + When set, the custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not @@ -249288,8 +249452,9 @@ async def list_evaluation_metrics(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # Optional. - Timestamp when the custom metric was last updated. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # @@ -249445,20 +249610,19 @@ async def create_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str" # Optional. @@ -249482,20 +249646,21 @@ async def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -249584,20 +249749,21 @@ async def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -249664,20 +249830,19 @@ async def create_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str" # Optional. @@ -249701,20 +249866,21 @@ async def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -249884,20 +250050,19 @@ async def update_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str", # Optional. @@ -249922,20 +250087,21 @@ async def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -250027,20 +250193,21 @@ async def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -250112,20 +250279,19 @@ async def update_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str", # Optional. @@ -250150,20 +250316,21 @@ async def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -250699,6 +250866,8 @@ async def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> J "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -250708,6 +250877,14 @@ async def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> J metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -250717,6 +250894,8 @@ async def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> J "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID (built-in or + custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", "METRIC_VALUE_TYPE_NUMBER", @@ -250725,6 +250904,13 @@ async def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> J a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -250885,6 +251071,8 @@ async def get_evaluation_run_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -250894,6 +251082,14 @@ async def get_evaluation_run_results( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -250903,6 +251099,8 @@ async def get_evaluation_run_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID (built-in or + custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", "METRIC_VALUE_TYPE_NUMBER", @@ -250911,6 +251109,13 @@ async def get_evaluation_run_results( a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -250974,6 +251179,9 @@ async def get_evaluation_run_results( calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # + Optional. Metric UUID (built-in or custom); stable key + for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values @@ -250984,6 +251192,16 @@ async def get_evaluation_run_results( Optional. The value of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known + values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -251186,6 +251404,9 @@ async def get_evaluation_run_results( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -251195,6 +251416,14 @@ async def get_evaluation_run_results( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -251351,6 +251580,9 @@ async def get_evaluation_run_prompt_results( calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. + Metric UUID (built-in or custom); stable key for results UI + and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -251361,6 +251593,15 @@ async def get_evaluation_run_prompt_results( value of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -251564,6 +251805,8 @@ async def get_evaluation_run_prompt_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -251573,6 +251816,14 @@ async def get_evaluation_run_prompt_results( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -251752,12 +252003,12 @@ async def list_evaluation_test_cases(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -251765,8 +252016,9 @@ async def list_evaluation_test_cases(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. Alternative way of authentication for internal usage only - @@ -252256,6 +252508,9 @@ async def list_evaluation_runs_by_test_case( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -252265,6 +252520,14 @@ async def list_evaluation_runs_by_test_case( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -252274,6 +252537,8 @@ async def list_evaluation_runs_by_test_case( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -252283,6 +252548,14 @@ async def list_evaluation_runs_by_test_case( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -252465,12 +252738,12 @@ async def get_evaluation_test_case( "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -252478,8 +252751,9 @@ async def get_evaluation_test_case( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", @@ -258684,12 +258958,12 @@ async def list_model_evaluation_metrics(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # Optional. - Timestamp when the custom metric was created. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. - When set, the custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; ignored on - create/update requests. + When set, the custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not @@ -258697,8 +258971,9 @@ async def list_model_evaluation_metrics(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # Optional. - Timestamp when the custom metric was last updated. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. List of model evaluation metrics. @@ -258849,18 +259124,48 @@ async def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: response == { "presets": [ { + "candidate_inference_config": { + "max_tokens": 0, # Optional. Inference configuration + for the candidate model during evaluation. + "stop_token": "str", # Optional. Inference + configuration for the candidate model during evaluation. + "system_prompt": "str", # Optional. Inference + configuration for the candidate model during evaluation. + "temperature": 0.0 # Optional. Inference + configuration for the candidate model during evaluation. + }, + "candidate_model_name": "str", # Optional. Model slug used + to call the candidate model API. Empty when the CANDIDATE section was not + saved. + "candidate_model_source": + "CANDIDATE_MODEL_SOURCE_SERVERLESS", # Optional. Default value is + "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether inference runs against the + serverless platform, a dedicated deployment, or a model router. Known + values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the + candidate model stored on this preset. Empty when the CANDIDATE section + was not saved. For DEDICATED candidates this is the dedicated inference + deployment UUID. + "candidate_system_prompt": "str", # Optional. System prompt + / instructions to send to the candidate model. Empty when the + SYSTEM_PROMPT section was not saved (check ``saved_sections``"" ). "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the preset was created. - "dataset_name": "str", # Optional. List of explicitly saved - evaluation presets (reusable configs). - "dataset_uuid": "str", # Optional. Dataset used for - evaluation. + "dataset_name": "str", # Optional. Display name of the + dataset stored on this preset. Empty when the DATASET section was not + saved or the dataset no longer exists. + "dataset_uuid": "str", # Optional. UUID of the dataset + stored on this preset. Empty when the DATASET section was not saved + (check ``saved_sections``"" ). "eval_preset_uuid": "str", # Optional. UUID of the evaluation preset. - "judge_model_name": "str", # Optional. List of explicitly - saved evaluation presets (reusable configs). - "judge_model_uuid": "str", # Optional. Judge model used to - score responses. + "judge_model_name": "str", # Optional. Display name of the + judge model stored on this preset. Empty when the JUDGE section was not + saved or the model no longer exists. + "judge_model_uuid": "str", # Optional. UUID of the judge + model stored on this preset. Empty when the JUDGE section was not saved + (check ``saved_sections``"" ). "metrics": [ { "associated_presets": [ @@ -258881,12 +259186,12 @@ async def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -258894,11 +259199,13 @@ async def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. Default value is "EVALUATION_SCOPE_UNSPECIFIED". Scope that determines whether a @@ -258910,18 +259217,22 @@ async def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "inverted": bool, # Optional. If true, the metric is inverted, meaning that a lower value is better. "is_metric_goal": bool, # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_name": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_rank": 0, # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_type": "METRIC_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_TYPE_UNSPECIFIED". Known values are: "METRIC_TYPE_UNSPECIFIED", "METRIC_TYPE_GENERAL_QUALITY", "METRIC_TYPE_RAG_AND_TOOL", "METRIC_TYPE_MODEL_QUALITY", and "METRIC_TYPE_MODEL_SAFETY". "metric_uuid": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -258941,6 +259252,14 @@ async def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: } ], "name": "str", # Optional. Name of the evaluation preset. + "saved_sections": [ + "str" # Optional. Sections of the inline evaluation + config that were persisted when this preset was created. Use this to + tell "section was saved with an empty value" apart from "section was + not saved" "u2014 scalar fields like ``dataset_uuid`` or + ``candidate_system_prompt`` are always emitted as the empty string + when the section was not saved. + ], "star_metric": { "metric_uuid": "str", # Optional. List of explicitly saved evaluation presets (reusable configs). @@ -259067,17 +259386,45 @@ async def get_model_evaluation_preset( # response body for status code(s): 200 response == { "preset": { + "candidate_inference_config": { + "max_tokens": 0, # Optional. Inference configuration for the + candidate model during evaluation. + "stop_token": "str", # Optional. Inference configuration for + the candidate model during evaluation. + "system_prompt": "str", # Optional. Inference configuration + for the candidate model during evaluation. + "temperature": 0.0 # Optional. Inference configuration for + the candidate model during evaluation. + }, + "candidate_model_name": "str", # Optional. Model slug used to call + the candidate model API. Empty when the CANDIDATE section was not saved. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model stored on this preset. Empty when the CANDIDATE section was not saved. + For DEDICATED candidates this is the dedicated inference deployment UUID. + "candidate_system_prompt": "str", # Optional. System prompt / + instructions to send to the candidate model. Empty when the SYSTEM_PROMPT + section was not saved (check ``saved_sections``"" ). "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the preset was created. - "dataset_name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. - "dataset_uuid": "str", # Optional. Dataset used for evaluation. + "dataset_name": "str", # Optional. Display name of the dataset + stored on this preset. Empty when the DATASET section was not saved or the + dataset no longer exists. + "dataset_uuid": "str", # Optional. UUID of the dataset stored on + this preset. Empty when the DATASET section was not saved (check + ``saved_sections``"" ). "eval_preset_uuid": "str", # Optional. UUID of the evaluation preset. - "judge_model_name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. - "judge_model_uuid": "str", # Optional. Judge model used to score - responses. + "judge_model_name": "str", # Optional. Display name of the judge + model stored on this preset. Empty when the JUDGE section was not saved or + the model no longer exists. + "judge_model_uuid": "str", # Optional. UUID of the judge model + stored on this preset. Empty when the JUDGE section was not saved (check + ``saved_sections``"" ). "metrics": [ { "associated_presets": [ @@ -259097,12 +259444,12 @@ async def get_model_evaluation_preset( "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -259110,11 +259457,12 @@ async def get_model_evaluation_preset( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. Default value is "EVALUATION_SCOPE_UNSPECIFIED". Scope that determines whether a metric belongs to agent evaluation or model @@ -259125,18 +259473,18 @@ async def get_model_evaluation_preset( "inverted": bool, # Optional. If true, the metric is inverted, meaning that a lower value is better. "is_metric_goal": bool, # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_name": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_rank": 0, # Optional. Metrics selected for - this preset. + this preset. Empty when the METRICS section was not saved. "metric_type": "METRIC_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_TYPE_UNSPECIFIED". Known values are: "METRIC_TYPE_UNSPECIFIED", "METRIC_TYPE_GENERAL_QUALITY", "METRIC_TYPE_RAG_AND_TOOL", "METRIC_TYPE_MODEL_QUALITY", and "METRIC_TYPE_MODEL_SAFETY". "metric_uuid": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -259155,11 +259503,29 @@ async def get_model_evaluation_preset( } ], "name": "str", # Optional. Name of the evaluation preset. + "saved_sections": [ + "str" # Optional. Sections of the inline evaluation config + that were persisted when this preset was created. Use this to tell + "section was saved with an empty value" apart from "section was not + saved" "u2014 scalar fields like ``dataset_uuid`` or + ``candidate_system_prompt`` are always emitted as the empty string when + the section was not saved. + ], "star_metric": { "metric_uuid": "str", # Optional. Model Evaluation Preset - - a saved, reusable configuration for model evaluation runs. + a saved, reusable configuration for model evaluation runs. Each section + (dataset, judge, metrics, candidate, system prompt) is independent and + may be empty; sections the preset omits must be supplied inline on the + run that references it. Use ``saved_sections`` to tell "section saved + with empty value" apart from "section not saved at all" "u2014 each + section's scalar fields are left empty when the section was not saved. "name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. + saved, reusable configuration for model evaluation runs. Each section + (dataset, judge, metrics, candidate, system prompt) is independent and + may be empty; sections the preset omits must be supplied inline on the + run that references it. Use ``saved_sections`` to tell "section saved + with empty value" apart from "section not saved at all" "u2014 each + section's scalar fields are left empty when the section was not saved. "success_threshold": 0.0, # Optional. The success threshold for the star metric. This is a value that the metric must reach to be considered successful. @@ -259653,6 +260019,19 @@ async def create_model_evaluation_run( ], "name": "str", # Optional. "preset_name": "str", # Optional. + "preset_save_sections": [ + "str" # Optional. Which sections of this run's resolved + configuration to persist as a reusable preset. Each selected section saves + only its own fields; the remaining sections stay empty on the preset and must + be supplied inline on future runs that reference it. Empty means do not save + a preset (unless the deprecated ``save_as_preset`` boolean is true, in which + case all sections are saved). Ignored when ``eval_preset_uuid`` is set. Use + ``preset_name`` to label the saved preset. + ], + "save_as_preset": bool, # Optional. Deprecated: use + ``preset_save_sections``. When ``true`` and ``preset_save_sections`` is empty, + all five sections of the resolved configuration are saved as a reusable preset + (legacy behavior). Ignored when ``eval_preset_uuid`` is set. "source": "str", # Optional. Source of the run creation (api, sdk, cli). "star_metric": { "metric_uuid": "str", # Optional. @@ -259775,6 +260154,19 @@ async def create_model_evaluation_run( ], "name": "str", # Optional. "preset_name": "str", # Optional. + "preset_save_sections": [ + "str" # Optional. Which sections of this run's resolved + configuration to persist as a reusable preset. Each selected section saves + only its own fields; the remaining sections stay empty on the preset and must + be supplied inline on future runs that reference it. Empty means do not save + a preset (unless the deprecated ``save_as_preset`` boolean is true, in which + case all sections are saved). Ignored when ``eval_preset_uuid`` is set. Use + ``preset_name`` to label the saved preset. + ], + "save_as_preset": bool, # Optional. Deprecated: use + ``preset_save_sections``. When ``true`` and ``preset_save_sections`` is empty, + all five sections of the resolved configuration are saved as a reusable preset + (legacy behavior). Ignored when ``eval_preset_uuid`` is set. "source": "str", # Optional. Source of the run creation (api, sdk, cli). "star_metric": { "metric_uuid": "str", # Optional. @@ -259947,6 +260339,8 @@ async def get_model_evaluation_run( per-prompt evaluation results. "candidate_model_uuid": "str", # Optional. Paginated per-prompt evaluation results. + "candidate_routed_task": "str", # Optional. Paginated + per-prompt evaluation results. "ground_truth": "str", # Optional. Paginated per-prompt evaluation results. "input": "str", # Optional. The input query sent to the @@ -259957,6 +260351,9 @@ async def get_model_evaluation_run( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -259966,6 +260363,14 @@ async def get_model_evaluation_run( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -260031,12 +260436,12 @@ async def get_model_evaluation_run( "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -260044,8 +260449,9 @@ async def get_model_evaluation_run( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Metrics selected for this evaluation. @@ -260105,14 +260511,24 @@ async def get_model_evaluation_run( { "description": "str", # Optional. Per-metric aggregated pass/fail statistics. + "fail_count": 0, # Optional. Rows where the + metric failed to score or completed below the threshold. "fail_percent": 0.0, # Optional. Per-metric aggregated pass/fail statistics. "metric_name": "str", # Optional. Per-metric aggregated pass/fail statistics. "metric_uuid": "str", # Optional. Per-metric aggregated pass/fail statistics. - "pass_percent": 0.0 # Optional. Per-metric + "pass_count": 0, # Optional. Rows where the + metric completed and passed the configured threshold. + "pass_percent": 0.0, # Optional. Per-metric aggregated pass/fail statistics. + "skip_percent": 0.0, # Optional. Percentage + of rows that were skipped for this metric, computed as + skipped_count divided by the total rows the metric saw + (pass_count + fail_count + skipped_count). + "skipped_count": 0 # Optional. Rows where + the metric was not evaluated for this prompt. } ], "overall_score_percent": 0.0, # Optional. Aggregated result @@ -260123,24 +260539,39 @@ async def get_model_evaluation_run( "metric_summaries": [ { "description": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "fail_count": 0, # + Optional. Rows where the metric failed to score or + completed below the threshold. "fail_percent": 0.0, - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. "metric_name": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. "metric_uuid": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. - "pass_percent": 0.0 - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "pass_count": 0, # + Optional. Rows where the metric completed and passed + the configured threshold. + "pass_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "skip_percent": 0.0, + # Optional. Percentage of rows that were skipped for + this metric, computed as skipped_count divided by the + total rows the metric saw (pass_count + fail_count + + skipped_count). + "skipped_count": 0 # + Optional. Rows where the metric was not evaluated for + this prompt. } ], - "model_name": "str", # Optional. - Name/slug of the model (matches routed_model from results). + "model_name": "str", # Optional. The + underlying model these results are for, such as ``Llama 3.3 + Instruct (70B)``. "performance_metrics": { "candidate_latency": { "avg_e2e_latency_ms": @@ -260181,7 +260612,91 @@ async def get_model_evaluation_run( } }, "prompt_count": 0 # Optional. Number - of prompts routed to this model. + of prompts in the run that were routed to this model. + } + ] + }, + "per_task_summaries": { + "summaries": [ + { + "metric_summaries": [ + { + "description": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "fail_count": 0, # + Optional. Rows where the metric failed to score or + completed below the threshold. + "fail_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "metric_name": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "metric_uuid": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "pass_count": 0, # + Optional. Rows where the metric completed and passed + the configured threshold. + "pass_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "skip_percent": 0.0, + # Optional. Percentage of rows that were skipped for + this metric, computed as skipped_count divided by the + total rows the metric saw (pass_count + fail_count + + skipped_count). + "skipped_count": 0 # + Optional. Rows where the metric was not evaluated for + this prompt. + } + ], + "performance_metrics": { + "candidate_latency": { + "avg_e2e_latency_ms": + 0.0, # Optional. Average end-to-end latency across + all invocations. + "max_e2e_latency_ms": + 0.0, # Optional. Maximum end-to-end latency + observed. + "min_e2e_latency_ms": + 0.0, # Optional. Minimum end-to-end latency + observed. + "p50_latency_ms": + 0.0, # Optional. P50 (median) latency. + "p90_latency_ms": + 0.0, # Optional. P90 latency. + "p95_latency_ms": 0.0 + # Optional. P95 latency. + }, + "token_usage": { + "total_candidate_input_tokens": "str", # Optional. + All performance metrics are for the candidate model + unless noted otherwise. + "total_candidate_output_tokens": "str", # Optional. + All performance metrics are for the candidate model + unless noted otherwise. + "total_candidate_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_input_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_output_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_tokens": + "str" # Optional. All performance metrics are for + the candidate model unless noted otherwise. + } + }, + "prompt_count": 0, # Optional. + Number of prompts in the run that were classified into this + task category. + "task_name": "str" # Optional. The + routing task category these results are for, such as ``Coding + & brainstorming`` or ``Summarization``. } ] }, @@ -260506,6 +261021,374 @@ async def delete_model_evaluation_run( return cast(JSON, deserialized) # type: ignore + @overload + async def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[JSON] = None, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Default value is None. + :type body: JSON + :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. + Default value is "application/json". + :paramtype content_type: str + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # JSON input template you can fill out and use as your body input. + body = { + "eval_run_uuid": "str", # Optional. UUID of the model evaluation run to + update. Returned by ``CreateModelEvaluationRun`` and listed via + ``ListModelEvaluationRuns``. + "name": "str" # Optional. Optional new display name for the evaluation run + (max 255 characters). + } + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + + @overload + async def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[IO[bytes]] = None, + *, + content_type: str = "application/json", + **kwargs: Any + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Default value is None. + :type body: IO[bytes] + :keyword content_type: Body Parameter content-type. Content type parameter for binary body. + Default value is "application/json". + :paramtype content_type: str + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + + @distributed_trace_async + async def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[Union[JSON, IO[bytes]]] = None, + **kwargs: Any + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Is either a JSON type or a IO[bytes] type. Default value is None. + :type body: JSON or IO[bytes] + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # JSON input template you can fill out and use as your body input. + body = { + "eval_run_uuid": "str", # Optional. UUID of the model evaluation run to + update. Returned by ``CreateModelEvaluationRun`` and listed via + ``ListModelEvaluationRuns``. + "name": "str" # Optional. Optional new display name for the evaluation run + (max 255 characters). + } + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + error_map: MutableMapping[int, Type[HttpResponseError]] = { + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + 401: cast( + Type[HttpResponseError], + lambda response: ClientAuthenticationError(response=response), + ), + 429: HttpResponseError, + 500: HttpResponseError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {}) + _params = kwargs.pop("params", {}) or {} + + content_type: Optional[str] = kwargs.pop( + "content_type", _headers.pop("Content-Type", None) + ) + cls: ClsType[JSON] = kwargs.pop("cls", None) + + content_type = content_type or "application/json" + _json = None + _content = None + if isinstance(body, (IOBase, bytes)): + _content = body + else: + if body is not None: + _json = body + else: + _json = None + + _request = build_genai_update_model_evaluation_run_request( + eval_run_uuid=eval_run_uuid, + content_type=content_type, + json=_json, + content=_content, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _stream = False + pipeline_response: PipelineResponse = ( + await self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + ) + + response = pipeline_response.http_response + + if response.status_code not in [200, 404]: + if _stream: + await response.read() # Load the body in memory and close the socket + map_error(status_code=response.status_code, response=response, error_map=error_map) # type: ignore + raise HttpResponseError(response=response) + + response_headers = {} + if response.status_code == 200: + response_headers["ratelimit-limit"] = self._deserialize( + "int", response.headers.get("ratelimit-limit") + ) + response_headers["ratelimit-remaining"] = self._deserialize( + "int", response.headers.get("ratelimit-remaining") + ) + response_headers["ratelimit-reset"] = self._deserialize( + "int", response.headers.get("ratelimit-reset") + ) + + if response.content: + deserialized = response.json() + else: + deserialized = None + + if response.status_code == 404: + response_headers["ratelimit-limit"] = self._deserialize( + "int", response.headers.get("ratelimit-limit") + ) + response_headers["ratelimit-remaining"] = self._deserialize( + "int", response.headers.get("ratelimit-remaining") + ) + response_headers["ratelimit-reset"] = self._deserialize( + "int", response.headers.get("ratelimit-reset") + ) + + if response.content: + deserialized = response.json() + else: + deserialized = None + + if cls: + return cls(pipeline_response, cast(JSON, deserialized), response_headers) # type: ignore + + return cast(JSON, deserialized) # type: ignore + @overload async def cancel_model_evaluation_run( self, @@ -261105,6 +261988,27 @@ async def list_models( "parent_uuid": "str", # Optional. Unique id of the model, this model is based on. "pricing": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing @@ -261123,8 +262027,22 @@ async def list_models( million tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the model does not charge + separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs image tokens + separately (e.g. OpenAI gpt-image-2). Standard chat models leave + these at 0 and use input_price_per_million / output_price_per_million + instead. Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: @@ -262144,20 +263062,52 @@ async def list_model_catalog( "availability": [ "str" # Optional. ], + "badges": [ + "str" # Optional. Badges for models. + ], "benchmark_score": {}, # Optional. Benchmark scores for this model, stored as arbitrary JSON. "capabilities": [ "str" # Optional. ], "context_window": "str", # Optional. Specs (flat). + "created_at": "2020-02-20 00:00:00", # Optional. RFC 3339 + timestamp indicating when the model was added to the catalog. "creator": "str", # Optional. Model creator/developer (e.g., "Meta", "Anthropic", "OpenAI"). + "hugging_face_id": "str", # Optional. The Hugging Face + repository ID (e.g. "meta-llama/Llama-3.3-70B-Instruct") the model is + based on, when applicable. Omitted for models not sourced from Hugging + Face. "id": "str", # Optional. Identity. + "max_output_tokens": "str", # Optional. The maximum number + of output tokens the model can generate in a single response. "model_id": "str", # Optional. Model identifier used for API calls (e.g., "llama3.1-70b-instruct"). "name": "str", # Optional. "parameter_count": 0.0, # Optional. "pricing": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing @@ -262176,13 +263126,30 @@ async def list_model_catalog( million tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the model does not charge + separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs image tokens + separately (e.g. OpenAI gpt-image-2). Standard chat models leave + these at 0 and use input_price_per_million / output_price_per_million + instead. Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: "MODEL_PROVIDER_DIGITALOCEAN", "MODEL_PROVIDER_ANTHROPIC", and "MODEL_PROVIDER_OPENAI". + "scaled_pricing_enabled": bool, # Optional. True when this + model's pricing varies over time. Retrieve the model's details for the + full pricing schedule. False for models with fixed pricing. "short_description": "str", # Optional. "type": "str" # Optional. } @@ -262312,6 +263279,9 @@ async def get_model_catalog_card( "availability": [ "str" # Optional. Detail view for GetModelCatalogCard. ], + "badges": [ + "str" # Optional. Badges for models. + ], "benchmark_score": {}, # Optional. Benchmark scores for this model, stored as arbitrary JSON. "capabilities": [ @@ -262330,7 +263300,12 @@ async def get_model_catalog_card( "creator": "str", # Optional. Model creator/developer (e.g., "Meta", "Anthropic", "OpenAI"). "description": "str", # Optional. Card-specific. + "hugging_face_id": "str", # Optional. The Hugging Face repository ID + (e.g. "meta-llama/Llama-3.3-70B-Instruct") the model is based on, when + applicable. Omitted for models not sourced from Hugging Face. "id": "str", # Optional. Identity (same as Entry). + "max_output_tokens": "str", # Optional. The maximum number of output + tokens the model can generate in a single response. "modalities": { "input": [ "str" # Optional. Input/output modalities. @@ -262345,6 +263320,25 @@ async def get_model_catalog_card( "parameter_count": 0.0, # Optional. Detail view for GetModelCatalogCard. "pricing": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models (token_type "cache + read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a 1-hour + lifetime. + "cache_write_5m_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a 5-minute + lifetime. + "image_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. Pricing + per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. Pricing + per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read input price + per single token. Equivalent to cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing per @@ -262362,8 +263356,344 @@ async def get_model_catalog_card( tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per million + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price per + million reasoning tokens. 0 if the model does not charge separately for + reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. Per-million + token rates for models that bill text vs image tokens separately (e.g. + OpenAI gpt-image-2). Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million instead. Values align + with usage token_type / internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing per + million tokens (aligns with existing ModelPrice pattern). + }, + "pricing_detail": { + "variants": [ + { + "currency": "str", # Optional. Currency code + for this variant's rates (e.g. ``USD``"" ). + "label": "str", # Optional. Display name for + the variant (e.g. ``Standard``"" , ``Fast Mode``"" , ``1M + Context``"" ). + "mode": "MODEL_BILLING_MODE_UNSPECIFIED", # + Optional. Default value is "MODEL_BILLING_MODE_UNSPECIFIED". + Whether rates apply to real-time or batch requests. * + MODEL_BILLING_MODE_INTERACTIVE: Real-time request pricing. * + MODEL_BILLING_MODE_BATCH: Discounted pricing for asynchronous + batch requests. Known values are: + "MODEL_BILLING_MODE_UNSPECIFIED", + "MODEL_BILLING_MODE_INTERACTIVE", and "MODEL_BILLING_MODE_BATCH". + "prices": { + "cache_read_input_price_per_million": + 0.0, # Optional. Per-million cache read rate for standard + chat models (token_type "cache read input"). Multimodal + models use text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. + Cache read input price per single token. Equivalent to + cache_read_input_price_per_million. + "input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "output_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_audio": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "price_per_image": 0.0, # Optional. + Unit-based pricing for non-token models (e.g., Fal AI + image/video/audio generation, speech models). At most one of + these is typically populated per model. Token-based models + (chat, embeddings) leave all of these at 0 and populate + input_price_per_million / output_price_per_million instead. + "price_per_megapixel": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_second": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "price_per_thousand_characters": 0.0, + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "reasoning_price_per_million": 0.0, + # Optional. Price per million reasoning tokens. 0 if the + model does not charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, + # Optional. Per-million token rates for models that bill text + vs image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million instead. + Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + }, + "scaled_pricing": { + "intervals": [ + { + "end_time": + "2020-02-20 00:00:00", # Optional. End of the + interval, exclusive. Always on the hour. + "prices": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for + standard chat models (token_type "cache read + input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the + prompt cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the + prompt cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns + with existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. + "input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_audio": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "price_per_image": 0.0, # Optional. Unit-based + pricing for non-token models (e.g., Fal AI + image/video/audio generation, speech models). At + most one of these is typically populated per + model. Token-based models (chat, embeddings) + leave all of these at 0 and populate + input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, # Optional. Pricing + per million tokens (aligns with existing + ModelPrice pattern). + "price_per_second": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "price_per_thousand_characters": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_video": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the + model does not charge separately for reasoning + tokens. + "text_cache_read_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns + with existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text + vs image tokens separately (e.g. OpenAI + gpt-image-2). Standard chat models leave these at + 0 and use input_price_per_million / + output_price_per_million instead. Values align + with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + }, + "start_time": + "2020-02-20 00:00:00" # Optional. Start of the + interval, inclusive. Always on the hour. + } + ], + "max_prices": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "input_cache_read": 0.0, # + Optional. Cache read input price per single token. + Equivalent to cache_read_input_price_per_million. + "input_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_audio": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_image": 0.0, # + Optional. Unit-based pricing for non-token models (e.g., + Fal AI image/video/audio generation, speech models). At + most one of these is typically populated per model. + Token-based models (chat, embeddings) leave all of these + at 0 and populate input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, + # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_second": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_thousand_characters": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price + per million reasoning tokens. 0 if the model does not + charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs + image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million + instead. Values align with usage token_type / + internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing + per million tokens (aligns with existing ModelPrice + pattern). + }, + "min_prices": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "input_cache_read": 0.0, # + Optional. Cache read input price per single token. + Equivalent to cache_read_input_price_per_million. + "input_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_audio": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_image": 0.0, # + Optional. Unit-based pricing for non-token models (e.g., + Fal AI image/video/audio generation, speech models). At + most one of these is typically populated per model. + Token-based models (chat, embeddings) leave all of these + at 0 and populate input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, + # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_second": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_thousand_characters": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price + per million reasoning tokens. 0 if the model does not + charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs + image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million + instead. Values align with usage token_type / + internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing + per million tokens (aligns with existing ModelPrice + pattern). + } + }, + "tier": "MODEL_PRICING_TIER_UNSPECIFIED" # + Optional. Default value is "MODEL_PRICING_TIER_UNSPECIFIED". A + pricing variant of a model, such as a faster serving option or a + larger context window. * MODEL_PRICING_TIER_STANDARD: Default + pricing. * MODEL_PRICING_TIER_FAST_MODE: Faster, higher-priority + serving at a premium price. * MODEL_PRICING_TIER_EXTENDED_1M: + Pricing for the 1M-token context window. * + MODEL_PRICING_TIER_EXTENDED_272K: Pricing for the 272K-token + context window. * MODEL_PRICING_TIER_BYOK: Pricing when using + your own model API key. Known values are: + "MODEL_PRICING_TIER_UNSPECIFIED", "MODEL_PRICING_TIER_STANDARD", + "MODEL_PRICING_TIER_FAST_MODE", "MODEL_PRICING_TIER_EXTENDED_1M", + "MODEL_PRICING_TIER_EXTENDED_272K", and + "MODEL_PRICING_TIER_BYOK". + } + ] }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: @@ -262665,9 +263995,11 @@ async def create_model_router( body = { "description": "str", # Optional. Model router description. "fallback_models": [ - "str" # Optional. Fallback models. + "str" # Optional. At least one fallback model is required; order + defines failover priority. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -262686,7 +264018,9 @@ async def create_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ] } @@ -262837,9 +264171,11 @@ async def create_model_router( body = { "description": "str", # Optional. Model router description. "fallback_models": [ - "str" # Optional. Fallback models. + "str" # Optional. At least one fallback model is required; order + defines failover priority. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -262858,7 +264194,9 @@ async def create_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ] } @@ -263503,7 +264841,8 @@ async def update_model_router( "fallback_models": [ {} # Optional. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -263522,7 +264861,9 @@ async def update_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ], "uuid": "str" # Optional. Model router id. } @@ -263681,7 +265022,8 @@ async def update_model_router( "fallback_models": [ {} # Optional. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -263700,7 +265042,9 @@ async def update_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ], "uuid": "str" # Optional. Model router id. } @@ -267370,6 +268714,10 @@ async def list_agents_by_openai_key( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -267447,16 +268795,14 @@ async def list_agents_by_openai_key( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -267465,10 +268811,10 @@ async def list_agents_by_openai_key( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -269581,6 +270927,10 @@ async def list_workspaces(self, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. + Whether this agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. + Whether this agent can use the built-in web_search tool. "workspace": ... } ], @@ -269656,14 +271006,14 @@ async def list_workspaces(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -269672,9 +271022,10 @@ async def list_workspaces(self, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -271028,6 +272379,10 @@ async def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -271095,14 +272450,13 @@ async def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -271111,9 +272465,10 @@ async def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -272376,6 +273731,10 @@ async def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -272443,14 +273802,13 @@ async def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -272459,9 +273817,10 @@ async def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -273726,6 +275085,10 @@ async def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -273793,14 +275156,13 @@ async def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -273809,9 +275171,10 @@ async def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -275158,6 +276521,10 @@ async def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -275225,14 +276592,13 @@ async def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -275241,9 +276607,10 @@ async def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -276593,6 +277960,10 @@ async def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -276660,14 +278031,13 @@ async def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -276676,9 +278046,10 @@ async def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -277944,6 +279315,10 @@ async def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -278011,14 +279386,13 @@ async def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -278027,9 +279401,10 @@ async def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -279297,6 +280672,10 @@ async def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -279364,14 +280743,13 @@ async def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -279380,9 +280758,10 @@ async def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -280794,6 +282173,10 @@ async def list_agents_by_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -280871,16 +282254,14 @@ async def list_agents_by_workspace( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -280889,10 +282270,10 @@ async def list_agents_by_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -282274,6 +283655,10 @@ async def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -282341,14 +283726,13 @@ async def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -282357,9 +283741,10 @@ async def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -283625,6 +285010,10 @@ async def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -283692,14 +285081,13 @@ async def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -283708,9 +285096,10 @@ async def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -284979,6 +286368,10 @@ async def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -285046,14 +286439,13 @@ async def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -285062,9 +286454,10 @@ async def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -285318,12 +286711,12 @@ async def list_evaluation_test_cases_by_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -285331,8 +286724,9 @@ async def list_evaluation_test_cases_by_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. "evaluation_scope": diff --git a/src/pydo/operations/_operations.py b/src/pydo/operations/_operations.py index 493a7c47..d587a331 100644 --- a/src/pydo/operations/_operations.py +++ b/src/pydo/operations/_operations.py @@ -15203,6 +15203,34 @@ def build_genai_delete_model_evaluation_run_request( # pylint: disable=name-too return HttpRequest(method="DELETE", url=_url, headers=_headers, **kwargs) +def build_genai_update_model_evaluation_run_request( # pylint: disable=name-too-long + eval_run_uuid: str, **kwargs: Any +) -> HttpRequest: + _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {}) + + content_type: Optional[str] = kwargs.pop( + "content_type", _headers.pop("Content-Type", None) + ) + accept = _headers.pop("Accept", "application/json") + + # Construct URL + _url = "/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}" + path_format_arguments = { + "eval_run_uuid": _SERIALIZER.url("eval_run_uuid", eval_run_uuid, "str"), + } + + _url: str = _url.format(**path_format_arguments) # type: ignore + + # Construct headers + if content_type is not None: + _headers["Content-Type"] = _SERIALIZER.header( + "content_type", content_type, "str" + ) + _headers["Accept"] = _SERIALIZER.header("accept", accept, "str") + + return HttpRequest(method="PATCH", url=_url, headers=_headers, **kwargs) + + def build_genai_cancel_model_evaluation_run_request( # pylint: disable=name-too-long eval_run_uuid: str, **kwargs: Any ) -> HttpRequest: @@ -223795,8 +223823,14 @@ def list_agents( "user_id": "str", # Optional. Id of user that created the agent. "uuid": "str", # Optional. Unique agent id. - "version_hash": "str" # Optional. The latest version of the + "version_hash": "str", # Optional. The latest version of the agent. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool to retrieve content from public web + pages. + "web_search_enabled": bool # Optional. Whether this agent + can use the built-in web_search tool to search the public web for current + information. } ], "links": { @@ -223978,6 +224012,10 @@ def create_agent( "str" # Optional. Agent tag to organize related resources. ], "thinking_token_budget": 0, # Optional. + "web_fetch_enabled": bool, # Optional. Whether the agent can use the + built-in web_fetch tool to retrieve content from public web pages. + "web_search_enabled": bool, # Optional. Whether the agent can use the + built-in web_search tool to search the public web for current information. "workspace_uuid": "str" # Optional. Identifier for the workspace. } @@ -225008,6 +225046,10 @@ def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -225084,14 +225126,14 @@ def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -225100,9 +225142,10 @@ def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -226251,6 +226294,10 @@ def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -226327,14 +226374,14 @@ def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -226343,9 +226390,10 @@ def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -226507,6 +226555,10 @@ def create_agent( "str" # Optional. Agent tag to organize related resources. ], "thinking_token_budget": 0, # Optional. + "web_fetch_enabled": bool, # Optional. Whether the agent can use the + built-in web_fetch tool to retrieve content from public web pages. + "web_search_enabled": bool, # Optional. Whether the agent can use the + built-in web_search tool to search the public web for current information. "workspace_uuid": "str" # Optional. Identifier for the workspace. } @@ -227537,6 +227589,10 @@ def create_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -227613,14 +227669,14 @@ def create_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -227629,9 +227685,10 @@ def create_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -229813,6 +229870,10 @@ def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -229889,14 +229950,14 @@ def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -229905,9 +229966,10 @@ def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -231059,6 +231121,10 @@ def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -231135,14 +231201,14 @@ def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -231151,9 +231217,10 @@ def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -232315,6 +232382,10 @@ def attach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -232391,14 +232462,14 @@ def attach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -232407,9 +232478,10 @@ def attach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -233674,6 +233746,10 @@ def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -233750,14 +233826,14 @@ def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -233766,9 +233842,10 @@ def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -234923,6 +235000,10 @@ def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -234999,14 +235080,14 @@ def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -235015,9 +235096,10 @@ def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -236183,6 +236265,10 @@ def update_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -236259,14 +236345,14 @@ def update_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -236275,9 +236361,10 @@ def update_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -237517,6 +237604,10 @@ def detach_agent_function( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -237593,14 +237684,14 @@ def detach_agent_function( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -237609,9 +237700,10 @@ def detach_agent_function( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -238852,6 +238944,10 @@ def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -238928,14 +239024,14 @@ def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -238944,9 +239040,10 @@ def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -240098,6 +240195,10 @@ def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -240174,14 +240275,14 @@ def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -240190,9 +240291,10 @@ def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -241350,6 +241452,10 @@ def attach_agent_guardrails( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -241426,14 +241532,14 @@ def attach_agent_guardrails( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -241442,9 +241548,10 @@ def attach_agent_guardrails( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -242682,6 +242789,10 @@ def detach_agent_guardrail( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -242758,14 +242869,14 @@ def detach_agent_guardrail( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -242774,9 +242885,10 @@ def detach_agent_guardrail( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -243994,6 +244106,10 @@ def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -244070,14 +244186,14 @@ def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -244086,9 +244202,10 @@ def attach_knowledge_bases(self, agent_uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -245309,6 +245426,10 @@ def attach_knowledge_base( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -245385,14 +245506,14 @@ def attach_knowledge_base( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -245401,9 +245522,10 @@ def attach_knowledge_base( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -246625,6 +246747,10 @@ def detach_knowledge_base( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -246701,14 +246827,14 @@ def detach_knowledge_base( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -246717,9 +246843,10 @@ def detach_knowledge_base( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -248586,6 +248713,10 @@ def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -248662,14 +248793,14 @@ def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -248678,9 +248809,10 @@ def get_agent(self, uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -248953,7 +249085,13 @@ def update_agent( "top_p": 0.0, # Optional. Defines the cumulative probability threshold for word selection, specified as a number between 0 and 1. Higher values allow for more diverse outputs, while lower values ensure focused and coherent responses. - "uuid": "str" # Optional. Unique agent id. + "uuid": "str", # Optional. Unique agent id. + "web_fetch_enabled": bool, # Optional. Optional. Set to true to let the + agent use the built-in web_fetch tool to retrieve content from public web pages, + or false to disable it. + "web_search_enabled": bool # Optional. Optional. Set to true to let the + agent use the built-in web_search tool to search the public web for current + information, or false to disable it. } # response body for status code(s): 200 @@ -249983,6 +250121,10 @@ def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -250059,14 +250201,14 @@ def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -250075,9 +250217,10 @@ def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -251229,6 +251372,10 @@ def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -251305,14 +251452,14 @@ def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -251321,9 +251468,10 @@ def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -251511,7 +251659,13 @@ def update_agent( "top_p": 0.0, # Optional. Defines the cumulative probability threshold for word selection, specified as a number between 0 and 1. Higher values allow for more diverse outputs, while lower values ensure focused and coherent responses. - "uuid": "str" # Optional. Unique agent id. + "uuid": "str", # Optional. Unique agent id. + "web_fetch_enabled": bool, # Optional. Optional. Set to true to let the + agent use the built-in web_fetch tool to retrieve content from public web pages, + or false to disable it. + "web_search_enabled": bool # Optional. Optional. Set to true to let the + agent use the built-in web_search tool to search the public web for current + information, or false to disable it. } # response body for status code(s): 200 @@ -252541,6 +252695,10 @@ def update_agent( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -252617,14 +252775,14 @@ def update_agent( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -252633,9 +252791,10 @@ def update_agent( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -253868,6 +254027,10 @@ def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -253944,14 +254107,14 @@ def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -253960,9 +254123,10 @@ def delete_agent(self, uuid: str, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -255241,6 +255405,10 @@ def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Child agents. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -255318,16 +255486,14 @@ def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -255336,10 +255502,10 @@ def get_agent_children(self, uuid: str, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -256591,6 +256757,10 @@ def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -256667,14 +256837,14 @@ def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -256683,9 +256853,10 @@ def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -257837,6 +258008,10 @@ def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -257913,14 +258088,14 @@ def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -257929,9 +258104,10 @@ def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -259089,6 +259265,10 @@ def update_agent_deployment_visibility( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. An Agent. + "web_fetch_enabled": bool, # Optional. Whether this agent can use + the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent can use + the built-in web_search tool. "workspace": { "agents": [ ... @@ -259165,14 +259345,14 @@ def update_agent_deployment_visibility( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -259181,9 +259361,10 @@ def update_agent_deployment_visibility( "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -262042,6 +262223,10 @@ def list_agents_by_anthropic_key( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -262119,16 +262304,14 @@ def list_agents_by_anthropic_key( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -262137,10 +262320,10 @@ def list_agents_by_anthropic_key( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -262423,8 +262606,8 @@ def list_custom_models( when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the - custom model import or processing failed. + "error_message": "str", # Optional. User-facing reason the + most recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., @@ -262602,6 +262785,8 @@ def import_custom_model( # JSON input template you can fill out and use as your body input. body = { + "accept_hf_token_storage": bool, # Optional. Whether the caller accepts + storage of their HuggingFace token for gated model access. "accept_terms_and_conditions": bool, # Optional. Whether the caller accepts the terms and conditions for importing this model. "description": "str", # Optional. Description of the model. @@ -262688,8 +262873,8 @@ def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -262844,8 +263029,8 @@ def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -262941,6 +263126,8 @@ def import_custom_model( # JSON input template you can fill out and use as your body input. body = { + "accept_hf_token_storage": bool, # Optional. Whether the caller accepts + storage of their HuggingFace token for gated model access. "accept_terms_and_conditions": bool, # Optional. Whether the caller accepts the terms and conditions for importing this model. "description": "str", # Optional. Description of the model. @@ -263027,8 +263214,8 @@ def import_custom_model( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -263251,8 +263438,8 @@ def get_custom_model(self, uuid: str, **kwargs: Any) -> JSON: "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -263543,14 +263730,16 @@ def update_custom_model_metadata( body = { "description": "str", # Optional. "input_modalities": [ - "str" # Optional. Input modalities supported (e.g., text, image). + "str" # Optional. Optional new input modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "license": "str", # Optional. License under which the model is distributed. + "license": "str", # Optional. "name": "str", # Optional. "output_modalities": [ - "str" # Optional. Output modalities supported (e.g., text, image). + "str" # Optional. Optional new output modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "parameters": "str", # Optional. Number of parameters in the model. + "parameters": "str", # Optional. "tags": { "tags": [ "str" # Optional. List of tag strings. @@ -263596,8 +263785,8 @@ def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -263730,8 +263919,8 @@ def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -263823,14 +264012,16 @@ def update_custom_model_metadata( body = { "description": "str", # Optional. "input_modalities": [ - "str" # Optional. Input modalities supported (e.g., text, image). + "str" # Optional. Optional new input modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "license": "str", # Optional. License under which the model is distributed. + "license": "str", # Optional. "name": "str", # Optional. "output_modalities": [ - "str" # Optional. Output modalities supported (e.g., text, image). + "str" # Optional. Optional new output modalities for the model + (replaces existing list when non-empty). Spaces-imported models only. ], - "parameters": "str", # Optional. Number of parameters in the model. + "parameters": "str", # Optional. "tags": { "tags": [ "str" # Optional. List of tag strings. @@ -263876,8 +264067,8 @@ def update_custom_model_metadata( "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the model was created. "description": "str", # Optional. Description of the custom model. - "error_message": "str", # Optional. Error message if the custom - model import or processing failed. + "error_message": "str", # Optional. User-facing reason the most + recent import failed; empty otherwise. "file_count": 0, # Optional. Number of files in the model. "input_modalities": [ "str" # Optional. Input modalities supported (e.g., text, @@ -264947,12 +265138,12 @@ def list_evaluation_metrics(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # Optional. - Timestamp when the custom metric was created. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. - When set, the custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; ignored on - create/update requests. + When set, the custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not @@ -264960,8 +265151,9 @@ def list_evaluation_metrics(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # Optional. - Timestamp when the custom metric was last updated. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # @@ -265117,20 +265309,19 @@ def create_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str" # Optional. @@ -265154,20 +265345,21 @@ def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -265256,20 +265448,21 @@ def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -265336,20 +265529,19 @@ def create_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str" # Optional. @@ -265373,20 +265565,21 @@ def create_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -265556,20 +265749,19 @@ def update_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str", # Optional. @@ -265594,20 +265786,21 @@ def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -265699,20 +265892,21 @@ def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -265784,20 +265978,19 @@ def update_custom_evaluation_metric( # JSON input template you can fill out and use as your body input. body = { "config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the - custom metric was created. Server-assigned; ignored on create/update - requests. + "created_at": "2020-02-20 00:00:00", # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; ignored on create/update - requests. + "updated_at": "2020-02-20 00:00:00" # Optional. Configuration for a + custom model-evaluation metric scored by an LLM judge. Prompt and model + response are always included in the judge context. }, "description": "str", # Optional. "metric_name": "str", # Optional. @@ -265822,20 +266015,21 @@ def update_custom_evaluation_metric( "METRIC_CATEGORY_USER_OUTCOMES", "METRIC_CATEGORY_SAFETY_AND_SECURITY", "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { - "created_at": "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. Server-assigned; ignored on - create/update requests. + "created_at": "2020-02-20 00:00:00", # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, - the custom metric has been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update requests. + the custom metric is soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not sent to the judge. "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). - "updated_at": "2020-02-20 00:00:00" # Optional. Timestamp - when the custom metric was last updated. Server-assigned; ignored on - create/update requests. + "updated_at": "2020-02-20 00:00:00" # Optional. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. @@ -266369,6 +266563,8 @@ def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> JSON: "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -266378,6 +266574,14 @@ def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> JSON: metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -266387,6 +266591,8 @@ def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> JSON: "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID (built-in or + custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", "METRIC_VALUE_TYPE_NUMBER", @@ -266395,6 +266601,13 @@ def get_evaluation_run(self, evaluation_run_uuid: str, **kwargs: Any) -> JSON: a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -266555,6 +266768,8 @@ def get_evaluation_run_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -266564,6 +266779,14 @@ def get_evaluation_run_results( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -266573,6 +266796,8 @@ def get_evaluation_run_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID (built-in or + custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", "METRIC_VALUE_TYPE_NUMBER", @@ -266581,6 +266806,13 @@ def get_evaluation_run_results( a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -266644,6 +266876,9 @@ def get_evaluation_run_results( calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # + Optional. Metric UUID (built-in or custom); stable key + for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values @@ -266654,6 +266889,16 @@ def get_evaluation_run_results( Optional. The value of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # + Optional. Default value is + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known + values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -266856,6 +267101,9 @@ def get_evaluation_run_results( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -266865,6 +267113,14 @@ def get_evaluation_run_results( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -267021,6 +267277,9 @@ def get_evaluation_run_prompt_results( calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. + Metric UUID (built-in or custom); stable key for results UI + and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -267031,6 +267290,15 @@ def get_evaluation_run_prompt_results( value of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values + are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -267234,6 +267502,8 @@ def get_evaluation_run_prompt_results( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -267243,6 +267513,14 @@ def get_evaluation_run_prompt_results( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -267422,12 +267700,12 @@ def list_evaluation_test_cases(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -267435,8 +267713,9 @@ def list_evaluation_test_cases(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. Alternative way of authentication for internal usage only - @@ -267926,6 +268205,9 @@ def list_evaluation_runs_by_test_case( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -267935,6 +268217,14 @@ def list_evaluation_runs_by_test_case( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -267944,6 +268234,8 @@ def list_evaluation_runs_by_test_case( "error_description": "str", # Optional. Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric UUID + (built-in or custom); stable key for results UI and aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -267953,6 +268245,14 @@ def list_evaluation_runs_by_test_case( metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. Default + value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". Outcome of + scoring a single metric for one prompt or span. Known values are: + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. }, @@ -268135,12 +268435,12 @@ def get_evaluation_test_case( "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -268148,8 +268448,9 @@ def get_evaluation_test_case( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", @@ -274354,12 +274655,12 @@ def list_model_evaluation_metrics(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # Optional. - Timestamp when the custom metric was created. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. "deleted_at": "2020-02-20 00:00:00", # Optional. - When set, the custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; ignored on - create/update requests. + When set, the custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required and is not @@ -274367,8 +274668,9 @@ def list_model_evaluation_metrics(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # Optional. - Timestamp when the custom metric was last updated. Server-assigned; - ignored on create/update requests. + Configuration for a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included in the judge + context. }, "description": "str", # Optional. List of model evaluation metrics. @@ -274519,18 +274821,48 @@ def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: response == { "presets": [ { + "candidate_inference_config": { + "max_tokens": 0, # Optional. Inference configuration + for the candidate model during evaluation. + "stop_token": "str", # Optional. Inference + configuration for the candidate model during evaluation. + "system_prompt": "str", # Optional. Inference + configuration for the candidate model during evaluation. + "temperature": 0.0 # Optional. Inference + configuration for the candidate model during evaluation. + }, + "candidate_model_name": "str", # Optional. Model slug used + to call the candidate model API. Empty when the CANDIDATE section was not + saved. + "candidate_model_source": + "CANDIDATE_MODEL_SOURCE_SERVERLESS", # Optional. Default value is + "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether inference runs against the + serverless platform, a dedicated deployment, or a model router. Known + values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the + candidate model stored on this preset. Empty when the CANDIDATE section + was not saved. For DEDICATED candidates this is the dedicated inference + deployment UUID. + "candidate_system_prompt": "str", # Optional. System prompt + / instructions to send to the candidate model. Empty when the + SYSTEM_PROMPT section was not saved (check ``saved_sections``"" ). "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the preset was created. - "dataset_name": "str", # Optional. List of explicitly saved - evaluation presets (reusable configs). - "dataset_uuid": "str", # Optional. Dataset used for - evaluation. + "dataset_name": "str", # Optional. Display name of the + dataset stored on this preset. Empty when the DATASET section was not + saved or the dataset no longer exists. + "dataset_uuid": "str", # Optional. UUID of the dataset + stored on this preset. Empty when the DATASET section was not saved + (check ``saved_sections``"" ). "eval_preset_uuid": "str", # Optional. UUID of the evaluation preset. - "judge_model_name": "str", # Optional. List of explicitly - saved evaluation presets (reusable configs). - "judge_model_uuid": "str", # Optional. Judge model used to - score responses. + "judge_model_name": "str", # Optional. Display name of the + judge model stored on this preset. Empty when the JUDGE section was not + saved or the model no longer exists. + "judge_model_uuid": "str", # Optional. UUID of the judge + model stored on this preset. Empty when the JUDGE section was not saved + (check ``saved_sections``"" ). "metrics": [ { "associated_presets": [ @@ -274551,12 +274883,12 @@ def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -274564,11 +274896,13 @@ def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. Default value is "EVALUATION_SCOPE_UNSPECIFIED". Scope that determines whether a @@ -274580,18 +274914,22 @@ def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: "inverted": bool, # Optional. If true, the metric is inverted, meaning that a lower value is better. "is_metric_goal": bool, # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_name": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_rank": 0, # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_type": "METRIC_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_TYPE_UNSPECIFIED". Known values are: "METRIC_TYPE_UNSPECIFIED", "METRIC_TYPE_GENERAL_QUALITY", "METRIC_TYPE_RAG_AND_TOOL", "METRIC_TYPE_MODEL_QUALITY", and "METRIC_TYPE_MODEL_SAFETY". "metric_uuid": "str", # Optional. Metrics - selected for this preset. + selected for this preset. Empty when the METRICS section was not + saved. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -274611,6 +274949,14 @@ def list_model_evaluation_presets(self, **kwargs: Any) -> JSON: } ], "name": "str", # Optional. Name of the evaluation preset. + "saved_sections": [ + "str" # Optional. Sections of the inline evaluation + config that were persisted when this preset was created. Use this to + tell "section was saved with an empty value" apart from "section was + not saved" "u2014 scalar fields like ``dataset_uuid`` or + ``candidate_system_prompt`` are always emitted as the empty string + when the section was not saved. + ], "star_metric": { "metric_uuid": "str", # Optional. List of explicitly saved evaluation presets (reusable configs). @@ -274735,17 +275081,45 @@ def get_model_evaluation_preset(self, eval_preset_uuid: str, **kwargs: Any) -> J # response body for status code(s): 200 response == { "preset": { + "candidate_inference_config": { + "max_tokens": 0, # Optional. Inference configuration for the + candidate model during evaluation. + "stop_token": "str", # Optional. Inference configuration for + the candidate model during evaluation. + "system_prompt": "str", # Optional. Inference configuration + for the candidate model during evaluation. + "temperature": 0.0 # Optional. Inference configuration for + the candidate model during evaluation. + }, + "candidate_model_name": "str", # Optional. Model slug used to call + the candidate model API. Empty when the CANDIDATE section was not saved. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model stored on this preset. Empty when the CANDIDATE section was not saved. + For DEDICATED candidates this is the dedicated inference deployment UUID. + "candidate_system_prompt": "str", # Optional. System prompt / + instructions to send to the candidate model. Empty when the SYSTEM_PROMPT + section was not saved (check ``saved_sections``"" ). "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the preset was created. - "dataset_name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. - "dataset_uuid": "str", # Optional. Dataset used for evaluation. + "dataset_name": "str", # Optional. Display name of the dataset + stored on this preset. Empty when the DATASET section was not saved or the + dataset no longer exists. + "dataset_uuid": "str", # Optional. UUID of the dataset stored on + this preset. Empty when the DATASET section was not saved (check + ``saved_sections``"" ). "eval_preset_uuid": "str", # Optional. UUID of the evaluation preset. - "judge_model_name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. - "judge_model_uuid": "str", # Optional. Judge model used to score - responses. + "judge_model_name": "str", # Optional. Display name of the judge + model stored on this preset. Empty when the JUDGE section was not saved or + the model no longer exists. + "judge_model_uuid": "str", # Optional. UUID of the judge model + stored on this preset. Empty when the JUDGE section was not saved (check + ``saved_sections``"" ). "metrics": [ { "associated_presets": [ @@ -274765,12 +275139,12 @@ def get_model_evaluation_preset(self, eval_preset_uuid: str, **kwargs: Any) -> J "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -274778,11 +275152,12 @@ def get_model_evaluation_preset(self, eval_preset_uuid: str, **kwargs: Any) -> J "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "evaluation_scope": "EVALUATION_SCOPE_UNSPECIFIED", # Optional. Default value is "EVALUATION_SCOPE_UNSPECIFIED". Scope that determines whether a metric belongs to agent evaluation or model @@ -274793,18 +275168,18 @@ def get_model_evaluation_preset(self, eval_preset_uuid: str, **kwargs: Any) -> J "inverted": bool, # Optional. If true, the metric is inverted, meaning that a lower value is better. "is_metric_goal": bool, # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_name": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_rank": 0, # Optional. Metrics selected for - this preset. + this preset. Empty when the METRICS section was not saved. "metric_type": "METRIC_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_TYPE_UNSPECIFIED". Known values are: "METRIC_TYPE_UNSPECIFIED", "METRIC_TYPE_GENERAL_QUALITY", "METRIC_TYPE_RAG_AND_TOOL", "METRIC_TYPE_MODEL_QUALITY", and "METRIC_TYPE_MODEL_SAFETY". "metric_uuid": "str", # Optional. Metrics selected - for this preset. + for this preset. Empty when the METRICS section was not saved. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: "METRIC_VALUE_TYPE_UNSPECIFIED", @@ -274823,11 +275198,29 @@ def get_model_evaluation_preset(self, eval_preset_uuid: str, **kwargs: Any) -> J } ], "name": "str", # Optional. Name of the evaluation preset. + "saved_sections": [ + "str" # Optional. Sections of the inline evaluation config + that were persisted when this preset was created. Use this to tell + "section was saved with an empty value" apart from "section was not + saved" "u2014 scalar fields like ``dataset_uuid`` or + ``candidate_system_prompt`` are always emitted as the empty string when + the section was not saved. + ], "star_metric": { "metric_uuid": "str", # Optional. Model Evaluation Preset - - a saved, reusable configuration for model evaluation runs. + a saved, reusable configuration for model evaluation runs. Each section + (dataset, judge, metrics, candidate, system prompt) is independent and + may be empty; sections the preset omits must be supplied inline on the + run that references it. Use ``saved_sections`` to tell "section saved + with empty value" apart from "section not saved at all" "u2014 each + section's scalar fields are left empty when the section was not saved. "name": "str", # Optional. Model Evaluation Preset - a - saved, reusable configuration for model evaluation runs. + saved, reusable configuration for model evaluation runs. Each section + (dataset, judge, metrics, candidate, system prompt) is independent and + may be empty; sections the preset omits must be supplied inline on the + run that references it. Use ``saved_sections`` to tell "section saved + with empty value" apart from "section not saved at all" "u2014 each + section's scalar fields are left empty when the section was not saved. "success_threshold": 0.0, # Optional. The success threshold for the star metric. This is a value that the metric must reach to be considered successful. @@ -275321,6 +275714,19 @@ def create_model_evaluation_run( ], "name": "str", # Optional. "preset_name": "str", # Optional. + "preset_save_sections": [ + "str" # Optional. Which sections of this run's resolved + configuration to persist as a reusable preset. Each selected section saves + only its own fields; the remaining sections stay empty on the preset and must + be supplied inline on future runs that reference it. Empty means do not save + a preset (unless the deprecated ``save_as_preset`` boolean is true, in which + case all sections are saved). Ignored when ``eval_preset_uuid`` is set. Use + ``preset_name`` to label the saved preset. + ], + "save_as_preset": bool, # Optional. Deprecated: use + ``preset_save_sections``. When ``true`` and ``preset_save_sections`` is empty, + all five sections of the resolved configuration are saved as a reusable preset + (legacy behavior). Ignored when ``eval_preset_uuid`` is set. "source": "str", # Optional. Source of the run creation (api, sdk, cli). "star_metric": { "metric_uuid": "str", # Optional. @@ -275443,6 +275849,19 @@ def create_model_evaluation_run( ], "name": "str", # Optional. "preset_name": "str", # Optional. + "preset_save_sections": [ + "str" # Optional. Which sections of this run's resolved + configuration to persist as a reusable preset. Each selected section saves + only its own fields; the remaining sections stay empty on the preset and must + be supplied inline on future runs that reference it. Empty means do not save + a preset (unless the deprecated ``save_as_preset`` boolean is true, in which + case all sections are saved). Ignored when ``eval_preset_uuid`` is set. Use + ``preset_name`` to label the saved preset. + ], + "save_as_preset": bool, # Optional. Deprecated: use + ``preset_save_sections``. When ``true`` and ``preset_save_sections`` is empty, + all five sections of the resolved configuration are saved as a reusable preset + (legacy behavior). Ignored when ``eval_preset_uuid`` is set. "source": "str", # Optional. Source of the run creation (api, sdk, cli). "star_metric": { "metric_uuid": "str", # Optional. @@ -275615,6 +276034,8 @@ def get_model_evaluation_run( per-prompt evaluation results. "candidate_model_uuid": "str", # Optional. Paginated per-prompt evaluation results. + "candidate_routed_task": "str", # Optional. Paginated + per-prompt evaluation results. "ground_truth": "str", # Optional. Paginated per-prompt evaluation results. "input": "str", # Optional. The input query sent to the @@ -275625,6 +276046,9 @@ def get_model_evaluation_run( Error description if the metric could not be calculated. "metric_name": "str", # Optional. Metric name. + "metric_uuid": "str", # Optional. Metric + UUID (built-in or custom); stable key for results UI and + aggregation. "metric_value_type": "METRIC_VALUE_TYPE_UNSPECIFIED", # Optional. Default value is "METRIC_VALUE_TYPE_UNSPECIFIED". Known values are: @@ -275634,6 +276058,14 @@ def get_model_evaluation_run( of the metric as a number. "reasoning": "str", # Optional. Reasoning of the metric result. + "status": + "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", # Optional. + Default value is "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED". + Outcome of scoring a single metric for one prompt or span. Known + values are: "EVALUATION_METRIC_RESULT_STATUS_UNSPECIFIED", + "EVALUATION_METRIC_RESULT_STATUS_COMPLETED", + "EVALUATION_METRIC_RESULT_STATUS_FAILED", and + "EVALUATION_METRIC_RESULT_STATUS_SKIPPED". "string_value": "str" # Optional. The value of the metric as a string. } @@ -275699,12 +276131,12 @@ def get_model_evaluation_run( "METRIC_CATEGORY_CONTEXT_QUALITY", and "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", # - Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # - Optional. When set, the custom metric has been deleted and is no - longer available for use in evaluations. Server-assigned; ignored - on create/update requests. + Optional. When set, the custom metric is soft-deleted and must + not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth is not required @@ -275712,8 +276144,9 @@ def get_model_evaluation_run( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" # - Optional. Timestamp when the custom metric was last updated. - Server-assigned; ignored on create/update requests. + Optional. Configuration for a custom model-evaluation metric + scored by an LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Metrics selected for this evaluation. @@ -275773,14 +276206,24 @@ def get_model_evaluation_run( { "description": "str", # Optional. Per-metric aggregated pass/fail statistics. + "fail_count": 0, # Optional. Rows where the + metric failed to score or completed below the threshold. "fail_percent": 0.0, # Optional. Per-metric aggregated pass/fail statistics. "metric_name": "str", # Optional. Per-metric aggregated pass/fail statistics. "metric_uuid": "str", # Optional. Per-metric aggregated pass/fail statistics. - "pass_percent": 0.0 # Optional. Per-metric + "pass_count": 0, # Optional. Rows where the + metric completed and passed the configured threshold. + "pass_percent": 0.0, # Optional. Per-metric aggregated pass/fail statistics. + "skip_percent": 0.0, # Optional. Percentage + of rows that were skipped for this metric, computed as + skipped_count divided by the total rows the metric saw + (pass_count + fail_count + skipped_count). + "skipped_count": 0 # Optional. Rows where + the metric was not evaluated for this prompt. } ], "overall_score_percent": 0.0, # Optional. Aggregated result @@ -275791,24 +276234,39 @@ def get_model_evaluation_run( "metric_summaries": [ { "description": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "fail_count": 0, # + Optional. Rows where the metric failed to score or + completed below the threshold. "fail_percent": 0.0, - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. "metric_name": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. "metric_uuid": "str", - # Optional. Per-metric pass/fail for only this - model's prompts. - "pass_percent": 0.0 - # Optional. Per-metric pass/fail for only this - model's prompts. + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "pass_count": 0, # + Optional. Rows where the metric completed and passed + the configured threshold. + "pass_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts routed to this model. + "skip_percent": 0.0, + # Optional. Percentage of rows that were skipped for + this metric, computed as skipped_count divided by the + total rows the metric saw (pass_count + fail_count + + skipped_count). + "skipped_count": 0 # + Optional. Rows where the metric was not evaluated for + this prompt. } ], - "model_name": "str", # Optional. - Name/slug of the model (matches routed_model from results). + "model_name": "str", # Optional. The + underlying model these results are for, such as ``Llama 3.3 + Instruct (70B)``. "performance_metrics": { "candidate_latency": { "avg_e2e_latency_ms": @@ -275849,7 +276307,91 @@ def get_model_evaluation_run( } }, "prompt_count": 0 # Optional. Number - of prompts routed to this model. + of prompts in the run that were routed to this model. + } + ] + }, + "per_task_summaries": { + "summaries": [ + { + "metric_summaries": [ + { + "description": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "fail_count": 0, # + Optional. Rows where the metric failed to score or + completed below the threshold. + "fail_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "metric_name": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "metric_uuid": "str", + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "pass_count": 0, # + Optional. Rows where the metric completed and passed + the configured threshold. + "pass_percent": 0.0, + # Optional. Pass/fail rate for each metric, computed + over only the prompts in this task category. + "skip_percent": 0.0, + # Optional. Percentage of rows that were skipped for + this metric, computed as skipped_count divided by the + total rows the metric saw (pass_count + fail_count + + skipped_count). + "skipped_count": 0 # + Optional. Rows where the metric was not evaluated for + this prompt. + } + ], + "performance_metrics": { + "candidate_latency": { + "avg_e2e_latency_ms": + 0.0, # Optional. Average end-to-end latency across + all invocations. + "max_e2e_latency_ms": + 0.0, # Optional. Maximum end-to-end latency + observed. + "min_e2e_latency_ms": + 0.0, # Optional. Minimum end-to-end latency + observed. + "p50_latency_ms": + 0.0, # Optional. P50 (median) latency. + "p90_latency_ms": + 0.0, # Optional. P90 latency. + "p95_latency_ms": 0.0 + # Optional. P95 latency. + }, + "token_usage": { + "total_candidate_input_tokens": "str", # Optional. + All performance metrics are for the candidate model + unless noted otherwise. + "total_candidate_output_tokens": "str", # Optional. + All performance metrics are for the candidate model + unless noted otherwise. + "total_candidate_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_input_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_output_tokens": "str", # Optional. All + performance metrics are for the candidate model + unless noted otherwise. + "total_judge_tokens": + "str" # Optional. All performance metrics are for + the candidate model unless noted otherwise. + } + }, + "prompt_count": 0, # Optional. + Number of prompts in the run that were classified into this + task category. + "task_name": "str" # Optional. The + routing task category these results are for, such as ``Coding + & brainstorming`` or ``Summarization``. } ] }, @@ -276172,6 +276714,374 @@ def delete_model_evaluation_run(self, eval_run_uuid: str, **kwargs: Any) -> JSON return cast(JSON, deserialized) # type: ignore + @overload + def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[JSON] = None, + *, + content_type: str = "application/json", + **kwargs: Any, + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Default value is None. + :type body: JSON + :keyword content_type: Body Parameter content-type. Content type parameter for JSON body. + Default value is "application/json". + :paramtype content_type: str + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # JSON input template you can fill out and use as your body input. + body = { + "eval_run_uuid": "str", # Optional. UUID of the model evaluation run to + update. Returned by ``CreateModelEvaluationRun`` and listed via + ``ListModelEvaluationRuns``. + "name": "str" # Optional. Optional new display name for the evaluation run + (max 255 characters). + } + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + + @overload + def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[IO[bytes]] = None, + *, + content_type: str = "application/json", + **kwargs: Any, + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Default value is None. + :type body: IO[bytes] + :keyword content_type: Body Parameter content-type. Content type parameter for binary body. + Default value is "application/json". + :paramtype content_type: str + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + + @distributed_trace + def update_model_evaluation_run( + self, + eval_run_uuid: str, + body: Optional[Union[JSON, IO[bytes]]] = None, + **kwargs: Any, + ) -> JSON: + # pylint: disable=line-too-long + """Update Model Evaluation Run. + + To update a model evaluation run's display name, send a PATCH request to + ``/v2/gen-ai/model_evaluation_runs/{eval_run_uuid}``. + + :param eval_run_uuid: UUID of the model evaluation run to update. Returned by + ``CreateModelEvaluationRun`` + and listed via ``ListModelEvaluationRuns``. Required. + :type eval_run_uuid: str + :param body: Is either a JSON type or a IO[bytes] type. Default value is None. + :type body: JSON or IO[bytes] + :return: JSON object + :rtype: JSON + :raises ~azure.core.exceptions.HttpResponseError: + + Example: + .. code-block:: python + + # JSON input template you can fill out and use as your body input. + body = { + "eval_run_uuid": "str", # Optional. UUID of the model evaluation run to + update. Returned by ``CreateModelEvaluationRun`` and listed via + ``ListModelEvaluationRuns``. + "name": "str" # Optional. Optional new display name for the evaluation run + (max 255 characters). + } + + # response body for status code(s): 200 + response == { + "run": { + "candidate_model_name": "str", # Optional. Name of the candidate + model being evaluated. + "candidate_model_source": "CANDIDATE_MODEL_SOURCE_SERVERLESS", # + Optional. Default value is "CANDIDATE_MODEL_SOURCE_SERVERLESS". Whether + inference runs against the serverless platform, a dedicated deployment, or a + model router. Known values are: "CANDIDATE_MODEL_SOURCE_SERVERLESS", + "CANDIDATE_MODEL_SOURCE_DEDICATED", and "CANDIDATE_MODEL_SOURCE_ROUTER". + "candidate_model_uuid": "str", # Optional. UUID of the candidate + model being evaluated. + "created_at": "2020-02-20 00:00:00", # Optional. Timestamp when the + run was created. + "dataset_name": "str", # Optional. Name of the dataset used for + evaluation. + "dataset_uuid": "str", # Optional. UUID of the dataset used for + evaluation. + "eval_run_uuid": "str", # Optional. UUID of the evaluation run. + "judge_model_name": "str", # Optional. Model Evaluation Run Summary + - lightweight view used in run history list. + "judge_model_uuid": "str", # Optional. Judge model used to score + responses. + "name": "str", # Optional. Name of the evaluation run. + "progress": { + "candidate_rows_evaluated": 0, # Optional. Dataset rows + whose candidate model call has completed (success or failure). + "judge_rows_evaluated": 0, # Optional. Candidate-success + rows the judge has finished (scored or skipped). Caps at the number of + candidate successes, which may be below total_rows. + "total_rows": 0 # Optional. Total dataset rows for the run, + sourced from the evaluation dataset. + }, + "status": "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED" # Optional. + Default value is "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED". Model Evaluation + Run Statuses. Known values are: "MODEL_EVALUATION_RUN_STATUS_UNSPECIFIED", + "MODEL_EVALUATION_RUN_QUEUED", "MODEL_EVALUATION_RUN_RUNNING_DATASET", + "MODEL_EVALUATION_RUN_EVALUATING_RESULTS", "MODEL_EVALUATION_RUN_CANCELLING", + "MODEL_EVALUATION_RUN_CANCELLED", "MODEL_EVALUATION_RUN_SUCCESSFUL", + "MODEL_EVALUATION_RUN_PARTIALLY_SUCCESSFUL", and + "MODEL_EVALUATION_RUN_FAILED". + } + } + # response body for status code(s): 404 + response == { + "id": "str", # A short identifier corresponding to the HTTP status code + returned. For example, the ID for a response returning a 404 status code would + be "not_found.". Required. + "message": "str", # A message providing additional information about the + error, including details to help resolve it when possible. Required. + "request_id": "str" # Optional. Optionally, some endpoints may include a + request ID that should be provided when reporting bugs or opening support + tickets to help identify the issue. + } + """ + error_map: MutableMapping[int, Type[HttpResponseError]] = { + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + 401: cast( + Type[HttpResponseError], + lambda response: ClientAuthenticationError(response=response), + ), + 429: HttpResponseError, + 500: HttpResponseError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {}) + _params = kwargs.pop("params", {}) or {} + + content_type: Optional[str] = kwargs.pop( + "content_type", _headers.pop("Content-Type", None) + ) + cls: ClsType[JSON] = kwargs.pop("cls", None) + + content_type = content_type or "application/json" + _json = None + _content = None + if isinstance(body, (IOBase, bytes)): + _content = body + else: + if body is not None: + _json = body + else: + _json = None + + _request = build_genai_update_model_evaluation_run_request( + eval_run_uuid=eval_run_uuid, + content_type=content_type, + json=_json, + content=_content, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _stream = False + pipeline_response: PipelineResponse = ( + self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + ) + + response = pipeline_response.http_response + + if response.status_code not in [200, 404]: + if _stream: + response.read() # Load the body in memory and close the socket + map_error(status_code=response.status_code, response=response, error_map=error_map) # type: ignore + raise HttpResponseError(response=response) + + response_headers = {} + if response.status_code == 200: + response_headers["ratelimit-limit"] = self._deserialize( + "int", response.headers.get("ratelimit-limit") + ) + response_headers["ratelimit-remaining"] = self._deserialize( + "int", response.headers.get("ratelimit-remaining") + ) + response_headers["ratelimit-reset"] = self._deserialize( + "int", response.headers.get("ratelimit-reset") + ) + + if response.content: + deserialized = response.json() + else: + deserialized = None + + if response.status_code == 404: + response_headers["ratelimit-limit"] = self._deserialize( + "int", response.headers.get("ratelimit-limit") + ) + response_headers["ratelimit-remaining"] = self._deserialize( + "int", response.headers.get("ratelimit-remaining") + ) + response_headers["ratelimit-reset"] = self._deserialize( + "int", response.headers.get("ratelimit-reset") + ) + + if response.content: + deserialized = response.json() + else: + deserialized = None + + if cls: + return cls(pipeline_response, cast(JSON, deserialized), response_headers) # type: ignore + + return cast(JSON, deserialized) # type: ignore + @overload def cancel_model_evaluation_run( self, @@ -276771,6 +277681,27 @@ def list_models( "parent_uuid": "str", # Optional. Unique id of the model, this model is based on. "pricing": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing @@ -276789,8 +277720,22 @@ def list_models( million tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the model does not charge + separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs image tokens + separately (e.g. OpenAI gpt-image-2). Standard chat models leave + these at 0 and use input_price_per_million / output_price_per_million + instead. Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: @@ -277810,20 +278755,52 @@ def list_model_catalog( "availability": [ "str" # Optional. ], + "badges": [ + "str" # Optional. Badges for models. + ], "benchmark_score": {}, # Optional. Benchmark scores for this model, stored as arbitrary JSON. "capabilities": [ "str" # Optional. ], "context_window": "str", # Optional. Specs (flat). + "created_at": "2020-02-20 00:00:00", # Optional. RFC 3339 + timestamp indicating when the model was added to the catalog. "creator": "str", # Optional. Model creator/developer (e.g., "Meta", "Anthropic", "OpenAI"). + "hugging_face_id": "str", # Optional. The Hugging Face + repository ID (e.g. "meta-llama/Llama-3.3-70B-Instruct") the model is + based on, when applicable. Omitted for models not sourced from Hugging + Face. "id": "str", # Optional. Identity. + "max_output_tokens": "str", # Optional. The maximum number + of output tokens the model can generate in a single response. "model_id": "str", # Optional. Model identifier used for API calls (e.g., "llama3.1-70b-instruct"). "name": "str", # Optional. "parameter_count": 0.0, # Optional. "pricing": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing @@ -277842,13 +278819,30 @@ def list_model_catalog( million tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the model does not charge + separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs image tokens + separately (e.g. OpenAI gpt-image-2). Standard chat models leave + these at 0 and use input_price_per_million / output_price_per_million + instead. Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: "MODEL_PROVIDER_DIGITALOCEAN", "MODEL_PROVIDER_ANTHROPIC", and "MODEL_PROVIDER_OPENAI". + "scaled_pricing_enabled": bool, # Optional. True when this + model's pricing varies over time. Retrieve the model's details for the + full pricing schedule. False for models with fixed pricing. "short_description": "str", # Optional. "type": "str" # Optional. } @@ -277978,6 +278972,9 @@ def get_model_catalog_card( "availability": [ "str" # Optional. Detail view for GetModelCatalogCard. ], + "badges": [ + "str" # Optional. Badges for models. + ], "benchmark_score": {}, # Optional. Benchmark scores for this model, stored as arbitrary JSON. "capabilities": [ @@ -277996,7 +278993,12 @@ def get_model_catalog_card( "creator": "str", # Optional. Model creator/developer (e.g., "Meta", "Anthropic", "OpenAI"). "description": "str", # Optional. Card-specific. + "hugging_face_id": "str", # Optional. The Hugging Face repository ID + (e.g. "meta-llama/Llama-3.3-70B-Instruct") the model is based on, when + applicable. Omitted for models not sourced from Hugging Face. "id": "str", # Optional. Identity (same as Entry). + "max_output_tokens": "str", # Optional. The maximum number of output + tokens the model can generate in a single response. "modalities": { "input": [ "str" # Optional. Input/output modalities. @@ -278011,6 +279013,25 @@ def get_model_catalog_card( "parameter_count": 0.0, # Optional. Detail view for GetModelCatalogCard. "pricing": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models (token_type "cache + read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a 1-hour + lifetime. + "cache_write_5m_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a 5-minute + lifetime. + "image_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. Pricing + per million tokens (aligns with existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. Pricing + per million tokens (aligns with existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read input price + per single token. Equivalent to cache_read_input_price_per_million. "input_price_per_million": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). "output_price_per_million": 0.0, # Optional. Pricing per @@ -278028,8 +279049,344 @@ def get_model_catalog_card( tokens (aligns with existing ModelPrice pattern). "price_per_thousand_characters": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). - "price_per_video": 0.0 # Optional. Pricing per million + "price_per_video": 0.0, # Optional. Pricing per million tokens (aligns with existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price per + million reasoning tokens. 0 if the model does not charge separately for + reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. Per-million + token rates for models that bill text vs image tokens separately (e.g. + OpenAI gpt-image-2). Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million instead. Values align + with usage token_type / internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing per + million tokens (aligns with existing ModelPrice pattern). + }, + "pricing_detail": { + "variants": [ + { + "currency": "str", # Optional. Currency code + for this variant's rates (e.g. ``USD``"" ). + "label": "str", # Optional. Display name for + the variant (e.g. ``Standard``"" , ``Fast Mode``"" , ``1M + Context``"" ). + "mode": "MODEL_BILLING_MODE_UNSPECIFIED", # + Optional. Default value is "MODEL_BILLING_MODE_UNSPECIFIED". + Whether rates apply to real-time or batch requests. * + MODEL_BILLING_MODE_INTERACTIVE: Real-time request pricing. * + MODEL_BILLING_MODE_BATCH: Discounted pricing for asynchronous + batch requests. Known values are: + "MODEL_BILLING_MODE_UNSPECIFIED", + "MODEL_BILLING_MODE_INTERACTIVE", and "MODEL_BILLING_MODE_BATCH". + "prices": { + "cache_read_input_price_per_million": + 0.0, # Optional. Per-million cache read rate for standard + chat models (token_type "cache read input"). Multimodal + models use text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a + 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # Optional. + Price per million tokens written to the prompt cache with a + 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "image_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. + Cache read input price per single token. Equivalent to + cache_read_input_price_per_million. + "input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "output_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_audio": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "price_per_image": 0.0, # Optional. + Unit-based pricing for non-token models (e.g., Fal AI + image/video/audio generation, speech models). At most one of + these is typically populated per model. Token-based models + (chat, embeddings) leave all of these at 0 and populate + input_price_per_million / output_price_per_million instead. + "price_per_megapixel": 0.0, # + Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_second": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "price_per_thousand_characters": 0.0, + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "reasoning_price_per_million": 0.0, + # Optional. Price per million reasoning tokens. 0 if the + model does not charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing ModelPrice + pattern). + "text_input_price_per_million": 0.0, + # Optional. Per-million token rates for models that bill text + vs image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million instead. + Values align with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 + # Optional. Pricing per million tokens (aligns with existing + ModelPrice pattern). + }, + "scaled_pricing": { + "intervals": [ + { + "end_time": + "2020-02-20 00:00:00", # Optional. End of the + interval, exclusive. Always on the hour. + "prices": { + "cache_read_input_price_per_million": 0.0, # + Optional. Per-million cache read rate for + standard chat models (token_type "cache read + input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the + prompt cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the + prompt cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns + with existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_output_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "input_cache_read": 0.0, # Optional. Cache read + input price per single token. Equivalent to + cache_read_input_price_per_million. + "input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_audio": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "price_per_image": 0.0, # Optional. Unit-based + pricing for non-token models (e.g., Fal AI + image/video/audio generation, speech models). At + most one of these is typically populated per + model. Token-based models (chat, embeddings) + leave all of these at 0 and populate + input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, # Optional. Pricing + per million tokens (aligns with existing + ModelPrice pattern). + "price_per_second": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "price_per_thousand_characters": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_video": 0.0, # Optional. Pricing per + million tokens (aligns with existing ModelPrice + pattern). + "reasoning_price_per_million": 0.0, # Optional. + Price per million reasoning tokens. 0 if the + model does not charge separately for reasoning + tokens. + "text_cache_read_input_price_per_million": 0.0, + # Optional. Pricing per million tokens (aligns + with existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text + vs image tokens separately (e.g. OpenAI + gpt-image-2). Standard chat models leave these at + 0 and use input_price_per_million / + output_price_per_million instead. Values align + with usage token_type / internal/usage.Type + string values for each meter. + "text_output_price_per_million": 0.0 # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + }, + "start_time": + "2020-02-20 00:00:00" # Optional. Start of the + interval, inclusive. Always on the hour. + } + ], + "max_prices": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "input_cache_read": 0.0, # + Optional. Cache read input price per single token. + Equivalent to cache_read_input_price_per_million. + "input_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_audio": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_image": 0.0, # + Optional. Unit-based pricing for non-token models (e.g., + Fal AI image/video/audio generation, speech models). At + most one of these is typically populated per model. + Token-based models (chat, embeddings) leave all of these + at 0 and populate input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, + # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_second": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_thousand_characters": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price + per million reasoning tokens. 0 if the model does not + charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs + image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million + instead. Values align with usage token_type / + internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing + per million tokens (aligns with existing ModelPrice + pattern). + }, + "min_prices": { + "cache_read_input_price_per_million": 0.0, # Optional. + Per-million cache read rate for standard chat models + (token_type "cache read input"). Multimodal models use + text_cache_read_input_price_per_million / + image_cache_read_input_price_per_million instead. + "cache_write_1h_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 1-hour lifetime. + "cache_write_5m_input_price_per_million": 0.0, # + Optional. Price per million tokens written to the prompt + cache with a 5-minute lifetime. + "image_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "image_input_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "image_output_price_per_million": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "input_cache_read": 0.0, # + Optional. Cache read input price per single token. + Equivalent to cache_read_input_price_per_million. + "input_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "output_price_per_million": + 0.0, # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_audio": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_image": 0.0, # + Optional. Unit-based pricing for non-token models (e.g., + Fal AI image/video/audio generation, speech models). At + most one of these is typically populated per model. + Token-based models (chat, embeddings) leave all of these + at 0 and populate input_price_per_million / + output_price_per_million instead. + "price_per_megapixel": 0.0, + # Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_second": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "price_per_thousand_characters": 0.0, # Optional. + Pricing per million tokens (aligns with existing + ModelPrice pattern). + "price_per_video": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "reasoning_price_per_million": 0.0, # Optional. Price + per million reasoning tokens. 0 if the model does not + charge separately for reasoning tokens. + "text_cache_read_input_price_per_million": 0.0, # + Optional. Pricing per million tokens (aligns with + existing ModelPrice pattern). + "text_input_price_per_million": 0.0, # Optional. + Per-million token rates for models that bill text vs + image tokens separately (e.g. OpenAI gpt-image-2). + Standard chat models leave these at 0 and use + input_price_per_million / output_price_per_million + instead. Values align with usage token_type / + internal/usage.Type string values for each meter. + "text_output_price_per_million": 0.0 # Optional. Pricing + per million tokens (aligns with existing ModelPrice + pattern). + } + }, + "tier": "MODEL_PRICING_TIER_UNSPECIFIED" # + Optional. Default value is "MODEL_PRICING_TIER_UNSPECIFIED". A + pricing variant of a model, such as a faster serving option or a + larger context window. * MODEL_PRICING_TIER_STANDARD: Default + pricing. * MODEL_PRICING_TIER_FAST_MODE: Faster, higher-priority + serving at a premium price. * MODEL_PRICING_TIER_EXTENDED_1M: + Pricing for the 1M-token context window. * + MODEL_PRICING_TIER_EXTENDED_272K: Pricing for the 272K-token + context window. * MODEL_PRICING_TIER_BYOK: Pricing when using + your own model API key. Known values are: + "MODEL_PRICING_TIER_UNSPECIFIED", "MODEL_PRICING_TIER_STANDARD", + "MODEL_PRICING_TIER_FAST_MODE", "MODEL_PRICING_TIER_EXTENDED_1M", + "MODEL_PRICING_TIER_EXTENDED_272K", and + "MODEL_PRICING_TIER_BYOK". + } + ] }, "provider": "MODEL_PROVIDER_DIGITALOCEAN", # Optional. Default value is "MODEL_PROVIDER_DIGITALOCEAN". Known values are: @@ -278331,9 +279688,11 @@ def create_model_router( body = { "description": "str", # Optional. Model router description. "fallback_models": [ - "str" # Optional. Fallback models. + "str" # Optional. At least one fallback model is required; order + defines failover priority. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -278352,7 +279711,9 @@ def create_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ] } @@ -278503,9 +279864,11 @@ def create_model_router( body = { "description": "str", # Optional. Model router description. "fallback_models": [ - "str" # Optional. Fallback models. + "str" # Optional. At least one fallback model is required; order + defines failover priority. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -278524,7 +279887,9 @@ def create_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ] } @@ -279169,7 +280534,8 @@ def update_model_router( "fallback_models": [ {} # Optional. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -279188,7 +280554,9 @@ def update_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ], "uuid": "str" # Optional. Model router id. } @@ -279347,7 +280715,8 @@ def update_model_router( "fallback_models": [ {} # Optional. ], - "name": "str", # Optional. Model router name. + "name": "str", # Optional. Model router name: lowercase, at most 255 + characters, only a-z, 0-9, and hyphens. "policies": [ { "custom_task": { @@ -279366,7 +280735,9 @@ def update_model_router( } ], "regions": [ - "str" # Optional. Target regions for the router. + "str" # Optional. DEPRECATED: this field does not affect deployment + and model routers are always deployed to all regions. Must be omitted or set + to ["all"]. ], "uuid": "str" # Optional. Model router id. } @@ -283036,6 +284407,10 @@ def list_agents_by_openai_key( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -283113,16 +284488,14 @@ def list_agents_by_openai_key( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -283131,10 +284504,10 @@ def list_agents_by_openai_key( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -285245,6 +286618,10 @@ def list_workspaces(self, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. + Whether this agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. + Whether this agent can use the built-in web_search tool. "workspace": ... } ], @@ -285320,14 +286697,14 @@ def list_workspaces(self, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp when - the custom metric was created. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00", # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. Server-assigned; - ignored on create/update requests. + custom metric is soft-deleted and must not appear in + pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground @@ -285336,9 +286713,10 @@ def list_workspaces(self, **kwargs: Any) -> JSON: "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when the - custom metric was last updated. Server-assigned; - ignored on create/update requests. + "2020-02-20 00:00:00" # Optional. Configuration for + a custom model-evaluation metric scored by an LLM + judge. Prompt and model response are always included + in the judge context. }, "description": "str", # Optional. Evaluations. @@ -286692,6 +288070,10 @@ def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -286759,14 +288141,13 @@ def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -286775,9 +288156,10 @@ def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -288040,6 +289422,10 @@ def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -288107,14 +289493,13 @@ def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -288123,9 +289508,10 @@ def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -289390,6 +290776,10 @@ def create_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -289457,14 +290847,13 @@ def create_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -289473,9 +290862,10 @@ def create_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -290822,6 +292212,10 @@ def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -290889,14 +292283,13 @@ def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -290905,9 +292298,10 @@ def get_workspace(self, workspace_uuid: str, **kwargs: Any) -> JSON: "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -292257,6 +293651,10 @@ def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -292324,14 +293722,13 @@ def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -292340,9 +293737,10 @@ def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -293608,6 +295006,10 @@ def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -293675,14 +295077,13 @@ def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -293691,9 +295092,10 @@ def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -294961,6 +296363,10 @@ def update_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -295028,14 +296434,13 @@ def update_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -295044,9 +296449,10 @@ def update_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -296458,6 +297864,10 @@ def list_agents_by_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. + "web_fetch_enabled": bool, # Optional. Whether this agent + can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this agent + can use the built-in web_search tool. "workspace": { "agents": [ ... @@ -296535,16 +297945,14 @@ def list_agents_by_workspace( "custom_eval_config": { "created_at": - "2020-02-20 00:00:00", # Optional. Timestamp - when the custom metric was created. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00", # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. "deleted_at": "2020-02-20 00:00:00", # Optional. When set, the - custom metric has been deleted and is no longer - available for use in evaluations. - Server-assigned; ignored on create/update - requests. + custom metric is soft-deleted and must not appear + in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, @@ -296553,10 +297961,10 @@ def list_agents_by_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": - "2020-02-20 00:00:00" # Optional. Timestamp when - the custom metric was last updated. - Server-assigned; ignored on create/update - requests. + "2020-02-20 00:00:00" # Optional. Configuration + for a custom model-evaluation metric scored by an + LLM judge. Prompt and model response are always + included in the judge context. }, "description": "str", # Optional. Evaluations. @@ -297938,6 +299346,10 @@ def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -298005,14 +299417,13 @@ def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -298021,9 +299432,10 @@ def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -299289,6 +300701,10 @@ def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -299356,14 +300772,13 @@ def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -299372,9 +300787,10 @@ def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -300643,6 +302059,10 @@ def update_agents_workspace( "str" # Optional. VPC Egress IPs. ], "vpc_uuid": "str", # Optional. Agents. + "web_fetch_enabled": bool, # Optional. Whether this + agent can use the built-in web_fetch tool. + "web_search_enabled": bool, # Optional. Whether this + agent can use the built-in web_search tool. "workspace": ... } ], @@ -300710,14 +302130,13 @@ def update_agents_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 - 00:00:00", # Optional. Timestamp when the custom metric - was created. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. "deleted_at": "2020-02-20 - 00:00:00", # Optional. When set, the custom metric has - been deleted and is no longer available for use in - evaluations. Server-assigned; ignored on create/update - requests. + 00:00:00", # Optional. When set, the custom metric is + soft-deleted and must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. @@ -300726,9 +302145,10 @@ def update_agents_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 - 00:00:00" # Optional. Timestamp when the custom metric - was last updated. Server-assigned; ignored on - create/update requests. + 00:00:00" # Optional. Configuration for a custom + model-evaluation metric scored by an LLM judge. Prompt + and model response are always included in the judge + context. }, "description": "str", # Optional. Evaluations. @@ -300982,12 +302402,12 @@ def list_evaluation_test_cases_by_workspace( "METRIC_CATEGORY_MODEL_FIT". "custom_eval_config": { "created_at": "2020-02-20 00:00:00", - # Optional. Timestamp when the custom metric was created. - Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. "deleted_at": "2020-02-20 00:00:00", - # Optional. When set, the custom metric has been deleted and - is no longer available for use in evaluations. - Server-assigned; ignored on create/update requests. + # Optional. When set, the custom metric is soft-deleted and + must not appear in pickers. "requires_ground_truth": bool, # Optional. When true, each row must provide ground truth and it is included in the judge context. When false, ground truth @@ -300995,8 +302415,9 @@ def list_evaluation_test_cases_by_workspace( "scoring_prompt": "str", # Optional. Instructions for the judge model (multi-line). "updated_at": "2020-02-20 00:00:00" - # Optional. Timestamp when the custom metric was last - updated. Server-assigned; ignored on create/update requests. + # Optional. Configuration for a custom model-evaluation + metric scored by an LLM judge. Prompt and model response are + always included in the judge context. }, "description": "str", # Optional. "evaluation_scope":