From 24d00d0a5624774577023928e7b017ff370347e7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 22 Sep 2026 17:17:13 +0000 Subject: [PATCH] fix: enable apimap --- go.mod | 17 +- go.sum | 206 +- pkg/apis/constants/constants.go | 16 +- pkg/apis/onecloud/v1alpha1/defaults.go | 7 +- pkg/apis/onecloud/v1alpha1/types.go | 13 +- .../v1alpha1/zz_generated.deepcopy.go | 30 +- pkg/manager/component/apimap.go | 49 +- pkg/manager/component/component.go | 2 +- pkg/manager/component/vpcagent.go | 9 +- pkg/service-init/component/apimap.go | 47 - .../github.com/go-openapi/swag/.golangci.yml | 2 + vendor/github.com/go-yaml/yaml/.travis.yml | 12 + vendor/github.com/go-yaml/yaml/LICENSE | 201 ++ .../github.com/go-yaml/yaml/LICENSE.libyaml | 31 + vendor/github.com/go-yaml/yaml/NOTICE | 13 + vendor/github.com/go-yaml/yaml/README.md | 135 + vendor/github.com/go-yaml/yaml/apic.go | 739 +++++ vendor/github.com/go-yaml/yaml/decode.go | 764 +++++ vendor/github.com/go-yaml/yaml/emitterc.go | 1685 ++++++++++ vendor/github.com/go-yaml/yaml/encode.go | 358 +++ vendor/github.com/go-yaml/yaml/parserc.go | 1095 +++++++ vendor/github.com/go-yaml/yaml/readerc.go | 394 +++ vendor/github.com/go-yaml/yaml/resolve.go | 245 ++ vendor/github.com/go-yaml/yaml/scannerc.go | 2702 +++++++++++++++++ vendor/github.com/go-yaml/yaml/sorter.go | 104 + vendor/github.com/go-yaml/yaml/writerc.go | 26 + vendor/github.com/go-yaml/yaml/yaml.go | 466 +++ vendor/github.com/go-yaml/yaml/yamlh.go | 738 +++++ .../github.com/go-yaml/yaml/yamlprivateh.go | 173 ++ .../go-cmp/cmp/internal/function/func.go | 7 + .../github.com/google/go-cmp/cmp/options.go | 10 +- vendor/modules.txt | 21 +- .../pkg/cloudprovider/cloudprovider.go | 16 + .../x/cloudmux/pkg/cloudprovider/instance.go | 5 + .../x/cloudmux/pkg/cloudprovider/ipset.go | 42 + .../x/cloudmux/pkg/cloudprovider/metrics.go | 4 +- .../x/cloudmux/pkg/cloudprovider/resources.go | 5 + .../pkg/cloudprovider/securitygroup.go | 35 +- .../cloudmux/pkg/multicloud/esxi/devtools.go | 1 + .../x/cloudmux/pkg/multicloud/esxi/monitor.go | 44 +- .../x/cloudmux/pkg/multicloud/esxi/tags.go | 5 +- .../pkg/multicloud/esxi/virtualmachine.go | 149 +- .../x/cloudmux/pkg/multicloud/esxi/vnic.go | 29 +- .../x/cloudmux/pkg/multicloud/region_base.go | 12 + .../cloudmux/pkg/multicloud/secgroup_base.go | 7 + vendor/yunion.io/x/jsonutils/Makefile | 2 +- vendor/yunion.io/x/jsonutils/access.go | 16 +- vendor/yunion.io/x/jsonutils/consts.go | 3 + vendor/yunion.io/x/jsonutils/equals.go | 10 +- vendor/yunion.io/x/jsonutils/errors.go | 6 + vendor/yunion.io/x/jsonutils/interface.go | 12 +- vendor/yunion.io/x/jsonutils/jsonutils.go | 181 +- vendor/yunion.io/x/jsonutils/marshal.go | 21 +- vendor/yunion.io/x/jsonutils/parse_session.go | 33 +- vendor/yunion.io/x/jsonutils/querystring.go | 10 +- vendor/yunion.io/x/jsonutils/std_json.go | 8 +- vendor/yunion.io/x/jsonutils/string.go | 4 + vendor/yunion.io/x/jsonutils/unmarshal.go | 36 +- .../x/jsonutils/unmarshal_session.go | 39 +- vendor/yunion.io/x/jsonutils/write.go | 21 +- .../x/onecloud/cmd/climc/shell/utils.go | 115 +- .../yunion.io/x/onecloud/locales/locales.go | 21 +- .../x/onecloud/pkg/aiproxy/options/options.go | 4 +- .../x/onecloud/pkg/apihelper/apihelper.go | 108 +- .../yunion.io/x/onecloud/pkg/apihelper/db.go | 123 - .../x/onecloud/pkg/apihelper/interface.go | 83 +- .../x/onecloud/pkg/apihelper/options.go | 25 + .../x/onecloud/pkg/apimap/options/options.go | 36 +- .../pkg/apis/ansibleserver/ansible.go | 5 + .../x/onecloud/pkg/apis/compute/api.go | 15 +- .../x/onecloud/pkg/apis/compute/backup.go | 9 + .../pkg/apis/compute/cloudprovider.go | 2 +- .../x/onecloud/pkg/apis/compute/container.go | 27 +- .../x/onecloud/pkg/apis/compute/disk.go | 29 +- .../x/onecloud/pkg/apis/compute/disk_const.go | 12 + .../onecloud/pkg/apis/compute/guest_const.go | 7 + .../x/onecloud/pkg/apis/compute/guest_disk.go | 11 +- .../onecloud/pkg/apis/compute/guestnetwork.go | 4 + .../x/onecloud/pkg/apis/compute/guests.go | 111 +- .../x/onecloud/pkg/apis/compute/host.go | 5 +- .../pkg/apis/compute/instance_backup.go | 11 + .../x/onecloud/pkg/apis/compute/ipset.go | 130 + .../pkg/apis/compute/isolated_device.go | 63 +- .../pkg/apis/compute/isolated_device_const.go | 132 +- .../x/onecloud/pkg/apis/compute/pod.go | 4 + .../x/onecloud/pkg/apis/compute/secgroup.go | 44 +- .../onecloud/pkg/apis/compute/secgrouprule.go | 12 + .../x/onecloud/pkg/apis/compute/snapshot.go | 5 +- .../pkg/apis/compute/zz_generated.model.go | 82 +- vendor/yunion.io/x/onecloud/pkg/apis/const.go | 1 + .../x/onecloud/pkg/apis/container.go | 34 +- .../x/onecloud/pkg/apis/host/container.go | 3 + .../pkg/apis/identity/identityprovider.go | 8 + .../x/onecloud/pkg/apis/identity/oidc.go | 2 + .../onecloud/pkg/apis/identity/rolepolicy.go | 8 + .../x/onecloud/pkg/apis/identity/saml.go | 5 + .../x/onecloud/pkg/apis/image/consts.go | 6 + .../pkg/apis/image/container_image.go | 89 + .../pkg/apis/image/container_registry.go | 122 + vendor/yunion.io/x/onecloud/pkg/apis/input.go | 2 + .../x/onecloud/pkg/apis/llm/image.go | 4 +- .../x/onecloud/pkg/apis/llm/instantmodel.go | 4 + .../yunion.io/x/onecloud/pkg/apis/llm/llm.go | 5 +- .../x/onecloud/pkg/apis/llm/llm_benchmark.go | 166 + .../pkg/apis/llm/llm_benchmark_package.go | 42 + .../x/onecloud/pkg/apis/llm/llm_const.go | 2 + .../x/onecloud/pkg/apis/llm/llm_container.go | 11 +- .../x/onecloud/pkg/apis/llm/llm_deployment.go | 3 + .../x/onecloud/pkg/apis/llm/llm_router.go | 12 - .../onecloud/pkg/apis/llm/llm_router_agent.go | 98 + .../x/onecloud/pkg/apis/llm/llm_spec.go | 40 +- .../x/onecloud/pkg/apis/llm/mcp_agent.go | 55 +- .../x/onecloud/pkg/apis/llm/sglang_const.go | 2 +- .../yunion.io/x/onecloud/pkg/apis/llm/sku.go | 69 +- .../x/onecloud/pkg/apis/llm/vllm_const.go | 2 +- .../x/onecloud/pkg/apis/monitor/commalert.go | 24 +- .../x/onecloud/pkg/apis/scheduler/api.go | 1 + .../yunion.io/x/onecloud/pkg/appsrv/cors.go | 10 + .../x/onecloud/pkg/appsrv/response.go | 40 +- .../pkg/cloudcommon/cmdline/helper.go | 3 + .../pkg/cloudcommon/cmdline/parser.go | 3 +- .../x/onecloud/pkg/cloudcommon/db/domain.go | 18 +- .../x/onecloud/pkg/cloudcommon/db/fetch.go | 65 +- .../x/onecloud/pkg/cloudcommon/db/metadata.go | 2 + .../onecloud/pkg/cloudcommon/db/modelbase.go | 7 +- .../x/onecloud/pkg/cloudcommon/db/opslog.go | 3 + .../x/onecloud/pkg/cloudcommon/db/project.go | 22 +- .../pkg/cloudcommon/db/quotas/handler.go | 3 + .../onecloud/pkg/cloudcommon/db/rolecache.go | 3 + .../pkg/cloudcommon/db/scoperesource.go | 2 +- .../pkg/cloudcommon/db/sharablebase.go | 18 + .../pkg/cloudcommon/db/standalone_anon.go | 72 +- .../pkg/cloudcommon/db/taskman/tasks.go | 17 +- .../pkg/cloudcommon/db/tenantcache.go | 3 + .../onecloud/pkg/cloudcommon/db/usercache.go | 3 + .../pkg/cloudcommon/options/options.go | 3 +- .../onecloud/pkg/cloudcommon/policy/policy.go | 3 +- .../x/onecloud/pkg/cloudcommon/types/ipmi.go | 22 +- .../pkg/cloudmon/misc/usageservice.go | 3 + .../onecloud/pkg/cloudmon/options/options.go | 2 + .../pkg/compute/models/access_group_rules.go | 2 +- .../pkg/compute/models/backup_storage.go | 15 +- .../compute/models/billingresourcecheck.go | 1 + .../pkg/compute/models/cachedimages.go | 198 +- .../pkg/compute/models/capabilities.go | 43 +- .../compute/models/cloudaccount_proxmox.go | 128 + .../pkg/compute/models/cloudaccount_vmware.go | 4 +- .../pkg/compute/models/cloudaccounts.go | 7 +- .../pkg/compute/models/cloudproviders.go | 31 +- .../pkg/compute/models/cloudregions.go | 3 + .../onecloud/pkg/compute/models/cloudsync.go | 112 +- .../pkg/compute/models/container_drivers.go | 2 +- .../pkg/compute/models/container_hostpath.go | 64 + .../onecloud/pkg/compute/models/containers.go | 253 +- .../pkg/compute/models/diskbackups.go | 167 +- .../x/onecloud/pkg/compute/models/disks.go | 76 +- .../onecloud/pkg/compute/models/elasticips.go | 2 +- .../pkg/compute/models/guest_actions.go | 761 +++-- .../pkg/compute/models/guest_convert.go | 127 + .../pkg/compute/models/guest_deploy.go | 42 + .../compute/models/guest_isolated_devices.go | 319 ++ .../pkg/compute/models/guest_queries.go | 24 +- .../pkg/compute/models/guest_sshable.go | 10 +- .../onecloud/pkg/compute/models/guestdisks.go | 21 +- .../pkg/compute/models/guestdrivers.go | 8 +- .../pkg/compute/models/guestnetworks.go | 27 +- .../x/onecloud/pkg/compute/models/guests.go | 234 +- .../x/onecloud/pkg/compute/models/hosts.go | 161 +- .../x/onecloud/pkg/compute/models/initdb.go | 3 + .../pkg/compute/models/instance_backup.go | 11 +- .../pkg/compute/models/instance_snapshots.go | 30 +- .../x/onecloud/pkg/compute/models/ipsets.go | 793 +++++ .../compute/models/isolated_device_models.go | 5 - .../pkg/compute/models/isolated_devices.go | 1087 +++++-- .../compute/models/isolateddeviceresouce.go | 202 ++ .../pkg/compute/models/networkaddresses.go | 4 +- .../x/onecloud/pkg/compute/models/networks.go | 2 +- .../x/onecloud/pkg/compute/models/purge.go | 6 +- .../pkg/compute/models/regiondrivers.go | 1 + .../pkg/compute/models/scaling_group.go | 6 + .../pkg/compute/models/secgrouprules.go | 189 +- .../onecloud/pkg/compute/models/secgroups.go | 21 + .../onecloud/pkg/compute/models/snapshots.go | 119 +- .../pkg/compute/models/storagecachedimages.go | 3 + .../x/onecloud/pkg/compute/models/storages.go | 50 +- .../x/onecloud/pkg/compute/models/vpcs.go | 6 +- .../x/onecloud/pkg/compute/models/wires.go | 1 + .../x/onecloud/pkg/compute/models/zones.go | 6 + .../x/onecloud/pkg/compute/options/options.go | 3 + .../pkg/devtool/utils/gen_telegraf_conf.go | 5 - .../x/onecloud/pkg/devtool/utils/ssh.go | 1 + .../pkg/hostman/guestfs/fsdriver/base.go | 10 + .../pkg/hostman/guestfs/fsdriver/drivers.go | 2 +- .../pkg/hostman/guestfs/fsdriver/interface.go | 4 +- .../pkg/hostman/guestfs/fsdriver/linux.go | 25 +- .../hostman/guestfs/fsdriver/linux_init.go | 21 +- .../pkg/hostman/guestfs/fsdriver/netplan.go | 27 +- .../pkg/hostman/guestfs/fsdriver/openruyi.go | 73 + .../pkg/hostman/guestfs/fsdriver/windows.go | 30 +- .../hostman/guestfs/fsdriver/winscripts.go | 8 +- .../pkg/hostman/guestman/desc/desc.go | 11 +- .../hostman/hostdeployer/apis/deploy.pb.go | 162 +- .../hostman/hostdeployer/apis/deploy.proto | 1 + .../hostman/hostinfo/hostbridge/hostbridge.go | 6 +- .../x/onecloud/pkg/hostman/options/options.go | 46 +- .../backupstorage/backup_storage.go | 29 +- .../storageman/backupstorage/nfs/factory.go | 8 +- .../storageman/backupstorage/nfs/nfs.go | 65 +- .../backupstorage/object/factory.go | 8 +- .../storageman/backupstorage/object/object.go | 67 +- .../pkg/hostman/system_service/telegraf.go | 50 + .../x/onecloud/pkg/httperrors/general.go | 5 + .../x/onecloud/pkg/image/options/options.go | 6 +- .../onecloud/pkg/lbagent/gobetween/config.go | 15 +- .../onecloud/pkg/lbagent/models/gobetween.go | 27 +- .../x/onecloud/pkg/lbagent/models/modelset.go | 36 + .../x/onecloud/pkg/llm/options/option.go | 55 +- .../x/onecloud/pkg/mcclient/auth/aksk.go | 39 + .../x/onecloud/pkg/mcclient/auth/auth.go | 3 +- .../pkg/mcclient/modulebase/modules.go | 4 +- .../mcclient/modules/compute/mod_ipsets.go | 32 + .../modules/compute/mod_isolatedevices.go | 18 +- .../mcclient/modules/compute/mod_snapshots.go | 2 +- .../modules/image/mod_container_image.go | 29 + .../modules/image/mod_container_registry.go | 111 + .../modules/k8s/container_registry.go | 6 +- .../mcclient/modules/llm/mod_llm_benchmark.go | 60 + .../modules/llm/mod_llm_benchmark_package.go | 35 + .../modules/llm/mod_llm_router_agent.go | 24 + .../pkg/mcclient/modules/monitor/helper.go | 11 + .../modules/scheduler/mod_scheduler.go | 6 + .../x/onecloud/pkg/mcclient/options/base.go | 30 +- .../pkg/mcclient/options/compute/backup.go | 11 +- .../pkg/mcclient/options/compute/buckets.go | 8 +- .../mcclient/options/compute/cachedimage.go | 13 +- .../mcclient/options/compute/cdn_domains.go | 2 +- .../mcclient/options/compute/cloudaccounts.go | 54 +- .../options/compute/cloudproviders.go | 6 +- .../mcclient/options/compute/containers.go | 74 +- .../mcclient/options/compute/dbinstances.go | 33 +- .../pkg/mcclient/options/compute/disks.go | 62 +- .../pkg/mcclient/options/compute/eips.go | 29 +- .../options/compute/elastic_search.go | 2 +- .../mcclient/options/compute/elasticache.go | 13 +- .../pkg/mcclient/options/compute/host.go | 49 +- .../options/compute/instancegroup_guest.go | 2 +- .../pkg/mcclient/options/compute/ipset.go | 77 + .../options/compute/isolated_devices.go | 32 +- .../pkg/mcclient/options/compute/kafka.go | 11 +- .../pkg/mcclient/options/compute/keypairs.go | 4 +- .../mcclient/options/compute/loadbalancers.go | 16 +- .../pkg/mcclient/options/compute/mongodb.go | 11 +- .../mcclient/options/compute/natgateways.go | 2 +- .../pkg/mcclient/options/compute/network.go | 22 +- .../pkg/mcclient/options/compute/schedtags.go | 6 +- .../mcclient/options/compute/schedulers.go | 4 +- .../pkg/mcclient/options/compute/secgroup.go | 53 +- .../mcclient/options/compute/secgrouprule.go | 80 +- .../mcclient/options/compute/server_pod.go | 2 +- .../pkg/mcclient/options/compute/servers.go | 283 +- .../mcclient/options/compute/serverskus.go | 94 +- .../pkg/mcclient/options/compute/storage.go | 12 +- .../mcclient/options/compute/storagecache.go | 2 +- .../pkg/mcclient/options/compute/vpc.go | 8 +- .../pkg/mcclient/options/identity/domain.go | 6 +- .../pkg/mcclient/options/identity/project.go | 12 +- .../pkg/mcclient/options/identity/users.go | 16 +- .../pkg/mcclient/options/k8s/cluster.go | 12 + .../pkg/mcp-server/options/options.go | 43 +- .../x/onecloud/pkg/notify/options/options.go | 1 + vendor/yunion.io/x/onecloud/pkg/proxy/http.go | 16 +- .../x/onecloud/pkg/util/ansible/module.go | 18 +- .../x/onecloud/pkg/util/ansible/playbook.go | 8 +- .../x/onecloud/pkg/util/ansible/validate.go | 175 ++ .../pkg/util/ansible/validate_yaml.go | 217 ++ .../onecloud/pkg/util/fileutils2/fileutils.go | 9 +- .../onecloud/pkg/util/fileutils2/hostpath.go | 153 + .../onecloud/pkg/util/fileutils2/nofollow.go | 103 + .../x/onecloud/pkg/util/logclient/consts.go | 2 + .../pkg/util/logclient/consts_i18n.go | 5 + .../x/onecloud/pkg/util/netutils2/netutils.go | 35 +- .../pkg/util/procutils/remote_path.go | 36 + .../x/onecloud/pkg/util/seclib2/crypto.go | 9 + .../yunion.io/x/onecloud/pkg/util/ssh/ssh.go | 4 + .../x/onecloud/pkg/util/sysutils/kvm.go | 62 +- .../x/onecloud/pkg/util/winutils/winutils.go | 17 + .../x/pkg/util/cloudinit/cloudconfig.go | 164 +- vendor/yunion.io/x/pkg/util/httputils/fs.go | 11 +- .../x/pkg/util/httputils/httputils.go | 288 +- .../x/pkg/util/imagetools/imagetools.go | 49 +- vendor/yunion.io/x/pkg/util/netutils/http.go | 47 +- vendor/yunion.io/x/pkg/util/netutils/ipv6.go | 4 + .../yunion.io/x/pkg/util/netutils/netutils.go | 95 +- .../x/pkg/util/reflectutils/ambiguous.go | 105 +- .../yunion.io/x/pkg/util/reflectutils/doc.go | 24 + .../x/pkg/util/reflectutils/jsonfield.go | 180 +- .../x/pkg/util/reflectutils/reflectutils.go | 103 +- .../yunion.io/x/pkg/util/samlutils/encrypt.go | 37 +- vendor/yunion.io/x/pkg/util/samlutils/util.go | 17 +- vendor/yunion.io/x/pkg/util/seclib/certs.go | 12 +- vendor/yunion.io/x/pkg/util/seclib/seclib.go | 38 +- vendor/yunion.io/x/pkg/util/secrules/cut.go | 10 +- .../yunion.io/x/pkg/util/secrules/secrules.go | 18 +- .../x/pkg/util/streamutils/streamutils.go | 38 +- vendor/yunion.io/x/pkg/utils/crypt.go | 156 + vendor/yunion.io/x/pkg/utils/misc.go | 27 +- vendor/yunion.io/x/pkg/utils/net.go | 17 +- vendor/yunion.io/x/pkg/utils/pidfile.go | 4 +- vendor/yunion.io/x/pkg/utils/tags.go | 34 +- vendor/yunion.io/x/sqlchemy/debug.go | 3 +- vendor/yunion.io/x/sqlchemy/parser.go | 2 +- vendor/yunion.io/x/sqlchemy/update.go | 8 +- vendor/yunion.io/x/structarg/structarg.go | 4 +- 313 files changed, 21069 insertions(+), 2520 deletions(-) delete mode 100644 pkg/service-init/component/apimap.go create mode 100644 vendor/github.com/go-yaml/yaml/.travis.yml create mode 100644 vendor/github.com/go-yaml/yaml/LICENSE create mode 100644 vendor/github.com/go-yaml/yaml/LICENSE.libyaml create mode 100644 vendor/github.com/go-yaml/yaml/NOTICE create mode 100644 vendor/github.com/go-yaml/yaml/README.md create mode 100644 vendor/github.com/go-yaml/yaml/apic.go create mode 100644 vendor/github.com/go-yaml/yaml/decode.go create mode 100644 vendor/github.com/go-yaml/yaml/emitterc.go create mode 100644 vendor/github.com/go-yaml/yaml/encode.go create mode 100644 vendor/github.com/go-yaml/yaml/parserc.go create mode 100644 vendor/github.com/go-yaml/yaml/readerc.go create mode 100644 vendor/github.com/go-yaml/yaml/resolve.go create mode 100644 vendor/github.com/go-yaml/yaml/scannerc.go create mode 100644 vendor/github.com/go-yaml/yaml/sorter.go create mode 100644 vendor/github.com/go-yaml/yaml/writerc.go create mode 100644 vendor/github.com/go-yaml/yaml/yaml.go create mode 100644 vendor/github.com/go-yaml/yaml/yamlh.go create mode 100644 vendor/github.com/go-yaml/yaml/yamlprivateh.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/cloudprovider/ipset.go delete mode 100644 vendor/yunion.io/x/onecloud/pkg/apihelper/db.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/compute/ipset.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/image/container_image.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/image/container_registry.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark_package.go delete mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router_agent.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_proxmox.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/container_hostpath.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/guest_deploy.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/guest_isolated_devices.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/ipsets.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/compute/models/isolateddeviceresouce.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/openruyi.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_ipsets.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_image.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_registry.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_router_agent.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/ipset.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/util/ansible/validate.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/util/ansible/validate_yaml.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/util/fileutils2/hostpath.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/util/fileutils2/nofollow.go create mode 100644 vendor/yunion.io/x/onecloud/pkg/util/procutils/remote_path.go diff --git a/go.mod b/go.mod index 50b054df7..87e8f9bdb 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 k8s.io/klog v0.3.3 - yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 + yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 - yunion.io/x/onecloud v0.0.0-20260714035258-73d7cbbb1f53 - yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978 - yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c + yunion.io/x/onecloud v0.0.0-20260922165722-171561b0780b + yunion.io/x/pkg v1.10.4-0.20260918012554-27cd9d2e093b + yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac ) require ( @@ -48,10 +48,11 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/swag v0.19.15 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/go-yaml/yaml v2.1.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-plus/errors v1.0.0 // indirect github.com/golang-plus/uuid v1.0.0 // indirect @@ -59,7 +60,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect @@ -141,10 +142,10 @@ require ( sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260713095312-995dd1831cd6 // indirect + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf // indirect yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 // indirect yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 // indirect - yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab // indirect + yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 // indirect ) replace ( diff --git a/go.sum b/go.sum index 994a41755..fa7edd35f 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,32 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitee.com/chunanyong/dm v1.8.14 h1:1S9+aD0fY/HXkcm8dKh2HjYsDlcdXNG+4IOX9JtrSjA= gitee.com/chunanyong/dm v1.8.14/go.mod h1:EPRJnuPFgbyOFgJ0TRYCTGzhq+ZT4wdyaj/GW/LLcNg= @@ -46,6 +67,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -66,6 +88,7 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -138,11 +161,15 @@ github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -195,8 +222,8 @@ github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= @@ -212,6 +239,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -229,15 +258,22 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -245,6 +281,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -260,19 +297,28 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -309,10 +355,12 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -320,6 +368,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -414,6 +463,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -455,6 +505,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -463,10 +515,14 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= @@ -569,7 +625,9 @@ github.com/xuri/excelize/v2 v2.7.1 h1:gm8q0UCAyaTt3MEF5wWMjVdmthm2EHAWesGSKS9tdV github.com/xuri/excelize/v2 v2.7.1/go.mod h1:qc0+2j4TvAUrBw36ATtcTeC1VCM0fFdAXZOmcF4nTpY= github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Qsdt4+M5+ltca05dA5bG2M= github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= @@ -596,6 +654,8 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -631,7 +691,12 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= @@ -642,12 +707,16 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -670,16 +739,29 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= @@ -689,6 +771,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -697,7 +781,10 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= @@ -719,33 +806,49 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -763,10 +866,12 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -802,10 +907,32 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -818,12 +945,24 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -834,8 +973,27 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 h1:SzXBGiWM1LNVYLCRP3e0/Gsze804l4jGoJ5lYysEO5I= google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= @@ -850,6 +1008,10 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= @@ -862,6 +1024,7 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= @@ -895,6 +1058,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -909,6 +1073,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= k8s.io/api v0.0.0-20210712061623-33e331166f19/go.mod h1:3YUNrS3YZCXipiLZ72bx8O/lWZN7wopU4mia01/MEg8= k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= @@ -945,6 +1111,8 @@ k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= @@ -956,26 +1124,28 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260713095312-995dd1831cd6 h1:8rIQvC0ii9q0QzPueyDX5BopV5VeFRPs59Rjp0ht38w= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260713095312-995dd1831cd6/go.mod h1:QgrcekfkD3fELNYipZnlUfhR8dPI8CCUsuzC2SkP3YE= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf h1:weQS1w9PxBeOgQFhXDkhc9UUsvpY/6AWXkk6d3Pc9fk= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf/go.mod h1:IwmeqVkld+4XylS80cyVBXyN8fJ5ynAspd/6tcvqJuc= yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 h1:3sWwcjGXGjG9mLBWa7AyLq+QSi0udTAx21pfVQRFMBE= yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= -yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= -yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 h1:/+THlvf/MvgCW+7KeCDCr33e81KSRa5JmdZ1IIyLOXQ= -yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo= +yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= +yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea h1:lB7Kwanf7REFaA19YmDc7p0HVkssVRoNQejMlqtpecw= +yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea/go.mod h1:8ctyIiHT+Af1Ldr0xhvZZvuf+kyghlBUuvTy6pOKwbo= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE= yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= +yunion.io/x/log v1.0.0/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= -yunion.io/x/onecloud v0.0.0-20260714035258-73d7cbbb1f53 h1:T7/6s3SGuQ+b+dguBo6OvTRtodJuco5+WGU2Ut9rjaM= -yunion.io/x/onecloud v0.0.0-20260714035258-73d7cbbb1f53/go.mod h1:tKH1RwXeSrSeGtfKQ1Qt3RLjKGSVSHG2av1pQzmKgMc= -yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= +yunion.io/x/onecloud v0.0.0-20260922165722-171561b0780b h1:FjnGq/fyiYbjKoFcdj9CRVbeWVu4mGAOE2MVnk2018A= +yunion.io/x/onecloud v0.0.0-20260922165722-171561b0780b/go.mod h1:yxUxtrN8Op8AkGmCqp4XmHzJeHfG1JIFF/WazLcpZYM= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978 h1:+BS9DValwi0xHa92xrB+RR6Cxp7bDbXVysa/KiC18rY= -yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= +yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= +yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758/go.mod h1:Crp6aUYB+Yq16DFjZPHhHq7ha5HL2DfyPvzCo2LJ+LU= +yunion.io/x/pkg v1.10.4-0.20260918012554-27cd9d2e093b h1:XUJLpD48685DE8hwxEC+ms1F4T6/g2KKAfGuVn8rZeo= +yunion.io/x/pkg v1.10.4-0.20260918012554-27cd9d2e093b/go.mod h1:S8ITTpENaL3qSXi8Y4L6wVbVlP8XZ5BvABO10DFqF7M= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab h1:5m/bSzW3uTAk83rp9eethbYsxJFYInFVeU1RDkteW4E= -yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A= -yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= -yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= +yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 h1:xU8Z/Zg+zc199Y58eB3fA8mvOHsA28UCDfd7OuK3QKA= +yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12/go.mod h1:RNiIM7ZFF7VFrk18DbwAa8PYR00YP7Y23XjqKIlfr3I= +yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac h1:hJAthswj6YfWd3GCOBl9eM7mv0+4EUmjnD0b9jdBYi0= +yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac/go.mod h1:DyMCdRyfV5PdEe/BDz2vBbemnmALUF3Rj1JmUUlu3KY= diff --git a/pkg/apis/constants/constants.go b/pkg/apis/constants/constants.go index 1b76e2b3a..1340d146b 100644 --- a/pkg/apis/constants/constants.go +++ b/pkg/apis/constants/constants.go @@ -89,10 +89,10 @@ const ( MonitorLoki = "monitor-loki" MonitorPrometheus = "prometheus-monitor-monitor-stack-prometheus" - MonitorBucketThanos = "thanos" - MonitorBucketLoki = "loki" - MonitorBucketAiProxy = "aiproxy-log" - MonitorBucketAiProxyRetentionDays = 90 + MonitorBucketThanos = "thanos" + MonitorBucketLoki = "loki" + MonitorBucketAiProxy = "aiproxy-log" + MonitorBucketAiProxyRetentionDays = 90 ) const ( @@ -411,9 +411,11 @@ const ( ServiceTypeScheduledtask = "scheduledtask" ScheduledtaskPort = 30978 - ServiceNameAPIMap = "apimap" - ServiceTypeAPIMap = "apimap" - APIMapPort = 31999 + ServiceNameApiMap = "apimap" + ServiceTypeApiMap = "apimap" + ApiMapAdminUser = "apimapadmin" + ApiMapAdminProject = SysAdminProject + ApiMapPort = 31999 ServiceNameReport = "report" ServiceTypeReport = "report" diff --git a/pkg/apis/onecloud/v1alpha1/defaults.go b/pkg/apis/onecloud/v1alpha1/defaults.go index 49d9f7e17..505bc4ef3 100644 --- a/pkg/apis/onecloud/v1alpha1/defaults.go +++ b/pkg/apis/onecloud/v1alpha1/defaults.go @@ -210,7 +210,7 @@ func SetDefaults_OnecloudClusterSpec(obj *OnecloudClusterSpec, isEE bool, isEEOr SuggestionComponentType: nHP(&obj.Suggestion.DeploymentSpec, useHyperImage), ScheduledtaskComponentType: nHP(&obj.Scheduledtask.DeploymentSpec, useHyperImage), ReportComponentType: nHP(&obj.Report.DeploymentSpec, useHyperImage), - APIMapComponentType: nHP(&obj.APIMap.DeploymentSpec, useHyperImage), + ApiMapComponentType: nHP(&obj.ApiMap.DeploymentSpec, useHyperImage), BastionHostComponentType: nHP(&obj.BastionHost.DeploymentSpec, useHyperImage), ExtdbComponentType: nHP(&obj.Extdb.DeploymentSpec, useHyperImage), BillingComponentType: nHP(&obj.Billing.DeploymentSpec, useHyperImage), @@ -224,8 +224,6 @@ func SetDefaults_OnecloudClusterSpec(obj *OnecloudClusterSpec, isEE bool, isEEOr spec.Supported, isEE, )) } - // disable the apimap service defaultly - obj.APIMap.Disable = true // CE or EE parts for cType, spec := range map[ComponentType]*hyperImagePair{ @@ -493,7 +491,7 @@ func setDefaults_Components_ServicePort(obj *OnecloudClusterSpec) { newSP(&obj.VictoriaMetrics.Service, constants.VictoriaMetricsPort), newSP(&obj.Monitor.Service, constants.MonitorPort), newSP(&obj.Scheduledtask.Service, constants.ScheduledtaskPort), - newSP(&obj.APIMap.Service, constants.APIMapPort), + newSP(&obj.ApiMap.Service, constants.ApiMapPort), newSP(&obj.Report.Service, constants.ReportPort), newSP(&obj.APIGateway.APIService, constants.APIGatewayPort), newSP(&obj.APIGateway.WSService, constants.APIWebsocketPort), @@ -808,6 +806,7 @@ func SetDefaults_OnecloudClusterConfig(obj *OnecloudClusterConfig) { &obj.S3gateway: {constants.S3gatewayAdminUser, constants.S3gatewayPort}, &obj.EsxiAgent.ServiceCommonOptions: {constants.EsxiAgentAdminUser, constants.EsxiAgentPort}, &obj.VpcAgent.ServiceCommonOptions: {constants.VpcAgentAdminUser, constants.VpcAgentPort}, + &obj.ApiMap.ServiceCommonOptions: {constants.ApiMapAdminUser, constants.ApiMapPort}, &obj.ServiceOperator: {constants.ServiceOperatorAdminUser, constants.ServiceOperatorPort}, &obj.Lbagent.ServiceCommonOptions: {constants.LbagentAdminUser, constants.LbagentPort}, &obj.McpServer.ServiceCommonOptions: {constants.McpServerAdminUser, constants.McpServerPort}, diff --git a/pkg/apis/onecloud/v1alpha1/types.go b/pkg/apis/onecloud/v1alpha1/types.go index 569712489..ab294700a 100644 --- a/pkg/apis/onecloud/v1alpha1/types.go +++ b/pkg/apis/onecloud/v1alpha1/types.go @@ -137,7 +137,7 @@ const ( ScheduledtaskComponentType ComponentType = "scheduledtask" - APIMapComponentType ComponentType = "apimap" + ApiMapComponentType ComponentType = "apimap" MonitorStackComponentType ComponentType = "monitor-stack" @@ -348,7 +348,8 @@ type OnecloudClusterSpec struct { MonitorStack MonitorStackSpec `json:"monitorStack"` Scheduledtask DeploymentServicePortSpec `json:"scheduledtask"` - APIMap DeploymentServicePortSpec `json:"apiMap"` + + ApiMap DeploymentServicePortSpec `json:"apimap"` Report DeploymentServicePortSpec `json:"report"` EChartsSSR EChartsSSRSpec `json:"echartsSSR"` @@ -415,7 +416,7 @@ type OnecloudClusterStatus struct { Cloudmon DeploymentStatus `json:"cloudmon,omitempty"` MonitorStack MonitorStackStatus `json:"monitorStack,omitempty"` Scheduledtask DeploymentStatus `json:"scheduledtask,omitempty"` - APIMap DeploymentStatus `json:"apiMap,omitempty"` + APIMap DeploymentStatus `json:"apimap,omitempty"` Report DeploymentStatus `json:"report,omitempty"` ECharts DeploymentStatus `json:"echarts,omitempty"` BastionHost DeploymentStatus `json:"bastionHost,omitempty"` @@ -646,6 +647,7 @@ type WebconsoleSpec struct { type DeploymentServicePortSpec struct { DeploymentSpec + Service ServiceSpec `json:"service"` DbEngine TDBEngineType `json:"db_engine"` } @@ -1230,6 +1232,10 @@ type VpcAgentConfig struct { ServiceCommonOptions } +type APIMapConfig struct { + ServiceCommonOptions +} + type ItsmConfig struct { ServiceDBCommonOptions SecondDatabase string `json:"secondDatabase"` @@ -1290,4 +1296,5 @@ type OnecloudClusterConfig struct { LLM ServiceDBCommonOptions `json:"llm"` McpServer ServiceDBCommonOptions `json:"mcpServer"` AiProxy ServiceDBCommonOptions `json:"aiproxy"` + ApiMap APIMapConfig `json:"apimap"` } diff --git a/pkg/apis/onecloud/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/onecloud/v1alpha1/zz_generated.deepcopy.go index 429f86787..a691d62dc 100644 --- a/pkg/apis/onecloud/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/onecloud/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,23 @@ func (in *APIGatewaySpec) DeepCopy() *APIGatewaySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIMapConfig) DeepCopyInto(out *APIMapConfig) { + *out = *in + out.ServiceCommonOptions = in.ServiceCommonOptions + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIMapConfig. +func (in *APIMapConfig) DeepCopy() *APIMapConfig { + if in == nil { + return nil + } + out := new(APIMapConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BaremetalAgentStatus) DeepCopyInto(out *BaremetalAgentStatus) { *out = *in @@ -480,6 +497,11 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { *out = new(ImageStatus) **out = **in } + if in.SlaveDeployment != nil { + in, out := &in.SlaveDeployment, &out.SlaveDeployment + *out = new(appsv1.DeploymentStatus) + (*in).DeepCopyInto(*out) + } return } @@ -626,6 +648,11 @@ func (in *Etcd) DeepCopyInto(out *Etcd) { *out = new(bool) **out = **in } + if in.EtcdBackendQuotaSize != nil { + in, out := &in.EtcdBackendQuotaSize, &out.EtcdBackendQuotaSize + *out = new(int64) + **out = **in + } return } @@ -1438,6 +1465,7 @@ func (in *OnecloudClusterConfig) DeepCopyInto(out *OnecloudClusterConfig) { out.LLM = in.LLM out.McpServer = in.McpServer out.AiProxy = in.AiProxy + out.ApiMap = in.ApiMap return } @@ -1557,7 +1585,7 @@ func (in *OnecloudClusterSpec) DeepCopyInto(out *OnecloudClusterSpec) { in.Suggestion.DeepCopyInto(&out.Suggestion) in.MonitorStack.DeepCopyInto(&out.MonitorStack) in.Scheduledtask.DeepCopyInto(&out.Scheduledtask) - in.APIMap.DeepCopyInto(&out.APIMap) + in.ApiMap.DeepCopyInto(&out.ApiMap) in.Report.DeepCopyInto(&out.Report) in.EChartsSSR.DeepCopyInto(&out.EChartsSSR) in.BastionHost.DeepCopyInto(&out.BastionHost) diff --git a/pkg/manager/component/apimap.go b/pkg/manager/component/apimap.go index dcdccdc7e..f71fae2b6 100644 --- a/pkg/manager/component/apimap.go +++ b/pkg/manager/component/apimap.go @@ -15,19 +15,20 @@ package component import ( + "path" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "yunion.io/x/pkg/errors" - + "yunion.io/x/onecloud/pkg/apimap/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud-operator/pkg/apis/constants" "yunion.io/x/onecloud-operator/pkg/apis/onecloud/v1alpha1" "yunion.io/x/onecloud-operator/pkg/controller" "yunion.io/x/onecloud-operator/pkg/manager" - "yunion.io/x/onecloud-operator/pkg/service-init/component" "yunion.io/x/onecloud-operator/pkg/util/onecloud" + "yunion.io/x/onecloud-operator/pkg/util/option" ) type apiMapManager struct { @@ -42,42 +43,60 @@ func (m *apiMapManager) getProductVersions() []v1alpha1.ProductVersion { return []v1alpha1.ProductVersion{ v1alpha1.ProductVersionFullStack, v1alpha1.ProductVersionEdge, + v1alpha1.ProductVersionLightEdge, + v1alpha1.ProductVersionAI, } } func (m *apiMapManager) GetComponentType() v1alpha1.ComponentType { - return v1alpha1.APIMapComponentType + return v1alpha1.ApiMapComponentType } func (m *apiMapManager) IsDisabled(oc *v1alpha1.OnecloudCluster) bool { - return oc.Spec.APIMap.Disable || !isInProductVersion(m, oc) + return oc.Spec.ApiMap.Disable || !isInProductVersion(m, oc) } func (m *apiMapManager) GetServiceName() string { - return constants.ServiceNameAPIMap + return constants.ServiceNameApiMap } func (m *apiMapManager) Sync(oc *v1alpha1.OnecloudCluster) error { return syncComponent(m, oc, "") } -func (m *apiMapManager) getPhaseControl(man controller.ComponentManager, zone string) controller.PhaseControl { - return component.NewAPIMap().GetPhaseControl(man) +func (m *apiMapManager) getCloudUser(cfg *v1alpha1.OnecloudClusterConfig) *v1alpha1.CloudUser { + return &cfg.ApiMap.CloudUser } func (m *apiMapManager) getConfigMap(oc *v1alpha1.OnecloudCluster, cfg *v1alpha1.OnecloudClusterConfig, zone string) (*corev1.ConfigMap, bool, error) { - opt, err := component.NewAPIMap().GetConfig(oc, cfg) - if err != nil { - return nil, false, errors.Wrap(err, "apimap: SetOptionsDefault") + opts := options.GetOptions() + if err := option.SetOptionsDefault(opts, constants.ServiceTypeApiMap); err != nil { + return nil, false, err } - return m.newServiceConfigMap(v1alpha1.APIMapComponentType, "", oc, opt), false, nil + config := cfg.ApiMap + option.SetOptionsServiceTLS(&opts.BaseOptions, false) + option.SetServiceCommonOptions(&opts.CommonOptions, oc, config.ServiceCommonOptions, cfg.CommonConfig) + opts.SslCertfile = path.Join(constants.CertDir, constants.ServiceCertName) + opts.SslKeyfile = path.Join(constants.CertDir, constants.ServiceKeyName) + opts.Port = constants.ApiMapPort + return m.newServiceConfigMap(v1alpha1.ApiMapComponentType, "", oc, opts), false, nil + // return m.shouldSyncConfigmap(oc, v1alpha1.ApiMapComponentType, opts, nil) } + +func (m *apiMapManager) getPhaseControl(man controller.ComponentManager, zone string) controller.PhaseControl { + return controller.NewRegisterEndpointComponent(man, + v1alpha1.ApiMapComponentType, + constants.ServiceNameApiMap, + constants.ServiceTypeApiMap, + man.GetCluster().Spec.ApiMap.Service.NodePort, "") +} + func (m *apiMapManager) getService(oc *v1alpha1.OnecloudCluster, cfg *v1alpha1.OnecloudClusterConfig, zone string) []*corev1.Service { - return m.newSinglePortService(v1alpha1.APIMapComponentType, oc, oc.Spec.APIMap.Service.InternalOnly, int32(oc.Spec.APIMap.Service.NodePort), constants.APIMapPort, oc.Spec.APIMap.SlaveReplicas > 0) + return m.newSinglePortService(v1alpha1.ApiMapComponentType, oc, oc.Spec.ApiMap.Service.InternalOnly, int32(oc.Spec.ApiMap.Service.NodePort), constants.ApiMapPort, oc.Spec.ApiMap.SlaveReplicas > 0) } func (m *apiMapManager) getDeployment(oc *v1alpha1.OnecloudCluster, cfg *v1alpha1.OnecloudClusterConfig, zone string) (*apps.Deployment, error) { - return m.newCloudServiceSinglePortDeployment(v1alpha1.APIMapComponentType, "", oc, &oc.Spec.APIMap.DeploymentSpec, constants.APIMapPort, false, false) + return m.newCloudServiceSinglePortDeployment(v1alpha1.ApiMapComponentType, "", oc, &oc.Spec.ApiMap.DeploymentSpec, constants.ApiMapPort, false, false) } func (m *apiMapManager) getDeploymentStatus(oc *v1alpha1.OnecloudCluster, zone string) *v1alpha1.DeploymentStatus { @@ -97,7 +116,7 @@ func (m *apiMapManager) getMcclientSyncFunc(oc *v1alpha1.OnecloudCluster) func(* if m.IsDisabled(oc) { return onecloud.EnsureDisableService(s, m.GetServiceName()) } else { - return onecloud.EnsureEnableService(s, m.GetServiceName(), m.supportsReadOnlyService() && oc.Spec.APIMap.SlaveReplicas > 0) + return onecloud.EnsureEnableService(s, m.GetServiceName(), m.supportsReadOnlyService() && oc.Spec.ApiMap.SlaveReplicas > 0) } } } diff --git a/pkg/manager/component/component.go b/pkg/manager/component/component.go index 32cf9413e..d34e74620 100644 --- a/pkg/manager/component/component.go +++ b/pkg/manager/component/component.go @@ -1829,7 +1829,7 @@ func (m *ComponentManager) shouldSyncConfigmap( if oldCfgMap != nil { optStr, ok := oldCfgMap.Data["config"] if ok { - if cond(optStr) { + if cond != nil && cond(optStr) { return cfgMap, true, nil } } diff --git a/pkg/manager/component/vpcagent.go b/pkg/manager/component/vpcagent.go index 70c851df6..a9f675f75 100644 --- a/pkg/manager/component/vpcagent.go +++ b/pkg/manager/component/vpcagent.go @@ -17,13 +17,10 @@ package component import ( "fmt" "path" - "strings" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/vpcagent/options" @@ -90,8 +87,8 @@ func (m *vpcAgentManager) getConfigMap(oc *v1alpha1.OnecloudCluster, cfg *v1alph opts.SslCertfile = path.Join(constants.CertDir, constants.ServiceCertName) opts.SslKeyfile = path.Join(constants.CertDir, constants.ServiceKeyName) opts.Port = constants.VpcAgentPort - // return m.newServiceConfigMap(v1alpha1.VpcAgentComponentType, "", oc, opts), false, nil - return m.shouldSyncConfigmap(oc, v1alpha1.VpcAgentComponentType, opts, func(oldOpt string) bool { + return m.newServiceConfigMap(v1alpha1.VpcAgentComponentType, "", oc, opts), false, nil + /*return m.shouldSyncConfigmap(oc, v1alpha1.VpcAgentComponentType, opts, func(oldOpt string) bool { for _, k := range []string{ "fetch_data_from_compute_service: false", } { @@ -102,7 +99,7 @@ func (m *vpcAgentManager) getConfigMap(oc *v1alpha1.OnecloudCluster, cfg *v1alph } } return false - }) + })*/ } func (m *vpcAgentManager) getPhaseControl(man controller.ComponentManager, zone string) controller.PhaseControl { diff --git a/pkg/service-init/component/apimap.go b/pkg/service-init/component/apimap.go deleted file mode 100644 index 046141c54..000000000 --- a/pkg/service-init/component/apimap.go +++ /dev/null @@ -1,47 +0,0 @@ -package component - -import ( - "yunion.io/x/onecloud/pkg/apimap/options" - "yunion.io/x/pkg/errors" - - "yunion.io/x/onecloud-operator/pkg/apis/constants" - "yunion.io/x/onecloud-operator/pkg/apis/onecloud/v1alpha1" - "yunion.io/x/onecloud-operator/pkg/controller" - "yunion.io/x/onecloud-operator/pkg/util/option" -) - -type apiMap struct { - *baseService -} - -func NewAPIMap() Component { - return &apiMap{newBaseService(v1alpha1.APIMapComponentType, new(options.SOptions))} -} - -func (am *apiMap) GetConfig(oc *v1alpha1.OnecloudCluster, cfg *v1alpha1.OnecloudClusterConfig) (interface{}, error) { - opt := options.GetOptions() - if err := option.SetOptionsDefault(opt, constants.ServiceTypeAPIMap); err != nil { - return nil, errors.Wrap(err, "apimap: SetOptionsDefault") - } - // apimap use region config directly - config := cfg.RegionServer - - switch oc.Spec.GetDbEngine(oc.Spec.APIMap.DbEngine) { - case v1alpha1.DBEngineDameng: - option.SetDamengOptions(&opt.DBOptions, oc.Spec.Dameng, config.DB) - case v1alpha1.DBEngineMySQL: - fallthrough - default: - option.SetMysqlOptions(&opt.DBOptions, oc.Spec.Mysql, config.DB) - } - - option.SetOptionsServiceTLS(&opt.BaseOptions, false) - option.SetServiceCommonOptions(&opt.CommonOptions, oc, config.ServiceDBCommonOptions.ServiceCommonOptions, cfg.CommonConfig) - - opt.Port = constants.APIMapPort - return opt, nil -} - -func (am *apiMap) GetPhaseControl(man controller.ComponentManager) controller.PhaseControl { - return controller.NewRegisterEndpointComponent(man, v1alpha1.APIMapComponentType, constants.ServiceNameAPIMap, constants.ServiceTypeAPIMap, man.GetCluster().Spec.APIMap.Service.NodePort, "") -} diff --git a/vendor/github.com/go-openapi/swag/.golangci.yml b/vendor/github.com/go-openapi/swag/.golangci.yml index 813c47aa6..842ac1c09 100644 --- a/vendor/github.com/go-openapi/swag/.golangci.yml +++ b/vendor/github.com/go-openapi/swag/.golangci.yml @@ -37,3 +37,5 @@ linters: - gci - gocognit - paralleltest + - thelper + - ifshort diff --git a/vendor/github.com/go-yaml/yaml/.travis.yml b/vendor/github.com/go-yaml/yaml/.travis.yml new file mode 100644 index 000000000..9f556934d --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/.travis.yml @@ -0,0 +1,12 @@ +language: go + +go: + - 1.4 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + - tip + +go_import_path: gopkg.in/yaml.v2 diff --git a/vendor/github.com/go-yaml/yaml/LICENSE b/vendor/github.com/go-yaml/yaml/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/go-yaml/yaml/LICENSE.libyaml b/vendor/github.com/go-yaml/yaml/LICENSE.libyaml new file mode 100644 index 000000000..8da58fbf6 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/LICENSE.libyaml @@ -0,0 +1,31 @@ +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original copyright and license: + + apic.go + emitterc.go + parserc.go + readerc.go + scannerc.go + writerc.go + yamlh.go + yamlprivateh.go + +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/go-yaml/yaml/NOTICE b/vendor/github.com/go-yaml/yaml/NOTICE new file mode 100644 index 000000000..866d74a7a --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/vendor/github.com/go-yaml/yaml/README.md b/vendor/github.com/go-yaml/yaml/README.md new file mode 100644 index 000000000..2ed3314c7 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/README.md @@ -0,0 +1,135 @@ +# YAML support for the Go language + +Introduction +------------ + +The yaml package enables Go programs to comfortably encode and decode YAML +values. It was developed within [Canonical](https://www.canonical.com) as +part of the [juju](https://juju.ubuntu.com) project, and is based on a +pure Go port of the well-known [libyaml](http://pyyaml.org/wiki/LibYAML) +C library to parse and generate YAML data quickly and reliably. + +Compatibility +------------- + +The yaml package supports most of YAML 1.1 and 1.2, including support for +anchors, tags, map merging, etc. Multi-document unmarshalling is not yet +implemented, and base-60 floats from YAML 1.1 are purposefully not +supported since they're a poor design and are gone in YAML 1.2. + +Installation and usage +---------------------- + +The import path for the package is *gopkg.in/yaml.v2*. + +To install it, run: + + go get gopkg.in/yaml.v2 + +API documentation +----------------- + +If opened in a browser, the import path itself leads to the API documentation: + + * [https://gopkg.in/yaml.v2](https://gopkg.in/yaml.v2) + +API stability +------------- + +The package API for yaml v2 will remain stable as described in [gopkg.in](https://gopkg.in). + + +License +------- + +The yaml package is licensed under the Apache License 2.0. Please see the LICENSE file for details. + + +Example +------- + +Some more examples can be found in the "examples" folder. + +```Go +package main + +import ( + "fmt" + "log" + + "gopkg.in/yaml.v2" +) + +var data = ` +a: Easy! +b: + c: 2 + d: [3, 4] +` + +// Note: struct fields must be public in order for unmarshal to +// correctly populate the data. +type T struct { + A string + B struct { + RenamedC int `yaml:"c"` + D []int `yaml:",flow"` + } +} + +func main() { + t := T{} + + err := yaml.Unmarshal([]byte(data), &t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t:\n%v\n\n", t) + + d, err := yaml.Marshal(&t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t dump:\n%s\n\n", string(d)) + + m := make(map[interface{}]interface{}) + + err = yaml.Unmarshal([]byte(data), &m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m:\n%v\n\n", m) + + d, err = yaml.Marshal(&m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m dump:\n%s\n\n", string(d)) +} +``` + +This example will generate the following output: + +``` +--- t: +{Easy! {2 [3 4]}} + +--- t dump: +a: Easy! +b: + c: 2 + d: [3, 4] + + +--- m: +map[a:Easy! b:map[c:2 d:[3 4]]] + +--- m dump: +a: Easy! +b: + c: 2 + d: + - 3 + - 4 +``` + diff --git a/vendor/github.com/go-yaml/yaml/apic.go b/vendor/github.com/go-yaml/yaml/apic.go new file mode 100644 index 000000000..3e24a0d7d --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/apic.go @@ -0,0 +1,739 @@ +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +//// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +///* +// * Create ALIAS. +// */ +// +//YAML_DECLARE(int) +//yaml_alias_event_initialize(event *yaml_event_t, anchor *yaml_char_t) +//{ +// mark yaml_mark_t = { 0, 0, 0 } +// anchor_copy *yaml_char_t = NULL +// +// assert(event) // Non-NULL event object is expected. +// assert(anchor) // Non-NULL anchor is expected. +// +// if (!yaml_check_utf8(anchor, strlen((char *)anchor))) return 0 +// +// anchor_copy = yaml_strdup(anchor) +// if (!anchor_copy) +// return 0 +// +// ALIAS_EVENT_INIT(*event, anchor_copy, mark, mark) +// +// return 1 +//} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compliler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/vendor/github.com/go-yaml/yaml/decode.go b/vendor/github.com/go-yaml/yaml/decode.go new file mode 100644 index 000000000..c8eac1642 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/decode.go @@ -0,0 +1,764 @@ +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +const ( + documentNode = 1 << iota + mappingNode + sequenceNode + scalarNode + aliasNode +) + +type node struct { + kind int + line, column int + tag string + // For an alias node, alias holds the resolved alias. + alias *node + value string + implicit bool + children []*node + anchors map[string]*node +} + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *node, anchor []byte) { + if anchor != nil { + p.doc.anchors[string(anchor)] = n + } +} + +func (p *parser) parse() *node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + default: + panic("attempted to parse unknown event: " + p.event.typ.String()) + } +} + +func (p *parser) node(kind int) *node { + return &node{ + kind: kind, + line: p.event.start_mark.line, + column: p.event.start_mark.column, + } +} + +func (p *parser) document() *node { + n := p.node(documentNode) + n.anchors = make(map[string]*node) + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + n.children = append(n.children, p.parse()) + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *node { + n := p.node(aliasNode) + n.value = string(p.event.anchor) + n.alias = p.doc.anchors[n.value] + if n.alias == nil { + failf("unknown anchor '%s' referenced", n.value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *node { + n := p.node(scalarNode) + n.value = string(p.event.value) + n.tag = string(p.event.tag) + n.implicit = p.event.implicit + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *node { + n := p.node(sequenceNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + n.children = append(n.children, p.parse()) + } + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *node { + n := p.node(mappingNode) + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + n.children = append(n.children, p.parse(), p.parse()) + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *node + aliases map[*node]bool + mapType reflect.Type + terrors []string + strict bool +} + +var ( + mapItemType = reflect.TypeOf(MapItem{}) + durationType = reflect.TypeOf(time.Duration(0)) + defaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = defaultMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder(strict bool) *decoder { + d := &decoder{mapType: defaultMapType, strict: strict} + d.aliases = make(map[*node]bool) + return d +} + +func (d *decoder) terror(n *node, tag string, out reflect.Value) { + if n.tag != "" { + tag = n.tag + } + value := n.value + if tag != yaml_SEQ_TAG && tag != yaml_MAP_TAG { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.line+1, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *node, u Unmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.tag == yaml_NULL_TAG || n.kind == scalarNode && n.tag == "" && (n.value == "null" || n.value == "~" || n.value == "" && n.implicit) { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + if u, ok := out.Addr().Interface().(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) unmarshal(n *node, out reflect.Value) (good bool) { + switch n.kind { + case documentNode: + return d.document(n, out) + case aliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.kind { + case scalarNode: + good = d.scalar(n, out) + case mappingNode: + good = d.mapping(n, out) + case sequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(n.kind)) + } + return good +} + +func (d *decoder) document(n *node, out reflect.Value) (good bool) { + if len(n.children) == 1 { + d.doc = n + d.unmarshal(n.children[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.value) + } + d.aliases[n] = true + good = d.unmarshal(n.alias, out) + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.tag == "" && !n.implicit { + tag = yaml_STR_TAG + resolved = n.value + } else { + tag, resolved = resolve(n.tag, n.value) + if tag == yaml_BINARY_TAG { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.Kind() == reflect.Map && !out.CanAddr() { + resetMap(out) + } else { + out.Set(reflect.Zero(out.Type())) + } + return true + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == yaml_BINARY_TAG { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == yaml_BINARY_TAG { + out.SetString(resolved.(string)) + return true + } + if resolved != nil { + out.SetString(n.value) + return true + } + case reflect.Interface: + if resolved == nil { + out.Set(reflect.Zero(out.Type())) + } else if tag == yaml_TIMESTAMP_TAG { + // It looks like a timestamp but for backward compatibility + // reasons we set it as a string, so that code that unmarshals + // timestamp-like values into interface{} will continue to + // see a string and not a time.Time. + out.Set(reflect.ValueOf(n.value)) + } else { + out.Set(reflect.ValueOf(resolved)) + } + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + switch resolved := resolved.(type) { + case int: + if !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + if out.Type().Elem() == reflect.TypeOf(resolved) { + // TODO DOes this make sense? When is out a Ptr except when decoding a nil value? + elem := reflect.New(out.Type().Elem()) + elem.Elem().Set(reflect.ValueOf(resolved)) + out.Set(elem) + return true + } + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *node, out reflect.Value) (good bool) { + l := len(n.children) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, yaml_SEQ_TAG, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.children[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + out.Set(out.Slice(0, j)) + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *node, out reflect.Value) (good bool) { + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Slice: + return d.mappingSlice(n, out) + case reflect.Map: + // okay + case reflect.Interface: + if d.mapType.Kind() == reflect.Map { + iface := out + out = reflect.MakeMap(d.mapType) + iface.Set(out) + } else { + slicev := reflect.New(d.mapType).Elem() + if !d.mappingSlice(n, slicev) { + return false + } + out.Set(slicev) + return true + } + default: + d.terror(n, yaml_MAP_TAG, out) + return false + } + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + mapType := d.mapType + if outt.Key() == ifaceType && outt.Elem() == ifaceType { + d.mapType = outt + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + l := len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.children[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.children[i+1], e) { + d.setMapIndex(n.children[i+1], out, k, e) + } + } + } + d.mapType = mapType + return true +} + +func (d *decoder) setMapIndex(n *node, out, k, v reflect.Value) { + if d.strict && out.MapIndex(k) != zeroValue { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: key %#v already set in map", n.line+1, k.Interface())) + return + } + out.SetMapIndex(k, v) +} + +func (d *decoder) mappingSlice(n *node, out reflect.Value) (good bool) { + outt := out.Type() + if outt.Elem() != mapItemType { + d.terror(n, yaml_MAP_TAG, out) + return false + } + + mapType := d.mapType + d.mapType = outt + + var slice []MapItem + var l = len(n.children) + for i := 0; i < l; i += 2 { + if isMerge(n.children[i]) { + d.merge(n.children[i+1], out) + continue + } + item := MapItem{} + k := reflect.ValueOf(&item.Key).Elem() + if d.unmarshal(n.children[i], k) { + v := reflect.ValueOf(&item.Value).Elem() + if d.unmarshal(n.children[i+1], v) { + slice = append(slice, item) + } + } + } + out.Set(reflect.ValueOf(slice)) + d.mapType = mapType + return true +} + +func (d *decoder) mappingStruct(n *node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + name := settableValueOf("") + l := len(n.children) + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + var doneFields []bool + if d.strict { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + for i := 0; i < l; i += 2 { + ni := n.children[i] + if isMerge(ni) { + d.merge(n.children[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.strict { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.line+1, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = out.FieldByIndex(info.Inline) + } + d.unmarshal(n.children[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.children[i+1], value) + d.setMapIndex(n.children[i+1], inlineMap, name, value) + } else if d.strict { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.line+1, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *node, out reflect.Value) { + switch n.kind { + case mappingNode: + d.unmarshal(n, out) + case aliasNode: + an, ok := d.doc.anchors[n.value] + if ok && an.kind != mappingNode { + failWantMap() + } + d.unmarshal(n, out) + case sequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.children) - 1; i >= 0; i-- { + ni := n.children[i] + if ni.kind == aliasNode { + an, ok := d.doc.anchors[ni.value] + if ok && an.kind != mappingNode { + failWantMap() + } + } else if ni.kind != mappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *node) bool { + return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG) +} diff --git a/vendor/github.com/go-yaml/yaml/emitterc.go b/vendor/github.com/go-yaml/yaml/emitterc.go new file mode 100644 index 000000000..cf0db118a --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/emitterc.go @@ -0,0 +1,1685 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + emitter.column = 0 + emitter.line++ + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + emitter.column = 0 + emitter.line++ + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + return yaml_emitter_emit_node(emitter, event, true, false, false, false) +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, emitter.mapping_context && !emitter.indention) { + return false + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + return yaml_emitter_emit_node(emitter, event, false, true, false, false) +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an achor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + emitter.indention = true + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + emitter.whitespace = false + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} diff --git a/vendor/github.com/go-yaml/yaml/encode.go b/vendor/github.com/go-yaml/yaml/encode.go new file mode 100644 index 000000000..1e730eff6 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/encode.go @@ -0,0 +1,358 @@ +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + // doneInit holds whether the initial stream_start_event has been + // emitted. + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch m := iface.(type) { + case time.Time, *time.Time: + // Although time.Time implements TextMarshaler, + // we don't want to treat it as a string for YAML + // purposes because YAML has special support for + // timestamps. + case Marshaler: + v, err := m.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + in = reflect.ValueOf(v) + case encoding.TextMarshaler: + text, err := m.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + if in.Type() == ptrTimeType { + e.timev(tag, in.Elem()) + } else { + e.marshal(tag, in.Elem()) + } + case reflect.Struct: + if in.Type() == timeType { + e.timev(tag, in) + } else { + e.structv(tag, in) + } + case reflect.Slice: + if in.Type().Elem() == mapItemType { + e.itemsv(tag, in) + } else { + e.slicev(tag, in) + } + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if in.Type() == durationType { + e.stringv(tag, reflect.ValueOf(iface.(time.Duration).String())) + } else { + e.intv(tag, in) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) itemsv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + slice := in.Convert(reflect.TypeOf([]MapItem{})).Interface().([]MapItem) + for _, item := range slice { + e.marshal("", reflect.ValueOf(item.Key)) + e.marshal("", reflect.ValueOf(item.Value)) + } + }) +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = in.FieldByIndex(info.Inline) + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == yaml_BINARY_TAG { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = yaml_BINARY_TAG + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == yaml_STR_TAG && !isBase60Float(s) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + if tag == "" { + tag = yaml_TIMESTAMP_TAG + } + e.emitScalar(t.Format(time.RFC3339Nano), "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + s := strconv.FormatFloat(in.Float(), 'g', -1, 64) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) { + implicit := tag == "" + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.emit() +} diff --git a/vendor/github.com/go-yaml/yaml/parserc.go b/vendor/github.com/go-yaml/yaml/parserc.go new file mode 100644 index 000000000..81d05dfe5 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/parserc.go @@ -0,0 +1,1095 @@ +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + return &parser.tokens[parser.tokens_head] + } + return nil +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + return true +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/vendor/github.com/go-yaml/yaml/readerc.go b/vendor/github.com/go-yaml/yaml/readerc.go new file mode 100644 index 000000000..f45079171 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/readerc.go @@ -0,0 +1,394 @@ +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/vendor/github.com/go-yaml/yaml/resolve.go b/vendor/github.com/go-yaml/yaml/resolve.go new file mode 100644 index 000000000..ea90bd5e0 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/resolve.go @@ -0,0 +1,245 @@ +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}}, + {true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}}, + {true, yaml_BOOL_TAG, []string{"on", "On", "ON"}}, + {false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}}, + {false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}}, + {false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}}, + {nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", yaml_MERGE_TAG, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + // TODO This can easily be made faster and produce less garbage. + if strings.HasPrefix(tag, longTagPrefix) { + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG, yaml_TIMESTAMP_TAG: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+][0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG: + return + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == yaml_TIMESTAMP_TAG { + t, ok := parseTimestamp(in) + if ok { + return yaml_TIMESTAMP_TAG, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return yaml_FLOAT_TAG, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, int(intv) + } else { + return yaml_INT_TAG, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return yaml_INT_TAG, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt(plain[3:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return yaml_INT_TAG, -int(intv) + } else { + return yaml_INT_TAG, -intv + } + } + } + default: + panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")") + } + } + return yaml_STR_TAG, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5Z07:00", + "2006-1-2t15:4:5Z07:00", // RFC3339 with lower-case "t". + "2006-1-2 15:4:5", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/vendor/github.com/go-yaml/yaml/scannerc.go b/vendor/github.com/go-yaml/yaml/scannerc.go new file mode 100644 index 000000000..492a9845d --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/scannerc.go @@ -0,0 +1,2702 @@ +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // Check if we really need to fetch more tokens. + need_more_tokens := false + + if parser.tokens_head == len(parser.tokens) { + // Queue is empty. + need_more_tokens = true + } else { + // Check if any potential simple key may occupy the head position. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + if simple_key.possible && simple_key.token_number == parser.tokens_parsed { + need_more_tokens = true + break + } + } + } + + // We are finished. + if !need_more_tokens { + break + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // Remove obsolete potential simple keys. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +// Check the list of potential simple keys and remove the positions that +// cannot contain simple keys anymore. +func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { + // Check for a potential simple key for each flow level. + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + + // The specification requires that a simple key + // + // - is limited to a single line, + // - is shorter than 1024 characters. + if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { + + // Check if the potential simple key to be removed is required. + if simple_key.required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + } + } + return true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // A simple key is required only when it is the first token in the current + // line. Therefore it is always allowed. But we add a check anyway. + if required && !parser.simple_key_allowed { + panic("should not happen") + } + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + } + simple_key.mark = parser.mark + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + return true +} + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // Increase the flow level. + parser.flow_level++ + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + } + return true +} + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + // Loop through the indentation levels in the stack. + for parser.indent > column { + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if simple_key.possible { + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab character that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violate indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} diff --git a/vendor/github.com/go-yaml/yaml/sorter.go b/vendor/github.com/go-yaml/yaml/sorter.go new file mode 100644 index 000000000..5958822f9 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/sorter.go @@ -0,0 +1,104 @@ +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + return bl + } + var ai, bi int + var an, bn int64 + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/vendor/github.com/go-yaml/yaml/writerc.go b/vendor/github.com/go-yaml/yaml/writerc.go new file mode 100644 index 000000000..a2dde608c --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/writerc.go @@ -0,0 +1,26 @@ +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/vendor/github.com/go-yaml/yaml/yaml.go b/vendor/github.com/go-yaml/yaml/yaml.go new file mode 100644 index 000000000..483aae587 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/yaml.go @@ -0,0 +1,466 @@ +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" +) + +// MapSlice encodes and decodes as a YAML map. +// The order of keys is preserved when encoding and decoding. +type MapSlice []MapItem + +// MapItem is an item in a MapSlice. +type MapItem struct { + Key, Value interface{} +} + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. The UnmarshalYAML +// method receives a function that may be called to unmarshal the original +// YAML value into a field or variable. It is safe to call the unmarshal +// function parameter more than once if necessary. +type Unmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// UnmarshalStrict is like Unmarshal except that any fields that are found +// in the data that do not have corresponding struct members, or mapping +// keys that are duplicates, will result in +// an error. +func UnmarshalStrict(in []byte, out interface{}) (err error) { + return unmarshal(in, out, true) +} + +// A Decorder reads and decodes YAML values from an input stream. +type Decoder struct { + strict bool + parser *parser +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// SetStrict sets whether strict decoding behaviour is enabled when +// decoding items in the data (see UnmarshalStrict). By default, decoding is not strict. +func (dec *Decoder) SetStrict(strict bool) { + dec.strict = strict +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder(dec.strict) + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder(strict) + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only unmarshalled if they are exported (have an upper case +// first letter), and are unmarshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be included if that method returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("Multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct: + sinfo, err := getStructInfo(field.Type) + if err != nil { + return nil, err + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + default: + //return nil, errors.New("Option ,inline needs a struct value or map field") + return nil, errors.New("Option ,inline needs a struct value field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "Duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/vendor/github.com/go-yaml/yaml/yamlh.go b/vendor/github.com/go-yaml/yaml/yamlh.go new file mode 100644 index 000000000..e25cee563 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/yamlh.go @@ -0,0 +1,738 @@ +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = iota + + yaml_PLAIN_SCALAR_STYLE // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/vendor/github.com/go-yaml/yaml/yamlprivateh.go b/vendor/github.com/go-yaml/yaml/yamlprivateh.go new file mode 100644 index 000000000..8110ce3c3 --- /dev/null +++ b/vendor/github.com/go-yaml/yaml/yamlprivateh.go @@ -0,0 +1,173 @@ +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/vendor/github.com/google/go-cmp/cmp/internal/function/func.go b/vendor/github.com/google/go-cmp/cmp/internal/function/func.go index d127d4362..def01a6be 100644 --- a/vendor/github.com/google/go-cmp/cmp/internal/function/func.go +++ b/vendor/github.com/google/go-cmp/cmp/internal/function/func.go @@ -19,6 +19,7 @@ const ( tbFunc // func(T) bool ttbFunc // func(T, T) bool + ttiFunc // func(T, T) int trbFunc // func(T, R) bool tibFunc // func(T, I) bool trFunc // func(T) R @@ -28,11 +29,13 @@ const ( Transformer = trFunc // func(T) R ValueFilter = ttbFunc // func(T, T) bool Less = ttbFunc // func(T, T) bool + Compare = ttiFunc // func(T, T) int ValuePredicate = tbFunc // func(T) bool KeyValuePredicate = trbFunc // func(T, R) bool ) var boolType = reflect.TypeOf(true) +var intType = reflect.TypeOf(0) // IsType reports whether the reflect.Type is of the specified function type. func IsType(t reflect.Type, ft funcType) bool { @@ -49,6 +52,10 @@ func IsType(t reflect.Type, ft funcType) bool { if ni == 2 && no == 1 && t.In(0) == t.In(1) && t.Out(0) == boolType { return true } + case ttiFunc: // func(T, T) int + if ni == 2 && no == 1 && t.In(0) == t.In(1) && t.Out(0) == intType { + return true + } case trbFunc: // func(T, R) bool if ni == 2 && no == 1 && t.Out(0) == boolType { return true diff --git a/vendor/github.com/google/go-cmp/cmp/options.go b/vendor/github.com/google/go-cmp/cmp/options.go index 754496f3b..ba3fce81f 100644 --- a/vendor/github.com/google/go-cmp/cmp/options.go +++ b/vendor/github.com/google/go-cmp/cmp/options.go @@ -232,7 +232,15 @@ func (validator) apply(s *state, vx, vy reflect.Value) { if t := s.curPath.Index(-2).Type(); t.Name() != "" { // Named type with unexported fields. name = fmt.Sprintf("%q.%v", t.PkgPath(), t.Name()) // e.g., "path/to/package".MyType - if _, ok := reflect.New(t).Interface().(error); ok { + isProtoMessage := func(t reflect.Type) bool { + m, ok := reflect.PointerTo(t).MethodByName("ProtoReflect") + return ok && m.Type.NumIn() == 1 && m.Type.NumOut() == 1 && + m.Type.Out(0).PkgPath() == "google.golang.org/protobuf/reflect/protoreflect" && + m.Type.Out(0).Name() == "Message" + } + if isProtoMessage(t) { + help = `consider using "google.golang.org/protobuf/testing/protocmp".Transform to compare proto.Message types` + } else if _, ok := reflect.New(t).Interface().(error); ok { help = "consider using cmpopts.EquateErrors to compare error values" } else if t.Comparable() { help = "consider using cmpopts.EquateComparable to compare comparable Go types" diff --git a/vendor/modules.txt b/vendor/modules.txt index 7c5c48ebd..412fa9e5c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -91,7 +91,7 @@ github.com/go-openapi/jsonpointer ## explicit; go 1.13 github.com/go-openapi/jsonreference github.com/go-openapi/jsonreference/internal -# github.com/go-openapi/swag v0.19.14 +# github.com/go-openapi/swag v0.19.15 ## explicit; go 1.11 github.com/go-openapi/swag # github.com/go-playground/locales v0.13.0 @@ -107,6 +107,9 @@ github.com/go-playground/validator/v10 # github.com/go-sql-driver/mysql v1.6.0 ## explicit; go 1.10 github.com/go-sql-driver/mysql +# github.com/go-yaml/yaml v2.1.0+incompatible +## explicit +github.com/go-yaml/yaml # github.com/gogo/protobuf v1.3.2 ## explicit; go 1.15 github.com/gogo/protobuf/gogoproto @@ -146,8 +149,8 @@ github.com/google/gnostic/extensions github.com/google/gnostic/jsonschema github.com/google/gnostic/openapiv2 github.com/google/gnostic/openapiv3 -# github.com/google/go-cmp v0.6.0 -## explicit; go 1.13 +# github.com/google/go-cmp v0.7.0 +## explicit; go 1.21 github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags @@ -1038,7 +1041,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260713095312-995dd1831cd6 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf ## explicit; go 1.24 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1054,14 +1057,14 @@ yunion.io/x/cloudmux/pkg/multicloud/objectstore/provider ## explicit; go 1.12 yunion.io/x/executor/apis yunion.io/x/executor/client -# yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 +# yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea ## explicit; go 1.18 yunion.io/x/jsonutils # yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 ## explicit; go 1.12 yunion.io/x/log yunion.io/x/log/hooks -# yunion.io/x/onecloud v0.0.0-20260714035258-73d7cbbb1f53 +# yunion.io/x/onecloud v0.0.0-20260922165722-171561b0780b ## explicit; go 1.24 yunion.io/x/onecloud/cmd/climc/shell yunion.io/x/onecloud/locales @@ -1233,7 +1236,7 @@ yunion.io/x/onecloud/pkg/util/yunionmeta yunion.io/x/onecloud/pkg/vpcagent/options yunion.io/x/onecloud/pkg/webconsole/options yunion.io/x/onecloud/pkg/yunionconf/options -# yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978 +# yunion.io/x/pkg v1.10.4-0.20260918012554-27cd9d2e093b ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors @@ -1281,11 +1284,11 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab +# yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends/clickhouse -# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c +# yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac ## explicit; go 1.12 yunion.io/x/structarg # github.com/jaypipes/ghw => github.com/zexi/ghw v0.9.1 diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/cloudprovider.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/cloudprovider.go index 7851b207e..d449aecd2 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/cloudprovider.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/cloudprovider.go @@ -312,6 +312,10 @@ type ICloudProvider interface { GetISSLCertificates() ([]ICloudSSLCertificate, error) CreateISSLCertificate(opts *SSLCertificateCreateOptions) (ICloudSSLCertificate, error) + GetIIpSets() ([]ICloudIpSet, error) + GetIIpSetById(id string) (ICloudIpSet, error) + CreateIIpSet(opts *IpSetCreateOptions) (ICloudIpSet, error) + GetIAiGateways() ([]IAiGateway, error) GetIAiGatewayById(id string) (IAiGateway, error) CreateIAiGateway(opts *AiGatewayCreateOptions) (IAiGateway, error) @@ -675,6 +679,18 @@ func (base *SBaseProvider) GetNotices() ([]INotice, error) { return nil, errors.Wrapf(ErrNotImplemented, "GetNotices") } +func (base *SBaseProvider) GetIIpSets() ([]ICloudIpSet, error) { + return nil, errors.Wrapf(ErrNotImplemented, "GetIIpSets") +} + +func (base *SBaseProvider) GetIIpSetById(id string) (ICloudIpSet, error) { + return nil, errors.Wrapf(ErrNotImplemented, "GetIIpSetById") +} + +func (base *SBaseProvider) CreateIIpSet(opts *IpSetCreateOptions) (ICloudIpSet, error) { + return nil, errors.Wrapf(ErrNotImplemented, "CreateIIpSet") +} + func NewBaseProvider(factory ICloudProviderFactory) SBaseProvider { return SBaseProvider{factory: factory} } diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/instance.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/instance.go index 3e3e86104..cda780e64 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/instance.go @@ -151,6 +151,11 @@ type SManagedVMCreateConfig struct { ExternalNetworkId string ExternalVpcId string IpAddr string + MacAddr string + Bridge string + Gateway string + Masklen int8 + Dns string Description string SysDisk SDiskInfo DataDisks []SDiskInfo diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/ipset.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/ipset.go new file mode 100644 index 000000000..94c9d3d4c --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/ipset.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +const ( + IpSetTypeIpv4CidrList = "ipv4_cidr_list" + IpSetTypeIpv6CidrList = "ipv6_cidr_list" +) + +type IpSetCreateOptions struct { + Name string + Desc string + IpSetType string + Addresses []string +} + +type IpSetUpdateOptions struct { + Name string + Addresses []string +} + +type ICloudIpSet interface { + IVirtualResource + + GetIpSetType() string + GetAddresses() []string + + Update(opts *IpSetUpdateOptions) error + Delete() error +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go index 76e3d0cf0..f84f08ff4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go @@ -485,8 +485,8 @@ var ( ) type MetricListOptions struct { - ResourceType TResourceType - MetricType TMetricType + ResourceType TResourceType `choices:"rds|server|host|redis|lb|bucket|k8s|storage|wire|cloudaccount_balance|modelarts|eip"` + MetricType TMetricType `choices:"rds_cpu.usage_active|rds_mem.used_percent|rds_netio.bps_recv|rds_netio.bps_sent|rds_disk.used_percent|rds_diskio.read_bps|rds_diskio.write_bps|rds_diskio.used_percent|rds_conn.used_count|rds_conn.active_count|rds_conn.used_percent|rds_conn.failed_count|rds_qps.query_qps|rds_tps.trans_qps|rds_innodb.read_bps|rds_innodb.write_bps|vm_cpu.usage_active|vm_mem.used_percent|vm_disk.used_percent|vm_diskio.read_bps|vm_diskio.write_bps|vm_diskio.read_iops|vm_diskio.write_iops|vm_netio.bps_recv|vm_netio.bps_sent|vm_netio.out_bandwidth_usage|vm_netio.tcp_connections|vm_process.number|cpu.usage_active|mem.used_percent|diskio.read_bps|diskio.write_bps|net.bps_recv|net.bps_sent|diskio.read_iops|diskio.write_iops|system.uptime|system.load1_pcore|processes.zombies|processes.total|disk.used_percent|mem.available|dcs_cpu.usage_active|dcs_mem.used_percent|dcs_netio.bps_recv|dcs_netio.bps_sent|dcs_conn.used_conn|dcs_instantopt.opt_sec|dcs_cachekeys.key_count|dcs_cachekeys.expire_key_count|dcs_datamem.used_byte|haproxy.used_snat_port|haproxy.snat_conn_count|haproxy.bin|haproxy.bout|haproxy.packet_rx|haproxy.packet_tx|haproxy.inactive_connection|haproxy.active_connection|haproxy.max_connection|haproxy.unhealthy_server_count|haproxy.hrsp_Nxx|haproxy.drop_traffic_tx|haproxy.drop_traffic_rx|haproxy.drop_packet_tx|haproxy.drop_packet_rx|oss_netio.bps_sent|oss_netio.bps_recv|oss_latency.req_late|oss_req.req_count|oss_req.5xx_count|oss_req.4xx_count|oss_req.3xx_count|oss_req.2xx_count|oss_storage.size|k8s_node_cpu.usage_active|k8s_node_mem.used_percent|k8s_node_disk.used_percent|k8s_node_netio.bps_recv|k8s_node_netio.bps_sent|modelarts_pool_cpu.usage_percent|modelarts_pool_mem.usage_percent|modelarts_pool_gpu_mem.usage_percent|modelarts_pool_gpu_util.percent|modelarts_pool_npu_util.percent|modelarts_pool_npu_mem.usage_percent|modelarts_pool_disk.available_capacity|modelarts_pool_disk.capacity|modelarts_pool_disk.usage_percent|wire_cpu.usage_percent|wire_mem.usage_percent|wire_net.rt|wire_net.unreachable_rate|eip_net.bps_recv|eip_net.bps_sent|eip_net.drop_speed_rx"` ResourceId string // batch metric pull for tencentcloud diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go index fe2a201ee..bb62f5868 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go @@ -94,6 +94,10 @@ type ICloudRegion interface { GetISecurityGroupById(secgroupId string) (ICloudSecurityGroup, error) CreateISecurityGroup(opts *SecurityGroupCreateInput) (ICloudSecurityGroup, error) + GetIIpSets() ([]ICloudIpSet, error) + GetIIpSetById(id string) (ICloudIpSet, error) + CreateIIpSet(opts *IpSetCreateOptions) (ICloudIpSet, error) + CreateIVpc(opts *VpcCreateOptions) (ICloudVpc, error) CreateInternetGateway() (ICloudInternetGateway, error) CreateEIP(eip *SEip) (ICloudEIP, error) @@ -539,6 +543,7 @@ type ISecurityGroupRule interface { GetPorts() string GetDescription() string GetCIDRs() []string + GetTargetType() string Update(opts *SecurityGroupRuleUpdateOptions) error Delete() error diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/securitygroup.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/securitygroup.go index e46b9610c..735a278ba 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/securitygroup.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/securitygroup.go @@ -26,6 +26,13 @@ type SecurityGroupReference struct { Name string } +const ( + SecurityGroupRuleTargetTypeCidr = "cidr" + SecurityGroupRuleTargetTypeIpSet = "ip_set" + SecurityGroupRuleTargetTypeIpSetGroup = "ip_set_group" + SecurityGroupRuleTargetTypeSecurityGroup = "security_group" +) + type SecurityGroupCreateInput struct { Name string Desc string @@ -36,22 +43,24 @@ type SecurityGroupCreateInput struct { } type SecurityGroupRuleCreateOptions struct { - Desc string - Priority int - Protocol string - Ports string - Direction secrules.TSecurityRuleDirection - CIDR string - Action secrules.TSecurityRuleAction + Desc string + Priority int + Protocol string + Ports string + Direction secrules.TSecurityRuleDirection + CIDR string + TargetType string + Action secrules.TSecurityRuleAction } type SecurityGroupRuleUpdateOptions struct { - CIDR string - Action secrules.TSecurityRuleAction - Desc string - Ports string - Protocol string - Priority int + CIDR string + TargetType string + Action secrules.TSecurityRuleAction + Desc string + Ports string + Protocol string + Priority int } func (rule *SecurityGroupRuleCreateOptions) String() string { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go index f277cc135..47b174c11 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go @@ -33,6 +33,7 @@ type SDiskConfig struct { UnitNumber int32 Key int32 ImagePath string + DestPath string IsRoot bool Datastore *SDatastore Preallocation string diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/monitor.go index 86ed37ac1..6dac0186c 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/monitor.go @@ -16,11 +16,13 @@ package esxi import ( "fmt" + "sync" "time" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/performance" "github.com/vmware/govmomi/view" + "github.com/vmware/govmomi/vim25/methods" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/log" @@ -34,6 +36,34 @@ const ( MONTYPE_VIRTUALMACHINE = "VirtualMachine" ) +var ( + metricTimeSkewLogMu sync.Mutex + metricTimeSkewLogAt = map[string]time.Time{} +) + +// alignMetricTimeRange 将本地计算的查询窗口对齐到 vCenter 时钟。 +// cloudmon 比 vCenter 快时，未对齐的时间窗对 vCenter 是「未来」，QueryPerf 会返回空。 +func (self *SESXiClient) alignMetricTimeRange(start, end time.Time) (time.Time, time.Time) { + startUTC := start.UTC() + endUTC := end.UTC() + vcNow, err := methods.GetCurrentTime(self.context, self.client.Client) + if err != nil || vcNow == nil { + return startUTC, endUTC + } + skew := time.Since(*vcNow) + if skew > time.Minute { + startUTC = startUTC.Add(-skew) + endUTC = endUTC.Add(-skew) + metricTimeSkewLogMu.Lock() + if time.Since(metricTimeSkewLogAt[self.host]) >= time.Hour { + log.Warningf("esxi metric time skew %s ahead of vCenter %s, aligned query window", skew.Round(time.Second), self.host) + metricTimeSkewLogAt[self.host] = time.Now() + } + metricTimeSkewLogMu.Unlock() + } + return startUTC, endUTC +} + func (self *SESXiClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { metricName := "" switch opts.MetricType { @@ -58,6 +88,8 @@ func (self *SESXiClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) ([ } func (self *SESXiClient) getEcsMetrics(metricName string, metricType cloudprovider.TMetricType, start, end time.Time) ([]cloudprovider.MetricValues, error) { + start, end = self.alignMetricTimeRange(start, end) + m := view.NewManager(self.client.Client) v, err := m.CreateContainerView(self.context, self.client.Client.ServiceContent.RootFolder, nil, true) if err != nil { @@ -80,6 +112,7 @@ func (self *SESXiClient) getEcsMetrics(metricName string, metricType cloudprovid queries := []types.PerfQuerySpec{} for i := range vms { + s, e := start, end query := types.PerfQuerySpec{ Entity: vms[i].Reference(), MetricId: []types.PerfMetricId{ @@ -89,8 +122,8 @@ func (self *SESXiClient) getEcsMetrics(metricName string, metricType cloudprovid }, Format: "normal", IntervalId: 20, - StartTime: &start, - EndTime: &end, + StartTime: &s, + EndTime: &e, } if metricType == cloudprovider.VM_METRIC_TYPE_DISK_USAGE { query.IntervalId = 0 @@ -165,6 +198,8 @@ func (self *SESXiClient) GetHostMetrics(opts *cloudprovider.MetricListOptions) ( } func (self *SESXiClient) getHostMetrics(metricName string, metricType cloudprovider.TMetricType, start, end time.Time) ([]cloudprovider.MetricValues, error) { + start, end = self.alignMetricTimeRange(start, end) + m := view.NewManager(self.client.Client) v, err := m.CreateContainerView(self.context, self.client.Client.ServiceContent.RootFolder, nil, true) if err != nil { @@ -193,6 +228,7 @@ func (self *SESXiClient) getHostMetrics(metricName string, metricType cloudprovi queries := []types.PerfQuerySpec{} for i := range vms { + s, e := start, end query := types.PerfQuerySpec{ Entity: vms[i].Reference(), MetricId: []types.PerfMetricId{ @@ -202,8 +238,8 @@ func (self *SESXiClient) getHostMetrics(metricName string, metricType cloudprovi }, Format: "normal", IntervalId: 0, - StartTime: &start, - EndTime: &end, + StartTime: &s, + EndTime: &e, } queries = append(queries, query) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/tags.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/tags.go index 9a068d1f0..8825109fb 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/tags.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/tags.go @@ -61,7 +61,10 @@ func (svm *SVirtualMachine) GetSysTags() map[string]string { for i := 3; i < len(paths); i++ { meta[fmt.Sprintf("folder_%d", i-3)] = paths[i] } - meta["networks"] = svm.getNetTags() + networks := svm.getNetTags() + if len(networks) > 0 { + meta["networks"] = networks + } // meta["datacenter"] = svm.GetDatacenterPathString() rp, _ := svm.getResourcePool() if rp != nil { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go index 21f25a3bc..f07245349 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go @@ -285,17 +285,105 @@ func (svm *SVirtualMachine) DoRebuildRoot(ctx context.Context, imagePath string, return svm.rebuildDisk(ctx, &svm.vdisks[0], imagePath, uefi) } +func rebuildRootDiskBackupPath(filename string) string { + const suffix = ".vmdk" + if strings.HasSuffix(strings.ToLower(filename), suffix) { + return filename[:len(filename)-len(suffix)] + ".rebuild-bak.vmdk" + } + return filename + ".rebuild-bak.vmdk" +} + +func (svm *SVirtualMachine) moveVirtualDiskFile(ctx context.Context, ds *SDatastore, src, dst string) error { + obj, err := ds.getDatastoreObj(ctx) + if err != nil { + return errors.Wrapf(err, "getDatastoreObj") + } + fm := obj.NewFileManager(ds.datacenter.getObjectDatacenter(), true) + err = fm.Move(ctx, src, dst) + if err != nil { + return errors.Wrapf(err, "move %s -> %s", src, dst) + } + return nil +} + +func (svm *SVirtualMachine) reattachNonRootDisks(ctx context.Context) error { + for i := 1; i < len(svm.vdisks); i++ { + err := svm.doDetachDisk(ctx, &svm.vdisks[i], false) + if err != nil { + return errors.Wrapf(err, "doDetachDisk %d", i) + } + err = svm.doAttachDisk(ctx, &svm.vdisks[i]) + if err != nil { + return errors.Wrapf(err, "doAttachDisk: %d", i) + } + } + return nil +} + +func (svm *SVirtualMachine) attachDisksInOriginalOrder(ctx context.Context, root *SVirtualDisk) error { + for i := 1; i < len(svm.vdisks); i++ { + err := svm.doDetachDisk(ctx, &svm.vdisks[i], false) + if err != nil { + return errors.Wrapf(err, "doDetachDisk %d", i) + } + } + err := svm.doAttachDisk(ctx, root) + if err != nil { + for i := 1; i < len(svm.vdisks); i++ { + if attachErr := svm.doAttachDisk(ctx, &svm.vdisks[i]); attachErr != nil { + log.Errorf("reattach disk %d after root attach failed: %s", i, attachErr) + } + } + return errors.Wrap(err, "doAttachDisk root") + } + for i := 1; i < len(svm.vdisks); i++ { + err = svm.doAttachDisk(ctx, &svm.vdisks[i]) + if err != nil { + return errors.Wrapf(err, "doAttachDisk: %d", i) + } + } + return nil +} + +func (svm *SVirtualMachine) restoreRootDiskAfterRebuild(ctx context.Context, disk *SVirtualDisk, ds *SDatastore, origPath, backupPath string) error { + if err := ds.Delete2(ctx, origPath, false, true); err != nil { + log.Errorf("delete rebuilt root disk %s before restore: %s", origPath, err) + } + if err := svm.moveVirtualDiskFile(ctx, ds, backupPath, origPath); err != nil { + return errors.Wrap(err, "rename backup root disk") + } + return svm.attachDisksInOriginalOrder(ctx, disk) +} + func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, imagePath string, uefi bool) error { uuid := disk.GetId() sizeMb := disk.GetDiskSizeMB() diskKey := disk.getKey() ctlKey := disk.getControllerKey() unitNumber := *disk.dev.GetVirtualDevice().UnitNumber + origPath := disk.GetFilename() + backupPath := rebuildRootDiskBackupPath(origPath) + + istorage, err := disk.GetIStorage() + if err != nil { + return errors.Wrap(err, "GetIStorage") + } + ds := istorage.(*SDatastore) - err := svm.doDetachAndDeleteDisk(ctx, disk) + err = svm.doDetachDisk(ctx, disk, false) if err != nil { return err } + + err = svm.moveVirtualDiskFile(ctx, ds, origPath, backupPath) + if err != nil { + if attachErr := svm.attachDisksInOriginalOrder(ctx, disk); attachErr != nil { + log.Errorf("reattach disks in original order after backup failed: %s", attachErr) + } + return errors.Wrapf(err, "backup root disk %s", origPath) + } + log.Infof("backup root disk %s -> %s", origPath, backupPath) + err = svm.createDiskInternal(ctx, SDiskConfig{ Uefi: uefi, SizeMb: int64(sizeMb), @@ -304,20 +392,21 @@ func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, UnitNumber: unitNumber, Key: diskKey, ImagePath: imagePath, + DestPath: origPath, IsRoot: len(imagePath) > 0, + Datastore: ds, }, false) if err != nil { + if restoreErr := svm.restoreRootDiskAfterRebuild(ctx, disk, ds, origPath, backupPath); restoreErr != nil { + log.Errorf("restore root disk %s from %s failed: %s", origPath, backupPath, restoreErr) + } return errors.Wrapf(err, "createDiskInternal") } - for i := 1; i < len(svm.vdisks); i++ { - err = svm.doDetachDisk(ctx, &svm.vdisks[i], false) - if err != nil { - return errors.Wrapf(err, "doDetachDisk %d", i) - } - err = svm.doAttachDisk(ctx, &svm.vdisks[i]) - if err != nil { - return errors.Wrapf(err, "doAttachDisk: %d", i) - } + if err = svm.reattachNonRootDisks(ctx); err != nil { + return err + } + if err = ds.Delete2(ctx, backupPath, false, true); err != nil { + log.Errorf("delete backup root disk %s: %s", backupPath, err) } return nil } @@ -966,6 +1055,18 @@ func (svm *SVirtualMachine) getNetTags() string { info = append(info, nicConf.Mac, nicConf.Network) info = append(info, nicConf.IPs...) } + if len(info) > 0 { + return strings.Join(info, "/") + } + // guest.net 为空时回退读取虚拟网卡硬件配置 + for i := range svm.vnics { + mac := svm.vnics[i].GetMAC() + network := svm.vnics[i].GetNetworkName() + if len(mac) == 0 && len(network) == 0 { + continue + } + info = append(info, mac, network) + } return strings.Join(info, "/") } @@ -978,6 +1079,9 @@ type sNicConfig struct { func (svm *SVirtualMachine) fetchGuestIps() map[string]sNicConfig { guestIps := make(map[string]sNicConfig) moVM := svm.getVirtualMachine() + if moVM.Guest == nil { + return guestIps + } for _, net := range moVM.Guest.Net { if len(net.Network) == 0 { continue @@ -1241,10 +1345,24 @@ func (svm *SVirtualMachine) GetRootImagePath() (string, error) { return path, nil } -func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath string) (string, error) { - newImagePath, datastore, err := svm.getDatastoreAndRootImagePath(false) - if err != nil { - return "", errors.Wrapf(err, "GetRootImagePath") +func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath, destPath string, datastore *SDatastore) (string, error) { + var ( + newImagePath string + err error + ) + if len(destPath) > 0 { + newImagePath = destPath + if datastore == nil { + _, datastore, err = svm.getDatastoreAndRootImagePath(false) + if err != nil { + return "", errors.Wrapf(err, "GetRootImagePath") + } + } + } else { + newImagePath, datastore, err = svm.getDatastoreAndRootImagePath(false) + if err != nil { + return "", errors.Wrapf(err, "GetRootImagePath") + } } ds, err := datastore.getDatastoreObj(ctx) if err != nil { @@ -1263,7 +1381,7 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi // copy disk if len(config.ImagePath) > 0 { config.IsRoot = true - config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath) + config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath, config.DestPath, config.Datastore) if err != nil { return errors.Wrap(err, "unable to copyRootDisk") } @@ -1312,7 +1430,6 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi } func (svm *SVirtualMachine) createDiskInternal(ctx context.Context, config SDiskConfig, check bool) error { - return svm.createDiskWithDeviceChange(ctx, nil, config, check) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/vnic.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/vnic.go index 8c25f5acb..4bb387ffe 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/vnic.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/vnic.go @@ -63,7 +63,34 @@ func (nic *SVirtualNIC) GetDriver() string { } func (nic *SVirtualNIC) GetMAC() string { - return netutils.FormatMacAddr(nic.getVirtualEthernetCard().MacAddress) + card := nic.getVirtualEthernetCard() + if card == nil { + return "" + } + return netutils.FormatMacAddr(card.MacAddress) +} + +func (nic *SVirtualNIC) GetNetworkName() string { + card := nic.getVirtualEthernetCard() + if card == nil { + return "" + } + switch bk := card.Backing.(type) { + case *types.VirtualEthernetCardNetworkBackingInfo: + if len(bk.DeviceName) > 0 { + return bk.DeviceName + } + case *types.VirtualEthernetCardOpaqueNetworkBackingInfo: + if len(bk.OpaqueNetworkId) > 0 { + return bk.OpaqueNetworkId + } + } + if card.DeviceInfo != nil { + if desc := card.DeviceInfo.GetDescription(); desc != nil && len(desc.Summary) > 0 { + return desc.Summary + } + } + return "" } func (nic *SVirtualNIC) InClassicNetwork() bool { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go index 9b1f6d409..ab5517b87 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/region_base.go @@ -51,6 +51,18 @@ func (r *SRegion) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, err return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISecurityGroups") } +func (r *SRegion) GetIIpSets() ([]cloudprovider.ICloudIpSet, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIIpSets") +} + +func (r *SRegion) GetIIpSetById(id string) (cloudprovider.ICloudIpSet, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIIpSetById") +} + +func (r *SRegion) CreateIIpSet(opts *cloudprovider.IpSetCreateOptions) (cloudprovider.ICloudIpSet, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "CreateIIpSet") +} + func (r *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIStorageById") } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/secgroup_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/secgroup_base.go index 531881875..d58644da9 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/secgroup_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/secgroup_base.go @@ -23,6 +23,13 @@ type SSecurityGroup struct { SVirtualResourceBase } +type SSecurityGroupRuleBase struct { +} + +func (self *SSecurityGroupRuleBase) GetTargetType() string { + return cloudprovider.SecurityGroupRuleTargetTypeCidr +} + func (self *SSecurityGroup) GetReferences() ([]cloudprovider.SecurityGroupReference, error) { return []cloudprovider.SecurityGroupReference{}, nil } diff --git a/vendor/yunion.io/x/jsonutils/Makefile b/vendor/yunion.io/x/jsonutils/Makefile index f101cd8b0..2ab03a3a4 100644 --- a/vendor/yunion.io/x/jsonutils/Makefile +++ b/vendor/yunion.io/x/jsonutils/Makefile @@ -6,4 +6,4 @@ GOPROXY ?= direct mod: GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x \ go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p')) - go mod tidy + GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go mod tidy diff --git a/vendor/yunion.io/x/jsonutils/access.go b/vendor/yunion.io/x/jsonutils/access.go index 7d687d3e1..f236cc956 100644 --- a/vendor/yunion.io/x/jsonutils/access.go +++ b/vendor/yunion.io/x/jsonutils/access.go @@ -20,10 +20,18 @@ import ( "time" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" "yunion.io/x/pkg/util/timeutils" ) +func normalizeJSONObject(obj JSONObject) JSONObject { + if gotypes.IsNil(obj) { + return JSONNull + } + return obj +} + type JSONPair struct { key string val JSONObject @@ -40,7 +48,7 @@ func NewDict(objs ...JSONPair) *JSONDict { func NewArray(objs ...JSONObject) *JSONArray { arr := JSONArray{data: make([]JSONObject, 0, len(objs))} for _, o := range objs { - arr.data = append(arr.data, o) + arr.data = append(arr.data, normalizeJSONObject(o)) } return &arr } @@ -74,7 +82,7 @@ func NewBool(val bool) *JSONBool { } func (this *JSONDict) Set(key string, value JSONObject) { - this.data = sortedmap.Add(this.data, key, value) + this.data = sortedmap.Add(this.data, key, normalizeJSONObject(value)) } func (this *JSONDict) Remove(key string) bool { @@ -130,12 +138,12 @@ func (this *JSONDict) Add(o JSONObject, keys ...string) error { } func (this *JSONArray) SetAt(idx int, obj JSONObject) { - this.data[idx] = obj + this.data[idx] = normalizeJSONObject(obj) } func (this *JSONArray) Add(objs ...JSONObject) { for _, o := range objs { - this.data = append(this.data, o) + this.data = append(this.data, normalizeJSONObject(o)) } } diff --git a/vendor/yunion.io/x/jsonutils/consts.go b/vendor/yunion.io/x/jsonutils/consts.go index 53eb20fd6..7658de9df 100644 --- a/vendor/yunion.io/x/jsonutils/consts.go +++ b/vendor/yunion.io/x/jsonutils/consts.go @@ -20,4 +20,7 @@ import ( const ( TAG_DEPRECATED_BY = reflectutils.TAG_DEPRECATED_BY + + // maxParseDepth is the maximum nesting level accepted by the parser + maxParseDepth = 10000 ) diff --git a/vendor/yunion.io/x/jsonutils/equals.go b/vendor/yunion.io/x/jsonutils/equals.go index b49cc0f7c..ad341834f 100644 --- a/vendor/yunion.io/x/jsonutils/equals.go +++ b/vendor/yunion.io/x/jsonutils/equals.go @@ -15,6 +15,7 @@ package jsonutils import ( + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) @@ -51,7 +52,14 @@ func (arr *JSONArray) Equals(json JSONObject) bool { return false } for i, v := range arr.data { - if !v.Equals(arr2.data[i]) { + oth := arr2.data[i] + if gotypes.IsNil(v) || gotypes.IsNil(oth) { + if gotypes.IsNil(v) != gotypes.IsNil(oth) { + return false + } + continue + } + if !v.Equals(oth) { return false } } diff --git a/vendor/yunion.io/x/jsonutils/errors.go b/vendor/yunion.io/x/jsonutils/errors.go index c6851ccac..0038481bc 100644 --- a/vendor/yunion.io/x/jsonutils/errors.go +++ b/vendor/yunion.io/x/jsonutils/errors.go @@ -26,10 +26,16 @@ const ( ErrInvalidHex = errors.Error("invalid hex") ErrInvalidRune = errors.Error("invalid 4 byte rune") + ErrInvalidNodeId = errors.Error("invalid node id") + ErrDuplicateNodeId = errors.Error("duplicate node id") + ErrNestedTooDeep = errors.Error("json nested too deep") + ErrNodeReferenceDisabled = errors.Error("node reference is not enabled, see ParseTrusted") + ErrTypeMismatch = errors.Error("unmarshal type mismatch") ErrArrayLengthMismatch = errors.Error("unmarshal array length mismatch") ErrInterfaceUnsupported = errors.Error("do not known how to deserialize json into this interface type") ErrMapKeyMustString = errors.Error("map key must be string") + ErrNodeNotFound = errors.Error("no node found for the reference") ErrMissingInputField = errors.Error("missing input field") ErrNilInputField = errors.Error("nil input field") diff --git a/vendor/yunion.io/x/jsonutils/interface.go b/vendor/yunion.io/x/jsonutils/interface.go index 7d2b7edcd..af6edccd4 100644 --- a/vendor/yunion.io/x/jsonutils/interface.go +++ b/vendor/yunion.io/x/jsonutils/interface.go @@ -15,6 +15,7 @@ package jsonutils import ( + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) @@ -41,6 +42,10 @@ func (self *JSONString) Interface() interface{} { func (self *JSONArray) Interface() interface{} { ret := make([]interface{}, len(self.data)) for i := 0; i < len(self.data); i += 1 { + if gotypes.IsNil(self.data[i]) { + ret[i] = nil + continue + } ret[i] = self.data[i].Interface() } return ret @@ -51,7 +56,12 @@ func (self *JSONDict) Interface() interface{} { for iter := sortedmap.NewIterator(self.data); iter.HasMore(); iter.Next() { k, v := iter.Get() - mapping[k] = v.(JSONObject).Interface() + jo, _ := v.(JSONObject) + if gotypes.IsNil(jo) { + mapping[k] = nil + continue + } + mapping[k] = jo.Interface() } return mapping diff --git a/vendor/yunion.io/x/jsonutils/jsonutils.go b/vendor/yunion.io/x/jsonutils/jsonutils.go index 6f1ab1548..36f007b0b 100644 --- a/vendor/yunion.io/x/jsonutils/jsonutils.go +++ b/vendor/yunion.io/x/jsonutils/jsonutils.go @@ -16,10 +16,13 @@ package jsonutils import ( "bytes" + "math" "reflect" + "sort" "strconv" "strings" "time" + "unicode/utf16" "unicode/utf8" "yunion.io/x/log" @@ -117,6 +120,11 @@ func skipEmpty(str []byte, offset int) int { return i } +// isFiniteFloat reports whether the value has a json representation +func isFiniteFloat(val float64) bool { + return !math.IsNaN(val) && !math.IsInf(val, 0) +} + func hexchar2num(v byte) (byte, error) { switch { case v >= '0' && v <= '9': @@ -183,9 +191,22 @@ ret: if e != nil { return "", i, NewJSONError(str, i, e.Error()) } + i += 4 + if utf16.IsSurrogate(r) { + // a character outside the BMP is written as a + // surrogate pair, e.g. 😀 + if i+6 <= len(str) && str[i] == '\\' && str[i+1] == 'u' { + r2, e2 := hexstr2rune(str[i+2 : i+6]) + if e2 == nil { + if combined := utf16.DecodeRune(r, r2); combined != utf8.RuneError { + r = combined + i += 6 + } + } + } + } runen = utf8.EncodeRune(runebytes, r) buffer = append(buffer, runebytes[0:runen]...) - i += 4 case 'x': i++ if i+2 >= len(str) { @@ -253,13 +274,23 @@ ret2: return string(str[offset:i]), false, i, nil } +// isNodeReference reports whether the token has the form of a node +// reference, that is a bare with an integer N +func isNodeReference(val string) bool { + if len(val) < 3 || val[0] != '<' || val[len(val)-1] != '>' { + return false + } + _, err := strconv.ParseInt(val[1:len(val)-1], 10, 64) + return err == nil +} + func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, int, error) { val, quote, i, e := parseString(str, offset) if e != nil { return nil, i, errors.Wrap(e, "parseString") } else if quote { return &JSONString{data: val}, i, nil - } else if val[0] == '<' && val[len(val)-1] == '>' { + } else if s.allowNodeReference && len(val) > 1 && val[0] == '<' && val[len(val)-1] == '>' { // Pointer val = val[1 : len(val)-1] ival, err := strconv.ParseInt(val, 10, 64) @@ -272,6 +303,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, } s.saveReferer(nodeId, ptr) return ptr, i, nil + } else if !s.allowNodeReference && isNodeReference(val) { + // a node reference can not be left unresolved and kept as a plain + // value: a caller could not tell it apart from a real string + return nil, i, errors.Wrap(ErrNodeReferenceDisabled, val) } else { lval := strings.ToLower(val) if len(lval) == 0 || lval == "null" || lval == "none" { @@ -288,9 +323,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, return &JSONInt{data: ival}, i, nil } fval, err := strconv.ParseFloat(val, 64) - if err == nil { + if err == nil && isFiniteFloat(fval) { return &JSONFloat{data: fval}, i, nil } + // nan and +-inf have no json representation, keep them as strings return &JSONString{data: val}, i, nil } } @@ -333,6 +369,12 @@ func escapeJsonChar(sb *strings.Builder, ch byte) { case '\t': sb.Write([]byte{'\\', 't'}) default: + // RFC 8259: U+0000–U+001F must be escaped as \uXXXX. + if ch < 0x20 { + const hexdigits = "0123456789abcdef" + sb.Write([]byte{'\\', 'u', '0', '0', hexdigits[ch>>4], hexdigits[ch&0xf]}) + return + } sb.WriteByte(ch) /*if ((ch >= 0x20 && ch <= 0x21) || (ch >= 0x23 || ch <= 0x5B) || (ch >= 0x5D && ch <= 0x10FFFF)) && ch != 0x81 && ch != 0x8d && ch != 0x8f && ch != 0x90 && ch != 0x9d { sb.WriteRune(ch) @@ -351,12 +393,33 @@ func escapeJsonChar(sb *strings.Builder, ch byte) { } } +// escapeJsonByte writes a byte that is not part of a valid utf-8 sequence +// as a \xXX escape, which parseQuoteString reads back unchanged +func escapeJsonByte(sb *strings.Builder, ch byte) { + const hexdigits = "0123456789abcdef" + sb.Write([]byte{'\\', 'x', hexdigits[ch>>4], hexdigits[ch&0xf]}) +} + func quoteString(str string) string { sb := &strings.Builder{} sb.Grow(len(str) + 2) sb.WriteByte('"') - for i := 0; i < len(str); i += 1 { - escapeJsonChar(sb, str[i]) + for i := 0; i < len(str); { + ch := str[i] + if ch < utf8.RuneSelf { + escapeJsonChar(sb, ch) + i++ + continue + } + r, size := utf8.DecodeRuneInString(str[i:]) + if r == utf8.RuneError && size <= 1 { + // keep a non utf-8 byte reversible instead of writing it out + escapeJsonByte(sb, ch) + i++ + continue + } + sb.WriteString(str[i : i+size]) + i += size } sb.WriteByte('"') return sb.String() @@ -425,6 +488,11 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted var e error = nil var key string var stop = false + // collect the keys first so that the sorted map can be built in key + // order: adding to a sorted map out of order shifts the whole tail + // on every insert + values := make(map[string]JSONObject) + keys := make([]string, 0) for !stop && i < len(str) { i = skipEmpty(str, i) if i >= len(str) { @@ -468,11 +536,18 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted if e != nil { return smap, i, nodeId, errors.Wrap(e, "parse misc") } - if key == jsonPointerKey { + if s.allowNodeReference && key == jsonPointerKey { // node id - nodeId = int(val.(*JSONInt).data) + jval, ok := val.(*JSONInt) + if !ok { + return smap, i, nodeId, errors.Wrap(ErrInvalidNodeId, jsonPointerKey) + } + nodeId = int(jval.data) } else { - smap = sortedmap.Add(smap, key, val) + if _, ok := values[key]; !ok { + keys = append(keys, key) + } + values[key] = val } i = skipEmpty(str, i) if i >= len(str) { @@ -488,6 +563,10 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted return smap, i, nodeId, NewJSONError(str, i, "Unexpected char") } } + sort.Strings(keys) + for _, key := range keys { + smap = sortedmap.Add(smap, key, values[key]) + } return smap, i, nodeId, nil } @@ -546,12 +625,20 @@ func (s *sJsonParseSession) parseArray(str []byte, offset int) ([]JSONObject, in } func (this *JSONDict) parse(s *sJsonParseSession, str []byte, offset int) (int, error) { + e := s.enter() + if e != nil { + return offset, errors.Wrap(e, "enter") + } + defer s.leave() smap, i, nodeId, e := s.parseDict(str, offset) if e == nil { this.nodeId = nodeId this.data = smap if this.nodeId > 0 { - s.saveNode(nodeId, this) + e = s.saveNode(nodeId, this) + if e != nil { + return i, errors.Wrap(e, "saveNode") + } } return i, nil } @@ -585,17 +672,21 @@ func (this *JSONDict) prettyString(level int) string { buffer.WriteByte('\n') buffer.WriteString(tab) buffer.WriteString(" ") - buffer.WriteByte('"') - buffer.WriteString(k) - buffer.WriteString("\":") - _, okdict := v.(*JSONDict) - _, okarray := v.(*JSONArray) - if okdict || okarray { - buffer.WriteByte('\n') - buffer.WriteString(v.prettyString(level + 2)) - } else { + buffer.WriteString(quoteString(k)) + buffer.WriteByte(':') + if gotypes.IsNil(v) { buffer.WriteByte(' ') - buffer.WriteString(v.String()) + buffer.WriteString("null") + } else { + _, okdict := v.(*JSONDict) + _, okarray := v.(*JSONArray) + if okdict || okarray { + buffer.WriteByte('\n') + buffer.WriteString(v.prettyString(level + 2)) + } else { + buffer.WriteByte(' ') + buffer.WriteString(v.String()) + } } idx++ } @@ -608,6 +699,11 @@ func (this *JSONDict) prettyString(level int) string { } func (this *JSONArray) parse(s *sJsonParseSession, str []byte, offset int) (int, error) { + e := s.enter() + if e != nil { + return offset, errors.Wrap(e, "enter") + } + defer s.leave() val, i, e := s.parseArray(str, offset) if e == nil { this.data = val @@ -633,7 +729,12 @@ func (this *JSONArray) prettyString(level int) string { buffer.WriteString(",") } buffer.WriteByte('\n') - buffer.WriteString(v.prettyString(level + 1)) + if gotypes.IsNil(v) { + buffer.WriteString(tab) + buffer.WriteString(" null") + } else { + buffer.WriteString(v.prettyString(level + 1)) + } } if len(this.data) > 0 { buffer.WriteByte('\n') @@ -648,12 +749,46 @@ func ParseString(str string) (JSONObject, error) { } func Parse(str []byte) (JSONObject, error) { - json, _, err := ParseStream(str, 0) - return json, err + json, offset, err := ParseStream(str, 0) + if err != nil { + return nil, err + } + if i := skipEmpty(str, offset); i < len(str) { + return nil, NewJSONError(str, i, "Unexpected content after the value") + } + return json, nil } +// ParseStream parses one value starting at offset, it returns the value and +// the offset just after it, so that a stream of concatenated values can be +// walked. Node references are not resolved, see ParseTrusted. func ParseStream(str []byte, offset int) (JSONObject, int, error) { - s := newJsonParseSession() + return parseStream(str, offset, false) +} + +// ParseTrusted parses a document from a trusted source, resolving the node +// references that Marshal writes for a cyclic object: the ___jnid_ key inside +// an object and a bare value referring to it. +// +// A document from an untrusted source must be parsed with Parse instead. A +// resolved reference makes two fields of the target struct point at the same +// object, which is what the round trip of a cyclic object needs, but it also +// lets a forged document do the same. +// +// Note that Marshal needs this syntax to terminate on a cyclic object, so it +// keeps writing it either way. +func ParseTrusted(str []byte) (JSONObject, error) { + json, _, err := parseStream(str, 0, true) + return json, err +} + +// ParseTrustedString is ParseTrusted for a string +func ParseTrustedString(str string) (JSONObject, error) { + return ParseTrusted([]byte(str)) +} + +func parseStream(str []byte, offset int, allowNodeReference bool) (JSONObject, int, error) { + s := newJsonParseSession(allowNodeReference) i := offset i = skipEmpty(str, i) var val JSONObject = nil diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index 12d6c1e50..5b8f41ea5 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -48,7 +48,7 @@ func (s *sJsonMarshalSession) marshalSlice(val reflect.Value, info *reflectutils objs := make([]JSONObject, 0) for i := 0; i < val.Len(); i += 1 { val := s.marshalValue(val.Index(i), nil, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objs = append(objs, val) } } @@ -84,7 +84,7 @@ func (s *sJsonMarshalSession) marshalMap(val reflect.Value, info *reflectutils.S for i := 0; i < len(keys); i += 1 { key := keys[i] val := s.marshalValue(val.MapIndex(key), nil, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objPairs = append(objPairs, JSONPair{key: fmt.Sprintf("%s", key), val: val}) } } @@ -128,12 +128,12 @@ func (s *sJsonMarshalSession) struct2JSONPairs(val reflect.Value, omitEmpty bool continue } key := jsonInfo.MarshalName() - if deprecatedBy, ok := fields[i].Info.Tags[TAG_DEPRECATED_BY]; ok { + if deprecatedBy, ok := fields[i].Info.Tag(TAG_DEPRECATED_BY); ok { depFields[key] = deprecatedBy continue } val := s.marshalValue(fields[i].Value, jsonInfo, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objPair := JSONPair{key: key, val: val} objPairs = append(objPairs, objPair) } @@ -181,6 +181,9 @@ func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo, bit int, o return nil } else if info != nil && info.ForceString { return NewString(fmt.Sprintf("%f", val)) + } else if !isFiniteFloat(val) { + // nan and +-inf have no json representation + return JSONNull } else { return NewFloat64(val) } @@ -191,6 +194,9 @@ func marshalFloat32(val float32, info *reflectutils.SStructFieldInfo, bit int, o return nil } else if info != nil && info.ForceString { return NewString(fmt.Sprintf("%f", val)) + } else if !isFiniteFloat(float64(val)) { + // nan and +-inf have no json representation + return JSONNull } else { return NewFloat32(val) } @@ -243,6 +249,13 @@ func marshalTime(val time.Time, info *reflectutils.SStructFieldInfo, omitEmpty b } } +// Marshal converts an object to a JSONObject, which can be written out with +// String. +// +// A cyclic object is written with the node reference syntax, that is the +// ___jnid_ key inside an object plus a bare value referring to it. That +// syntax is not standard json and its result is not a valid json document, +// and Parse does not resolve it: read such a document back with ParseTrusted. func Marshal(obj interface{}) JSONObject { if obj == nil { return JSONNull diff --git a/vendor/yunion.io/x/jsonutils/parse_session.go b/vendor/yunion.io/x/jsonutils/parse_session.go index 632dce9d3..f9a0485b9 100644 --- a/vendor/yunion.io/x/jsonutils/parse_session.go +++ b/vendor/yunion.io/x/jsonutils/parse_session.go @@ -15,7 +15,7 @@ package jsonutils import ( - "fmt" + "yunion.io/x/pkg/errors" ) type sNodeReferer struct { @@ -25,14 +25,36 @@ type sNodeReferer struct { type sJsonParseSession struct { objectMap map[int]*sNodeReferer + + // depth is the current nesting level of the object/array being parsed + depth int + + // allowNodeReference tells whether the ___jnid_ key and a bare + // value are read as node references, see ParseTrusted + allowNodeReference bool } -func newJsonParseSession() *sJsonParseSession { +func newJsonParseSession(allowNodeReference bool) *sJsonParseSession { return &sJsonParseSession{ - objectMap: make(map[int]*sNodeReferer), + objectMap: make(map[int]*sNodeReferer), + allowNodeReference: allowNodeReference, } } +// enter records entering one more nesting level, it fails if the nesting +// level exceeds maxParseDepth +func (s *sJsonParseSession) enter() error { + if s.depth >= maxParseDepth { + return ErrNestedTooDeep + } + s.depth++ + return nil +} + +func (s *sJsonParseSession) leave() { + s.depth-- +} + func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) { if nr, ok := s.objectMap[nodeId]; ok { nr.pointers = append(nr.pointers, ptr) @@ -43,10 +65,10 @@ func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) { } } -func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) { +func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) error { if nr, ok := s.objectMap[nodeId]; ok { if nr.node != nil { - panic(fmt.Sprintf("nodeId %d alreayd exists: %s != %s", nodeId, nr.node, node)) + return errors.Wrapf(ErrDuplicateNodeId, "node id %d", nodeId) } else { nr.node = node } @@ -56,4 +78,5 @@ func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) { pointers: nil, } } + return nil } diff --git a/vendor/yunion.io/x/jsonutils/querystring.go b/vendor/yunion.io/x/jsonutils/querystring.go index faded210f..91ce91a41 100644 --- a/vendor/yunion.io/x/jsonutils/querystring.go +++ b/vendor/yunion.io/x/jsonutils/querystring.go @@ -55,7 +55,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO if err != nil { return nil, errors.Wrapf(err, "get jsondict %s with %s fail", body, key) } - addQueryStringSeg(next, segs[1:], val) + _, err = addQueryStringSeg(next, segs[1:], val) + if err != nil { + return nil, errors.Wrapf(err, "addQueryStringSeg %s with %s fail", body, key) + } } return jbody, nil case *JSONArray: @@ -66,7 +69,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO if err != nil { return nil, errors.Wrapf(err, "get jsonarray %s at %d fail", body, index) } - addQueryStringSeg(next, segs[1:], val) + _, err = addQueryStringSeg(next, segs[1:], val) + if err != nil { + return nil, errors.Wrapf(err, "addQueryStringSeg %s at %d fail", body, index) + } } else if arrSize == index { // new next, err := addQueryStringSeg(nil, segs[1:], val) diff --git a/vendor/yunion.io/x/jsonutils/std_json.go b/vendor/yunion.io/x/jsonutils/std_json.go index f70091d8d..e5b4b9ea8 100644 --- a/vendor/yunion.io/x/jsonutils/std_json.go +++ b/vendor/yunion.io/x/jsonutils/std_json.go @@ -16,9 +16,9 @@ package jsonutils import ( "encoding/json" - "fmt" "reflect" + "yunion.io/x/log" "yunion.io/x/pkg/gotypes" ) @@ -110,11 +110,13 @@ func tryStdMarshal(v reflect.Value, marshalFunc func(v reflect.Value) JSONObject if m != nil { data, err := m.MarshalJSON() if err != nil { - panic(fmt.Sprintf("MarshalJSON of %q error: %v", v.String(), err)) + log.Errorf("MarshalJSON of %s error: %v", v.Type(), err) + return JSONNull } jo, err := Parse(data) if err != nil { - panic(fmt.Sprintf("Parse data %q to json of %q error: %v", data, v.String(), err)) + log.Errorf("Parse data %s to json of %s error: %v", data, v.Type(), err) + return JSONNull } return jo } diff --git a/vendor/yunion.io/x/jsonutils/string.go b/vendor/yunion.io/x/jsonutils/string.go index dfa791b59..e033ccbfc 100644 --- a/vendor/yunion.io/x/jsonutils/string.go +++ b/vendor/yunion.io/x/jsonutils/string.go @@ -33,6 +33,10 @@ func (this *JSONInt) String() string { } func (this *JSONFloat) String() string { + if !isFiniteFloat(this.data) { + // nan and +-inf have no json representation + return "null" + } if this.bit != 32 && this.bit != 64 { this.bit = 64 } diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index 4209253e7..fa088df9b 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -62,12 +62,26 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error { return errors.Wrap(err, "Get") } } - s := newJsonUnmarshalSession() value := reflect.ValueOf(o) - err := jo.unmarshalValue(s, reflect.Indirect(value)) + if value.IsValid() && value.Kind() == reflect.Ptr && value.IsNil() { + return errors.Wrapf(ErrTypeMismatch, "unmarshal into a nil pointer of type %s", value.Type()) + } + target := reflect.Indirect(value) + // a non addressable value can not be written to, unless it is a + // non nil map, which is a reference type + if target.IsValid() && !target.CanAddr() && + (target.Kind() != reflect.Map || target.IsNil()) { + return errors.Wrapf(ErrTypeMismatch, "unmarshal into a value of type %s, a pointer is required", target.Type()) + } + s := newJsonUnmarshalSession() + err := jo.unmarshalValue(s, target) if err != nil { return errors.Wrap(err, "jo.unmarshalValue") } + err = s.checkUnboundNodes() + if err != nil { + return errors.Wrap(err, "checkUnboundNodes") + } return nil } @@ -385,6 +399,11 @@ func (this *JSONString) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Va if err != nil { return err } + if !isFiniteFloat(floatVal) { + // nan and +-inf have no json representation, they would + // silently break any comparison the field takes part in + return errors.Wrap(ErrInvalidJsonFloat, "not a finite number") + } val.SetFloat(floatVal) } case reflect.Bool: @@ -476,7 +495,10 @@ func (this *JSONArray) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Val func (this *JSONDict) unmarshalValue(s *sJsonUnmarshalSession, val reflect.Value) error { if this.nodeId > 0 && val.CanAddr() { - s.saveNodeValue(this.nodeId, val.Addr()) + err := s.saveNodeValue(this.nodeId, val.Addr()) + if err != nil { + return errors.Wrap(err, "saveNodeValue") + } } return tryStdUnmarshal(s, this, val, this._unmarshalValue) } @@ -510,7 +532,11 @@ func (this *JSONDict) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Valu return err } if objPtr == nil { - val.Set(reflect.ValueOf(this.data)) // ??? + dataVal := reflect.ValueOf(this.data) + if !dataVal.Type().AssignableTo(val.Type()) { + return errors.Wrapf(ErrInterfaceUnsupported, "JSONDict.unmarshalValue: %s", val.Type()) + } + val.Set(dataVal) return nil } err = this.unmarshalValue(s, reflect.ValueOf(objPtr)) @@ -600,7 +626,7 @@ func setStructFieldAt(s *sJsonUnmarshalSession, key string, v JSONObject, fieldV if err != nil { return errors.Wrap(err, "JSONDict.unmarshalStruct") } - depInfo, ok := fieldValues[index].Info.Tags[TAG_DEPRECATED_BY] + depInfo, ok := fieldValues[index].Info.Tag(TAG_DEPRECATED_BY) if ok { err := setStructFieldAt(s, depInfo, v, fieldValues, keyIndexMap, visited) if err != nil { diff --git a/vendor/yunion.io/x/jsonutils/unmarshal_session.go b/vendor/yunion.io/x/jsonutils/unmarshal_session.go index 1b5d89c34..c0fe3833a 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal_session.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal_session.go @@ -16,6 +16,9 @@ package jsonutils import ( "reflect" + "sort" + + "yunion.io/x/pkg/errors" ) type sJsonNodeValues struct { @@ -34,13 +37,27 @@ func newJsonUnmarshalSession() *sJsonUnmarshalSession { } } -func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) { +// assignable reports whether the node value can be assigned to the value +// referring to that node +func assignable(nodeValue, refValue reflect.Value) bool { + if !nodeValue.IsValid() || !refValue.IsValid() { + return false + } + return nodeValue.Type().AssignableTo(refValue.Type()) +} + +func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) error { if nv, ok := s.objectMap[nodeId]; !ok { s.objectMap[nodeId] = &sJsonNodeValues{ nodeValue: val, nodeValueSet: true, } } else { + for i := range nv.targetValues { + if !assignable(val, nv.targetValues[i]) { + return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, nv.targetValues[i].Type()) + } + } nv.nodeValue = val nv.nodeValueSet = true for i := range nv.targetValues { @@ -48,10 +65,30 @@ func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) { } nv.targetValues = nil } + return nil +} + +// checkUnboundNodes reports the node ids that were referred to but never +// resolved, which would otherwise leave the referring field untouched +func (s *sJsonUnmarshalSession) checkUnboundNodes() error { + ids := make([]int, 0) + for nodeId, nv := range s.objectMap { + if !nv.nodeValueSet { + ids = append(ids, nodeId) + } + } + if len(ids) == 0 { + return nil + } + sort.Ints(ids) + return errors.Wrapf(ErrNodeNotFound, "node ids %v", ids) } func (s *sJsonUnmarshalSession) setPointerValue(nodeId int, val reflect.Value) error { if nv, ok := s.objectMap[nodeId]; ok && nv.nodeValueSet { + if !assignable(nv.nodeValue, val) { + return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, val.Type()) + } val.Set(nv.nodeValue) } else if ok && !nv.nodeValueSet { nv.targetValues = append(nv.targetValues, val) diff --git a/vendor/yunion.io/x/jsonutils/write.go b/vendor/yunion.io/x/jsonutils/write.go index f9ebc4a70..3295be0a0 100644 --- a/vendor/yunion.io/x/jsonutils/write.go +++ b/vendor/yunion.io/x/jsonutils/write.go @@ -18,9 +18,18 @@ import ( "fmt" "strings" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) +func writeJSONObject(sb *strings.Builder, v JSONObject) { + if gotypes.IsNil(v) { + sb.WriteString("null") + return + } + v.buildString(sb) +} + type writeSource interface { buildString(sb *strings.Builder) } @@ -46,6 +55,10 @@ func (this *JSONBool) buildString(sb *strings.Builder) { } func (this *JSONDict) buildString(sb *strings.Builder) { + if this == nil { + sb.WriteString("null") + return + } sb.WriteByte('{') var idx = 0 if this.nodeId > 0 { @@ -63,19 +76,23 @@ func (this *JSONDict) buildString(sb *strings.Builder) { sb.WriteString(quoteString(k)) sb.WriteByte(':') - v.buildString(sb) + writeJSONObject(sb, v) idx++ } sb.WriteByte('}') } func (this *JSONArray) buildString(sb *strings.Builder) { + if this == nil { + sb.WriteString("null") + return + } sb.WriteByte('[') for idx, v := range this.data { if idx > 0 { sb.WriteByte(',') } - v.buildString(sb) + writeJSONObject(sb, v) } sb.WriteByte(']') } diff --git a/vendor/yunion.io/x/onecloud/cmd/climc/shell/utils.go b/vendor/yunion.io/x/onecloud/cmd/climc/shell/utils.go index e66974e97..de343c403 100644 --- a/vendor/yunion.io/x/onecloud/cmd/climc/shell/utils.go +++ b/vendor/yunion.io/x/onecloud/cmd/climc/shell/utils.go @@ -15,10 +15,15 @@ package shell import ( + "bytes" "fmt" + "io" "os" + "runtime" "sort" + "strconv" "strings" + "sync" "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/printutils" @@ -37,45 +42,111 @@ const ( var outputFormat = OUTPUT_FORMAT_TABLE +// goroutine 本地输出：MCP 并发 tools/call 时避免劫持全局 os.Stdout。 +type outputState struct { + writer io.Writer + format string +} + +var outputStates sync.Map // uint64(goid) -> *outputState + func OutputFormat(s string) { outputFormat = s } +// PushOutput 将当前 goroutine 的 shell 输出重定向到 w，并可选覆盖格式。 +// 返回的 restore 必须在同一 goroutine 调用。 +func PushOutput(w io.Writer, format string) (restore func()) { + id := goroutineID() + prev, _ := outputStates.Load(id) + outputStates.Store(id, &outputState{writer: w, format: format}) + return func() { + if prev != nil { + outputStates.Store(id, prev) + } else { + outputStates.Delete(id) + } + } +} + +func currentWriter() io.Writer { + if v, ok := outputStates.Load(goroutineID()); ok { + if s := v.(*outputState); s != nil && s.writer != nil { + return s.writer + } + } + return os.Stdout +} + +func currentFormat() string { + if v, ok := outputStates.Load(goroutineID()); ok { + if s := v.(*outputState); s != nil && s.format != "" { + return s.format + } + } + return outputFormat +} + +func goroutineID() uint64 { + b := make([]byte, 64) + b = b[:runtime.Stack(b, false)] + b = bytes.TrimPrefix(b, []byte("goroutine ")) + i := bytes.IndexByte(b, ' ') + if i <= 0 { + return 0 + } + n, _ := strconv.ParseUint(string(b[:i]), 10, 64) + return n +} + func PrintList(list *printutils.ListResult, columns []string) { - switch outputFormat { + w := currentWriter() + switch currentFormat() { case OUTPUT_FORMAT_TABLE: - printutils.PrintJSONList(list, columns) + if w == os.Stdout { + printutils.PrintJSONList(list, columns) + return + } + fmt.Fprint(w, jsonutils.Marshal(list).PrettyString()) + fmt.Fprint(w, "\n") case OUTPUT_FORMAT_JSON: - fmt.Print(jsonutils.Marshal(list).PrettyString()) - fmt.Print("\n") + fmt.Fprint(w, jsonutils.Marshal(list).PrettyString()) + fmt.Fprint(w, "\n") case OUTPUT_FORMAT_YAML: - fmt.Print(jsonutils.Marshal(list).YAMLString()) + fmt.Fprint(w, jsonutils.Marshal(list).YAMLString()) default: - fmt.Fprintf(os.Stderr, "unknown output format: %q\n", outputFormat) + fmt.Fprintf(os.Stderr, "unknown output format: %q\n", currentFormat()) } } func PrintObject(obj jsonutils.JSONObject) { - switch outputFormat { + w := currentWriter() + switch currentFormat() { case OUTPUT_FORMAT_TABLE: - printutils.PrintJSONObject(obj) + if w == os.Stdout { + printutils.PrintJSONObject(obj) + return + } + fmt.Fprint(w, obj.PrettyString()) + fmt.Fprint(w, "\n") case OUTPUT_FORMAT_KV: printObjectFmtKv(obj) case OUTPUT_FORMAT_JSON: - fmt.Print(obj.PrettyString()) - fmt.Print("\n") + fmt.Fprint(w, obj.PrettyString()) + fmt.Fprint(w, "\n") case OUTPUT_FORMAT_YAML: - fmt.Print(obj.YAMLString()) + fmt.Fprint(w, obj.YAMLString()) case OUTPUT_FORMAT_FLATTEN_TABLE: printObjectRecursive(obj) case OUTPUT_FORMAT_FLATTEN_KV: printObjectRecursiveEx(obj, printObjectFmtKv) default: - fmt.Fprintf(os.Stderr, "unknown output format: %q\n", outputFormat) + fmt.Fprintf(os.Stderr, "unknown output format: %q\n", currentFormat()) } } func printObjectFmtKv(obj jsonutils.JSONObject) { + w := currentWriter() m, _ := obj.GetMap() maxWidth := 0 keys := make([]string, 0, len(m)) @@ -95,7 +166,7 @@ func printObjectFmtKv(obj jsonutils.JSONObject) { } else { s = objV.String() } - fmt.Printf("%*s: %s\n", maxWidth, k, s) + fmt.Fprintf(w, "%*s: %s\n", maxWidth, k, s) } } @@ -107,8 +178,24 @@ func printObjectRecursiveEx(obj jsonutils.JSONObject, cb printutils.PrintJSONObj printutils.PrintJSONObjectRecursiveEx(obj, cb) } +func PrintBatchResults(results []printutils.SubmitResult, columns []string) { + w := currentWriter() + switch currentFormat() { + case OUTPUT_FORMAT_JSON: + fmt.Fprint(w, jsonutils.Marshal(results).PrettyString()) + fmt.Fprint(w, "\n") + default: + if w == os.Stdout { + printutils.PrintJSONBatchResults(results, columns) + return + } + fmt.Fprint(w, jsonutils.Marshal(results).PrettyString()) + fmt.Fprint(w, "\n") + } +} + func printBatchResults(results []printutils.SubmitResult, columns []string) { - printutils.PrintJSONBatchResults(results, columns) + PrintBatchResults(results, columns) } func ExportList(list *printutils.ListResult, file string, exportKeys string, exportTexts string, columns []string) { diff --git a/vendor/yunion.io/x/onecloud/locales/locales.go b/vendor/yunion.io/x/onecloud/locales/locales.go index 99b6ee904..9291fe5de 100644 --- a/vendor/yunion.io/x/onecloud/locales/locales.go +++ b/vendor/yunion.io/x/onecloud/locales/locales.go @@ -1609,6 +1609,7 @@ var messageKeyToIndex = map[string]int{ "loadbalancer is already associated with eip": 780, "loadbalancer is locked, cannot delete": 1366, "local host storage is not empty": 1177, + "local_path SKU with HAMi requires per-GPU VRAM: set devices[].memory_mb on the LLM SKU": 2131, "locally stored disks do not support detaching": 394, "locally stored system disks do not support changing configuration": 391, "login_account exceeds 32 characters": 820, @@ -2172,7 +2173,7 @@ var messageKeyToIndex = map[string]int{ "zone_ids must be in the same cloudregion": 1582, } -var en_USIndex = []uint32{ // 2132 elements +var en_USIndex = []uint32{ // 2133 elements // Entry 0 - 1F 0x00000000, 0x0000001b, 0x00000034, 0x00000049, 0x00000067, 0x0000007a, 0x00000099, 0x000000d4, @@ -2773,9 +2774,10 @@ var en_USIndex = []uint32{ // 2132 elements 0x00011c60, 0x00011c78, 0x00011c8e, 0x00011c99, 0x00011ca7, 0x00011cba, 0x00011ccf, 0x00011ce4, 0x00011cf3, 0x00011d12, 0x00011d2c, 0x00011d41, -} // Size: 8552 bytes + 0x00011d98, +} // Size: 8532 bytes -const en_USData string = "" + // Size: 73025 bytes +const en_USData string = "" + // Size: 73112 bytes "\x02not found tenantId in body\x02failed to change project\x02get passwo" + "rd in body\x02username or password is empty\x02missing credential\x02inc" + "orrect username or password\x02The user has been locked, please contact " + @@ -3881,9 +3883,10 @@ const en_USData string = "" + // Size: 73025 bytes "vn controller configuration\x02unmarshal input fail %s\x02invalid charac" + "ters %s\x02bad config\x02network error\x02ssh protocol error\x02invalid " + "vpc provider\x02invalid ovn database\x02user not found\x02check name dup" + - "lication fail %s\x02getReferenceCount fail %s\x02policy is referenced" + "lication fail %s\x02getReferenceCount fail %s\x02policy is referenced" + + "\x02local_path SKU with HAMi requires per-GPU VRAM: set devices[].memory_mb on the LLM SKU" -var zh_CNIndex = []uint32{ // 2132 elements +var zh_CNIndex = []uint32{ // 2133 elements // Entry 0 - 1F 0x00000000, 0x0000001b, 0x0000002e, 0x00000049, 0x00000062, 0x00000075, 0x00000094, 0x000000bf, @@ -4484,9 +4487,10 @@ var zh_CNIndex = []uint32{ // 2132 elements 0x00010c9c, 0x00010cb1, 0x00010cc7, 0x00010cd7, 0x00010ce4, 0x00010cf4, 0x00010d0a, 0x00010d20, 0x00010d30, 0x00010d4e, 0x00010d71, 0x00010d8a, -} // Size: 8552 bytes + 0x00010e12, +} // Size: 8532 bytes -const zh_CNData string = "" + // Size: 69002 bytes +const zh_CNData string = "" + // Size: 69138 bytes "\x02获取tenantId参数失败\x02切换项目失败\x02获取password参数失败\x02用户名或密码为空\x02缺少认证信息\x02" + "用户名或者密码不正确\x02用户已经被锁定，请联系管理员\x02用户已经被禁用，请联系管理员\x02用户已经被禁用或处于不可用状态\x02无" + "效的认证信息\x02获取请求的JSON内容失败：%v\x02此用户禁止从Web控制台登录\x02请求内容为空\x02缺少id参数\x02fe" + @@ -5070,6 +5074,7 @@ const zh_CNData string = "" + // Size: 69002 bytes "告警条件类型: %s\x02解析URL %q出错：%v\x02无效的IP\x02停止dap搜索\x02ovn controller\x02o" + "vn controller配置\x02解析input失败 %s\x02不支持的字符 %s\x02无效的配置\x02网络错误\x02ssh协议错误" + "\x02无效的vpc provider\x02无效的ovn database\x02找不到用户\x02检查重复名称失败：%s\x02getRef" + - "erenceCount调用出错：%s\x02权限定义正被使用" + "erenceCount调用出错：%s\x02权限定义正被使用" + + "\x02主机本地模型（local_path）推理模板使用 HAMi 时必须设置每卡显存：请到推理模板中填写 GPU 显存（MiB）" // Total table size 159131 bytes (155KiB); checksum: A1EC1008 diff --git a/vendor/yunion.io/x/onecloud/pkg/aiproxy/options/options.go b/vendor/yunion.io/x/onecloud/pkg/aiproxy/options/options.go index 1d4ae8f80..996deaaf0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/aiproxy/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/aiproxy/options/options.go @@ -22,12 +22,12 @@ type SAiProxyOptions struct { common_options.CommonOptions common_options.DBOptions - AdvertiseAddress string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address and port" default:""` + AdvertiseAddress string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address or local NIC IP and port" default:""` NodeHeartbeatIntervalSeconds int `help:"Interval in seconds for standby node registration heartbeat" default:"60"` APILogEnabled bool `help:"Enable OpenAI API request JSONL logs" default:"true"` APILogLocalDir string `help:"Local directory for API JSONL logs" default:"/tmp/aiproxy-apilog"` - APILogUploadEnabled bool `help:"Upload closed API log hour files to S3" default:"true"` + APILogUploadEnabled bool `help:"Upload API logs to S3" default:"true"` APILogUploadIntervalSeconds int `help:"API log upload interval in seconds" default:"10"` APILogSegmentMinutes int `help:"API log file segment duration in minutes" default:"60"` APILogS3Endpoint string `help:"S3 endpoint for API log upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"` diff --git a/vendor/yunion.io/x/onecloud/pkg/apihelper/apihelper.go b/vendor/yunion.io/x/onecloud/pkg/apihelper/apihelper.go index 92b31dcc1..5d5f8021b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apihelper/apihelper.go +++ b/vendor/yunion.io/x/onecloud/pkg/apihelper/apihelper.go @@ -28,6 +28,8 @@ import ( api "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/cloudcommon/policy" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify" @@ -46,6 +48,12 @@ type APIHelper struct { mcclientSession *mcclient.ClientSession tick *time.Timer + + // apiMapTS is the version of the model sets that modelSets reflects, as + // served by the apimap service; hasAPIMapTS tells whether that version + // is known at all. Both are only touched from the sync loop. + apiMapTS int64 + hasAPIMapTS bool } func NewAPIHelper(opts *Options, modelSets IModelSets) (*APIHelper, error) { @@ -76,13 +84,26 @@ func (h *APIHelper) getRunDelay() time.Duration { func (h *APIHelper) addSyncHandler(app *appsrv.Application, prefix string) { path := httputils.JoinPath(prefix, "sync") - app.AddHandler("POST", path, h.handlerSync) + app.AddHandler("POST", path, auth.Authenticate(RequireSystemAdmin(h.handlerSync))) } func (h *APIHelper) handlerSync(ctx context.Context, w http.ResponseWriter, r *http.Request) { h.scheduleSync() } +// RequireSystemAdmin wraps a handler so that only a caller with system scope +// privilege reaches it. +func RequireSystemAdmin(h func(context.Context, http.ResponseWriter, *http.Request)) func(context.Context, http.ResponseWriter, *http.Request) { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + userCred := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential) + if userCred == nil || !userCred.HasSystemAdminPrivilege() { + httperrors.ForbiddenError(ctx, w, "only system admin is allowed to access this resource") + return + } + h(ctx, w, r) + } +} + func (h *APIHelper) Start(ctx context.Context, app *appsrv.Application, prefix string) { defer func() { log.Infoln("apihelper: bye") @@ -133,11 +154,11 @@ func (h *APIHelper) RunManually(ctx context.Context) { } func (h *APIHelper) run(ctx context.Context) { - changed, err := h.doSync(ctx) - if err != nil { - log.Errorf("doSync error: %v", err) + status := h.doSync(ctx) + if status.Err != nil { + log.Errorf("doSync error: %v", status.Err) } - if changed { + if status.Changed { mssCopy := h.modelSets.CopyJoined() select { case h.modelSetsCh <- mssCopy: @@ -146,20 +167,35 @@ func (h *APIHelper) run(ctx context.Context) { } } -func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) { - { - stime := time.Now() - defer func() { - elapsed := time.Since(stime) - log.Infof("sync data done, changed: %v, elapsed: %s", changed, elapsed.String()) - }() - } +func (h *APIHelper) doSync(ctx context.Context) (status SyncStatus) { + status.At = time.Now() + defer func() { + status.Elapsed = time.Since(status.At) + log.Infof("sync data done, changed: %v, elapsed: %s, err: %v", status.Changed, status.Elapsed.String(), status.Err) + if h.opts.OnSyncDone != nil { + h.opts.OnSyncDone(status) + } + }() s := h.adminClientSession(ctx) - mss := h.modelSets.Copy() - r, err := SyncModelSets(mss, s, h.opts) - if err != nil { - return false, errors.Wrap(err, "SyncModelSets") + var ( + mss IModelSets + r ModelSetsUpdateResult + err error + ) + if h.opts.FetchFromComputeService { + mss = h.modelSets.Copy() + r, err = SyncModelSets(mss, s, h.opts) + if err != nil { + status.Err = errors.Wrap(err, "SyncModelSets") + return + } + } else { + mss, r, err = h.syncModelSetsFromAPIMap(s) + if err != nil { + status.Err = err + return + } } h.modelSets = mss if !r.Correct { @@ -168,10 +204,42 @@ func (h *APIHelper) doSync(ctx context.Context) (changed bool, err error) { if err != nil { log.Errorf("unable to EventNotify: %s", err) } - return false, errors.Errorf("sync error") + status.Err = errors.Errorf("sync error") + return + } + status.Changed = r.Changed + status.Correct = true + return +} + +// syncModelSetsFromAPIMap refreshes the model sets from the apimap service. +// +// The apimap service publishes a version with the model sets, so a round that +// finds the version unchanged is skipped without copying anything. The +// version returned along with the payload is the authoritative one: the model +// sets may have moved on between the version query and the fetch. +func (h *APIHelper) syncModelSetsFromAPIMap(s *mcclient.ClientSession) (mss IModelSets, r ModelSetsUpdateResult, err error) { + apims, ok := h.modelSets.(IAPIMapModelSets) + if !ok { + return nil, r, errors.Errorf("model sets %T cannot be fetched from apimap", h.modelSets) + } + curTS, err := apims.APIMapTimestamp(s) + if err != nil { + return nil, r, errors.Wrap(err, "APIMapTimestamp") + } + if h.hasAPIMapTS && curTS == h.apiMapTS { + // nothing new since the last round + return h.modelSets, ModelSetsUpdateResult{Correct: true, Changed: false}, nil + } + mssNews, newTS, err := apims.FetchFromAPIMap(s) + if err != nil { + return nil, r, errors.Wrap(err, "FetchFromAPIMap") } - changed = r.Changed - return changed, nil + mss = h.modelSets.Copy() + r = mss.ApplyUpdates(mssNews) + h.apiMapTS = newTS + h.hasAPIMapTS = true + return mss, r, nil } func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSession { diff --git a/vendor/yunion.io/x/onecloud/pkg/apihelper/db.go b/vendor/yunion.io/x/onecloud/pkg/apihelper/db.go deleted file mode 100644 index d563c0f6d..000000000 --- a/vendor/yunion.io/x/onecloud/pkg/apihelper/db.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package apihelper - -import ( - "context" - "fmt" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/mcclient/options" -) - -type GetDBModelsOptions struct { - modelOptions *GetModelsOptions - modelDBManager db.IModelManager -} - -func (o GetDBModelsOptions) IncludeOtherCloudEnv() bool { - return o.modelOptions.InCludeOtherCloudEnv -} - -func (o GetDBModelsOptions) GetModelSet() IModelSet { - return o.modelOptions.ModelSet -} - -func (o GetDBModelsOptions) IncludeDetails() bool { - return o.modelOptions.IncludeDetails -} - -func (o GetDBModelsOptions) IncludeEmulated() bool { - return o.modelOptions.IncludeEmulated -} - -func (o GetDBModelsOptions) BatchListSize() int { - return o.modelOptions.BatchListSize -} - -func (o GetDBModelsOptions) InCludeOtherCloudEnv() bool { - return o.modelOptions.InCludeOtherCloudEnv -} - -func GetDBModels(opts *GetDBModelsOptions) error { - man := opts.modelDBManager - manKeyPlural := man.KeywordPlural() - - limit := opts.BatchListSize() - // limit := 5 - listOptions := options.BaseListOptions{ - System: options.Bool(true), - Admin: options.Bool(true), - Scope: "system", - Details: options.Bool(opts.IncludeDetails()), - ShowEmulated: options.Bool(opts.IncludeEmulated()), - OrderBy: []string{"updated_at"}, - Order: "asc", - Limit: &limit, - } - if !opts.InCludeOtherCloudEnv() { - listOptions.Filter = append(listOptions.Filter, - "manager_id.isnullorempty()", // len(manager_id) > 0 is for pubcloud objects - // "external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects - ) - listOptions.CloudEnv = "onpremise" - // listOptions.Provider = []string{"OneCloud"} - } - if inter, ok := opts.GetModelSet().(IModelSetFilter); ok { - filter := inter.ModelFilter() - listOptions.Filter = append(listOptions.Filter, filter...) - } - params, err := listOptions.Params() - if err != nil { - return fmt.Errorf("%s: making list params: %s", manKeyPlural, err) - } - if inter, ok := opts.GetModelSet().(IModelListParam); ok { - filter := inter.ModelParamFilter() - params.Update(filter) - } - //XXX - //params.Set(api.LBAGENT_QUERY_ORIG_KEY, jsonutils.NewString(api.LBAGENT_QUERY_ORIG_VAL)) - - entriesJson := []jsonutils.JSONObject{} - for { - log.Debugf("list %s with params: %s", manKeyPlural, params.String()) - var err error - listResult, err := db.ListItems(man, context.Background(), auth.AdminCredential(), params, nil) - if err != nil { - return fmt.Errorf("%s: list failed: %s", - manKeyPlural, err) - } - entriesJson = append(entriesJson, listResult.Data...) - if listResult.Offset+len(listResult.Data) >= listResult.Total { - break - } else { - offset := listResult.Offset + len(listResult.Data) - params.Set("offset", jsonutils.NewInt(int64(offset))) - } - } - { - err := InitializeModelSetFromJSON(opts.GetModelSet(), entriesJson) - if err != nil { - return fmt.Errorf("%s: initializing model set failed: %s", - manKeyPlural, err) - } - } - - return nil -} diff --git a/vendor/yunion.io/x/onecloud/pkg/apihelper/interface.go b/vendor/yunion.io/x/onecloud/pkg/apihelper/interface.go index 1b24f8089..b3bf70a25 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apihelper/interface.go +++ b/vendor/yunion.io/x/onecloud/pkg/apihelper/interface.go @@ -36,9 +36,19 @@ type IModelSets interface { CopyJoined() IModelSets } -type IDBModelSets interface { +// IAPIMapModelSets is implemented by the model sets that can be fetched from +// the apimap service instead of from the compute APIs. +type IAPIMapModelSets interface { IModelSets - FetchFromAPIMap(s *mcclient.ClientSession) (IModelSets, error) + + // APIMapTimestamp returns the version of the model sets currently served + // by the apimap service, without fetching the payload. + APIMapTimestamp(s *mcclient.ClientSession) (int64, error) + + // FetchFromAPIMap fetches the model sets from the apimap service. The + // returned timestamp is the version of the returned payload, which is + // what the caller has to compare against on the next round. + FetchFromAPIMap(s *mcclient.ClientSession) (IModelSets, int64, error) } type IModelSet interface { @@ -46,11 +56,7 @@ type IModelSet interface { NewModel() db.IModel AddModel(db.IModel) Copy() IModelSet -} - -type IDBModelSet interface { - IModelSet - DBModelManager() db.IModelManager + IncludeDetails() bool } type IModelSetEmulatedIncluder interface { @@ -69,21 +75,14 @@ type IModelListSetParams interface { SetModelListParams(params *jsonutils.JSONDict) *jsonutils.JSONDict } +// SyncModelSets refreshes mssOld with the model sets listed from the compute +// service APIs. Its counterpart, for the model sets served by the apimap +// service, is APIHelper's own sync path, which also has to keep track of the +// version that the model sets reflect. func SyncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (ModelSetsUpdateResult, error) { - var ( - mssNews IModelSets - err error - ) - if mssDB, ok := mssOld.(IDBModelSets); ok && !opt.FetchFromComputeService { - mssNews, err = mssDB.FetchFromAPIMap(s) - if err != nil { - return ModelSetsUpdateResult{}, errors.Wrap(err, "FetchFromAPIMap") - } - } else { - mssNews, err = syncModelSets(mssOld, s, opt) - if err != nil { - return ModelSetsUpdateResult{}, errors.Wrap(err, "syncModelSets") - } + mssNews, err := syncModelSets(mssOld, s, opt) + if err != nil { + return ModelSetsUpdateResult{}, errors.Wrap(err, "syncModelSets") } r := mssOld.ApplyUpdates(mssNews) return r, nil @@ -107,7 +106,7 @@ func syncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) ( ModelSet: msNew, BatchListSize: opt.ListBatchSize, - IncludeDetails: opt.IncludeDetails, + IncludeDetails: msNew.IncludeDetails(), IncludeEmulated: includeEmulated, InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv, }) @@ -117,43 +116,3 @@ func syncModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) ( } return mssNews, nil } - -func SyncDBModelSets(mssOld IModelSets, s *mcclient.ClientSession, opt *Options) (r ModelSetsUpdateResult, err error) { - mssNews := mssOld.NewEmpty() - for _, msNew := range mssNews.ModelSetList() { - var ( - includeEmulated = false - ) - if optProvider, ok := msNew.(IModelSetEmulatedIncluder); ok { - includeEmulated = optProvider.IncludeEmulated() - } - msi, ok := msNew.(IDBModelSet) - opts := &GetModelsOptions{ - ClientSession: s, - ModelManager: msNew.ModelManager(), - ModelSet: msNew, - BatchListSize: opt.ListBatchSize, - - IncludeDetails: opt.IncludeDetails, - IncludeEmulated: includeEmulated, - InCludeOtherCloudEnv: opt.IncludeOtherCloudEnv, - } - if ok { - dbOpts := &GetDBModelsOptions{ - modelOptions: opts, - modelDBManager: msi.DBModelManager(), - } - err = GetDBModels(dbOpts) - if err != nil { - return - } - } else { - err = GetModels(opts) - if err != nil { - return - } - } - } - r = mssOld.ApplyUpdates(mssNews) - return r, nil -} diff --git a/vendor/yunion.io/x/onecloud/pkg/apihelper/options.go b/vendor/yunion.io/x/onecloud/pkg/apihelper/options.go index 24ccc7a14..1130dc130 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apihelper/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/apihelper/options.go @@ -15,9 +15,30 @@ package apihelper import ( + "time" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) +// SyncStatus reports the outcome of one sync attempt. +type SyncStatus struct { + // At is when the attempt started + At time.Time + // Elapsed is how long the attempt took + Elapsed time.Duration + // Changed tells whether the model sets changed in this attempt + Changed bool + // Correct tells whether the model sets are complete and consistent + Correct bool + // Err is the error of the attempt, nil on success + Err error +} + +// SyncDoneFunc is called after every sync attempt, successful or not. It is +// how a service that does not consume the ModelSets() channel learns about the +// result of a sync. +type SyncDoneFunc func(status SyncStatus) + type Options struct { common_options.CommonOptions @@ -27,4 +48,8 @@ type Options struct { IncludeDetails bool IncludeOtherCloudEnv bool FetchFromComputeService bool + + // OnSyncDone, when not nil, is called after every sync attempt with the + // outcome of that attempt. + OnSyncDone SyncDoneFunc } diff --git a/vendor/yunion.io/x/onecloud/pkg/apimap/options/options.go b/vendor/yunion.io/x/onecloud/pkg/apimap/options/options.go index 3aac2d041..7d7cf4cb0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apimap/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/apimap/options/options.go @@ -17,12 +17,24 @@ package options import ( "os" + "yunion.io/x/onecloud/pkg/apihelper" common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" - "yunion.io/x/onecloud/pkg/compute/options" ) type SOptions struct { - options.ComputeOptions + common_options.CommonOptions + + // APISyncIntervalSeconds is how often the model sets are synced from the + // region API. Keep it below the sync interval of the services that + // consume the model sets, so that a change reaches them without an extra + // delay. + APISyncIntervalSeconds int `default:"5" help:"interval in seconds to sync model sets from region"` + // APIRunDelayMilliseconds is how long a triggered sync waits before it + // runs, so that a burst of triggers collapses into one sync. + APIRunDelayMilliseconds int `default:"100" help:"delay in milliseconds before a scheduled sync runs"` + // APIListBatchSize is the batch size of the model list requests made + // against the region API. + APIListBatchSize int `default:"1024" help:"batch size of model list requests"` } var ( @@ -35,8 +47,21 @@ func GetOptions() *SOptions { func Init() { common_options.ParseOptions(&opts, os.Args, "apimap.conf", "apimap") - options.Options = opts.ComputeOptions } + +func (opts *SOptions) ValidateThenInit() error { + if opts.APISyncIntervalSeconds < apihelper.MinSyncIntervalSeconds { + opts.APISyncIntervalSeconds = apihelper.MinSyncIntervalSeconds + } + if opts.APIRunDelayMilliseconds < apihelper.MinRunDelayMilliseconds { + opts.APIRunDelayMilliseconds = apihelper.MinRunDelayMilliseconds + } + if opts.APIListBatchSize <= 20 { + opts.APIListBatchSize = 20 + } + return nil +} + func OnOptionsChange(oldO, newO interface{}) bool { oldOpts := oldO.(*SOptions) newOpts := newO.(*SOptions) @@ -45,11 +70,6 @@ func OnOptionsChange(oldO, newO interface{}) bool { if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) { changed = true } - if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) { - changed = true - } - - options.Options = newOpts.ComputeOptions return changed } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/ansibleserver/ansible.go b/vendor/yunion.io/x/onecloud/pkg/apis/ansibleserver/ansible.go index 8b0abbda3..eb724e275 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/ansibleserver/ansible.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/ansibleserver/ansible.go @@ -28,6 +28,11 @@ type AnsiblePlaybookCreateInput struct { Playbook ansible.Playbook `json:"playbook"` } +type AnsiblePlaybookListInput struct { + apis.VirtualResourceListInput + apis.EnabledResourceBaseListInput +} + type AnsiblePlaybookUpdateInput AnsiblePlaybookCreateInput type AnsibleHost struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go index a67accbdb..93332a5c0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/api.go @@ -315,17 +315,22 @@ type IsolatedDeviceConfig struct { Id string `json:"id"` DevType string `json:"dev_type"` Model string `json:"model"` + SharingMode string `json:"sharing_mode"` Vendor string `json:"vendor"` NetworkIndex *int `json:"network_index"` WireId string `json:"wire_id"` DiskIndex *int8 `json:"disk_index"` DevicePath string `json:"device_path"` + GpuType string `json:"gpu_type"` // MemoryMb is the minimum on-device memory in MiB required from the // candidate isolated_device (e.g. NVIDIA GPU VRAM). 0 means no constraint. // The scheduler excludes devices whose memory_size > 0 and is below this // threshold; devices with memory_size == 0 are treated as unknown and // allowed through to avoid penalising hosts that haven't reported yet. MemoryMb int `json:"memory_mb,omitempty"` + // Memory request for Devices allocate by Memory size + MemoryRequest int `json:"memory_request,omitempty"` + SmUtilLimit int `json:"sm_util_limit,omitempty"` } type BaremetalDiskConfig struct { @@ -408,7 +413,10 @@ type ServerConfigs struct { // default: kvm Hypervisor string `json:"hypervisor"` - // swagger:ignore + // specific qemu version, eg: 4.2.0, 10.0.7 + QemuVersion string `json:"qemu_version"` + + // swagger: ignore Provider string `json:"provider"` // 包年包月资源池 @@ -722,6 +730,11 @@ type ServerCreateInput struct { // 安全组Id列表 Secgroups []string `json:"secgroups"` + // 源 IP 检查 + SrcIpCheck *bool `json:"src_ip_check"` + // 源 MAC 检查 + SrcMacCheck *bool `json:"src_mac_check"` + // GCP 网络标记(network tags)，仅对 Google 云有效 // 传入后可不指定安全组，创建时直接作为实例 tags 下发 // required: false diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/backup.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/backup.go index 19f2822ac..77903a063 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/backup.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/backup.go @@ -106,6 +106,8 @@ type DiskBackupListInput struct { IsInstanceBackup *bool `json:"is_instance_backup"` // 按硬盘名称排序 OrderByDiskName string `json:"order_by_disk_name"` + // description: 主机ID + ServerId string `json:"server_id"` } type DiskBackupDetails struct { @@ -136,6 +138,10 @@ type DiskBackupCreateInput struct { apis.VirtualResourceCreateInput apis.EncryptedResourceCreateInput + // swagger:ignore + SizeMb int `json:"size_mb"` + // path to find backup file, in case of create backup from a backup file + BackupFilePath string `json:"backup_file_path"` // description: disk id DiskId string `json:"disk_id"` // swagger:ignore @@ -165,6 +171,9 @@ type DiskBackupPackMetadata struct { // 操作系统类型 OsType string `json:"os_type"` DiskConfig *SBackupDiskConfig `json:"disk_config"` + + // 备份文件路径 + BackupFilePath string `json:"backup_file_path"` } type DiskBackupExportInfo struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/cloudprovider.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/cloudprovider.go index aabedf94e..88d1af846 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/cloudprovider.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/cloudprovider.go @@ -249,7 +249,7 @@ type CloudproviderListInput struct { UsableResourceListInput - CloudregionResourceInput + CloudregionResourceListInput ZoneResourceInput diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/container.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/container.go index eff41ce6c..183a99992 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/container.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/container.go @@ -40,8 +40,15 @@ const ( CONTAINER_DEV_NVIDIA_GPU = "NVIDIA_GPU" CONTAINER_DEV_NVIDIA_MPS = "NVIDIA_MPS" CONTAINER_DEV_NVIDIA_GPU_SHARE = "NVIDIA_GPU_SHARE" + CONTAINER_DEV_NVIDIA_HAMI = "NVIDIA_HAMI" CONTAINER_DEV_ASCEND_NPU = "ASCEND_NPU" + CONTAINER_DEV_ASCEND_NPU_HAMI = "ASCEND_NPU_HAMI" CONTAINER_DEV_VASTAITECH_GPU = "VASTAITECH_GPU" + CONTAINER_DEV_HYGON_DCU = "HYGON_DCU" + CONTAINER_DEV_HYGON_DCU_HAMI = "HYGON_DCU_HAMI" + CONTAINER_DEV_ILUVATAR_GPU = "ILUVATAR_GPU" + CONTAINER_DEV_THEAD_PPU = "THEAD_PPU" + CONTAINER_DEV_KUNLUNXIN_XPU = "KUNLUNXIN_XPU" ) var ( @@ -49,14 +56,21 @@ var ( CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_VASTAITECH_GPU, + CONTAINER_DEV_HYGON_DCU, + CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } ) -var NVIDIA_GPU_TYPES = []string{ +var CONTAINER_NVIDIA_GPU_TYPES = []string{ CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_NVIDIA_GPU_SHARE, } @@ -198,10 +212,11 @@ type ContainerHostDevice struct { } type ContainerIsolatedDevice struct { - Index *int `json:"index"` - Id string `json:"id"` - OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` - CDI *apis.ContainerIsolatedDeviceCDI `json:"cdi"` + Index *int `json:"index"` + Id string `json:"id"` + GuestIsolatedDeviceIndex int `json:"guest_isolated_device_index"` + OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` + CDI *apis.ContainerIsolatedDeviceCDI `json:"cdi"` } type ContainerDevice struct { @@ -257,7 +272,7 @@ type ContainerCommitExternalRegistry struct { } type ContainerCommitInput struct { - // Container registry id from kubeserver + // Container registry id from glance (or kubeserver during migration) RegistryId string `json:"registry_id"` ExternalRegistry *ContainerCommitExternalRegistry `json:"external_registry"` // image name diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk.go index ce4e9c91a..894e4a8b4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk.go @@ -341,12 +341,11 @@ type DiskAllocateInput struct { BackupId string `json:"backup_id"` Backup *DiskAllocateFromBackupInput `json:"backup"` - SnapshotUrl string `json:"snapshot_url"` - SnapshotOutOfChain bool `json:"snapshot_out_of_chain"` - Protocol string `json:"protocol"` - SrcDiskId string `json:"src_disk_id"` - SrcPool string `json:"src_pool"` - ExistingPath string `json:"existing_path"` + SnapshotUrl string `json:"snapshot_url"` + Protocol string `json:"protocol"` + SrcDiskId string `json:"src_disk_id"` + SrcPool string `json:"src_pool"` + ExistingPath string `json:"existing_path"` // vmware HostIp string `json:"host_ip"` @@ -358,17 +357,19 @@ type DiskAllocateInput struct { } type DiskAllocateFromBackupInput struct { - BackupId string `json:"backup_id"` - BackupStorageId string `json:"backup_storage_id"` - BackupStorageAccessInfo *jsonutils.JSONDict `json:"backup_storage_access_info"` - DiskConfig *DiskConfig `json:"disk_config"` - BackupAsTar *DiskBackupAsTarInput `json:"backup_as_tar"` + BackupId string `json:"backup_id"` + BackupStorageId string `json:"backup_storage_id"` + BackupStorageAccessInfo *SBackupStorageAccessInfo `json:"backup_storage_access_info"` + DiskConfig *DiskConfig `json:"disk_config"` + BackupAsTar *DiskBackupAsTarInput `json:"backup_as_tar"` + BackupFilePath string `json:"backup_file_path"` } type DiskDeleteInput struct { - SkipRecycle *bool `json:"skip_recycle"` - EsxiFlatFilePath string `json:"esxi_flat_file_path"` - CleanSnapshots bool `json:"clean_snapshots"` + SkipRecycle *bool `json:"skip_recycle"` + EsxiFlatFilePath string `json:"esxi_flat_file_path"` + CleanSnapshots bool `json:"clean_snapshots"` + SnapshotIds []string `json:"snapshot_ids"` } type DiskResetInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk_const.go index 684b6097d..f9d4d5ba6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/disk_const.go @@ -97,3 +97,15 @@ const ( DISK_DRIVER_SATA = "sata" DISK_DRIVER_VFIO = "vfio-pci" ) + +const ( + DISK_CACHE_MODE_WRITETHROGH = "writethrough" + DISK_CACHE_MODE_NONE = "none" + DISK_CACHE_MODE_WRITEBACK = "writeback" + DISK_CACHE_MODE_DIRECTSYNC = "directsync" +) + +const ( + DISK_AIO_MODE_NATIVE = "native" + DISK_AIO_MOD_THREADS = "threads" +) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_const.go index e81d9c269..705f1f2a8 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_const.go @@ -180,6 +180,10 @@ const ( VM_QGA_SET_NETWORK = "qga_set_network" VM_QGA_SET_NETWORK_FAILED = "qga_set_network_failed" + // 设置端口映射 + VM_SET_PORTMAPPING = "set_portmapping" + VM_SET_PORTMAPPING_FAIL = "set_portmapping_fail" + // Kickstart自动安装相关状态 VM_KICKSTART_PENDING = "kickstart_pending" // kickstart配置完成，等待VM启动 VM_KICKSTART_INSTALLING = "kickstart_installing" // kickstart正在安装 @@ -319,6 +323,7 @@ var HYPERVISORS = []string{ HYPERVISOR_SANGFOR, HYPERVISOR_ZETTAKIT, HYPERVISOR_UIS, + HYPERVISOR_CAS, HYPERVISOR_CNWARE, HYPERVISOR_ROCKBASE, } @@ -352,6 +357,8 @@ const ( VM_METADATA_ENABLE_TPM = "enable_tpm" + VM_METADATA_QEMU_VERSION = "qemu_version" + VM_METADATA_RELEASED_DEVICES = "released_devices" VM_METADATA_CPU_NUMA_PIN = "__cpu_numa_pin" diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_disk.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_disk.go index ae555f7af..eae5ecc65 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_disk.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guest_disk.go @@ -56,11 +56,11 @@ type GuestdiskListInput struct { type GuestdiskUpdateInput struct { GuestJointBaseUpdateInput - Driver string `json:"driver"` - - CacheMode string `json:"cache_mode"` - - AioMode string `json:"aio_mode"` + //Driver string `json:"driver"` + // + //CacheMode string `json:"cache_mode"` + // + //AioMode string `json:"aio_mode"` Iops *int `json:"iops"` @@ -90,6 +90,7 @@ type GuestdiskJsonDesc struct { Index int8 `json:"index"` BootIndex *int8 `json:"boot_index"` MergeSnapshot bool `json:"merge_snapshot"` + SnapshotId string `json:"snapshot_id"` Fs string `json:"fs"` Mountpoint string `json:"mountpoint"` Dev string `json:"dev"` diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guestnetwork.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guestnetwork.go index a5d7be01d..988da38a9 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guestnetwork.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guestnetwork.go @@ -130,6 +130,10 @@ type GuestnetworkUpdateInput struct { PortMappings GuestPortMappings `json:"port_mappings"` ChargeType string `json:"charge_type"` + + // NoSync 为 true 时只更新数据库中网卡的 port_mappings，不触发宿主机同步 + // 用于宿主机回写（如分配 host_port）等场景，避免递归触发同步 + NoSync *bool `json:"no_sync"` } type GuestnetworkBaseDesc struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guests.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guests.go index 78d71b826..2d0bfb05c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/guests.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/guests.go @@ -133,6 +133,8 @@ type ServerListInput struct { SnapshotpolicyId string `json:"snapshotpolicy_id"` + IsolatedDeviceId string `json:"isolated_device_id"` + // 是否调度到宿主机上 WithHost *bool `json:"with_host"` @@ -515,6 +517,28 @@ type ConvertToKvmInput struct { // dest guest network configs Networks []*NetworkConfig `json:"networks"` + // dest guest disk storage configs; length must equal guest disks when set + // support per-disk backend/storage/medium/schedtags; overrides sys/data disk prefers + Disks []*DiskConfig `json:"disks"` + + // Prefer disk backend for system disk, e.g. local/lvm/slvm/nfs/rbd + SysDiskBackend string `json:"sys_disk_backend"` + // Prefer storage id or name for system disk + SysPreferStorage string `json:"sys_prefer_storage"` + // Prefer medium for system disk, e.g. rotate/ssd/hybrid + SysDiskMedium string `json:"sys_disk_medium"` + // Prefer disk schedtags for system disk + SysDiskSchedtags []*SchedtagConfig `json:"sys_disk_schedtags"` + + // Prefer disk backend for data disks, e.g. local/lvm/slvm/nfs/rbd + DataDiskBackend string `json:"data_disk_backend"` + // Prefer storage id or name for data disks + DataPreferStorage string `json:"data_prefer_storage"` + // Prefer medium for data disks, e.g. rotate/ssd/hybrid + DataDiskMedium string `json:"data_disk_medium"` + // Prefer disk schedtags for data disks + DataDiskSchedtags []*SchedtagConfig `json:"data_disk_schedtags"` + // deploy telegraf after convert DeployTelegraf bool `json:"deploy_telegraf"` } @@ -688,8 +712,8 @@ type ServerStopInput struct { // 是否强制关机 IsForce bool `json:"is_force"` - // 关机等待时间，如果是强制关机，则等待时间为0，如果不设置，默认为30秒 - TimeoutSecs int `json:"timeout_secs"` + // 关机等待时间，如果不是强制关机，超过关机时间可能关机失败。linux默认则等待时间为60秒，windows 120秒 + TimeoutSecs *int `json:"timeout_secs"` // 是否关机停止计费, 若平台不支持停止计费，此参数无作用 // 若包年包月机器关机设置此参数，则先转换计费模式到按量计费，再关机不收费 @@ -697,6 +721,14 @@ type ServerStopInput struct { StopCharging bool `json:"stop_charging"` } +type ServerRestartInput struct { + // 是否强制关机 + IsForce bool `json:"is_force"` + + // 关机等待时间，如果不是强制关机，超过关机时间可能关机失败。linux默认则等待时间为60秒，windows 120秒 + TimeoutSecs *int `json:"timeout_secs"` +} + type ServerSaveImageInput struct { // 镜像名称 Name string `json:"name"` @@ -891,8 +923,9 @@ type ServerChangeConfigInput struct { // 内存大小, 1024M, 1G VmemSize string `json:"vmem_size"` - // 是否强制关机 - // 若虚拟机不支持开机调整配置, 则需要指定此参数为true, 强制关机后, 再调整配置, 再启动虚拟机 + // 是否允许强制关机 + // 仅当虚拟机不支持开机调整配置, 或开机状态下降配(降低CPU/内存), 或ARM架构时时生效: 需指定为true以允许强制关机后再调整配置并启动; + // 若已支持开机变配, 即使传入该参数也不会强制关机 ForceStop bool `json:"force_stop"` // 调整完配置后是否自动启动 @@ -1066,8 +1099,10 @@ type ServerChangeDiskStorageInput struct { } type ServerChangeDiskDriverInput struct { - DiskId string `json:"disk_id"` - Driver string `json:"driver"` + DiskId string `json:"disk_id"` + Driver string `json:"driver"` + CacheMode string `json:"cache_mode"` + AioMode string `json:"aio_mode"` } type ServerChangeDiskStorageInternalInput struct { @@ -1302,6 +1337,12 @@ type ServerSetPasswordInput struct { AutoStart bool `json:"auto_start"` } +type ServerSetIsoInput struct { + CdromOrdinal int64 `json:"cdrom_ordinal"` + ImageId string `json:"image_id"` + BootIndex *int8 `json:"boot_index"` +} + type ServerInsertVfdInput struct { FloppyOrdinal int64 `json:"floppy_ordinal"` ImageId string `json:"image_id"` @@ -1484,6 +1525,13 @@ type ServerChangeBandwidthInput struct { NoSync *bool `json:"no_sync"` } +// ServerSetPortMappingInput 设置服务器指定网卡的端口映射 +type ServerSetPortMappingInput struct { + ServerNetworkInfo + // 端口映射规则列表；传空数组表示清空该网卡的所有端口映射 + PortMappings GuestPortMappings `json:"port_mappings"` +} + type ServerChangeConfigSpecs struct { CpuSockets int `json:"cpu_sockets"` VcpuCount int `json:"vcpu_count"` @@ -1511,6 +1559,8 @@ type ServerChangeConfigSettings struct { AutoStart bool `json:"auto_start"` GuestOnline bool `json:"guest_online"` + // 需要强制关机后再调整配置(仅不支持在线变配/降配时为true), 并在完成后自动启动 + ForceStop bool `json:"force_stop"` // 设置虚拟网卡的流量上限 SetTrafficLimits []ServerNicTrafficLimit `json:"set_traffic_limits"` @@ -1524,6 +1574,10 @@ func (conf ServerChangeConfigSettings) CpuChanged() bool { return conf.VcpuCount != conf.Old.VcpuCount } +func (conf ServerChangeConfigSettings) CpuReduced() bool { + return conf.VcpuCount < conf.Old.VcpuCount +} + func (conf ServerChangeConfigSettings) AddedCpu() int { addCpu := conf.VcpuCount - conf.Old.VcpuCount if addCpu < 0 { @@ -1536,6 +1590,10 @@ func (conf ServerChangeConfigSettings) ExtraCpuChanged() bool { return conf.ExtraCpuCount != conf.Old.ExtraCpuCount } +func (conf ServerChangeConfigSettings) ExtraCpuReduced() bool { + return conf.ExtraCpuCount < conf.Old.ExtraCpuCount +} + func (conf ServerChangeConfigSettings) AddedExtraCpu() int { addCpu := conf.ExtraCpuCount - conf.Old.ExtraCpuCount if addCpu < 0 { @@ -1548,6 +1606,10 @@ func (conf ServerChangeConfigSettings) MemChanged() bool { return conf.VmemSize != conf.Old.VmemSize } +func (conf ServerChangeConfigSettings) MemReduced() bool { + return conf.VmemSize < conf.Old.VmemSize +} + func (conf ServerChangeConfigSettings) InstanceTypeChanged() bool { return len(conf.InstanceType) > 0 && conf.InstanceType != conf.Old.InstanceType } @@ -1560,6 +1622,11 @@ func (conf ServerChangeConfigSettings) AddedMem() int { return addMem } +// ConfigReduced 是否为降配(降低CPU/内存) +func (conf ServerChangeConfigSettings) ConfigReduced() bool { + return conf.CpuReduced() || conf.MemReduced() || conf.ExtraCpuReduced() +} + func (conf ServerChangeConfigSettings) AddedDisk() int { var size int for _, resize := range conf.Resize { @@ -1572,8 +1639,36 @@ func (conf ServerChangeConfigSettings) AddedDisk() int { } type ServerReleasedIsolatedDevice struct { - DevType string `json:"dev_type"` - Model string `json:"model"` + DevType string `json:"dev_type"` + Model string `json:"model"` + GpuType string `json:"gpu_type"` + SharingMode string `json:"sharing_mode"` + MemoryRequest int `json:"memory_request"` +} + +type ServerAttachIsolatedDeviceBase struct { + AutoStart bool `json:"auto_start"` + GpuType string `json:"gpu_type"` + MemoryRequest *int `json:"memory_request"` + SharingMode string `json:"sharing_mode"` + Count *int `json:"count"` +} + +type ServerAttachIsolatedDeviceInput struct { + ServerAttachIsolatedDeviceBase + Device string `json:"device"` + Model string `json:"model"` +} + +type ServerDetachIsolatedDeviceInputBase struct { + Device string `json:"device"` + Index *int `json:"index"` +} +type ServerDetachIsolatedDeviceInput struct { + Devices []ServerDetachIsolatedDeviceInputBase `json:"devices"` + IsForce bool `json:"is_force"` + DetachAll bool `json:"detach_all"` + AutoStart bool `json:"auto_start"` } type ServerChangeBillingTypeInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/host.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/host.go index 4edc2f4ff..ec40ddbaf 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/host.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/host.go @@ -19,7 +19,6 @@ import ( cloudmux "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" "yunion.io/x/onecloud/pkg/cloudcommon/types" @@ -501,6 +500,8 @@ type HostIpmiAttributes struct { IpmiPresent *bool `json:"ipmi_present"` // lan channel IpmiLanChannel *uint8 `json:"ipmi_lan_channel"` + // RMCP+ cipher suite for ipmitool -C + IpmiCipherSuite *int `json:"ipmi_cipher_suite"` // verified IpmiVerified *bool `json:"ipmi_verified"` // Redfish API support @@ -776,7 +777,7 @@ type HostUploadGuestsStatusInput struct { } type HostIsolatedDeviceNumaStatsInput struct { - DevType string `json:"dev_type"` + Model string } type GuestUploadContainerStatusResponse struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/instance_backup.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/instance_backup.go index f8fbf2ab6..889e8a279 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/instance_backup.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/instance_backup.go @@ -95,6 +95,8 @@ type SSimpleBackup struct { EncryptKeyId string `json:"encrypt_key_id"` // 创建时间 CreatedAt time.Time `json:"created_at"` + // 备份文件路径 + BackupFilePath string `json:"backup_file_path"` } type InstanceBackupRecoveryInput struct { @@ -112,3 +114,12 @@ type InstanceBackupManagerCreateFromPackageInput struct { BackupStorageId string `json:"backup_storage_id"` PackageName string `json:"package_name"` } + +type SStoragePackInstanceBackup struct { + PackageName string `json:"package_name"` + BackupStorageId string `json:"backup_storage_id"` + BackupStorageAccessInfo *SBackupStorageAccessInfo `json:"backup_storage_access_info"` + BackupIds []string `json:"backup_ids"` + Metadata *InstanceBackupPackMetadata `json:"metadata"` + DiskBackups []SSimpleBackup `json:"disk_backups"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/ipset.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/ipset.go new file mode 100644 index 000000000..ef361a89f --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/ipset.go @@ -0,0 +1,130 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "strings" + + "yunion.io/x/pkg/util/regutils" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/httperrors" +) + +type TIpSetType string + +const ( + IpSetTypeIpv4CidrList TIpSetType = "ipv4_cidr_list" + IpSetTypeIpv6CidrList TIpSetType = "ipv6_cidr_list" +) + +func (t TIpSetType) IsValid() bool { + switch t { + case IpSetTypeIpv4CidrList, IpSetTypeIpv6CidrList: + return true + default: + return false + } +} + +type IpSetCreateInput struct { + apis.SharableVirtualResourceCreateInput + + CloudregionResourceInput + CloudproviderResourceInput + + // IP集合类型 + // enum: ["ipv4_cidr_list", "ipv6_cidr_list"] + // required: true + IpSetType TIpSetType `json:"ip_set_type"` + + // IP/CIDR 列表，逗号分隔 + // example: 192.168.1.0/24,10.0.0.1 + // required: true + Data string `json:"data"` +} + +type IpSetUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + + // IP/CIDR 列表，逗号分隔 + // example: 192.168.1.0/24,10.0.0.1 + Data *string `json:"data"` +} + +type IpSetListInput struct { + apis.SharableVirtualResourceListInput + apis.ExternalizedResourceBaseListInput + + ManagedResourceListInput + RegionalFilterListInput + + // 按 IP 集合类型过滤 + // enum: ["ipv4_cidr_list", "ipv6_cidr_list"] + IpSetType []TIpSetType `json:"ip_set_type"` + + // 根据 IP/CIDR 模糊匹配 + Ip string `json:"ip"` +} + +type IpSetDetails struct { + apis.SharableVirtualResourceDetails + ManagedResourceInfo + CloudregionResourceInfo + + SIpSet + + // 关联安全组数量 + SecurityGroupCount int `json:"security_group_count"` +} + +func ValidateIpSetData(ipSetType TIpSetType, data string) error { + if !ipSetType.IsValid() { + return httperrors.NewInputParameterError("invalid ip_set_type %s", ipSetType) + } + data = strings.TrimSpace(data) + if len(data) == 0 { + return httperrors.NewMissingParameterError("data") + } + parts := strings.Split(data, ",") + for i := range parts { + cidr := strings.TrimSpace(parts[i]) + if len(cidr) == 0 { + return httperrors.NewInputParameterError("empty cidr in data") + } + switch ipSetType { + case IpSetTypeIpv4CidrList: + if !regutils.MatchCIDR(cidr) && !regutils.MatchIP4Addr(cidr) { + return httperrors.NewInputParameterError("invalid ipv4 cidr or address %s", cidr) + } + case IpSetTypeIpv6CidrList: + if !regutils.MatchCIDR6(cidr) && !regutils.MatchIP6Addr(cidr) { + return httperrors.NewInputParameterError("invalid ipv6 cidr or address %s", cidr) + } + } + } + return nil +} + +func (input *IpSetCreateInput) Validate() error { + return ValidateIpSetData(input.IpSetType, input.Data) +} + +func (input *IpSetUpdateInput) Validate(currentType TIpSetType) error { + if input.Data != nil { + return ValidateIpSetData(currentType, *input.Data) + } + return nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device.go index 44f71edf2..76f7059fa 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device.go @@ -34,10 +34,17 @@ type IsolateDeviceDetails struct { SIsolatedDevice + // 设备厂商，由 vendor_device_id 经 ID_VENDOR_MAP 翻译 + Vendor string `json:"vendor"` + + MemoryAllocated int + AllocatedCount int + // 云主机名称 - Guest string `json:"guest"` + Guest []string `json:"guest"` + GuestIds []string `json:"guest_ids"` // 云主机状态 - GuestStatus string `json:"guest_status"` + GuestStatus []string `json:"guest_status"` } type IsolatedDeviceListInput struct { @@ -70,6 +77,9 @@ type IsolatedDeviceListInput struct { // 设备VENDOE编号 VendorDeviceId []string `json:"vendor_device_id"` + // 设备厂商，如 HYGON / NVIDIA + Vendor []string `json:"vendor"` + // NUMA节点序号 NumaNode []uint8 `json:"numa_node"` @@ -95,6 +105,9 @@ type IsolatedDeviceCreateInput struct { // example: GPU DevType string `json:"dev_type"` + // DEVICE sharing mode + SharingMode string `json:"sharing_mode"` + // 设备型号 // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `json:"model"` @@ -137,9 +150,28 @@ type IsolatedDeviceUpdateInput struct { DeviceMinor int `json:"device_minor"` } +type SDelIsolatedDeviceInput struct { + Device string + Index int +} + +type SAddIsolatedDeviceInput struct { + Device string + GpuType string + MemoryRequest *int +} + +type SetIsolatedDeviceInput struct { + AddDevices []SAddIsolatedDeviceInput + DelDevices []SDelIsolatedDeviceInput + AutoStart bool +} + type IsolatedDeviceJsonDesc struct { Id string `json:"id"` DevType string `json:"dev_type"` + GpuType string `json:"gpu_type"` + SharingMode string `json:"sharing_mode"` Model string `json:"model"` Addr string `json:"addr"` VendorDeviceId string `json:"vendor_device_id"` @@ -152,6 +184,8 @@ type IsolatedDeviceJsonDesc struct { MemorySize int `json:"memory_size"` MdevId string `json:"mdev_id"` NumaNode int8 `json:"numa_node"` + MemoryLimit int `json:"memory_limit"` + SmUtilLimit int `json:"sm_util_limit"` } type IsolatedDeviceModelCreateInput struct { @@ -368,3 +402,28 @@ type HostIsolatedDeviceModelDetails struct { HotPluggable bool `json:"hot_pluggable"` DisableAutoDetect bool `json:"disable_auto_detect"` } + +type IsolatedDeviceFilterListInput struct { + IsolateDeviceIds []string `json:"isolate_device_ids"` +} + +type GuestIsolatedDeviceListInput struct { + GuestJointsListInput + + IsolatedDeviceListInput + IsolatedDeviceFilterListInput +} + +type GuestIsolatedDeviceDetails struct { + GuestJointResourceDetails + SGuestIsolatedDevice + SIsolatedDevice + HostResourceInfo + apis.SharableResourceBaseInfo + + // 设备厂商，由 vendor_device_id 经 ID_VENDOR_MAP 翻译 + Vendor string `json:"vendor"` + + MemoryAllocated int + AllocatedCount int +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device_const.go index 05d5bff96..3898eebbb 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/isolated_device_const.go @@ -15,26 +15,108 @@ package compute const ( - DIRECT_PCI_TYPE = "PCI" - GPU_HPC_TYPE = "GPU-HPC" // # for compute - GPU_VGA_TYPE = "GPU-VGA" // # for display - SRIOV_VGPU_TYPE = "SRIOV-VGPU" - LEGACY_VGPU_TYPE = "LEGACY-VGPU" - USB_TYPE = "USB" - NIC_TYPE = "NIC" // nic sriov - NVME_PT_TYPE = "NVME-PT" // nvme passthrough - - NVIDIA_VENDOR_ID = "10de" - AMD_VENDOR_ID = "1002" + DIRECT_PCI_TYPE = "PCI" + GPU_TYPE = "GPU" + USB_TYPE = "USB" + NIC_TYPE = "NIC" // nic sriov + NVME_PT_TYPE = "NVME-PT" // nvme passthrough + NETINT_TYPE = "NETINT" + NPU_TYPE = "NPU" + BINDER_TYPE = "BINDER" ) -var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} -var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE} +var VALID_TYPES = []string{DIRECT_PCI_TYPE, GPU_TYPE, USB_TYPE, NIC_TYPE, NVME_PT_TYPE, NETINT_TYPE, NPU_TYPE, BINDER_TYPE} + +const ( + DEVICE_SHARING_MODE_EXCLUSIVE = "EXCLUSIVE" + DEVICE_SHARING_MODE_SRIOV = "SRIOV" + DEVICE_SHARING_MODE_MPS = "MPS" + DEVICE_SHARING_MODE_HAMI = "HAMI" + DEVICE_SHARING_MODE_UNLIMITED = "UNLIMITED" + DEVICE_SHARING_MODE_MDEV = "MDEV" +) + +var VIRTUAL_SHARING_MODES = []string{DEVICE_SHARING_MODE_HAMI, DEVICE_SHARING_MODE_UNLIMITED, DEVICE_SHARING_MODE_MPS} +var VAILD_SHARING_MODES = []string{DEVICE_SHARING_MODE_EXCLUSIVE, DEVICE_SHARING_MODE_SRIOV, DEVICE_SHARING_MODE_MDEV, DEVICE_SHARING_MODE_HAMI, DEVICE_SHARING_MODE_UNLIMITED, DEVICE_SHARING_MODE_MPS} + +const ( + GPU_HPC = "HPC" + GPU_VGA = "VGA" +) + +const ( + NVIDIA_VENDOR_ID = "10de" + AMD_VENDOR_ID = "1002" + VASTAITECH_VENDOR_ID = "1ec6" + HYGON_VENDOR_ID = "1d94" + ASCEND_VENDOR_ID = "19e5" + ILUVATAR_VENDOR_ID = "1e3e" + THEAD_VENDOR_ID = "1ded" + KUNLUNXIN_VENDOR_ID = "1d22" +) + +var ID_VENDOR_MAP = map[string]string{ + NVIDIA_VENDOR_ID: "NVIDIA", + AMD_VENDOR_ID: "AMD", + VASTAITECH_VENDOR_ID: "VASTAITECH", + ASCEND_VENDOR_ID: "ASCEND", + HYGON_VENDOR_ID: "HYGON", + ILUVATAR_VENDOR_ID: "ILUVATAR", + THEAD_VENDOR_ID: "THEAD", + KUNLUNXIN_VENDOR_ID: "KUNLUNXIN", +} + +var VENDOR_ID_MAP = map[string]string{ + "NVIDIA": NVIDIA_VENDOR_ID, + "AMD": AMD_VENDOR_ID, + "VASTAITECH": VASTAITECH_VENDOR_ID, + "ASCEND": ASCEND_VENDOR_ID, + "HYGON": HYGON_VENDOR_ID, + "ILUVATAR": ILUVATAR_VENDOR_ID, + "THEAD": THEAD_VENDOR_ID, + "KUNLUNXIN": KUNLUNXIN_VENDOR_ID, +} + +const ( + ISOLATED_DEVICE_MODEL_METADATA_BANDWIDTH = "bandwidth" + ISOLATED_DEVICE_MODEL_METADATA_MEMORY_MB = "memory_mb" + ISOLATED_DEVICE_MODEL_METADATA_TFLOPS = "tflops" +) + +////////////// deprecated + +const ( + GPU_HPC_TYPE = "GPU-HPC" // # for compute + GPU_VGA_TYPE = "GPU-VGA" // # for display + SRIOV_VGPU_TYPE = "SRIOV-VGPU" //done + LEGACY_VGPU_TYPE = "LEGACY-VGPU" //done +) + +var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE} //done +var VALID_ATTACH_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE, USB_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE} //done + +var GPU_TYPES = []string{ + GPU_HPC_TYPE, GPU_VGA_TYPE, SRIOV_VGPU_TYPE, LEGACY_VGPU_TYPE, + CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_VASTAITECH_GPU, + CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, +} + +var NETINT_TYPES = []string{ + CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, +} var VALID_CONTAINER_DEVICE_TYPES = []string{ CONTAINER_DEV_CPH_AMD_GPU, CONTAINER_DEV_CPH_AOSP_BINDER, CONTAINER_DEV_NETINT_CA_QUADRA, - CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, + CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_GPU, CONTAINER_DEV_NVIDIA_MPS, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_VASTAITECH_GPU, + CONTAINER_DEV_HYGON_DCU, CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } var VALID_PASSTHROUGH_TYPES = []string{ @@ -49,27 +131,17 @@ var VITRUAL_DEVICE_TYPES = []string{ CONTAINER_DEV_NETINT_CA_QUADRA, CONTAINER_DEV_NETINT_CA_ASIC, CONTAINER_DEV_NVIDIA_MPS, + CONTAINER_DEV_NVIDIA_HAMI, CONTAINER_DEV_ASCEND_NPU, CONTAINER_DEV_NVIDIA_GPU_SHARE, CONTAINER_DEV_VASTAITECH_GPU, + CONTAINER_DEV_HYGON_DCU, + CONTAINER_DEV_HYGON_DCU_HAMI, + CONTAINER_DEV_ILUVATAR_GPU, + CONTAINER_DEV_THEAD_PPU, + CONTAINER_DEV_KUNLUNXIN_XPU, } -var ID_VENDOR_MAP = map[string]string{ - NVIDIA_VENDOR_ID: "NVIDIA", - AMD_VENDOR_ID: "AMD", -} - -var VENDOR_ID_MAP = map[string]string{ - "NVIDIA": NVIDIA_VENDOR_ID, - "AMD": AMD_VENDOR_ID, -} - -const ( - ISOLATED_DEVICE_MODEL_METADATA_BANDWIDTH = "bandwidth" - ISOLATED_DEVICE_MODEL_METADATA_MEMORY_MB = "memory_mb" - ISOLATED_DEVICE_MODEL_METADATA_TFLOPS = "tflops" -) - func init() { VALID_PASSTHROUGH_TYPES = append(VALID_PASSTHROUGH_TYPES, VALID_CONTAINER_DEVICE_TYPES...) VALID_ATTACH_TYPES = append(VALID_ATTACH_TYPES, VALID_CONTAINER_DEVICE_TYPES...) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/pod.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/pod.go index 98e01a458..42eeb3dec 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/pod.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/pod.go @@ -84,6 +84,10 @@ type PodCreateInput struct { HostIPC bool `json:"host_ipc"` //PortMappings []*PodPortMapping `json:"port_mappings"` SecurityContext *PodSecurityContext `json:"security_context,omitempty"` + // DisableCgroupCpuLimit skips setting CPU CFS quota on the pod sandbox and containers. + DisableCgroupCpuLimit bool `json:"disable_cgroup_cpu_limit"` + // DisableCgroupMemoryLimit skips setting memory hard limit on the pod sandbox and containers. + DisableCgroupMemoryLimit bool `json:"disable_cgroup_memory_limit"` } type PodStartResponse struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgroup.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgroup.go index a11314524..e436e0ed4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgroup.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgroup.go @@ -15,7 +15,6 @@ package compute import ( - "fmt" "strings" "yunion.io/x/pkg/errors" @@ -71,6 +70,11 @@ type SSecgroupRuleResource struct { // example: 192.168.222.121 CIDR string `json:"cidr"` + // 目标类型 + // enum: ["cidr", "ip_set", "security_group"] + // required: true + TargetType TSecgroupTargetType `json:"target_type"` + // 行为 // deny: 拒绝 // allow: 允许 @@ -135,6 +139,9 @@ type SSecgroupRuleUpdateInput struct { // requried: false // example: test to create rule Description string `json:"description"` + + // swagger:ignore + TargetType TSecgroupTargetType `json:"target_type"` } func IsValidSecgroupRuleCIDR(cidr string) bool { @@ -177,13 +184,31 @@ func (input *SSecgroupRuleResource) Check() error { } } - if len(input.CIDR) > 0 { - if !IsValidSecgroupRuleCIDR(input.CIDR) { - return fmt.Errorf("invalid cidr: %s", input.CIDR) + if len(input.TargetType) == 0 { + input.TargetType = SecurityGroupRuleTargetTypeCidr + } + switch input.TargetType { + case SecurityGroupRuleTargetTypeCidr: + if len(input.CIDR) > 0 { + if !IsValidSecgroupRuleCIDR(input.CIDR) { + return errors.Wrapf(errors.ErrInvalidFormat, "invalid cidr: %s", input.CIDR) + } + } else { + // empty CIDR means both IPv4 and IPv6 + // input.CIDR = "0.0.0.0/0" + } + case SecurityGroupRuleTargetTypeIpSet: + if len(input.CIDR) > 0 { + + } else { + return errors.Wrap(errors.ErrEmpty, "empty ip set id") } - } else { - // empty CIDR means both IPv4 and IPv6 - // input.CIDR = "0.0.0.0/0" + case SecurityGroupRuleTargetTypeIpSetGroup: + fallthrough + case SecurityGroupRuleTargetTypeSecurityGroup: + fallthrough + default: + return errors.Wrapf(errors.ErrNotSupported, "unsupported target type %s", input.TargetType) } return rule.ValidateRule() @@ -239,6 +264,9 @@ type SecgroupListInput struct { LoadbalancerId string `json:"loadbalancer_id"` RegionalFilterListInput ManagedResourceListInput + + // 指定过滤规则中含有指定ip集的安全组 + IpSetId []string `json:"ip_set_id"` } type SecurityGroupRuleListInput struct { @@ -258,6 +286,8 @@ type SecurityGroupRuleListInput struct { Ports string `json:"ports"` // 根据ip模糊匹配安全组规则 Ip string `json:"ip"` + // 根据target_type字段过滤安全组规则 + TargetType []string `json:"target_type"` } type SecgroupResourceInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgrouprule.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgrouprule.go index 74ba469e9..2839001a4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgrouprule.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/secgrouprule.go @@ -16,6 +16,15 @@ package compute import "yunion.io/x/onecloud/pkg/apis" +type TSecgroupTargetType string + +const ( + SecurityGroupRuleTargetTypeIpSet TSecgroupTargetType = "ip_set" + SecurityGroupRuleTargetTypeIpSetGroup TSecgroupTargetType = "ip_set_group" + SecurityGroupRuleTargetTypeSecurityGroup TSecgroupTargetType = "security_group" + SecurityGroupRuleTargetTypeCidr TSecgroupTargetType = "cidr" +) + type SecgroupRuleDetails struct { apis.ResourceBaseDetails apis.ProjectizedResourceInfo @@ -23,4 +32,7 @@ type SecgroupRuleDetails struct { SecurityGroupResourceInfo ProjectId string `json:"tenant_id"` + + TargetIpSet string `json:"target_ip_set"` + TargetSecurityGroup string `json:"target_security_group"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/snapshot.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/snapshot.go index 14bcb15a0..d9ff31931 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/snapshot.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/snapshot.go @@ -60,8 +60,6 @@ type SnapshotListInput struct { StorageShareFilterListInput - // filter snapshot that is fake deleted - FakeDeleted *bool `json:"fake_deleted"` // filter by disk type DiskType string `json:"disk_type"` // filter instance snapshot @@ -70,8 +68,7 @@ type SnapshotListInput struct { DiskFilterListInputBase StorageFilterListInputBase - OutOfChain *bool `json:"out_of_chain"` - OsType []string `json:"os_type"` + OsType []string `json:"os_type"` // list server snapshots ServerId string `json:"server_id"` diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/compute/zz_generated.model.go b/vendor/yunion.io/x/onecloud/pkg/apis/compute/zz_generated.model.go index 5dd622812..5c460f226 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/compute/zz_generated.model.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/compute/zz_generated.model.go @@ -23,6 +23,13 @@ import ( "yunion.io/x/onecloud/pkg/apis/cloudprovider" ) +// IsolatedDeviceAllocateStat is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.IsolatedDeviceAllocateStat. +type IsolatedDeviceAllocateStat struct { + SIsolatedDevice + GuestCount int `json:"guest_count"` + MemoryAllocated int `json:"memory_allocated"` +} + // SAccessGroup is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SAccessGroup. type SAccessGroup struct { apis.SStatusInfrasResourceBase @@ -943,6 +950,10 @@ type SElasticip struct { BgpType string `json:"bgp_type"` // 是否跟随主机删除而自动释放 AutoDellocate *bool `json:"auto_dellocate,omitempty"` + // 下行带宽限制，单位mbps + RxBwLimit int `json:"rx_bw_limit"` + // 上行带宽限制，单位mbps + TxBwLimit int `json:"tx_bw_limit"` } // SExternalProject is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SExternalProject. @@ -1117,6 +1128,23 @@ type SGuest struct { ResourcePool string `json:"resource_pool"` } +// SGuestIsolatedDevice is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestIsolatedDevice. +type SGuestIsolatedDevice struct { + SGuestJointsBase + SIsolatedDeviceResourceBase + // guest isolated device memory size limit + DeviceMemorySize int `json:"device_memory_size"` + // guest isolated device Streaming Multiprocessor Utilization limit + SmUtilLimit int `json:"sm_util_limit"` + // gpu device work type: HPC VGA + GpuType string `json:"gpu_type"` + // guest network index + NetworkIndex int `json:"network_index"` + // guest disk index + DiskIndex byte `json:"disk_index"` + Index byte `json:"index"` +} + // SGuestJointsBase is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestJointsBase. type SGuestJointsBase struct { apis.SVirtualJointResourceBase @@ -1253,6 +1281,10 @@ type SGuestnetwork struct { PortMappings []*GuestPortMapping `json:"port_mappings"` SBillingTypeBase SBillingChargeTypeBase + // 下行带宽限制，单位mbps + RxBwLimit int `json:"rx_bw_limit"` + // 上行带宽限制，单位mbps + TxBwLimit int `json:"tx_bw_limit"` } // SGuestnetworksecgroup is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SGuestnetworksecgroup. @@ -1538,7 +1570,7 @@ type SInstanceSnapshot struct { // 套餐名称 InstanceType string `json:"instance_type"` // 主机快照磁盘容量和 - SizeMb int `json:"size_mb"` + // SizeMb int `nullable:"false" list:"user"` // 镜像ID ImageId string `json:"image_id"` // 是否保存内存 @@ -1584,21 +1616,33 @@ type SInterVpcNetworkVpc struct { VpcId string `json:"vpc_id"` } +// SIpSet is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIpSet. +type SIpSet struct { + apis.SSharableVirtualResourceBase + apis.SExternalizedResourceBase + SManagedResourceBase + SCloudregionResourceBase + // IP集合类型 + IpSetType TIpSetType `json:"ip_set_type"` + // IP/CIDR 列表，逗号分隔 + Data string `json:"data"` +} + // SIsolatedDevice is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDevice. type SIsolatedDevice struct { apis.SStandaloneResourceBase apis.SExternalizedResourceBase apis.SSharableBaseResource SHostResourceBase - // # PCI / GPU-HPC / GPU-VGA / USB / NIC + // # PCI / GPU / USB / NIC ... // 设备类型 DevType string `json:"dev_type"` + // EXCLUSIVE / SRIOV / MPS / HAMI / SHARE / MIG + SharingMode string `json:"sharing_mode"` + // Device is hot pluggable + HotPluggable bool `json:"hot_pluggable"` // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `json:"model"` - // 云主机Id - GuestId string `json:"guest_id"` - // guest network index - NetworkIndex int `json:"network_index"` // Nic wire id WireId string `json:"wire_id"` // Offload interface name @@ -1607,9 +1651,7 @@ type SIsolatedDevice struct { IsInfinibandNic bool `json:"is_infiniband_nic"` // NVME disk size NvmeSizeMB int `json:"nvme_size_mb"` - // guest disk index - DiskIndex byte `json:"disk_index"` - // # pci address of `Bus:Device.Function` format, or usb bus address of `bus.addr` + // # pci address of `Bus:Device.Function` format, or usb bus address of `bus:addr:port` Addr string `json:"addr"` DevicePath string `json:"device_path"` // GPU card path, like /dev/dri/cardX @@ -1655,6 +1697,11 @@ type SIsolatedDevice struct { PcieInfo *IsolatedDevicePCIEInfo `json:"pcie_info"` // device numa node NumaNode byte `json:"numa_node"` + // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, + // or per-slice quota for MPS share mode). 0 means unknown / not applicable. + MemorySize int `json:"memory_size"` + // some of isolated device type support virtual num, like NVIDIA_GPU_SHARE, NVIDIA_MPS + VirtualNum int `json:"virtual_num"` } // SIsolatedDeviceModel is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDeviceModel. @@ -1669,6 +1716,11 @@ type SIsolatedDeviceModel struct { DisableAutoDetect *bool `json:"disable_auto_detect,omitempty"` } +// SIsolatedDeviceResourceBase is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SIsolatedDeviceResourceBase. +type SIsolatedDeviceResourceBase struct { + IsolatedDeviceId string `json:"isolated_device_id"` +} + // SKafka is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SKafka. type SKafka struct { apis.SVirtualResourceBase @@ -2731,6 +2783,8 @@ type SSecurityGroupRule struct { CIDR string `json:"cidr"` Action string `json:"action"` Description string `json:"description"` + + TargetType TSecgroupTargetType `json:"target_type"` } // SServerSku is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SServerSku. @@ -2739,7 +2793,6 @@ type SServerSku struct { apis.SExternalizedResourceBase SCloudregionResourceBase SZoneResourceBase - // SkuId string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"admin_required"` // x2.large InstanceTypeFamily string `json:"instance_type_family"` // x2 InstanceTypeCategory string `json:"instance_type_category"` @@ -2816,18 +2869,15 @@ type SSnapshotPolicy struct { apis.SExternalizedResourceBase SManagedResourceBase SCloudregionResourceBase - // 快照保留天数, -1: 表示永久保留 + // 快照保留天数 RetentionDays int `json:"retention_days"` // 快照保留数量, 优先级高于 RetentionDays, 且仅对本地IDC资源有效 RetentionCount int `json:"retention_count"` // 快照类型, 目前支持 disk, server - // disk: 自动磁盘快照策略， 只能关联磁盘 - // server: 自动主机快照策略, 只能关联主机 Type string `json:"type"` - // 1~7, 1 is Monday, 7 is Sunday + // 1~7, 1 is Monday RepeatWeekdays []int `json:"repeat_weekdays"` - // 0~23, 每小时 - // 创建自动快照策略的时间必须与 RepeatWeekdays 对应的创建周期相一致 + // 0~23 TimePoints []int `json:"time_points"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/const.go b/vendor/yunion.io/x/onecloud/pkg/apis/const.go index cd8e7aeac..722fe388b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/const.go @@ -61,6 +61,7 @@ const ( STATUS_CREATING = "creating" STATUS_AVAILABLE = "available" STATUS_CREATE_FAILED = "create_failed" + STATUS_UPDATE_FAILED = "update_failed" // 更改计费模式 STATUS_CHANGE_BILLING_TYPE = "change_billing_type" diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/container.go b/vendor/yunion.io/x/onecloud/pkg/apis/container.go index a5952d7f7..13322ecd4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/container.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/container.go @@ -70,8 +70,10 @@ const ( ) type ContainerSecurityContext struct { - RunAsUser *int64 `json:"run_as_user,omitempty"` - RunAsGroup *int64 `json:"run_as_group,omitempty"` + RunAsUser *int64 `json:"run_as_user,omitempty"` + RunAsGroup *int64 `json:"run_as_group,omitempty"` + SupplementalGroups []int64 `json:"supplemental_groups,omitempty"` + SupplementalGroupNames []string `json:"supplemental_group_names,omitempty"` // procMount denotes the type of proc mount to use for the containers. // The default is DefaultProcMount which uses the container runtime defaults for ProcMount ContainerProcMountType `json:"proc_mount"` @@ -121,6 +123,8 @@ type ContainerIsolatedDeviceCDI struct { type ContainerSpec struct { // Image to use. Image string `json:"image"` + // Container image id managed by glance + ContainerImageId string `json:"container_image_id"` // Image pull policy ImagePullPolicy ImagePullPolicy `json:"image_pull_policy"` // Image credential id @@ -134,17 +138,21 @@ type ContainerSpec struct { // List of environment variable to set in the container. Envs []*ContainerKeyValue `json:"envs"` // Enable lxcfs - EnableLxcfs bool `json:"enable_lxcfs"` - Capabilities *ContainerCapability `json:"capabilities"` - Privileged bool `json:"privileged"` - DisableNoNewPrivs bool `json:"disable_no_new_privs"` - Lifecyle *ContainerLifecyle `json:"lifecyle"` - CgroupDevicesAllow []string `json:"cgroup_devices_allow"` - CgroupPidsMax int `json:"cgroup_pids_max"` - ResourcesLimit *ContainerResources `json:"resources_limit"` - SimulateCpu bool `json:"simulate_cpu"` - ShmSizeMB int `json:"shm_size_mb"` - SecurityContext *ContainerSecurityContext `json:"security_context,omitempty"` + EnableLxcfs bool `json:"enable_lxcfs"` + Capabilities *ContainerCapability `json:"capabilities"` + Privileged bool `json:"privileged"` + DisableNoNewPrivs bool `json:"disable_no_new_privs"` + Lifecyle *ContainerLifecyle `json:"lifecyle"` + CgroupDevicesAllow []string `json:"cgroup_devices_allow"` + CgroupPidsMax int `json:"cgroup_pids_max"` + ResourcesLimit *ContainerResources `json:"resources_limit"` + // DisableCgroupCpuLimit skips setting CPU CFS quota for this container. + DisableCgroupCpuLimit bool `json:"disable_cgroup_cpu_limit"` + // DisableCgroupMemoryLimit skips setting memory hard limit for this container. + DisableCgroupMemoryLimit bool `json:"disable_cgroup_memory_limit"` + SimulateCpu bool `json:"simulate_cpu"` + ShmSizeMB int `json:"shm_size_mb"` + SecurityContext *ContainerSecurityContext `json:"security_context,omitempty"` // Periodic probe of container liveness. // Container will be restarted if the probe fails. // Cannot be updated. diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/host/container.go b/vendor/yunion.io/x/onecloud/pkg/apis/host/container.go index f361a4d1d..c9f4ba50b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/host/container.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/host/container.go @@ -93,8 +93,11 @@ type ContainerIsolatedDevice struct { Addr string `json:"addr"` Path string `json:"path"` DeviceType string `json:"device_type"` + SharingMode string `json:"sharing_mode"` CardPath string `json:"card_path"` RenderPath string `json:"render_path"` + MemoryLimit int `json:"memory_limit"` + SmUtilLimit int `json:"sm_util_limit"` Index int `json:"index"` DeviceMinor int `json:"device_minor"` OnlyEnv []*apis.ContainerIsolatedDeviceOnlyEnv `json:"only_env"` diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/identity/identityprovider.go b/vendor/yunion.io/x/onecloud/pkg/apis/identity/identityprovider.go index 218d9c4fb..6a795c761 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/identity/identityprovider.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/identity/identityprovider.go @@ -104,6 +104,14 @@ type IdentityProviderCreateInput struct { IconUri string `json:"icon_uri"` } +type IdentityProviderPropertyAttributeNamesInput struct { + // 后端驱动名称 + Driver string `json:"driver"` + + // 模板名称 + Template string `json:"template"` +} + type GetIdpSamlMetadataInput struct { // 缩进展示SAML sp metadata Pretty *bool `json:"pretty"` diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/identity/oidc.go b/vendor/yunion.io/x/onecloud/pkg/apis/identity/oidc.go index b53f0cd52..0ef0d2552 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/identity/oidc.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/identity/oidc.go @@ -29,6 +29,8 @@ type SOIDCIdpConfigOptions struct { TimeoutSecs int `json:"timeout_secs"` SIdpAttributeOptions + + AttributeNames map[string]string `json:"attribute_names"` } type SOIDCDexConfigOptions struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/identity/rolepolicy.go b/vendor/yunion.io/x/onecloud/pkg/apis/identity/rolepolicy.go index 180cddf7d..4ab66e9ca 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/identity/rolepolicy.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/identity/rolepolicy.go @@ -23,6 +23,14 @@ import ( "yunion.io/x/onecloud/pkg/util/rbacutils" ) +type RolePolicyCreateInput struct { + apis.ResourceBaseCreateInput + + RoleId string `json:"role_id"` + ProjectId string `json:"project_id"` + PolicyId string `json:"policy_id"` +} + type RolePolicyListInput struct { apis.ResourceBaseListInput diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/identity/saml.go b/vendor/yunion.io/x/onecloud/pkg/apis/identity/saml.go index f7135fb27..3713917d5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/identity/saml.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/identity/saml.go @@ -34,6 +34,9 @@ type SIdpAttributeOptions struct { type SSAMLIdpBaseConfigOptions struct { AllowIdpInit *bool `json:"allow_idp_init"` + // VerifySignature, when true, checks the SAML response signature with SigningCert. + VerifySignature *bool `json:"verify_signature"` + SigningCert string `json:"signing_cert"` } type SSAMLIdpConfigOptions struct { @@ -43,6 +46,8 @@ type SSAMLIdpConfigOptions struct { SSAMLIdpBaseConfigOptions SIdpAttributeOptions + + AttributeNames map[string]string `json:"attribute_names"` } type SSAMLTestIdpConfigOptions struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/image/consts.go b/vendor/yunion.io/x/onecloud/pkg/apis/image/consts.go index 3a7ec7a4b..709242fee 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/image/consts.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/image/consts.go @@ -51,9 +51,13 @@ const ( LocalFilePrefix = "file://" S3Prefix = "s3://" + NfsPrefix = "nfs://" + + NfsSubDirName = "images" IMAGE_STORAGE_DRIVER_LOCAL = "local" IMAGE_STORAGE_DRIVER_S3 = "s3" + IMAGE_STORAGE_DRIVER_NFS = "nfs" // image properties IMAGE_OS_ARCH = "os_arch" @@ -70,9 +74,11 @@ const ( IMAGE_PARTITION_TYPE = "partition_type" IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init" IMAGE_DISABLE_USB_KBD = "disable_usb_kbd" + IMAGE_MACHINE_TYPE = "machine_type" IMAGE_VDI_PROTOCOL = "vdi_protocol" IMAGE_INTERNAL_PATH_MAP = "internal_path_map" IMAGE_USED_BY_POST_OVERLAY = "used_by_post_overlay" + IMAGE_WIN_VIRTIO_NET = "win_virtio_net" IMAGE_STATUS_UPDATING = "updating" ) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/image/container_image.go b/vendor/yunion.io/x/onecloud/pkg/apis/image/container_image.go new file mode 100644 index 000000000..d3fbb6001 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/image/container_image.go @@ -0,0 +1,89 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "reflect" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" +) + +const ( + CONTAINER_IMAGE_STATUS_READY = "ready" +) + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(&ContainerImageEnvs{}), func() gotypes.ISerializable { + return &ContainerImageEnvs{} + }) +} + +// ContainerImageEnvs is the default environment variables of a container image. +// It is a named serializable type so that it can be stored as a compound column. +type ContainerImageEnvs []*apis.ContainerKeyValue + +func (e ContainerImageEnvs) String() string { + return jsonutils.Marshal(e).String() +} + +func (e ContainerImageEnvs) IsZero() bool { + return len(e) == 0 +} + +type ContainerImageListInput struct { + apis.SharableVirtualResourceListInput + + ImageLabel string `json:"image_label"` + ImageName string `json:"image_name"` + RegistryId string `json:"registry_id"` +} + +type ContainerImageCreateInput struct { + apis.SharableVirtualResourceCreateInput + + ImageName string `json:"image_name"` + ImageLabel string `json:"image_label"` + CredentialId string `json:"credential_id"` + RegistryId string `json:"registry_id"` + + // Default startup command when a container does not specify one + Command []string `json:"command"` + // Default startup args when a container does not specify one + Args []string `json:"args"` + // Default environment variables, appended to the container envs + Envs *ContainerImageEnvs `json:"envs"` +} + +type ContainerImageUpdateInput struct { + apis.SharableVirtualResourceCreateInput + + ImageName *string `json:"image_name,omitempty"` + ImageLabel *string `json:"image_label,omitempty"` + CredentialId *string `json:"credential_id,omitempty"` + RegistryId *string `json:"registry_id,omitempty"` + + Command *[]string `json:"command,omitempty"` + Args *[]string `json:"args,omitempty"` + Envs *ContainerImageEnvs `json:"envs,omitempty"` +} + +type ContainerImageDetails struct { + apis.SharableVirtualResourceDetails + + Registry string `json:"registry"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/image/container_registry.go b/vendor/yunion.io/x/onecloud/pkg/apis/image/container_registry.go new file mode 100644 index 000000000..b6881dbd6 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/image/container_registry.go @@ -0,0 +1,122 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type ContainerRegistryType string + +const ( + ContainerRegistryTypeHarbor = "harbor" + ContainerRegistryTypeCommon = "common" + ContainerRegistryTypeCustom = "custom" +) + +type ContainerRegistryListInput struct { + apis.SharableVirtualResourceListInput + + Type string `json:"type"` + Url string `json:"url"` +} + +type ContainerRegistryConfigCommon struct { + apis.ContainerPullImageAuthConfig +} + +type ContainerRegistryConfigHarbor struct { + ContainerRegistryConfigCommon +} + +type ContainerRegistryConfigCustom struct { + ContainerRegistryConfigCommon +} + +type ContainerRegistryConfig struct { + Type ContainerRegistryType `json:"type"` + Common *ContainerRegistryConfigCommon `json:"common"` + Harbor *ContainerRegistryConfigHarbor `json:"harbor"` + Custom *ContainerRegistryConfigCustom `json:"custom"` +} + +type ContainerRegistryCreateInput struct { + apis.SharableVirtualResourceCreateInput + + // Repo type + // required: true + // enum: harbor,common,custom + Type ContainerRegistryType `json:"type"` + + // Repo URL + // required: true + // example: https://10.127.190.187 + Url string `json:"url"` + + // Configuration info + Config ContainerRegistryConfig `json:"config"` + + // Credential ID (used when importing existing credentials) + CredentialId string `json:"credential_id"` +} + +type ContainerRegistryUploadImageInput struct { + // Repository is the path on server, e.g. 'yunion/influxdb' + Repository string `json:"repository"` + // Tag is image tag + Tag string `json:"tag"` +} + +type ContainerRegistryGetImageTagsInput struct { + Repository string `json:"repository"` +} + +type ContainerRegistryManagerDownloadImageInput struct { + Insecure bool `json:"insecure"` + Image string `json:"image"` + Username string `json:"username"` + Password string `json:"password"` +} + +type ContainerRegistryDownloadImageInput struct { + ImageName string `json:"image_name"` + Tag string `json:"tag"` +} + +type ContainerRegistryListImagesInput struct { + Details bool `json:"details"` + RepositoryName string `json:"repository_name"` +} + +type ContainerRegistryImportFromKubeserverInput struct { + // Force re-import even if a registry with the same URL already exists + Force bool `json:"force"` +} + +type ContainerRegistryImportFromKubeserverOutput struct { + Imported []string `json:"imported"` + Skipped []string `json:"skipped"` + Failed []string `json:"failed"` +} + +type ContainerRegistryDetails struct { + apis.SharableVirtualResourceDetails + + Url string `json:"url"` + Type string `json:"type"` + CredentialId string `json:"credential_id"` + Credential string `json:"credential"` + Config *ContainerRegistryConfig `json:"config"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/input.go b/vendor/yunion.io/x/onecloud/pkg/apis/input.go index ec71ac03f..5201a44c2 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/input.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/input.go @@ -423,6 +423,8 @@ type AutoRenewInput struct { } type RenewInput struct { + // 续费时长 + // example: 1d, 1w, 1m Duration string `json:"duration"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/image.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/image.go index bdfce3968..48b349d58 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/image.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/image.go @@ -16,8 +16,8 @@ const ( LLM_IMAGE_TYPE_COMFYUI LLMImageType = "comfyui" LLM_IMAGE_TYPE_OPENCLAW LLMImageType = "openclaw" LLM_IMAGE_TYPE_HERMES_AGENT LLMImageType = "hermes-agent" - LLM_IMAGE_TYPE_LLM_ROUTER LLMImageType = "llm-router" LLM_IMAGE_TYPE_DESKTOP LLMImageType = "desktop" + LLM_IMAGE_TYPE_BENCHMARK LLMImageType = "benchmark" ) var ( @@ -29,8 +29,8 @@ var ( string(LLM_IMAGE_TYPE_COMFYUI), string(LLM_IMAGE_TYPE_OPENCLAW), string(LLM_IMAGE_TYPE_HERMES_AGENT), - string(LLM_IMAGE_TYPE_LLM_ROUTER), string(LLM_IMAGE_TYPE_DESKTOP), + string(LLM_IMAGE_TYPE_BENCHMARK), ) ) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/instantmodel.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/instantmodel.go index 42a0a401e..a93d0ca21 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/instantmodel.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/instantmodel.go @@ -17,6 +17,7 @@ type InstantModelListInput struct { ModelTag string `json:"model_tag"` ModelId string `json:"model_id"` LlmType string `json:"llm_type"` + Source string `json:"source"` Image string `json:"image"` Mounts string `json:"mounts"` @@ -89,6 +90,9 @@ type MountedByLLMInfo struct { type InstantModelSyncstatusInput struct { } +type InstantModelResumeImportInput struct { +} + // InstantModelVramRequirement reports the heuristic VRAM needed to run this // model, mirroring GPUStack's `estimate_model_vram`. Returns // `vram_required_mb=0` when `weight_size_bytes` is unknown — callers should diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm.go index 488c31513..6a2fc01e3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm.go @@ -9,7 +9,8 @@ import ( ) const ( - SERVICE_TYPE = "llm" + SERVICE_TYPE = "llm" + SERVICE_VERSION = "" ) type LLMBaseListDetails struct { @@ -204,6 +205,7 @@ type LLMSyncStatusInput struct { } type LLMRestartInput struct { + Force bool `json:"force"` } type LLMRestartTaskInput struct { @@ -217,6 +219,7 @@ type LLMRestartTaskInput struct { RebindVolumeId string OnlyStop bool + Force bool `json:"force"` } type LLMChangeNetworkInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark.go new file mode 100644 index 000000000..0c7b91f77 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark.go @@ -0,0 +1,166 @@ +package llm + +import ( + "reflect" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" +) + +const ( + LLMBenchmarkStatePending = "pending" + LLMBenchmarkStateQueued = "queued" + LLMBenchmarkStateValidating = "validating" + LLMBenchmarkStateRunning = "running" + LLMBenchmarkStateCompleted = "completed" + LLMBenchmarkStateStopped = "stopped" + LLMBenchmarkStateError = "error" +) + +const ( + LLMBenchmarkDatasetSyntheticText = "synthetic_text" + LLMBenchmarkDatasetPackage = "benchmark_package" + LLMBenchmarkProfileConstant = "constant" +) + +const ( + LLMBenchmarkEvaluationStateEvaluating = "evaluating" + LLMBenchmarkEvaluationStateCompleted = "completed" + LLMBenchmarkEvaluationStateSkipped = "skipped" + LLMBenchmarkEvaluationStateError = "error" +) + +const ( + LLMBenchmarkArtifactStorageLocal = "local" + LLMBenchmarkArtifactStorageMinio = "minio" +) + +const ( + LLMBenchmarkDefaultRequestFormat = "/v1/chat/completions" + LLMBenchmarkDefaultImage = "registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64" +) + +type LLMBenchmarkCreateInput struct { + apis.VirtualResourceCreateInput + + LLMId string `json:"llm_id"` + BenchmarkImage string `json:"benchmark_image,omitempty"` + BenchmarkPackage string `json:"benchmark_package,omitempty"` + + RequestFormat string `json:"request_format,omitempty"` + Model string `json:"model,omitempty"` + + Profile string `json:"profile,omitempty"` + RequestRate int `json:"request_rate,omitempty"` + + TotalRequests int `json:"total_requests,omitempty"` + MaxDurationSeconds int `json:"max_duration_seconds,omitempty"` + MaxErrors int `json:"max_errors,omitempty"` + + DatasetName string `json:"dataset_name,omitempty"` + DatasetInputTokens int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens int `json:"dataset_output_tokens,omitempty"` + DatasetPath string `json:"dataset_path,omitempty"` + + LLMDeploymentId string `json:"llm_deployment_id,omitempty"` + LLMSkuId string `json:"llm_sku_id,omitempty"` + LLMImageId string `json:"llm_image_id,omitempty"` + BenchmarkPackageId string `json:"benchmark_package_id,omitempty"` + Backend string `json:"backend,omitempty"` + TargetUrl string `json:"target_url,omitempty"` + WorkDir string `json:"work_dir,omitempty"` + TargetSnapshot string `json:"target_snapshot,omitempty"` + GuideLLMSpec string `json:"guide_llm_spec,omitempty"` +} + +type LLMBenchmarkCopyInput struct { + Name string `json:"name"` + LLMDeploymentId string `json:"llm_deployment_id"` + + Description *string `json:"description,omitempty"` + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +type LLMBenchmarkUpdateInput struct { + apis.VirtualResourceBaseUpdateInput + + Model *string `json:"model,omitempty"` + RequestRate *int `json:"request_rate,omitempty"` + TotalRequests *int `json:"total_requests,omitempty"` + MaxDurationSeconds *int `json:"max_duration_seconds,omitempty"` + MaxErrors *int `json:"max_errors,omitempty"` + DatasetInputTokens *int `json:"dataset_input_tokens,omitempty"` + DatasetOutputTokens *int `json:"dataset_output_tokens,omitempty"` +} + +type LLMBenchmarkRetestInput struct{} + +type LLMBenchmarkListInput struct { + apis.VirtualResourceListInput + + LLMId string `json:"llm_id"` + LLMDeploymentId string `json:"llm_deployment_id"` + State string `json:"state"` +} + +type LLMBenchmarkDatasetPreflight struct { + State string `json:"state"` + ExpectedSamples int `json:"expected_samples"` + ActualSamples int `json:"actual_samples"` + Successful int `json:"successful"` + Errored int `json:"errored"` + ErrorRate float64 `json:"error_rate"` + LatencyMeanSeconds float64 `json:"latency_mean_sec"` + Message string `json:"message"` +} + +func (p *LLMBenchmarkDatasetPreflight) String() string { + return jsonutils.Marshal(p).String() +} + +func (p *LLMBenchmarkDatasetPreflight) IsZero() bool { + return p == nil || *p == (LLMBenchmarkDatasetPreflight{}) +} + +type LLMBenchmarkDatasetEvaluation struct { + State string `json:"state"` + AnswerColumn string `json:"answer_column,omitempty"` + RequestTotal int `json:"request_total"` + Evaluated int `json:"evaluated"` + Correct int `json:"correct"` + Incorrect int `json:"incorrect"` + Unscored int `json:"unscored"` + Accuracy float64 `json:"accuracy"` + Message string `json:"message,omitempty"` +} + +func (e *LLMBenchmarkDatasetEvaluation) String() string { + return jsonutils.Marshal(e).String() +} + +func (e *LLMBenchmarkDatasetEvaluation) IsZero() bool { + return e == nil || *e == (LLMBenchmarkDatasetEvaluation{}) +} + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetPreflight)), func() gotypes.ISerializable { + return new(LLMBenchmarkDatasetPreflight) + }) + gotypes.RegisterSerializable(reflect.TypeOf(new(LLMBenchmarkDatasetEvaluation)), func() gotypes.ISerializable { + return new(LLMBenchmarkDatasetEvaluation) + }) +} + +type LLMBenchmarkDetails struct { + apis.VirtualResourceDetails + + LLMDeployment string `json:"llm_deployment"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark_package.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark_package.go new file mode 100644 index 000000000..c007f887d --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_benchmark_package.go @@ -0,0 +1,42 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + LLMBenchmarkPackageSourceHuggingFace = "huggingface" + LLMBenchmarkPackageFormatGuideLLMJSONL = "guidellm_jsonl" + + LLMBenchmarkPackageMountBase = "/data/benchmark-packages" +) + +type LLMBenchmarkPackageCreateInput struct { + apis.SharableVirtualResourceCreateInput + + Source string `json:"source,omitempty"` + RepoId string `json:"repo_id,omitempty"` + Revision string `json:"revision,omitempty"` + FilePath string `json:"file_path,omitempty"` + Format string `json:"format,omitempty"` + AnswerColumn string `json:"answer_column,omitempty"` + + ImageId string `json:"image_id,omitempty"` + Size int64 `json:"size,omitempty"` + ActualSizeMb int32 `json:"actual_size_mb,omitempty"` + + MountPath string `json:"mount_path,omitempty"` + DatasetPath string `json:"dataset_path,omitempty"` + Manifest string `json:"manifest,omitempty"` +} + +type LLMBenchmarkPackageImportInput struct { + LLMBenchmarkPackageCreateInput + BenchmarkSpec *LLMBenchmarkCreateInput `json:"benchmark_spec,omitempty"` +} + +type LLMBenchmarkPackageListInput struct { + apis.SharableVirtualResourceListInput + + Source string `json:"source"` + RepoId string `json:"repo_id"` + Format string `json:"format"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_const.go index df93dd41a..16647edef 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_const.go @@ -4,6 +4,8 @@ const ( STATUS_READY = "ready" ) +const LLM_DEFAULT_CONTEXT_TOKENS = 8192 + const ( /* 未知 */ LLM_STATUS_UNKNOWN = "unknown" diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_container.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_container.go index b8937b2f4..cba23d2eb 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_container.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_container.go @@ -16,7 +16,6 @@ const ( LLM_CONTAINER_COMFYUI LLMContainerType = "comfyui" LLM_CONTAINER_OPENCLAW LLMContainerType = "openclaw" LLM_CONTAINER_HERMES_AGENT LLMContainerType = "hermes-agent" - LLM_CONTAINER_LLM_ROUTER LLMContainerType = "llm-router" LLM_CONTAINER_DESKTOP LLMContainerType = "desktop" ) @@ -29,7 +28,6 @@ var ( string(LLM_CONTAINER_COMFYUI), string(LLM_CONTAINER_OPENCLAW), string(LLM_CONTAINER_HERMES_AGENT), - string(LLM_CONTAINER_LLM_ROUTER), string(LLM_CONTAINER_DESKTOP), ) LLM_INSTANT_MODEL_TYPES = sets.NewString( @@ -39,6 +37,11 @@ var ( string(LLM_CONTAINER_COMFYUI), string(LLM_CONTAINER_OPENCLAW), ) + LLM_INFERENCE_TYPES = sets.NewString( + string(LLM_CONTAINER_OLLAMA), + string(LLM_CONTAINER_VLLM), + string(LLM_CONTAINER_SGLANG), + ) ) func IsLLMContainerType(t string) bool { @@ -49,6 +52,10 @@ func IsLLMInstantModelType(t string) bool { return LLM_INSTANT_MODEL_TYPES.Has(t) } +func IsLLMInferenceType(t string) bool { + return LLM_INFERENCE_TYPES.Has(t) +} + func GetLLMInstantModelContainerType(t LLMContainerType) LLMContainerType { return t } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_deployment.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_deployment.go index 63b893514..ab8f22164 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_deployment.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_deployment.go @@ -87,6 +87,8 @@ const ( LLM_DEPLOYMENT_STATUS_PARTIAL = "partial" // Replica reconcile or syncstatus in progress. LLM_DEPLOYMENT_STATUS_SYNCING = "syncing" + // Restart of all replicas in progress. + LLM_DEPLOYMENT_STATUS_RESTARTING = "restarting" ) // AiproxySyncStatus values stored on SLLMDeployment.AiproxySyncStatus. @@ -183,6 +185,7 @@ type LLMDeploymentUpdateInput struct { } type LLMDeploymentRestartInput struct { + Force bool `json:"force"` } type LLMDeploymentSyncstatusInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router.go deleted file mode 100644 index 6920ae10a..000000000 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router.go +++ /dev/null @@ -1,12 +0,0 @@ -package llm - -const ( - LLM_ROUTER = "llm-router" - LLM_ROUTER_EXEC_PATH = "llm-router-runtime" - LLM_ROUTER_DEFAULT_RUNTIME = "llmrouter-lib" - LLM_ROUTER_DEFAULT_MODEL_DIR = "/models" - LLM_ROUTER_DEFAULT_ROUTE_PATH = "/v1/route" - LLM_ROUTER_DEFAULT_HEALTH_PATH = "/health" - LLM_ROUTER_DEFAULT_PORT = 8000 - LLM_ROUTER_CACHE_DIR = "/root/.cache" -) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router_agent.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router_agent.go new file mode 100644 index 000000000..b42da7495 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_router_agent.go @@ -0,0 +1,98 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + LLM_ROUTER_ROUTE_SIMPLE = "simple" + LLM_ROUTER_ROUTE_COMPLEX = "complex" + + LLM_ROUTER_DEFAULT_ROUTE = LLM_ROUTER_ROUTE_COMPLEX + LLM_ROUTER_DEFAULT_MAX_PROMPT_CHARS = LLM_DEFAULT_CONTEXT_TOKENS * 3 / 4 + LLM_ROUTER_DEFAULT_MAX_DECISION_TOKENS = 64 + LLM_ROUTER_DEFAULT_SIMPLE_DEFINITION = "适合翻译、摘要、短问答和格式转换。" + LLM_ROUTER_DEFAULT_COMPLEX_DEFINITION = "适合代码、数学、长上下文、复杂推理和日志分析。" +) + +type LLMRouterAgentListInput struct { + apis.SharableVirtualResourceListInput + + LLMDriver string `json:"llm_driver"` +} + +type LLMRouterAgentCreateInput struct { + apis.SharableVirtualResourceCreateInput + + LLMId string `json:"llm_id" help:"LLM instance ID; if set, llm_url is resolved from it"` + LLMUrl string `json:"llm_url" help:"decision model OpenAI-compatible base URL"` + LLMDriver string `json:"llm_driver" help:"decision model driver, default openai"` + Model string `json:"model" help:"decision model name"` + ApiKey string `json:"api_key" help:"decision model API key"` + DefaultRoute string `json:"default_route" help:"fallback route, default complex"` + MaxPromptChars int `json:"max_prompt_chars" help:"request char limit; 0 auto-detects /v1/models context, longer requests use complex"` + MaxDecisionTokens int `json:"max_decision_tokens" help:"decision answer token hint"` + CandidateMapping map[string][]string `json:"candidate_mapping" help:"route to candidate model names"` + SimpleDefinition string `json:"simple_definition" help:"definition of simple requests"` + ComplexDefinition string `json:"complex_definition" help:"definition of complex requests"` + SimpleExamples []string `json:"simple_examples" help:"examples routed to simple"` + ComplexExamples []string `json:"complex_examples" help:"examples routed to complex"` +} + +type LLMRouterAgentUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + + LLMId *string `json:"llm_id,omitempty" help:"LLM instance ID; if set, llm_url is resolved from it"` + LLMUrl *string `json:"llm_url,omitempty" help:"decision model OpenAI-compatible base URL"` + LLMDriver *string `json:"llm_driver,omitempty" help:"decision model driver"` + Model *string `json:"model,omitempty" help:"decision model name"` + ApiKey *string `json:"api_key,omitempty" help:"decision model API key"` + DefaultRoute *string `json:"default_route,omitempty" help:"fallback route"` + MaxPromptChars *int `json:"max_prompt_chars,omitempty" help:"request char limit; 0 recomputes from /v1/models context"` + MaxDecisionTokens *int `json:"max_decision_tokens,omitempty" help:"decision answer token hint"` + CandidateMapping *map[string][]string `json:"candidate_mapping,omitempty" help:"route to candidate model names"` + SimpleDefinition *string `json:"simple_definition,omitempty" help:"definition of simple requests"` + ComplexDefinition *string `json:"complex_definition,omitempty" help:"definition of complex requests"` + SimpleExamples *[]string `json:"simple_examples,omitempty" help:"replace examples routed to simple"` + ComplexExamples *[]string `json:"complex_examples,omitempty" help:"replace examples routed to complex"` +} + +type LLMRouterAgentDetails struct { + apis.SharableVirtualResourceDetails + + LLMId string `json:"llm_id"` + LLMName string `json:"llm_name"` +} + +type LLMRouterPromptConfig struct { + SimpleDefinition string `json:"simple_definition"` + ComplexDefinition string `json:"complex_definition"` + SimpleExamples []string `json:"simple_examples"` + ComplexExamples []string `json:"complex_examples"` +} + +type LLMRouterExpandExamplesInput struct { + SimpleDefinition *string `json:"simple_definition,omitempty"` + ComplexDefinition *string `json:"complex_definition,omitempty"` + SimpleExamples *[]string `json:"simple_examples,omitempty"` + ComplexExamples *[]string `json:"complex_examples,omitempty"` +} + +type LLMRouterMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` +} + +type LLMRouterRouteRequest struct { + Model string `json:"model"` + Messages []LLMRouterMessage `json:"messages"` + Candidates []string `json:"candidates"` +} + +type LLMRouterRouteResponse struct { + Model string `json:"model"` +} + +type LLMRouterDecision struct { + Route string `json:"route"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason,omitempty"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_spec.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_spec.go index 0d2224e9a..f22d88038 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_spec.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/llm_spec.go @@ -30,7 +30,6 @@ type LLMSpec struct { ComfyUI *LLMSpecComfyUI `json:"comfyui,omitempty"` OpenClaw *LLMSpecOpenClaw `json:"openclaw,omitempty"` HermesAgent *LLMSpecHermesAgent `json:"hermes_agent,omitempty"` - LLMRouter *LLMSpecLLMRouter `json:"llm_router,omitempty"` } func (s *LLMSpec) String() string { @@ -41,7 +40,7 @@ func (s *LLMSpec) IsZero() bool { if s == nil { return true } - return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil && s.LLMRouter == nil + return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil } // LLMSpecOllama holds type-specific fields for ollama SKUs. @@ -196,43 +195,6 @@ func (s *LLMSpecHermesAgent) IsZero() bool { return s.LLMId == "" && s.LLMUrl == "" && s.Model == "" && s.ApiKey == "" && s.ContextLength == 0 } -type LLMRouterEnv struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type LLMRouterArg struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type LLMSpecLLMRouter struct { - Runtime string `json:"runtime"` - RouterMethod string `json:"router_method"` - ConfigPath string `json:"config_path,omitempty"` - ModelDir string `json:"model_dir,omitempty"` - RoutePath string `json:"route_path,omitempty"` - HealthPath string `json:"health_path,omitempty"` - CandidateMappingPath string `json:"candidate_mapping_path,omitempty"` - CustomizedEnvs []*LLMRouterEnv `json:"customized_envs,omitempty"` - CustomizedArgs []*LLMRouterArg `json:"customized_args,omitempty"` - Extra map[string]interface{} `json:"extra,omitempty"` -} - -func (s *LLMSpecLLMRouter) String() string { - return jsonutils.Marshal(s).String() -} - -func (s *LLMSpecLLMRouter) IsZero() bool { - if s == nil { - return true - } - return s.Runtime == "" && s.RouterMethod == "" && s.ConfigPath == "" && - s.ModelDir == "" && s.RoutePath == "" && s.HealthPath == "" && - s.CandidateMappingPath == "" && len(s.CustomizedEnvs) == 0 && - len(s.CustomizedArgs) == 0 && len(s.Extra) == 0 -} - func init() { gotypes.RegisterSerializable(reflect.TypeOf(new(LLMSpec)), func() gotypes.ISerializable { return new(LLMSpec) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/mcp_agent.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/mcp_agent.go index 9280a65da..0af6c782c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/mcp_agent.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/mcp_agent.go @@ -13,15 +13,16 @@ const ( LLM_CLIENT_OLLAMA LLMClientType = "ollama" LLM_CLIENT_OPENAI LLMClientType = "openai" - MCP_AGENT_SYSTEM_PROMPT = `你是一个 Cloudpods 云平台管理助手。你可以使用提供的工具来帮助用户管理云资源。 + MCP_AGENT_SYSTEM_PROMPT = `你是一个 %s 云平台管理助手。你可以使用提供的工具来帮助用户管理云资源。 ## 你的能力 -- 查询云平台资源（虚拟机、镜像、网络、存储、区域等） +- 查询云平台资源（虚拟机、镜像、网络、存储、区域、套餐等） - 管理虚拟机（创建、启动、停止、重启、删除、重置密码） - 获取虚拟机监控信息和实时统计数据 ## 重要规则（必须严格遵守） **如果用户的问题涉及查询、创建、修改或删除云资源，你必须先调用相应的工具，而不是直接回答。** +- 严禁空口编造：在未真正调用工具并拿到返回结果前，禁止声称“已查到区域/镜像/网络”“正在创建成功”等。 - 对于需要查询资源的问题（如"列出虚拟机"、"查询状态"等），必须调用工具获取数据后再回答 - 对于需要操作资源的问题（如"创建"、"启动"、"停止"等），必须调用工具执行操作后再回答 - 只有在以下情况才可以直接回复： @@ -29,9 +30,17 @@ const ( 2. 没有合适的工具可以解决用户的问题 3. 工具调用失败后需要向用户说明错误原因 +## 创建虚拟机标准流程（同一轮对话中连续调用工具，不要只说不做；查询可并行以节省轮次） +1. climc_cloud_region_list（公有云必须 provider=["Aliyun"] 等；创建时 usable=true） +2. climc_cloud_region_capability（ID=区域 id；从 storage_types2 取系统盘 backend） +3. climc_cached_image_list（公有云）或 climc_image_list（KVM）；公有云必须带 provider + region=区域 id；不要重复调用 +4. climc_server_sku_list（公有云带 provider+cloudregion；用户说 2c2g/2核2G 时传 spec="2c2g"） +5. climc_server_create（name、disk 须含 image+backend、instance-type 或 ncpu/mem-spec；公有云 hypervisor=aliyun、prefer-region=区域 id）。net 可省略：未指定时自动 random（nets:[{exit:false}]）调度，默认不要先 network-list/vpc-list +查询工具的返回不等于任务完成；必须最终调用 climc_server_create。创建失败时根据工具错误向用户说明原因。 + ## 工作流程 1. 理解用户的需求 -2. **优先检查是否有合适的工具可以完成任务，如果有则必须调用工具** +2. **优先检查是否有合适的工具可以完成任务，如果有则必须调用工具（发 tool_calls，不要只输出计划文字）** 3. 分析工具返回的结果 4. 如果需要更多信息，继续调用其他工具 5. 最后用自然语言总结结果给用户 @@ -67,33 +76,39 @@ type MCPAgentListInput struct { type MCPAgentCreateInput struct { apis.SharableVirtualResourceCreateInput - LLMId string `json:"llm_id" help:"LLM 实例 ID，如果提供则自动获取 llm_url"` - LLMUrl string `json:"llm_url" help:"后端大模型的 base 请求地址"` - LLMDriver string `json:"llm_driver" help:"使用的大模型驱动，可以是 ollama 或 openai"` - Model string `json:"model" help:"使用的模型名称"` - ApiKey string `json:"api_key" help:"在 llm_driver 中需要用到的认证"` - McpServer string `json:"mcp_server" help:"mcp 服务器的后端地址"` - DefaultAgent *bool `json:"default_agent,omitempty" help:"set as default MCP agent (only one can be true globally)"` + LLMId string `json:"llm_id" help:"deprecated; ignored"` + LLMUrl string `json:"llm_url" help:"AI 网关 OpenAI 兼容 base 请求地址"` + LLMDriver string `json:"llm_driver" help:"使用的大模型驱动，固定为 openai"` + Model string `json:"model" help:"使用的模型名称"` + ApiKey string `json:"api_key" help:"deprecated; ignored"` + McpServer string `json:"mcp_server" help:"mcp 服务器的后端地址"` + AiProxyRoutingId string `json:"aiproxy_routing_id" help:"关联的 AI 网关路由规则 ID"` + AiproxyVirtualKeyId string `json:"aiproxy_virtual_key_id" help:"关联的 AI 网关 API Key ID"` + DefaultAgent *bool `json:"default_agent,omitempty" help:"set as default MCP agent (only one can be true globally)"` } type MCPAgentUpdateInput struct { apis.SharableVirtualResourceCreateInput - LLMId *string `json:"llm_id,omitempty" help:"LLM 实例 ID，如果提供则自动获取 llm_url"` - LLMUrl *string `json:"llm_url,omitempty" help:"后端大模型的 base 请求地址"` - LLMDriver *string `json:"llm_driver,omitempty" help:"使用的大模型驱动，可以是 ollama 或 openai"` - Model *string `json:"model,omitempty" help:"使用的模型名称"` - ApiKey *string `json:"api_key,omitempty" help:"在 llm_driver 中需要用到的认证"` - McpServer *string `json:"mcp_server,omitempty" help:"mcp 服务器的后端地址"` - DefaultAgent *bool `json:"default_agent,omitempty" help:"set as default MCP agent (only one can be true globally)"` + LLMId *string `json:"llm_id,omitempty" help:"deprecated; ignored"` + LLMUrl *string `json:"llm_url,omitempty" help:"AI 网关 OpenAI 兼容 base 请求地址"` + LLMDriver *string `json:"llm_driver,omitempty" help:"使用的大模型驱动，固定为 openai"` + Model *string `json:"model,omitempty" help:"使用的模型名称"` + ApiKey *string `json:"api_key,omitempty" help:"deprecated; ignored"` + McpServer *string `json:"mcp_server,omitempty" help:"mcp 服务器的后端地址"` + AiProxyRoutingId *string `json:"aiproxy_routing_id,omitempty" help:"关联的 AI 网关路由规则 ID"` + AiproxyVirtualKeyId *string `json:"aiproxy_virtual_key_id,omitempty" help:"关联的 AI 网关 API Key ID"` + DefaultAgent *bool `json:"default_agent,omitempty" help:"set as default MCP agent (only one can be true globally)"` } type MCPAgentDetails struct { apis.SharableVirtualResourceDetails - LLMId string `json:"llm_id"` - LLMName string `json:"llm_name"` - DefaultAgent bool `json:"default_agent"` + LLMId string `json:"llm_id"` + LLMName string `json:"llm_name"` + AiProxyRoutingId string `json:"aiproxy_routing_id"` + AiproxyVirtualKeyId string `json:"aiproxy_virtual_key_id"` + DefaultAgent bool `json:"default_agent"` } type LLMToolRequestInput struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/sglang_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/sglang_const.go index 777838f95..a6213ae37 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/sglang_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/sglang_const.go @@ -8,5 +8,5 @@ const ( LLM_SGLANG_HF_ENDPOINT = LLM_VLLM_HF_ENDPOINT LLM_SGLANG_CACHE_DIR = "/root/.cache/huggingface" LLM_SGLANG_BASE_PATH = "/data/models" - LLM_SGLANG_MODELS_PATH = "/data/models/huggingface" + LLM_SGLANG_MODELS_PATH = LLM_SGLANG_BASE_PATH ) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/sku.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/sku.go index e5acd49d4..da717b294 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/sku.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/sku.go @@ -78,9 +78,15 @@ func (s PortMappings) IsZero() bool { } type Device struct { - DevType string `json:"dev_type"` - Model string `json:"model"` - DevicePath string `json:"device_path"` + DevType string `json:"dev_type"` + SharingMode string `json:"sharing_mode,omitempty"` + Vendor string `json:"vendor,omitempty"` + Model string `json:"model"` + DevicePath string `json:"device_path"` + // MemoryMb is optional per-device VRAM (MiB) for HAMI. When > 0 it is used + // as MemoryMb/MemoryRequest on pod create; otherwise claim is split evenly. + MemoryMb int `json:"memory_mb,omitempty"` + SmUtilLimit int `json:"sm_util_limit,omitempty"` } type Devices []Device @@ -139,6 +145,10 @@ type LLMSkuDetails struct { // Inference backend version and parameters BackendVersion string `json:"backend_version"` BackendParameters []string `json:"backend_parameters,omitempty"` + + // VramClaimMb is computed from mounted InstantModel weight_size_bytes + // (EstimateClaimMb). Not persisted on the SKU row. + VramClaimMb int `json:"vram_claim_mb"` } type MountedAppResourceDetails struct { @@ -151,10 +161,11 @@ type LLMSKuBaseCreateInput struct { Cpu int `json:"cpu"` Memory int `json:"memory"` Bandwidth int `json:"bandwidth"` - // VramClaimMb is the estimated VRAM (MiB) the inference instance will - // require. Optional — if 0, the deployment create task will auto-fill it - // from the mounted InstantModel's weight_size_bytes. - VramClaimMb int `json:"vram_claim_mb,omitempty"` + + // EnableCgroupCpu enables CPU CFS quota on the pod/container. Inference SKUs default to false. + EnableCgroupCpu *bool `json:"enable_cgroup_cpu"` + // EnableCgroupMemory enables memory hard limit on the pod/container. Inference SKUs default to false. + EnableCgroupMemory *bool `json:"enable_cgroup_memory"` Volumes *Volumes `json:"volumes"` HostPaths *HostPaths `json:"host_paths"` @@ -170,6 +181,9 @@ type LLMSkuBaseUpdateInput struct { Cpu *int `json:"cpu"` Memory *int `json:"memory"` + EnableCgroupCpu *bool `json:"enable_cgroup_cpu"` + EnableCgroupMemory *bool `json:"enable_cgroup_memory"` + // RequstSyncImage *bool `json:"request_sync_image"` DiskSize *int `json:"disk_size" yunion-deprecated-by:"disk_size_mb"` @@ -245,6 +259,9 @@ type LLMSkuUpdateInput struct { ModelScopeModelId *string `json:"model_scope_model_id,omitempty"` ModelScopeFilePath *string `json:"model_scope_file_path,omitempty"` LocalPath *string `json:"local_path,omitempty"` + // PreferHosts updates SKU prefer_hosts. Omitted means unchanged; an explicit + // value is required for local_path SKUs and rejected otherwise. + PreferHosts []string `json:"prefer_hosts"` // Model categories Categories *[]string `json:"categories,omitempty"` // Inference backend version and parameters @@ -252,9 +269,13 @@ type LLMSkuUpdateInput struct { BackendParameters *[]string `json:"backend_parameters,omitempty"` } -// type LLMModelCloneInput struct { -// Name string `json:"name"` -// } +// LLMSkuCloneInput is the body for POST /llm_skus/{id}/clone. +// Specs are copied from the source SKU; the caller supplies a new name. +type LLMSkuCloneInput struct { + Name string `json:"name"` + GenerateName string `json:"generate_name"` + Description string `json:"description"` +} // type LLMModelSyncImageRequestTaskInput struct { // Request bool `json:"request"` @@ -292,3 +313,31 @@ type LLMSkuUpdateInput struct { // DifySSRFImageId string `json:"dify_ssrf_image_id"` // DifyWeaviateImageId string `json:"dify_weaviate_image_id"` // } + +// LLMSchedulableCheckInput is the body for +// POST /llm_skus/{id}/schedulable-check. Specs come from the SKU; the body is empty. +type LLMSchedulableCheckInput struct{} + +// LLMSchedulableHostInfo describes GPU availability on one candidate host. +type LLMSchedulableHostInfo struct { + HostId string `json:"host_id"` + HostName string `json:"host_name"` + GpuAvailable int `json:"gpu_available"` + BestGpuVramMb int `json:"best_gpu_vram_mb"` + BestGpuModel string `json:"best_gpu_model,omitempty"` +} + +// LLMSchedulableCheckOutput mirrors GPUStack's ModelEvaluationResult: +// a yes/no verdict plus per-host detail so the caller can surface a +// meaningful message ("not enough VRAM on any host", "host X qualifies", …). +type LLMSchedulableCheckOutput struct { + Schedulable bool `json:"schedulable"` + VramClaimMb int `json:"vram_claim_mb"` + PerDevMinMb int `json:"per_dev_min_mb"` + GpuCount int `json:"gpu_count"` + Reason string `json:"reason,omitempty"` + FilteredCandidates jsonutils.JSONObject `json:"filtered_candidates,omitempty"` + Hosts []LLMSchedulableHostInfo `json:"hosts,omitempty"` + TotalGpuHosts int `json:"total_gpu_hosts"` + QualifiedHosts int `json:"qualified_hosts"` +} diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/llm/vllm_const.go b/vendor/yunion.io/x/onecloud/pkg/apis/llm/vllm_const.go index 74dae9672..baa8c64ac 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/llm/vllm_const.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/llm/vllm_const.go @@ -8,5 +8,5 @@ const ( LLM_VLLM_HF_ENDPOINT = "https://hf-mirror.com" LLM_VLLM_CACHE_DIR = "/root/.cache/huggingface" LLM_VLLM_BASE_PATH = "/data/models" - LLM_VLLM_MODELS_PATH = "/data/models/huggingface" + LLM_VLLM_MODELS_PATH = LLM_VLLM_BASE_PATH ) diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/monitor/commalert.go b/vendor/yunion.io/x/onecloud/pkg/apis/monitor/commalert.go index b60b27fae..6b734804b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/monitor/commalert.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/monitor/commalert.go @@ -196,15 +196,17 @@ type CommonAlertMetricDetails struct { ConditionType string `json:"condition_type"` ThresholdStr string `json:"threshold_str"` // metric points'value的运算方式 - Reduce string `json:"reduce"` - DB string `json:"db"` - Measurement string `json:"measurement"` - MeasurementDisplayName string `json:"measurement_display_name"` - ResType string `json:"res_type"` - Field string `json:"field"` - Groupby string `json:"groupby"` - Filters []MetricQueryTag `json:"filters"` - FieldDescription MetricFieldDetail - FieldOpt string `json:"field_opt"` - GetPointStr bool `json:"get_point_str"` + Reduce string `json:"reduce"` + DB string `json:"db"` + Measurement string `json:"measurement"` + MeasurementDisplayName string `json:"measurement_display_name"` + ResType string `json:"res_type"` + // ResIdKey 为该资源类型在 TSDB 中的资源 ID tag，如 host_id / vm_id + ResIdKey string `json:"res_id_key"` + Field string `json:"field"` + Groupby string `json:"groupby"` + Filters []MetricQueryTag `json:"filters"` + FieldDescription MetricFieldDetail + FieldOpt string `json:"field_opt"` + GetPointStr bool `json:"get_point_str"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/apis/scheduler/api.go b/vendor/yunion.io/x/onecloud/pkg/apis/scheduler/api.go index 0f6f9ce36..2598fbc85 100644 --- a/vendor/yunion.io/x/onecloud/pkg/apis/scheduler/api.go +++ b/vendor/yunion.io/x/onecloud/pkg/apis/scheduler/api.go @@ -89,6 +89,7 @@ type ScheduleInput struct { CpuMode string `json:"cpu_mode"` OsArch string `json:"os_arch"` ResetCpuNumaPin bool `json:"reset_cpu_numa_pin"` + QemuVersion string `json:"qemu_version"` ExtraCpuCount int `json:"extra_cpu_count"` CpuNumaPin []SCpuNumaPin `json:"cpu_numa_pin"` diff --git a/vendor/yunion.io/x/onecloud/pkg/appsrv/cors.go b/vendor/yunion.io/x/onecloud/pkg/appsrv/cors.go index 3d13b94f6..4720e9cbc 100644 --- a/vendor/yunion.io/x/onecloud/pkg/appsrv/cors.go +++ b/vendor/yunion.io/x/onecloud/pkg/appsrv/cors.go @@ -138,6 +138,9 @@ type CorsOptions struct { ExposedHeaders []string // AllowCredentials indicates whether the request can include user credentials like // cookies, HTTP authentication or client side SSL certificates. + // Credentials are only honored for explicitly listed origins: when AllowedOrigins + // is empty or contains "*", echoing any origin with credentials would allow any + // website to make credentialed requests. AllowCredentials bool // MaxAge indicates how long (in seconds) the results of a preflight request // can be cached @@ -215,6 +218,13 @@ func NewCors(options CorsOptions) *Cors { } } + // Credentials with a wildcard/unset origin allowlist would let any + // website send authenticated cross-origin requests, so credentials are + // only honored for explicitly listed origins + if c.allowedOriginsAll { + c.allowCredentials = false + } + // Allowed Headers if len(options.AllowedHeaders) == 0 { // Use sensible defaults diff --git a/vendor/yunion.io/x/onecloud/pkg/appsrv/response.go b/vendor/yunion.io/x/onecloud/pkg/appsrv/response.go index fa44d0a94..7f4f5202a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/appsrv/response.go +++ b/vendor/yunion.io/x/onecloud/pkg/appsrv/response.go @@ -18,8 +18,10 @@ import ( "bufio" "context" "fmt" + "mime" "net" "net/http" + "strings" "yunion.io/x/onecloud/pkg/httperrors" ) @@ -36,6 +38,8 @@ type responseWriterChannel struct { bodyResp chan responseWriterResponse statusChan chan int statusResp chan bool + flushChan chan struct{} + flushResp chan struct{} isClosed bool } @@ -47,6 +51,8 @@ func newResponseWriterChannel(backend http.ResponseWriter) responseWriterChannel bodyResp: make(chan responseWriterResponse), statusChan: make(chan int), statusResp: make(chan bool), + flushChan: make(chan struct{}), + flushResp: make(chan struct{}), isClosed: false, } } @@ -76,16 +82,34 @@ func (w *responseWriterChannel) WriteHeader(status int) { <-w.statusResp } -// implent http.Flusher +// Flush implements http.Flusher. Flush must run on the same goroutine as Write +// (wait loop); otherwise SSE via ReverseProxy never reaches the client. func (w *responseWriterChannel) Flush() { if w.isClosed { return } + w.flushChan <- struct{}{} + <-w.flushResp +} + +func (w *responseWriterChannel) flushBackend() { if f, ok := w.backend.(http.Flusher); ok { f.Flush() } } +func (w *responseWriterChannel) isEventStream() bool { + ct := w.backend.Header().Get("Content-Type") + if ct == "" { + return false + } + baseCT, _, err := mime.ParseMediaType(ct) + if err != nil { + return strings.HasPrefix(ct, "text/event-stream") + } + return baseCT == "text/event-stream" +} + // Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter // and a Hijacker. func (w *responseWriterChannel) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { @@ -121,6 +145,11 @@ func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWork // log.Infof("Recive body: %s, more: %v", len(bytes), more) if more { c, e := w.backend.Write(bytes) + // SSE / streaming: flush immediately after each chunk so clients + // see events without waiting for handler completion. + if e == nil && w.isEventStream() { + w.flushBackend() + } w.bodyResp <- responseWriterResponse{count: c, err: e} } else { stop = true @@ -133,6 +162,13 @@ func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWork } else { stop = true } + case _, more := <-w.flushChan: + if more { + w.flushBackend() + w.flushResp <- struct{}{} + } else { + stop = true + } } } return err @@ -148,4 +184,6 @@ func (w *responseWriterChannel) closeChannels() { close(w.bodyResp) close(w.statusChan) close(w.statusResp) + close(w.flushChan) + close(w.flushResp) } diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/helper.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/helper.go index 612e7b603..2ebcaad7e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/helper.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/helper.go @@ -476,6 +476,9 @@ func FetchDiskCreateInputByJSON(data jsonutils.JSONObject) (*compute.DiskCreateI } if data.Contains("disk") { desc, err := data.GetString("disk") + if err != nil { + return nil, err + } config, err = ParseDiskConfig(desc, -1) if err != nil { return nil, err diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/parser.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/parser.go index 7bc27ef6f..c5cd585e6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/parser.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/cmdline/parser.go @@ -513,8 +513,7 @@ func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, e } dev := new(compute.IsolatedDeviceConfig) parts := strings.Split(desc, ":") - devTypes := sets.NewString(compute.VALID_PASSTHROUGH_TYPES...) - devTypes.Insert(compute.VALID_CONTAINER_DEVICE_TYPES...) + devTypes := sets.NewString(compute.VALID_TYPES...) for _, p := range parts { if regutils.MatchUUIDExact(p) { dev.Id = p diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/domain.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/domain.go index c0eb90513..32a66ef26 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/domain.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/domain.go @@ -57,24 +57,26 @@ func (manager *SDomainizedResourceBaseManager) ResourceScope() rbacscope.TRbacSc func (manager *SDomainizedResourceBaseManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man FilterByOwnerProvider, userCred mcclient.TokenCredential, owner mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { if owner != nil { switch scope { - case rbacscope.ScopeProject, rbacscope.ScopeDomain: - q = q.Equals("domain_id", owner.GetProjectDomainId()) + case rbacscope.ScopeSystem: if userCred != nil { result := policy.PolicyManager.Allow(scope, userCred, consts.GetServiceType(), man.KeywordPlural(), policy.PolicyActionList) + if !result.DomainTags.IsEmpty() { + policyFilters := tagutils.STagFilters{} + policyFilters.AddFilters(result.DomainTags) + q = ObjectIdQueryWithTagFilters(ctx, q, "domain_id", "domain", policyFilters) + } if !result.ObjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ObjectTags) q = ObjectIdQueryWithTagFilters(ctx, q, "id", man.Keyword(), policyTagFilters) } } - case rbacscope.ScopeSystem: + default: + // domain view, also used when the requested scope has no + // dedicated case + q = q.Equals("domain_id", owner.GetProjectDomainId()) if userCred != nil { result := policy.PolicyManager.Allow(scope, userCred, consts.GetServiceType(), man.KeywordPlural(), policy.PolicyActionList) - if !result.DomainTags.IsEmpty() { - policyFilters := tagutils.STagFilters{} - policyFilters.AddFilters(result.DomainTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "domain_id", "domain", policyFilters) - } if !result.ObjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ObjectTags) diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/fetch.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/fetch.go index a7b59fc73..1c2b20933 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/fetch.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/fetch.go @@ -437,25 +437,9 @@ func FetchCheckQueryOwnerScope( } else { ownerId = userCred reqScopeStr, _ := data.GetString("scope") - if len(reqScopeStr) > 0 { - if reqScopeStr == "max" || reqScopeStr == "maxallowed" { - queryScope = allowScope - } else { - queryScope = rbacscope.String2Scope(reqScopeStr) - } - } else if data.Contains("admin") { - isAdmin := jsonutils.QueryBoolean(data, "admin", false) - if isAdmin && allowScope.HigherThan(rbacscope.ScopeProject) { - queryScope = allowScope - } - } else if action == policy.PolicyActionGet { - queryScope = allowScope - } else { - queryScope = resScope - } - // if resScope.HigherThan(queryScope) { - // queryScope = resScope - // } + hasAdmin := data.Contains("admin") + isAdmin := hasAdmin && jsonutils.QueryBoolean(data, "admin", false) + queryScope = resolveQueryScope(reqScopeStr, hasAdmin, isAdmin, allowScope, resScope, action) requireScope = queryScope } if doCheckRbac && (requireScope.HigherThan(allowScope) || policyTagFilters.Result.IsDeny()) { @@ -466,6 +450,49 @@ func FetchCheckQueryOwnerScope( return ownerId, queryScope, nil, policyTagFilters } +// resolveQueryScope computes the scope at which a list/get query is filtered +// when the request carries no explicit owner (no project_id/domain_id/user_id +// filter). +// +// Semantics: +// - an explicit scope=system/domain/project selects that view; the RBAC +// caller check (requireScope) enforces the caller is allowed that scope. +// Querying a resource below its natural scope (e.g. project scope on a +// domain-scoped resource, resolved by custom FilterByOwner overrides) is +// a supported feature, so no blanket clamping to resScope is done here. +// - scope=max/maxallowed selects the widest scope the caller is allowed. +// - admin=true selects the widest allowed scope for callers above project +// scope; project-level callers fall back to the default resource view. +// - an unset (empty) or none scope, and a user scope on a non user-scoped +// resource, fall back to the natural scope of the resource. +func resolveQueryScope(reqScopeStr string, hasAdmin, isAdmin bool, allowScope, resScope rbacscope.TRbacScope, action string) rbacscope.TRbacScope { + var queryScope rbacscope.TRbacScope + if len(reqScopeStr) > 0 { + if reqScopeStr == "max" || reqScopeStr == "maxallowed" { + queryScope = allowScope + } else { + queryScope = rbacscope.String2Scope(reqScopeStr) + } + } else if hasAdmin { + if isAdmin && allowScope.HigherThan(rbacscope.ScopeProject) { + queryScope = allowScope + } else { + // a project-level caller asking for the admin view (or sending + // admin=false) falls back to the default view of the resource + queryScope = resScope + } + } else if action == policy.PolicyActionGet { + queryScope = allowScope + } else { + queryScope = resScope + } + if queryScope == "" || queryScope == rbacscope.ScopeNone || + (queryScope == rbacscope.ScopeUser && resScope != rbacscope.ScopeUser) { + queryScope = resScope + } + return queryScope +} + func mapKeys(idMap map[string]string) []string { keys := make([]string, len(idMap)) idx := 0 diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/metadata.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/metadata.go index 323c6c3c0..d5fdd9012 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/metadata.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/metadata.go @@ -426,6 +426,8 @@ func (manager *SMetadataManager) ListItemFilter(ctx context.Context, q *sqlchemy } if len(conditions) > 0 { q = q.Filter(sqlchemy.OR(conditions...)) + } else { + q = q.Equals("obj_id", "") } } diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/modelbase.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/modelbase.go index e3f4b970a..ca4c88e23 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/modelbase.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/modelbase.go @@ -413,16 +413,17 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context, if !ok { im = manager } - fn, err := query.GetArray("field") + fn, _ := query.GetArray("field") efs, _ := query.GetArray("extra_field") fields := make([]string, len(fn)) // validate field for i, f := range fn { - fields[i], err = f.GetString() + fs, err := f.GetString() if err != nil { return nil, httperrors.NewInputParameterError("can't get string field") } + fields[i] = fs var hasField = false for _, field := range manager.getTable().Fields() { if field.Name() == fields[i] { @@ -436,7 +437,7 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context, } q := im.Query() - q, err = ListItemQueryFilters(im, ctx, q, userCred, query, policy.PolicyActionList) + q, err := ListItemQueryFilters(im, ctx, q, userCred, query, policy.PolicyActionList) if err != nil { return nil, err } diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/opslog.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/opslog.go index bb80be3a3..1f0028e4e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/opslog.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/opslog.go @@ -366,6 +366,9 @@ func (manager *SOpsLogManager) LogSyncUpdate(m IModel, uds sqlchemy.UpdateDiffs, func (self *SOpsLogManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man FilterByOwnerProvider, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { if ownerId != nil { + if scope == "" || scope == rbacscope.ScopeNone { + scope = rbacscope.ScopeProject + } switch scope { case rbacscope.ScopeUser: if len(ownerId.GetUserId()) > 0 { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/project.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/project.go index 7f50f9b7d..050882368 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/project.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/project.go @@ -55,16 +55,6 @@ func (model *SProjectizedResourceBase) GetOwnerId() mcclient.IIdentityProvider { func (manager *SProjectizedResourceBaseManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man FilterByOwnerProvider, userCred mcclient.TokenCredential, owner mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { if owner != nil { switch scope { - case rbacscope.ScopeProject: - q = q.Equals("tenant_id", owner.GetProjectId()) - if userCred != nil { - result := policy.PolicyManager.Allow(scope, userCred, consts.GetServiceType(), man.KeywordPlural(), policy.PolicyActionList) - if !result.ObjectTags.IsEmpty() { - policyTagFilters := tagutils.STagFilters{} - policyTagFilters.AddFilters(result.ObjectTags) - q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) - } - } case rbacscope.ScopeDomain: q = q.Equals("domain_id", owner.GetProjectDomainId()) if userCred != nil { @@ -99,6 +89,18 @@ func (manager *SProjectizedResourceBaseManager) FilterByOwner(ctx context.Contex q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) } } + default: + // project view, also used when the requested scope has no + // dedicated case + q = q.Equals("tenant_id", owner.GetProjectId()) + if userCred != nil { + result := policy.PolicyManager.Allow(scope, userCred, consts.GetServiceType(), man.KeywordPlural(), policy.PolicyActionList) + if !result.ObjectTags.IsEmpty() { + policyTagFilters := tagutils.STagFilters{} + policyTagFilters.AddFilters(result.ObjectTags) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) + } + } } } return q diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/quotas/handler.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/quotas/handler.go index bd3b26b2f..1d030edcd 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/quotas/handler.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/quotas/handler.go @@ -580,6 +580,9 @@ func (manager *SQuotaBaseManager) listQuotas(ctx context.Context, userCred mccli keyList = append(keyList, quota.GetKeys()) } idNameMap, err := manager.keyList2IdNameMap(ctx, keyList) + if err != nil { + return nil, errors.Wrap(err, "keyList2IdNameMap") + } var fields []string for i := range quotaList { keys := quotaList[i].GetKeys() diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/rolecache.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/rolecache.go index 5a723aa38..4c21f548d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/rolecache.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/rolecache.go @@ -172,6 +172,9 @@ func (manager *SRoleCacheManager) Save(ctx context.Context, idStr string, name s } } else { objm, err := NewModelObject(manager) + if err != nil { + return nil, err + } obj := objm.(*SRole) obj.Id = idStr obj.Name = name diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/scoperesource.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/scoperesource.go index 2aa810d89..ba80bafb1 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/scoperesource.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/scoperesource.go @@ -117,7 +117,7 @@ func (m *SScopedResourceBaseManager) FilterByOwner(ctx context.Context, q *sqlch switch scope { case rbacscope.ScopeDomain: q = q.Equals("domain_id", owner.GetProjectDomainId()) - case rbacscope.ScopeProject: + default: q = q.Equals("tenant_id", owner.GetProjectId()) } return q diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/sharablebase.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/sharablebase.go index d38590e54..251699353 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/sharablebase.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/sharablebase.go @@ -217,6 +217,24 @@ func SharableManagerValidateCreateData( func SharableManagerFilterByOwner(ctx context.Context, manager IStandaloneModelManager, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, owner mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { if owner != nil { resScope := manager.ResourceScope() + // map unrecognized scopes to the resource's default view + if scope != rbacscope.ScopeSystem && scope != rbacscope.ScopeDomain && + scope != rbacscope.ScopeProject && + !(scope == rbacscope.ScopeUser && resScope == rbacscope.ScopeUser) { + switch resScope { + case rbacscope.ScopeUser: + scope = rbacscope.ScopeUser + case rbacscope.ScopeDomain: + scope = rbacscope.ScopeDomain + default: + scope = rbacscope.ScopeProject + } + } + // domain-level owner has no project id: use the domain view + if scope == rbacscope.ScopeProject && len(owner.GetProjectId()) == 0 && + len(owner.GetProjectDomainId()) > 0 { + scope = rbacscope.ScopeDomain + } if resScope == rbacscope.ScopeUser { targetProjectId := owner.GetProjectId() if len(targetProjectId) == 0 && userCred != nil { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/standalone_anon.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/standalone_anon.go index 9afa11d25..3f8141940 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/standalone_anon.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/standalone_anon.go @@ -255,6 +255,12 @@ func (model *SStandaloneAnonResourceBase) SetMetadata(ctx context.Context, key s if err != nil { return errors.Wrap(err, "SetValue") } + { + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } + } if isUserMetadata(key) { model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred) } @@ -266,6 +272,12 @@ func (model *SStandaloneAnonResourceBase) SetAllMetadata(ctx context.Context, di if err != nil { return errors.Wrap(err, "SetValuesWithLog") } + { + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } + } if containsUserMetadata(dictstore) { model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred) } @@ -316,7 +328,10 @@ func (model *SStandaloneAnonResourceBase) SetUserMetadataValues(ctx context.Cont return errors.Wrap(err, "SetValuesWithLog") } { - model.GetModelManager().TableSpec().InformUpdate(ctx, model, jsonutils.Marshal(model).(*jsonutils.JSONDict)) + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } } model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred) return nil @@ -333,7 +348,10 @@ func (model *SStandaloneAnonResourceBase) SetUserMetadataAll(ctx context.Context return errors.Wrap(err, "SetAll") } { - model.GetModelManager().TableSpec().InformUpdate(ctx, model, jsonutils.Marshal(model).(*jsonutils.JSONDict)) + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } } model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred) return nil @@ -361,9 +379,23 @@ func (model *SStandaloneAnonResourceBase) SetCloudMetadataAll(ctx context.Contex userTags[strings.Replace(k, CLOUD_TAG_PREFIX, USER_TAG_PREFIX, 1)] = v } if readOnly { - return Metadata.SetAllWithoutDelelte(ctx, model, userTags, userCred) + err := Metadata.SetAllWithoutDelelte(ctx, model, userTags, userCred) + if err != nil { + return errors.Wrap(err, "SetAllWithoutDelelte") + } + } else { + err := Metadata.SetAll(ctx, model, userTags, userCred, USER_TAG_PREFIX) + if err != nil { + return errors.Wrap(err, "SetAll") + } } - return Metadata.SetAll(ctx, model, userTags, userCred, USER_TAG_PREFIX) + { + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } + } + return nil } func (model *SStandaloneAnonResourceBase) SetOrganizationMetadataAll(ctx context.Context, meta map[string]string, userCred mcclient.TokenCredential) error { @@ -377,10 +409,15 @@ func (model *SStandaloneAnonResourceBase) SetOrganizationMetadataAll(ctx context return errors.Wrap(err, "SetAllOrganization") } } - Update(model, func() error { + _, err := Update(model, func() error { model.OrgNodeMd5 = fmt.Sprintf("%x", md5.Sum([]byte(jsonutils.Marshal(meta).String()))) + model.UpdatedAt = time.Now() + model.UpdateVersion += 1 return nil }) + if err != nil { + return errors.Wrap(err, "Update") + } // 避免加入组织架构后，项目所在的层级会移除此项目 //{ @@ -400,6 +437,15 @@ func (model *SStandaloneAnonResourceBase) SetOrganizationMetadataAll(ctx context return nil } +func (model *SStandaloneAnonResourceBase) forceUpdate() error { + _, err := Update(model, func() error { + model.UpdateVersion += 1 + model.UpdatedAt = time.Now() + return nil + }) + return err +} + func (model *SStandaloneAnonResourceBase) SetClassMetadataValues(ctx context.Context, dictstore map[string]string, userCred mcclient.TokenCredential) error { var err error dictStore, err := ensurePrefixString(dictstore, CLASS_TAG_PREFIX) @@ -410,6 +456,12 @@ func (model *SStandaloneAnonResourceBase) SetClassMetadataValues(ctx context.Con if err != nil { return errors.Wrap(err, "SetValuesWithLog") } + { + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } + } return nil } @@ -423,6 +475,12 @@ func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Contex if err != nil { return errors.Wrap(err, "SetAll") } + { + err := model.forceUpdate() + if err != nil { + return errors.Wrap(err, "forceUpdate") + } + } return nil } @@ -671,7 +729,7 @@ func (model *SStandaloneAnonResourceBase) PerformSetUserMetadata(ctx context.Con func (model *SStandaloneAnonResourceBase) PerformClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformClassMetadataInput) (jsonutils.JSONObject, error) { err := model.SetClassMetadataValues(ctx, input, userCred) if err != nil { - return nil, errors.Wrap(err, "SetUserMetadataValues") + return nil, errors.Wrap(err, "SetClassMetadataValues") } return nil, nil } @@ -681,7 +739,7 @@ func (model *SStandaloneAnonResourceBase) PerformClassMetadata(ctx context.Conte func (model *SStandaloneAnonResourceBase) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { err := model.SetClassMetadataAll(ctx, input, userCred) if err != nil { - return nil, errors.Wrap(err, "SetUserMetadataAll") + return nil, errors.Wrap(err, "SetClassMetadataAll") } return nil, nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/taskman/tasks.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/taskman/tasks.go index 4d898ff91..a9d1dc61b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/taskman/tasks.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/taskman/tasks.go @@ -951,7 +951,22 @@ func (task *STask) ClearPendingUsage(index int) error { } func (task *STask) GetParams() *jsonutils.JSONDict { - return task.Params + result := jsonutils.NewDict() + if task.Params == nil { + return result + } + copied := task.Params.DeepCopy() + copyParams, ok := copied.(*jsonutils.JSONDict) + if !ok || copyParams == nil { + return result + } + paramsJsonMap, _ := copyParams.GetMap() + for k, v := range paramsJsonMap { + if !strings.HasPrefix(k, "__") { + result.Set(k, v) + } + } + return result } func (task *STask) GetUserCred() mcclient.TokenCredential { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/tenantcache.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/tenantcache.go index 279665f4a..e1d987fd4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/tenantcache.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/tenantcache.go @@ -391,6 +391,9 @@ func (manager *STenantCacheManager) Save(ctx context.Context, item SCachedTenant } } else { objm, err := NewModelObject(manager) + if err != nil { + return nil, err + } obj := objm.(*STenant) obj.Id = item.Id obj.Name = item.Name diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/usercache.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/usercache.go index d599c3e4d..a2f3d3483 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/usercache.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/db/usercache.go @@ -177,6 +177,9 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s } } else { objm, err := NewModelObject(manager) + if err != nil { + return nil, err + } obj := objm.(*SUser) obj.Id = idStr obj.Name = name diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/options/options.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/options/options.go index 29db74263..e2082fbbe 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/options/options.go @@ -66,7 +66,7 @@ type BaseOptions struct { LogVerboseLevel int `help:"log verbosity level" default:"0"` LogFilePrefix string `help:"prefix of log files"` - CorsHosts []string `help:"List of hostname that allow CORS"` + CorsHosts []string `help:"List of hostname that allow CORS, credentials are only honored for explicitly listed origins"` TempPath string `help:"Path for store temp file, at least 40G space" default:"/opt/yunion/tmp"` ApplicationID string `help:"Application ID"` @@ -175,6 +175,7 @@ type HostCommonOptions struct { ImageDeployDriver string `help:"Image deploy driver" default:"qemu-kvm" choices:"qemu-kvm|nbd|libguestfs"` DeployConcurrent int `help:"qemu-kvm deploy driver concurrent" default:"5"` Qcow2Preallocation string `help:"Qcow2 image create preallocation" default:"metadata" choices:"disable|metadata|falloc|full"` + EnableNestedVirtualization bool `help:"Enable nested virtualization" default:"true"` } type S3CommonOptions struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/policy/policy.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/policy/policy.go index 632ef1c93..9707e61a9 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/policy/policy.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/policy/policy.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" identity_api "yunion.io/x/onecloud/pkg/apis/identity" @@ -334,7 +335,7 @@ func (manager *SPolicyManager) allowWithoutCache(policies rbacutils.TPolicySet, user = userCred.GetUserName() token = userCred.GetTokenString() } - log.Warningf("no policies fetched for scope %s user %s token %s key %s", scope, user, token, policyKey(userCred)) + log.Warningf("no policies fetched for scope %s user %s token %s key %s", scope, user, utils.TruncateString(token, 16), policyKey(userCred)) } else { matchRules = policies.GetMatchRules(service, resource, action, extra...) if consts.IsRbacDebug() { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/types/ipmi.go b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/types/ipmi.go index f9d0366d5..c97a8162e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudcommon/types/ipmi.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudcommon/types/ipmi.go @@ -24,15 +24,16 @@ const ( ) type SIPMIInfo struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - IpAddr string `json:"ip_addr,omitempty"` - Present bool `json:"present,omitempty"` - LanChannel uint8 `json:"lan_channel,omitzero"` - Verified bool `json:"verified,omitfalse"` - RedfishApi bool `json:"redfish_api,omitfalse"` - CdromBoot bool `json:"cdrom_boot,omitfalse"` - PxeBoot bool `json:"pxe_boot,omitfalse"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + IpAddr string `json:"ip_addr,omitempty"` + Present bool `json:"present,omitempty"` + LanChannel uint8 `json:"lan_channel,omitzero"` + CipherSuite int `json:"cipher_suite,omitzero"` + Verified bool `json:"verified,omitfalse"` + RedfishApi bool `json:"redfish_api,omitfalse"` + CdromBoot bool `json:"cdrom_boot,omitfalse"` + PxeBoot bool `json:"pxe_boot,omitfalse"` } func (info SIPMIInfo) ToPrepareParams() jsonutils.JSONObject { @@ -48,6 +49,9 @@ func (info SIPMIInfo) ToPrepareParams() jsonutils.JSONObject { } data.Add(jsonutils.NewBool(info.Present), "ipmi_present") data.Add(jsonutils.NewInt(int64(info.LanChannel)), "ipmi_lan_channel") + if info.CipherSuite > 0 { + data.Add(jsonutils.NewInt(int64(info.CipherSuite)), "ipmi_cipher_suite") + } if info.Verified { data.Add(jsonutils.JSONTrue, "ipmi_verified") } diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudmon/misc/usageservice.go b/vendor/yunion.io/x/onecloud/pkg/cloudmon/misc/usageservice.go index 6fd188e57..0ffbe1b7c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudmon/misc/usageservice.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudmon/misc/usageservice.go @@ -83,6 +83,9 @@ func UsegReport(ctx context.Context, userCred mcclient.TokenCredential, isStart } //查询host-type==""的情况，对应onecloud-控制面板-全部 要展示的内容 dataList, err = packMetricList(s, dataList, imageUsageFieldsDict, "host-type", "", nowTime) + if err != nil { + return err + } //获取域/项目 虚拟机usage data, err := getDomainAndProjectServerUsage(s, nowTime) if err != nil { diff --git a/vendor/yunion.io/x/onecloud/pkg/cloudmon/options/options.go b/vendor/yunion.io/x/onecloud/pkg/cloudmon/options/options.go index 444b42b6c..f97257cd2 100644 --- a/vendor/yunion.io/x/onecloud/pkg/cloudmon/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/cloudmon/options/options.go @@ -26,6 +26,8 @@ type CloudMonOptions struct { CollectMetricInterval int64 `help:"Increment Sync Interval unit:minute" default:"6"` SkipMetricPullProviders string `help:"Skip indicate provider metric pull" default:""` + CloudpodsCollectDelayMinutes int64 `help:"Cloudpods Collect Delay Duration unit:minute" default:"0"` + InfluxDatabase string `help:"influxdb database name, default telegraf" default:"telegraf"` DisableServiceMetric bool `help:"disable service metric collect"` diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/access_group_rules.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/access_group_rules.go index 69c7400b6..74c27d97c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/access_group_rules.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/access_group_rules.go @@ -61,7 +61,7 @@ type SAccessGroupRule struct { // 云上Id, 对应云上资源自身Id ExternalId string `width:"256" charset:"utf8" index:"true" list:"user" create:"domain_optional" update:"admin" json:"external_id"` - Priority int `default:"1" list:"user" update:"user" list:"user"` + Priority int `default:"1" list:"user" update:"user"` Source string `width:"16" charset:"ascii" list:"user" update:"user" create:"required"` RWAccessType string `width:"16" charset:"ascii" list:"user" update:"user" create:"required"` UserAccessType string `width:"16" charset:"ascii" list:"user" update:"user" create:"required"` diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/backup_storage.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/backup_storage.go index 5c767f908..0a2268007 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/backup_storage.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/backup_storage.go @@ -396,9 +396,22 @@ func (bs *SBackupStorage) GetIBackupStorage() (backupstorage.IBackupStorage, err return nil, errors.Wrap(err, "GetAccessInfo") } log.Infof("GetIBackupStorage %s %s", bs.Id, accessInfo.String()) - ibs, err := backupstorage.GetBackupStorage(bs.Id, jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict)) + ibs, err := backupstorage.GetBackupStorage(bs.Id, accessInfo) if err != nil { return nil, errors.Wrap(err, "GetBackupStorage") } return ibs, nil } + +func (manager *SBackupStorageManager) fetchItems(filterFunc func(q *sqlchemy.SQuery) *sqlchemy.SQuery) ([]SBackupStorage, error) { + q := manager.Query() + if filterFunc != nil { + q = filterFunc(q) + } + ret := make([]SBackupStorage, 0) + err := db.FetchModelObjects(manager, q, &ret) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + return ret, nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/billingresourcecheck.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/billingresourcecheck.go index a23f88cf3..7428f9899 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/billingresourcecheck.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/billingresourcecheck.go @@ -302,6 +302,7 @@ func CheckBillingResourceExpireAt(ctx context.Context, userCred mcclient.TokenCr notifyclient.EventNotify(ctx, userCred, notifyclient.SEventNotifyParam{ Obj: &resources[i], ObjDetailsDecorator: detailsDecro, + ResourceType: resources[i].ResourceType, Action: notifyclient.ActionExpiredRelease, AdvanceDays: advanceDay, }) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cachedimages.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cachedimages.go index cdb92b54a..9ceb33d47 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cachedimages.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cachedimages.go @@ -41,6 +41,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules/image" + "yunion.io/x/onecloud/pkg/util/hashcache" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -51,6 +52,10 @@ type SCachedimageManager struct { var CachedimageManager *SCachedimageManager +const cachedimageStorageFilterIdLimit = 500 + +var cachedimageStorageFilterIdsCache = hashcache.NewCache(256, 5*time.Minute) + func init() { CachedimageManager = &SCachedimageManager{ SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( @@ -611,7 +616,7 @@ func (cachedImage *SCachedimage) addRefCount() { } } -func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) { +func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType []string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) { storageCachedImage := StoragecachedimageManager.Query().SubQuery() storage := StorageManager.Query().SubQuery() hostStorage := HoststorageManager.Query().SubQuery() @@ -635,7 +640,7 @@ func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes)) } if len(hostType) > 0 { - q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType)) + q = q.Filter(sqlchemy.In(host.Field("host_type"), hostType)) } for _, rangeObj := range rangeObjs { @@ -816,6 +821,10 @@ func (image *SCachedimage) getValidStoragecache() []SStoragecache { func (image *SCachedimage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { db.SharedResourceManager.CleanModelShares(ctx, userCred, image.GetISharableVirtualModel()) + // glance 缓存（无 external_id）软删，便于再次从 glance 同步时恢复；公有云/私有云镜像物理删除 + if len(image.ExternalId) == 0 { + return db.DeleteModel(ctx, userCred, image) + } return db.RealDeleteModel(ctx, userCred, image) } @@ -880,7 +889,121 @@ func (image *SCachedimage) GetCloudprovider() (*SCloudprovider, error) { return cloudprovider, nil } -// 缓存镜像列表 +type sCachedimageStorageFilterCacheKey struct { + Valid bool `json:"valid"` + CloudproviderId []string `json:"cloudprovider_id,omitempty"` + CloudEnv string `json:"cloud_env,omitempty"` + HostSchedtagId string `json:"host_schedtag_id,omitempty"` + ZoneId string `json:"zone_id,omitempty"` + CloudregionId []string `json:"cloudregion_id,omitempty"` +} + +func cachedimageStorageFilterCacheKey(query api.CachedimageListInput) string { + key := sCachedimageStorageFilterCacheKey{ + Valid: query.Valid, + CloudproviderId: query.CloudproviderId, + CloudEnv: query.CloudEnv, + HostSchedtagId: query.HostSchedtagId, + ZoneId: query.ZoneId, + CloudregionId: query.CloudregionId, + } + return jsonutils.Marshal(key).String() +} + +func (manager *SCachedimageManager) buildStorageCachedImagesQuery(query api.CachedimageListInput) *sqlchemy.SQuery { + storagecachedImages := StoragecachedimageManager.Query("cachedimage_id").Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE) + storagecachedImages = storagecachedImages.Snapshot() + + storagesQ := StorageManager.Query("storagecache_id") + storagesQ = storagesQ.Snapshot() + if query.Valid { + storagesQ = storagesQ.In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled") + } + if len(query.CloudproviderId) > 0 { + storagesQ = storagesQ.In("manager_id", query.CloudproviderId) + } + if len(query.ZoneId) > 0 { + storagesQ = storagesQ.Equals("zone_id", query.ZoneId) + } + if len(query.CloudEnv) > 0 { + switch query.CloudEnv { + case api.CLOUD_ENV_PUBLIC_CLOUD: + pubQ := CloudproviderManager.GetPublicProviderIdsQuery() + storagesQ = storagesQ.In("manager_id", pubQ) + case api.CLOUD_ENV_PRIVATE_CLOUD: + privQ := CloudproviderManager.GetPrivateProviderIdsQuery() + storagesQ = storagesQ.In("manager_id", privQ) + case api.CLOUD_ENV_ON_PREMISE: + storagesQ = storagesQ.IsNullOrEmpty("manager_id") + case api.CLOUD_ENV_PRIVATE_ON_PREMISE: + privQ := CloudproviderManager.GetPrivateProviderIdsQuery() + storagesQ = storagesQ.Filter( + sqlchemy.OR( + sqlchemy.In(storagesQ.Field("manager_id"), privQ), + sqlchemy.IsNullOrEmpty(storagesQ.Field("manager_id")), + ), + ) + } + } + if len(query.HostSchedtagId) > 0 { + hostschedtags := HostschedtagManager.Query("host_id").Equals("schedtag_id", query.HostSchedtagId) + hoststorages := HoststorageManager.Query("storage_id").In("host_id", hostschedtags.Distinct().SubQuery()) + storagesQ = storagesQ.In("id", hoststorages.Distinct().SubQuery()) + } + + zonesQ := ZoneManager.Query("id") + zonesQ = zonesQ.Snapshot() + if len(query.CloudregionId) > 0 { + zonesQ = zonesQ.In("cloudregion_id", query.CloudregionId) + } + if zonesQ.IsAltered() { + storagesQ = storagesQ.Filter( + sqlchemy.In(storagesQ.Field("zone_id"), zonesQ.Distinct().SubQuery()), + ) + } + + if storagesQ.IsAltered() { + storagecachedImages = storagecachedImages.Filter( + sqlchemy.In(storagecachedImages.Field("storagecache_id"), storagesQ.Distinct().SubQuery()), + ) + } + + return storagecachedImages +} + +func (manager *SCachedimageManager) fetchStorageCachedImageIds(query api.CachedimageListInput, storagecachedImages *sqlchemy.SQuery) ([]string, error) { + cacheKey := cachedimageStorageFilterCacheKey(query) + if cached := cachedimageStorageFilterIdsCache.AtomicGet(cacheKey); cached != nil { + return cached.([]string), nil + } + + ids, err := db.FetchIds(storagecachedImages.Distinct()) + if err != nil { + return nil, errors.Wrap(err, "FetchIds storage cachedimage ids") + } + cachedimageStorageFilterIdsCache.AtomicSet(cacheKey, ids) + return ids, nil +} + +func (manager *SCachedimageManager) filterByStorage( + q *sqlchemy.SQuery, + query api.CachedimageListInput, +) (*sqlchemy.SQuery, error) { + storagecachedImages := manager.buildStorageCachedImagesQuery(query) + if !storagecachedImages.IsAltered() { + return q, nil + } + + ids, err := manager.fetchStorageCachedImageIds(query, storagecachedImages) + if err != nil { + return nil, err + } + if len(ids) <= cachedimageStorageFilterIdLimit { + return q.In("id", ids), nil + } + return q.In("id", storagecachedImages.Distinct().SubQuery()), nil +} + func (manager *SCachedimageManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, @@ -888,6 +1011,12 @@ func (manager *SCachedimageManager) ListItemFilter( query api.CachedimageListInput, ) (*sqlchemy.SQuery, error) { var err error + + q, err = manager.filterByStorage(q, query) + if err != nil { + return nil, err + } + q, err = manager.SSharableBaseResourceManager.ListItemFilter(ctx, q, userCred, query.SharableResourceBaseListInput) if err != nil { return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter") @@ -903,69 +1032,6 @@ func (manager *SCachedimageManager) ListItemFilter( return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter") } - { - storagecachedImages := StoragecachedimageManager.Query("cachedimage_id").Equals("status", api.CACHED_IMAGE_STATUS_ACTIVE) // .SubQuery() - storagecachedImages = storagecachedImages.Snapshot() - - storagesQ := StorageManager.Query("storagecache_id") - storagesQ = storagesQ.Snapshot() - if query.Valid { - storagesQ = storagesQ.In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled") - } - if len(query.CloudproviderId) > 0 { - storagesQ = storagesQ.In("manager_id", query.CloudproviderId) - } - if len(query.CloudEnv) > 0 { - switch query.CloudEnv { - case api.CLOUD_ENV_PUBLIC_CLOUD: - pubQ := CloudproviderManager.GetPublicProviderIdsQuery() - storagesQ = storagesQ.In("manager_id", pubQ) - case api.CLOUD_ENV_PRIVATE_CLOUD: - privQ := CloudproviderManager.GetPrivateProviderIdsQuery() - storagesQ = storagesQ.In("manager_id", privQ) - case api.CLOUD_ENV_ON_PREMISE: - storagesQ = storagesQ.IsNullOrEmpty("manager_id") - case api.CLOUD_ENV_PRIVATE_ON_PREMISE: - privQ := CloudproviderManager.GetPrivateProviderIdsQuery() - storagesQ = storagesQ.Filter( - sqlchemy.OR( - sqlchemy.In(storagesQ.Field("manager_id"), privQ), - sqlchemy.IsNullOrEmpty(storagesQ.Field("manager_id")), - ), - ) - } - } - if len(query.HostSchedtagId) > 0 { - hostschedtags := HostschedtagManager.Query("host_id").Equals("schedtag_id", query.HostSchedtagId) - hoststorages := HoststorageManager.Query("storage_id").In("host_id", hostschedtags.Distinct().SubQuery()) - storagesQ = storagesQ.In("id", hoststorages.Distinct().SubQuery()) - } - - zonesQ := ZoneManager.Query("id") - zonesQ = zonesQ.Snapshot() - if len(query.ZoneId) > 0 { - zonesQ = zonesQ.Equals("id", query.ZoneId) - } - if len(query.CloudregionId) > 0 { - zonesQ = zonesQ.In("cloudregion_id", query.CloudregionId) - } - if zonesQ.IsAltered() { - storagesQ = storagesQ.Filter( - sqlchemy.In(storagesQ.Field("zone_id"), zonesQ.Distinct().SubQuery()), - ) - } - - if storagesQ.IsAltered() { - storagecachedImages = storagecachedImages.Filter( - sqlchemy.In(storagecachedImages.Field("storagecache_id"), storagesQ.Distinct().SubQuery()), - ) - } - - if storagecachedImages.IsAltered() { - q = q.In("id", storagecachedImages.Distinct().SubQuery()) - } - } - if len(query.ImageType) > 0 { q = q.Equals("image_type", query.ImageType) } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/capabilities.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/capabilities.go index 07c84c126..0e263b75c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/capabilities.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/capabilities.go @@ -869,12 +869,18 @@ func getStorageTypes( } type PCIDevModelTypes struct { - Model string - DevType string - SizeMB int + Model string + DevType string + SharingMode string + NvmeSizeMB int + DevMemorySize int VirtualDev bool Hypervisor string + + Vendor string `json:"vendor,allowempty"` + VendorDeviceId string `json:"vendor_device_id,allowempty"` + PciId string `json:"pci_id,allowempty"` } func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, zone *SZone, domainId, tenantId string) ([]string, []PCIDevModelTypes) { @@ -885,7 +891,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia hostQuery = StorageManager.FilterByOwner(ctx, hostQuery, StorageManager, userCred, ownerId, rbacscope.ScopeDomain) } if len(tenantId) > 0 { - devicesQ = devicesQ.IsNullOrEmpty("guest_id") + devicesQ = IsolatedDeviceManager.GetAvailableIsolatedDeviceQuery(devicesQ) subq := db.SharedResourceManager.Query("resource_id") subq = subq.Equals("resource_type", IsolatedDeviceManager.Keyword()) subq = subq.Equals("target_project_id", tenantId) @@ -916,7 +922,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia devices := devicesQ.SubQuery() hosts := hostQuery.SubQuery() - q := devices.Query(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb")) + q := devices.Query(hosts.Field("host_type"), devices.Field("vendor_device_id"), devices.Field("model"), devices.Field("dev_type"), devices.Field("sharing_mode"), devices.Field("nvme_size_mb"), devices.Field("memory_size")) q = q.Filter(sqlchemy.NotIn(devices.Field("dev_type"), []string{api.USB_TYPE, api.NIC_TYPE})) if zone != nil { q = q.Join(hosts, sqlchemy.Equals(devices.Field("host_id"), hosts.Field("id"))) @@ -935,7 +941,7 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia sqlchemy.IsNullOrEmpty(hosts.Field("manager_id")), )) }*/ - q = q.GroupBy(hosts.Field("host_type"), devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb")) + q = q.GroupBy(hosts.Field("host_type"), devices.Field("vendor_device_id"), devices.Field("model"), devices.Field("dev_type"), devices.Field("sharing_mode"), devices.Field("nvme_size_mb"), devices.Field("memory_size")) rows, err := q.Rows() if err != nil { @@ -946,20 +952,22 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia gpus := make([]PCIDevModelTypes, 0) gpuModels := make([]string, 0) for rows.Next() { - var m, t string - var sizeMB int + var m, t, sharingMode string + var vendorDeviceId string + var nvmeSizeMB int + var memSizeMB int var vdev bool var hypervisor string var hostType string - rows.Scan(&hostType, &m, &t, &sizeMB) + rows.Scan(&hostType, &vendorDeviceId, &m, &t, &sharingMode, &nvmeSizeMB, &memSizeMB) if m == "" { continue } - if utils.IsInStringArray(t, api.VITRUAL_DEVICE_TYPES) { + if utils.IsInStringArray(sharingMode, api.VIRTUAL_SHARING_MODES) { vdev = true } - if utils.IsInStringArray(t, api.VALID_CONTAINER_DEVICE_TYPES) { + if hostType == api.HOST_TYPE_CONTAINER { hypervisor = api.HYPERVISOR_POD } else { hypervisor = api.HYPERVISOR_KVM @@ -969,7 +977,18 @@ func getIsolatedDeviceInfo(ctx context.Context, userCred mcclient.TokenCredentia hypervisor = api.HYPERVISOR_ZETTAKIT } - gpus = append(gpus, PCIDevModelTypes{m, t, sizeMB, vdev, hypervisor}) + gpus = append(gpus, PCIDevModelTypes{ + Model: m, + DevType: t, + SharingMode: sharingMode, + NvmeSizeMB: nvmeSizeMB, + DevMemorySize: memSizeMB, + VirtualDev: vdev, + Hypervisor: hypervisor, + Vendor: GetVendorByVendorDeviceId(vendorDeviceId), + VendorDeviceId: vendorDeviceId, + PciId: vendorDeviceId, + }) if !utils.IsInStringArray(m, gpuModels) { gpuModels = append(gpuModels, m) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_proxmox.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_proxmox.go new file mode 100644 index 000000000..d877bb056 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_proxmox.go @@ -0,0 +1,128 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +// PrepareProxmoxHostNetwork prepares on-premise L2 wires and IP subnets for Proxmox hosts. +// Proxmox does not sync remote L2 wires; host-nics only attach to on-premise wires. +// Flow: +// 1. collect host IP list from Proxmox host nics (bridges included) +// 2. check whether local KVM/on-premise networks already cover those IPs +// 3. if not, create a L2 wire and corresponding IP subnet +// 4. later SyncHostExternalNics associates host-nics onto the prepared on-premise wire by IP +func (account *SCloudaccount) PrepareProxmoxHostNetwork(ctx context.Context, userCred mcclient.TokenCredential, zoneId string) error { + cProvider, err := account.GetProvider(ctx) + if err != nil { + return errors.Wrap(err, "GetProvider") + } + region, err := cProvider.GetOnPremiseIRegion() + if err != nil { + return errors.Wrap(err, "GetOnPremiseIRegion") + } + iHosts, err := region.GetIHosts() + if err != nil { + return errors.Wrap(err, "GetIHosts") + } + + hostIps := make([]netutils.IPV4Addr, 0) + for i := range iHosts { + accessIp := iHosts[i].GetAccessIp() + hostNics, err := iHosts[i].GetIHostNics() + if err != nil { + return errors.Wrapf(err, "iHosts[%d].GetIHostNics()", i) + } + findAccessIp := false + for _, hn := range hostNics { + ipAddrStr := hn.GetIpAddr() + if len(ipAddrStr) == 0 { + // skip interface without a valid ip address + continue + } + if ipAddrStr == "127.0.0.1" { + continue + } + if accessIp == ipAddrStr { + findAccessIp = true + } + ipAddr, err := netutils.NewIPV4Addr(ipAddrStr) + if err != nil { + log.Errorf("fail to parse ipv4 addr %s: %s", ipAddrStr, err) + continue + } + hostIps = append(hostIps, ipAddr) + } + if !findAccessIp && len(accessIp) > 0 { + log.Errorf("Fail to find access ip %s NIC for proxmox host %s", accessIp, iHosts[i].GetName()) + } + } + + onPremiseNets, err := NetworkManager.fetchAllOnpremiseNetworks("", tristate.None) + if err != nil { + return errors.Wrap(err, "NetworkManager.fetchAllOnpremiseNetworks") + } + + if zoneId == "" { + zoneIds, err := fetchOnpremiseZoneIds(onPremiseNets) + if err != nil { + return errors.Wrap(err, "fetchOnpremiseZoneIds") + } + if len(zoneIds) == 0 { + zoneIds, err = ZoneManager.getOnpremiseZoneIds() + if err != nil { + return errors.Wrap(err, "getOnpremiseZoneIds") + } + } + if len(zoneIds) == 0 { + return errors.Wrap(httperrors.ErrInvalidStatus, "empty zone id?") + } + if len(zoneIds) == 1 { + zoneId = zoneIds[0] + } else { + zoneId, err = guessEsxiZoneId(hostIps, onPremiseNets) + if err != nil { + return errors.Wrap(err, "fail to find zone of proxmox") + } + } + } + + netConfs, err := guessEsxiNetworks(hostIps, account.Name, onPremiseNets) + if err != nil { + return errors.Wrap(err, "guessEsxiNetworks") + } + for i := range netConfs { + netConfs[i].Name = fmt.Sprintf("%s-proxmox-host-network", account.Name) + netConfs[i].Description = fmt.Sprintf("Auto created network for proxmox cloudaccount %q", account.Name) + } + log.Infof("proxmox netConfs: %s", jsonutils.Marshal(netConfs)) + err = account.createNetworks(ctx, account.Name, zoneId, netConfs) + if err != nil { + return errors.Wrap(err, "account.createNetworks") + } + + return nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_vmware.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_vmware.go index d0266ea2f..dcc3479f6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_vmware.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccount_vmware.go @@ -74,7 +74,7 @@ func (account *SCloudaccount) PrepareEsxiHostNetwork(ctx context.Context, userCr accessIp := iHosts[i].GetAccessIp() hostNics, err := iHosts[i].GetIHostNics() if err != nil { - return errors.Wrapf(err, "iHosts[%d].GetIHostNics()", i) + return errors.Wrapf(err, "iHosts(%s,%s).GetIHostNics()", iHosts[i].GetName(), iHosts[i].GetId()) } findAccessIp := false for _, hn := range hostNics { @@ -145,7 +145,7 @@ func fetchOnpremiseZoneIds(onPremiseNets []SNetwork) ([]string, error) { for i := range onPremiseNets { zone, err := onPremiseNets[i].GetZone() if err != nil { - return nil, errors.Wrapf(err, "onPremiseNets[%d].GetZone", i) + return nil, errors.Wrapf(err, "onPremiseNets(%s,%s).GetZone()", onPremiseNets[i].GetName(), onPremiseNets[i].GetId()) } if !utils.IsInArray(zone.Id, zoneIds) { zoneIds = append(zoneIds, zone.Id) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccounts.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccounts.go index e406b7edc..d88263b4f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccounts.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudaccounts.go @@ -2523,10 +2523,11 @@ func (account *SCloudaccount) SubmitSyncAccountTask(ctx context.Context, userCre }() log.Debugf("syncAccountStatus %s %s", account.Id, account.Name) err := account.syncAccountStatus(ctx, userCred, true) + if err != nil { + log.Errorf("syncAccountStatus %s %s fail: %s", account.Id, account.Name, err) + err = errors.Wrap(err, "account.syncAccountStatus") + } if waitChan != nil { - if err != nil { - err = errors.Wrap(err, "account.syncAccountStatus") - } waitChan <- err } }) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudproviders.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudproviders.go index 320c6f54f..ecbe736c4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudproviders.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudproviders.go @@ -1251,7 +1251,7 @@ func (manager *SCloudproviderManager) ListItemFilter( } var zone *SZone - var region *SCloudregion + regionIds := make([]string, 0) if len(query.ZoneId) > 0 { zoneObj, err := validators.ValidateModel(ctx, userCred, ZoneManager, &query.ZoneId) @@ -1259,10 +1259,11 @@ func (manager *SCloudproviderManager) ListItemFilter( return nil, err } zone = zoneObj.(*SZone) - region, err = zone.GetRegion() + region, err := zone.GetRegion() if err != nil { return nil, err } + regionIds = append(regionIds, region.Id) vpcs := VpcManager.Query("manager_id").Equals("cloudregion_id", region.Id).Distinct() if !utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) { wires := WireManager.Query().Equals("zone_id", query.ZoneId).SubQuery() @@ -1276,17 +1277,21 @@ func (manager *SCloudproviderManager) ListItemFilter( ), ) } else if len(query.CloudregionId) > 0 { - regionObj, err := CloudregionManager.FetchByIdOrName(ctx, userCred, query.CloudregionId) - if err != nil { - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2("cloudregion", query.CloudregionId) + for _, regionId := range query.CloudregionId { + if len(regionId) == 0 { + continue } - return nil, httperrors.NewGeneralError(err) + regionObj, err := ValidateCloudregionId(ctx, userCred, regionId) + if err != nil { + return nil, errors.Wrapf(err, "ValidateCloudregionId %s", regionId) + } + regionIds = append(regionIds, regionObj.GetId()) + } + if len(regionIds) > 0 { + pr := CloudproviderRegionManager.Query().SubQuery() + sq := pr.Query(pr.Field("cloudprovider_id")).In("cloudregion_id", regionIds).Distinct() + q = q.In("id", sq) } - region = regionObj.(*SCloudregion) - pr := CloudproviderRegionManager.Query().SubQuery() - sq := pr.Query(pr.Field("cloudprovider_id")).Equals("cloudregion_id", region.Id).Distinct() - q = q.In("id", sq) } if query.Usable != nil && *query.Usable { @@ -1310,8 +1315,8 @@ func (manager *SCloudproviderManager) ListItemFilter( if zone != nil { zoneFilter := sqlchemy.OR(sqlchemy.Equals(wires.Field("zone_id"), zone.GetId()), sqlchemy.IsNullOrEmpty(wires.Field("zone_id"))) sq = sq.Filter(zoneFilter) - } else if region != nil { - sq = sq.Filter(sqlchemy.Equals(vpcs.Field("cloudregion_id"), region.GetId())) + } else if len(regionIds) > 0 { + sq = sq.Filter(sqlchemy.In(vpcs.Field("cloudregion_id"), regionIds)) } q = q.Filter(sqlchemy.In(q.Field("id"), sq.SubQuery())) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudregions.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudregions.go index 378f90343..0a7c62f4f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudregions.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudregions.go @@ -910,6 +910,9 @@ func (manager *SCloudregionManager) ListItemFilter( } domainId, err := db.FetchQueryDomain(ctx, userCred, jsonutils.Marshal(query)) + if err != nil { + return nil, err + } if len(domainId) > 0 { q = q.In("id", getCloudRegionIdByDomainId(domainId)) } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go index f98ab0af4..dbdb70f22 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/cloudsync.go @@ -513,6 +513,42 @@ func syncRegionSecGroup( } } +func syncRegionIpSets( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { + ipSets, err := func() ([]cloudprovider.ICloudIpSet, error) { + defer syncResults.AddRequestCost(IpSetManager)() + return remoteRegion.GetIIpSets() + }() + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotImplemented || errors.Cause(err) == cloudprovider.ErrNotSupported { + return + } + msg := fmt.Sprintf("GetIIpSets for region %s provider %s failed %s", localRegion.Name, provider.Name, err) + log.Errorf("%s", msg) + return + } + + result := func() compare.SyncResult { + defer syncResults.AddSqlCost(IpSetManager)() + return localRegion.SyncIpSets(ctx, userCred, provider, ipSets, syncRange.Xor) + }() + syncResults.Add(IpSetManager, result) + + notes := fmt.Sprintf("SyncIpSets for region %s provider %s result: %s", localRegion.Name, provider.Name, result.Result()) + log.Infof("%s", notes) + provider.SyncError(result, notes, userCred) + if result.IsError() { + return + } +} + func syncVpcSecGroup( ctx context.Context, userCred mcclient.TokenCredential, @@ -1294,35 +1330,31 @@ func (self *SGuest) SyncVMIsolateDevices(ctx context.Context, userCred mcclient. if err != nil { return err } - devs, err := self.GetIsolatedDevices() + gdevs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrapf(err, "GetIsolatedDevices") } result := compare.SyncResult{} - for i := range devs { - if !utils.IsInStringArray(devs[i].ExternalId, externalIds) { - _, err = db.Update(&devs[i], func() error { - devs[i].GuestId = "" - return nil - }) + for i := range gdevs { + dev := gdevs[i].GetIsolatedDevice() + if !utils.IsInStringArray(dev.ExternalId, externalIds) { + err = gdevs[i].Detach(ctx, userCred) if err != nil { return err } result.Delete() } } - devs = []SIsolatedDevice{} + devs := []SIsolatedDevice{} sq := HostManager.Query("id").Equals("manager_id", host.ManagerId).SubQuery() q := IsolatedDeviceManager.Query().In("host_id", sq).In("external_id", externalIds) err = db.FetchModelObjects(IsolatedDeviceManager, q, &devs) if err != nil { return err } + for i := range devs { - _, err = db.Update(&devs[i], func() error { - devs[i].GuestId = self.Id - return nil - }) + err = self.attachIsolatedDevice(ctx, userCred, &devs[i], nil, nil, nil, "") if err != nil { return err } @@ -2492,6 +2524,9 @@ func syncPublicCloudProviderInfo( syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } + if syncRange.IsNotSkipSyncResource(IpSetManager) { + syncRegionIpSets(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) + } if syncRange.IsNotSkipSyncResource(SecurityGroupManager) { syncRegionSecGroup(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } @@ -2775,16 +2810,19 @@ func syncOnPremiseCloudProviderInfo( } if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK) { - remoteVpcs, err := iregion.GetIVpcs() - if err != nil { - msg := fmt.Sprintf("GetIVpcs for provider %s failed %s", provider.GetName(), err) - log.Errorf("%s", msg) - return err - } - { - // sync wires - localVpc := VpcManager.FetchDefaultVpc() - syncVpcWires(ctx, userCred, syncResults, provider, localVpc, remoteVpcs[0], zone, syncRange) + // Proxmox does not sync remote L2 wires; host-nics attach to on-premise wires only + if provider.Provider != api.CLOUD_PROVIDER_PROXMOX { + remoteVpcs, err := iregion.GetIVpcs() + if err != nil { + msg := fmt.Sprintf("GetIVpcs for provider %s failed %s", provider.GetName(), err) + log.Errorf("%s", msg) + return err + } + { + // sync wires + localVpc := VpcManager.FetchDefaultVpc() + syncVpcWires(ctx, userCred, syncResults, provider, localVpc, remoteVpcs[0], zone, syncRange) + } } } @@ -3034,6 +3072,13 @@ func SyncCloudproviderResources(ctx context.Context, userCred mcclient.TokenCred syncSSLCertificates(ctx, userCred, SSyncResultSet{}, provider, driver, syncRange.Xor) } + if syncRange.IsNotSkipSyncResource(IpSetManager) { + err = syncProviderIpSets(ctx, userCred, SSyncResultSet{}, provider, driver, syncRange.Xor) + if err != nil { + log.Errorf("syncProviderIpSets error: %v", err) + } + } + return nil } @@ -3158,6 +3203,29 @@ func syncSSLCertificates(ctx context.Context, userCred mcclient.TokenCredential, return nil } +func syncProviderIpSets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, xor bool) error { + ipSets, err := func() ([]cloudprovider.ICloudIpSet, error) { + defer syncResults.AddRequestCost(IpSetManager)() + return driver.GetIIpSets() + }() + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotImplemented || errors.Cause(err) == cloudprovider.ErrNotSupported { + return nil + } + return err + } + + result := func() compare.SyncResult { + defer syncResults.AddSqlCost(IpSetManager)() + return provider.SyncIpSets(ctx, userCred, ipSets, xor) + }() + syncResults.Add(IpSetManager, result) + notes := fmt.Sprintf("Sync ip sets for provider %s result: %s", provider.Name, result.Result()) + log.Infof("%s", notes) + provider.SyncError(result, notes, userCred) + return nil +} + func syncTablestore( ctx context.Context, userCred mcclient.TokenCredential, diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/container_drivers.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/container_drivers.go index fac10ad6a..d639bfa00 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/container_drivers.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/container_drivers.go @@ -134,7 +134,7 @@ type IContainerDeviceDriver interface { GetType() apis.ContainerDeviceType ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, dev *api.ContainerDevice, input *api.ServerCreateInput) error ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) - ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) + ToHostDevice(dev *api.ContainerDevice, guestId string) (*hostapi.ContainerDevice, error) } type IContainerLifecyleDriver interface { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/container_hostpath.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/container_hostpath.go new file mode 100644 index 000000000..db5b6ce52 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/container_hostpath.go @@ -0,0 +1,64 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or authorized to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/fileutils2" +) + +// ValidateHostBindPath cleans an absolute host bind path and rejects +// restricted host locations for non-admin callers. +func ValidateHostBindPath(userCred mcclient.TokenCredential, path string) (string, error) { + clean, err := fileutils2.CleanHostBindPath(path) + if err != nil { + return "", httperrors.NewInputParameterError("%v", err) + } + if userCred == nil || !userCred.HasSystemAdminPrivilege() { + if fileutils2.IsRestrictedHostBindPath(clean) { + return "", httperrors.NewForbiddenError("host path %s is not allowed", clean) + } + } + return clean, nil +} + +// ValidateHostPathAutoCreate checks auto-create options for a host bind path. +func ValidateHostPathAutoCreate(autoCreate bool, cfg *apis.ContainerVolumeMountHostPathAutoCreateConfig) error { + if !autoCreate && cfg == nil { + return nil + } + if cfg == nil { + return nil + } + if cfg.Permissions != "" { + _, canon, err := fileutils2.ParseUnixFileMode(cfg.Permissions) + if err != nil { + return httperrors.NewInputParameterError("invalid auto_create permissions: %v", err) + } + cfg.Permissions = canon + } + return nil +} + +// ValidateRelSubpath cleans a relative path that must stay inside a parent dir. +func ValidateRelSubpath(path string) (string, error) { + clean, err := fileutils2.CleanRelSubpath(path) + if err != nil { + return "", httperrors.NewInputParameterError("%v", err) + } + return clean, nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go index 9ede8be49..60685fc42 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/containers.go @@ -43,6 +43,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" identitymod "yunion.io/x/onecloud/pkg/mcclient/modules/identity" + imagemod "yunion.io/x/onecloud/pkg/mcclient/modules/image" kubemod "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" ) @@ -173,6 +174,15 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient. if !sets.NewString(apis.ImagePullPolicyAlways, apis.ImagePullPolicyIfNotPresent).Has(string(spec.ImagePullPolicy)) { return httperrors.NewInputParameterError("invalid image_pull_policy %s", spec.ImagePullPolicy) } + // ctr == nil means the container is being created, apply the defaults + // (command/args/envs) configured on the container_image; when updating the + // spec of an existing container keep what the user has set. + if err := m.resolveContainerImageId(ctx, userCred, spec, ctr == nil); err != nil { + return errors.Wrap(err, "resolve container_image_id") + } + if spec.Image == "" { + return httperrors.NewNotEmptyError("image is required") + } if spec.ImageCredentialId != "" { if _, err := m.GetImageCredential(ctx, userCred, spec.ImageCredentialId); err != nil { return errors.Wrapf(err, "get image credential by id: %s", spec.ImageCredentialId) @@ -234,6 +244,146 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient. return nil } +// fetchContainerImageObj fetches the container_image resource through the +// glance service. +func fetchContainerImageObj(ctx context.Context, userCred mcclient.TokenCredential, imageId string) (jsonutils.JSONObject, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + obj, err := imagemod.ContainerImages.Get(s, imageId, nil) + if err != nil { + return nil, httperrors.NewGeneralError(errors.Wrapf(err, "get container_image %s", imageId)) + } + return obj, nil +} + +// fetchContainerRegistryObj fetches the container_registry resource through the +// glance service. +func fetchContainerRegistryObj(ctx context.Context, userCred mcclient.TokenCredential, registryId string) (jsonutils.JSONObject, error) { + s := auth.GetSession(ctx, userCred, options.Options.Region) + return imagemod.ContainerRegistries.Get(s, registryId, nil) +} + +func (m *SContainerManager) resolveContainerImageId(ctx context.Context, userCred mcclient.TokenCredential, spec *api.ContainerSpec, applyDefaults bool) error { + if spec.ContainerImageId == "" { + return nil + } + obj, err := fetchContainerImageObj(ctx, userCred, spec.ContainerImageId) + if err != nil { + return err + } + imageName, _ := obj.GetString("image_name") + imageLabel, _ := obj.GetString("image_label") + if imageName == "" || imageLabel == "" { + return httperrors.NewInputParameterError("container_image %s missing image_name or image_label", spec.ContainerImageId) + } + spec.ContainerImageId, _ = obj.GetString("id") + spec.Image = fmt.Sprintf("%s:%s", imageName, imageLabel) + // Credential is owned by container_image (fallback to registry); clear client-provided value first. + spec.ImageCredentialId = "" + credId, _ := obj.GetString("credential_id") + if credId == "" { + if registryId, _ := obj.GetString("registry_id"); registryId != "" { + regObj, regErr := fetchContainerRegistryObj(ctx, userCred, registryId) + if regErr != nil { + return httperrors.NewGeneralError(errors.Wrapf(regErr, "get container_registry %s for credential fallback", registryId)) + } + credId, _ = regObj.GetString("credential_id") + } + } + if credId != "" { + spec.ImageCredentialId = credId + } + if applyDefaults { + if err := applyContainerImageDefaults(spec, obj); err != nil { + return errors.Wrap(err, "apply container_image defaults") + } + } + return nil +} + +// applyContainerImageDefaults applies the defaults of the referenced +// container_image to the container spec. The container_image wins for +// command/args when it configures a non-empty value; the envs of the image are +// merged into the container envs and win on key conflict. +func applyContainerImageDefaults(spec *api.ContainerSpec, image jsonutils.JSONObject) error { + if gotypes.IsNil(image) { + return nil + } + if command, err := image.Get("command"); err == nil { + values := make([]string, 0) + if err := command.Unmarshal(&values); err != nil { + return errors.Wrap(err, "unmarshal container_image command") + } + // an empty command on the image means "unset", keep what the container has + if len(values) > 0 { + spec.Command = values + } + } + if args, err := image.Get("args"); err == nil { + values := make([]string, 0) + if err := args.Unmarshal(&values); err != nil { + return errors.Wrap(err, "unmarshal container_image args") + } + if len(values) > 0 { + spec.Args = values + } + } + imageEnvs := make([]*apis.ContainerKeyValue, 0) + if envs, err := image.Get("envs"); err == nil { + if err := envs.Unmarshal(&imageEnvs); err != nil { + return errors.Wrap(err, "unmarshal container_image envs") + } + } + if len(imageEnvs) == 0 && len(spec.Envs) == 0 { + return nil + } + spec.Envs = mergeContainerImageEnvs(spec.Envs, imageEnvs) + return nil +} + +// mergeContainerImageEnvs merges the image envs into the container envs. The +// container envs are the base and keep their order, the image envs override the +// value of the same key and append the missing keys. +func mergeContainerImageEnvs(ctrEnvs, imageEnvs []*apis.ContainerKeyValue) []*apis.ContainerKeyValue { + out := make([]*apis.ContainerKeyValue, 0, len(ctrEnvs)+len(imageEnvs)) + indexByKey := make(map[string]int, len(ctrEnvs)+len(imageEnvs)) + for _, env := range ctrEnvs { + if env == nil { + continue + } + key := strings.TrimSpace(env.Key) + if key == "" { + continue + } + kv := env + if env.Key != key { + kv = &apis.ContainerKeyValue{Key: key, Value: env.Value, ValueFrom: env.ValueFrom} + } + indexByKey[key] = len(out) + out = append(out, kv) + } + for _, env := range imageEnvs { + if env == nil { + continue + } + key := strings.TrimSpace(env.Key) + if key == "" { + continue + } + if idx, ok := indexByKey[key]; ok { + // keep the key spelling of the container side, take the value of the image + out[idx] = &apis.ContainerKeyValue{ + Key: out[idx].Key, + Value: env.Value, + ValueFrom: env.ValueFrom, + } + continue + } + indexByKey[key] = len(out) + out = append(out, &apis.ContainerKeyValue{Key: key, Value: env.Value, ValueFrom: env.ValueFrom}) + } + return out +} + func (m *SContainerManager) ValidateSpecEnvs(ctx context.Context, userCred mcclient.TokenCredential, spec *api.ContainerSpec) error { var errs []error for _, env := range spec.Envs { @@ -696,6 +846,13 @@ func (c *SContainer) PerformStart(ctx context.Context, userCred mcclient.TokenCr } func (c *SContainer) StartStartTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + // Refresh the defaults inherited from the container_image so that changes + // made on the image take effect on start/restart. All the start paths + // (container-start, container-restart, pod-start, pod-restart and the batch + // variants) funnel through here. A failure must not block the start. + if err := c.applyContainerImageDefaultsOnStart(ctx, userCred); err != nil { + log.Warningf("refresh container_image defaults for container %s: %v", c.GetId(), err) + } c.SetStatus(ctx, userCred, api.CONTAINER_STATUS_STARTING, "") task, err := taskman.TaskManager.NewTask(ctx, "ContainerStartTask", c, userCred, nil, parentTaskId, "", nil) if err != nil { @@ -853,7 +1010,58 @@ func (c *SContainer) GetImageCredential(ctx context.Context, userCred mcclient.T return GetContainerManager().GetImageCredential(ctx, userCred, c.Spec.ImageCredentialId) } +func (c *SContainer) ensureContainerImageResolved(ctx context.Context, userCred mcclient.TokenCredential) error { + if c.Spec == nil { + return httperrors.NewNotEmptyError("container spec is empty") + } + needResolve := c.Spec.ContainerImageId != "" && c.Spec.Image == "" + if needResolve { + if _, err := db.Update(c, func() error { + if err := GetContainerManager().resolveContainerImageId(ctx, userCred, c.Spec, false); err != nil { + return err + } + if c.Spec.Image == "" { + return httperrors.NewInputParameterError("container_image %s resolved empty image", c.Spec.ContainerImageId) + } + return nil + }); err != nil { + return errors.Wrap(err, "resolve and persist container_image_id") + } + } + if c.Spec.Image == "" { + return httperrors.NewNotEmptyError("image is required") + } + return nil +} + +// applyContainerImageDefaultsOnStart refreshes command/args/envs of the +// container spec from the referenced container_image, so that changes made on +// the image take effect when the container is started or restarted. +func (c *SContainer) applyContainerImageDefaultsOnStart(ctx context.Context, userCred mcclient.TokenCredential) error { + if c.Spec == nil || c.Spec.ContainerImageId == "" { + return nil + } + obj, err := fetchContainerImageObj(ctx, userCred, c.Spec.ContainerImageId) + if err != nil { + return errors.Wrapf(err, "fetch container_image %s", c.Spec.ContainerImageId) + } + // db.Update is a no-op when nothing changed, so it is safe to call it here + _, err = db.Update(c, func() error { + if c.Spec == nil { + return httperrors.NewNotEmptyError("container spec is empty") + } + return applyContainerImageDefaults(c.Spec, obj) + }) + if err != nil { + return errors.Wrap(err, "update container spec") + } + return nil +} + func (c *SContainer) GetHostPullImageInput(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerPullImageInput, error) { + if err := c.ensureContainerImageResolved(ctx, userCred); err != nil { + return nil, err + } input := &hostapi.ContainerPullImageInput{ Image: c.Spec.Image, PullPolicy: c.Spec.ImagePullPolicy, @@ -944,13 +1152,18 @@ func (c *SContainer) ToHostContainerSpec(ctx context.Context, userCred mcclient. } ctrDevs := make([]*hostapi.ContainerDevice, 0) for _, dev := range c.Spec.Devices { - ctrDev, err := GetContainerDeviceDriver(dev.Type).ToHostDevice(dev) + ctrDev, err := GetContainerDeviceDriver(dev.Type).ToHostDevice(dev, c.GuestId) if err != nil { return nil, errors.Wrapf(err, "ToHostDevice %s", jsonutils.Marshal(dev)) } ctrDevs = append(ctrDevs, ctrDev) } + pullInput, err := c.GetHostPullImageInput(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "GetHostPullImageInput") + } + // Copy after resolve so host receives filled Image (never relies on ContainerImageId). spec := c.Spec.ContainerSpec hSpec := &hostapi.ContainerSpec{ ContainerSpec: spec, @@ -958,10 +1171,7 @@ func (c *SContainer) ToHostContainerSpec(ctx context.Context, userCred mcclient. VolumeMounts: mounts, Devices: ctrDevs, } - pullInput, err := c.GetHostPullImageInput(ctx, userCred) - if err != nil { - return nil, errors.Wrap(err, "GetHostPullImageInput") - } + hSpec.Image = pullInput.Image hSpec.ImageCredentialToken = base64.StdEncoding.EncodeToString([]byte(jsonutils.Marshal(pullInput.Auth).String())) secretCredentials, err := c.GetSecretCredentials(ctx, userCred) if err != nil { @@ -1223,22 +1433,39 @@ func (c *SContainer) getContainerHostCommitInput(ctx context.Context, userCred m } if input.RegistryId != "" { s := auth.GetSession(ctx, userCred, consts.GetRegion()) - obj, err := kubemod.ContainerRegistries.Get(s, input.RegistryId, nil) - if err != nil { - return nil, httperrors.NewGeneralError(err) + var ( + obj jsonutils.JSONObject + err error + fromGlance bool + ) + // Prefer glance-managed registry; fall back to kubeserver during migration. + obj, err = imagemod.ContainerRegistries.Get(s, input.RegistryId, nil) + if err == nil { + fromGlance = true + } else { + obj, err = kubemod.ContainerRegistries.Get(s, input.RegistryId, nil) + if err != nil { + return nil, httperrors.NewGeneralError(err) + } } reg := new(api.KubeServerContainerRegistryDetails) if err := obj.Unmarshal(reg); err != nil { - return nil, errors.Wrap(err, "Unmarshal kube server registry details") + return nil, errors.Wrap(err, "Unmarshal registry details") } if reg.Config == nil { - confObj, err := kubemod.ContainerRegistries.GetSpecific(s, input.RegistryId, "config", nil) - if err != nil { - return nil, errors.Wrap(err, "Get kube server registry config") + var confObj jsonutils.JSONObject + var confErr error + if fromGlance { + confObj, confErr = imagemod.ContainerRegistries.GetSpecific(s, input.RegistryId, "config", nil) + } else { + confObj, confErr = kubemod.ContainerRegistries.GetSpecific(s, input.RegistryId, "config", nil) + } + if confErr != nil { + return nil, errors.Wrap(confErr, "Get registry config") } reg.Config = new(api.KubeServerContainerRegistryConfig) if err := confObj.Unmarshal(reg.Config); err != nil { - return nil, errors.Wrap(err, "Unmarshal kube server registry config") + return nil, errors.Wrap(err, "Unmarshal registry config") } } repoUrl = reg.Url diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/diskbackups.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/diskbackups.go index 12c5a8d6b..dd7be958a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/diskbackups.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/diskbackups.go @@ -60,7 +60,7 @@ type SDiskBackup struct { db.SEncryptedResource - DiskId string `width:"36" charset:"ascii" nullable:"true" create:"required" list:"user" index:"true"` + DiskId string `width:"36" charset:"ascii" nullable:"true" create:"optional" list:"user" index:"true"` BackupStorageId string `width:"36" charset:"ascii" nullable:"true" create:"required" list:"user" index:"true"` StorageId string `width:"36" charset:"ascii" nullable:"true" list:"user"` @@ -71,6 +71,8 @@ type SDiskBackup struct { // 操作系统类型 OsType string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional"` DiskConfig *SBackupDiskConfig + + BackupFilePath string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional"` } var DiskBackupManager *SDiskBackupManager @@ -137,6 +139,17 @@ func (dm *SDiskBackupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ q = q.Join(insjsq, sqlchemy.Equals(q.Field("id"), insjsq.Field("disk_backup_id"))) } } + if len(input.ServerId) > 0 { + iG, err := GuestManager.FetchByIdOrName(ctx, userCred, input.ServerId) + if err != nil && err == sql.ErrNoRows { + return nil, httperrors.NewNotFoundError("guest %s not found", input.ServerId) + } else if err != nil { + return nil, errors.Wrap(err, "fetch guest") + } + guest := iG.(*SGuest) + gdq := GuestdiskManager.Query("disk_id").Equals("guest_id", guest.Id).SubQuery() + q = q.In("disk_id", gdq) + } return q, nil } @@ -229,6 +242,76 @@ func (db *SDiskBackup) GetRegionDriver() (IRegionDriver, error) { return cloudRegion.GetDriver(), nil } +func (dm *SDiskBackupManager) validateCreateDataFromBackupFile( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.DiskBackupCreateInput, +) (api.DiskBackupCreateInput, error) { + var backupStorages []*SBackupStorage + if len(input.DiskId) > 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "disk_id is not allowed") + } + if len(input.BackupStorageId) > 0 { + bsObj, err := BackupStorageManager.FetchByIdOrName(ctx, userCred, input.BackupStorageId) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return input, httperrors.NewResourceNotFoundError2(BackupStorageManager.Keyword(), input.BackupStorageId) + } + if errors.Cause(err) == sqlchemy.ErrDuplicateEntry { + return input, httperrors.NewDuplicateResourceError(BackupStorageManager.Keyword(), input.BackupStorageId) + } + return input, httperrors.NewGeneralError(err) + } + backupStorages = append(backupStorages, bsObj.(*SBackupStorage)) + } else { + bsItems, err := BackupStorageManager.fetchItems(func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + q = q.Equals("status", api.BACKUPSTORAGE_STATUS_ONLINE) + q = q.Equals("enabled", 1) + return q + }) + if err != nil { + return input, httperrors.NewGeneralError(err) + } + for i := range bsItems { + backupStorages = append(backupStorages, &bsItems[i]) + } + } + if len(backupStorages) == 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "no backup storage found") + } + var errs []error + input.SizeMb = -1 + for i := range backupStorages { + ibs, err := backupStorages[i].GetIBackupStorage() + if err != nil { + return input, errors.Wrap(err, "GetIBackupStorage") + } + exists, sizeBytes, offlineReason, err := ibs.IsBackupExists("", input.BackupFilePath) + if !exists { + if err != nil { + errs = append(errs, errors.Wrapf(err, "Backup storage error %s(%s)", backupStorages[i].GetName(), backupStorages[i].GetId())) + } + if len(offlineReason) > 0 { + errs = append(errs, errors.Wrapf(errors.ErrInvalidStatus, "Backup storage %s(%s) is offline: %s", backupStorages[i].GetName(), backupStorages[i].GetId(), offlineReason)) + } + errs = append(errs, errors.Wrapf(errors.ErrNotFound, "Backup file %s not found in backup storage %s(%s)", input.BackupFilePath, backupStorages[i].GetName(), backupStorages[i].GetId())) + } else { + input.BackupStorageId = backupStorages[i].GetId() + input.SizeMb = int(sizeBytes / 1024 / 1024) + break + } + } + if input.SizeMb == -1 { + if len(errs) > 0 { + return input, errors.NewAggregate(errs) + } + return input, errors.Wrap(httperrors.ErrInputParameter, "no backup storage found") + } + return input, nil +} + func (dm *SDiskBackupManager) ValidateCreateData( ctx context.Context, userCred mcclient.TokenCredential, @@ -236,6 +319,10 @@ func (dm *SDiskBackupManager) ValidateCreateData( query jsonutils.JSONObject, input api.DiskBackupCreateInput, ) (api.DiskBackupCreateInput, error) { + if len(input.BackupFilePath) > 0 { + // create disk backup from a backup file + return dm.validateCreateDataFromBackupFile(ctx, userCred, ownerId, query, input) + } if input.NeedEncrypt() { return input, errors.Wrap(httperrors.ErrInputParameter, "encryption should not be specified") } @@ -272,9 +359,7 @@ func (dm *SDiskBackupManager) ValidateCreateData( } return input, httperrors.NewGeneralError(err) } - if err != nil { - return input, err - } + bs := ibs.(*SBackupStorage) if bs.Status != api.BACKUPSTORAGE_STATUS_ONLINE { return input, httperrors.NewForbiddenError("can't backup guest to backup storage with status %s", bs.Status) @@ -329,36 +414,42 @@ func (db *SDiskBackup) CustomizeCreate(ctx context.Context, userCred mcclient.To if err != nil { return err } - diskObj, err := DiskManager.FetchById(db.DiskId) - if err != nil { - return errors.Wrap(err, "DiskManager.FetchById") - } - disk := diskObj.(*SDisk) - db.DiskConfig = &SBackupDiskConfig{ - DiskConfig: *disk.ToDiskConfig(), - Name: disk.GetName(), - BackupAsTar: input.BackupAsTar, + if len(db.DiskId) > 0 { + diskObj, err := DiskManager.FetchById(db.DiskId) + if err != nil { + return errors.Wrap(err, "DiskManager.FetchById") + } + disk := diskObj.(*SDisk) + db.DiskConfig = &SBackupDiskConfig{ + DiskConfig: *disk.ToDiskConfig(), + Name: disk.GetName(), + BackupAsTar: input.BackupAsTar, + } + db.DiskType = disk.DiskType + db.DiskSizeMb = disk.DiskSize + db.OsArch = disk.OsArch + db.StorageId = disk.StorageId + db.DomainId = disk.DomainId + db.ProjectId = disk.ProjectId } - db.DiskType = disk.DiskType - db.DiskSizeMb = disk.DiskSize - db.OsArch = disk.OsArch - db.StorageId = disk.StorageId - db.DomainId = disk.DomainId - db.ProjectId = disk.ProjectId return nil } func (db *SDiskBackup) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { db.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) - disk, err := db.GetDisk() - if err != nil { - log.Errorf("unable to GetDisk: %s", err.Error()) - } - err = disk.InheritTo(ctx, userCred, db) - if err != nil { - log.Errorf("unable to inherit from disk %s to backup %s: %s", disk.GetId(), db.GetId(), err.Error()) + if len(db.DiskId) > 0 { + disk, err := db.GetDisk() + if err != nil { + log.Errorf("unable to GetDisk: %s", err.Error()) + } + err = disk.InheritTo(ctx, userCred, db) + if err != nil { + log.Errorf("unable to inherit from disk %s to backup %s: %s", disk.GetId(), db.GetId(), err.Error()) + } + db.StartBackupCreateTask(ctx, userCred, nil, "") + } else { + db.SetStatus(ctx, userCred, api.BACKUP_STATUS_READY, "import from backup file") } - db.StartBackupCreateTask(ctx, userCred, nil, "") } func (db *SDiskBackup) StartBackupCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { @@ -568,6 +659,8 @@ func (diskBackup *SDiskBackup) PackMetadata() *api.DiskBackupPackMetadata { DiskConfig: diskBackup.DiskConfig.DiskConfig, Name: diskBackup.DiskConfig.Name, }, + // 备份文件路径 + BackupFilePath: diskBackup.BackupFilePath, } } @@ -591,6 +684,7 @@ func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, o backup.Name = name backup.Id = id backup.Status = api.BACKUP_STATUS_READY + backup.BackupFilePath = metadata.BackupFilePath err := DiskBackupManager.TableSpec().Insert(ctx, backup) if err != nil { return nil, err @@ -631,7 +725,7 @@ func (diskBackup *SDiskBackup) GetDetailsExportInfo(ctx context.Context, userCre return nil, errors.Wrap(err, "GetIBackupStorage") } - exportInfo.AccessUrl, err = ibs.GetExternalAccessUrl(diskBackup.Id) + exportInfo.AccessUrl, err = ibs.GetExternalAccessUrl(diskBackup.Id, diskBackup.BackupFilePath) if err != nil { log.Errorf("SDiskBackup %s(%s) GetExternalAccessUrl fail: %v", diskBackup.GetName(), diskBackup.GetId(), err) } @@ -744,10 +838,25 @@ func (diskBackup *SDiskBackup) DoImport(ctx context.Context, userCred mcclient.T } defer resp.Body.Close() - err = ibs.SaveBackupFrom(ctx, resp.Body, resp.ContentLength, diskBackup.Id) + err = ibs.SaveBackupFrom(ctx, resp.Body, resp.ContentLength, diskBackup.Id, "") if err != nil { return errors.Wrap(err, "SaveBackupFrom") } return nil } + +func (diskBackup SDiskBackup) ToSimpleBackup() api.SSimpleBackup { + return api.SSimpleBackup{ + Id: diskBackup.Id, + Name: diskBackup.Name, + SizeMb: diskBackup.SizeMb, + DiskSizeMb: diskBackup.DiskSizeMb, + DiskType: diskBackup.DiskType, + Status: diskBackup.Status, + EncryptKeyId: diskBackup.EncryptKeyId, + CreatedAt: diskBackup.CreatedAt, + + BackupFilePath: diskBackup.BackupFilePath, + } +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/disks.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/disks.go index 0bca7b558..995ddc665 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/disks.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/disks.go @@ -893,16 +893,21 @@ func (self *SDisk) GetSnapshotFuseUrl() (string, error) { func (self *SDisk) GetSnapshotCount() (int, error) { q := SnapshotManager.Query() - return q.Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), self.Id), - sqlchemy.Equals(q.Field("fake_deleted"), false))).CountWithError() + return q.Filter(sqlchemy.Equals(q.Field("disk_id"), self.Id)).CountWithError() } func (self *SDisk) GetManualSnapshotCount() (int, error) { return SnapshotManager.Query(). - Equals("disk_id", self.Id).Equals("fake_deleted", false). + Equals("disk_id", self.Id). Equals("created_by", api.SNAPSHOT_MANUAL).CountWithError() } +func (self *SDisk) GetAutoSnapshotCount() (int, error) { + return SnapshotManager.Query(). + Equals("disk_id", self.Id). + Equals("created_by", api.SNAPSHOT_AUTO).CountWithError() +} + func (self *SDisk) getDiskAllocateFromBackupInput(ctx context.Context, backupId string) (*api.DiskAllocateFromBackupInput, error) { ibackup, err := DiskBackupManager.FetchById(backupId) if err != nil { @@ -920,9 +925,10 @@ func (self *SDisk) getDiskAllocateFromBackupInput(ctx context.Context, backupId return &api.DiskAllocateFromBackupInput{ BackupId: backupId, BackupStorageId: bs.GetId(), - BackupStorageAccessInfo: jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict), + BackupStorageAccessInfo: accessInfo, DiskConfig: &backup.DiskConfig.DiskConfig, BackupAsTar: backup.DiskConfig.BackupAsTar, + BackupFilePath: backup.BackupFilePath, }, nil } @@ -987,36 +993,6 @@ func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SSto return driver.RequestAllocateDiskOnStorage(ctx, userCred, host, storage, self, task, input) } -// make snapshot after reset out of chain -func (self *SDisk) CleanUpDiskSnapshots(ctx context.Context, userCred mcclient.TokenCredential, snapshot *SSnapshot) error { - dest := make([]SSnapshot, 0) - query := SnapshotManager.Query() - query.Filter(sqlchemy.Equals(query.Field("disk_id"), self.Id)). - GT("created_at", snapshot.CreatedAt).Asc("created_at").All(&dest) - if len(dest) == 0 { - return nil - } - convertSnapshots := jsonutils.NewArray() - deleteSnapshots := jsonutils.NewArray() - for i := 0; i < len(dest); i++ { - if !dest[i].FakeDeleted && !dest[i].OutOfChain { - convertSnapshots.Add(jsonutils.NewString(dest[i].Id)) - } else if dest[i].FakeDeleted { - deleteSnapshots.Add(jsonutils.NewString(dest[i].Id)) - } - } - params := jsonutils.NewDict() - params.Set("convert_snapshots", convertSnapshots) - params.Set("delete_snapshots", deleteSnapshots) - task, err := taskman.TaskManager.NewTask(ctx, "DiskCleanUpSnapshotsTask", self, userCred, params, "", "", nil) - if err != nil { - return err - } else { - task.ScheduleRun(nil) - } - return nil -} - func (self *SDisk) PerformDiskReset(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.DiskResetInput) (jsonutils.JSONObject, error) { err := self.ValidateEncryption(ctx, userCred) if err != nil { @@ -2393,11 +2369,14 @@ func (self *SDisk) fetchDiskInfo(diskConfig *api.DiskConfig) error { self.Nonpersistent = true } else { if len(self.DiskType) == 0 { - diskType := api.DISK_TYPE_DATA - if diskConfig.DiskType == api.DISK_TYPE_VOLUME { - diskType = api.DISK_TYPE_VOLUME + // 无镜像的空白盘（如 pod rootfs）也要尊重传入的 disk_type； + // 否则只能依赖 ImageId 才能标成 sys，pod 两块盘都会变成 data。 + switch diskConfig.DiskType { + case api.DISK_TYPE_SYS, api.DISK_TYPE_DATA, api.DISK_TYPE_VOLUME: + self.DiskType = diskConfig.DiskType + default: + self.DiskType = api.DISK_TYPE_DATA } - self.DiskType = diskType } self.Nonpersistent = false } @@ -3018,6 +2997,19 @@ func (manager *SDiskManager) AutoDiskSnapshot(ctx context.Context, userCred mccl log.Errorf("get disk error: %v", err) continue } + snapCnt, err := disk.GetAutoSnapshotCount() + if err != nil { + log.Errorf("failed get snapshot count: %v", err) + continue + } + if snapCnt > options.Options.RetentionCountLimit { + msg := fmt.Sprintf("disk %s auto snapshot count %d more than retention count limit %d", disk.GetId(), snapCnt, options.Options.RetentionCountLimit) + log.Errorf("auto snapshot %s error: %v", disk.Name, msg) + db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, msg, userCred) + notifyclient.NotifySystemErrorWithCtx(ctx, disk.Id, disk.Name, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, msg) + continue + } + if guest := disk.GetGuest(); guest != nil { if dps, ok := guestDps[guest.Id]; ok { guestDps[guest.Id] = append(dps, disks[i]) @@ -3101,12 +3093,6 @@ func (self *SDisk) CreateSnapshotAuto( // inherit encrypt_key_id snapshot.EncryptKeyId = self.EncryptKeyId - - driver, err := storage.GetRegionDriver() - if err != nil { - return nil, errors.Wrapf(err, "GetRegionDriver") - } - snapshot.OutOfChain = driver.SnapshotIsOutOfChain(self) snapshot.VirtualSize = self.DiskSize snapshot.DiskType = self.DiskType snapshot.Location = "" @@ -3218,7 +3204,7 @@ func (self *SDisk) UpdataSnapshotsBackingDisk(backingDiskId string) error { func (self *SDisk) GetSnapshotsNotInInstanceSnapshot() ([]SSnapshot, error) { snapshots := make([]SSnapshot, 0) sq := InstanceSnapshotJointManager.Query("snapshot_id").SubQuery() - q := SnapshotManager.Query().IsFalse("fake_deleted").Equals("disk_id", self.Id) + q := SnapshotManager.Query().Equals("disk_id", self.Id) q = q.LeftJoin(sq, sqlchemy.Equals(q.Field("id"), sq.Field("snapshot_id"))). Filter(sqlchemy.IsNull(sq.Field("snapshot_id"))) err := db.FetchModelObjects(SnapshotManager, q, &snapshots) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/elasticips.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/elasticips.go index 70866b4b8..13c542b58 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/elasticips.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/elasticips.go @@ -1118,7 +1118,7 @@ func (manager *SElasticipManager) ValidateCreateData(ctx context.Context, userCr func (eip *SElasticip) GetQuotaKeys() (quotas.IQuotaKeys, error) { region, err := eip.GetRegion() - if region == nil { + if err != nil { return nil, errors.Wrapf(err, "eip.GetRegion") } return fetchRegionalQuotaKeys( diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_actions.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_actions.go index 52b882248..f51c0b3e6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_actions.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_actions.go @@ -542,7 +542,7 @@ func (self *SGuest) PerformMigrateForecast(ctx context.Context, userCred mcclien } } - schedParams := self.GetSchedMigrateParams(userCred, input) + schedParams := self.GetSchedMigrateParams(ctx, userCred, input) if input.ConvertToKvm { schedParams.Hypervisor = api.HYPERVISOR_KVM } @@ -557,12 +557,19 @@ func (self *SGuest) PerformMigrateForecast(ctx context.Context, userCred mcclien } func (self *SGuest) GetSchedMigrateParams( - userCred mcclient.TokenCredential, + ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerMigrateForecastInput, ) *schedapi.ScheduleInput { schedDesc := self.ToSchedDesc() if input.PreferHostId != "" { schedDesc.ServerConfig.PreferHost = input.PreferHostId + } else { + createParams, err := self.GetCreateParams(ctx, userCred) + if err != nil { + log.Errorf("GetSchedMigrateParams failed get create params: %s", err) + } else { + schedDesc.Schedtags = createParams.Schedtags + } } schedDesc.ResetCpuNumaPin = input.ResetCpuNumaPin @@ -925,6 +932,10 @@ func (self *SGuest) PerformDeploy( input.ResetPassword = true } + if err := ValidateDeployConfigs(input.DeployConfigs); err != nil { + return nil, err + } + driver, err := self.GetDriver() if err != nil { return nil, errors.Wrapf(err, "GetDriver") @@ -1168,6 +1179,18 @@ func (self *SGuest) StartResumeTask(ctx context.Context, userCred mcclient.Token return driver.StartResumeTask(ctx, userCred, self, nil, parentTaskId) } +func isolatedDeviceRestoreKey(devType, sharingMode, model string) string { + return devType + "\x00" + sharingMode + "\x00" + model +} + +func parseIsolatedDeviceRestoreKey(key string) (devType, sharingMode, model string, ok bool) { + segs := strings.SplitN(key, "\x00", 3) + if len(segs) != 3 { + return "", "", "", false + } + return segs[0], segs[1], segs[2], true +} + func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { config := self.GetMetadataJson(ctx, api.VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG, userCred) if config == nil { @@ -1178,21 +1201,18 @@ func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, us if err != nil { return nil, errors.Wrap(err, "unmarshal virtual dev configs") } - devs, err := self.GetIsolatedDevices() + devs, err := self.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrap(err, "GetIsolatedDevices") } devCount := map[string]int{} for i := range devConfigs { - key := devConfigs[i].DevType + "-" + devConfigs[i].Model - if cnt, ok := devCount[key]; ok { - devCount[key] = cnt + 1 - } else { - devCount[key] = 1 - } + key := isolatedDeviceRestoreKey(devConfigs[i].DevType, devConfigs[i].SharingMode, devConfigs[i].Model) + devCount[key] = devCount[key] + 1 } for i := range devs { - key := devConfigs[i].DevType + "-" + devConfigs[i].Model + isoDev := devs[i].GetIsolatedDevice() + key := isolatedDeviceRestoreKey(isoDev.DevType, isoDev.SharingMode, isoDev.Model) if cnt, ok := devCount[key]; ok { devCount[key] = cnt - 1 } @@ -1210,10 +1230,14 @@ func (self *SGuest) PerformRestoreVirtualIsolatedDevices(ctx context.Context, us if cnt <= 0 { continue } - segs := strings.SplitN(key, "-", 2) + devType, sharingMode, model, ok := parseIsolatedDeviceRestoreKey(key) + if !ok { + return nil, errors.Errorf("invalid restore key %q", key) + } devConfig := &api.IsolatedDeviceConfig{ - Model: segs[1], - DevType: segs[0], + Model: model, + DevType: devType, + SharingMode: sharingMode, } err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil) if err != nil { @@ -1397,15 +1421,20 @@ func (self *SGuest) NotifyAdminServerEvent(ctx context.Context, event string, pr notifyclient.SystemNotifyWithCtx(ctx, priority, event, kwargs) } -func (self *SGuest) StartGuestStopTask(ctx context.Context, userCred mcclient.TokenCredential, timeoutSecs int, isForce, stopCharging bool, parentTaskId string) error { +func (self *SGuest) StartGuestStopTask(ctx context.Context, userCred mcclient.TokenCredential, timeoutSecs *int, isForce, stopCharging bool, parentTaskId string) error { if len(parentTaskId) == 0 { self.SetStatus(ctx, userCred, api.VM_START_STOP, "") } params := jsonutils.NewDict() + timeout := options.Options.DefaultGuestStopTimeout if isForce { + timeout = options.Options.DefaultGuestForceStopTimeout params.Add(jsonutils.NewBool(isForce), "is_force") + } + if timeoutSecs != nil { + params.Add(jsonutils.NewInt(int64(*timeoutSecs)), "timeout") } else { - params.Add(jsonutils.NewInt(int64(timeoutSecs)), "timeout") + params.Add(jsonutils.NewInt(int64(timeout)), "timeout") } params.Add(jsonutils.NewBool(stopCharging), "stop_charging") if len(parentTaskId) > 0 { @@ -1490,6 +1519,56 @@ func (self *SGuest) GetDetailsKickstart(ctx context.Context, userCred mcclient.T return result, nil } +func (self *SGuest) PerformSetIso(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.ServerSetIsoInput) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(self.Hypervisor, []string{api.HYPERVISOR_KVM, api.HYPERVISOR_BAREMETAL}) { + return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) + } + if !utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_READY}) { + return nil, httperrors.NewServerStatusError("Set ISO not allowed in status %s", self.Status) + } + if input.ImageId != "" { + isoImage, err := parseIsoInfo(ctx, userCred, input.ImageId) + if err != nil { + return nil, errors.Wrap(err, "parseIsoInfo") + } + input.ImageId = isoImage.Id + } + + cdrom := self.getCdrom(false, input.CdromOrdinal) + if input.BootIndex != nil { + bd8 := *input.BootIndex + if cdrom == nil || cdrom.BootIndex != bd8 { + if isDup, err := self.isBootIndexDuplicated(bd8); err != nil { + return nil, err + } else if isDup { + return nil, httperrors.NewInputParameterError("boot index %d is duplicated", bd8) + } + } + } + srcImage := "" + if cdrom != nil { + srcImage = cdrom.ImageId + } + if srcImage == "" && input.ImageId == "" { + return nil, nil + } else if srcImage != "" && input.ImageId != "" { + if srcImage == input.ImageId { + return nil, nil + } + // eject && insert + err := self.StartEjectisoTask(ctx, input.CdromOrdinal, userCred, input.BootIndex, input.ImageId, "") + return nil, err + } else if srcImage == "" && input.ImageId != "" { + // insert only + err := self.StartInsertIsoTask(ctx, input.CdromOrdinal, input.ImageId, false, input.BootIndex, self.HostId, userCred, "") + return nil, err + } else { + // eject only + err := self.StartEjectisoTask(ctx, input.CdromOrdinal, userCred, nil, "", "") + return nil, err + } +} + // 挂载ISO镜像 func (self *SGuest) PerformInsertiso(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if !utils.IsInStringArray(self.Hypervisor, []string{api.HYPERVISOR_KVM, api.HYPERVISOR_BAREMETAL}) { @@ -1544,16 +1623,23 @@ func (self *SGuest) PerformEjectiso(ctx context.Context, userCred mcclient.Token return nil, httperrors.NewBadRequestError("No ISO to eject") } if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_READY}) { - err := self.StartEjectisoTask(ctx, cdromOrdinal, userCred, "") + err := self.StartEjectisoTask(ctx, cdromOrdinal, userCred, nil, "", "") return nil, err } else { return nil, httperrors.NewServerStatusError("Eject ISO not allowed in status %s", self.Status) } } -func (self *SGuest) StartEjectisoTask(ctx context.Context, cdromOrdinal int64, userCred mcclient.TokenCredential, parentTaskId string) error { +func (self *SGuest) StartEjectisoTask(ctx context.Context, cdromOrdinal int64, userCred mcclient.TokenCredential, bootIndex *int8, newImageId, parentTaskId string) error { data := jsonutils.NewDict() data.Add(jsonutils.NewInt(cdromOrdinal), "cdrom_ordinal") + if newImageId != "" { + data.Add(jsonutils.NewString(newImageId), "new_image_id") + if bootIndex != nil { + data.Add(jsonutils.NewInt(int64(*bootIndex)), "boot_index") + } + } + task, err := taskman.TaskManager.NewTask(ctx, "GuestEjectISOTask", self, userCred, data, parentTaskId, "", nil) if err != nil { return err @@ -1689,11 +1775,11 @@ func (self *SGuest) StartInsertVfdTask(ctx context.Context, floppyOrdinal int64, } func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) error { - devs, err := self.GetIsolatedDevices() + guestDevs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrap(err, "guest get isolated devices") } - if len(devs) == 0 { + if len(guestDevs) == 0 { return nil } host, err := self.GetHost() @@ -1701,16 +1787,19 @@ func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCre return errors.Wrap(err, "guest get host") } var numaNodeBalance = true - detachDevs := make([]SIsolatedDevice, 0) + detachDevs := make([]SGuestIsolatedDevice, 0) originDevConfigs := make([]api.IsolatedDeviceConfig, 0) - for i := range devs { - if utils.IsInStringArray(devs[i].DevType, api.VITRUAL_DEVICE_TYPES) { + for i := range guestDevs { + dev := guestDevs[i].GetIsolatedDevice() + if dev.SharingMode == api.DEVICE_SHARING_MODE_UNLIMITED { originDevConfigs = append(originDevConfigs, api.IsolatedDeviceConfig{ - Model: devs[i].Model, - DevType: devs[i].DevType, + Model: dev.Model, + DevType: dev.DevType, + MemoryRequest: guestDevs[i].DeviceMemorySize, + SmUtilLimit: guestDevs[i].SmUtilLimit, }) - detachDevs = append(detachDevs, devs[i]) - isBalance, err := host.VirtualDeviceNumaBalance(devs[i].DevType, devs[i].NumaNode) + detachDevs = append(detachDevs, guestDevs[i]) + isBalance, err := host.VirtualDeviceNumaBalance(dev.Model, dev.NumaNode) if err != nil { return errors.Wrap(err, "VirtualDeviceNumaBalance") } @@ -1728,20 +1817,24 @@ func (self *SGuest) RebalanceVirtualIsolatedDevices(ctx context.Context, userCre lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) + detachedDevConfigs := []*api.IsolatedDeviceConfig{} for i := 0; i < len(detachDevs); i++ { err := self.detachIsolateDevice(ctx, userCred, &detachDevs[i]) if err != nil { return errors.Wrapf(err, "detach device %s", detachDevs[i].GetId()) } + dev := detachDevs[i].GetIsolatedDevice() + detachedDevConfigs = append(detachedDevConfigs, &api.IsolatedDeviceConfig{ + Model: dev.Model, + DevType: dev.DevType, + MemoryRequest: detachDevs[i].DeviceMemorySize, + SmUtilLimit: detachDevs[i].SmUtilLimit, + }) } usedDeviceMap := map[string]*SIsolatedDevice{} - for i := range detachDevs { - devConfig := &api.IsolatedDeviceConfig{ - Model: detachDevs[i].Model, - DevType: detachDevs[i].DevType, - } - err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, devConfig, userCred, usedDeviceMap, nil) + for i := range detachedDevConfigs { + err := IsolatedDeviceManager.attachHostDeviceToGuestByModel(ctx, self, host, detachedDevConfigs[i], userCred, usedDeviceMap, nil) if err != nil { return errors.Wrap(err, "attachHostDeviceToGuestByModel") } @@ -2347,12 +2440,16 @@ func (self *SGuest) GetReleasedIsolatedDevices(ctx context.Context, userCred mcc return devs, nil } -func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { +func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SGuestIsolatedDevice) error { records := make([]api.ServerReleasedIsolatedDevice, 0) - for _, dev := range devs { + for i := range devs { + dev := devs[i].GetIsolatedDevice() record := api.ServerReleasedIsolatedDevice{ - DevType: dev.DevType, - Model: dev.Model, + DevType: dev.DevType, + Model: dev.Model, + GpuType: devs[i].GpuType, + SharingMode: dev.SharingMode, + MemoryRequest: devs[i].DeviceMemorySize, } records = append(records, record) } @@ -2362,14 +2459,14 @@ func (self *SGuest) SetReleasedIsolatedDevices(ctx context.Context, userCred mcc return nil } -func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SIsolatedDevice) error { +func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devs []SGuestIsolatedDevice) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) for i := 0; i < len(devs); i++ { // check first - dev := devs[i] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { + dev := devs[i].GetIsolatedDevice() + if !dev.IsValidAttachDev() { if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { msg := fmt.Sprintf("Can't separately detach dev type %s", dev.DevType) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) @@ -2393,12 +2490,11 @@ func (self *SGuest) DetachIsolatedDevices(ctx context.Context, userCred mcclient } // 卸载透传设备 -func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.ServerDetachIsolatedDeviceInput) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } - forceDetach := jsonutils.QueryBoolean(data, "is_force", false) - if !forceDetach { + if !input.IsForce { if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) || (self.Hypervisor == api.HYPERVISOR_POD && self.GetStatus() != api.VM_READY) { msg := fmt.Sprintf("Can't detach isolated device when guest is %s", self.GetStatus()) @@ -2413,36 +2509,52 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc } } - var detachAllDevice = jsonutils.QueryBoolean(data, "detach_all", false) - devs := make([]SIsolatedDevice, 0) - if !detachAllDevice { - device, err := data.GetString("device") - if err != nil { - msg := "Missing isolated device" - logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return nil, httperrors.NewBadRequestError("%s", msg) + devs := make([]SGuestIsolatedDevice, 0) + if !input.DetachAll { + for i := range input.Devices { + device := input.Devices[i].Device + if input.Devices[i].Index == nil { + msg := "Missing isolated device index" + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError("%s", msg) + } + index := *input.Devices[i].Index + iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) + if err != nil { + msgFmt := "Isolated device %s not found" + msg := fmt.Sprintf(msgFmt, device) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msgFmt, device) + } + dev := iDev.(*SIsolatedDevice) + gdev, err := dev.GetGuestIsolatedDevice(self.Id, index) + if err != nil { + msg := err.Error() + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError("%s", msg) + } + devs = append(devs, *gdev) } - iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) + + } else { + var err error + devs, err = self.GetGuestIsolatedDevices() if err != nil { - msgFmt := "Isolated device %s not found" - msg := fmt.Sprintf(msgFmt, device) + msg := err.Error() logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) - return nil, httperrors.NewBadRequestError(msgFmt, device) + return nil, httperrors.NewBadRequestError("%s", msg) } - devs = append(devs, *iDev.(*SIsolatedDevice)) - } else { - devs, _ = self.GetIsolatedDevices() } if err := self.DetachIsolatedDevices(ctx, userCred, devs); err != nil { return nil, err } - if forceDetach { + if input.IsForce { return nil, nil } - return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") + return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, input.AutoStart, "") } -func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, userCred mcclient.TokenCredential, device string, index int) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -2451,25 +2563,28 @@ func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, user return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - return self.DetachIsolatedDevices(ctx, userCred, []SIsolatedDevice{*dev}) + gdev, err := dev.GetGuestIsolatedDevice(self.Id, index) + if err != nil { + msgFmt := "Isolated device %s index %d not found" + msg := fmt.Sprintf(msgFmt, device, index) + logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return httperrors.NewBadRequestError(msgFmt, device) + } + return self.DetachIsolatedDevices(ctx, userCred, []SGuestIsolatedDevice{*gdev}) } -func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice) error { - if dev.GuestId != self.Id { +func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, gdev *SGuestIsolatedDevice) error { + if gdev.GuestId != self.Id { msg := "Isolated device is not attached to this guest" logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return httperrors.NewBadRequestError("%s", msg) } + dev := gdev.GetIsolatedDevice() drv, _ := self.GetDriver() - if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, dev); err != nil { - return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + if err := drv.BeforeDetachIsolatedDevice(ctx, userCred, self, gdev); err != nil { + return errors.Wrapf(err, "BeforeDetachIsolatedDevice %s of guest %s", jsonutils.Marshal(gdev), self.GetId()) } - _, err := db.Update(dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - dev.DiskIndex = -1 - return nil - }) + err := gdev.Detach(ctx, userCred) if err != nil { return err } @@ -2478,7 +2593,7 @@ func (self *SGuest) detachIsolateDevice(ctx context.Context, userCred mcclient.T } // 挂载透传设备 -func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.ServerAttachIsolatedDeviceInput) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } @@ -2489,20 +2604,29 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc return nil, httperrors.NewInvalidStatusError("%s", msg) } var err error - autoStart := jsonutils.QueryBoolean(data, "auto_start", false) - if data.Contains("device") { - device, _ := data.GetString("device") - err = self.StartAttachIsolatedDeviceGpuOrUsb(ctx, userCred, device, autoStart) - } else if data.Contains("model") { - vmodel, _ := data.GetString("model") - var count int64 = 1 - if data.Contains("count") { - count, _ = data.Int("count") + if input.GpuType != "" && !utils.IsInStringArray(input.GpuType, []string{api.GPU_VGA, api.GPU_HPC}) { + return nil, httperrors.NewInputParameterError("gpu_type %s not vaild", input.GpuType) + } + if input.MemoryRequest != nil && *input.MemoryRequest <= 0 { + return nil, httperrors.NewBadRequestError("guest attach gpu memory request count must > 0") + } + + if input.Device != "" { + err = self.StartAttachIsolatedDeviceGpuOrUsb(ctx, userCred, input.Device, input.GpuType, input.MemoryRequest, input.AutoStart) + } else if input.Model != "" { + if !utils.IsInStringArray(input.SharingMode, api.VAILD_SHARING_MODES) { + return nil, httperrors.NewInputParameterError("shaing_mode %s not valid", input.SharingMode) + } + + var count int = 1 + if input.Count != nil { + count = *input.Count } if count < 1 { return nil, httperrors.NewBadRequestError("guest attach gpu count must > 0") } - err = self.StartAttachIsolatedDevices(ctx, userCred, vmodel, int(count), autoStart) + input.Count = &count + err = self.StartAttachIsolatedDevices(ctx, userCred, input.Model, input.GpuType, input.SharingMode, count, input.MemoryRequest, input.AutoStart) } else { return nil, httperrors.NewMissingParameterError("device||model") } @@ -2513,101 +2637,104 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc return nil, nil } -func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int, autoStart bool) error { - if err := self.startAttachIsolatedDevices(ctx, userCred, devModel, count); err != nil { +func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel, gpuType, sharingMode string, count int, memoryRequest *int, autoStart bool) error { + if err := self.startAttachIsolatedDevices(ctx, userCred, devModel, gpuType, sharingMode, count, memoryRequest); err != nil { return err } // perform post attach task return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int) error { +func (self *SGuest) AttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModelCount map[string]int, gpuType, sharingMode string, memoryRequest *int) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) + attachedGpus, err := self.GetGuestIsolatedDevices() + if err != nil { + return errors.Wrap(err, "get isolated devices") + } + attachedAddrs := map[string]struct{}{} + for i := range attachedGpus { + dev := attachedGpus[i].GetIsolatedDevice() + attachedAddrs[dev.Addr] = struct{}{} + gdev := attachedGpus[i].GetIsolatedDevice() + if sharingMode == api.DEVICE_SHARING_MODE_MDEV { + if gdev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") + } else if gdev.DevType == api.GPU_TYPE { + return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") + } + } + } - unusedDevs := []SIsolatedDevice{} for devModel, count := range devModelCount { - devs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, count) + devs, err := IsolatedDeviceManager.GetAvailableIsolatedDeviceOnHost(host.Id, devModel, sharingMode) if err != nil { return httperrors.NewInternalServerError("fetch gpu failed %s", err) } - if len(devs) == 0 || len(devs) != count { + if len(devs) == 0 { return httperrors.NewBadRequestError("required %d %s isolated devices on host %s, but not enough are available", count, devModel, host.GetName()) } - dev := devs[0] - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - } - } - } - if dev.DevType == api.LEGACY_VGPU_TYPE { - attachedGpus, err := self.GetIsolatedDevices() - if err != nil { - return errors.Wrap(err, "get isolated devices") + avaDevs := make([]SIsolatedDevice, 0) + avaDevCnt := 0 + for i := range devs { + dev := devs[i] + if _, ok := attachedAddrs[dev.Addr]; ok { + continue } - for i := range attachedGpus { - if attachedGpus[i].DevType == api.LEGACY_VGPU_TYPE { - return httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1") - } else if utils.IsInStringArray(attachedGpus[i].DevType, api.VALID_GPU_TYPES) { - return httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus") + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if memoryRequest == nil || *memoryRequest <= 0 { + return httperrors.NewBadRequestError("dev sharing_mode %s memory request invalid", dev.SharingMode) } - } - } else if dev.DevType == api.CONTAINER_DEV_NVIDIA_MPS { - allDevs, err := IsolatedDeviceManager.GetUnusedDevsOnHost(host.Id, devModel, -1) - if err != nil { - return httperrors.NewInternalServerError("fetch gpu failed %s", err) - } - attachedGpus, err := self.GetIsolatedDevices() - if err != nil { - return httperrors.NewInternalServerError("get attached isolated devices %s", err) - } - attachedAddrs := map[string]struct{}{} - for i := range attachedGpus { - addr := strings.Split(attachedGpus[i].Addr, "-")[0] - attachedAddrs[addr] = struct{}{} - } - validDevs := []SIsolatedDevice{} - for i := range allDevs { - devAddr := strings.Split(allDevs[i].Addr, "-")[0] - if _, ok := attachedAddrs[devAddr]; ok { - continue + if dev.IsEnough(*memoryRequest) { + avaDevs = append(avaDevs, dev) + avaDevCnt += 1 + } + } else { + cnt, err := dev.getAllocatedCount() + if err != nil { + return errors.Wrap(err, "getAllocatedCount") + } + if dev.VirtualNum > cnt { + avaCnt := dev.VirtualNum - cnt + avaDevCnt += avaCnt + for j := 0; j < avaCnt; j++ { + avaDevs = append(avaDevs, dev) + } } - validDevs = append(validDevs, allDevs[i]) - } - if len(validDevs) < count { - return httperrors.NewInsufficientResourceError("required %d %s isolated devices on host %s, but not enough are available", count, devModel, host.GetName()) } - devs = validDevs[:count] } - unusedDevs = append(unusedDevs, devs...) - } - defer func() { go host.ClearSchedDescCache() }() - for i := 0; i < len(unusedDevs); i++ { - if err := self.attachIsolatedDevice(ctx, userCred, &unusedDevs[i], nil, nil); err != nil { - return errors.Wrapf(err, "attach device %s", unusedDevs[i].GetId()) + if avaDevCnt < count { + return httperrors.NewInsufficientResourceError("Available device count %d less then request count %d", avaDevCnt, count) + } + dev := avaDevs[0] + if !dev.HotPluggable && self.GetStatus() == api.VM_RUNNING { + return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, dev.Model) + } + + for k := 0; k < count; k++ { + if err := self.attachIsolatedDevice(ctx, userCred, &avaDevs[k], nil, nil, memoryRequest, gpuType); err != nil { + return errors.Wrapf(err, "attach device %s", avaDevs[k].GetId()) + } } } + go host.ClearSchedDescCache() return nil } -func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error { - return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count}) +func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel, gpuType, sharingMode string, count int, memoryRequest *int) error { + return self.AttachIsolatedDevices(ctx, userCred, map[string]int{devModel: count}, gpuType, sharingMode, memoryRequest) } -func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { - if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device); err != nil { +func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device, gpuType string, memoryRequest *int, autoStart bool) error { + if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device, gpuType, memoryRequest); err != nil { return err } // perform post attach task return self.StartIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") } -func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device string) error { +func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device, gpuType string, memoryRequest *int) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(ctx, userCred, device) if err != nil { msgFmt := "Isolated device %s not found" @@ -2616,29 +2743,45 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) - if !utils.IsInStringArray(dev.DevType, api.VALID_ATTACH_TYPES) { - if devModel, err := IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil { - return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType) - } else { - if !devModel.HotPluggable.Bool() && self.GetStatus() == api.VM_RUNNING { - return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, devModel.Model) - } + gdevs, err := self.GetGuestIsolatedDevices() + if err != nil { + return err + } + for i := range gdevs { + if gdevs[i].IsolatedDeviceId == dev.Id { + return httperrors.NewBadRequestError("device %s is already in used by guest %s", dev.Id, self.GetName()) } } + if !dev.HotPluggable && self.GetStatus() == api.VM_RUNNING { + return httperrors.NewBadRequestError("dev type %s model %s unhotpluggable", dev.DevType, dev.Model) + } + if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { - return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus()) + return httperrors.NewInvalidStatusError("Can't attach isolated device when status is %q", self.GetStatus()) + } + + reqCnt := 1 + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if memoryRequest == nil || *memoryRequest <= 0 { + return httperrors.NewBadRequestError("Dev sharing_mode %s memory request invalid", dev.SharingMode) + } + reqCnt = *memoryRequest + } + if !dev.IsEnough(reqCnt) { + return httperrors.NewBadRequestError("Dev %s is not enough", dev.GetName()) } - if dev.DevType == api.LEGACY_VGPU_TYPE { - devs, err := self.GetIsolatedDevices() + if dev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + devs, err := self.GetGuestIsolatedDevices() if err != nil { return errors.Wrap(err, "get isolated devices") } for i := range devs { - if devs[i].DevType == api.LEGACY_VGPU_TYPE { - return httperrors.NewBadRequestError("Nvidia vGPU count cannot exceed 1") - } else if utils.IsInStringArray(devs[i].DevType, api.VALID_GPU_TYPES) { - return httperrors.NewBadRequestError("Nvidia vGPU cannot passthrough with other gpus") + gdev := devs[i].GetIsolatedDevice() + if gdev.SharingMode == api.DEVICE_SHARING_MODE_MDEV { + return httperrors.NewBadRequestError("Nvidia vgpu count exceed > 1") + } else if gdev.DevType == api.GPU_TYPE { + return httperrors.NewBadRequestError("Nvidia vgpu can't passthrough with other gpus") } } } @@ -2646,7 +2789,7 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - err = self.attachIsolatedDevice(ctx, userCred, dev, nil, nil) + err = self.attachIsolatedDevice(ctx, userCred, dev, nil, nil, memoryRequest, gpuType) var msg string if err != nil { msg = err.Error() @@ -2657,39 +2800,53 @@ func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred return err } -func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice, networkIndex *int, diskIndex *int8) error { - if len(dev.GuestId) > 0 { - return fmt.Errorf("Isolated device already attached to another guest: %s", dev.GuestId) +func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, dev *SIsolatedDevice, networkIndex *int, diskIndex *int8, memoryRequest *int, gpuType string) error { + if dev.IsFull() { + return fmt.Errorf("Isolated device already allocated") } if dev.HostId != self.HostId { return fmt.Errorf("Isolated device and guest are not located in the same host") } + + lockman.LockObject(ctx, self) + defer lockman.ReleaseObject(ctx, self) + // guest isolated device attach + guestIsolatedDevice := SGuestIsolatedDevice{} + guestIsolatedDevice.SetModelManager(GuestIsolatedDeviceManager, &guestIsolatedDevice) + guestIsolatedDevice.GuestId = self.Id + guestIsolatedDevice.IsolatedDeviceId = dev.Id + guestIsolatedDevice.Index = self.getIsolatedDeviceIndex() + + if networkIndex != nil { + guestIsolatedDevice.NetworkIndex = *networkIndex + } + if diskIndex != nil { + guestIsolatedDevice.DiskIndex = *diskIndex + } + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI && memoryRequest != nil { + guestIsolatedDevice.DeviceMemorySize = *memoryRequest + guestIsolatedDevice.SmUtilLimit = int((float64(*memoryRequest) / float64(dev.MemorySize)) * 100.0) + } + if utils.IsInStringArray(gpuType, []string{api.GPU_HPC, api.GPU_VGA}) { + guestIsolatedDevice.GpuType = gpuType + } + drv, _ := self.GetDriver() - if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, dev); err != nil { - return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(dev), self.GetId()) + if err := drv.BeforeAttachIsolatedDevice(ctx, userCred, self, &guestIsolatedDevice); err != nil { + return errors.Wrapf(err, "BeforeAttachIsolatedDevice %s of guest %s", jsonutils.Marshal(guestIsolatedDevice), self.GetId()) } - if _, err := db.Update(dev, func() error { - dev.GuestId = self.Id - if networkIndex != nil { - dev.NetworkIndex = *networkIndex - } else { - dev.NetworkIndex = -1 - } - if diskIndex != nil { - dev.DiskIndex = *diskIndex - } else { - dev.DiskIndex = -1 - } - return nil - }); err != nil { - return errors.Wrap(err, "db.Update") + + err := GuestIsolatedDeviceManager.TableSpec().Insert(ctx, &guestIsolatedDevice) + if err != nil { + return err } + db.OpsLog.LogEvent(self, db.ACT_GUEST_ATTACH_ISOLATED_DEVICE, dev.GetShortDesc(ctx), userCred) return nil } // 设置透传设备 -func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input *api.SetIsolatedDeviceInput) (jsonutils.JSONObject, error) { if self.Hypervisor != api.HYPERVISOR_KVM && self.Hypervisor != api.HYPERVISOR_POD { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } @@ -2697,46 +2854,27 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli (self.Hypervisor == api.HYPERVISOR_POD && self.GetStatus() != api.VM_READY) { return nil, httperrors.NewInvalidStatusError("Can't set isolated device when guest is %s", self.GetStatus()) } - var addDevs []string - { - addDevices, err := data.Get("add_devices") - if err == nil { - arrAddDev, ok := addDevices.(*jsonutils.JSONArray) - if ok { - addDevs = arrAddDev.GetStringArray() - } else { - return nil, httperrors.NewInputParameterError("attach devices is not string array") - } - } - } - - var delDevs []string - { - delDevices, err := data.Get("del_devices") - if err == nil { - arrDelDev, ok := delDevices.(*jsonutils.JSONArray) - if ok { - delDevs = arrDelDev.GetStringArray() - } else { - return nil, httperrors.NewInputParameterError("detach devices is not string array") - } - } - } + var addDevs = input.AddDevices + var delDevs = input.DelDevices // detach first for i := 0; i < len(delDevs); i++ { - err := self.startDetachIsolateDeviceWithoutNic(ctx, userCred, delDevs[i]) + err := self.startDetachIsolateDeviceWithoutNic(ctx, userCred, delDevs[i].Device, delDevs[i].Index) if err != nil { return nil, err } } for i := 0; i < len(addDevs); i++ { - err := self.startAttachIsolatedDevGeneral(ctx, userCred, addDevs[i]) + if addDevs[i].GpuType != "" && !utils.IsInStringArray(addDevs[i].GpuType, []string{api.GPU_VGA, api.GPU_HPC}) { + return nil, httperrors.NewInputParameterError("gpu_type %s not vaild", addDevs[i].GpuType) + } + + err := self.startAttachIsolatedDevGeneral(ctx, userCred, addDevs[i].Device, addDevs[i].GpuType, addDevs[i].MemoryRequest) if err != nil { return nil, err } } - return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") + return nil, self.StartIsolatedDevicesSyncTask(ctx, userCred, input.AutoStart, "") } func (self *SGuest) StartIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parentId string) error { @@ -2753,7 +2891,8 @@ func (self *SGuest) StartIsolatedDevicesSyncTask(ctx context.Context, userCred m } } -func (self *SGuest) findGuestnetworkByInfo(info api.ServerNetworkInfo) (*SGuestnetwork, error) { +// FindGuestnetworkByInfo 根据 ip / ip6 / mac / index 定位虚机网卡 +func (self *SGuest) FindGuestnetworkByInfo(info api.ServerNetworkInfo) (*SGuestnetwork, error) { if len(info.IpAddr) > 0 { gn, err := self.GetGuestnetworkByIp(info.IpAddr) if err != nil { @@ -2824,9 +2963,9 @@ func (self *SGuest) PerformChangeIpaddr( reserve := (input.Reserve != nil && *input.Reserve) - gn, err := self.findGuestnetworkByInfo(input.ServerNetworkInfo) + gn, err := self.FindGuestnetworkByInfo(input.ServerNetworkInfo) if err != nil { - return nil, errors.Wrap(err, "findGuestnetworkByInfo") + return nil, errors.Wrap(err, "FindGuestnetworkByInfo") } var conf *api.NetworkConfig @@ -3251,6 +3390,9 @@ func (self *SGuest) PerformAttachnetwork( if err != nil { return nil, err } + if len(input.Nets[i].PortMappings) > 0 && !self.SupportPortMapping() { + return nil, httperrors.NewUnsupportOperationError("hypervisor %s does not support port_mapping", self.Hypervisor) + } if IsExitNetworkInfo(ctx, userCred, input.Nets[i]) { enicCnt += 1 // ebw = input.BwLimit @@ -3396,9 +3538,9 @@ func (guest *SGuest) PerformChangeBandwidth( return nil, httperrors.NewBadRequestError("Bandwidth, tx_bw_limit and rx_bw_limit must be non-negative") } - guestnic, err := guest.findGuestnetworkByInfo(input.ServerNetworkInfo) + guestnic, err := guest.FindGuestnetworkByInfo(input.ServerNetworkInfo) if err != nil { - return nil, errors.Wrap(err, "findGuestnetworkByInfo") + return nil, errors.Wrap(err, "FindGuestnetworkByInfo") } if guestnic.BwLimit != int(input.Bandwidth) || guestnic.TxBwLimit != int(input.TxBwLimit) || guestnic.RxBwLimit != int(input.RxBwLimit) { @@ -3430,6 +3572,45 @@ func (guest *SGuest) PerformChangeBandwidth( return nil, nil } +// Set port mappings of a guest nic +// 由独立的 GuestSetPortMappingTask 完成：先请求宿主机设置/分配 host_port，再同步配置到宿主机 +// 仅 kvm / pod 支持 +func (self *SGuest) PerformSetPortMapping( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.ServerSetPortMappingInput, +) (jsonutils.JSONObject, error) { + if !self.SupportPortMapping() { + return nil, httperrors.NewUnsupportOperationError("hypervisor %s does not support port_mapping", self.Hypervisor) + } + if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) { + return nil, httperrors.NewBadRequestError("Cannot set port mapping in status %s", self.Status) + } + for i := range input.PortMappings { + if err := validatePortMapping(input.PortMappings[i]); err != nil { + return nil, errors.Wrapf(err, "validate port mapping %s", jsonutils.Marshal(input.PortMappings[i])) + } + } + // 预先确认网卡存在，避免启动任务后才报错 + if _, err := self.FindGuestnetworkByInfo(input.ServerNetworkInfo); err != nil { + return nil, errors.Wrap(err, "FindGuestnetworkByInfo") + } + return nil, self.StartGuestSetPortMappingTask(ctx, userCred, input) +} + +func (self *SGuest) StartGuestSetPortMappingTask(ctx context.Context, userCred mcclient.TokenCredential, input api.ServerSetPortMappingInput) error { + // 先置进行中状态，让列表与操作入口立刻可见 + if err := self.SetStatus(ctx, userCred, api.VM_SET_PORTMAPPING, "set port mappings"); err != nil { + return errors.Wrap(err, "SetStatus") + } + task, err := taskman.TaskManager.NewTask(ctx, "GuestSetPortMappingTask", self, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), "", "") + if err != nil { + return errors.Wrap(err, "New GuestSetPortMappingTask") + } + return task.ScheduleRun(nil) +} + // 修改源地址检查 func (self *SGuest) PerformModifySrcCheck(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) { @@ -3486,6 +3667,16 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T if !utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_READY}) { return nil, httperrors.NewInvalidStatusError("Cannot change config in status %s", self.Status) } + + if self.RescueMode { + return nil, httperrors.NewInvalidStatusError("Cannot change config in rescue mode") + } + + // 停止计费模式下, 不允许调整配置 + if len(self.ExternalId) > 0 && self.ShutdownMode == api.VM_SHUTDOWN_MODE_STOP_CHARGING && self.Status == api.VM_READY { + return nil, httperrors.NewInvalidStatusError("Cannot change config in ready status when shutdown mode is stop charging") + } + driver, err := self.GetDriver() if err != nil { return nil, err @@ -3513,12 +3704,20 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T if err != nil { return nil, err } - if !runningOk && !input.ForceStop { + // 仅在不支持开机变配, 或开机降配时需要强制关机,或ARM架构时需要强制关机; force_stop 表示允许关机, 而非一定关机 + needForceStop := !runningOk || confs.ConfigReduced() || apis.IsARM(self.OsArch) + if needForceStop && !input.ForceStop { return nil, httperrors.NewInvalidStatusError("Cannot change config in %s for %s, requires force_stop to change config", self.Status, self.GetHypervisor()) } + if needForceStop { + // 走离线变配路径, 变配完成后自动启动 + confs.ForceStop = true + confs.GuestOnline = false + confs.AutoStart = true + } } - if self.PowerStates == api.VM_POWER_STATES_ON && (confs.CpuChanged() || confs.MemChanged()) { + if self.PowerStates == api.VM_POWER_STATES_ON && (confs.CpuChanged() || confs.MemChanged()) && !confs.ForceStop { confs, err = driver.ValidateGuestHotChangeConfigInput(ctx, self, confs) if err != nil { return nil, httperrors.NewInvalidStatusError("cannot change CPU/Memory spec in power status %s: %s", self.PowerStates, err) @@ -3569,7 +3768,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T func (self *SGuest) ChangeConfToSchedDesc(addCpu, addExtraCpu, addMem int, schedInputDisks []*api.DiskConfig) *schedapi.ScheduleInput { region, _ := self.GetRegion() - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() desc := &schedapi.ScheduleInput{ ServerConfig: schedapi.ServerConfig{ ServerConfigs: &api.ServerConfigs{ @@ -3646,6 +3845,21 @@ func (self *SGuest) DoCancelPendingDelete(ctx context.Context, userCred mcclient disk.DoCancelPendingDelete(ctx, userCred) } + instanceSnapshots, err := self.GetInstanceSnapshots() + if err != nil { + return err + } + for i := range instanceSnapshots { + instanceSnapshots[i].DoCancelPendingDelete(ctx, userCred) + } + snapshots, err := self.GetDiskSnapshotsNotInInstanceSnapshots(true) + if err != nil { + return err + } + for i := range snapshots { + snapshots[i].DoCancelPendingDelete(ctx, userCred) + } + if self.BillingType == billing_api.BILLING_TYPE_POSTPAID && !self.ExpiredAt.IsZero() { err := SaveReleaseAt(ctx, self, userCred, time.Time{}) if err != nil { @@ -3981,14 +4195,13 @@ func (self *SGuest) StartGuestStopAndFreezeTask(ctx context.Context, userCred mc } // 重启 -func (self *SGuest) PerformRestart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - isForce := jsonutils.QueryBoolean(data, "is_force", false) - if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_STOP_FAILED, api.VM_KICKSTART_INSTALLING, api.VM_KICKSTART_FAILED, api.VM_KICKSTART_COMPLETED}) || (isForce && self.Status == api.VM_STOPPING) { +func (self *SGuest) PerformRestart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerRestartInput) (jsonutils.JSONObject, error) { + if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_STOP_FAILED, api.VM_KICKSTART_INSTALLING, api.VM_KICKSTART_FAILED, api.VM_KICKSTART_COMPLETED}) || (input.IsForce && self.Status == api.VM_STOPPING) { driver, err := self.GetDriver() if err != nil { return nil, err } - return nil, driver.StartGuestRestartTask(self, ctx, userCred, isForce, "") + return nil, driver.StartGuestRestartTask(self, ctx, userCred, input.IsForce, input.TimeoutSecs, "") } return nil, httperrors.NewInvalidStatusError("Cannot do restart server in status %s", self.Status) } @@ -4644,7 +4857,7 @@ func (self *SGuest) PerformCreateBackup( if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewBadRequestError("Backup only support hypervisor kvm") } - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() if len(devs) > 0 { return nil, httperrors.NewBadRequestError("Cannot create backup with isolated devices") } @@ -4859,15 +5072,18 @@ func (self *SGuest) PerformPostpaidExpire(ctx context.Context, userCred mcclient } // 续费 -func (self *SGuest) PerformRenew(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - durationStr, _ := data.GetString("duration") - if len(durationStr) == 0 { - return nil, httperrors.NewInputParameterError("missong duration") +func (self *SGuest) PerformRenew(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.RenewInput) (jsonutils.JSONObject, error) { + if self.BillingType != billing_api.BILLING_TYPE_PREPAID { + return nil, httperrors.NewUnsupportOperationError("Only %s guest support renew operation", billing_api.BILLING_TYPE_PREPAID) } - bc, err := billing.ParseBillingCycle(durationStr) + if len(input.Duration) == 0 { + return nil, httperrors.NewMissingParameterError("duration") + } + + bc, err := billing.ParseBillingCycle(input.Duration) if err != nil { - return nil, httperrors.NewInputParameterError("invalid duration %s: %s", durationStr, err) + return nil, httperrors.NewInputParameterError("invalid duration %s: %s", input.Duration, err) } driver, err := self.GetDriver() @@ -4876,10 +5092,10 @@ func (self *SGuest) PerformRenew(ctx context.Context, userCred mcclient.TokenCre } if !driver.IsSupportedBillingCycle(bc) { - return nil, httperrors.NewInputParameterError("unsupported duration %s", durationStr) + return nil, httperrors.NewInputParameterError("unsupported duration %s", input.Duration) } - err = self.startGuestRenewTask(ctx, userCred, durationStr, "") + err = self.startGuestRenewTask(ctx, userCred, input.Duration, "") if err != nil { return nil, err } @@ -5264,9 +5480,10 @@ func (self *SGuest) GenerateVirtInstallCommandLine( } // isolated devices - isolatedDevices, _ := self.GetIsolatedDevices() + isolatedDevices, _ := self.GetGuestIsolatedDevices() for _, isolatedDev := range isolatedDevices { - cmd += L(fmt.Sprintf("--hostdev %s", isolatedDev.Addr)) + dev := isolatedDev.GetIsolatedDevice() + cmd += L(fmt.Sprintf("--hostdev %s", dev.Addr)) } if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_BLOCK_STREAM}) { @@ -5532,7 +5749,7 @@ func (self *SGuest) validateForBatchMigrate(ctx context.Context, rescueMode bool if len(guest.BackupHostId) > 0 { return guest, httperrors.NewBadRequestError("guest %s has backup, can't migrate", guest.Name) } - devs, _ := guest.GetIsolatedDevices() + devs, _ := guest.GetGuestIsolatedDevices() if len(devs) > 0 { return guest, httperrors.NewBadRequestError("guest %s has isolated device, can't migrate", guest.Name) } @@ -5930,7 +6147,6 @@ func (self *SGuest) StartSnapshotResetTask(ctx context.Context, userCred mcclien data.Add(jsonutils.NewBool(withMemory), "with_memory") self.SetStatus(ctx, userCred, api.VM_START_SNAPSHOT_RESET, "start snapshot reset task") instanceSnapshot.SetStatus(ctx, userCred, api.INSTANCE_SNAPSHOT_RESET, "start snapshot reset task") - log.Errorf("====data: %s", data) if task, err := taskman.TaskManager.NewTask( ctx, "InstanceSnapshotResetTask", instanceSnapshot, userCred, data, "", "", nil, ); err != nil { @@ -5984,6 +6200,9 @@ func (self *SGuest) PerformSnapshotAndClone( input.ServerCreateSnapshotParams = params // set guest pending usage pendingUsage, pendingRegionUsage, err := self.getGuestUsage(count) + if err != nil { + return nil, errors.Wrap(err, "getGuestUsage") + } keys, err := self.GetQuotaKeys() if err != nil { quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false) @@ -6108,8 +6327,12 @@ func (self *SGuest) GetDetailsJnlp(ctx context.Context, userCred mcclient.TokenC return host.GetDetailsJnlp(ctx, userCred, query) } -func (guest *SGuest) StartDeleteGuestSnapshots(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { - task, err := taskman.TaskManager.NewTask(ctx, "GuestDeleteSnapshotsTask", guest, userCred, nil, parentTaskId, "", nil) +func (guest *SGuest) StartDeleteGuestSnapshots(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, deletePendingSnapshots bool) error { + data := jsonutils.NewDict() + if deletePendingSnapshots { + data.Add(jsonutils.JSONTrue, "delete_pending_snapshots") + } + task, err := taskman.TaskManager.NewTask(ctx, "GuestDeleteSnapshotsTask", guest, userCred, data, parentTaskId, "", nil) if err != nil { return err } @@ -6663,20 +6886,52 @@ func (self *SGuest) PerformChangeDiskDriver(ctx context.Context, userCred mcclie if gd == nil { return nil, httperrors.NewBadRequestError("failed get guest disk by disk id %s", input.DiskId) } - if input.Driver == gd.Driver { - return nil, nil + var driverChanged = false + if input.Driver != "" { + if input.Driver != gd.Driver { + driverChanged = true + } + if !utils.IsInStringArray(input.Driver, []string{api.DISK_DRIVER_VIRTIO, api.DISK_DRIVER_PVSCSI, api.DISK_DRIVER_IDE, api.DISK_DRIVER_SCSI}) { + return nil, httperrors.NewInputParameterError("unknown driver %s", input.Driver) + } } - if !utils.IsInStringArray(input.Driver, []string{api.DISK_DRIVER_VIRTIO, api.DISK_DRIVER_PVSCSI, api.DISK_DRIVER_IDE, api.DISK_DRIVER_SCSI}) { - return nil, httperrors.NewInputParameterError("unknown driver %s", input.Driver) + if input.CacheMode != "" && gd.CacheMode != input.CacheMode { + if input.CacheMode != "none" { + input.AioMode = "threads" + } + if !utils.IsInStringArray(input.CacheMode, []string{api.DISK_CACHE_MODE_WRITETHROGH, api.DISK_CACHE_MODE_DIRECTSYNC, api.DISK_CACHE_MODE_WRITEBACK, api.DISK_CACHE_MODE_NONE}) { + return nil, httperrors.NewInputParameterError("unknown cache_mode %s", input.CacheMode) + } } + if input.AioMode != "" && gd.AioMode != input.AioMode { + if !utils.IsInStringArray(input.AioMode, []string{api.DISK_AIO_MODE_NATIVE, api.DISK_AIO_MOD_THREADS}) { + return nil, httperrors.NewInputParameterError("unknown aio_mode %s", input.AioMode) + } + cacheMode := gd.CacheMode + if input.CacheMode != "" { + cacheMode = input.CacheMode + } + if input.AioMode == "native" && cacheMode != "none" { + return nil, httperrors.NewBadRequestError("AIO mode %s with cache mode %s is not supported", input.AioMode, cacheMode) + } + } + _, err := db.Update(gd, func() error { - gd.Driver = input.Driver + if input.Driver != "" { + gd.Driver = input.Driver + } + if input.AioMode != "" { + gd.AioMode = input.AioMode + } + if input.CacheMode != "" { + gd.CacheMode = input.CacheMode + } return nil }) if err != nil { return nil, errors.Wrap(err, "failed update disk driver") } - if self.Bios == api.VM_BOOT_MODE_UEFI { + if driverChanged && self.Bios == api.VM_BOOT_MODE_UEFI { drv, err := self.GetDriver() if err != nil { return nil, err @@ -6937,7 +7192,7 @@ func (self *SGuest) PerformProbeIsolatedDevices(ctx context.Context, userCred mc return nil, errors.Wrapf(err, "FetchById %q", id) } dev := devObj.(*SIsolatedDevice) - if dev.GuestId == "" { + if !dev.IsFull() { devs = append(devs, dev) } } @@ -7075,12 +7330,13 @@ func (self *SGuest) GetDetailsCpusetCores(ctx context.Context, userCred mcclient func (self *SGuest) GetDetailsNumaInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *api.ServerGetNumaInfoInput) (*api.ServerGetNumaInfoResp, error) { ret := new(api.ServerGetNumaInfoResp) - devs, _ := self.GetIsolatedDevices() + devs, _ := self.GetGuestIsolatedDevices() if len(devs) > 0 { ret.IsolatedDevicesNumaNode = make([]int8, 0) for i := range devs { - if devs[i].NumaNode > 0 { - ret.IsolatedDevicesNumaNode = append(ret.IsolatedDevicesNumaNode, devs[i].NumaNode) + dev := devs[i].GetIsolatedDevice() + if dev.NumaNode > 0 { + ret.IsolatedDevicesNumaNode = append(ret.IsolatedDevicesNumaNode, dev.NumaNode) } } } @@ -7146,12 +7402,13 @@ func (self *SGuest) GetDetailsHardwareInfo(ctx context.Context, userCred mcclien } // fill GPU info - devs, err := self.GetIsolatedDevices() + devs, err := self.GetGuestIsolatedDevices() if err != nil { return nil, errors.Wrap(err, "get isolated devices") } gpuInfos := make([]*api.ServerHardwareInfoGPU, 0) - for _, dev := range devs { + for i := range devs { + dev := devs[i].GetIsolatedDevice() if !dev.IsGPU() { continue } @@ -7197,6 +7454,14 @@ func (self *SGuest) PerformEnableMemclean(ctx context.Context, userCred mcclient return nil, self.SetMetadata(ctx, api.VM_METADATA_ENABLE_MEMCLEAN, "true", userCred) } +func (self *SGuest) PerformSetQemuVersion(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if data.Contains(api.VM_METADATA_QEMU_VERSION) { + qemuVersion, _ := data.GetString(api.VM_METADATA_QEMU_VERSION) + return nil, self.SetMetadata(ctx, api.VM_METADATA_QEMU_VERSION, qemuVersion, userCred) + } + return nil, self.SetMetadata(ctx, api.VM_METADATA_QEMU_VERSION, "", userCred) +} + func (self *SGuest) PerformSetTpm(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { enableTpm := jsonutils.QueryBoolean(data, api.VM_METADATA_ENABLE_TPM, false) if enableTpm { @@ -7374,7 +7639,7 @@ func (self *SGuest) PerformSetKickstart(ctx context.Context, userCred mcclient.T return nil, errors.Wrap(err, "get driver for restart") } - if err := driver.StartGuestRestartTask(self, ctx, userCred, false, "kickstart config updated"); err != nil { + if err := driver.StartGuestRestartTask(self, ctx, userCred, true, nil, "kickstart config updated"); err != nil { return nil, errors.Wrap(err, "start restart task") } @@ -7420,7 +7685,7 @@ func (self *SGuest) PerformKickstartComplete(ctx context.Context, userCred mccli if err != nil { return nil, errors.Wrap(err, "get driver") } - driver.StartGuestRestartTask(self, ctx, userCred, false, "") + driver.StartGuestRestartTask(self, ctx, userCred, true, nil, "") return jsonutils.Marshal(map[string]string{ "status": "success", "message": "kickstart marked as completed, restarting VM", diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_convert.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_convert.go index 28c0cbc9c..cf8ba88cf 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_convert.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_convert.go @@ -16,6 +16,7 @@ package models import ( "context" + "database/sql" "fmt" "yunion.io/x/jsonutils" @@ -168,9 +169,119 @@ func (self *SGuest) StartConvertToKvmTask( } } +func isConvertSysDisk(disk *api.DiskConfig) bool { + if disk.DiskType == api.DISK_TYPE_SYS { + return true + } + if len(disk.DiskType) > 0 { + return false + } + return disk.Index == 0 +} + +func fetchPreferStorageId(ctx context.Context, userCred mcclient.TokenCredential, preferStorage string) (string, error) { + if len(preferStorage) == 0 { + return "", nil + } + storageObj, err := StorageManager.FetchByIdOrName(ctx, userCred, preferStorage) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return "", httperrors.NewResourceNotFoundError2(StorageManager.Keyword(), preferStorage) + } + return "", errors.Wrapf(err, "StorageManager.FetchByIdOrName %s", preferStorage) + } + return storageObj.GetId(), nil +} + +type convertDiskPrefer struct { + backend string + storageId string + medium string + schedtags []*api.SchedtagConfig +} + +func convertDiskTypePrefer(disk *api.DiskConfig, data *api.ConvertToKvmInput, sysStorageId, dataStorageId string) convertDiskPrefer { + if isConvertSysDisk(disk) { + return convertDiskPrefer{ + backend: data.SysDiskBackend, + storageId: sysStorageId, + medium: data.SysDiskMedium, + schedtags: data.SysDiskSchedtags, + } + } + return convertDiskPrefer{ + backend: data.DataDiskBackend, + storageId: dataStorageId, + medium: data.DataDiskMedium, + schedtags: data.DataDiskSchedtags, + } +} + +// applyConvertDiskConfigs applies target storage preference for convert-to-kvm. +// Priority: per-disk Disks configs > sys/data DiskBackend / PreferStorage / Medium / DiskSchedtags. +// When nothing is specified, disks keep cleared Backend/Storage (scheduler default, usually local). +func applyConvertDiskConfigs(ctx context.Context, userCred mcclient.TokenCredential, disks []*api.DiskConfig, data *api.ConvertToKvmInput) error { + if data == nil || len(disks) == 0 { + return nil + } + + if data.Disks != nil && len(data.Disks) != len(disks) { + return httperrors.NewInputParameterError("input disk configs length must equal guest disks length") + } + + sysPreferStorageId, err := fetchPreferStorageId(ctx, userCred, data.SysPreferStorage) + if err != nil { + return err + } + dataPreferStorageId, err := fetchPreferStorageId(ctx, userCred, data.DataPreferStorage) + if err != nil { + return err + } + + for i := range disks { + prefer := convertDiskTypePrefer(disks[i], data, sysPreferStorageId, dataPreferStorageId) + var perDisk *api.DiskConfig + if data.Disks != nil { + perDisk = data.Disks[i] + } + + if perDisk != nil && len(perDisk.Backend) > 0 { + disks[i].Backend = perDisk.Backend + } else if len(prefer.backend) > 0 { + disks[i].Backend = prefer.backend + } + + if perDisk != nil && len(perDisk.Storage) > 0 { + id, err := fetchPreferStorageId(ctx, userCred, perDisk.Storage) + if err != nil { + return err + } + disks[i].Storage = id + } else if len(prefer.storageId) > 0 { + disks[i].Storage = prefer.storageId + } + + if perDisk != nil && len(perDisk.Medium) > 0 { + disks[i].Medium = perDisk.Medium + } else if len(prefer.medium) > 0 { + disks[i].Medium = prefer.medium + } + + if perDisk != nil && perDisk.Schedtags != nil { + disks[i].Schedtags = perDisk.Schedtags + } else if prefer.schedtags != nil { + disks[i].Schedtags = prefer.schedtags + } + } + return nil +} + func (self *SGuest) createConvertedServer(ctx context.Context, userCred mcclient.TokenCredential, data *api.ConvertToKvmInput) (*SGuest, *api.ServerCreateInput, error) { // set guest pending usage pendingUsage, pendingRegionUsage, err := self.getGuestUsage(1) + if err != nil { + return nil, nil, errors.Wrap(err, "getGuestUsage") + } keys, err := self.GetQuotaKeys() if err != nil { return nil, nil, errors.Wrap(err, "GetQuotaKeys") @@ -210,6 +321,7 @@ func (self *SGuest) createConvertedServer(ctx context.Context, userCred mcclient createInput.Disks[i].Format = "" createInput.Disks[i].Backend = "" createInput.Disks[i].Medium = "" + createInput.Disks[i].Storage = "" } gns, err := self.GetNetworks("") if err != nil { @@ -229,6 +341,12 @@ func (self *SGuest) createConvertedServer(ctx context.Context, userCred mcclient createInput.Disks[0].ImageId = "" } + err = applyConvertDiskConfigs(ctx, userCred, createInput.Disks, data) + if err != nil { + quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false) + return nil, nil, errors.Wrap(err, "applyConvertDiskConfigs") + } + if data.Networks != nil && len(data.Networks) != len(createInput.Networks) { return nil, nil, httperrors.NewInputParameterError("input network configs length must equal guestnetworks length") } @@ -246,11 +364,20 @@ func (self *SGuest) createConvertedServer(ctx context.Context, userCred mcclient } schedDesc := self.ToSchedDesc() + // convert creates a new guest; do not treat as migrate (HostId would force shared + // backends to require an existing storage_id accessible on the candidate host) + schedDesc.HostId = "" schedDesc.PreferHost = data.PreferHost for i := range schedDesc.Disks { schedDesc.Disks[i].Backend = "" schedDesc.Disks[i].Medium = "" schedDesc.Disks[i].Storage = "" + schedDesc.Disks[i].DiskId = "" + } + err = applyConvertDiskConfigs(ctx, userCred, schedDesc.Disks, data) + if err != nil { + quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false) + return nil, nil, errors.Wrap(err, "applyConvertDiskConfigs schedDesc") } schedDesc.Networks = data.Networks schedDesc.Hypervisor = api.HYPERVISOR_KVM diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_deploy.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_deploy.go new file mode 100644 index 000000000..396e3b904 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_deploy.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or authorized to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/pkg/util/sets" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/util/fileutils2" +) + +var deployConfigActions = sets.NewString("", "create", "append") + +func ValidateDeployConfigs(deploys []*api.DeployConfig) error { + for i, d := range deploys { + if d == nil { + continue + } + if !deployConfigActions.Has(d.Action) { + return httperrors.NewInputParameterError("invalid deploy action %q", d.Action) + } + clean, err := fileutils2.CleanGuestDeployPath(d.Path) + if err != nil { + return httperrors.NewInputParameterError("deploy_configs[%d].path: %v", i, err) + } + d.Path = clean + } + return nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_isolated_devices.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_isolated_devices.go new file mode 100644 index 000000000..4bcf05178 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_isolated_devices.go @@ -0,0 +1,319 @@ +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// +onecloud:swagger-gen-ignore +type SGuestIsolatedDeviceManager struct { + SGuestJointsManager + SIsolatedDeviceResourceBaseManager +} + +var GuestIsolatedDeviceManager *SGuestIsolatedDeviceManager + +func init() { + db.InitManager(func() { + GuestIsolatedDeviceManager = &SGuestIsolatedDeviceManager{ + SGuestJointsManager: NewGuestJointsManager( + SGuestIsolatedDevice{}, + "guestisolateddevices_tbl", + "guestisolateddevice", + "guestisolateddevices", + IsolatedDeviceManager, + ), + } + GuestIsolatedDeviceManager.SetVirtualObject(GuestIsolatedDeviceManager) + GuestIsolatedDeviceManager.TableSpec().AddIndex(false, "isolated_device_id", "guest_id") + }) +} + +// +onecloud:model-api-gen +type SGuestIsolatedDevice struct { + SGuestJointsBase + SIsolatedDeviceResourceBase + + // guest isolated device memory size limit + DeviceMemorySize int `nullable:"true" default:"0" list:"user" update:"user" create:"optional"` + // guest isolated device Streaming Multiprocessor Utilization limit + SmUtilLimit int `nullable:"true" default:"0" list:"user" update:"user" create:"optional"` + // gpu device work type: HPC VGA + GpuType string `width:"16" charset:"ascii" nullable:"true" default:"" index:"true" list:"user" create:"optional" update:"user"` + + // guest network index + NetworkIndex int `nullable:"true" default:"-1" list:"user"` + // guest disk index + DiskIndex int8 `nullable:"true" default:"-1" list:"user"` + + Index int8 `nullable:"false" default:"0" list:"user"` +} + +func (manager *SGuestIsolatedDeviceManager) GetSlaveFieldName() string { + return "isolated_device_id" +} + +func (self *SGuestIsolatedDevice) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DeleteModel(ctx, userCred, self) +} + +func (self *SGuestIsolatedDevice) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { + return db.DetachJoint(ctx, userCred, self) +} + +func (manager *SGuestIsolatedDeviceManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = manager.SGuestJointsManager.ListItemFilter(ctx, q, userCred, query.GuestJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemFilter") + } + if len(query.IsolateDeviceIds) > 0 { + query.IsolatedDeviceListInput.Ids = query.IsolateDeviceIds + } + q, err = manager.SIsolatedDeviceResourceBaseManager.ListItemFilter(ctx, q, userCred, query) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemFilter") + } + return q, nil +} + +func (manager *SGuestIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SGuestJointsManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.ListItemExportKeys") + } + if keys.ContainsAny(manager.SIsolatedDeviceResourceBaseManager.GetExportKeys()...) { + q, err = manager.SIsolatedDeviceResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SIsolatedDeviceManager.ListItemExportKeys") + } + } + + return q, nil +} + +func (manager *SGuestIsolatedDeviceManager) CreateByInsertOrUpdate() bool { + return false +} + +func (manager *SGuestIsolatedDeviceManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.GuestIsolatedDeviceDetails { + rows := make([]api.GuestIsolatedDeviceDetails, len(objs)) + guestRows := manager.SGuestJointsManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + isolatedDeviceRows := manager.SIsolatedDeviceResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for i := range rows { + rows[i].GuestJointResourceDetails = guestRows[i] + rows[i].SIsolatedDevice = isolatedDeviceRows[i].SIsolatedDevice + rows[i].HostResourceInfo = isolatedDeviceRows[i].HostResourceInfo + rows[i].SharableResourceBaseInfo = isolatedDeviceRows[i].SharableResourceBaseInfo + rows[i].Vendor = isolatedDeviceRows[i].Vendor + rows[i].MemoryAllocated = isolatedDeviceRows[i].MemoryAllocated + rows[i].AllocatedCount = isolatedDeviceRows[i].AllocatedCount + } + + return rows +} + +func (manager *SGuestIsolatedDeviceManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + var err error + + q, err = manager.SGuestJointsManager.OrderByExtraFields(ctx, q, userCred, query.GuestJointsListInput) + if err != nil { + return nil, errors.Wrap(err, "SGuestJointsManager.OrderByExtraFields") + } + q, err = manager.SIsolatedDeviceResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query) + if err != nil { + return nil, errors.Wrap(err, "SNetworkResourceBaseManager.OrderByExtraFields") + } + + return q, nil +} + +func (dev *SIsolatedDevice) getAllocatedMemorySize() (int, error) { + sq := GuestIsolatedDeviceManager.Query(). + Equals("isolated_device_id", dev.Id). + SubQuery() + + q := sq.Query( + sqlchemy.SUM("memory_used", sq.Field("device_memory_size")), + ) + + var result struct { + MemoryUsed int + } + + err := q.First(&result) + if err != nil { + return 0, err + } + return result.MemoryUsed, nil +} + +func (dev *SIsolatedDevice) getAllocatedCount() (int, error) { + return GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", dev.Id).CountWithError() +} + +func (dev *SIsolatedDevice) getAttachedGuestIds() []string { + q := GuestIsolatedDeviceManager.Query("guest_id").Equals("isolated_device_id", dev.Id) + type sGuestId struct { + GuestId string + } + + guestIsolatedDevices := make([]sGuestId, 0) + err := q.All(&guestIsolatedDevices) + if err != nil { + log.Errorf("failed get guest isolated devices %s", err) + return nil + } + ret := make([]string, len(guestIsolatedDevices)) + for i := range guestIsolatedDevices { + ret[i] = guestIsolatedDevices[i].GuestId + } + return ret +} + +func (dev *SIsolatedDevice) getAttachedGuests() []SGuestIsolatedDevice { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", dev.Id) + guestIsolatedDevices := make([]SGuestIsolatedDevice, 0) + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &guestIsolatedDevices) + if err != nil { + log.Errorf("failed get guest isolated devices %s", err) + return nil + } + return guestIsolatedDevices +} + +func (guest *SGuest) GetIsolatedDevicesQuery() *sqlchemy.SQuery { + return GuestIsolatedDeviceManager.Query().Equals("guest_id", guest.Id) +} + +func (guest *SGuest) GetGuestIsolatedDevices() ([]SGuestIsolatedDevice, error) { + gdevs := make([]SGuestIsolatedDevice, 0) + q := guest.GetIsolatedDevicesQuery().Asc("index") + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return gdevs, nil +} + +func (guest *SGuest) GetGuestGpuIsolatedDevices() ([]SGuestIsolatedDevice, error) { + gdevs := make([]SGuestIsolatedDevice, 0) + q := guest.GetIsolatedDevicesQuery() + isq := IsolatedDeviceManager.Query() + + cond := sqlchemy.Equals(isq.Field("dev_type"), api.GPU_TYPE) + isq = isq.Filter(cond) + sidq := isq.SubQuery() + q = q.Join(sidq, sqlchemy.Equals(q.Field("isolated_device_id"), sidq.Field("id"))) + q = q.Asc("index") + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return gdevs, nil +} + +func (self *SGuest) getIsolatedDeviceIndex() int8 { + guestDevs, _ := self.GetGuestIsolatedDevices() + var max uint + for i := 0; i < len(guestDevs); i++ { + if uint(guestDevs[i].Index) > max { + max = uint(guestDevs[i].Index) + } + } + + idxs := make([]int, max+1) + for i := 0; i < len(guestDevs); i++ { + idxs[guestDevs[i].Index] = 1 + } + + // find first idx not set + for i := 0; i < len(idxs); i++ { + if idxs[i] != 1 { + return int8(i) + } + } + + return int8(max + 1) +} + +func (self *SGuestIsolatedDevice) GetIsolatedDevice() *SIsolatedDevice { + dev, err := IsolatedDeviceManager.FetchById(self.IsolatedDeviceId) + if err != nil { + log.Errorf("IsolatedDeviceManager.FetchById %s", err) + return nil + } + return dev.(*SIsolatedDevice) +} + +func (self *SIsolatedDevice) GetGuestIsolatedDevice(guestId string, index int) (*SGuestIsolatedDevice, error) { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", self.Id).Equals("guest_id", guestId).Equals("index", index) + ret := &SGuestIsolatedDevice{} + err := q.First(ret) + if err != nil { + return nil, errors.Wrapf(err, "fetch guest %s isolated device by id %s index %d", guestId, self.Id, index) + } + ret.SetModelManager(GuestIsolatedDeviceManager, ret) + return ret, nil +} + +func (self *SIsolatedDevice) GetAllGuestIsolatedDevices() ([]SGuestIsolatedDevice, error) { + q := GuestIsolatedDeviceManager.Query().Equals("isolated_device_id", self.Id) + ret := []SGuestIsolatedDevice{} + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &ret) + if err != nil { + return nil, errors.Wrapf(err, "fetch guest isolated device by id %s", self.Id) + } + return ret, nil +} + +func (self *SGuestIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { + dev := self.GetIsolatedDevice() + devDesc := dev.getDesc() + devDesc.DiskIndex = self.DiskIndex + devDesc.NetworkIndex = self.NetworkIndex + devDesc.MemoryLimit = self.DeviceMemorySize + devDesc.SmUtilLimit = self.SmUtilLimit + devDesc.GpuType = self.GpuType + return devDesc +} + +func (self *SGuestIsolatedDevice) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { + devDesc := self.getDesc() + desc := jsonutils.NewDict() + desc.Update(jsonutils.Marshal(devDesc)) + desc.Add(jsonutils.NewString(GuestIsolatedDeviceManager.Keyword()), "res_name") + return desc +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go index f40310669..ff5afdcd5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_queries.go @@ -952,23 +952,27 @@ func fetchContainers(guestIds []string) (map[string][]*api.PodContainerDesc, err } func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevice { - q := IsolatedDeviceManager.Query().In("guest_id", guestIds) - devs := make([]SIsolatedDevice, 0) + q := GuestIsolatedDeviceManager.Query().In("guest_id", guestIds) + devs := make([]SGuestIsolatedDevice, 0) err := q.All(&devs) if err != nil { return nil } ret := make(map[string][]api.SIsolatedDevice) for i := range devs { + gdev := devs[i].GetIsolatedDevice() + if gdev == nil { + continue + } dev := api.SIsolatedDevice{} - dev.Id = devs[i].Id - dev.HostId = devs[i].HostId - dev.DevType = devs[i].DevType - dev.Model = devs[i].Model - dev.GuestId = devs[i].GuestId - dev.Addr = devs[i].Addr - dev.VendorDeviceId = devs[i].VendorDeviceId - dev.NumaNode = byte(devs[i].NumaNode) + dev.Id = gdev.Id + dev.HostId = gdev.HostId + dev.DevType = gdev.DevType + dev.SharingMode = gdev.SharingMode + dev.Model = gdev.Model + dev.Addr = gdev.Addr + dev.VendorDeviceId = gdev.VendorDeviceId + dev.NumaNode = byte(gdev.NumaNode) gdevs, ok := ret[devs[i].GuestId] if !ok { gdevs = make([]api.SIsolatedDevice, 0) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_sshable.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_sshable.go index 35d9fede5..2f99878a8 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_sshable.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guest_sshable.go @@ -405,9 +405,15 @@ func (guest *SGuest) PerformMakeSshable( if input.User == "" { return output, httperrors.NewBadRequestError("missing username") } + if !ansible.IsValidAnsibleUser(input.User) { + return output, httperrors.NewInputParameterError("invalid username") + } if input.PrivateKey == "" && input.Password == "" { return output, httperrors.NewBadRequestError("private_key and password cannot both be empty") } + if strings.ContainsAny(input.Password, "\x00\r\n") { + return output, httperrors.NewInputParameterError("invalid password") + } _, projectPublicKey, err := sshkeys.GetSshProjectKeypair(ctx, guest.ProjectId) if err != nil { @@ -496,7 +502,9 @@ func (guest *SGuest) PerformMakeSshable( host.SetVar("ansible_password", input.Password) } - cliSess := auth.GetSession(ctx, userCred, "") + // the playbook content here is server-generated; use the admin session + // since ansible playbook creation requires system admin privilege + cliSess := auth.GetAdminSession(ctx, "") pbId := "" pbName := "make-sshable-" + guest.Id pbModel, err := ansible_modules.AnsiblePlaybooks.UpdateOrCreatePbModel( diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdisks.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdisks.go index 9f3d98755..626c49d39 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdisks.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdisks.go @@ -65,9 +65,9 @@ type SGuestdisk struct { ImagePath string `width:"256" charset:"ascii" nullable:"false" get:"user" create:"required"` // Column(VARCHAR(256, charset='ascii'), nullable=False) - Driver string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) - CacheMode string `width:"32" charset:"ascii" nullable:"true" list:"user" update:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) - AioMode string `width:"32" charset:"ascii" nullable:"true" get:"user" update:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) + Driver string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) + CacheMode string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) + AioMode string `width:"32" charset:"ascii" nullable:"true" get:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) Iops int `nullable:"true" default:"0" list:"user" update:"user"` Bps int `nullable:"true" default:"0" list:"user" update:"user"` // Mb @@ -95,20 +95,6 @@ func (self *SGuestdisk) ValidateUpdateData(ctx context.Context, userCred mcclien return input, httperrors.NewInputParameterError("DISK Index %d has been occupied", index) } } - if self.CacheMode != input.CacheMode { - if input.CacheMode != "none" { - input.AioMode = "threads" - } - } - if self.AioMode != input.AioMode { - cacheMode := self.CacheMode - if input.CacheMode != "" { - cacheMode = input.CacheMode - } - if input.AioMode == "native" && cacheMode != "none" { - return input, httperrors.NewBadRequestError("AIO mode %s with cache mode %s is not supported", input.AioMode, cacheMode) - } - } var err error input.GuestJointBaseUpdateInput, err = self.SGuestJointsBase.ValidateUpdateData(ctx, userCred, query, input.GuestJointBaseUpdateInput) @@ -251,6 +237,7 @@ func (self *SGuestdisk) GetDiskJsonDescAtHost(ctx context.Context, host *SHost, desc.Url = url } } + desc.SnapshotId = disk.SnapshotId } if fpath := disk.GetMetadata(ctx, api.DISK_META_REMOTE_ACCESS_PATH, nil); len(fpath) > 0 { guest := self.getGuest() diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdrivers.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdrivers.go index 3a0cb1ea3..ce463f8ec 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdrivers.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestdrivers.go @@ -87,7 +87,7 @@ type IGuestDriver interface { CanStop(guest *SGuest) error StartGuestStopTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error StartGuestResetTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isHard bool, parentTaskId string) error - StartGuestRestartTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isForce bool, parentTaskId string) error + StartGuestRestartTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, isForce bool, timeout *int, parentTaskId string) error RequestSoftReset(ctx context.Context, guest *SGuest, task taskman.ITask) error @@ -109,6 +109,8 @@ type IGuestDriver interface { RequestSyncConfigOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error RequestSyncSecgroupsOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error + RequestSetPortMappingOnHost(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, host *SHost, task taskman.ITask, input api.ServerSetPortMappingInput) error + RequestSyncstatusOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) error RequestStartOnHost(ctx context.Context, guest *SGuest, host *SHost, userCred mcclient.TokenCredential, task taskman.ITask) error @@ -258,8 +260,8 @@ type IGuestDriver interface { ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error - BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error - BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error + BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SGuestIsolatedDevice) error + BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SGuestIsolatedDevice) error RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestnetworks.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestnetworks.go index a15318741..4287c5469 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guestnetworks.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guestnetworks.go @@ -969,6 +969,12 @@ func (gn *SGuestnetwork) ValidateUpdateData( return input, err } } + if len(input.PortMappings) > 0 { + guest := gn.GetGuest() + if guest != nil && !guest.SupportPortMapping() { + return input, httperrors.NewUnsupportOperationError("hypervisor %s does not support port_mapping", guest.Hypervisor) + } + } var err error input.GuestJointBaseUpdateInput, err = gn.SGuestJointsBase.ValidateUpdateData(ctx, userCred, query, input.GuestJointBaseUpdateInput) @@ -1001,6 +1007,20 @@ func (gn *SGuestnetwork) PostUpdate(ctx context.Context, userCred mcclient.Token log.Errorf("fail to fixDefaultGateway %s", err) } } + // port_mappings 变更后，启动独立 task 设置端口映射（先由宿主机分配 host_port，再同步配置） + // no_sync 为 true 时跳过，供宿主机回写（如自动分配 host_port）避免递归触发 + if data.Contains("port_mappings") && (input.NoSync == nil || !*input.NoSync) { + guest := gn.GetGuest() + if guest != nil && (guest.Status == api.VM_READY || guest.Status == api.VM_RUNNING) { + pms := gn.PortMappings + if err := guest.StartGuestSetPortMappingTask(ctx, userCred, api.ServerSetPortMappingInput{ + ServerNetworkInfo: api.ServerNetworkInfo{Mac: gn.MacAddr}, + PortMappings: pms, + }); err != nil { + log.Errorf("fail to start set port mapping task of nic %s: %s", gn.GetDetailedString(), err) + } + } + } } func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, userCred mcclient.TokenCredential, gns []SGuestnetwork, reserve bool) error { @@ -1010,7 +1030,7 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, userCr return errors.Wrapf(httperrors.ErrInvalidStatus, "eip associate with %s", gn.IpAddr) } guest := gn.GetGuest() - dev, err := guest.GetIsolatedDeviceByNetworkIndex(gn.Index) + dev, err := guest.GetGuestIsolatedDeviceByNetworkIndex(gn.Index) if err != nil { return errors.Wrap(err, "GetIsolatedDeviceByNetworkIndex") } @@ -1274,7 +1294,10 @@ func (gn *SGuestnetwork) GetVirtualIPs() []string { func (gn *SGuestnetwork) GetIsolatedDevice() (*SIsolatedDevice, error) { dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", gn.GuestId).Equals("network_index", gn.Index) + q := IsolatedDeviceManager.Query() + gidq := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", gn.GuestId).Equals("network_index", gn.Index).SubQuery() + q = q.Join(gidq, sqlchemy.Equals(q.Field("id"), gidq.Field("isolated_device_id"))) if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go index 63265c2b2..72968b0c3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/guests.go @@ -561,12 +561,14 @@ func (manager *SGuestManager) ListItemFilter( devTypeQ := func(q *sqlchemy.SQuery, checkType, backup *bool, dType string, conditions []sqlchemy.ICondition) []sqlchemy.ICondition { if checkType != nil { - isodev := IsolatedDeviceManager.Query().SubQuery() - isodevCons := []sqlchemy.ICondition{sqlchemy.IsNotNull(isodev.Field("guest_id"))} + guestIdev := GuestIsolatedDeviceManager.Query().SubQuery() + sgq := guestIdev.Query(guestIdev.Field("guest_id")).GroupBy(guestIdev.Field("guest_id")) + if len(dType) > 0 { - isodevCons = append(isodevCons, sqlchemy.Startswith(isodev.Field("dev_type"), dType)) + isodev := IsolatedDeviceManager.Query().SubQuery() + sgq = sgq.Join(isodev, sqlchemy.Equals(guestIdev.Field("isolated_device_id"), isodev.Field("id"))) + sgq = sgq.Filter(sqlchemy.Startswith(isodev.Field("dev_type"), dType)) } - sgq := isodev.Query(isodev.Field("guest_id")).Filter(sqlchemy.AND(isodevCons...)) cond := sqlchemy.NotIn if *checkType { cond = sqlchemy.In @@ -729,6 +731,12 @@ func (manager *SGuestManager) ListItemFilter( q = q.IsNullOrEmpty("host_id") } } + if len(query.IsolatedDeviceId) > 0 { + sq := GuestIsolatedDeviceManager.Query("guest_id"). + Equals("isolated_device_id", query.IsolatedDeviceId).SubQuery() + q = q.In("id", sq) + } + if len(query.SnapshotpolicyId) > 0 { sp := SnapshotPolicyResourceManager.Query("resource_id"). Equals("resource_type", api.SNAPSHOT_POLICY_TYPE_SERVER). @@ -1687,6 +1695,9 @@ func parseInstanceSnapshot(ctx context.Context, input *api.ServerCreateInput) (* return nil, httperrors.NewBadRequestError("Instance snapshot not ready") } input, err = isp.ToInstanceCreateInput(input) + if err != nil { + return nil, errors.Wrap(err, "ToInstanceCreateInput") + } if len(input.Disks) == 0 { return nil, httperrors.NewInputParameterError("there are no disks in this instance snapshot, try another one") } @@ -1706,6 +1717,9 @@ func parseInstanceBackup(ctx context.Context, input *api.ServerCreateInput) (*ap return nil, httperrors.NewBadRequestError("Instance backup not ready") } input, err = isp.ToInstanceCreateInput(input) + if err != nil { + return nil, errors.Wrap(err, "ToInstanceCreateInput") + } if len(input.Disks) == 0 { return nil, httperrors.NewInputParameterError("there are no disks in this instance backup, try another one") } @@ -1748,6 +1762,10 @@ func (manager *SGuestManager) validateCreateData( return nil, err } + if err := ValidateDeployConfigs(input.DeployConfigs); err != nil { + return nil, err + } + if len(input.Metadata) > 20 { return nil, httperrors.NewInputParameterError("metadata must be less than 20") } @@ -1876,6 +1894,10 @@ func (manager *SGuestManager) validateCreateData( var osProf osprofile.SOSProfile hypervisor = input.Hypervisor if hypervisor != api.HYPERVISOR_POD { + if hypervisor == api.HYPERVISOR_BAREMETAL { + // enable deploy telegraf + input.DeployTelegraf = true + } if len(input.Disks) == 0 && input.Cdrom == "" { return nil, httperrors.NewInputParameterError("No bootable disk information provided") } @@ -1938,13 +1960,17 @@ func (manager *SGuestManager) validateCreateData( input.EnableTpm = true } - // use uefi boot and q35 machine type on enable tpm + // TPM requires UEFI. x86 uses q35; ARM/RISC-V uses virt. if input.EnableTpm { input.Bios = "UEFI" - input.Machine = api.VM_MACHINE_TYPE_Q35 + if apis.IsARM(input.OsArch) || apis.IsRISCV(input.OsArch) { + input.Machine = api.VM_MACHINE_TYPE_VIRT + } else { + input.Machine = api.VM_MACHINE_TYPE_Q35 + } } - if imageDiskFormat != imageapi.IMAGE_DISK_FORMAT_ISO { + if imageDiskFormat != "" && imageDiskFormat != imageapi.IMAGE_DISK_FORMAT_ISO { var imgSupportUEFI *bool var imgSupportBIOS *bool if desc, ok := imgProperties[imageapi.IMAGE_UEFI_SUPPORT]; ok { @@ -1976,10 +2002,17 @@ func (manager *SGuestManager) validateCreateData( return nil, httperrors.NewInputParameterError("BIOS boot mode requires BIOS image") } default: - if imgSupportUEFI != nil && *imgSupportUEFI { - input.Bios = "UEFI" + supportUEFI := imgSupportUEFI != nil && *imgSupportUEFI + supportBIOS := imgSupportBIOS != nil && *imgSupportBIOS + if input.Hypervisor == api.HYPERVISOR_BAREMETAL && supportUEFI && supportBIOS { + // decided by selected host + input.Bios = "" } else { - input.Bios = "BIOS" + if imgSupportUEFI != nil && *imgSupportUEFI { + input.Bios = "UEFI" + } else { + input.Bios = "BIOS" + } } } } else { @@ -1997,11 +2030,15 @@ func (manager *SGuestManager) validateCreateData( imgProperties = map[string]string{"os_type": "Linux"} } input.DisableUsbKbd = imgProperties[imageapi.IMAGE_DISABLE_USB_KBD] == "true" + imageMachineType := imgProperties[imageapi.IMAGE_MACHINE_TYPE] + if len(input.Machine) == 0 && len(imageMachineType) > 0 { + input.Machine = imageMachineType + } imgIsWindows := imgProperties[imageapi.IMAGE_OS_TYPE] == "Windows" hasGpuVga := func() bool { for i := 0; i < len(input.IsolatedDevices); i++ { - if input.IsolatedDevices[i].DevType == api.GPU_VGA_TYPE { + if input.IsolatedDevices[i].GpuType == api.GPU_VGA { return true } } @@ -2044,6 +2081,13 @@ func (manager *SGuestManager) validateCreateData( osType = osProf.OSType input.OsType = osType } + if imgIsWindows { + netDriver := api.NETWORK_DRIVER_E1000 + if hasVirtioNetDrvier := imgProperties[imageapi.IMAGE_WIN_VIRTIO_NET] == "true"; hasVirtioNetDrvier { + netDriver = api.NETWORK_DRIVER_VIRTIO + } + osProf.NetDriver = netDriver + } input.OsProfile = jsonutils.Marshal(osProf) } @@ -2241,6 +2285,9 @@ func (manager *SGuestManager) validateCreateData( if err != nil { return nil, err } + if len(netConfig.PortMappings) > 0 && !IsHypervisorSupportPortMapping(input.Hypervisor) { + return nil, httperrors.NewInputParameterError("hypervisor %s does not support port_mapping", input.Hypervisor) + } if len(netConfig.Driver) == 0 { netConfig.Driver = osProf.NetDriver } @@ -2288,6 +2335,14 @@ func (manager *SGuestManager) validateCreateData( return nil, errors.Wrapf(httperrors.ErrInputParameter, "more than 1 nic(%d) assigned as default gateway", defaultGwCnt) } + // default: both check on + // switch: mac check off, also implies ip check off + // router: mac check on, ip check off + if input.SrcMacCheck != nil && !*input.SrcMacCheck { + srcIpCheck := false + input.SrcIpCheck = &srcIpCheck + } + isoDevArray := input.IsolatedDevices for idx := 0; idx < len(isoDevArray); idx += 1 { // .Contains(fmt.Sprintf("isolated_device.%d", idx)); idx += 1 { if input.Backup { @@ -2307,9 +2362,9 @@ func (manager *SGuestManager) validateCreateData( nvidiaVgpuCnt := 0 gpuCnt := 0 for i := 0; i < len(input.IsolatedDevices); i++ { - if input.IsolatedDevices[i].DevType == api.LEGACY_VGPU_TYPE { + if input.IsolatedDevices[i].SharingMode == api.DEVICE_SHARING_MODE_MDEV { nvidiaVgpuCnt += 1 - } else if utils.IsInStringArray(input.IsolatedDevices[i].DevType, api.VALID_GPU_TYPES) { + } else if input.IsolatedDevices[i].DevType == api.GPU_TYPE { gpuCnt += 1 } } @@ -2900,6 +2955,9 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred if matcherJson != nil { guest.SetMetadata(ctx, api.BAREMETAL_SERVER_METATA_ROOT_DISK_MATCHER, matcherJson, userCred) } + if qemuVersion, _ := data.GetString(api.VM_METADATA_QEMU_VERSION); qemuVersion != "" { + guest.SetMetadata(ctx, api.VM_METADATA_QEMU_VERSION, qemuVersion, userCred) + } userData, _ := data.GetString("user_data") if len(userData) > 0 { @@ -3354,6 +3412,22 @@ func (self *SGuest) IsExitOnly() bool { return true } +// IsHypervisorSupportPortMapping 指定 hypervisor 是否支持端口映射（port_mapping） +// 目前仅 kvm 与 pod 两种 hypervisor 支持，baremetal 等其它类型均不支持 +func IsHypervisorSupportPortMapping(hypervisor string) bool { + switch hypervisor { + case api.HYPERVISOR_KVM, api.HYPERVISOR_POD: + return true + default: + return false + } +} + +// SupportPortMapping 当前虚拟机是否支持端口映射（port_mapping） +func (self *SGuest) SupportPortMapping() bool { + return IsHypervisorSupportPortMapping(self.Hypervisor) +} + func (self *SGuest) getVirtualIPs() []string { ips := make([]string, 0) for _, guestgroup := range self.GetGroups() { @@ -4224,6 +4298,24 @@ func (self *SGuest) attach2NetworkOnce( } func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host *SHost, ipList []string, index int) (*SNetwork, error) { + findByIp := func() (*SNetwork, error) { + ip := vnic.GetIP() + if len(ip) == 0 { + if index < len(ipList) { + ip = ipList[index] + } + if len(ip) == 0 { + return nil, fmt.Errorf("Cannot find inetwork for vnics %s: no ip", vnic.GetMAC()) + } + } + return host.getNetworkOfIPOnHost(ctx, ip) + } + + // Proxmox does not sync remote L2/networks; always resolve on-premise network by IP + if host.HostType == api.HOST_TYPE_PROXMOX { + return findByIp() + } + vnetId := vnic.GetINetworkId() if len(vnetId) == 0 { if vnic.InClassicNetwork() { @@ -4240,17 +4332,7 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host } return NetworkManager.GetOrCreateClassicNetwork(ctx, wire) } - ip := vnic.GetIP() - if len(ip) == 0 { - if index < len(ipList) { - ip = ipList[index] - } - if len(ip) == 0 { - return nil, fmt.Errorf("Cannot find inetwork for vnics %s: no ip", vnic.GetMAC()) - } - } - // find network by IP - return host.getNetworkOfIPOnHost(ctx, ip) + return findByIp() } localNetObj, err := db.FetchByExternalIdAndManagerId(NetworkManager, vnetId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { // vpc := VpcManager.Query().SubQuery() @@ -4318,13 +4400,22 @@ func (self *SGuest) SyncVMNics( ip := commonext[i].GetIP() ip6 := commonext[i].GetIP6() if len(ip) > 0 { - if !network.Contains(ip) { + needRebind := network == nil || !network.Contains(ip) + // ESXi: force rebind if current network is not under the same manager wire + if !needRebind && host.HostType == api.HOST_TYPE_ESXI { + wire, _ := network.GetWire() + if wire == nil || wire.ManagerId != host.ManagerId { + needRebind = true + } + } + if needRebind { localNet, err := getCloudNicNetwork(ctx, commonext[i], host, ipList, i) if err != nil { return errors.Wrapf(err, "getCloudNicNetwork") } commondb[i].NetworkId = localNet.Id commondb[i].IpAddr = ip + commondb[i].Ip6Addr = ip6 } else { commondb[i].IpAddr = ip commondb[i].Ip6Addr = ip6 @@ -4650,7 +4741,7 @@ func usageTotalGuestResourceCount( policyResult rbacutils.SPolicyResult, ) map[string]SGuestCountStat { countStat := make(map[string]SGuestCountStat) - for _, arch := range []string{apis.OS_ARCH_ALL, apis.OS_ARCH_X86_64, apis.OS_ARCH_AARCH64} { + for _, arch := range []string{apis.OS_ARCH_ALL, apis.OS_ARCH_X86_64, apis.OS_ARCH_AARCH64, apis.OS_ARCH_RISCV64} { allStat := usageTotalGuestResourceCountByArch( ctx, scope, ownerId, rangeObjs, status, hypervisors, includeSystem, pendingDelete, @@ -4734,11 +4825,10 @@ func _guestResourceCountQuery( diskBackupSubQuery := backupDiskQuery.SubQuery() // diskBackupSubQuery := diskQuery.IsNotEmpty("backup_storage_id").SubQuery() - isolated := IsolatedDeviceManager.Query().SubQuery() + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() - isoDevQuery := isolated.Query(isolated.Field("guest_id"), sqlchemy.COUNT("device_sum")) - isoDevQuery = isoDevQuery.Filter(sqlchemy.IsNotNull(isolated.Field("guest_id"))) - isoDevQuery = isoDevQuery.GroupBy(isolated.Field("guest_id")) + isoDevQuery := guestIdevs.Query(guestIdevs.Field("guest_id"), sqlchemy.COUNT("device_sum")) + isoDevQuery = isoDevQuery.GroupBy(guestIdevs.Field("guest_id")) isoDevSubQuery := isoDevQuery.SubQuery() @@ -4890,10 +4980,11 @@ func (self *SGuest) allocSriovNicDevice( if err != nil { return errors.Wrap(err, "self.createIsolatedDeviceOnHost") } - dev, err := self.GetIsolatedDeviceByNetworkIndex(gn.Index) + gdev, err := self.GetGuestIsolatedDeviceByNetworkIndex(gn.Index) if err != nil { return errors.Wrap(err, "self.GetIsolatedDeviceByNetworkIndex") } + dev := gdev.GetIsolatedDevice() if dev.OvsOffloadInterface != "" { _, err = db.Update(gn, func() error { gn.Ifname = dev.OvsOffloadInterface @@ -4970,9 +5061,9 @@ func (self *SGuest) attach2NamedNetworkDesc(ctx context.Context, userCred mcclie dev, _ := idev.(*SIsolatedDevice) sriovWires = []string{dev.WireId} } else { - wires, err := IsolatedDeviceManager.FindUnusedNicWiresByModel(netConfig.SriovDevice.Model) + wires, err := IsolatedDeviceManager.FindAvailableNicWiresByModel(netConfig.SriovDevice.Model) if err != nil { - return nil, errors.Wrap(err, "FindUnusedNicWiresByModel") + return nil, errors.Wrap(err, "FindAvailableNicWiresByModel") } sriovWires = wires } @@ -5228,6 +5319,9 @@ func (self *SGuest) createDiskOnHost( } if autoAttach { err = self.attach2Disk(ctx, disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint, diskConfig.BootIndex) + if err != nil { + return nil, err + } } err = self.InheritTo(ctx, userCred, disk) if err != nil { @@ -5554,6 +5648,9 @@ func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, userCred mcclient if err != nil { return nil, err } + if err := ValidateDeployConfigs(deploys); err != nil { + return nil, err + } if len(deploys) > 0 { config.Add(jsonutils.Marshal(deploys), "deploys") @@ -5712,7 +5809,9 @@ func (self *SGuest) getExtraOptions(ctx context.Context) jsonutils.JSONObject { } func (self *SGuest) GetIsolatedDevices() ([]SIsolatedDevice, error) { - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id) + gq := GuestIsolatedDeviceManager.Query().Equals("guest_id", self.Id).SubQuery() + q := IsolatedDeviceManager.Query() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) devs := []SIsolatedDevice{} err := db.FetchModelObjects(IsolatedDeviceManager, q, &devs) if err != nil { @@ -5721,9 +5820,10 @@ func (self *SGuest) GetIsolatedDevices() ([]SIsolatedDevice, error) { return devs, nil } -func (self *SGuest) GetIsolatedDeviceByNetworkIndex(index int) (*SIsolatedDevice, error) { - dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id).Equals("network_index", index) +func (self *SGuest) GetGuestIsolatedDeviceByNetworkIndex(index int) (*SGuestIsolatedDevice, error) { + dev := SGuestIsolatedDevice{} + q := GuestIsolatedDeviceManager.Query().Equals("network_index", index).Equals("guest_id", self.Id) + if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { @@ -5733,13 +5833,17 @@ func (self *SGuest) GetIsolatedDeviceByNetworkIndex(index int) (*SIsolatedDevice if err != nil { return nil, err } - dev.SetModelManager(IsolatedDeviceManager, &dev) + dev.SetModelManager(GuestIsolatedDeviceManager, &dev) return &dev, nil } func (self *SGuest) GetIsolatedDeviceByDiskIndex(index int8) (*SIsolatedDevice, error) { dev := SIsolatedDevice{} - q := IsolatedDeviceManager.Query().Equals("guest_id", self.Id).Equals("disk_index", index) + q := IsolatedDeviceManager.Query() + gidq := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", self.Id).Equals("disk_index", index).SubQuery() + q = q.Join(gidq, sqlchemy.Equals(q.Field("id"), gidq.Field("isolated_device_id"))) + if cnt, err := q.CountWithError(); err != nil { return nil, err } else if cnt == 0 { @@ -5797,8 +5901,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *a desc.IsVolatileHost = true } - // isolated devices - isolatedDevs, _ := self.GetIsolatedDevices() + isolatedDevs, _ := self.GetGuestIsolatedDevices() for _, dev := range isolatedDevs { desc.IsolatedDevices = append(desc.IsolatedDevices, dev.getDesc()) } @@ -6058,10 +6161,11 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { spec.Set("nic", nicSpecs) // get isolate device spec - guestgpus, _ := self.GetIsolatedDevices() + guestgpus, _ := self.GetGuestIsolatedDevices() gpuSpecs := []GpuSpec{} - for _, guestgpu := range guestgpus { - if strings.HasPrefix(guestgpu.DevType, "GPU") { + for i := range guestgpus { + guestgpu := guestgpus[i].GetIsolatedDevice() + if guestgpu.DevType == api.GPU_TYPE { gs := guestgpu.GetGpuSpec() gpuSpecs = append(gpuSpecs, *gs) } @@ -6780,6 +6884,19 @@ func (self *SGuest) PendingDetachScalingGroup() error { return nil } +func (self *SGuest) PendingDeleteSnapshots(ctx context.Context, userCred mcclient.TokenCredential) error { + instanceSnapshots, _ := self.GetInstanceSnapshots() + for i := range instanceSnapshots { + instanceSnapshots[i].DoPendingDelete(ctx, userCred) + } + + snapshots, _ := self.GetDiskSnapshotsNotInInstanceSnapshots(false) + for i := range snapshots { + snapshots[i].DoPendingDelete(ctx, userCred) + } + return nil +} + func (self *SGuest) DeleteEip(ctx context.Context, userCred mcclient.TokenCredential) error { eip, err := self.GetEipOrPublicIp() if err != nil { @@ -7191,16 +7308,18 @@ func (self *SGuest) ToNetworksConfig() []*api.NetworkConfig { } func (self *SGuest) ToIsolatedDevicesConfig() []*api.IsolatedDeviceConfig { - guestIsolatedDevices, _ := self.GetIsolatedDevices() + guestIsolatedDevices, _ := self.GetGuestIsolatedDevices() if len(guestIsolatedDevices) == 0 { return nil } ret := make([]*api.IsolatedDeviceConfig, len(guestIsolatedDevices)) - for idx, guestIsolatedDevice := range guestIsolatedDevices { + for idx := range guestIsolatedDevices { + dev := guestIsolatedDevices[idx].GetIsolatedDevice() devConf := new(api.IsolatedDeviceConfig) - devConf.Model = guestIsolatedDevice.Model - devConf.Vendor = guestIsolatedDevice.getVendor() - devConf.DevType = guestIsolatedDevice.DevType + devConf.Model = dev.Model + devConf.Vendor = dev.getVendor() + devConf.DevType = dev.DevType + devConf.SharingMode = dev.SharingMode ret[idx] = devConf } return ret @@ -7249,12 +7368,22 @@ func (self *SGuest) GetInstanceSnapshots() ([]SInstanceSnapshot, error) { return instanceSnapshots, nil } +func (self *SGuest) GetPendingDeleteInstanceSnapshots() ([]SInstanceSnapshot, error) { + instanceSnapshots := make([]SInstanceSnapshot, 0) + q := InstanceSnapshotManager.Query().Equals("guest_id", self.Id).IsTrue("pending_deleted") + err := db.FetchModelObjects(InstanceSnapshotManager, q, &instanceSnapshots) + if err != nil { + return nil, err + } + return instanceSnapshots, nil +} + func (self *SGuest) GetInstanceSnapshotCount() (int, error) { q := InstanceSnapshotManager.Query().Equals("guest_id", self.Id) return q.CountWithError() } -func (self *SGuest) GetDiskSnapshotsNotInInstanceSnapshots() ([]SSnapshot, error) { +func (self *SGuest) GetDiskSnapshotsNotInInstanceSnapshots(pendingDelted bool) ([]SSnapshot, error) { guestDisks, err := self.GetGuestDisks() if err != nil { return nil, errors.Wrapf(err, "GetGuestDisks") @@ -7264,7 +7393,10 @@ func (self *SGuest) GetDiskSnapshotsNotInInstanceSnapshots() ([]SSnapshot, error diskIds[i] = guestDisks[i].DiskId } snapshots := make([]SSnapshot, 0) - q := SnapshotManager.Query().IsFalse("fake_deleted").In("disk_id", diskIds) + q := SnapshotManager.Query().In("disk_id", diskIds) + if pendingDelted { + q = q.IsTrue("pending_deleted") + } sq := InstanceSnapshotJointManager.Query("snapshot_id").SubQuery() q = q.LeftJoin(sq, sqlchemy.Equals(q.Field("id"), sq.Field("snapshot_id"))). Filter(sqlchemy.IsNull(sq.Field("snapshot_id"))) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/hosts.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/hosts.go index a9dc3329d..2c3252013 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/hosts.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/hosts.go @@ -1443,17 +1443,35 @@ func (hh *SHostManager) GetPropertyK8sMasterNodeIps(ctx context.Context, userCre if err != nil { return nil, errors.Wrap(err, "list master nodes") } - ips := make([]string, 0) + ips := map[string]struct{}{} for i := range nodes.Items { for j := range nodes.Items[i].Status.Addresses { if nodes.Items[i].Status.Addresses[j].Type == v1.NodeInternalIP { - ips = append(ips, nodes.Items[i].Status.Addresses[j].Address) + ips[nodes.Items[i].Status.Addresses[j].Address] = struct{}{} } } } log.Infof("k8s master nodes ips %v", ips) + if jsonutils.QueryBoolean(query, "kvm_hosts", false) { + hostq := hh.Query("access_ip") + hostq = hostq.In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_KVM, api.HOST_TYPE_CONTAINER}) + type HostIp struct { + AccessIp string + } + hostIps := make([]HostIp, 0) + if err := hostq.All(&hostIps); err != nil && err != sql.ErrNoRows { + return nil, err + } + for i := range hostIps { + ips[hostIps[i].AccessIp] = struct{}{} + } + } + ipArr := make([]string, 0, len(ips)) + for k := range ips { + ipArr = append(ipArr, k) + } res := jsonutils.NewDict() - res.Set("ips", jsonutils.Marshal(ips)) + res.Set("ips", jsonutils.Marshal(ipArr)) return res, nil } @@ -3307,6 +3325,23 @@ func (hh *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredent func (hh *SHost) getNetworkOfIPOnHost(ctx context.Context, ipAddr string) (*SNetwork, error) { netInterfaces := hh.GetHostNetInterfaces() + // VMware: only associate networks under wires of the same manager_id + if hh.HostType == api.HOST_TYPE_ESXI { + if len(hh.ManagerId) == 0 { + return nil, fmt.Errorf("ESXi host %s has empty manager_id, cannot resolve network for IP %s", hh.Id, ipAddr) + } + for _, netInterface := range netInterfaces { + wire := netInterface.GetWire() + if wire == nil || wire.ManagerId != hh.ManagerId { + continue + } + network, err := netInterface.GetCandidateNetworkForIp(ctx, nil, nil, rbacscope.ScopeNone, ipAddr) + if err == nil && network != nil { + return network, nil + } + } + return nil, fmt.Errorf("IP %s not reachable on ESXi host %s under manager %s", ipAddr, hh.Id, hh.ManagerId) + } for _, netInterface := range netInterfaces { network, err := netInterface.GetCandidateNetworkForIp(ctx, nil, nil, rbacscope.ScopeNone, ipAddr) if err == nil && network != nil { @@ -3526,13 +3561,13 @@ func (manager *SHostManager) totalCountQ( q = db.ObjectIdQueryWithPolicyResult(ctx, q, HostManager, policyResult) - isolatedDevices := IsolatedDeviceManager.Query().SubQuery() + isolatedDevices := IsolatedDeviceManager.queryWithoutGuest(IsolatedDeviceManager.Query()).SubQuery() iq := isolatedDevices.Query( isolatedDevices.Field("host_id"), sqlchemy.SUM("isolated_reserved_memory", isolatedDevices.Field("reserved_memory")), sqlchemy.SUM("isolated_reserved_cpu", isolatedDevices.Field("reserved_cpu")), sqlchemy.SUM("isolated_reserved_storage", isolatedDevices.Field("reserved_storage")), - ).IsNullOrEmpty("guest_id").GroupBy(isolatedDevices.Field("host_id")).SubQuery() + ).GroupBy(isolatedDevices.Field("host_id")).SubQuery() q = q.LeftJoin(iq, sqlchemy.Equals(q.Field("id"), iq.Field("host_id"))) q.AppendField( iq.Field("isolated_reserved_memory"), @@ -4110,7 +4145,25 @@ func (hh *SHost) GetDevsReservedResource(devs []SIsolatedDevice) *api.IsolatedDe ReservedCpu: &reservedCpu, } for _, dev := range devs { - if !utils.IsInStringArray(dev.DevType, api.VALID_GPU_TYPES) { + if !dev.IsKvmExclusiveGPU() { + continue + } + reservedCpu += dev.ReservedCpu + reservedMem += dev.ReservedMemory + reservedStorage += dev.ReservedStorage + } + return &reservedResourceForGpu +} + +func (hh *SHost) GetDevsReservedResourceByDevStats(devs []IsolatedDeviceAllocateStat) *api.IsolatedDeviceReservedResourceInput { + reservedCpu, reservedMem, reservedStorage := 0, 0, 0 + reservedResourceForGpu := api.IsolatedDeviceReservedResourceInput{ + ReservedStorage: &reservedStorage, + ReservedMemory: &reservedMem, + ReservedCpu: &reservedCpu, + } + for _, dev := range devs { + if !dev.IsKvmExclusiveGPU() { continue } reservedCpu += dev.ReservedCpu @@ -4422,6 +4475,19 @@ func (hh *SHost) GetDetailsIpmi(ctx context.Context, userCred mcclient.TokenCred return ret, nil } +func (hh *SHost) GetDetailsGuestIsolatedDevicesInitialized(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + inited, err := IsolatedDeviceManager.isInitializeDataDone() + if err != nil { + return nil, err + } + if !inited { + return nil, httperrors.NewResourceNotReadyError("isolated device not isitialized") + } + ret := jsonutils.NewDict() + ret.Set("initialized", jsonutils.NewString("ok")) + return ret, nil +} + func (manager *SHostManager) GetHostsByManagerAndRegion(managerId string, regionId string) []SHost { zones := ZoneManager.Query().Equals("cloudregion_id", regionId).SubQuery() hosts := HostManager.Query() @@ -5046,6 +5112,9 @@ func fetchIpmiInfo(data api.HostIpmiAttributes, hostId string) (types.SIPMIInfo, if data.IpmiLanChannel != nil { info.LanChannel = *data.IpmiLanChannel } + if data.IpmiCipherSuite != nil { + info.CipherSuite = *data.IpmiCipherSuite + } if data.IpmiVerified != nil { info.Verified = *data.IpmiVerified } @@ -6915,14 +6984,24 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T enables = append(enables, extNics[j]) } } else { - wireId := "" - extWire := extNics[j].GetIWire() - if extWire != nil { - wire, err := WireManager.FetchWireByExternalId(provider.Id, extWire.GetGlobalId()) - if err != nil { - result.AddError(err) + wireId := netIfs[i].WireId + ipAddr := extNics[j].GetIpAddr() + // Proxmox only associates to on-premise wires; do not sync remote L2 wire ids + if provider.Provider != api.CLOUD_PROVIDER_PROXMOX { + extWire := extNics[j].GetIWire() + if extWire != nil { + wire, err := WireManager.FetchWireByExternalId(provider.Id, extWire.GetGlobalId()) + if err != nil { + result.AddError(err) + } else { + wireId = wire.Id + } } else { - wireId = wire.Id + wireId = "" + } + } else if len(ipAddr) > 0 { + if ipWire, werr := WireManager.GetOnPremiseWireOfIp(ipAddr); werr == nil { + wireId = ipWire.Id } } // in sync, sync interface and bridge @@ -7007,7 +7086,25 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T netif := host.GetNetInterface(enables[i].GetMac(), enables[i].GetVlanId()) // always true reserved address pool log.Debugf("enable netif %s", enables[i].GetMac()) - err = host.EnableNetif(ctx, userCred, netif, "", enables[i].GetIpAddr(), "", "", "", true, true, false, false) + ipAddr := enables[i].GetIpAddr() + if provider.Provider == api.CLOUD_PROVIDER_PROXMOX && len(ipAddr) > 0 { + ipWire, werr := WireManager.GetOnPremiseWireOfIp(ipAddr) + if werr != nil { + result.AddError(werr) + continue + } + if netif.WireId != ipWire.Id { + _, err := db.Update(netif, func() error { + netif.WireId = ipWire.Id + return nil + }) + if err != nil { + result.AddError(err) + continue + } + } + } + err = host.EnableNetif(ctx, userCred, netif, "", ipAddr, "", "", "", true, true, false, false) if err != nil { result.AddError(err) } else { @@ -7029,16 +7126,21 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T strBridge = &bridge } wireId := "" - extWire := extNic.GetIWire() - if extWire != nil { - wire, err := WireManager.FetchWireByExternalId(provider.Id, extWire.GetGlobalId()) - if err != nil { - result.AddError(err) - } else { - wireId = wire.Id + ipAddr := extNic.GetIpAddr() + // Proxmox does not sync remote L2 wires; leave wireId empty so addNetif + // resolves the on-premise wire via GetOnPremiseWireOfIp. + if provider.Provider != api.CLOUD_PROVIDER_PROXMOX { + extWire := extNic.GetIWire() + if extWire != nil { + wire, err := WireManager.FetchWireByExternalId(provider.Id, extWire.GetGlobalId()) + if err != nil { + result.AddError(err) + } else { + wireId = wire.Id + } } } - err = host.addNetif(ctx, userCred, extNic.GetMac(), extNic.GetVlanId(), wireId, extNic.GetIpAddr(), "", 0, + err = host.addNetif(ctx, userCred, extNic.GetMac(), extNic.GetVlanId(), wireId, ipAddr, "", 0, compute.TNicType(extNic.GetNicType()), int(extNic.GetIndex()), extNic.IsLinkUp(), int16(extNic.GetMtu()), false, strNetIf, strBridge, true, true, false, false) if err != nil { @@ -7335,6 +7437,10 @@ func (host *SHost) RemoteHealthStatus(ctx context.Context) string { } func (host *SHost) GetHostnameByName() string { + if host.Hostname != "" { + return host.Hostname + } + hostname := host.Name accessIp := strings.Replace(host.AccessIp, ".", "-", -1) if strings.HasSuffix(host.Name, "-"+accessIp) { @@ -7619,7 +7725,10 @@ func (hh *SHost) GetDetailsJnlp(ctx context.Context, userCred mcclient.TokenCred func (hh *SHost) PerformInsertIso(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if utils.IsInStringArray(hh.Status, []string{api.BAREMETAL_READY, api.BAREMETAL_RUNNING}) { - imageStr, err := data.GetString("image") + imageStr, _ := data.GetString("image") + if len(imageStr) == 0 { + return nil, httperrors.NewInputParameterError("missing image") + } image, err := CachedimageManager.getImageInfo(ctx, userCred, imageStr, false) if err != nil { if err == sql.ErrNoRows { @@ -8202,10 +8311,10 @@ func (h *SHost) GetDetailsApiStats(ctx context.Context, userCred mcclient.TokenC } func (hh *SHost) GetDetailsIsolatedDeviceNumaStats(ctx context.Context, userCred mcclient.TokenCredential, input *api.HostIsolatedDeviceNumaStatsInput) (jsonutils.JSONObject, error) { - if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { - return nil, httperrors.NewInputParameterError("dev_type %s is invalid", input.DevType) + if input.Model == "" { + return nil, httperrors.NewMissingParameterError("model") } - stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.DevType, hh.Id) + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(input.Model, hh.Id) if err != nil { return nil, err } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/initdb.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/initdb.go index b706ad3cb..53ada08ef 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/initdb.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/initdb.go @@ -45,12 +45,15 @@ func InitDB() error { WireManager, StorageManager, SecurityGroupManager, + IpSetManager, NetworkManager, NetworkAddressManager, NetworkIpMacManager, GuestManager, GuestnetworkManager, HostManager, + GuestIsolatedDeviceManager, + IsolatedDeviceManager, HostDmesgLogManager, LoadbalancerCertificateManager, LoadbalancerAclManager, diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_backup.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_backup.go index 7dca37347..0c30f5d61 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_backup.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_backup.go @@ -225,16 +225,7 @@ func (self *SInstanceBackup) getMoreDetails(userCred mcclient.TokenCredential, o backups, _ := self.GetBackups() out.DiskBackups = []api.SSimpleBackup{} for i := 0; i < len(backups); i++ { - out.DiskBackups = append(out.DiskBackups, api.SSimpleBackup{ - Id: backups[i].Id, - Name: backups[i].Name, - SizeMb: backups[i].SizeMb, - DiskSizeMb: backups[i].DiskSizeMb, - DiskType: backups[i].DiskType, - Status: backups[i].Status, - EncryptKeyId: backups[i].EncryptKeyId, - CreatedAt: backups[i].CreatedAt, - }) + out.DiskBackups = append(out.DiskBackups, backups[i].ToSimpleBackup()) } out.Size = self.SizeMb * 1024 * 1024 return out diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go index e0944fc99..6bebba99c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/instance_snapshots.go @@ -34,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -875,7 +876,20 @@ func (manager *SInstanceSnapshotManager) AutoServerSnapshot(ctx context.Context, log.Errorf("get server error: %v", err) continue } + cnt, err := server.GetInstanceSnapshotCount() + if err != nil { + log.Errorf("get instance snapshot error: %v", err) + continue + } + if cnt > options.Options.RetentionDaysLimit { + msg := fmt.Sprintf("server %s auto snapshot count %d more than retention count limit %d", server.GetId(), cnt, options.Options.RetentionCountLimit) + log.Errorf("auto snapshot %s error: %v", server.Name, msg) + db.OpsLog.LogEvent(server, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, msg, userCred) + notifyclient.NotifySystemErrorWithCtx(ctx, server.Id, server.Name, db.ACT_DISK_AUTO_SNAPSHOT_FAIL, msg) + continue + } serverMap[server.Id] = server + } for i := range serverMap { input := api.ServerInstanceSnapshot{} @@ -895,7 +909,7 @@ func (manager *SInstanceSnapshotManager) CleanupInstanceSnapshots(ctx context.Co defer func() { instanceSnapshotCleanupTaskRunning = 0 }() - sq := manager.Query().Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").SubQuery() + sq := manager.Query().Startswith("name", "auto-").SubQuery() iss := []struct { GuestCnt int @@ -949,6 +963,7 @@ func (manager *SInstanceSnapshotManager) CleanupInstanceSnapshots(ctx context.Co for guestId, retentionCnt := range guestRetentionMap { if cnt, ok := guestCount[guestId]; ok && cnt > retentionCnt { manager.startCleanupRetentionCount(ctx, userCred, guestId, cnt-retentionCnt) + return } } } @@ -984,20 +999,19 @@ func (manager *SInstanceSnapshotManager) CleanupInstanceSnapshots(ctx context.Co } for guestId, retentionDays := range guestRetentionMap { manager.startCleanupRetentionDays(ctx, userCred, guestId, retentionDays) + return } } } func (manager *SInstanceSnapshotManager) startCleanupRetentionCount(ctx context.Context, userCred mcclient.TokenCredential, guestId string, cnt int) error { - q := manager.Query().Equals("guest_id", guestId).Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").Asc("created_at").Limit(cnt) - vms := []SInstanceSnapshot{} - err := db.FetchModelObjects(manager, q, &vms) + is := new(SInstanceSnapshot) + err := manager.Query().Equals("guest_id", guestId).Equals("status", api.INSTANCE_SNAPSHOT_READY).Startswith("name", "auto-").Asc("created_at").First(is) if err != nil { - return errors.Wrapf(err, "FetchModelObjects") - } - for i := range vms { - vms[i].StartInstanceSnapshotDeleteTask(ctx, userCred, "") + return err } + is.SetModelManager(manager, is) + is.StartInstanceSnapshotDeleteTask(ctx, userCred, "") return nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/ipsets.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/ipsets.go new file mode 100644 index 000000000..12fcc18e8 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/ipsets.go @@ -0,0 +1,793 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "fmt" + "net" + "strings" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// +onecloud:swagger-gen-model-singular=ipset +// +onecloud:swagger-gen-model-plural=ipsets +type SIpSetManager struct { + db.SSharableVirtualResourceBaseManager + db.SExternalizedResourceBaseManager + SManagedResourceBaseManager + SCloudregionResourceBaseManager +} + +var IpSetManager *SIpSetManager + +func init() { + IpSetManager = &SIpSetManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SIpSet{}, + "ipsets_tbl", + "ipset", + "ipsets", + ), + } + IpSetManager.SetVirtualObject(IpSetManager) +} + +type SIpSet struct { + db.SSharableVirtualResourceBase + db.SExternalizedResourceBase + SManagedResourceBase + SCloudregionResourceBase `width:"36" charset:"ascii" nullable:"true" list:"user" create:"optional"` + + // IP集合类型 + IpSetType api.TIpSetType `width:"32" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + // IP/CIDR 列表，逗号分隔 + Data string `width:"2048" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` +} + +func (manager *SIpSetManager) InitializeData() error { + q := manager.Query().IsNullOrEmpty("manager_id").Equals("status", "init") + ipsets := []SIpSet{} + err := db.FetchModelObjects(manager, q, &ipsets) + if err != nil { + return errors.Wrapf(err, "db.FetchModelObjects") + } + for i := range ipsets { + _, err := db.Update(&ipsets[i], func() error { + ipsets[i].Status = apis.STATUS_AVAILABLE + return nil + }) + if err != nil { + return errors.Wrapf(err, "Update ipset %s", ipsets[i].Id) + } + } + return nil +} + +func ipSetRegionListFilter(ctx context.Context, q *sqlchemy.SQuery, query api.RegionalFilterListInput) (*sqlchemy.SQuery, error) { + regionIds := []string{} + for _, region := range query.CloudregionId { + if len(region) == 0 { + continue + } + regionObj, err := ValidateCloudregionId(ctx, nil, region) + if err != nil { + return nil, errors.Wrap(err, "ValidateCloudregionId") + } + regionIds = append(regionIds, regionObj.GetId()) + } + if len(regionIds) > 0 { + providerSubq := CloudproviderRegionManager.Query("cloudprovider_id").In("cloudregion_id", regionIds).Distinct().SubQuery() + q = q.Filter(sqlchemy.OR( + sqlchemy.In(q.Field("cloudregion_id"), regionIds), + sqlchemy.AND( + sqlchemy.IsNullOrEmpty(q.Field("cloudregion_id")), + sqlchemy.In(q.Field("manager_id"), providerSubq), + ), + )) + } + if len(query.City) > 0 { + subq := CloudregionManager.Query("id").Equals("city", query.City).SubQuery() + q = q.Filter(sqlchemy.OR( + sqlchemy.In(q.Field("cloudregion_id"), subq), + sqlchemy.IsNullOrEmpty(q.Field("cloudregion_id")), + )) + } + return q, nil +} + +// IP集合列表 +func (manager *SIpSetManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.IpSetListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter") + } + q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter") + } + q, err = manager.SManagedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ManagedResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemFilter") + } + q, err = ipSetRegionListFilter(ctx, q, query.RegionalFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "ipSetRegionListFilter") + } + if len(query.IpSetType) > 0 { + types := make([]string, 0, len(query.IpSetType)) + for _, t := range query.IpSetType { + types = append(types, string(t)) + } + q = q.In("ip_set_type", types) + } + if len(query.Ip) > 0 { + q = q.Contains("data", query.Ip) + } + return q, nil +} + +func (manager *SIpSetManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.IpSetListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SVirtualResourceBaseManager.OrderByExtraFields") + } + q, err = manager.SManagedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.ManagedResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.OrderByExtraFields") + } + q, err = manager.SCloudregionResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.RegionalFilterListInput) + if err != nil { + return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.OrderByExtraFields") + } + return q, nil +} + +func (manager *SIpSetManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + q, err = manager.SManagedResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + q, err = manager.SCloudregionResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + return q, httperrors.ErrNotFound +} + +func (manager *SIpSetManager) QueryDistinctExtraFields(q *sqlchemy.SQuery, resource string, fields []string) (*sqlchemy.SQuery, error) { + q, err := manager.SManagedResourceBaseManager.QueryDistinctExtraFields(q, resource, fields) + if err == nil { + return q, nil + } + return q, httperrors.ErrNotFound +} + +type sIpSetUsageCount struct { + Cidr string + SecurityGroupCount int +} + +func (manager *SIpSetManager) TotalResourceCount(ipSetIds []string) (map[string]int, error) { + ret := map[string]int{} + if len(ipSetIds) == 0 { + return ret, nil + } + sq := SecurityGroupRuleManager.Query("cidr", "secgroup_id"). + Equals("target_type", api.SecurityGroupRuleTargetTypeIpSet). + In("cidr", ipSetIds). + Distinct().SubQuery() + q := sq.Query( + sq.Field("cidr"), + sqlchemy.COUNT("security_group_count"), + ).GroupBy(sq.Field("cidr")) + counts := []sIpSetUsageCount{} + err := q.All(&counts) + if err != nil { + return nil, errors.Wrapf(err, "q.All") + } + for i := range counts { + ret[counts[i].Cidr] = counts[i].SecurityGroupCount + } + return ret, nil +} + +func (manager *SIpSetManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.IpSetDetails { + rows := make([]api.IpSetDetails, len(objs)) + virtRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + managerRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + regionRows := manager.SCloudregionResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + ipSetIds := make([]string, len(objs)) + for i := range rows { + ipSetIds[i] = objs[i].(*SIpSet).Id + rows[i] = api.IpSetDetails{ + SharableVirtualResourceDetails: virtRows[i], + ManagedResourceInfo: managerRows[i], + CloudregionResourceInfo: regionRows[i], + } + } + counts, err := manager.TotalResourceCount(ipSetIds) + if err != nil { + log.Errorf("TotalResourceCount error: %v", err) + return rows + } + for i := range rows { + rows[i].SecurityGroupCount = counts[ipSetIds[i]] + } + return rows +} + +func (manager *SIpSetManager) ListItemExportKeys( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemExportKeys") + } + if keys.ContainsAny(manager.SManagedResourceBaseManager.GetExportKeys()...) { + q, err = manager.SManagedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SManagedResourceBaseManager.ListItemExportKeys") + } + } + if keys.ContainsAny(manager.SCloudregionResourceBaseManager.GetExportKeys()...) { + q, err = manager.SCloudregionResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SCloudregionResourceBaseManager.ListItemExportKeys") + } + } + return q, nil +} + +func (manager *SIpSetManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input *api.IpSetCreateInput, +) (*api.IpSetCreateInput, error) { + if err := input.Validate(); err != nil { + return nil, err + } + input.Data = normalizeIpSetData(input.IpSetType, input.Data) + if len(input.Data) == 0 { + return nil, httperrors.NewInputParameterError("Data is empty") + } + + var err error + input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return nil, err + } + if len(input.CloudproviderId) > 0 { + providerObj, err := validators.ValidateModel(ctx, userCred, CloudproviderManager, &input.CloudproviderId) + if err != nil { + return nil, err + } + input.ManagerId = input.CloudproviderId + provider := providerObj.(*SCloudprovider) + if len(input.CloudregionId) > 0 { + regionObj, err := validators.ValidateModel(ctx, userCred, CloudregionManager, &input.CloudregionId) + if err != nil { + return nil, err + } + region := regionObj.(*SCloudregion) + if provider.Provider != region.Provider { + return nil, httperrors.NewConflictError("conflict region %s and cloudprovider %s", region.Name, provider.Name) + } + } else if provider.Provider != api.CLOUD_PROVIDER_QCLOUD { + return nil, httperrors.NewMissingParameterError("cloudregion_id") + } else { + input.CloudregionId = "" + } + return input, nil + } + if len(input.CloudregionId) == 0 { + input.CloudregionId = api.DEFAULT_REGION_ID + } + regionObj, err := validators.ValidateModel(ctx, userCred, CloudregionManager, &input.CloudregionId) + if err != nil { + return nil, err + } + _ = regionObj.(*SCloudregion) + return input, nil +} + +func (ipset *SIpSet) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.IpSetUpdateInput, +) (*api.IpSetUpdateInput, error) { + if input.Data != nil { + normalized := normalizeIpSetData(ipset.IpSetType, *input.Data) + if len(normalized) == 0 { + return nil, httperrors.NewInputParameterError("Data is empty") + } + input.Data = &normalized + } + + var err error + input.SharableVirtualResourceBaseUpdateInput, err = ipset.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) + if err != nil { + return nil, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData") + } + return input, nil +} + +func (ipset *SIpSet) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + ipset.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + if ipset.IsManaged() { + regionId, _ := data.GetString("cloudregion_id") + if len(regionId) == 0 { + provider := ipset.GetCloudprovider() + if provider != nil && provider.Provider == api.CLOUD_PROVIDER_QCLOUD && len(ipset.CloudregionId) > 0 { + _, err := db.Update(ipset, func() error { + ipset.CloudregionId = "" + return nil + }) + if err != nil { + log.Errorf("clear cloudregion_id for ipset %s: %v", ipset.Id, err) + } + } + } + ipset.StartCreateTask(ctx, userCred, "") + return + } + ipset.SetStatus(ctx, userCred, apis.STATUS_AVAILABLE, "") + ipset.DoSync(ctx, userCred) +} + +func (ipset *SIpSet) PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) { + ipset.SSharableVirtualResourceBase.PostUpdate(ctx, userCred, query, data) + if ipset.IsManaged() { + ipset.StartUpdateTask(ctx, userCred, "") + return + } + ipset.DoSync(ctx, userCred) +} + +func (ipset *SIpSet) GetSecurityGroups() ([]SSecurityGroup, error) { + sq := SecurityGroupRuleManager.Query("secgroup_id"). + Equals("target_type", api.SecurityGroupRuleTargetTypeIpSet). + Equals("cidr", ipset.Id). + Distinct().SubQuery() + q := SecurityGroupManager.Query().In("id", sq) + ret := []SSecurityGroup{} + err := db.FetchModelObjects(SecurityGroupManager, q, &ret) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return ret, nil +} + +func (ipset *SIpSet) DoSync(ctx context.Context, userCred mcclient.TokenCredential) { + secgroups, err := ipset.GetSecurityGroups() + if err != nil { + log.Errorf("GetSecurityGroups error: %s", err.Error()) + return + } + for i := range secgroups { + if len(secgroups[i].ManagerId) == 0 { + secgroups[i].DoSync(ctx, userCred) + } + } +} + +func normalizeIpSetData(ipSetType api.TIpSetType, data string) string { + parts := strings.Split(data, ",") + normalized := make([]string, 0, len(parts)) + for i := range parts { + cidr := strings.TrimSpace(parts[i]) + if len(cidr) == 0 { + continue + } + switch ipSetType { + case api.IpSetTypeIpv4CidrList: + if pref, err := netutils.NewIPV4Prefix(cidr); err == nil { + normalized = append(normalized, pref.String()) + } + case api.IpSetTypeIpv6CidrList: + if pref, err := netutils.NewIPV6Prefix(cidr); err == nil { + normalized = append(normalized, pref.String()) + } + } + } + return strings.Join(normalized, ",") +} + +func (ipset *SIpSet) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { + if info != nil && info.Contains("security_group_count") { + securityGroupCount, _ := info.Int("security_group_count") + if securityGroupCount > 0 { + return errors.Wrapf(httperrors.ErrNotEmpty, "ipset is being used by %d security groups", securityGroupCount) + } + } else if ipset.GetSecurityGroupCount() > 0 { + return errors.Wrapf(httperrors.ErrNotEmpty, "ipset is being used by %d security groups", ipset.GetSecurityGroupCount()) + } + err := ipset.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx, info) + if err != nil { + return errors.Wrapf(err, "SSharableVirtualResourceBase.ValidateDeleteCondition") + } + return nil +} + +func (ipset *SIpSet) GetSecurityGroupCount() int { + counts, err := IpSetManager.TotalResourceCount([]string{ipset.Id}) + if err != nil { + return 0 + } + return counts[ipset.Id] +} + +func (ipset *SIpSet) getIpNets() []*net.IPNet { + return netutils2.Str2IPNets(ipset.Data) +} + +func (ipset *SIpSet) GetIRegion(ctx context.Context) (cloudprovider.ICloudRegion, error) { + if ipset.IsManaged() && len(ipset.CloudregionId) == 0 { + return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "account level ipset") + } + region, err := ipset.SCloudregionResourceBase.GetRegion() + if err != nil { + return nil, errors.Wrapf(err, "GetRegion") + } + provider, err := ipset.GetDriver(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetDriver") + } + return provider.GetIRegionById(region.ExternalId) +} + +func (ipset *SIpSet) GetICloudIpSet(ctx context.Context) (cloudprovider.ICloudIpSet, error) { + if len(ipset.ExternalId) == 0 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "empty external id") + } + if ipset.IsManaged() && len(ipset.CloudregionId) == 0 { + provider := ipset.GetCloudprovider() + if provider == nil { + return nil, errors.Errorf("empty cloudprovider") + } + driver, err := provider.GetProvider(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetProvider") + } + return driver.GetIIpSetById(ipset.ExternalId) + } + iRegion, err := ipset.GetIRegion(ctx) + if err != nil { + return nil, errors.Wrapf(err, "GetIRegion") + } + return iRegion.GetIIpSetById(ipset.ExternalId) +} + +func (ipset *SIpSet) GetAddresses() []string { + if len(ipset.Data) == 0 { + return []string{} + } + parts := strings.Split(ipset.Data, ",") + ret := make([]string, 0, len(parts)) + for i := range parts { + addr := strings.TrimSpace(parts[i]) + if len(addr) > 0 { + ret = append(ret, addr) + } + } + return ret +} + +func (ipset *SIpSet) StartCreateTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "IpSetCreateTask", ipset, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + ipset.SetStatus(ctx, userCred, apis.STATUS_CREATING, "") + return task.ScheduleRun(nil) +} + +func (ipset *SIpSet) StartUpdateTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "IpSetUpdateTask", ipset, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + ipset.SetStatus(ctx, userCred, apis.STATUS_SYNC_STATUS, "") + return task.ScheduleRun(nil) +} + +func (ipset *SIpSet) StartDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error { + task, err := taskman.TaskManager.NewTask(ctx, "IpSetDeleteTask", ipset, userCred, nil, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + ipset.SetStatus(ctx, userCred, apis.STATUS_DELETING, "") + return task.ScheduleRun(nil) +} + +func (ipset *SIpSet) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { + if ipset.IsManaged() { + return nil + } + return ipset.SSharableVirtualResourceBase.Delete(ctx, userCred) +} + +func (ipset *SIpSet) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return ipset.SSharableVirtualResourceBase.Delete(ctx, userCred) +} + +func (ipset *SIpSet) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + if ipset.IsManaged() { + return ipset.StartDeleteTask(ctx, userCred, "") + } + return nil +} + +func (ipset *SIpSet) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if !ipset.IsManaged() { + return nil, ipset.SetStatus(ctx, userCred, apis.STATUS_AVAILABLE, "") + } + return nil, StartResourceSyncStatusTask(ctx, userCred, ipset, "IpSetSyncstatusTask", "") +} + +func (self *SCloudregion) GetIpSets(managerId string) ([]SIpSet, error) { + q := IpSetManager.Query().Equals("cloudregion_id", self.Id) + if len(managerId) > 0 { + q = q.Equals("manager_id", managerId) + } + ret := []SIpSet{} + err := db.FetchModelObjects(IpSetManager, q, &ret) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return ret, nil +} + +func (self *SCloudregion) SyncIpSets( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudIpSet, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, IpSetManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) + defer lockman.ReleaseRawObject(ctx, IpSetManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) + + result := compare.SyncResult{} + + dbIpSets, err := self.GetIpSets(provider.Id) + if err != nil { + result.Error(err) + return result + } + + removed := make([]SIpSet, 0) + commondb := make([]SIpSet, 0) + commonext := make([]cloudprovider.ICloudIpSet, 0) + added := make([]cloudprovider.ICloudIpSet, 0) + err = compare.CompareSets(dbIpSets, exts, &removed, &commondb, &commonext, &added) + if err != nil { + result.Error(err) + return result + } + + for i := 0; i < len(removed); i++ { + err := removed[i].syncRemove(ctx, userCred) + if err != nil { + result.DeleteError(err) + continue + } + result.Delete() + } + + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudIpSet(ctx, userCred, commonext[i], provider, self.Id) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() + } + } + + for i := 0; i < len(added); i++ { + err = self.newFromCloudIpSet(ctx, userCred, provider, added[i]) + if err != nil { + result.AddError(err) + continue + } + result.Add() + } + return result +} + +func (provider *SCloudprovider) GetIpSets() ([]SIpSet, error) { + q := IpSetManager.Query().Equals("manager_id", provider.Id) + ret := []SIpSet{} + err := db.FetchModelObjects(IpSetManager, q, &ret) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return ret, nil +} + +func (provider *SCloudprovider) SyncIpSets( + ctx context.Context, + userCred mcclient.TokenCredential, + exts []cloudprovider.ICloudIpSet, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, IpSetManager.Keyword(), provider.Id) + defer lockman.ReleaseRawObject(ctx, IpSetManager.Keyword(), provider.Id) + + result := compare.SyncResult{} + + dbIpSets, err := provider.GetIpSets() + if err != nil { + result.Error(err) + return result + } + + removed := make([]SIpSet, 0) + commondb := make([]SIpSet, 0) + commonext := make([]cloudprovider.ICloudIpSet, 0) + added := make([]cloudprovider.ICloudIpSet, 0) + err = compare.CompareSets(dbIpSets, exts, &removed, &commondb, &commonext, &added) + if err != nil { + result.Error(err) + return result + } + + for i := 0; i < len(removed); i++ { + if len(removed[i].CloudregionId) > 0 { + continue + } + err := removed[i].syncRemove(ctx, userCred) + if err != nil { + result.DeleteError(err) + continue + } + result.Delete() + } + + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudIpSet(ctx, userCred, commonext[i], provider, "") + if err != nil { + result.UpdateError(err) + continue + } + result.Update() + } + } + + for i := 0; i < len(added); i++ { + err = provider.newFromCloudIpSet(ctx, userCred, added[i]) + if err != nil { + result.AddError(err) + continue + } + result.Add() + } + return result +} + +func (provider *SCloudprovider) newFromCloudIpSet(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudIpSet) error { + ret := &SIpSet{} + ret.SetModelManager(IpSetManager, ret) + ret.Name = ext.GetName() + ret.CloudregionId = "" + ret.ManagerId = provider.Id + ret.ExternalId = ext.GetGlobalId() + ret.Status = apis.STATUS_AVAILABLE + ret.IpSetType = api.TIpSetType(ext.GetIpSetType()) + ret.Description = ext.GetDescription() + ret.Data = strings.Join(ext.GetAddresses(), ",") + err := IpSetManager.TableSpec().Insert(ctx, ret) + if err != nil { + return errors.Wrapf(err, "Insert") + } + syncVirtualResourceMetadata(ctx, userCred, ret, ext, false) + SyncCloudProject(ctx, userCred, ret, provider.GetOwnerId(), ext, provider) + return nil +} + +func (ipset *SIpSet) syncRemove(ctx context.Context, userCred mcclient.TokenCredential) error { + return ipset.RealDelete(ctx, userCred) +} + +func (ipset *SIpSet) SyncWithCloudIpSet(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudIpSet, provider *SCloudprovider, cloudregionId string) error { + _, err := db.Update(ipset, func() error { + ipset.Status = apis.STATUS_AVAILABLE + ipset.CloudregionId = cloudregionId + if options.Options.EnableSyncName { + ipset.Name = ext.GetName() + } + ipset.IpSetType = api.TIpSetType(ext.GetIpSetType()) + ipset.Data = strings.Join(ext.GetAddresses(), ",") + if desc := ext.GetDescription(); len(desc) > 0 { + ipset.Description = desc + } + return nil + }) + if err != nil { + return err + } + syncVirtualResourceMetadata(ctx, userCred, ipset, ext, false) + SyncCloudProject(ctx, userCred, ipset, provider.GetOwnerId(), ext, provider) + return nil +} + +func (self *SCloudregion) newFromCloudIpSet(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, ext cloudprovider.ICloudIpSet) error { + ret := &SIpSet{} + ret.SetModelManager(IpSetManager, ret) + ret.Name = ext.GetName() + ret.CloudregionId = self.Id + ret.ManagerId = provider.Id + ret.ExternalId = ext.GetGlobalId() + ret.Status = apis.STATUS_AVAILABLE + ret.IpSetType = api.TIpSetType(ext.GetIpSetType()) + ret.Description = ext.GetDescription() + ret.Data = strings.Join(ext.GetAddresses(), ",") + err := IpSetManager.TableSpec().Insert(ctx, ret) + if err != nil { + return errors.Wrapf(err, "Insert") + } + syncVirtualResourceMetadata(ctx, userCred, ret, ext, false) + SyncCloudProject(ctx, userCred, ret, provider.GetOwnerId(), ext, provider) + return nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_device_models.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_device_models.go index ee6b79555..1360721ea 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_device_models.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_device_models.go @@ -24,7 +24,6 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -77,10 +76,6 @@ func (manager *SIsolatedDeviceModelManager) ValidateCreateData(ctx context.Conte query jsonutils.JSONObject, input api.IsolatedDeviceModelCreateInput, ) (api.IsolatedDeviceModelCreateInput, error) { - if utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { - return input, httperrors.NewInputParameterError("device type %q is not supported", input.DevType) - } - input.VendorId = strings.ToLower(input.VendorId) input.DeviceId = strings.ToLower(input.DeviceId) deviceVendorReg := regexp.MustCompile(`^[a-f0-9]{4}$`) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_devices.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_devices.go index 1d22102d2..87ea1771e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_devices.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolated_devices.go @@ -31,7 +31,6 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/rbacscope" - "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -48,25 +47,6 @@ import ( "yunion.io/x/onecloud/pkg/util/stringutils2" ) -const ( -//DIRECT_PCI_TYPE = api.DIRECT_PCI_TYPE -//GPU_HPC_TYPE = api.GPU_HPC_TYPE // # for compute -//GPU_VGA_TYPE = api.GPU_VGA_TYPE // # for display -//USB_TYPE = api.USB_TYPE -//NIC_TYPE = api.NIC_TYPE - -// NVIDIA_VENDOR_ID = api.NVIDIA_VENDOR_ID -// AMD_VENDOR_ID = api.AMD_VENDOR_ID -) - -var VALID_GPU_TYPES = api.VALID_GPU_TYPES - -var VALID_PASSTHROUGH_TYPES = api.VALID_PASSTHROUGH_TYPES - -var ID_VENDOR_MAP = api.ID_VENDOR_MAP - -var VENDOR_ID_MAP = api.VENDOR_ID_MAP - type SIsolatedDeviceManager struct { db.SStandaloneResourceBaseManager db.SExternalizedResourceBaseManager @@ -76,6 +56,12 @@ type SIsolatedDeviceManager struct { var IsolatedDeviceManager *SIsolatedDeviceManager +const ( + isolatedDeviceInitializeDataObjType = "system" + isolatedDeviceInitializeDataObjId = "compute_isolated_device_initialize_data" + isolatedDeviceInitializeDataKey = "__initialized" +) + func init() { gotypes.RegisterSerializable(reflect.TypeOf(&api.IsolatedDevicePCIEInfo{}), func() gotypes.ISerializable { return &api.IsolatedDevicePCIEInfo{} @@ -98,17 +84,23 @@ type SIsolatedDevice struct { db.SSharableBaseResource `"is_public->create":"domain_optional" "public_scope->create":"domain_optional"` SHostResourceBase `width:"36" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required"` - // # PCI / GPU-HPC / GPU-VGA / USB / NIC + // # PCI / GPU / USB / NIC ... // 设备类型 DevType string `width:"128" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"` - + // EXCLUSIVE / SRIOV / MPS / HAMI / SHARE / MIG + SharingMode string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_required"` + // Device is hot pluggable + HotPluggable bool `default:"false" list:"domain" create:"domain_optional" update:"domain"` // # Specific device name read from lspci command, e.g. `Tesla K40m` ... Model string `width:"512" charset:"ascii" nullable:"false" default:"" index:"true" list:"domain" create:"domain_required" update:"domain"` - // 云主机Id - GuestId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain"` - // guest network index - NetworkIndex int `nullable:"true" default:"-1" list:"user" update:"user"` + // 云主机Id, keep for backward compatibility + // swagger:deprecated + GuestId string `width:"36" charset:"ascii" nullable:"true"` + // guest network index, keep for backward compatibility + // swagger:deprecated + NetworkIndex int `nullable:"true" default:"-1"` + // Nic wire id WireId string `width:"36" charset:"ascii" nullable:"true" index:"true" list:"domain" update:"domain" create:"domain_optional"` // Offload interface name @@ -117,11 +109,10 @@ type SIsolatedDevice struct { IsInfinibandNic bool `nullable:"false" default:"false" list:"user" create:"optional"` // NVME disk size NvmeSizeMB int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"` - // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, - // or per-slice quota for MPS share mode). 0 means unknown / not applicable. - MemorySize int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` - // guest disk index - DiskIndex int8 `nullable:"true" default:"-1" list:"user" update:"user"` + + // guest disk index, keep for backward compatibility + // swagger:deprecated + DiskIndex int8 `nullable:"true" default:"-1"` // # pci address of `Bus:Device.Function` format, or usb bus address of `bus:addr:port` Addr string `width:"16" charset:"ascii" nullable:"true" list:"domain" update:"domain" create:"domain_optional"` @@ -177,6 +168,20 @@ type SIsolatedDevice struct { PcieInfo *api.IsolatedDevicePCIEInfo `nullable:"true" create:"optional" list:"user" get:"user" update:"domain"` // device numa node NumaNode int8 `nullable:"true" default:"-1" list:"domain" update:"domain" create:"domain_optional"` + + // On-device memory in MiB (NVIDIA GPU VRAM via `nvidia-smi memory.total`, + // or per-slice quota for MPS share mode). 0 means unknown / not applicable. + MemorySize int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` + // some of isolated device type support virtual num, like NVIDIA_GPU_SHARE, NVIDIA_MPS + VirtualNum int `nullable:"true" default:"1" list:"user" update:"domain" create:"domain_optional"` +} + +func (manager *SIsolatedDeviceManager) GetIVirtualModelManager() db.IVirtualModelManager { + return manager +} + +func (manager *SIsolatedDeviceManager) GetResourceCount() ([]db.SScopeResourceCount, error) { + return []db.SScopeResourceCount{}, nil } func (manager *SIsolatedDeviceManager) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition { @@ -204,12 +209,16 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, if input.DevType == "" { return input, httperrors.NewNotEmptyError("dev_type is empty") } - if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) { + if !utils.IsInStringArray(input.DevType, api.VALID_TYPES) { if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil { return input, httperrors.NewInputParameterError("device type %q is not supported", input.DevType) } } + if !utils.IsInStringArray(input.SharingMode, api.VAILD_SHARING_MODES) { + return input, httperrors.NewNotEmptyError("sharing_mode %s is not valid", input.SharingMode) + } + input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput) if err != nil { return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData") @@ -225,7 +234,7 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, // validate reserverd resource // inject default reserverd resource for gpu: - if utils.IsInStringArray(input.DevType, []string{api.GPU_HPC_TYPE, api.GPU_VGA_TYPE}) { + if host.HostType == api.HOST_TYPE_HYPERVISOR && input.SharingMode == api.DEVICE_SHARING_MODE_EXCLUSIVE && input.DevType == api.GPU_TYPE { defaultCPU := 8 // 8 defaultMem := 8192 // 8g defaultStore := 102400 // 100g @@ -270,18 +279,6 @@ func (self *SIsolatedDevice) ValidateUpdateData( if input.ReservedStorage != nil && *input.ReservedStorage < 0 { return input, httperrors.NewInputParameterError("reserved storage must >= 0") } - if input.DevType != "" && input.DevType != self.DevType { - if !utils.IsInStringArray(input.DevType, api.VALID_GPU_TYPES) { - if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil { - return input, httperrors.NewInputParameterError("device type %q does not support update", input.DevType) - } - } else { - if !self.IsGPU() { - return input, httperrors.NewInputParameterError("Can't update for device %q", self.DevType) - } - } - } - return input, nil } @@ -342,7 +339,7 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( q = q.Equals("dev_type", "USB") } if query.Unused != nil && *query.Unused { - q = q.IsEmpty("guest_id") + q = manager.queryWithoutGuest(q) } if len(query.DevType) > 0 { @@ -360,6 +357,13 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( if len(query.VendorDeviceId) > 0 { q = q.In("vendor_device_id", query.VendorDeviceId) } + if len(query.Vendor) > 0 { + conds := make([]sqlchemy.ICondition, 0, len(query.Vendor)) + for _, v := range query.Vendor { + conds = append(conds, sqlchemy.Startswith(q.Field("vendor_device_id"), vendorDeviceIdPrefixForFilter(v))) + } + q = q.Filter(sqlchemy.OR(conds...)) + } if len(query.NumaNode) > 0 { q = q.In("numa_node", query.NumaNode) } @@ -380,7 +384,8 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( if err != nil { return nil, errors.Wrapf(err, "Fetch guest by %q", query.GuestId) } - q = q.Equals("guest_id", obj.GetId()) + gq := GuestIsolatedDeviceManager.Query().Equals("guest_id", obj.GetId()).SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) } return q, nil @@ -425,7 +430,9 @@ func (manager *SIsolatedDeviceManager) ListItemExportKeys(ctx context.Context, q } if keys.Contains("guest") { guestNameQuery := GuestManager.Query("name", "id").SubQuery() - q.LeftJoin(guestNameQuery, sqlchemy.Equals(q.Field("guest_id"), guestNameQuery.Field("id"))) + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + q.LeftJoin(guestNameQuery, sqlchemy.Equals(gq.Field("guest_id"), guestNameQuery.Field("id"))) q.AppendField(guestNameQuery.Field("name", "guest")) } if keys.Contains("host") { @@ -448,7 +455,11 @@ func (manager *SIsolatedDeviceManager) GetExportExtraKeys(ctx context.Context, k } func (self *SIsolatedDevice) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error { - if len(self.GuestId) > 0 { + gdevs, err := self.GetAllGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) > 0 { return httperrors.NewNotEmptyError("Isolated device used by server") } return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx, nil) @@ -458,18 +469,7 @@ func (self *SIsolatedDevice) getDetailedString() string { return fmt.Sprintf("%s:%s/%s/%s", self.Addr, self.Model, self.VendorDeviceId, self.DevType) } -func (manager *SIsolatedDeviceManager) findAttachedDevicesOfGuest(guest *SGuest) []SIsolatedDevice { - devs := make([]SIsolatedDevice, 0) - q := manager.Query().Equals("guest_id", guest.Id) - err := db.FetchModelObjects(manager, q, &devs) - if err != nil { - log.Errorf("findAttachedDevicesOfGuest error %s", err) - return nil - } - return devs -} - -func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr string, devType string) *SIsolatedDevice { +func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr, devType, sharingMode string) *SIsolatedDevice { dev := SIsolatedDevice{} dev.SetModelManager(manager, &dev) @@ -477,6 +477,9 @@ func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr string, devType if devType != "" { q = q.Equals("dev_type", devType) } + if sharingMode != "" { + q = q.Equals("sharing_mode", sharingMode) + } if fuzzyStr != "" { qe := q.Equals("model", fuzzyStr) @@ -501,7 +504,7 @@ func (self *SIsolatedDevice) getVendorId() string { func (self *SIsolatedDevice) getVendor() string { vendorId := self.getVendorId() - vendor, ok := ID_VENDOR_MAP[vendorId] + vendor, ok := api.ID_VENDOR_MAP[vendorId] if ok { return vendor } else { @@ -512,7 +515,7 @@ func (self *SIsolatedDevice) getVendor() string { func GetVendorByVendorDeviceId(vendorDeviceId string) string { parts := strings.Split(vendorDeviceId, ":") vendorId := parts[0] - vendor, ok := ID_VENDOR_MAP[vendorId] + vendor, ok := api.ID_VENDOR_MAP[vendorId] if ok { return vendor } else { @@ -520,8 +523,24 @@ func GetVendorByVendorDeviceId(vendorDeviceId string) string { } } +func vendorDeviceIdPrefixForFilter(vendor string) string { + return resolveVendorIdForFilter(vendor) + ":" +} + +func resolveVendorIdForFilter(vendor string) string { + if id, ok := api.VENDOR_ID_MAP[vendor]; ok { + return id + } + for name, id := range api.VENDOR_ID_MAP { + if strings.EqualFold(name, vendor) { + return id + } + } + return strings.ToLower(vendor) +} + func (self *SIsolatedDevice) IsGPU() bool { - return strings.HasPrefix(self.DevType, "GPU") || sets.NewString(api.CONTAINER_GPU_TYPES...).Has(self.DevType) + return self.DevType == api.GPU_TYPE } func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCredential, devConfig *api.IsolatedDeviceConfig) (*api.IsolatedDeviceConfig, error) { @@ -529,7 +548,7 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr var matchDev *SIsolatedDevice devId = devConfig.Id - matchDev = manager.fuzzyMatchModel(devConfig.Model, devConfig.DevType) + matchDev = manager.fuzzyMatchModel(devConfig.Model, devConfig.DevType, devConfig.SharingMode) devVendor = devConfig.Vendor devType = devConfig.DevType @@ -538,8 +557,10 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr return nil, httperrors.NewNotFoundError("Not found matched device by model: %q, dev_type: %q", devConfig.Model, devConfig.DevType) } devConfig.Model = matchDev.Model + devConfig.SharingMode = matchDev.SharingMode + if len(devVendor) > 0 { - vendorId, ok := VENDOR_ID_MAP[devVendor] + vendorId, ok := api.VENDOR_ID_MAP[devVendor] if ok { devConfig.Vendor = vendorId } else { @@ -558,17 +579,21 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr devConfig.Id = dev.Id devConfig.Model = dev.Model devConfig.DevType = dev.DevType + devConfig.SharingMode = dev.SharingMode devConfig.Vendor = dev.getVendor() devConfig.WireId = dev.WireId - if dev.IsGPU() && len(devType) > 0 { - if !utils.IsInStringArray(devType, VALID_GPU_TYPES) { - return nil, fmt.Errorf("%s not valid for GPU device", devType) - } + if devType != "" && devType != dev.DevType { + return nil, fmt.Errorf("request dev_type %s not match dev %s type %s", devType, dev.Id, dev.DevType) } } if len(devType) > 0 { devConfig.DevType = devType } + if devConfig.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + if devConfig.MemoryRequest <= 0 { + return nil, httperrors.NewBadRequestError("dev sharing_mode %s must give memory request", devConfig.SharingMode) + } + } return devConfig, nil } @@ -579,10 +604,13 @@ func (manager *SIsolatedDeviceManager) isValidDeviceInfo(config *api.IsolatedDev return httperrors.NewResourceNotFoundError("IsolatedDevice %s not found", config.Id) } dev := devObj.(*SIsolatedDevice) - if len(dev.GuestId) > 0 { - return httperrors.NewConflictError("Isolated device already attached to another guest: %s", dev.GuestId) + if dev.IsFull() { + return httperrors.NewConflictError("Isolated device already attached") } } + if config.GpuType != "" && !utils.IsInStringArray(config.GpuType, []string{api.GPU_HPC, api.GPU_VGA}) { + return httperrors.NewInputParameterError("Input gpu_type %s not valid", config.GpuType) + } return nil } @@ -601,8 +629,8 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe return httperrors.NewResourceNotFoundError("IsolatedDevice %s not found", config.Id) } dev := devObj.(*SIsolatedDevice) - if len(dev.GuestId) > 0 { - return httperrors.NewConflictError("Isolated device already attached to another guest: %s", dev.GuestId) + if dev.IsFull() { + return httperrors.NewConflictError("Isolated device already attached") } if dev.DevType != devType { return httperrors.NewBadRequestError("IsolatedDevice is not device type %s", devType) @@ -628,14 +656,21 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc( func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.Context, guest *SGuest, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { devObj, err := manager.FetchById(devConfig.Id) - if devObj == nil { - return fmt.Errorf("Device %s not found: %s", devConfig.Id, err) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return httperrors.NewResourceNotFoundError2(manager.Keyword(), devConfig.Id) + } else { + return errors.Wrap(err, "SIsolatedDeviceManager.FetchById") + } } dev := devObj.(*SIsolatedDevice) if len(devConfig.DevType) > 0 && devConfig.DevType != dev.DevType { dev.DevType = devConfig.DevType } - return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex) + if !dev.IsEnough(devConfig.MemoryRequest) { + return errors.Errorf("Dev %s not enough", dev.Id) + } + return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential, usedDevMap map[string]*SIsolatedDevice, preferNumaNodes []int) error { @@ -643,17 +678,27 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig) } // if dev type is not nic, wire is empty string - devs, err := manager.findHostUnusedByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId) + devs, err := manager.findHostAvailableByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId, devConfig.SharingMode) if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) } + devs = filterDevicesBySharingMode(devs, devConfig.SharingMode) + if len(devs) == 0 { + return fmt.Errorf("Can't found model %s device_path %s sharing_mode %s on host %s", + devConfig.Model, devConfig.DevicePath, devConfig.SharingMode, host.Id) + } devs = filterDevicesByMemoryMb(devs, devConfig.MemoryMb) if len(devs) == 0 { return fmt.Errorf("device_path %s on host %s does not satisfy memory_mb=%d", devConfig.DevicePath, host.Id, devConfig.MemoryMb) } + var selectedDev SIsolatedDevice for i := range devs { + if !devs[i].IsEnough(devConfig.MemoryRequest) { + continue + } + if _, ok := usedDevMap[devs[i].DevicePath]; !ok { selectedDev = devs[i] usedDevMap[devs[i].DevicePath] = &selectedDev @@ -662,7 +707,10 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c if selectedDev.Id == "" { selectedDev = devs[0] } - return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) + if !selectedDev.IsEnough(devConfig.MemoryRequest) { + return errors.Errorf("Dev %s not enough", selectedDev.Id) + } + return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } // filterDevicesByMemoryMb drops devices whose MemorySize > 0 and is below the @@ -683,6 +731,19 @@ func filterDevicesByMemoryMb(devs []SIsolatedDevice, minMemMb int) []SIsolatedDe return out } +func filterDevicesBySharingMode(devs []SIsolatedDevice, sharingMode string) []SIsolatedDevice { + if sharingMode == "" { + return devs + } + out := make([]SIsolatedDevice, 0, len(devs)) + for _, d := range devs { + if d.SharingMode == sharingMode { + out = append(out, d) + } + } + return out +} + type GroupDevs struct { DevPath string Devs []SIsolatedDevice @@ -721,7 +782,7 @@ type SNodeIsolateDevicesInfo struct { func (manager *SIsolatedDeviceManager) getDevNodesUsedRate( ctx context.Context, host *SHost, devConfig *api.IsolatedDeviceConfig, topo *hostapi.HostTopology, ) (map[string]SNodeIsolateDevicesInfo, error) { - devs, err := manager.findHostDevsByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId) + devs, err := manager.findHostDevsByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId, devConfig.SharingMode) if err != nil || len(devs) == 0 { return nil, fmt.Errorf("Can't found model %s on host %s", devConfig.Model, host.Id) } @@ -836,10 +897,15 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( return fmt.Errorf("Not found model from info: %#v", devConfig) } // if dev type is not nic, wire is empty string - devs, err := manager.findHostUnusedByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId) + devs, err := manager.findHostAvailableByDevConfig(devConfig.Model, devConfig.DevType, host.Id, devConfig.WireId, devConfig.SharingMode) if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s on host %s", devConfig.Model, host.Id) } + devs = filterDevicesBySharingMode(devs, devConfig.SharingMode) + if len(devs) == 0 { + return fmt.Errorf("Can't found model %s sharing_mode %s on host %s", + devConfig.Model, devConfig.SharingMode, host.Id) + } // Honour the request's VRAM floor. Predicate already verified enough // fitting devices exist on the host; here we make sure attach picks one // of them rather than a same-Model but smaller-VRAM card. @@ -852,6 +918,10 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( //groupDevs := make(SorttedGroupDevs, 0) mapDevs := map[string][]SIsolatedDevice{} for i := range devs { + if !devs[i].IsEnough(devConfig.MemoryRequest) { + continue + } + dev := devs[i] devPath := dev.DevicePath var gdevs []SIsolatedDevice @@ -1048,27 +1118,12 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( devAddr := strings.Split(selectedDev.Addr, "-")[0] usedDevMap[devAddr] = selectedDev - return guest.attachIsolatedDevice(ctx, userCred, selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) -} - -func (manager *SIsolatedDeviceManager) findUnusedQuery() *sqlchemy.SQuery { - isolateddevs := manager.Query().SubQuery() - q := isolateddevs.Query().Filter(sqlchemy.OR(sqlchemy.IsNull(isolateddevs.Field("guest_id")), - sqlchemy.IsEmpty(isolateddevs.Field("guest_id")))) - return q -} - -func (manager *SIsolatedDeviceManager) UnusedGpuQuery() *sqlchemy.SQuery { - q := manager.findUnusedQuery() - q = q.Filter(sqlchemy.OR( - sqlchemy.Equals(q.Field("dev_type"), api.GPU_HPC_TYPE), - sqlchemy.Equals(q.Field("dev_type"), api.GPU_VGA_TYPE))) - return q + return guest.attachIsolatedDevice(ctx, userCred, selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex, &devConfig.MemoryRequest, devConfig.GpuType) } -func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) FindAvailableByModels(models []string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.findUnusedQuery() + q := manager.GetAvailableIsolatedDeviceQuery(nil) q = q.In("model", models) err := db.FetchModelObjects(manager, q, &devs) if err != nil { @@ -1077,8 +1132,8 @@ func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SI return devs, nil } -func (manager *SIsolatedDeviceManager) FindUnusedNicWiresByModel(modelName string) ([]string, error) { - q := manager.Query().IsNullOrEmpty("guest_id").Equals("dev_type", api.NIC_TYPE) +func (manager *SIsolatedDeviceManager) FindAvailableNicWiresByModel(modelName string) ([]string, error) { + q := manager.Query().Equals("dev_type", api.NIC_TYPE) if len(modelName) > 0 { q = q.Equals("model", modelName) } @@ -1090,15 +1145,19 @@ func (manager *SIsolatedDeviceManager) FindUnusedNicWiresByModel(modelName strin } wires := make([]string, len(devs)) for i := 0; i < len(devs); i++ { + if devs[i].IsFull() { + continue + } + wires[i] = devs[i].WireId } return wires, err } -func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) FindAvailableGpusOnHost(hostId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.UnusedGpuQuery() - q = q.Equals("host_id", hostId) + q := manager.GetAvailableIsolatedDeviceQuery(nil) + q = q.Equals("dev_type", api.GPU_TYPE).Equals("host_id", hostId) err := db.FetchModelObjects(manager, q, &devs) if err != nil { return nil, err @@ -1106,19 +1165,22 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI return devs, nil } -func (manager *SIsolatedDeviceManager) findHostUnusedByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { - return manager.findHostUnusedByDevAttr(model, "dev_type", devType, hostId, wireId) +func (manager *SIsolatedDeviceManager) findHostAvailableByDevConfig(model, devType, hostId, wireId, sharingMode string) ([]SIsolatedDevice, error) { + return manager.findHostAvailableByDevAttr(model, "dev_type", devType, hostId, wireId, sharingMode) } -func (manager *SIsolatedDeviceManager) findHostDevsByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { - return manager.findHostDevsByDevAttr(model, "dev_type", devType, hostId, wireId) +func (manager *SIsolatedDeviceManager) findHostDevsByDevConfig(model, devType, hostId, wireId, sharingMode string) ([]SIsolatedDevice, error) { + return manager.findHostDevsByDevAttr(model, "dev_type", devType, hostId, wireId, sharingMode) } -func (manager *SIsolatedDeviceManager) findHostDevsByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) findHostDevsByDevAttr(model, attrKey, attrVal, hostId, wireId, sharingMode string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.Query() q = q.Equals("model", model).Equals("host_id", hostId) if attrVal != "" { - q.Equals(attrKey, attrVal) + q = q.Equals(attrKey, attrVal) + } + if sharingMode != "" { + q = q.Equals("sharing_mode", sharingMode) } if wireId != "" { wire := WireManager.FetchWireById(wireId) @@ -1133,12 +1195,15 @@ func (manager *SIsolatedDeviceManager) findHostDevsByDevAttr(model, attrKey, att return devs, nil } -func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) findHostAvailableByDevAttr(model, attrKey, attrVal, hostId, wireId, sharingMode string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.findUnusedQuery() + q := manager.GetAvailableIsolatedDeviceQuery(nil) q = q.Equals("model", model).Equals("host_id", hostId) if attrVal != "" { - q.Equals(attrKey, attrVal) + q = q.Equals(attrKey, attrVal) + } + if sharingMode != "" { + q = q.Equals("sharing_mode", sharingMode) } if wireId != "" { wire := WireManager.FetchWireById(wireId) @@ -1154,20 +1219,19 @@ func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, a } func (manager *SIsolatedDeviceManager) ReleaseGPUDevicesOfGuest(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential) error { - devs := manager.findAttachedDevicesOfGuest(guest) - if devs == nil { + gdevs, err := guest.GetGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) == 0 { return fmt.Errorf("fail to find attached devices") } - for _, dev := range devs { - if !utils.IsInStringArray(dev.DevType, api.VALID_GPU_TYPES) { + for _, gdev := range gdevs { + dev := gdev.GetIsolatedDevice() + if !dev.IsKvmExclusiveGPU() { continue } - _, err := db.Update(&dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - dev.DiskIndex = -1 - return nil - }) + err := gdev.Detach(ctx, userCred) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_GUEST_DETACH_ISOLATED_DEVICE_FAIL, dev.GetShortDesc(ctx), userCred) return err @@ -1178,16 +1242,16 @@ func (manager *SIsolatedDeviceManager) ReleaseGPUDevicesOfGuest(ctx context.Cont } func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential) error { - devs := manager.findAttachedDevicesOfGuest(guest) - if devs == nil { + gdevs, err := guest.GetGuestIsolatedDevices() + if err != nil { + return err + } + if len(gdevs) == 0 { return fmt.Errorf("fail to find attached devices") } - for _, dev := range devs { - _, err := db.Update(&dev, func() error { - dev.GuestId = "" - dev.NetworkIndex = -1 - return nil - }) + for _, gdev := range gdevs { + dev := gdev.GetIsolatedDevice() + err := gdev.Detach(ctx, userCred) if err != nil { db.OpsLog.LogEvent(guest, db.ACT_GUEST_DETACH_ISOLATED_DEVICE_FAIL, dev.GetShortDesc(ctx), userCred) return err @@ -1197,6 +1261,19 @@ func (manager *SIsolatedDeviceManager) ReleaseDevicesOfGuest(ctx context.Context return nil } +func (manager *SIsolatedDeviceManager) queryWithoutGuest(q *sqlchemy.SQuery) *sqlchemy.SQuery { + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.LeftJoin(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + q = q.Filter(sqlchemy.IsNull(gq.Field("isolated_device_id"))) + return q +} + +func (manager *SIsolatedDeviceManager) queryWithGuest(q *sqlchemy.SQuery) *sqlchemy.SQuery { + gq := GuestIsolatedDeviceManager.Query().SubQuery() + q = q.Join(gq, sqlchemy.Equals(q.Field("id"), gq.Field("isolated_device_id"))) + return q +} + func (manager *SIsolatedDeviceManager) totalCountQ( ctx context.Context, scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, devType []string, hostTypes []string, @@ -1213,7 +1290,9 @@ func (manager *SIsolatedDeviceManager) totalCountQ( hosts := hq.SubQuery() devs := manager.Query().SubQuery() q := devs.Query().Join(hosts, sqlchemy.Equals(devs.Field("host_id"), hosts.Field("id"))) - q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) + // 透传设备 usage 统计全部宿主机（含禁用），与 hosts 而非 enabled_hosts 口径一致。 + // 禁用宿主机上的 GPU 仍需在 usages/hosts 中展示，调度侧仍只使用启用宿主机。 + // q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled"))) if len(devType) != 0 { q = q.Filter(sqlchemy.In(devs.Field("dev_type"), devType)) } @@ -1260,12 +1339,14 @@ func (manager *SIsolatedDeviceManager) totalCount( policyResult, ) sq := iq.SubQuery() + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() q := sq.Query( sq.Field("dev_type"), - sq.Field("guest_id"), + guestIdevs.Field("guest_id"), sqlchemy.COUNT("count", sq.Field("id")), ) - q = q.GroupBy(q.Field("dev_type"), q.Field("guest_id")) + q = q.LeftJoin(guestIdevs, sqlchemy.Equals(sq.Field("id"), guestIdevs.Field("isolated_device_id"))) + q = q.GroupBy(sq.Field("dev_type"), guestIdevs.Field("guest_id")) ret := []IsolatedDeviceStat{} err := q.All(&ret) if err != nil { @@ -1297,12 +1378,12 @@ func (manager *SIsolatedDeviceManager) TotalCount( } for _, s := range stat { ret.Devices += s.Count - if utils.IsInStringArray(s.DevType, VALID_GPU_TYPES) { + if s.DevType == api.GPU_TYPE { ret.Gpus += s.Count } if len(s.GuestId) > 0 { ret.DevicesUsed += s.Count - if utils.IsInStringArray(s.DevType, VALID_GPU_TYPES) { + if s.DevType == api.GPU_TYPE { ret.GpusUsed += s.Count } } @@ -1315,12 +1396,11 @@ func (self *SIsolatedDevice) getDesc() *api.IsolatedDeviceJsonDesc { Id: self.Id, DevType: self.DevType, Model: self.Model, + SharingMode: self.SharingMode, Addr: self.Addr, VendorDeviceId: self.VendorDeviceId, Vendor: self.getVendor(), - NetworkIndex: self.NetworkIndex, OvsOffloadInterface: self.OvsOffloadInterface, - DiskIndex: self.DiskIndex, NvmeSizeMB: self.NvmeSizeMB, MemorySize: self.MemorySize, MdevId: self.MdevId, @@ -1334,9 +1414,9 @@ func (man *SIsolatedDeviceManager) GetSpecShouldCheckStatus(query *jsonutils.JSO func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonutils.JSONObject, error) { hostQ := HostManager.Query() - q := man.Query("vendor_device_id", "model", "dev_type") + q := man.Query("vendor_device_id", "model", "dev_type", "sharing_mode", "nvme_size_mb", "memory_size") if statusCheck { - q = q.IsNullOrEmpty("guest_id") + q = man.GetAvailableIsolatedDeviceQuery(q) hostQ = hostQ.Equals("status", api.BAREMETAL_RUNNING).IsTrue("enabled"). In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER, api.HOST_TYPE_ZETTAKIT}) } @@ -1344,7 +1424,7 @@ func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonuti q.Join(hostSQ, sqlchemy.Equals(q.Field("host_id"), hostSQ.Field("id"))) q.AppendField(hostSQ.Field("host_type")) - q.GroupBy(hostSQ.Field("host_type"), q.Field("vendor_device_id"), q.Field("model"), q.Field("dev_type")) + q.GroupBy(hostSQ.Field("host_type"), q.Field("vendor_device_id"), q.Field("model"), q.Field("dev_type"), q.Field("sharing_mode"), q.Field("nvme_size_mb"), q.Field("memory_size")) q.AppendField(sqlchemy.COUNT("*")) rows, err := q.Rows() @@ -1355,46 +1435,54 @@ func (man *SIsolatedDeviceManager) BatchGetModelSpecs(statusCheck bool) (jsonuti res := jsonutils.NewDict() for rows.Next() { - var hostType, vendorDeviceId, m, t string + var hostType, vendorDeviceId, m, t, s string + var nvmeSize, memorySize int var count int - if err := rows.Scan(&vendorDeviceId, &m, &t, &hostType, &count); err != nil { + if err := rows.Scan(&vendorDeviceId, &m, &t, &s, &nvmeSize, &memorySize, &hostType, &count); err != nil { return nil, errors.Wrap(err, "get model spec scan rows") } vendor := GetVendorByVendorDeviceId(vendorDeviceId) - specKeys := man.getSpecKeys(vendor, m, t) + specKeys := man.getSpecKeys(vendor, m, t, s) specKey := GetSpecIdentKey(specKeys) - spec := man.getSpecByRows(hostType, vendorDeviceId, m, t, &count) + spec := man.getSpecByRows(hostType, vendorDeviceId, m, t, s, &nvmeSize, &memorySize, &count) res.Set(specKey, spec) } return res, nil } -func (man *SIsolatedDeviceManager) getSpecByRows(hostType, vendorDeviceId, model, devType string, count *int) *jsonutils.JSONDict { +func (man *SIsolatedDeviceManager) getSpecByRows(hostType, vendorDeviceId, model, devType, sharingMode string, nvmeSize, memorySize, count *int) *jsonutils.JSONDict { var vdev bool var hypervisor string - if utils.IsInStringArray(devType, api.VITRUAL_DEVICE_TYPES) { + if utils.IsInStringArray(sharingMode, api.VIRTUAL_SHARING_MODES) { vdev = true } - if utils.IsInStringArray(devType, api.VALID_CONTAINER_DEVICE_TYPES) { + switch hostType { + case api.HOST_TYPE_CONTAINER: hypervisor = api.HYPERVISOR_POD - } else { - hypervisor = api.HYPERVISOR_KVM - } - if hostType == api.HOST_TYPE_ZETTAKIT { + case api.HOST_TYPE_ZETTAKIT: hypervisor = api.HYPERVISOR_ZETTAKIT + default: + hypervisor = api.HYPERVISOR_KVM } ret := jsonutils.NewDict() ret.Set("virtual_dev", jsonutils.NewBool(vdev)) ret.Set("hypervisor", jsonutils.NewString(hypervisor)) ret.Set("dev_type", jsonutils.NewString(devType)) + ret.Set("sharing_mode", jsonutils.NewString(sharingMode)) ret.Set("model", jsonutils.NewString(model)) ret.Set("pci_id", jsonutils.NewString(vendorDeviceId)) ret.Set("vendor", jsonutils.NewString(GetVendorByVendorDeviceId(vendorDeviceId))) if count != nil { ret.Set("count", jsonutils.NewInt(int64(*count))) } + if nvmeSize != nil { + ret.Set("nvme_size_mb", jsonutils.NewInt(int64(*nvmeSize))) + } + if memorySize != nil { + ret.Set("memory_size_mb", jsonutils.NewInt(int64(*memorySize))) + } return ret } @@ -1410,7 +1498,8 @@ type GpuSpec struct { func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { host := self.getHost() if statusCheck { - if len(self.GuestId) > 0 { + gdevs, _ := self.GetAllGuestIsolatedDevices() + if len(gdevs) > 0 { return nil } if host.Status != api.BAREMETAL_RUNNING || !host.GetEnabled() || @@ -1418,7 +1507,7 @@ func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { return nil } } - return IsolatedDeviceManager.getSpecByRows(host.HostType, self.VendorDeviceId, self.Model, self.DevType, nil) + return IsolatedDeviceManager.getSpecByRows(host.HostType, self.VendorDeviceId, self.Model, self.DevType, self.SharingMode, &self.NvmeSizeMB, &self.MemorySize, nil) } func (self *SIsolatedDevice) GetGpuSpec() *GpuSpec { @@ -1435,14 +1524,16 @@ func (man *SIsolatedDeviceManager) GetSpecIdent(spec *jsonutils.JSONDict) []stri devType, _ := spec.GetString("dev_type") vendor, _ := spec.GetString("vendor") model, _ := spec.GetString("model") - return man.getSpecKeys(vendor, model, devType) + sharingMode, _ := spec.GetString("sharing_mode") + return man.getSpecKeys(vendor, model, devType, sharingMode) } -func (man *SIsolatedDeviceManager) getSpecKeys(vendor, model, devType string) []string { +func (man *SIsolatedDeviceManager) getSpecKeys(vendor, model, devType, sharingMode string) []string { keys := []string{ fmt.Sprintf("type:%s", devType), fmt.Sprintf("vendor:%s", vendor), fmt.Sprintf("model:%s", model), + fmt.Sprintf("sharing_mode:%s", sharingMode), } return keys } @@ -1458,13 +1549,6 @@ func (self *SIsolatedDevice) getHost() *SHost { return HostManager.FetchHostById(self.HostId) } -func (self *SIsolatedDevice) getGuest() *SGuest { - if len(self.GuestId) > 0 { - return GuestManager.FetchGuestById(self.GuestId) - } - return nil -} - func (manager *SIsolatedDeviceManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, @@ -1478,28 +1562,50 @@ func (manager *SIsolatedDeviceManager) FetchCustomizeColumns( stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) hostRows := manager.SHostResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) shareRows := manager.SSharableBaseResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) - guestIds := make([]string, len(rows)) + guestIds := make([][]string, len(rows)) + guestIdsAll := make([]string, 0) for i := range rows { rows[i] = api.IsolateDeviceDetails{ StandaloneResourceDetails: stdRows[i], HostResourceInfo: hostRows[i], SharableResourceBaseInfo: shareRows[i], } - guestIds[i] = objs[i].(*SIsolatedDevice).GuestId + dev := objs[i].(*SIsolatedDevice) + rows[i].Vendor = dev.getVendor() + if dev.SharingMode == api.DEVICE_SHARING_MODE_HAMI { + rows[i].MemoryAllocated, _ = dev.getAllocatedMemorySize() + } else { + rows[i].AllocatedCount, _ = dev.getAllocatedCount() + } + guestIds[i] = dev.getAttachedGuestIds() + if len(guestIds[i]) > 0 { + guestIdsAll = append(guestIdsAll, guestIds[i]...) + } } guests := make(map[string]SGuest) - err := db.FetchStandaloneObjectsByIds(GuestManager, guestIds, &guests) + err := db.FetchStandaloneObjectsByIds(GuestManager, guestIdsAll, &guests) if err != nil { log.Errorf("db.FetchStandaloneObjectsByIds fail %s", err) return rows } for i := range rows { - if guest, ok := guests[guestIds[i]]; ok { - rows[i].Guest = guest.Name - rows[i].GuestStatus = guest.Status + nguests := guestIds[i] + if len(nguests) > 0 { + rows[i].Guest = make([]string, len(nguests)) + rows[i].GuestIds = make([]string, len(nguests)) + rows[i].GuestStatus = make([]string, len(nguests)) + } + + for j := range nguests { + if guest, ok := guests[nguests[j]]; ok { + rows[i].Guest[j] = guest.Name + rows[i].GuestIds[j] = guest.Id + rows[i].GuestStatus[j] = guest.Status + } } + } return rows @@ -1526,20 +1632,20 @@ func (self *SIsolatedDevice) PerformPurge(ctx context.Context, userCred mcclient } func (self *SIsolatedDevice) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - if len(self.GuestId) > 0 { + guestIsolatedDevices := self.getAttachedGuests() + if len(guestIsolatedDevices) > 0 { if !jsonutils.QueryBoolean(data, "purge", false) { - return httperrors.NewBadRequestError("%s: %s", api.ErrMsgIsolatedDeviceUsedByServer, self.GuestId) + return httperrors.NewBadRequestError("%s", api.ErrMsgIsolatedDeviceUsedByServer) } - iGuest, err := GuestManager.FetchById(self.GuestId) - if err != nil { - return err - } - guest := iGuest.(*SGuest) - err = guest.detachIsolateDevice(ctx, userCred, self) - if err != nil { - return err + + for i := range guestIsolatedDevices { + err := guestIsolatedDevices[i].Detach(ctx, userCred) + if err != nil { + return err + } } } + host := self.getHost() if host != nil { db.OpsLog.LogEvent(host, db.ACT_HOST_DETACH_ISOLATED_DEVICE, self.GetShortDesc(ctx), userCred) @@ -1581,12 +1687,9 @@ func (manager *SIsolatedDeviceManager) GetAllDevsOnHost(hostId string) ([]SIsola return devs, nil } -func (manager *SIsolatedDeviceManager) GetUnusedDevsOnHost(hostId string, model string, count int) ([]SIsolatedDevice, error) { +func (manager *SIsolatedDeviceManager) GetAvailableIsolatedDeviceOnHost(hostId string, model, sharingMode string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) - q := manager.Query().Equals("host_id", hostId).Equals("model", model).IsNullOrEmpty("guest_id") - if count > 0 { - q = q.Limit(count) - } + q := manager.GetAvailableIsolatedDeviceQuery(manager.Query().Equals("host_id", hostId).Equals("model", model)).Equals("sharing_mode", sharingMode) err := db.FetchModelObjects(manager, q, &devs) if err != nil { return nil, err @@ -1607,10 +1710,10 @@ func (manager *SIsolatedDeviceManager) hostHasDevAddr(hostId, addr, mdevId strin } func (manager *SIsolatedDeviceManager) CheckModelIsEmpty(model, vendor, device, devType string) (bool, error) { - cnt, err := manager.Query().Equals("model", model). + cnt, err := manager.queryWithGuest(manager.Query().Equals("model", model). Equals("dev_type", devType). - Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device)). - IsNotEmpty("guest_id").CountWithError() + Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device))). + CountWithError() if err != nil { return false, err } @@ -1741,14 +1844,6 @@ func (model *SIsolatedDevice) syncWithCloudIsolateDevice(ctx context.Context, us return nil } -func (model *SIsolatedDevice) SetNetworkIndex(idx int) error { - _, err := db.Update(model, func() error { - model.NetworkIndex = idx - return nil - }) - return err -} - func (model *SIsolatedDevice) GetRequiredSharedDomainIds() []string { host := model.getHost() if host != nil { @@ -1786,12 +1881,15 @@ type HostIsolatedDevicesNumaStat struct { NumaNodeDevCount int } -func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devType, hostId string) ([]HostIsolatedDevicesNumaStat, error) { - q := manager.Query().Equals("host_id", hostId).Equals("dev_type", devType) - guestQ := GuestManager.Query().SubQuery() +func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(devModel, hostId string) ([]HostIsolatedDevicesNumaStat, error) { + q := GuestIsolatedDeviceManager.Query() + guestQ := GuestManager.Query().NotEquals("status", api.VM_READY).SubQuery() + isq := manager.Query().Equals("host_id", hostId).Equals("model", devModel).SubQuery() + + q = q.Join(isq, sqlchemy.Equals(q.Field("isolated_device_id"), isq.Field("id"))) q = q.Join(guestQ, sqlchemy.Equals(q.Field("guest_id"), guestQ.Field("id"))) - q = q.Filter(sqlchemy.NotEquals(guestQ.Field("status"), api.VM_READY)).GroupBy(q.Field("numa_node")) - q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", q.Field("numa_node"))) + q = q.GroupBy(isq.Field("numa_node")) + q = q.AppendField(sqlchemy.COUNT("numa_node_dev_count", isq.Field("numa_node"))) subQ := q.SubQuery() numaQ := subQ.Query(subQ.Field("numa_node"), subQ.Field("numa_node_dev_count")) @@ -1803,11 +1901,11 @@ func (manager *SIsolatedDeviceManager) GetHostAllocatedIsolatedDeviceNumaStats(d return stats, nil } -func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool, error) { +func (host *SHost) VirtualDeviceNumaBalance(devModel string, numaNode int8) (bool, error) { if numaNode < 0 { return true, nil } - stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devType, host.Id) + stats, err := IsolatedDeviceManager.GetHostAllocatedIsolatedDeviceNumaStats(devModel, host.Id) if err != nil { return true, err } @@ -1826,3 +1924,588 @@ func (host *SHost) VirtualDeviceNumaBalance(devType string, numaNode int8) (bool } return true, nil } + +func (manager *SIsolatedDeviceManager) GetAvailableIsolatedDeviceQuery(isq *sqlchemy.SQuery) *sqlchemy.SQuery { + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() + guestIsQ := guestIdevs.Query( + guestIdevs.Field("isolated_device_id"), + sqlchemy.SUM("memory_allocated", guestIdevs.Field("device_memory_size")), + sqlchemy.COUNT("guest_count", guestIdevs.Field("guest_id")), + ).GroupBy("isolated_device_id").SubQuery() + + if isq == nil { + isq = manager.Query() + } + + isq = isq.LeftJoin(guestIsQ, sqlchemy.Equals(isq.Field("id"), guestIsQ.Field("isolated_device_id"))) + cond1 := sqlchemy.AND( + sqlchemy.Equals(isq.Field("sharing_mode"), api.DEVICE_SHARING_MODE_HAMI), + sqlchemy.OR( + sqlchemy.IsNull(guestIsQ.Field("memory_allocated")), + sqlchemy.GT(isq.Field("memory_size"), guestIsQ.Field("memory_allocated")), + ), + ) + cond2 := sqlchemy.AND( + sqlchemy.NotEquals(isq.Field("sharing_mode"), api.DEVICE_SHARING_MODE_HAMI), + sqlchemy.OR( + sqlchemy.IsNull(guestIsQ.Field("guest_count")), + sqlchemy.GT(isq.Field("virtual_num"), guestIsQ.Field("guest_count")), + ), + ) + + isq = isq.Filter(sqlchemy.OR(cond1, cond2)) + + return isq +} + +type IsolatedDeviceAllocateStat struct { + SIsolatedDevice + + GuestCount int + MemoryAllocated int +} + +func (manager *SIsolatedDeviceManager) GetHostsIsolatedDeviceStats(hostIds []string) []IsolatedDeviceAllocateStat { + guestIdevs := GuestIsolatedDeviceManager.Query().SubQuery() + guestIsQ := guestIdevs.Query( + guestIdevs.Field("isolated_device_id"), + sqlchemy.SUM("memory_allocated", guestIdevs.Field("device_memory_size")), + sqlchemy.COUNT("guest_count", guestIdevs.Field("guest_id")), + ).GroupBy("isolated_device_id").SubQuery() + + isq := manager.Query().In("host_id", hostIds) + isq = isq.LeftJoin(guestIsQ, sqlchemy.Equals(isq.Field("id"), guestIsQ.Field("isolated_device_id"))) + isq.AppendField(isq.QueryFields()...) + isq.AppendField(guestIsQ.Field("memory_allocated"), guestIsQ.Field("guest_count")) + stats := make([]IsolatedDeviceAllocateStat, 0) + err := isq.All(&stats) + if err != nil { + log.Errorf("GetHostsIsolatedDevicesDetails %s", err) + } + + return stats +} + +func (manager *SIsolatedDeviceManager) GetHostsGuestIsolatedDevices(hostIds []string) map[string][]string { + gidq := GuestIsolatedDeviceManager.Query().SubQuery() + isq := manager.Query().SubQuery() + + q := gidq.Query() + q = q.Join(isq, sqlchemy.Equals(isq.Field("id"), gidq.Field("isolated_device_id"))) + q = q.Filter(sqlchemy.In(isq.Field("host_id"), hostIds)) + + result := []struct { + IsolatedDeviceId string + GuestId string + }{} + err := q.All(&result) + if err != nil { + log.Errorf("GetHostsGuestIsolatedDevices query %s", err) + return nil + } + ret := map[string][]string{} + for i := range result { + if guests, ok := ret[result[i].IsolatedDeviceId]; ok { + ret[result[i].IsolatedDeviceId] = append(guests, result[i].GuestId) + } else { + ret[result[i].IsolatedDeviceId] = []string{result[i].GuestId} + } + } + return ret +} + +func (dev *SIsolatedDevice) IsEnough(memoryRequest int) bool { + if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI { + cnt, err := dev.getAllocatedCount() + if err != nil { + log.Errorf("failed getAllocatedCount %s", err) + return false + } + return dev.VirtualNum > cnt + } else { + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return false + } + return (dev.MemorySize - allocated) >= memoryRequest + } +} + +func (dev *SIsolatedDevice) IsFull() bool { + if dev.SharingMode != api.DEVICE_SHARING_MODE_HAMI { + cnt, err := dev.getAllocatedCount() + if err != nil { + log.Errorf("failed getAllocatedCount %s", err) + return true + } + return dev.VirtualNum <= cnt + } else { + allocated, err := dev.getAllocatedMemorySize() + if err != nil { + log.Errorf("failed getAllocatedMemorySize %s", err) + return true + } + return dev.MemorySize <= allocated + } +} + +func (manager *SIsolatedDeviceManager) InitializeData() error { + ctx := context.Background() + inited, err := manager.isInitializeDataDone() + if err != nil { + return errors.Wrap(err, "isInitializeDataDone") + } + if inited { + return nil + } + + if err := manager.migrateGuestIsolatedDevices(); err != nil { + return errors.Wrap(err, "migrateGuestIsolatedDevices") + } + if err := manager.mergeVirtualIsolatedDevices(); err != nil { + return errors.Wrap(err, "mergeVirtualIsolatedDevices") + } + if err := manager.initVirtualNum(); err != nil { + return errors.Wrap(err, "initVirtualNum") + } + if err := manager.migrateDevType(); err != nil { + return errors.Wrap(err, "migrateDevType") + } + if err := manager.markInitializeDataDone(ctx); err != nil { + return errors.Wrap(err, "markInitializeDataDone") + } + return nil +} + +func (manager *SIsolatedDeviceManager) getInitializeDataMetadataId() string { + return fmt.Sprintf("%s%s%s", isolatedDeviceInitializeDataObjType, db.OBJECT_TYPE_ID_SEP, isolatedDeviceInitializeDataObjId) +} + +func (manager *SIsolatedDeviceManager) isInitializeDataDone() (bool, error) { + md := db.SMetadata{} + err := db.Metadata.RawQuery("value", "deleted"). + Equals("id", manager.getInitializeDataMetadataId()). + Equals("key", isolatedDeviceInitializeDataKey). + First(&md) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return false, nil + } + return false, err + } + return !md.Deleted && md.Value == "true", nil +} + +func (manager *SIsolatedDeviceManager) markInitializeDataDone(ctx context.Context) error { + md := db.SMetadata{} + md.SetModelManager(db.Metadata, &md) + err := db.Metadata.RawQuery(). + Equals("id", manager.getInitializeDataMetadataId()). + Equals("key", isolatedDeviceInitializeDataKey). + First(&md) + if err != nil { + if errors.Cause(err) != sql.ErrNoRows { + return err + } + md.ObjType = isolatedDeviceInitializeDataObjType + md.ObjId = isolatedDeviceInitializeDataObjId + md.Id = manager.getInitializeDataMetadataId() + md.Key = isolatedDeviceInitializeDataKey + md.Value = "true" + return db.Metadata.TableSpec().Insert(ctx, &md) + } + _, err = db.Update(&md, func() error { + md.ObjType = isolatedDeviceInitializeDataObjType + md.ObjId = isolatedDeviceInitializeDataObjId + md.Value = "true" + md.Deleted = false + return nil + }) + return err +} + +func getIsolatedDeviceBaseAddr(addr string) string { + return strings.Split(addr, "-")[0] +} + +func isMergeableVirtualDevType(devType string) bool { + return utils.IsInStringArray(devType, api.VITRUAL_DEVICE_TYPES) +} + +type isolatedDeviceMergeKey struct { + HostId string + Addr string + MdevId string + VendorDeviceId string + DevType string +} + +func (manager *SIsolatedDeviceManager) migrateGuestIsolatedDevices() error { + rows, err := sqlchemy.GetDB().Query(fmt.Sprintf("SELECT id, guest_id, dev_type, network_index, disk_index FROM %s WHERE deleted = 0 AND guest_id IS NOT NULL AND LENGTH(guest_id) > 0", manager.TableSpec().Name())) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return errors.Wrap(err, "migrateGuestIsolatedDevices QueryRows") + } + if err != nil && errors.Cause(err) == sql.ErrNoRows { + return nil + } + defer rows.Close() + + ctx := context.Background() + migrated := 0 + guestIndexMap := map[string]int8{} + for rows.Next() { + var devId, guestId, devType string + var networkIndex int + var diskIndex int8 + if err = rows.Scan(&devId, &guestId, &devType, &networkIndex, &diskIndex); err != nil { + return errors.Wrap(err, "migrateGuestIsolatedDevices Scan") + } + cnt, err := GuestIsolatedDeviceManager.Query(). + Equals("guest_id", guestId). + Equals("isolated_device_id", devId). + CountWithError() + if err != nil { + return errors.Wrapf(err, "count guest isolated device for device %s", devId) + } + if cnt == 0 { + idx, ok := guestIndexMap[guestId] + if !ok { + maxIdx, err := manager.getGuestIsolatedDeviceMaxIndex(guestId) + if err != nil { + return errors.Wrapf(err, "getGuestIsolatedDeviceMaxIndex guest %s", guestId) + } + idx = maxIdx + 1 + } + guestIndexMap[guestId] = idx + 1 + + guestIsolatedDevice := SGuestIsolatedDevice{} + guestIsolatedDevice.SetModelManager(GuestIsolatedDeviceManager, &guestIsolatedDevice) + guestIsolatedDevice.GuestId = guestId + guestIsolatedDevice.IsolatedDeviceId = devId + guestIsolatedDevice.Index = idx + if devType == api.GPU_VGA_TYPE { + guestIsolatedDevice.GpuType = api.GPU_VGA + } else if utils.IsInStringArray(devType, api.GPU_TYPES) { + guestIsolatedDevice.GpuType = api.GPU_HPC + } + if networkIndex >= 0 { + guestIsolatedDevice.NetworkIndex = networkIndex + } + if diskIndex >= 0 { + guestIsolatedDevice.DiskIndex = diskIndex + } + if err := GuestIsolatedDeviceManager.TableSpec().Insert(ctx, &guestIsolatedDevice); err != nil { + return errors.Wrapf(err, "insert guest isolated device for device %s guest %s", devId, guestId) + } + migrated++ + } + } + + log.Infof("migrated %d legacy isolated device guest assign to guest_isolated_devices_tbl", migrated) + return nil +} + +func (manager *SIsolatedDeviceManager) getGuestIsolatedDeviceMaxIndex(guestId string) (int8, error) { + type maxIdxResult struct { + MaxIndex int8 + } + sq := GuestIsolatedDeviceManager.Query().Equals("guest_id", guestId).SubQuery() + q := sq.Query(sqlchemy.MAX("max_index", sq.Field("index"))) + ret := maxIdxResult{MaxIndex: -1} + err := q.First(&ret) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return -1, err + } + return ret.MaxIndex, nil +} + +func (manager *SIsolatedDeviceManager) mergeVirtualIsolatedDevices() error { + devs := make([]SIsolatedDevice, 0) + q := manager.Query().In("dev_type", api.VITRUAL_DEVICE_TYPES).NotEquals("dev_type", api.CONTAINER_DEV_NVIDIA_HAMI) + err := db.FetchModelObjects(manager, q, &devs) + if err != nil { + return errors.Wrap(err, "FetchModelObjects") + } + if len(devs) == 0 { + return nil + } + + grouped := map[isolatedDeviceMergeKey][]*SIsolatedDevice{} + for i := range devs { + dev := &devs[i] + key := isolatedDeviceMergeKey{ + HostId: dev.HostId, + Addr: getIsolatedDeviceBaseAddr(dev.Addr), + MdevId: dev.MdevId, + VendorDeviceId: dev.VendorDeviceId, + DevType: dev.DevType, + } + grouped[key] = append(grouped[key], dev) + } + + mergedGroups := 0 + deletedDevs := 0 + for _, group := range grouped { + if len(group) <= 1 { + continue + } + sort.Slice(group, func(i, j int) bool { + return group[i].CreatedAt.Before(group[j].CreatedAt) + }) + keeper := group[0] + ids := make([]string, len(group)-1) + for i := 1; i < len(group); i++ { + ids[i-1] = group[i].Id + } + err := manager.doMergeGuestIsolatedDevices(keeper.Id, getIsolatedDeviceBaseAddr(keeper.Addr), ids) + if err != nil { + return err + } + host := HostManager.FetchHostById(group[0].HostId) + if host != nil && host.HostType == api.HOST_TYPE_CONTAINER { + err = manager.doReplaceContainerIsolatedDeviceId(keeper.Id, ids) + if err != nil { + return err + } + } + mergedGroups++ + } + log.Infof("merged %d duplicate virtual isolated device groups, deleted %d duplicate devices", mergedGroups, deletedDevs) + return nil +} + +func (manager *SIsolatedDeviceManager) doReplaceContainerIsolatedDeviceId(keeperId string, originIds []string) error { + log.Infof("start replace contaienr isolated device id, originIds %v, keeper %s", originIds, keeperId) + ids := []string{keeperId} + ids = append(ids, originIds...) + gdevs := make([]SGuestIsolatedDevice, 0) + q := GuestIsolatedDeviceManager.Query().In("isolated_device_id", ids) + err := db.FetchModelObjects(GuestIsolatedDeviceManager, q, &gdevs) + if err != nil { + return errors.Wrap(err, "GuestIsolatedDeviceManager.FetchModelObjects") + } + for i := range gdevs { + ctrs, err := GetContainerManager().GetContainersByPod(gdevs[i].GuestId) + if err != nil { + return errors.Wrapf(err, "GetContainerManager().GetContainersByPod") + } + for j := range ctrs { + ctrPtr := &ctrs[j] + + spec := new(api.ContainerSpec) + if err := jsonutils.Marshal(ctrPtr.Spec).Unmarshal(spec); err != nil { + return errors.Wrap(err, "deep copy spec") + } + + updated := false + for k := range spec.Devices { + if spec.Devices[k].IsolatedDevice == nil { + continue + } + if !utils.IsInStringArray(spec.Devices[k].IsolatedDevice.Id, ids) { + continue + } + spec.Devices[k].IsolatedDevice.Id = keeperId + spec.Devices[k].IsolatedDevice.GuestIsolatedDeviceIndex = int(gdevs[i].Index) + updated = true + } + if !updated { + continue + } + _, err = db.Update(ctrPtr, func() error { + ctrPtr.Spec = spec + return nil + }) + if err != nil { + return errors.Wrap(err, "update ctr isolated device id") + } + log.Infof("replace container isolated device id %s to %s", ctrPtr.Id, keeperId) + } + } + return nil +} + +func (manager *SIsolatedDeviceManager) doMergeGuestIsolatedDevices(keeperId, keeperNewAddr string, originIds []string) error { + tx, err := sqlchemy.GetDB().Begin() + if err != nil { + return errors.Wrap(err, "failed begin TRANSACTION") + } + defer func() { + if err != nil { + tx.Rollback() + } + }() + buildInPlaceholders := func(n int) string { + parts := make([]string, n) + for i := range parts { + parts[i] = "?" + } + return strings.Join(parts, ",") + } + var res sql.Result + + if len(originIds) > 0 { + sql := fmt.Sprintf( + "update %s set isolated_device_id = ? where isolated_device_id in (%s)", + GuestIsolatedDeviceManager.TableSpec().Name(), buildInPlaceholders(len(originIds)), + ) + args := make([]interface{}, 1, 1+len(originIds)) + args[0] = keeperId + for i := range originIds { + args = append(args, originIds[i]) + } + res, err = tx.Exec(sql, args...) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ := res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + + sql = fmt.Sprintf( + "update %s set deleted = 1 where id in (%s)", + IsolatedDeviceManager.TableSpec().Name(), buildInPlaceholders(len(originIds)), + ) + args = make([]interface{}, len(originIds)) + for i := range originIds { + args[i] = originIds[i] + } + res, err = tx.Exec(sql, args...) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ = res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + if affected != int64(len(originIds)) { + return errors.Errorf("TRANSACTION: %s affected rows %d not equal to originIds length", sql, affected) + } + } + + virtualNum := 1 + len(originIds) + sql := fmt.Sprintf( + "update %s set virtual_num = ?, addr = ? where id = ?", + IsolatedDeviceManager.TableSpec().Name(), + ) + res, err = tx.Exec(sql, virtualNum, keeperNewAddr, keeperId) + if err != nil { + return errors.Wrapf(err, "failed exec TRANSACTION: %s", sql) + } + affected, _ := res.RowsAffected() + log.Infof("sql %s effect %d", sql, affected) + + if err = tx.Commit(); err != nil { + return errors.Wrap(err, "failed commit TRANSACTION") + } + return nil +} + +func (manager *SIsolatedDeviceManager) initVirtualNum() error { + devs := make([]SIsolatedDevice, 0) + q := manager.Query().NotEquals("dev_type", api.CONTAINER_DEV_NVIDIA_HAMI) + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("virtual_num")), + sqlchemy.LE(q.Field("virtual_num"), 0), + )) + err := db.FetchModelObjects(manager, q, &devs) + if err != nil { + return errors.Wrap(err, "FetchModelObjects") + } + updated := 0 + for i := range devs { + dev := &devs[i] + virtualNum := 1 + if isMergeableVirtualDevType(dev.DevType) { + cnt, err := dev.getAllocatedCount() + if err != nil { + return errors.Wrapf(err, "getAllocatedCount device %s", dev.Id) + } + if cnt > virtualNum { + virtualNum = cnt + } + } + if _, err := db.Update(dev, func() error { + dev.VirtualNum = virtualNum + return nil + }); err != nil { + return errors.Wrapf(err, "set virtual_num on device %s", dev.Id) + } + updated++ + } + log.Infof("initialized virtual_num for %d isolated devices", updated) + return nil +} + +func (manager *SIsolatedDeviceManager) migrateDevType() error { + hotPluggableDevTypes := []string{ + api.DIRECT_PCI_TYPE, api.USB_TYPE, api.GPU_VGA_TYPE, api.GPU_HPC_TYPE, + api.SRIOV_VGPU_TYPE, api.LEGACY_VGPU_TYPE, + } + + for _, devType := range api.VALID_PASSTHROUGH_TYPES { + var sharingMode string + switch devType { + case api.DIRECT_PCI_TYPE, api.USB_TYPE, api.GPU_VGA_TYPE, api.GPU_HPC_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_EXCLUSIVE + case api.SRIOV_VGPU_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_SRIOV + case api.CONTAINER_DEV_NVIDIA_MPS: + sharingMode = api.DEVICE_SHARING_MODE_MPS + case api.CONTAINER_DEV_NVIDIA_HAMI: + sharingMode = api.DEVICE_SHARING_MODE_HAMI + case api.LEGACY_VGPU_TYPE: + sharingMode = api.DEVICE_SHARING_MODE_MDEV + default: + sharingMode = api.DEVICE_SHARING_MODE_UNLIMITED + } + var hotPluggable = 0 + if utils.IsInStringArray(devType, hotPluggableDevTypes) { + hotPluggable = 1 + } + var targetDevType string + switch { + case utils.IsInStringArray(devType, api.GPU_TYPES): + targetDevType = api.GPU_TYPE + case utils.IsInStringArray(devType, api.NETINT_TYPES): + targetDevType = api.NETINT_TYPE + case devType == api.CONTAINER_DEV_CPH_AOSP_BINDER: + targetDevType = api.BINDER_TYPE + case devType == api.CONTAINER_DEV_ASCEND_NPU: + targetDevType = api.NPU_TYPE + default: + targetDevType = devType + } + + sql := fmt.Sprintf( + "update %s set dev_type = ?, sharing_mode = ?, hot_pluggable = ? where dev_type = ? and deleted = 0", + manager.TableSpec().Name(), + ) + res, err := sqlchemy.GetDB().Exec(sql, targetDevType, sharingMode, hotPluggable, devType) + if err != nil { + return errors.Wrapf(err, "update dev_type from %v to %s", devType, targetDevType) + } + effects, _ := res.RowsAffected() + log.Infof("updated dev_type from %v to %s, effects: %d", devType, targetDevType, effects) + } + return nil +} + +func (dev *SIsolatedDevice) IsValidAttachDev() bool { + if dev.DevType == api.NIC_TYPE || dev.DevType == api.NVME_PT_TYPE { + return false + } + return true +} + +func (dev *SIsolatedDevice) IsKvmExclusiveGPU() bool { + if dev.DevType != api.GPU_TYPE { + return false + } + if dev.SharingMode != api.DEVICE_SHARING_MODE_EXCLUSIVE { + return false + } + host := dev.GetHost() + if host.HostType != api.HOST_TYPE_HYPERVISOR { + return false + } + return true +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/isolateddeviceresouce.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolateddeviceresouce.go new file mode 100644 index 000000000..998b283ec --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/isolateddeviceresouce.go @@ -0,0 +1,202 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/reflectutils" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +type SIsolatedDeviceResourceBase struct { + IsolatedDeviceId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"` +} + +type SIsolatedDeviceResourceBaseManager struct { +} + +func (manager *SIsolatedDeviceResourceBaseManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.IsolateDeviceDetails { + rows := make([]api.IsolateDeviceDetails, len(objs)) + devIds := make([]string, len(objs)) + for i := range objs { + var base *SIsolatedDeviceResourceBase + err := reflectutils.FindAnonymouStructPointer(objs[i], &base) + if err != nil { + log.Errorf("Cannot find SIsolatedDeviceResourceBase in object %s", objs[i]) + continue + } + devIds[i] = base.IsolatedDeviceId + } + + devMap := make(map[string]SIsolatedDevice) + err := db.FetchStandaloneObjectsByIds(IsolatedDeviceManager, devIds, devMap) + if err != nil { + log.Errorf("FetchStandaloneObjectsByIds fail %s", err) + return rows + } + + devObjs := make([]interface{}, len(objs)) + devs := make([]SIsolatedDevice, len(objs)) + for i := range rows { + if dev, ok := devMap[devIds[i]]; ok { + devs[i] = dev + if err := jsonutils.Marshal(dev).Unmarshal(&rows[i].SIsolatedDevice); err != nil { + log.Errorf("unmarshal isolated device %s details failed %s", dev.Id, err) + } + } + devObjs[i] = &devs[i] + } + + devRows := IsolatedDeviceManager.FetchCustomizeColumns(ctx, userCred, query, devObjs, fields, isList) + for i := range rows { + rows[i].StandaloneResourceDetails = devRows[i].StandaloneResourceDetails + rows[i].SharableResourceBaseInfo = devRows[i].SharableResourceBaseInfo + rows[i].HostResourceInfo = devRows[i].HostResourceInfo + rows[i].Guest = devRows[i].Guest + rows[i].GuestIds = devRows[i].GuestIds + rows[i].GuestStatus = devRows[i].GuestStatus + rows[i].Vendor = devRows[i].Vendor + rows[i].MemoryAllocated = devRows[i].MemoryAllocated + rows[i].AllocatedCount = devRows[i].AllocatedCount + } + return rows +} + +func (manager *SIsolatedDeviceResourceBaseManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + devQ := IsolatedDeviceManager.Query("id") + devQ, err := IsolatedDeviceManager.ListItemFilter(ctx, devQ, userCred, query.IsolatedDeviceListInput) + if err != nil { + return nil, errors.Wrap(err, "IsolatedDeviceManager.ListItemFilter") + } + q = q.Filter(sqlchemy.In(q.Field("isolated_device_id"), devQ.SubQuery())) + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + switch field { + case "isolated_device": + devQ := IsolatedDeviceManager.Query("name", "id").SubQuery() + q = q.AppendField(devQ.Field("name", field)).Distinct() + q = q.Join(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + return q, nil + case "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node": + devQ := IsolatedDeviceManager.Query(field, "id").SubQuery() + q = q.AppendField(devQ.Field(field)).Distinct() + q = q.Join(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + return q, nil + default: + devQ := IsolatedDeviceManager.Query("id", "host_id").SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + q, err := IsolatedDeviceManager.SHostResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + } + return q, httperrors.ErrNotFound +} + +func (manager *SIsolatedDeviceResourceBaseManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, +) (*sqlchemy.SQuery, error) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, nil + } + orderQ := IsolatedDeviceManager.Query("id") + orderSubQ := orderQ.SubQuery() + orderQ, orders, fields := manager.GetOrderBySubQuery(orderQ, orderSubQ, orderQ.Field("id"), userCred, query, nil, nil) + q = q.LeftJoin(orderSubQ, sqlchemy.Equals(q.Field("isolated_device_id"), orderSubQ.Field("id"))) + q = db.OrderByFields(q, orders, fields) + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetOrderBySubQuery( + q *sqlchemy.SQuery, + subq *sqlchemy.SSubQuery, + joinField sqlchemy.IQueryField, + userCred mcclient.TokenCredential, + query api.GuestIsolatedDeviceListInput, + orders []string, + fields []sqlchemy.IQueryField, +) (*sqlchemy.SQuery, []string, []sqlchemy.IQueryField) { + if !db.NeedOrderQuery(manager.GetOrderByFields(query)) { + return q, orders, fields + } + devQ := IsolatedDeviceManager.Query().SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(joinField, devQ.Field("id"))) + q, orders, fields = IsolatedDeviceManager.SHostResourceBaseManager.GetOrderBySubQuery(q, subq, devQ.Field("host_id"), userCred, query.HostFilterListInput, orders, fields) + return q, orders, fields +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetOrderByFields(query api.GuestIsolatedDeviceListInput) []string { + return IsolatedDeviceManager.SHostResourceBaseManager.GetOrderByFields(query.HostFilterListInput) +} + +func (manager *SIsolatedDeviceResourceBaseManager) ListItemExportKeys(ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + keys stringutils2.SSortedStrings, +) (*sqlchemy.SQuery, error) { + if keys.ContainsAny(manager.GetExportKeys()...) { + devQ := IsolatedDeviceManager.Query("id", "name", "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node", "host_id").SubQuery() + q = q.LeftJoin(devQ, sqlchemy.Equals(q.Field("isolated_device_id"), devQ.Field("id"))) + if keys.Contains("isolated_device") { + q = q.AppendField(devQ.Field("name", "isolated_device")) + } + for _, key := range []string{"dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node"} { + if keys.Contains(key) { + q = q.AppendField(devQ.Field(key)) + } + } + if keys.ContainsAny(IsolatedDeviceManager.SHostResourceBaseManager.GetExportKeys()...) { + var err error + q, err = IsolatedDeviceManager.SHostResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrap(err, "SHostResourceBaseManager.ListItemExportKeys") + } + } + } + return q, nil +} + +func (manager *SIsolatedDeviceResourceBaseManager) GetExportKeys() []string { + keys := []string{"isolated_device", "dev_type", "model", "addr", "device_path", "vendor_device_id", "numa_node"} + keys = append(keys, IsolatedDeviceManager.SHostResourceBaseManager.GetExportKeys()...) + return keys +} diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/networkaddresses.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/networkaddresses.go index 2864cb20b..5cc440cd5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/networkaddresses.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/networkaddresses.go @@ -593,7 +593,7 @@ func (g *SGuest) PerformUpdateSubIps( query jsonutils.JSONObject, input api.GuestUpdateSubIpsInput, ) (jsonutils.JSONObject, error) { - gn, err := g.findGuestnetworkByInfo(input.ServerNetworkInfo) + gn, err := g.FindGuestnetworkByInfo(input.ServerNetworkInfo) if err != nil { return nil, errors.Wrapf(err, "getGuestnetworkByIpOrMac ip=%s mac=%s", input.IpAddr, input.Mac) } @@ -654,7 +654,7 @@ func (g *SGuest) PerformAddSubIps( query jsonutils.JSONObject, input api.GuestAddSubIpsInput, ) (jsonutils.JSONObject, error) { - gn, err := g.findGuestnetworkByInfo(input.ServerNetworkInfo) + gn, err := g.FindGuestnetworkByInfo(input.ServerNetworkInfo) if err != nil { return nil, errors.Wrapf(err, "getGuestnetworkByIpOrMac ip=%s mac=%s", input.IpAddr, input.Mac) } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/networks.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/networks.go index 97162cb7f..6d5941ada 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/networks.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/networks.go @@ -904,7 +904,7 @@ func (snet *SNetwork) isAddress6Used(ctx context.Context, address string) (bool, func (manager *SNetworkManager) fetchAllOnpremiseNetworks(serverType string, isPublic tristate.TriState) ([]SNetwork, error) { q := manager.Query() - wires := WireManager.Query().SubQuery() + wires := WireManager.Query().NotEquals("id", api.DEFAULT_HOST_LOCAL_WIRE_ID).SubQuery() q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id"))) q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), api.DEFAULT_VPC_ID)) if len(serverType) > 0 { diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/purge.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/purge.go index 3fbeaef0b..6106821e7 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/purge.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/purge.go @@ -303,6 +303,7 @@ func (self *SCloudregion) purgeResources(ctx context.Context, managerId string) tables := TablestoreManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) wafs := WafInstanceManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) ipsets := WafIPSetManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) + addressIpsets := IpSetManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) regsets := WafRegexSetManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) wafgroups := WafRuleGroupManager.Query("id").Equals("manager_id", managerId).Equals("cloudregion_id", self.Id) cprs := CloudproviderRegionManager.Query("row_id").Equals("cloudprovider_id", managerId).Equals("cloudregion_id", self.Id) @@ -312,6 +313,7 @@ func (self *SCloudregion) purgeResources(ctx context.Context, managerId string) {manager: WafRuleGroupManager, key: "id", q: wafgroups}, {manager: WafRegexSetManager, key: "id", q: regsets}, {manager: WafIPSetManager, key: "id", q: ipsets}, + {manager: IpSetManager, key: "id", q: addressIpsets}, {manager: WafInstanceManager, key: "id", q: wafs}, {manager: TablestoreManager, key: "id", q: tables}, {manager: SnapshotManager, key: "id", q: snapshots}, @@ -635,7 +637,7 @@ func (self *purgePair) purgeAll(ctx context.Context) error { func (self *SZone) purgeStorages(ctx context.Context, managerId string) error { storages := StorageManager.Query("id").Equals("manager_id", managerId).Equals("zone_id", self.Id) schedtags := StorageschedtagManager.Query("row_id").In("storage_id", storages.SubQuery()) - snapshots := SnapshotManager.Query("id").In("storage_id", storages.SubQuery()).IsTrue("fake_deleted") + snapshots := SnapshotManager.Query("id").In("storage_id", storages.SubQuery()) hoststorages := HoststorageManager.Query("row_id").In("storage_id", storages.SubQuery()) disks := DiskManager.Query("id").In("storage_id", storages.SubQuery()) diskbackups := DiskBackupManager.Query("id").In("disk_id", disks.SubQuery()) @@ -663,7 +665,7 @@ func (self *SZone) purgeStorages(ctx context.Context, managerId string) error { func (self *SStorage) purge(ctx context.Context, userCred mcclient.TokenCredential) error { schedtags := StorageschedtagManager.Query("row_id").Equals("storage_id", self.Id) - snapshots := SnapshotManager.Query("id").Equals("storage_id", self.Id).IsTrue("fake_deleted") + snapshots := SnapshotManager.Query("id").Equals("storage_id", self.Id) hoststorages := HoststorageManager.Query("row_id").Equals("storage_id", self.Id) disks := DiskManager.Query("id").Equals("storage_id", self.Id) diskbackups := DiskBackupManager.Query("id").In("disk_id", disks.SubQuery()) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/regiondrivers.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/regiondrivers.go index 053900c0e..1e290b071 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/regiondrivers.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/regiondrivers.go @@ -106,6 +106,7 @@ type ISecurityGroupDriver interface { RequestPrepareSecurityGroups(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, secgroups []SSecurityGroup, vpc *SVpc, callback func(ids []string) error, task taskman.ITask) error RequestDeleteSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, secgroup *SSecurityGroup, task taskman.ITask) error ValidateCreateSecurityGroupInput(ctx context.Context, userCred mcclient.TokenCredential, input *api.SSecgroupCreateInput) (*api.SSecgroupCreateInput, error) + ValidateCreateSecurityGroupRuleInput(ctx context.Context, userCred mcclient.TokenCredential, input *api.SSecgroupRuleCreateInput) (*api.SSecgroupRuleCreateInput, error) ValidateUpdateSecurityGroupRuleInput(ctx context.Context, userCred mcclient.TokenCredential, input *api.SSecgroupRuleUpdateInput) (*api.SSecgroupRuleUpdateInput, error) } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/scaling_group.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/scaling_group.go index 1a835e111..6ea39d372 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/scaling_group.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/scaling_group.go @@ -112,6 +112,9 @@ func (sgm *SScalingGroupManager) ValidateCreateData(ctx context.Context, userCre var err error input.VirtualResourceCreateInput, err = sgm.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, err + } // check InstanceNumber if input.MinInstanceNumber < 0 { return input, httperrors.NewInputParameterError("min_instance_number should not be smaller than 0") @@ -532,6 +535,9 @@ func (sg *SScalingGroup) Scale(ctx context.Context, triggerDesc IScalingTriggerD err = scalingActivity.SetReject("", fmt.Sprintf("The Cooling Time limit the execution time of the policy to at least: %s", sg.AllowScaleTime.In(CoolingTimeLocation).Format("2006-01-02 15:04:05 -0700"))) + if err != nil { + return errors.Wrap(err, "SetReject") + } return nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/secgrouprules.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/secgrouprules.go index 54aaa9c62..5da00bebe 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/secgrouprules.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/secgrouprules.go @@ -16,6 +16,7 @@ package models import ( "context" + "net" "strings" "yunion.io/x/cloudmux/pkg/cloudprovider" @@ -71,13 +72,15 @@ type SSecurityGroupRule struct { SSecurityGroupResourceBase `create:"required"` Id string `width:"128" charset:"ascii" primary:"true" list:"user"` - Priority int `list:"user" update:"user" list:"user"` + Priority int `list:"user" update:"user"` Protocol string `width:"32" charset:"ascii" nullable:"false" list:"user" update:"user" create:"required"` Ports string `width:"256" charset:"ascii" list:"user" update:"user" create:"optional"` Direction string `width:"3" charset:"ascii" list:"user" create:"required"` CIDR string `width:"256" charset:"ascii" list:"user" update:"user" create:"optional"` Action string `width:"5" charset:"ascii" nullable:"false" list:"user" update:"user" create:"required"` Description string `width:"256" charset:"utf8" list:"user" update:"user" create:"optional"` + + TargetType api.TSecgroupTargetType `width:"8" charset:"ascii" default:"cidr" list:"user" create:"optional"` } func (self *SSecurityGroupRule) GetId() string { @@ -173,6 +176,9 @@ func (manager *SSecurityGroupRuleManager) ListItemFilter( if len(query.Ip) > 0 { sql = sql.Like("cidr", "%"+query.Ip+"%") } + if len(query.TargetType) > 0 { + sql = sql.In("target_type", query.TargetType) + } return sql, nil } @@ -189,13 +195,20 @@ func (manager *SSecurityGroupRuleManager) FetchCustomizeColumns( bRows := manager.SResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) secRows := manager.SSecurityGroupResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) secIds := make([]string, len(objs)) + ipSetIds := make([]string, 0, len(objs)) for i := range rows { + rule := objs[i].(*SSecurityGroupRule) rows[i] = api.SecgroupRuleDetails{ ResourceBaseDetails: bRows[i], SecurityGroupResourceInfo: secRows[i], } - rule := objs[i].(*SSecurityGroupRule) secIds[i] = rule.SecgroupId + switch rule.TargetType { + case api.SecurityGroupRuleTargetTypeIpSet: + ipSetIds = append(ipSetIds, rule.CIDR) + case api.SecurityGroupRuleTargetTypeSecurityGroup: + secIds = append(secIds, rule.CIDR) + } } secgroups := make(map[string]SSecurityGroup) @@ -205,12 +218,31 @@ func (manager *SSecurityGroupRuleManager) FetchCustomizeColumns( return rows } + ipSets := make(map[string]SIpSet) + if len(ipSetIds) > 0 { + err := db.FetchStandaloneObjectsByIds(IpSetManager, ipSetIds, &ipSets) + if err != nil { + log.Errorf("FetchStandaloneObjectsByIds fail: %v", err) + } + } + virObjs := make([]interface{}, len(objs)) for i := range rows { + rule := objs[i].(*SSecurityGroupRule) if secgroup, ok := secgroups[secIds[i]]; ok { virObjs[i] = &secgroup rows[i].ProjectId = secgroup.ProjectId } + switch rule.TargetType { + case api.SecurityGroupRuleTargetTypeIpSet: + if ipSet, ok := ipSets[rule.CIDR]; ok { + rows[i].TargetIpSet = ipSet.Name + } + case api.SecurityGroupRuleTargetTypeSecurityGroup: + if secgroup, ok := secgroups[rule.CIDR]; ok { + rows[i].TargetSecurityGroup = secgroup.Name + } + } } projRows := SecurityGroupManager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, virObjs, fields, isList) @@ -294,9 +326,7 @@ func (manager *SSecurityGroupRuleManager) ValidateCreateData(ctx context.Context return nil, err } - opts := &api.SSecgroupCreateInput{} - opts.Rules = []api.SSecgroupRuleCreateInput{*input} - _, err = driver.ValidateCreateSecurityGroupInput(ctx, userCred, opts) + input, err = driver.ValidateCreateSecurityGroupRuleInput(ctx, userCred, input) if err != nil { return nil, err } @@ -323,6 +353,9 @@ func (self *SSecurityGroupRule) ValidateUpdateData(ctx context.Context, userCred return nil, err } + if len(input.TargetType) == 0 { + input.TargetType = self.TargetType + } input, err = driver.ValidateUpdateSecurityGroupRuleInput(ctx, userCred, input) if err != nil { return nil, err @@ -348,16 +381,16 @@ func (self *SSecurityGroupRule) Strings() []string { return ruleStrs } -func (self *SSecurityGroupRule) toRules() ([]*secrules.SecurityRule, error) { +func (sgr *SSecurityGroupRule) toRules() ([]*secrules.SecurityRule, error) { rule := secrules.SecurityRule{ - Priority: int(self.Priority), - Direction: secrules.TSecurityRuleDirection(self.Direction), - Action: secrules.TSecurityRuleAction(self.Action), - Protocol: self.Protocol, - Description: self.Description, + Priority: int(sgr.Priority), + Direction: secrules.TSecurityRuleDirection(sgr.Direction), + Action: secrules.TSecurityRuleAction(sgr.Action), + Protocol: sgr.Protocol, + Description: sgr.Description, } { - err := rule.ParsePorts(self.Ports) + err := rule.ParsePorts(sgr.Ports) if err != nil { return nil, errors.Wrap(err, "ParsePorts") } @@ -369,7 +402,7 @@ func (self *SSecurityGroupRule) toRules() ([]*secrules.SecurityRule, error) { } } - ipnets := netutils2.Str2IPNets(self.CIDR) + ipnets := sgr.getIpNets() if len(ipnets) == 0 { rule.IPNet = nil return []*secrules.SecurityRule{&rule}, nil @@ -383,6 +416,64 @@ func (self *SSecurityGroupRule) toRules() ([]*secrules.SecurityRule, error) { return rules, nil } +func (rule *SSecurityGroupRule) getIpNets() []*net.IPNet { + switch rule.TargetType { + case api.SecurityGroupRuleTargetTypeCidr: + return netutils2.Str2IPNets(rule.CIDR) + case api.SecurityGroupRuleTargetTypeIpSet: + ipSet, _ := rule.fetchIpSet() + if ipSet != nil { + return ipSet.getIpNets() + } + } + return nil +} + +func (rule *SSecurityGroupRule) fetchIpSet() (*SIpSet, error) { + ipSetObj, err := IpSetManager.FetchById(rule.CIDR) + if err != nil { + return nil, errors.Wrapf(err, "FetchById(%s)", rule.CIDR) + } + return ipSetObj.(*SIpSet), nil +} + +func GetCloudSecgroupRuleCIDR(targetType api.TSecgroupTargetType, cidr string) (string, error) { + switch targetType { + case api.SecurityGroupRuleTargetTypeIpSet: + if len(cidr) == 0 { + return "", errors.Wrap(errors.ErrEmpty, "empty ip set id") + } + ipSetObj, err := IpSetManager.FetchById(cidr) + if err != nil { + return "", errors.Wrapf(err, "FetchById(%s)", cidr) + } + ipSet := ipSetObj.(*SIpSet) + if len(ipSet.ExternalId) == 0 { + return "", errors.Wrapf(errors.ErrInvalidStatus, "ipset %s has empty external_id", ipSet.Id) + } + return ipSet.ExternalId, nil + case api.SecurityGroupRuleTargetTypeSecurityGroup: + if len(cidr) == 0 { + return "", errors.Wrap(errors.ErrEmpty, "empty security group id") + } + secgroupObj, err := SecurityGroupManager.FetchById(cidr) + if err != nil { + return "", errors.Wrapf(err, "FetchById(%s)", cidr) + } + secgroup := secgroupObj.(*SSecurityGroup) + if len(secgroup.ExternalId) == 0 { + return "", errors.Wrapf(errors.ErrInvalidStatus, "security group %s has empty external_id", secgroup.Id) + } + return secgroup.ExternalId, nil + default: + return cidr, nil + } +} + +func (rule *SSecurityGroupRule) GetCloudCIDR() (string, error) { + return GetCloudSecgroupRuleCIDR(rule.TargetType, rule.CIDR) +} + func (self *SSecurityGroupRule) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SResourceBase.PostCreate(ctx, userCred, ownerId, query, data) @@ -531,13 +622,79 @@ func (self *SSecurityGroup) SyncRules( return result } +func (self *SSecurityGroup) resolveCloudRuleTarget(ext cloudprovider.ISecurityGroupRule) (api.TSecgroupTargetType, string) { + cidr := strings.Join(ext.GetCIDRs(), ",") + targetType := api.TSecgroupTargetType(ext.GetTargetType()) + if len(targetType) == 0 { + targetType = api.SecurityGroupRuleTargetTypeCidr + } + switch targetType { + case api.SecurityGroupRuleTargetTypeIpSet: + if len(cidr) > 0 { + ipSet, err := self.fetchIpSetByExternalId(cidr) + if err == nil && ipSet != nil { + return targetType, ipSet.Id + } + } + case api.SecurityGroupRuleTargetTypeSecurityGroup: + if len(cidr) > 0 { + secgroup, err := self.fetchSecGroupByExternalId(cidr) + if err == nil && secgroup != nil { + return targetType, secgroup.Id + } + } + } + return targetType, cidr +} + +func (self *SSecurityGroup) fetchIpSetByExternalId(extId string) (*SIpSet, error) { + obj, err := db.FetchByExternalIdAndManagerId(IpSetManager, extId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + q = q.Equals("manager_id", self.ManagerId) + if len(self.CloudregionId) > 0 { + q = q.Filter(sqlchemy.OR( + sqlchemy.Equals(q.Field("cloudregion_id"), self.CloudregionId), + sqlchemy.IsNullOrEmpty(q.Field("cloudregion_id")), + )) + } + return q + }) + if err != nil { + return nil, err + } + return obj.(*SIpSet), nil +} + +func (self *SSecurityGroup) fetchSecGroupByExternalId(extId string) (*SSecurityGroup, error) { + obj, err := db.FetchByExternalIdAndManagerId(SecurityGroupManager, extId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + q = q.Equals("manager_id", self.ManagerId) + if len(self.CloudregionId) > 0 { + q = q.Equals("cloudregion_id", self.CloudregionId) + } + return q + }) + if err != nil { + return nil, err + } + return obj.(*SSecurityGroup), nil +} + func (rule *SSecurityGroupRule) syncWithCloudRule(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ISecurityGroupRule) error { + secgroup, _ := rule.GetSecGroup() + targetType := api.TSecgroupTargetType(ext.GetTargetType()) + if len(targetType) == 0 { + targetType = api.SecurityGroupRuleTargetTypeCidr + } + cidr := strings.Join(ext.GetCIDRs(), ",") + if secgroup != nil { + targetType, cidr = secgroup.resolveCloudRuleTarget(ext) + } _, err := db.Update(rule, func() error { rule.Action = string(ext.GetAction()) rule.Direction = string(ext.GetDirection()) rule.Protocol = string(ext.GetProtocol()) rule.Description = string(ext.GetDescription()) - rule.CIDR = strings.Join(ext.GetCIDRs(), ",") + rule.CIDR = cidr + rule.TargetType = targetType rule.Priority = ext.GetPriority() rule.Ports = ext.GetPorts() rule.Status = apis.STATUS_AVAILABLE @@ -547,6 +704,7 @@ func (rule *SSecurityGroupRule) syncWithCloudRule(ctx context.Context, userCred } func (self *SSecurityGroup) newFromCloudRule(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ISecurityGroupRule) error { + targetType, cidr := self.resolveCloudRuleTarget(ext) rule := &SSecurityGroupRule{} rule.SetModelManager(SecurityGroupRuleManager, rule) rule.SecgroupId = self.Id @@ -554,7 +712,8 @@ func (self *SSecurityGroup) newFromCloudRule(ctx context.Context, userCred mccli rule.Direction = string(ext.GetDirection()) rule.Protocol = string(ext.GetProtocol()) rule.Description = string(ext.GetDescription()) - rule.CIDR = strings.Join(ext.GetCIDRs(), ",") + rule.CIDR = cidr + rule.TargetType = targetType rule.Priority = ext.GetPriority() rule.Ports = ext.GetPorts() rule.ExternalId = ext.GetGlobalId() diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/secgroups.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/secgroups.go index c05eb7338..030d77122 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/secgroups.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/secgroups.go @@ -212,6 +212,22 @@ func (manager *SSecurityGroupManager) ListItemFilter( q = q.IsNullOrEmpty("manager_id") } + if len(input.IpSetId) > 0 { + for i := range input.IpSetId { + ipSetObj, err := IpSetManager.FetchByIdOrName(ctx, userCred, input.IpSetId[i]) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2("ip set %s not found", input.IpSetId[i]) + } else { + return nil, errors.Wrap(err, "IpSetManager.FetchByIdOrName") + } + } + input.IpSetId[i] = ipSetObj.GetId() + } + sq := SecurityGroupRuleManager.Query("secgroup_id").Equals("target_type", api.SecurityGroupRuleTargetTypeIpSet).In("cidr", input.IpSetId).Distinct().SubQuery() + q = q.Join(sq, sqlchemy.Equals(q.Field("id"), sq.Field("secgroup_id"))) + } + return q, nil } @@ -815,6 +831,10 @@ func (manager *SSecurityGroupManager) DelaySync(ctx context.Context, userCred mc return errors.Wrapf(err, "GetKvmGuests") } for _, guest := range guests { + // skip sync if guest has external id(cloudpods guest) + if len(guest.ExternalId) > 0 { + continue + } guest.StartSyncTask(ctx, userCred, true, "") } } @@ -1185,6 +1205,7 @@ func (self *SSecurityGroup) PerformImportRules(ctx context.Context, userCred mcc CIDR: r.CIDR, Action: r.Action, Description: r.Description, + TargetType: r.TargetType, } rule.SecgroupId = self.Id diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/snapshots.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/snapshots.go index facacb7fb..ebd8c7640 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/snapshots.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/snapshots.go @@ -76,10 +76,10 @@ type SSnapshot struct { // 快照大小,单位Mb Size int `nullable:"false" list:"user" create:"optional"` // Virtual size, for kvm is origin disk size - VirtualSize int `nullable:"false" list:"user" create:"optional"` - OutOfChain bool `nullable:"false" default:"false" list:"admin" create:"optional"` - FakeDeleted bool `nullable:"false" default:"false"` - DiskType string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional"` + VirtualSize int `nullable:"false" list:"user" create:"optional"` + //OutOfChain bool `nullable:"false" default:"false" list:"admin" create:"optional"` + //FakeDeleted bool `nullable:"false" default:"false"` + DiskType string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional"` // 操作系统类型 OsType string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional"` @@ -135,12 +135,6 @@ func (manager *SSnapshotManager) ListItemFilter( return nil, errors.Wrap(err, "SMultiArchResourceBaseManager.ListItemFilter") } - if query.FakeDeleted != nil && *query.FakeDeleted { - q = q.IsTrue("fake_deleted") - } else { - q = q.IsFalse("fake_deleted") - } - if query.Local != nil && *query.Local { storages := StorageManager.Query().SubQuery() sq := storages.Query(storages.Field("id")).Filter(sqlchemy.Equals(storages.Field("storage_type"), api.STORAGE_LOCAL)) @@ -184,13 +178,6 @@ func (manager *SSnapshotManager) ListItemFilter( return nil, errors.Wrap(err, "SStorageResourceBaseManager.ListItemFilter") } - if query.OutOfChain != nil { - if *query.OutOfChain { - q = q.IsTrue("out_of_chain") - } else { - q = q.IsFalse("out_of_chain") - } - } if len(query.OsType) > 0 { q = q.In("os_type", query.OsType) } @@ -652,8 +639,7 @@ func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string) dest := make([]SSnapshot, 0) q := self.Query().SubQuery() sq := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId), - sqlchemy.Equals(q.Field("created_by"), createdBy), - sqlchemy.Equals(q.Field("fake_deleted"), false))) + sqlchemy.Equals(q.Field("created_by"), createdBy))) err := db.FetchModelObjects(self, sq, &dest) if err != nil { log.Errorf("GetDiskSnapshots error: %s", err) @@ -674,27 +660,12 @@ func (self *SSnapshotManager) GetDiskSnapshots(diskId string) []SSnapshot { } func (self *SSnapshotManager) GetDiskManualSnapshotCount(diskId string) (int, error) { - return self.Query().Equals("disk_id", diskId).Equals("fake_deleted", false).CountWithError() -} - -func (self *SSnapshotManager) GetDiskFirstSnapshot(diskId string) *SSnapshot { - dest := &SSnapshot{} - q := self.Query().SubQuery() - err := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId), - sqlchemy.In(q.Field("status"), []string{api.SNAPSHOT_READY, api.SNAPSHOT_DELETING}), - sqlchemy.Equals(q.Field("out_of_chain"), false))).Asc("created_at").First(dest) - if err != nil { - log.Errorf("Get Disk First snapshot error: %s", err.Error()) - return nil - } - dest.SetModelManager(self, dest) - return dest + return self.Query().Equals("disk_id", diskId).CountWithError() } func (self *SSnapshotManager) GetDiskSnapshotCount(diskId string) (int, error) { q := self.Query().SubQuery() - return q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId), - sqlchemy.Equals(q.Field("fake_deleted"), false))).CountWithError() + return q.Query().Filter(sqlchemy.Equals(q.Field("disk_id"), diskId)).CountWithError() } func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, owner mcclient.IIdentityProvider, @@ -717,11 +688,6 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, owner mcclient // inherit encrypt_key_id snapshot.EncryptKeyId = disk.EncryptKeyId - driver, err := storage.GetRegionDriver() - if err != nil { - return nil, err - } - snapshot.OutOfChain = driver.SnapshotIsOutOfChain(disk) snapshot.VirtualSize = disk.DiskSize snapshot.DiskType = disk.DiskType snapshot.Location = location @@ -744,9 +710,8 @@ func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, owner mcclient return snapshot, nil } -func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, reloadDisk bool, parentTaskId string, deleteSnapshotTotalCnt int, deletedSnapshotCnt int) error { +func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, deleteSnapshotTotalCnt int, deletedSnapshotCnt int) error { params := jsonutils.NewDict() - params.Set("reload_disk", jsonutils.NewBool(reloadDisk)) if deleteSnapshotTotalCnt <= 0 { deleteSnapshotTotalCnt = 1 } @@ -788,6 +753,7 @@ func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context, info *api.Sn return httperrors.NewBadRequestError("Cannot delete snapshot on disk reset") } } + driver := self.GetRegionDriver() if driver != nil { return driver.ValidateSnapshotDelete(ctx, self) @@ -815,20 +781,12 @@ func (self *SSnapshot) GetRegionDriver() IRegionDriver { } func (self *SSnapshot) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { - return self.StartSnapshotDeleteTask(ctx, userCred, false, "", 0, 0) + return self.StartSnapshotDeleteTask(ctx, userCred, "", 0, 0) } // +onecloud:swagger-gen-ignore func (self *SSnapshot) PerformDeleted(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - _, err := db.Update(self, func() error { - self.OutOfChain = true - return nil - }) - if err != nil { - return nil, err - } - err = self.StartSnapshotDeleteTask(ctx, userCred, true, "", 0, 0) - return nil, err + return nil, nil } // 同步快照状态 @@ -845,23 +803,12 @@ func (self *SSnapshot) PerformSyncstatus(ctx context.Context, userCred mcclient. return nil, StartResourceSyncStatusTask(ctx, userCred, self, "SnapshotSyncstatusTask", "") } -func (self *SSnapshotManager) GetConvertSnapshot(deleteSnapshot *SSnapshot) (*SSnapshot, error) { - dest := &SSnapshot{} - q := self.Query() - err := q.Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), deleteSnapshot.DiskId), - sqlchemy.In(q.Field("status"), []string{api.SNAPSHOT_READY, api.SNAPSHOT_DELETING}), - sqlchemy.Equals(q.Field("out_of_chain"), false), - sqlchemy.GT(q.Field("created_at"), deleteSnapshot.CreatedAt))). - Asc("created_at").First(dest) - if err != nil { - return nil, err - } - return dest, nil -} - // +onecloud:swagger-gen-ignore func (self *SSnapshotManager) PerformDeleteDiskSnapshots(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SnapshotDeleteDiskSnapshotsInput) (jsonutils.JSONObject, error) { disk, err := DiskManager.FetchById(input.DiskId) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return nil, err + } if disk != nil { return nil, httperrors.NewBadRequestError("Cannot delete disk %s snapshots, disk still exists", input.DiskId) } @@ -871,9 +818,6 @@ func (self *SSnapshotManager) PerformDeleteDiskSnapshots(ctx context.Context, us } snapshotIds := []string{} for i := 0; i < len(snapshots); i++ { - if snapshots[i].FakeDeleted == false { - return nil, httperrors.NewBadRequestError("Cannot delete disk snapshots, has manual snapshots") - } snapshotIds = append(snapshotIds, snapshots[i].Id) } err = snapshots[0].StartSnapshotsDeleteTask(ctx, userCred, "", snapshotIds) @@ -977,7 +921,6 @@ func TotalSnapshotCount(ctx context.Context, scope rbacscope.TRbacScope, ownerId q = RangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id"), nil, nil) q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv) q = q.Equals("created_by", api.SNAPSHOT_MANUAL) - q = q.Equals("fake_deleted", false) return q.CountWithError() } @@ -1206,7 +1149,7 @@ func (self *SSnapshot) getCloudProviderInfo() SCloudProviderInfo { } func (manager *SSnapshotManager) GetResourceCount() ([]db.SScopeResourceCount, error) { - virts := manager.Query().IsFalse("fake_deleted") + virts := manager.Query() return db.CalculateResourceCount(virts, "tenant_id") } @@ -1221,10 +1164,10 @@ func (manager *SSnapshotManager) CleanupSnapshots(ctx context.Context, userCred log.Errorf("Previous CleanupSnapshots tasks still running !!!") return } + log.Infof("start clean expired snaps...") var now = time.Now() var snapshot = new(SSnapshot) err := manager.Query(). - Equals("fake_deleted", false). Equals("created_by", api.SNAPSHOT_AUTO). LE("expired_at", now).First(snapshot) if err != nil && err != sql.ErrNoRows { @@ -1232,17 +1175,17 @@ func (manager *SSnapshotManager) CleanupSnapshots(ctx context.Context, userCred return } else if err == sql.ErrNoRows { log.Infof("No snapshot need to clean ......") + } else { + snapshot.SetModelManager(manager, snapshot) + region, _ := snapshot.GetRegion() + if err = manager.StartSnapshotCleanupTask(ctx, userCred, region, now); err != nil { + log.Errorf("Start snaphsot cleanup task failed %s", err) + } return } - snapshot.SetModelManager(manager, snapshot) - region, _ := snapshot.GetRegion() - if err = manager.StartSnapshotCleanupTask(ctx, userCred, region, now); err != nil { - log.Errorf("Start snaphsot cleanup task failed %s", err) - return - } - - sq := manager.Query().Equals("status", api.SNAPSHOT_READY).Equals("created_by", api.SNAPSHOT_AUTO).Equals("fake_deleted", false).SubQuery() + log.Infof("start clean over retention count snaps...") + sq := manager.Query().Equals("created_by", api.SNAPSHOT_AUTO).SubQuery() disks := []struct { DiskCnt int @@ -1295,22 +1238,22 @@ func (manager *SSnapshotManager) CleanupSnapshots(ctx context.Context, userCred for diskId, retentionCnt := range diskRetentionMap { if cnt, ok := diskCount[diskId]; ok && cnt > retentionCnt { + log.Infof("disk %s snapshot count %d, retention count %d", diskId, cnt, retentionCnt) manager.startCleanupRetentionCount(ctx, userCred, diskId, cnt-retentionCnt) + return } } } } func (manager *SSnapshotManager) startCleanupRetentionCount(ctx context.Context, userCred mcclient.TokenCredential, diskId string, cnt int) error { - q := manager.Query().Equals("disk_id", diskId).Equals("created_by", api.SNAPSHOT_AUTO).Asc("created_at").Limit(cnt) - snapshots := []SSnapshot{} - err := db.FetchModelObjects(manager, q, &snapshots) + snap := new(SSnapshot) + err := manager.Query().Equals("disk_id", diskId).Equals("created_by", api.SNAPSHOT_AUTO).Asc("created_at").First(snap) if err != nil { - return errors.Wrapf(err, "FetchModelObjects") - } - for i := range snapshots { - snapshots[i].StartSnapshotDeleteTask(ctx, userCred, false, "", 0, 0) + return err } + snap.SetModelManager(manager, snap) + snap.StartSnapshotDeleteTask(ctx, userCred, "", 0, 0) return nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/storagecachedimages.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/storagecachedimages.go index 5a35f57f7..b85c7af1d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/storagecachedimages.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/storagecachedimages.go @@ -361,6 +361,9 @@ func (manager *SStoragecachedimageManager) RecoverStoragecachedImage( } return nil }) + if err != nil { + return nil, err + } db.OpsLog.LogEvent(&storagecachedImage, db.ACT_UPDATE, diff, userCred) return &storagecachedImage, nil diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/storages.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/storages.go index 05eff3cba..24e937128 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/storages.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/storages.go @@ -190,16 +190,6 @@ func (self *SStorage) StartStorageUpdateTask(ctx context.Context, userCred mccli return nil } -func (self *SStorage) getFakeDeletedSnapshots() ([]SSnapshot, error) { - q := SnapshotManager.Query().Equals("storage_id", self.Id).IsTrue("fake_deleted") - snapshots := make([]SSnapshot, 0) - err := db.FetchModelObjects(SnapshotManager, q, &snapshots) - if err != nil { - return nil, errors.Wrap(err, "FetchModelObjects") - } - return snapshots, nil -} - func (self *SStorage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { ok, err := self.IsNeedDeleteStoragecache() if err != nil { @@ -320,10 +310,7 @@ func (manager *SStorageManager) ValidateCreateData( func (self *SStorage) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { self.SetEnabled(true) - self.SetStatus(ctx, userCred, api.STORAGE_UNMOUNT, "CustomizeCreate") - if err := self.setHardwareInfoByData(ctx, userCred, data); err != nil { - return errors.Wrap(err, "setHardwareInfo") - } + self.SetStatusValue(api.STORAGE_UNMOUNT) return self.SEnabledStatusInfrasResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) } @@ -371,37 +358,16 @@ func (self *SStorage) ValidateDeleteCondition(ctx context.Context, info api.Stor func (self *SStorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { self.SEnabledStatusInfrasResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + if err := self.setHardwareInfoByData(ctx, userCred, data); err != nil { + log.Errorf("setHardwareInfoByData error: %v", err) + } + storageDriver := GetStorageDriver(self.StorageType) if storageDriver != nil { storageDriver.PostCreate(ctx, userCred, self, data) } } -func (self *SStorage) SetStatus(ctx context.Context, userCred mcclient.TokenCredential, status string, reason string) error { - if self.Status == status { - return nil - } - oldStatus := self.Status - _, err := db.Update(self, func() error { - self.Status = status - return nil - }) - if err != nil { - return err - } - if userCred != nil { - notes := fmt.Sprintf("%s=>%s", oldStatus, status) - if len(reason) > 0 { - notes = fmt.Sprintf("%s: %s", notes, reason) - } - db.OpsLog.LogEvent(self, db.ACT_UPDATE_STATUS, notes, userCred) - // if strings.Contains(notes, "fail") { - // logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_SYNC_STATUS, notes, userCred, false) - // } - } - return nil -} - func (self *SStorage) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { if self.Enabled.IsFalse() { _, err := db.Update(self, func() error { @@ -480,7 +446,7 @@ func (self *SStorage) GetDisks() []SDisk { } func (self *SStorage) GetVisibleSnapshotCount() (int, error) { - return SnapshotManager.Query().Equals("storage_id", self.Id).IsFalse("fake_deleted").CountWithError() + return SnapshotManager.Query().Equals("storage_id", self.Id).CountWithError() } func (self *SStorage) IsLocal() bool { @@ -568,9 +534,7 @@ func (manager *SStorageManager) TotalResourceCount(storageIds []string) (map[str sqlchemy.SUM("disk_wasted", _diskWastedSQ.Field("disk_size")), ).In("storage_id", storageIds).GroupBy(_diskWastedSQ.Field("storage_id")).SubQuery() - snapshotSQ := manager.query(SnapshotManager, "snapshot_cnt", storageIds, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { - return q.IsFalse("fake_deleted") - }) + snapshotSQ := manager.query(SnapshotManager, "snapshot_cnt", storageIds, nil) storages := manager.Query().SubQuery() storageQ := storages.Query( diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/vpcs.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/vpcs.go index 00a341a80..be883cc42 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/vpcs.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/vpcs.go @@ -1065,9 +1065,9 @@ func (svpc *SVpc) GetIVpc(ctx context.Context) (cloudprovider.ICloudVpc, error) if provider.GetFactory().IsOnPremise() { iregion, err = provider.GetOnPremiseIRegion() } else { - region, err := svpc.GetRegion() - if err != nil { - return nil, err + region, rerr := svpc.GetRegion() + if rerr != nil { + return nil, rerr } iregion, err = provider.GetIRegionById(region.ExternalId) } diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/wires.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/wires.go index d61007f2a..17340cb7f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/wires.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/wires.go @@ -978,6 +978,7 @@ func (swire *SWire) getNetworks(ctx context.Context, userCred mcclient.TokenCred func (swire *SWire) getGatewayNetworkQuery(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { q := swire.getNetworkQuery(ctx, userCred, ownerId, scope) q = q.Filter(sqlchemy.OR( + sqlchemy.In(q.Field("server_type"), []api.TNetworkType{api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_PXE, api.NETWORK_TYPE_IPMI}), sqlchemy.AND(sqlchemy.IsNotNull(q.Field("guest_gateway")), sqlchemy.IsNotEmpty(q.Field("guest_gateway"))), sqlchemy.AND(sqlchemy.IsNotNull(q.Field("guest_gateway6")), sqlchemy.IsNotEmpty(q.Field("guest_gateway6"))), )) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/models/zones.go b/vendor/yunion.io/x/onecloud/pkg/compute/models/zones.go index 36a6baa8f..34b121ce0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/models/zones.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/models/zones.go @@ -803,6 +803,9 @@ func (manager *SZoneManager) ListItemFilter( data := jsonutils.Marshal(query.DomainizedResourceListInput) domainId, err := db.FetchQueryDomain(ctx, userCred, data) + if err != nil { + return nil, err + } if len(domainId) > 0 { q = q.In("cloudregion_id", getCloudRegionIdByDomainId(domainId)) } @@ -869,6 +872,9 @@ func (manager *SZoneManager) ListItemFilter( } q, err = managedResourceFilterByRegion(ctx, q, query.RegionalFilterListInput, "", nil) + if err != nil { + return nil, err + } if len(query.Location) > 0 { q = q.In("location", query.Location) diff --git a/vendor/yunion.io/x/onecloud/pkg/compute/options/options.go b/vendor/yunion.io/x/onecloud/pkg/compute/options/options.go index 7b7099337..f18f3d31c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/compute/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/compute/options/options.go @@ -98,6 +98,9 @@ type ComputeOptions struct { DefaultDiskDriver string `help:"default disk driver" choices:"scsi|virtio|ide" default:"scsi"` DefaultDiskCacheMode string `help:"default kvm disk cache mode" choices:"writeback|none|writethrough" default:"none"` + DefaultGuestStopTimeout int `help:"default guest stop timeout" default:"300"` + DefaultGuestForceStopTimeout int `help:"default guest force stop timeout" default:"30"` + SystemAdminQuotaCheck bool `help:"Enable quota check for system admin, default False" default:"false"` CloudaccountHealthStatusCheck bool `help:"Enable cloudaccount health status check, default True" default:"true"` diff --git a/vendor/yunion.io/x/onecloud/pkg/devtool/utils/gen_telegraf_conf.go b/vendor/yunion.io/x/onecloud/pkg/devtool/utils/gen_telegraf_conf.go index 7f2b96f35..6f1813166 100644 --- a/vendor/yunion.io/x/onecloud/pkg/devtool/utils/gen_telegraf_conf.go +++ b/vendor/yunion.io/x/onecloud/pkg/devtool/utils/gen_telegraf_conf.go @@ -152,11 +152,6 @@ const TELEGRAF_INPUT_BAREMETAL = ` [[inputs.diskio]] name_prefix = "agent_" skip_serial_number = false -[[inputs.sensors]] - name_prefix = "agent_" -[[inputs.smart]] - name_prefix = "agent_" - use_sudo = true [[inputs.mem]] name_prefix = "agent_" [[inputs.processes]] diff --git a/vendor/yunion.io/x/onecloud/pkg/devtool/utils/ssh.go b/vendor/yunion.io/x/onecloud/pkg/devtool/utils/ssh.go index 3fdf5ba55..77cf58003 100644 --- a/vendor/yunion.io/x/onecloud/pkg/devtool/utils/ssh.go +++ b/vendor/yunion.io/x/onecloud/pkg/devtool/utils/ssh.go @@ -281,6 +281,7 @@ func CheckSSHable(session *mcclient.ClientSession, serverId string) (sshable SSH sshport, err = getServerSshport(session, serverDetail.Id, false) if err != nil { err = errors.Wrapf(err, "unable to get sshport of server %s", serverDetail.Id) + return } // create local forward createP := jsonutils.NewDict() diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/base.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/base.go index ee60266ba..ac1056a22 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/base.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/base.go @@ -31,6 +31,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -59,6 +60,11 @@ func (d *sGuestRootFsDriver) DeployFiles(deploys []*deployapi.DeployContent) err if len(deploy.Path) == 0 { return fmt.Errorf("Deploy file missing param path") } + clean, err := fileutils2.CleanGuestDeployPath(deploy.Path) + if err != nil { + return errors.Wrap(err, "deploy path") + } + deploy.Path = clean dirname := filepath.Dir(deploy.Path) if !d.GetPartition().Exists(dirname, caseInsensitive) { modeRWXOwner := syscall.S_IRWXU | syscall.S_IRGRP | syscall.S_IXGRP | syscall.S_IROTH | syscall.S_IXOTH @@ -160,6 +166,10 @@ func (l *sGuestRootFsDriver) MountProcfs() bool { return false } +func (l *sGuestRootFsDriver) IsWindowsVirtioNetSupport(IDiskPartition) bool { + return false +} + const ( modeAuthorizedKeysRWX = syscall.S_IRUSR | syscall.S_IWUSR | syscall.S_IXUSR modeAuthorizedKeysRW = syscall.S_IRUSR | syscall.S_IWUSR diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/drivers.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/drivers.go index 93f63c770..52d75dec8 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/drivers.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/drivers.go @@ -37,7 +37,7 @@ func GetRootfsDrivers() []newRootFsDriverFunc { func Init(cloudrootDir string) error { linuxFsDrivers := []newRootFsDriverFunc{ - NewFangdeRootFs, NewUnionOSRootFs, + NewOpenRuyiRootFs, NewFangdeRootFs, NewUnionOSRootFs, NewAnolisRootFs, NewRockyRootFs, NewOpenCloudOsRootFs, NewAlmaLinuxRootFs, NewOpenKylinRootfs, NewUOSDesktopRootfs, NewGalaxyKylinRootFs, NewNeoKylinRootFs, diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/interface.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/interface.go index 7c846718b..728588546 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/interface.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/interface.go @@ -46,7 +46,8 @@ type IDiskPartition interface { Cleandir(dir string, keepdir, caseInsensitive bool) error Zerofiles(dir string, caseInsensitive bool) error SupportSerialPorts() bool - //Copy(src, dest string) error + CopyFile(src, dest string) error + ExecCommand(name string, arg ...string) ([]string, error) GetPartDev() string IsMounted() bool @@ -95,6 +96,7 @@ type IRootFsDriver interface { IsCloudinitInstall() bool IsResizeFsPartitionSupport() bool MountProcfs() bool + IsWindowsVirtioNetSupport(part IDiskPartition) bool PrepareFsForTemplate(IDiskPartition) error CleanNetworkScripts(rootFs IDiskPartition) error diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux.go index cb85b167f..cb5348200 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux.go @@ -234,10 +234,10 @@ func (l *sLinuxRootFs) checkInputPasswd(rootFs IDiskPartition, config *pwquality err := config.Validate(password, account) if err != nil && errors.Cause(err) == pwquality.ErrPasswordTooWeak { - log.Infof("password %s too weak, try regenerate password", password) + log.Infof("password too weak, try regenerate password") npassword := config.GeneratePassword(seclib2.RandomPassword2) if len(npassword) > 0 { - log.Infof("regenerate password %s", npassword) + log.Infof("regenerate password (not logged)") password = npassword } } @@ -562,6 +562,9 @@ func (l *sLinuxRootFs) GetArch(rootFs IDiskPartition) string { log.Errorf("readlink of %s: %s", p, err) continue } + if mnt := rootFs.GetMountPath(); mnt != "" && !fileutils2.IsPathInside(mnt, rp) { + continue + } elfHeader, err := elf.Open(rp) if err != nil { log.Errorf("failed read file elf %s: %s", rp, err) @@ -824,15 +827,19 @@ func (d *sLinuxRootFs) DeployTelegraf(config string) (bool, error) { if err != nil { return false, errors.Wrap(err, "write telegraf config") } - output, err := procutils.NewCommand("cp", "-f", TELEGRAF_BINARY_PATH, path.Join(part.GetMountPath(), cloudMonitorPath)).Output() + telegrafBin := path.Base(TELEGRAF_BINARY_PATH) + err = part.CopyFile(TELEGRAF_BINARY_PATH, path.Join(cloudMonitorPath, telegrafBin)) if err != nil { - return false, errors.Wrapf(err, "cp telegraf failed %s", output) + return false, errors.Wrap(err, "copy telegraf file") } + // supervise - output, err = procutils.NewCommand("cp", "-f", SUPERVISE_BINARY_PATH, path.Join(part.GetMountPath(), cloudMonitorPath)).Output() + superviseBin := path.Base(SUPERVISE_BINARY_PATH) + err = part.CopyFile(SUPERVISE_BINARY_PATH, path.Join(cloudMonitorPath, superviseBin)) if err != nil { - return false, errors.Wrapf(err, "cp supervise failed %s", output) + return false, errors.Wrap(err, "copy supervise file") } + err = part.FilePutContents( path.Join(telegrafPath, "run"), fmt.Sprintf("#!/bin/sh\n%s/telegraf -config %s/telegraf.conf", cloudMonitorPath, cloudMonitorPath), @@ -2462,7 +2469,11 @@ func (d *SCoreOsRootFs) GetLoginAccount(rootFs IDiskPartition, user string, defa func (d *SCoreOsRootFs) DeployFiles(deploys []*deployapi.DeployContent) error { for _, deploy := range deploys { - d.GetConfig().AddWriteFile(deploy.Path, deploy.Content, "", "", false) + clean, err := fileutils2.CleanGuestDeployPath(deploy.Path) + if err != nil { + return errors.Wrap(err, "deploy path") + } + d.GetConfig().AddWriteFile(clean, deploy.Content, "", "", false) } return nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux_init.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux_init.go index 1eba46df8..74db48902 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux_init.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/linux_init.go @@ -16,10 +16,9 @@ package fsdriver import ( "fmt" + "strings" "yunion.io/x/pkg/errors" - - "yunion.io/x/onecloud/pkg/util/procutils" ) const ( @@ -41,15 +40,19 @@ func (d *sLinuxRootFs) installInitScript(name, cmd string, oneshot bool) error { func (d *sLinuxRootFs) installCrond(cmd string) error { cronJob := fmt.Sprintf("@reboot %s", cmd) - if procutils.NewCommand("chroot", d.rootFs.GetMountPath(), "crontab", "-l", "|", "grep", cronJob).Run() == nil { - // if cronjob exist, return success - return nil + outs, err := d.rootFs.ExecCommand("crontab", "-l") + if err != nil { + return errors.Wrapf(err, "exec crontab -l: %v", outs) + } + for _, line := range outs { + if strings.Contains(line, cronJob) { + // if cronjob exist, return success + return nil + } } - output, err := procutils.NewCommand("chroot", d.rootFs.GetMountPath(), "sh", "-c", - fmt.Sprintf("(crontab -l 2>/dev/null; echo '%s') |crontab -", cronJob), - ).Output() + outs, err = d.rootFs.ExecCommand("sh", "-c", fmt.Sprintf("(crontab -l 2>/dev/null; echo '%s') |crontab -", cronJob)) if err != nil { - return errors.Wrapf(err, "add crontab %s", output) + return errors.Wrapf(err, "add crontab %v", outs) } return nil } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/netplan.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/netplan.go index 3d6e5577f..3e56ce8d0 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/netplan.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/netplan.go @@ -86,17 +86,32 @@ func newNetplanNetwork(allNics []*types.SServerNic, bondNics []*types.SServerNic network.AddEthernet(sn.Name, nicConf) } - netConf := getNetplanEthernetConfig(bondNic, true, mainIp, mainIp6, nicCnt) - if netConf.Mtu == 0 { netConf.Mtu = defaultMtu } + if bondNic.VlanInterface { + ifname := fmt.Sprintf("%s.%d", bondNic.Name, bondNic.Vlan) + vlanConfig := &netplan.VlanConfig{ + EthernetConfig: *netConf, + Link: bondNic.Name, + Id: bondNic.Vlan, + } + network.AddVlan(ifname, vlanConfig) - // TODO: implement kinds of bond mode config - bondConf := netplan.NewBondMode4(netConf, interfaces) - - network.AddBond(bondNic.Name, bondConf) + bondNicConf := &netplan.EthernetConfig{ + DHCP4: false, + DHCP6: false, + MacAddress: bondNic.Mac, + Match: netplan.NewEthernetConfigMatchMac(bondNic.Mac), + } + bondConf := netplan.NewBondMode4(bondNicConf, interfaces) + network.AddBond(bondNic.Name, bondConf) + } else { + // TODO: implement kinds of bond mode config + bondConf := netplan.NewBondMode4(netConf, interfaces) + network.AddBond(bondNic.Name, bondConf) + } } return network diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/openruyi.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/openruyi.go new file mode 100644 index 000000000..3b296d87a --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/openruyi.go @@ -0,0 +1,73 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fsdriver + +import ( + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/types" + deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" +) + +type SOpenRuyiRootFs struct { + *sRedhatLikeRootFs +} + +func NewOpenRuyiRootFs(part IDiskPartition) IRootFsDriver { + return &SOpenRuyiRootFs{sRedhatLikeRootFs: newRedhatLikeRootFs(part)} +} + +func (d *SOpenRuyiRootFs) GetName() string { + return "openRuyi" +} + +func (d *SOpenRuyiRootFs) String() string { + return "OpenRuyiRootFs" +} + +func (d *SOpenRuyiRootFs) RootSignatures() []string { + sig := d.sLinuxRootFs.RootSignatures() + return append([]string{"/etc/os-release", "/usr/lib/rpm/openruyi"}, sig...) +} + +func parseOpenRuyiVersion(osRelease string) string { + for _, line := range strings.Split(osRelease, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "VERSION_ID=") { + return strings.Trim(strings.TrimSpace(strings.TrimPrefix(line, "VERSION_ID=")), `"'`) + } + } + return "" +} + +func (d *SOpenRuyiRootFs) GetReleaseInfo(rootFs IDiskPartition) *deployapi.ReleaseInfo { + rel, _ := rootFs.FileGetContents("/etc/os-release", false) + return deployapi.NewReleaseInfo(d.GetName(), parseOpenRuyiVersion(string(rel)), d.GetArch(rootFs)) +} + +func (d *SOpenRuyiRootFs) DeployNetworkingScripts(rootFs IDiskPartition, nics []*types.SServerNic) error { + return d.sRedhatLikeRootFs.deployNetworkingScripts(rootFs, nics, d.GetReleaseInfo(rootFs)) +} + +func (d *SOpenRuyiRootFs) EnableSerialConsole(rootFs IDiskPartition, sysInfo *jsonutils.JSONDict) error { + return d.enableSerialConsoleSystemd(rootFs) +} + +func (d *SOpenRuyiRootFs) DisableSerialConsole(rootFs IDiskPartition) error { + d.disableSerialConsoleSystemd(rootFs) + return nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/windows.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/windows.go index 0b7a1cfe9..330a015e4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/windows.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/windows.go @@ -546,8 +546,10 @@ func (w *SWindowsRootFs) deploySetupCompleteScripts(uname, passwd string) bool { "Net stop wuauserv", } for _, v := range [][3]string{ - {"AUOptions", "REG_DWORD", "3"}, - {"NoAutoUpdate", "REG_DWORD", "0"}, + // enable: 3, disable: 2 + {"AUOptions", "REG_DWORD", "2"}, + // enable: 0, disable: 1 + {"NoAutoUpdate", "REG_DWORD", "1"}, {"ScheduledInstallDay", "REG_DWORD", "0"}, {"ScheduledInstallTime", "REG_DWORD", "4"}, {"AutoInstallMinorUpdates", "REG_DWORD", "1"}, @@ -559,8 +561,9 @@ func (w *SWindowsRootFs) deploySetupCompleteScripts(uname, passwd string) bool { v[0], v[1], v[2])) } cmds = append(cmds, `REG ADD "HKLM\SYSTEM\CurrentControlSet\Control\TimeZoneInformation" /v RealTimeIsUniversal /t REG_DWORD /d 1 /f`) - cmds = append(cmds, "Net start wuauserv") - cmds = append(cmds, "wuauclt /detectnow") + cmds = append(cmds, `REG ADD "HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\PasswordLess\Device" /v DevicePasswordLessBuildVersion /t REG_DWORD /d 0 /f`) + // cmds = append(cmds, "Net start wuauserv") + // cmds = append(cmds, "wuauclt /detectnow") cmds = append(cmds, `del %SystemRoot%\chgpwd_setup.ps1`) cmds = append(cmds, `del %SystemRoot%\Setup\Scripts\SetupComplete.cmd`) if w.putGuestScriptContents(SETUP_SCRIPT_PATH, strings.Join(cmds, "\r\n")) != nil { @@ -621,7 +624,8 @@ func (w *SWindowsRootFs) DeployQgaService(part IDiskPartition) error { return errors.Wrap(err, "mkdir qemu-ga path") } - qgaInstallerPath := path.Join(w.rootFs.GetMountPath(), WIN_QGA_PATH, "qemu-ga-x86_64.msi") + winQgaPath := w.rootFs.GetLocalPath(WIN_QGA_PATH, true) + qgaInstallerPath := path.Join(winQgaPath, "qemu-ga-x86_64.msi") output, err := procutils.NewCommand("cp", "-f", QGA_WIN_MSI_INSTALLER_PATH, qgaInstallerPath).Output() if err != nil { return errors.Wrapf(err, "cp qga installer failed %s", output) @@ -643,16 +647,17 @@ func (w *SWindowsRootFs) DeployTelegraf(config string) (bool, error) { return false, errors.Wrap(err, "mkdir telegraf path") } - telegrafConfPath := path.Join(w.rootFs.GetMountPath(), WIN_TELEGRAF_PATH, "telegraf.conf") + winTelegrafPath := w.rootFs.GetLocalPath(WIN_TELEGRAF_PATH, true) + telegrafConfPath := path.Join(winTelegrafPath, "telegraf.conf") if err := w.rootFs.FilePutContents(telegrafConfPath, config, false, true); err != nil { return false, errors.Wrap(err, "write boot script") } telegrafConfPath = strings.ReplaceAll(path.Join("%PROGRAMFILES%", "Telegraf", "telegraf.conf"), "/", "\\") + telegrafBinaryPath := path.Join(winTelegrafPath, "telegraf.exe") - telegrafBinaryPath := path.Join(w.rootFs.GetMountPath(), WIN_TELEGRAF_PATH, "telegraf.exe") - output, err := procutils.NewCommand("cp", "-f", WIN_TELEGRAF_BINARY_PATH, telegrafBinaryPath).Output() + err := w.rootFs.CopyFile(WIN_TELEGRAF_BINARY_PATH, path.Join(WIN_TELEGRAF_PATH, "telegraf.exe")) if err != nil { - return false, errors.Wrapf(err, "cp telegraf failed %s", output) + return false, errors.Wrap(err, "cp telegraf failed") } telegrafBinaryPath = strings.ReplaceAll(path.Join("%PROGRAMFILES%", "Telegraf", "telegraf.exe"), "/", "\\") bootScript := strings.Join([]string{ @@ -686,3 +691,10 @@ func (w *SWindowsRootFs) DeployTelegraf(config string) (bool, error) { func (w *SWindowsRootFs) ConfigSshd(loginAccount, loginPassword string, sshPort int) error { return nil } + +func (w *SWindowsRootFs) IsWindowsVirtioNetSupport(part IDiskPartition) bool { + confPath := part.GetLocalPath("/windows/system32/config", true) + tool := winutils.NewWinRegTool(confPath) + tool.CheckPath() + return tool.IsNetKVMInstalled() +} diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/winscripts.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/winscripts.go index 59a779aac..a823b9b55 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/winscripts.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver/winscripts.go @@ -473,12 +473,14 @@ function mtw_mount_disk() { } function mtw_extend_c() { - cmd_lines = [ + mtw_execute_diskpart([ 'select volume c', 'extend', + ]); + mtw_execute_diskpart([ + 'select volume c', 'extend filesystem', - ]; - mtw_execute_diskpart(cmd_lines); + ]); mtw_append_debug(["extend c"], ["success"]); } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/guestman/desc/desc.go b/vendor/yunion.io/x/onecloud/pkg/hostman/guestman/desc/desc.go index 7dabd7b4b..b9865b2f6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/guestman/desc/desc.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/guestman/desc/desc.go @@ -266,12 +266,19 @@ type SGuestRng struct { type SoundCard struct { *PCIDevice `json:",omitempty"` Codec *Codec + Audio *AudioDev } -type Codec struct { +type AudioDev struct { Id string Type string - Cad int +} + +type Codec struct { + Id string + Type string + Cad int + AudioDev string } type SSpiceDesc struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.pb.go b/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.pb.go index a7e6615c1..2b36375fb 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.pb.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.20.0 -// protoc v3.21.5 +// protoc v3.20.3 // source: deploy.proto // protoc --version=libprotoc 3.11.3 @@ -1828,14 +1828,15 @@ type ImageInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - OsInfo *ReleaseInfo `protobuf:"bytes,1,opt,name=os_info,json=osInfo,proto3" json:"os_info,omitempty"` - OsType string `protobuf:"bytes,2,opt,name=os_type,json=osType,proto3" json:"os_type,omitempty"` - IsUefiSupport bool `protobuf:"varint,3,opt,name=is_uefi_support,json=isUefiSupport,proto3" json:"is_uefi_support,omitempty"` - IsLvmPartition bool `protobuf:"varint,4,opt,name=is_lvm_partition,json=isLvmPartition,proto3" json:"is_lvm_partition,omitempty"` - IsReadonly bool `protobuf:"varint,5,opt,name=is_readonly,json=isReadonly,proto3" json:"is_readonly,omitempty"` - PhysicalPartitionType string `protobuf:"bytes,6,opt,name=physical_partition_type,json=physicalPartitionType,proto3" json:"physical_partition_type,omitempty"` - IsInstalledCloudInit bool `protobuf:"varint,7,opt,name=is_installed_cloud_init,json=isInstalledCloudInit,proto3" json:"is_installed_cloud_init,omitempty"` - IsBiosSupport bool `protobuf:"varint,8,opt,name=is_bios_support,json=isBiosSupport,proto3" json:"is_bios_support,omitempty"` + OsInfo *ReleaseInfo `protobuf:"bytes,1,opt,name=os_info,json=osInfo,proto3" json:"os_info,omitempty"` + OsType string `protobuf:"bytes,2,opt,name=os_type,json=osType,proto3" json:"os_type,omitempty"` + IsUefiSupport bool `protobuf:"varint,3,opt,name=is_uefi_support,json=isUefiSupport,proto3" json:"is_uefi_support,omitempty"` + IsLvmPartition bool `protobuf:"varint,4,opt,name=is_lvm_partition,json=isLvmPartition,proto3" json:"is_lvm_partition,omitempty"` + IsReadonly bool `protobuf:"varint,5,opt,name=is_readonly,json=isReadonly,proto3" json:"is_readonly,omitempty"` + PhysicalPartitionType string `protobuf:"bytes,6,opt,name=physical_partition_type,json=physicalPartitionType,proto3" json:"physical_partition_type,omitempty"` + IsInstalledCloudInit bool `protobuf:"varint,7,opt,name=is_installed_cloud_init,json=isInstalledCloudInit,proto3" json:"is_installed_cloud_init,omitempty"` + IsBiosSupport bool `protobuf:"varint,8,opt,name=is_bios_support,json=isBiosSupport,proto3" json:"is_bios_support,omitempty"` + IsWindowsVirtioNetSupport bool `protobuf:"varint,9,opt,name=is_windows_virtio_net_support,json=isWindowsVirtioNetSupport,proto3" json:"is_windows_virtio_net_support,omitempty"` } func (x *ImageInfo) Reset() { @@ -1926,6 +1927,13 @@ func (x *ImageInfo) GetIsBiosSupport() bool { return false } +func (x *ImageInfo) GetIsWindowsVirtioNetSupport() bool { + if x != nil { + return x.IsWindowsVirtioNetSupport + } + return false +} + type EsxiDiskInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2458,7 +2466,7 @@ var file_deploy_proto_rawDesc = []byte{ 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x50, 0x72, 0x61, 0x6d, 0x61, 0x73, 0x12, 0x2b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x08, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xda, 0x02, 0x0a, 0x09, 0x49, 0x6d, + 0x08, 0x64, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x9c, 0x03, 0x0a, 0x09, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2a, 0x0a, 0x07, 0x6f, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x6f, 0x73, 0x49, @@ -2480,71 +2488,75 @@ var file_deploy_proto_rawDesc = []byte{ 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x69, 0x73, 0x5f, 0x62, 0x69, 0x6f, 0x73, 0x5f, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x69, 0x73, 0x42, 0x69, 0x6f, 0x73, 0x53, - 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x2b, 0x0a, 0x0c, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x50, - 0x61, 0x74, 0x68, 0x22, 0x7d, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, - 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x2e, 0x0a, - 0x09, 0x76, 0x64, 0x64, 0x6b, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x56, 0x44, 0x44, 0x4b, 0x43, 0x6f, 0x6e, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x76, 0x64, 0x64, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x33, 0x0a, - 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, - 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x49, 0x6e, - 0x66, 0x6f, 0x22, 0x43, 0x0a, 0x17, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x28, 0x0a, - 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, - 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x22, 0x67, 0x0a, 0x0b, 0x42, 0x6f, 0x6f, 0x74, 0x44, - 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, - 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x42, 0x6f, 0x6f, 0x74, 0x4f, - 0x72, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x65, 0x76, 0x54, 0x79, 0x70, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x44, 0x65, 0x76, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, - 0x0a, 0x0b, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0b, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x4f, 0x72, 0x64, 0x65, 0x72, - 0x22, 0x60, 0x0a, 0x13, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, - 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x4f, 0x76, 0x6d, 0x66, 0x56, - 0x61, 0x72, 0x73, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x4f, - 0x76, 0x6d, 0x66, 0x56, 0x61, 0x72, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x04, 0x64, - 0x65, 0x76, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x73, - 0x2e, 0x42, 0x6f, 0x6f, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x04, 0x64, 0x65, - 0x76, 0x73, 0x32, 0x82, 0x04, 0x0a, 0x0b, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x47, 0x75, 0x65, 0x73, - 0x74, 0x46, 0x73, 0x12, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, - 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x47, 0x75, 0x65, 0x73, 0x74, 0x46, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x46, 0x73, - 0x12, 0x14, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x46, 0x73, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x12, 0x2d, 0x0a, 0x08, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, 0x73, 0x12, - 0x14, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, 0x73, 0x50, - 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x44, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x54, 0x6f, 0x47, 0x6c, 0x61, 0x6e, - 0x63, 0x65, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x54, 0x6f, - 0x47, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1a, 0x2e, 0x61, - 0x70, 0x69, 0x73, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x54, 0x6f, 0x47, 0x6c, 0x61, 0x6e, 0x63, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x62, - 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1a, 0x2e, 0x61, 0x70, 0x69, - 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x50, 0x72, 0x61, 0x6d, 0x61, 0x73, 0x1a, 0x0f, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x49, 0x6d, - 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4f, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x12, 0x1c, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, - 0x73, 0x6b, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x73, - 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x41, 0x0a, 0x13, 0x44, 0x69, 0x73, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x12, - 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x0b, - 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3a, 0x0a, 0x10, 0x53, - 0x65, 0x74, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, - 0x19, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, 0x74, 0x4f, - 0x72, 0x64, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, - 0x73, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x34, 0x5a, 0x32, 0x79, 0x75, 0x6e, 0x69, 0x6f, - 0x6e, 0x2e, 0x69, 0x6f, 0x2f, 0x78, 0x2f, 0x6f, 0x6e, 0x65, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x68, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x2f, 0x68, 0x6f, 0x73, 0x74, - 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x40, 0x0a, 0x1d, 0x69, 0x73, 0x5f, 0x77, 0x69, 0x6e, + 0x64, 0x6f, 0x77, 0x73, 0x5f, 0x76, 0x69, 0x72, 0x74, 0x69, 0x6f, 0x5f, 0x6e, 0x65, 0x74, 0x5f, + 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x69, + 0x73, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x56, 0x69, 0x72, 0x74, 0x69, 0x6f, 0x4e, 0x65, + 0x74, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x2b, 0x0a, 0x0c, 0x45, 0x73, 0x78, 0x69, + 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, + 0x6b, 0x50, 0x61, 0x74, 0x68, 0x22, 0x7d, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, + 0x2e, 0x0a, 0x09, 0x76, 0x64, 0x64, 0x6b, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x56, 0x44, 0x44, 0x4b, 0x43, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x76, 0x64, 0x64, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x33, 0x0a, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, + 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x43, 0x0a, 0x17, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, + 0x73, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x28, 0x0a, 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x22, 0x67, 0x0a, 0x0b, 0x42, 0x6f, 0x6f, + 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x42, 0x6f, 0x6f, 0x74, + 0x4f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x42, 0x6f, 0x6f, + 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x65, 0x76, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x44, 0x65, 0x76, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x22, 0x60, 0x0a, 0x13, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, + 0x64, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x4f, 0x76, 0x6d, + 0x66, 0x56, 0x61, 0x72, 0x73, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x4f, 0x76, 0x6d, 0x66, 0x56, 0x61, 0x72, 0x73, 0x50, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, + 0x04, 0x64, 0x65, 0x76, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x61, 0x70, + 0x69, 0x73, 0x2e, 0x42, 0x6f, 0x6f, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x04, + 0x64, 0x65, 0x76, 0x73, 0x32, 0x82, 0x04, 0x0a, 0x0b, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x12, 0x40, 0x0a, 0x0d, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x47, 0x75, + 0x65, 0x73, 0x74, 0x46, 0x73, 0x12, 0x12, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x73, + 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x47, 0x75, 0x65, 0x73, 0x74, 0x46, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, + 0x46, 0x73, 0x12, 0x14, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, + 0x46, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x2d, 0x0a, 0x08, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, + 0x73, 0x12, 0x14, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x46, + 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x44, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x54, 0x6f, 0x47, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x53, 0x61, 0x76, 0x65, + 0x54, 0x6f, 0x47, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1a, + 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x53, 0x61, 0x76, 0x65, 0x54, 0x6f, 0x47, 0x6c, 0x61, 0x6e, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0e, 0x50, 0x72, + 0x6f, 0x62, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1a, 0x2e, 0x61, + 0x70, 0x69, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, + 0x66, 0x6f, 0x50, 0x72, 0x61, 0x6d, 0x61, 0x73, 0x1a, 0x0f, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4f, 0x0a, 0x10, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x12, 0x1c, 0x2e, + 0x61, 0x70, 0x69, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, + 0x44, 0x69, 0x73, 0x6b, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x1d, 0x2e, 0x61, 0x70, + 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, 0x73, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x41, 0x0a, 0x13, 0x44, 0x69, + 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, 0x6b, + 0x73, 0x12, 0x1d, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x73, 0x78, 0x69, 0x44, 0x69, 0x73, + 0x6b, 0x73, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, + 0x1a, 0x0b, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3a, 0x0a, + 0x10, 0x53, 0x65, 0x74, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, + 0x72, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x4f, 0x76, 0x6d, 0x66, 0x42, 0x6f, 0x6f, + 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x1a, 0x0b, 0x2e, 0x61, + 0x70, 0x69, 0x73, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x34, 0x5a, 0x32, 0x79, 0x75, 0x6e, + 0x69, 0x6f, 0x6e, 0x2e, 0x69, 0x6f, 0x2f, 0x78, 0x2f, 0x6f, 0x6e, 0x65, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x68, 0x6f, 0x73, 0x74, 0x6d, 0x61, 0x6e, 0x2f, 0x68, 0x6f, + 0x73, 0x74, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.proto b/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.proto index 4c7a73239..f20c3a37e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.proto +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis/deploy.proto @@ -203,6 +203,7 @@ message ImageInfo { string physical_partition_type = 6; bool is_installed_cloud_init = 7; bool is_bios_support = 8; + bool is_windows_virtio_net_support = 9; } message EsxiDiskInfo { diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge/hostbridge.go b/vendor/yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge/hostbridge.go index 7d1157671..747064699 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge/hostbridge.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge/hostbridge.go @@ -405,7 +405,7 @@ func (d *SBaseBridgeDriver) ConfirmToConfig() (bool, string, error) { return false, "", fmt.Errorf("bridge %s (%s) should have no ipv4 address", d.bridge, d.bridge.Addr) } if !d.bridge.IsSecretInterface() { - return false, "", fmt.Errorf("%s should have address in 169.254.0.0/16", d.bridge) + return false, fmt.Sprintf("bridge %s no ip and should have address in 169.254.0.0/16", d.bridge), nil } } if len(d.ip6) > 0 { @@ -477,7 +477,9 @@ func (d *SBaseBridgeDriver) SetupAddresses() error { if d.inter != nil { // first shutdown the origin interface ifname := d.inter.String() - tryUnmanageInterface(ifname) + if len(d.inter.Addr) > 0 || len(d.inter.Addr6) > 0 { + tryUnmanageInterface(ifname) + } if err := d.inter.FlushAddrs(); err != nil { return errors.Wrapf(err, "bridge %s slave ifname: %s flush addrs fail", br, ifname) } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/options/options.go b/vendor/yunion.io/x/onecloud/pkg/hostman/options/options.go index 1f4592ba3..541e28027 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/options/options.go @@ -42,6 +42,7 @@ type SHostBaseOptions struct { DefaultQemuVersion string `help:"Default qemu version" default:"10.0.7"` NoHpet bool `help:"Disable qemu hpet timer" default:"true"` + QgaStopTimeout int64 `default:"30" help:"Qemu guest agent stop timeout"` CdromCount int `help:"cdrom count" default:"1"` FloppyCount int `help:"floppy count" default:"1"` @@ -220,14 +221,15 @@ type SHostOptions struct { DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"` - DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` - DisableCustomDevice bool `help:"force disable custom pci device detect" default:"false" json:"disable_custom_device"` - DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` - SRIOVNics []string `help:"nics enable sriov" json:"sriov_nics"` - OvsOffloadNics []string `help:"nics enable ovs offload" json:"ovs_offload_nics"` - PTNVMEConfigs []string `help:"passthrough nvme disk pci address and size"` - AMDVgpuPFs []string `help:"amd vgpu pf pci addresses"` - NVIDIAVgpuPFs []string `help:"nvidia vgpu pf pci addresses"` + DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` + DisableCustomDevice bool `help:"force disable custom pci device detect" default:"false" json:"disable_custom_device"` + DisableUSB bool `help:"force disable USB detect" default:"true" json:"disable_usb"` + DisablePassthroughWithVendorDeviceId bool `help:"disable usb passthrough with vendor device id" default:"false" json:"disable_passthrough_with_vendor_device_id"` + SRIOVNics []string `help:"nics enable sriov" json:"sriov_nics"` + OvsOffloadNics []string `help:"nics enable ovs offload" json:"ovs_offload_nics"` + PTNVMEConfigs []string `help:"passthrough nvme disk pci address and size"` + AMDVgpuPFs []string `help:"amd vgpu pf pci addresses"` + NVIDIAVgpuPFs []string `help:"nvidia vgpu pf pci addresses"` EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"true" json:"ethtool_enable_gso"` @@ -269,9 +271,35 @@ type SHostOptions struct { CudaMPSLogDirectory string `help:"cuda mps log dir" default:"/tmp/nvidia-mps/log"` CudaMPSReplicas int `help:"cuda mps replicas" default:"10"` + EnableCudaHAMI bool `help:"enable cuda hami" default:"true"` + HAMICoreLibvgpuPath string `help:"hami core libvgpu.so path" default:"/opt/cloud/hami/libvgpu.so"` + SkipCheckKernelMods []string `help:"skip check kernel modules"` - EnableContainerAscendNPU bool `help:"enable container npu" default:"false"` + EnableContainerAscendNPU bool `help:"enable container npu" default:"false"` + EnableContainerAscendNPUHami bool `help:"enable container npu hami" default:"true"` + AscendNpuHamiShmPath string `help:"ascend npu hami shm path" default:"/opt/cloud/hami-shared-region"` + AscendNpuHamiLibvnpuPath string `help:"ascend npu hami libvnpu.so path" default:"/opt/cloud/hami/libvnpu.so"` + + EnableContainerHygonDCU bool `help:"enable container hygon dcu" default:"true"` + EnableContainerHygonDCUHami bool `help:"enable container hygon dcu hami" default:"false"` + HygonHyhalPath string `help:"hygon hyhal driver path" default:"/opt/hyhal"` + HygonDtkPath string `help:"hygon dtk toolkit path" default:"/opt/dtk"` + HygonHySmiPath string `help:"hygon hy-smi path" default:"/opt/hyhal/bin/hy-smi"` + HygonVdevConfDir string `help:"hygon vdcu config directory" default:"/etc/vdev"` + HygonVgpuCacheDir string `help:"hygon vgpu vdev cache directory" default:"/usr/local/vgpu/dcu"` + + EnableContainerIluvatarGPU bool `help:"enable container iluvatar gpu" default:"true"` + IluvatarCorexHome string `help:"iluvatar corex home" default:"/usr/local/corex-4.4.0"` + IluvatarIxsmiPath string `help:"iluvatar ixsmi path" default:"/usr/local/corex-4.4.0/bin/ixsmi"` + + EnableContainerTHeadPPU bool `help:"enable container t-head ppu" default:"true"` + THeadPpuSdkHome string `help:"t-head ppu sdk home" default:"/usr/local/PPU_SDK"` + THeadPpuSmiPath string `help:"t-head ppu-smi path" default:"/usr/local/bin/ppu-smi"` + + EnableContainerKunlunxinXPU bool `help:"enable container kunlunxin xpu" default:"true"` + KunlunxinXreHome string `help:"kunlunxin xre home" default:"/usr/local/xpu"` + KunlunxinXpuSmiPath string `help:"kunlunxin xpu-smi path" default:"/usr/local/bin/xpu-smi"` EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/backup_storage.go b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/backup_storage.go index 6a25360d3..275579970 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/backup_storage.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/backup_storage.go @@ -20,23 +20,24 @@ import ( "os" "sync" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis/compute" ) type IBackupStorageFactory interface { - NewBackupStore(storeId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) + NewBackupStore(storeId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) } type IBackupStorage interface { // 从指定路径拷贝磁盘文件到备份存储 - SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupId string) error + SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupId string, backupFilePath string) error // 将备份backupId对应的备份文件拷贝到指定的文件路径 - RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error + RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error // 删除备份 - RemoveBackup(ctx context.Context, backupId string) error + RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error // 备份是否存在 - IsBackupExists(backupId string) (bool, string, error) + IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) // 从指定路径拷贝主机备份文件到备份存储 SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupInstanceId string) error @@ -45,7 +46,7 @@ type IBackupStorage interface { // 删除备份 RemoveBackupInstance(ctx context.Context, backupInstanceId string) error // 备份是否存在 - IsBackupInstanceExists(backupInstanceId string) (bool, string, error) + IsBackupInstanceExists(bakcupInstanceFilePath string) (bool, int64, string, error) // ConvertTo(destPath string, format qemuimgfmt.TImageFormat, backupId string) error // ConvertFrom(srcPath string, format qemuimgfmt.TImageFormat, backupId string) (int, error) @@ -56,7 +57,7 @@ type IBackupStorage interface { IsOnline() (bool, string, error) // 获取外部访问地址 - GetExternalAccessUrl(backupId string) (string, error) + GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) } var factories []IBackupStorageFactory @@ -72,7 +73,7 @@ func RegisterFactory(factory IBackupStorageFactory) { factories = append(factories, factory) } -func newBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) { +func newBackupStorage(backupStroageId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) { errs := make([]error, 0) for _, factory := range factories { store, err := factory.NewBackupStore(backupStroageId, backupStorageAccessInfo) @@ -85,7 +86,7 @@ func newBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils return nil, errors.NewAggregate(errs) } -func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) { +func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) { backupStorageLock.Lock() defer backupStorageLock.Unlock() @@ -101,7 +102,7 @@ func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils } } -func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string, storage IBackupStorage) error { +func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string, backupFilePath string, storage IBackupStorage) error { fileInfo, err := os.Stat(srcFilename) if err != nil { return errors.Wrapf(err, "stat %s", srcFilename) @@ -112,10 +113,10 @@ func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string } defer file.Close() - return storage.SaveBackupFrom(ctx, file, fileInfo.Size(), bakcupId) + return storage.SaveBackupFrom(ctx, file, fileInfo.Size(), bakcupId, backupFilePath) } -func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupInstanceId string, storage IBackupStorage) error { +func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupInstanceFilePath string, storage IBackupStorage) error { fileInfo, err := os.Stat(srcFilename) if err != nil { return errors.Wrapf(err, "stat %s", srcFilename) @@ -125,5 +126,5 @@ func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupI return errors.Wrapf(err, "Open %s", srcFilename) } defer file.Close() - return storage.SaveBackupInstanceFrom(ctx, file, fileInfo.Size(), bakcupInstanceId) + return storage.SaveBackupInstanceFrom(ctx, file, fileInfo.Size(), bakcupInstanceFilePath) } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/factory.go b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/factory.go index 9484f0296..a4a52098a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/factory.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/factory.go @@ -15,7 +15,6 @@ package nfs import ( - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -25,12 +24,7 @@ import ( type sNfsBackupStorageFactory struct{} -func (factory *sNfsBackupStorageFactory) NewBackupStore(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (backupstorage.IBackupStorage, error) { - accessInfo := api.SBackupStorageAccessInfo{} - err := backupStorageAccessInfo.Unmarshal(&accessInfo) - if err != nil { - return nil, errors.Wrap(err, "Unmarshal access info") - } +func (factory *sNfsBackupStorageFactory) NewBackupStore(backupStroageId string, accessInfo *api.SBackupStorageAccessInfo) (backupstorage.IBackupStorage, error) { if len(accessInfo.NfsHost) == 0 { return nil, errors.Wrap(httperrors.ErrInputParameter, "need nfs_host in backup_storage_access_info") } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/nfs.go b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/nfs.go index 62c6ec3e6..ff911e9ef 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/nfs.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs/nfs.go @@ -57,7 +57,10 @@ func (s *SNFSBackupStorage) getBackupDir() string { return path.Join(s.Path, "backups") } -func (s *SNFSBackupStorage) getBackupDiskPath(backupId string) string { +func (s *SNFSBackupStorage) getBackupDiskPath(backupId string, backupFilePath string) string { + if len(backupFilePath) > 0 { + return backupFilePath + } return path.Join(s.getBackupDir(), backupId) } @@ -65,7 +68,7 @@ func (s *SNFSBackupStorage) getPackageDir() string { return path.Join(s.Path, "backuppacks") } -func (s *SNFSBackupStorage) getBackupInstancePath(backupInstanceId string) string { +func (s *SNFSBackupStorage) getBackupInstancePath(backupInstanceId string, backupFilePath string) string { return path.Join(s.getPackageDir(), backupInstanceId) } @@ -124,22 +127,22 @@ func (s *SNFSBackupStorage) unMount() error { return nil } -func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string, backupFilePath string) error { + return s.saveFile(ctx, srcFile, fileSize, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath) + return s.saveFile(ctx, srcFile, fileSize, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getPathFunc func(string) string) error { +func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fileSize int64, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - targetFilename := getPathFunc(id) + targetFilename := getPathFunc(id, backupFilePath) targetFile, err := os.Create(targetFilename) if err != nil { @@ -154,22 +157,22 @@ func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fil return nil } -func (s *SNFSBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreFile(ctx, targetFilename, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error { + return s.restoreFile(ctx, targetFilename, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) RestoreBackupInstanceTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreFile(ctx, targetFilename, backupId, s.getBackupInstancePath) + return s.restoreFile(ctx, targetFilename, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename string, id string, getPathFunc func(string) string) error { +func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename string, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - srcFilename := getPathFunc(id) + srcFilename := getPathFunc(id, backupFilePath) if output, err := procutils.NewCommand("cp", srcFilename, targetFilename).Output(); err != nil { log.Errorf("unable to cp %s to %s: %s", srcFilename, targetFilename, output) return errors.Wrapf(err, "cp %s to %s failed and output is %q", srcFilename, targetFilename, output) @@ -177,22 +180,22 @@ func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename stri return nil } -func (s *SNFSBackupStorage) RemoveBackup(ctx context.Context, backupId string) error { - return s.removeFile(ctx, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error { + return s.removeFile(ctx, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) RemoveBackupInstance(ctx context.Context, backupId string) error { - return s.removeFile(ctx, backupId, s.getBackupInstancePath) + return s.removeFile(ctx, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, getPathFunc func(id string) string) error { +func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - filename := getPathFunc(id) + filename := getPathFunc(id, backupFilePath) if !fileutils2.Exists(filename) { return nil } @@ -203,26 +206,34 @@ func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, getPathFu return nil } -func (s *SNFSBackupStorage) IsBackupExists(backupId string) (bool, string, error) { - return s.isFileExists(backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) { + return s.isFileExists(backupId, s.getBackupDiskPath, backupFilePath) } -func (s *SNFSBackupStorage) IsBackupInstanceExists(backupId string) (bool, string, error) { - return s.isFileExists(backupId, s.getBackupInstancePath) +func (s *SNFSBackupStorage) IsBackupInstanceExists(backupId string) (bool, int64, string, error) { + return s.isFileExists(backupId, s.getBackupInstancePath, "") } -func (s *SNFSBackupStorage) isFileExists(id string, getPathFunc func(id string) string) (bool, string, error) { +func (s *SNFSBackupStorage) isFileExists(id string, getPathFunc func(string, string) string, backupFilePath string) (bool, int64, string, error) { err := s.checkAndMount() if err != nil { if errors.Cause(err) == ErrorBackupStorageOffline { - return false, err.Error(), nil + return false, -1, err.Error(), nil } - return false, "", errors.Wrap(err, "unable to checkAndMount") + return false, -1, "", errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - filename := getPathFunc(id) - return fileutils2.Exists(filename), "", nil + filename := getPathFunc(id, backupFilePath) + exists := fileutils2.Exists(filename) + if !exists { + return false, -1, "", nil + } + stat, err := os.Stat(filename) + if err != nil { + return false, -1, "", errors.Wrap(err, "os.Stat") + } + return true, stat.Size(), "", nil } func (s *SNFSBackupStorage) IsOnline() (bool, string, error) { @@ -237,6 +248,6 @@ func (s *SNFSBackupStorage) IsOnline() (bool, string, error) { return true, "", nil } -func (s *SNFSBackupStorage) GetExternalAccessUrl(backupId string) (string, error) { +func (s *SNFSBackupStorage) GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) { return "", errors.ErrNotSupported } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/factory.go b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/factory.go index 45c5ef6c4..2a557b31b 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/factory.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/factory.go @@ -16,7 +16,6 @@ package object import ( "yunion.io/x/cloudmux/pkg/multicloud/objectstore" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -26,12 +25,7 @@ import ( type sObjectBackupStorageFactory struct{} -func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (backupstorage.IBackupStorage, error) { - accessInfo := api.SBackupStorageAccessInfo{} - err := backupStorageAccessInfo.Unmarshal(&accessInfo) - if err != nil { - return nil, errors.Wrap(err, "Unmarshal access info") - } +func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId string, accessInfo *api.SBackupStorageAccessInfo) (backupstorage.IBackupStorage, error) { if len(accessInfo.ObjectBucketUrl) == 0 { return nil, errors.Wrap(httperrors.ErrInputParameter, "need object_bucket_url in backup_storage_access_info") } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/object.go b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/object.go index 05bbcda80..64b63cb7d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/object.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object/object.go @@ -109,18 +109,21 @@ func parseBucketUrl(bucketUrl string) (string, string, error) { const backupPathPrefix = "backups" const backupInstancePathPrefix = "backuppacks" -func (s *SObjectBackupStorage) getBackupKey(backupId string) string { +func (s *SObjectBackupStorage) getBackupKey(backupId string, backupFilePath string) string { + if len(backupFilePath) > 0 { + return backupFilePath + } return fmt.Sprintf("%s/%s", backupPathPrefix, backupId) } -func (s *SObjectBackupStorage) getBackupInstanceKey(backupInstancePackName string) string { +func (s *SObjectBackupStorage) getBackupInstanceKey(backupInstancePackName string, _ string) string { return fmt.Sprintf("%s/%s", backupInstancePathPrefix, backupInstancePackName) } func (s *SObjectBackupStorage) getBucket() (cloudprovider.ICloudBucket, error) { bucket, err := s.store.GetIRegion().GetIBucketByName(s.bucket) if err != nil { - return nil, errors.Wrap(err, "IBucketExist") + return nil, errors.Wrapf(err, "IBucketExist %s", s.bucket) } return bucket, nil } @@ -136,42 +139,42 @@ func (s *SObjectBackupStorage) getExtBucket() (cloudprovider.ICloudBucket, error return bucket, nil } -func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string, backupFilePath string) error { + return s.saveObject(ctx, srcFile, fileSize, backupId, backupFilePath, s.getBackupKey) } -func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupInstanceKey) +func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupInstanceFilePath string) error { + return s.saveObject(ctx, srcFile, fileSize, bakcupInstanceFilePath, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFile io.Reader, fileSize int64, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id), 200*1024*1024, srcFile, fileSize, cloudprovider.ACLPrivate, "", nil, false) + err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id, backupFilePath), 200*1024*1024, srcFile, fileSize, cloudprovider.ACLPrivate, "", nil, false) if err != nil { - return errors.Wrapf(err, "UploadObject %d %s", fileSize, getKeyFunc(id)) + return errors.Wrapf(err, "UploadObject %d %s", fileSize, getKeyFunc(id, backupFilePath)) } return nil } -func (s *SObjectBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreObject(ctx, targetFilename, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error { + return s.restoreObject(ctx, targetFilename, backupId, backupFilePath, s.getBackupKey) } func (s *SObjectBackupStorage) RestoreBackupInstanceTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreObject(ctx, targetFilename, backupId, s.getBackupInstanceKey) + return s.restoreObject(ctx, targetFilename, backupId, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename string, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename string, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - reader, err := bucket.GetObject(ctx, getKeyFunc(id), nil) + reader, err := bucket.GetObject(ctx, getKeyFunc(id, backupFilePath), nil) if err != nil { return errors.Wrap(err, "GetObject") } @@ -187,47 +190,47 @@ func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename return nil } -func (s *SObjectBackupStorage) RemoveBackup(ctx context.Context, backupId string) error { - return s.removeObject(ctx, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error { + return s.removeObject(ctx, backupId, backupFilePath, s.getBackupKey) } func (s *SObjectBackupStorage) RemoveBackupInstance(ctx context.Context, backupId string) error { - return s.removeObject(ctx, backupId, s.getBackupInstanceKey) + return s.removeObject(ctx, backupId, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) removeObject(ctx context.Context, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) removeObject(ctx context.Context, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - err = bucket.DeleteObject(ctx, getKeyFunc(id)) + err = bucket.DeleteObject(ctx, getKeyFunc(id, backupFilePath)) if err != nil { return errors.Wrap(err, "DeleteObject") } return nil } -func (s *SObjectBackupStorage) IsBackupExists(backupId string) (bool, string, error) { - return s.isObjectExists(backupId, s.getBackupKey) +func (s *SObjectBackupStorage) IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) { + return s.isObjectExists(backupId, s.getBackupKey, backupFilePath) } -func (s *SObjectBackupStorage) IsBackupInstanceExists(backupId string) (bool, string, error) { - return s.isObjectExists(backupId, s.getBackupInstanceKey) +func (s *SObjectBackupStorage) IsBackupInstanceExists(backupId string) (bool, int64, string, error) { + return s.isObjectExists(backupId, s.getBackupInstanceKey, "") } -func (s *SObjectBackupStorage) isObjectExists(id string, getKeyFunc func(string) string) (bool, string, error) { +func (s *SObjectBackupStorage) isObjectExists(id string, getKeyFunc func(string, string) string, backupFilePath string) (bool, int64, string, error) { bucket, err := s.getBucket() if err != nil { - return false, "", errors.Wrap(err, "getBucket") + return false, -1, "", errors.Wrap(err, "getBucket") } - _, err = cloudprovider.GetIObject(bucket, getKeyFunc(id)) + obj, err := cloudprovider.GetIObject(bucket, getKeyFunc(id, backupFilePath)) if err != nil { if errors.Cause(err) == errors.ErrNotFound { - return false, "", nil + return false, -1, "", nil } - return false, "", errors.Wrap(err, "GetIObject") + return false, -1, "", errors.Wrap(err, "GetIObject") } - return true, "", nil + return true, obj.GetSizeBytes(), "", nil } func (s *SObjectBackupStorage) IsOnline() (bool, string, error) { @@ -238,7 +241,7 @@ func (s *SObjectBackupStorage) IsOnline() (bool, string, error) { return exist, "", nil } -func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string) (string, error) { +func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) { var bucket cloudprovider.ICloudBucket var err error bucket, err = s.getExtBucket() @@ -249,7 +252,7 @@ func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string) (string, er return "", errors.Wrap(err, "getBucket") } } - url, err := bucket.GetTempUrl(http.MethodGet, s.getBackupKey(backupId), 6*time.Hour) + url, err := bucket.GetTempUrl(http.MethodGet, s.getBackupKey(backupId, backupFilePath), 6*time.Hour) if err != nil { return "", errors.Wrap(err, "GetTempUrl") } diff --git a/vendor/yunion.io/x/onecloud/pkg/hostman/system_service/telegraf.go b/vendor/yunion.io/x/onecloud/pkg/hostman/system_service/telegraf.go index 568e0f999..afc09368a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/hostman/system_service/telegraf.go +++ b/vendor/yunion.io/x/onecloud/pkg/hostman/system_service/telegraf.go @@ -37,7 +37,13 @@ const ( TELEGRAF_INPUT_CONF_BIN_PATH = "bin_path" TELEGRAF_INPUT_NETDEV = "ni_rsrc_mon" TELEGRAF_INPUT_VASMI = "vasmi" + TELEGRAF_INPUT_HYSMI = "hysmi" + TELEGRAF_INPUT_IXSMI = "ixsmi" + TELEGRAF_INPUT_PPUSMI = "ppusmi" + TELEGRAF_INPUT_XPUSMI = "xpusmi" TELEGRAF_INPUT_NVIDIASMI = "nvidia-smi" + TELEGRAF_INPUT_NPUSMI = "npu-smi" + TELEGRAF_INPUT_CONF_LIB_PATH = "lib_path" ) type STelegraf struct { @@ -337,11 +343,55 @@ func (s *STelegraf) GetConfig(kwargs map[string]interface{}) string { conf += "\n" } + if hysmi, ok := kwargs[TELEGRAF_INPUT_HYSMI]; ok { + hysmiMap, _ := hysmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_HYSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", hysmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + conf += "\n" + } + + if ixsmi, ok := kwargs[TELEGRAF_INPUT_IXSMI]; ok { + ixsmiMap, _ := ixsmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_IXSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", ixsmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + if libPath, _ := ixsmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" { + conf += fmt.Sprintf(" lib_path = \"%s\"\n", libPath) + } + conf += "\n" + } + + if ppusmi, ok := kwargs[TELEGRAF_INPUT_PPUSMI]; ok { + ppusmiMap, _ := ppusmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_PPUSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", ppusmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + if libPath, _ := ppusmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" { + conf += fmt.Sprintf(" lib_path = \"%s\"\n", libPath) + } + conf += "\n" + } + + if xpusmi, ok := kwargs[TELEGRAF_INPUT_XPUSMI]; ok { + xpusmiMap, _ := xpusmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.%s]]\n", TELEGRAF_INPUT_XPUSMI) + conf += fmt.Sprintf(" bin_path = \"%s\"\n", xpusmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + if libPath, _ := xpusmiMap[TELEGRAF_INPUT_CONF_LIB_PATH].(string); libPath != "" { + conf += fmt.Sprintf(" lib_path = \"%s\"\n", libPath) + } + conf += "\n" + } + if _, ok := kwargs[TELEGRAF_INPUT_NVIDIASMI]; ok { conf += "[[inputs.nvidia_smi]]\n" conf += "\n" } + if npusmi, ok := kwargs[TELEGRAF_INPUT_NPUSMI]; ok { + npusmiMap, _ := npusmi.(map[string]interface{}) + conf += fmt.Sprintf("[[inputs.npu_smi]]\n") + conf += fmt.Sprintf(" bin_path = \"%s\"\n", npusmiMap[TELEGRAF_INPUT_CONF_BIN_PATH].(string)) + conf += "\n" + } + // 检查 IPMI 设备文件是否存在 if hasIPMIDevice() { conf += "[[inputs.ipmi_sensor]]\n" diff --git a/vendor/yunion.io/x/onecloud/pkg/httperrors/general.go b/vendor/yunion.io/x/onecloud/pkg/httperrors/general.go index f7e2de69d..9bb84f078 100644 --- a/vendor/yunion.io/x/onecloud/pkg/httperrors/general.go +++ b/vendor/yunion.io/x/onecloud/pkg/httperrors/general.go @@ -15,6 +15,8 @@ package httperrors import ( + "database/sql" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/httputils" ) @@ -31,6 +33,9 @@ func NewGeneralError(err error) *httputils.JSONClientError { return httputils.NewJsonClientError(code, string(nerr), "%s", err.Error()) default: root := errors.Cause(err) + if root == sql.ErrNoRows { + return NewNotFoundError("%s", err.Error()) + } switch nerr := root.(type) { case *httputils.JSONClientError: nerr.Details = err.Error() diff --git a/vendor/yunion.io/x/onecloud/pkg/image/options/options.go b/vendor/yunion.io/x/onecloud/pkg/image/options/options.go index 5c7bad6c6..12d2b549f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/image/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/image/options/options.go @@ -48,7 +48,7 @@ type SImageOptions struct { // DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"` - StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local"` + StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local|nfs"` S3MountPoint string `help:"s3fs mount point" default:"/opt/cloud/workspace/data/glance/s3images"` S3CheckImageStatus bool `help:"Enable s3 check image status"` @@ -58,6 +58,10 @@ type SImageOptions struct { S3DirectDownload bool `help:"enable s3 direct download" default:"false"` + NfsStorageId string `help:"region nfs storage id or name used as glance filesystem backend"` + NfsMountOptions string `help:"nfs mount options for glance filesystem backend"` + NfsMountPoint string `help:"nfs mount point" default:"/opt/cloud/workspace/data/glance/nfsimages"` + ImageStreamWorkerCount int `help:"Image stream worker count" default:"10"` VerifyImageStatusIntervalMinutes int `help:"verify image status periodically, default 15 minutes" default:"15"` diff --git a/vendor/yunion.io/x/onecloud/pkg/lbagent/gobetween/config.go b/vendor/yunion.io/x/onecloud/pkg/lbagent/gobetween/config.go index 4352d61af..62545da19 100644 --- a/vendor/yunion.io/x/onecloud/pkg/lbagent/gobetween/config.go +++ b/vendor/yunion.io/x/onecloud/pkg/lbagent/gobetween/config.go @@ -301,18 +301,21 @@ type HealthcheckConfig struct { *PingHealthcheckConfig *ExecHealthcheckConfig - *UdpHealthcheckConfig + *ProbeHealthcheckConfig } type PingHealthcheckConfig struct{} +type ProbeHealthcheckConfig struct { + ProbeProtocol string `toml:"probe_protocol" json:"probe_protocol"` + ProbeStrategy string `toml:"probe_strategy" json:"probe_strategy"` + ProbeSend string `toml:"probe_send" json:"probe_send"` + ProbeRecv string `toml:"probe_recv" json:"probe_recv"` + ProbeRecvLen int `toml:"probe_recv_len" json:"probe_recv_len"` +} + type ExecHealthcheckConfig struct { ExecCommand string `toml:"exec_command" json:"exec_command,omitempty"` ExecExpectedPositiveOutput string `toml:"exec_expected_positive_output" json:"exec_expected_positive_output"` ExecExpectedNegativeOutput string `toml:"exec_expected_negative_output" json:"exec_expected_negative_output"` } - -type UdpHealthcheckConfig struct { - Receive string - Send string -} diff --git a/vendor/yunion.io/x/onecloud/pkg/lbagent/models/gobetween.go b/vendor/yunion.io/x/onecloud/pkg/lbagent/models/gobetween.go index c4a0b1b76..71b1a9343 100644 --- a/vendor/yunion.io/x/onecloud/pkg/lbagent/models/gobetween.go +++ b/vendor/yunion.io/x/onecloud/pkg/lbagent/models/gobetween.go @@ -94,16 +94,23 @@ func (b *LoadbalancerCorpus) GenGobetweenConfigs(dir string, opts *GenGobetweenC // healthcheck var serverHealthcheck *gobetween.HealthcheckConfig if listener.HealthCheck == "on" && listener.HealthCheckType == "udp" { - serverHealthcheck = &gobetween.HealthcheckConfig{ - Kind: "pingudp", - Interval: fmt.Sprintf("%ds", listener.HealthCheckInterval), - Timeout: fmt.Sprintf("%ds", listener.HealthCheckTimeout), - Passes: listener.HealthCheckRise, - Fails: listener.HealthCheckFall, - UdpHealthcheckConfig: &gobetween.UdpHealthcheckConfig{ - Send: listener.HealthCheckReq, - Receive: listener.HealthCheckExp, - }, + if listener.HealthCheckReq == "" || listener.HealthCheckExp == "" { + log.Warningf("listener %s(%s): skip UDP healthcheck with empty request or expected response", + listener.Name, listener.Id) + } else { + serverHealthcheck = &gobetween.HealthcheckConfig{ + Kind: "probe", + Interval: fmt.Sprintf("%ds", listener.HealthCheckInterval), + Timeout: fmt.Sprintf("%ds", listener.HealthCheckTimeout), + Passes: listener.HealthCheckRise, + Fails: listener.HealthCheckFall, + ProbeHealthcheckConfig: &gobetween.ProbeHealthcheckConfig{ + ProbeProtocol: "udp", + ProbeStrategy: "starts_with", + ProbeSend: listener.HealthCheckReq, + ProbeRecv: listener.HealthCheckExp, + }, + } } } diff --git a/vendor/yunion.io/x/onecloud/pkg/lbagent/models/modelset.go b/vendor/yunion.io/x/onecloud/pkg/lbagent/models/modelset.go index 1718a76c2..23d18cdba 100644 --- a/vendor/yunion.io/x/onecloud/pkg/lbagent/models/modelset.go +++ b/vendor/yunion.io/x/onecloud/pkg/lbagent/models/modelset.go @@ -59,6 +59,10 @@ func (set Networks) Copy() apihelper.IModelSet { return setCopy } +func (set Networks) IncludeDetails() bool { + return false +} + func (set LoadbalancerNetworks) ModelManager() modulebase.IBaseManager { return &modules.Loadbalancernetworks } @@ -82,6 +86,10 @@ func (set LoadbalancerNetworks) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerNetworks) IncludeDetails() bool { + return false +} + func (set LoadbalancerNetworks) JoinLoadbalancers(entries Loadbalancers) bool { for mKey, m := range set { lbId := m.LoadbalancerId @@ -150,6 +158,10 @@ func (set Loadbalancers) Copy() apihelper.IModelSet { return setCopy } +func (set Loadbalancers) IncludeDetails() bool { + return false +} + func (ms Loadbalancers) JoinListeners(subEntries LoadbalancerListeners) bool { for _, m := range ms { m.Listeners = LoadbalancerListeners{} @@ -225,6 +237,10 @@ func (set LoadbalancerListeners) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerListeners) IncludeDetails() bool { + return false +} + func (ms LoadbalancerListeners) JoinListenerRules(subEntries LoadbalancerListenerRules) bool { for _, m := range ms { m.rules = LoadbalancerListenerRules{} @@ -311,6 +327,10 @@ func (set LoadbalancerListenerRules) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerListenerRules) IncludeDetails() bool { + return false +} + type OrderedLoadbalancerListenerRuleList []*LoadbalancerListenerRule func (lst OrderedLoadbalancerListenerRuleList) Len() int { @@ -375,6 +395,10 @@ func (set LoadbalancerBackendGroups) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerBackendGroups) IncludeDetails() bool { + return false +} + func (ms LoadbalancerBackendGroups) JoinBackends(subEntries LoadbalancerBackends) bool { for _, m := range ms { m.Backends = LoadbalancerBackends{} @@ -425,6 +449,10 @@ func (set LoadbalancerBackends) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerBackends) IncludeDetails() bool { + return false +} + func (set LoadbalancerAcls) ModelManager() modulebase.IBaseManager { return &modules.LoadbalancerAcls } @@ -451,6 +479,10 @@ func (set LoadbalancerAcls) Copy() apihelper.IModelSet { return setCopy } +func (set LoadbalancerAcls) IncludeDetails() bool { + return false +} + func (set LoadbalancerCertificates) ModelManager() modulebase.IBaseManager { return &modules.LoadbalancerCertificates } @@ -476,3 +508,7 @@ func (set LoadbalancerCertificates) Copy() apihelper.IModelSet { } return setCopy } + +func (set LoadbalancerCertificates) IncludeDetails() bool { + return false +} diff --git a/vendor/yunion.io/x/onecloud/pkg/llm/options/option.go b/vendor/yunion.io/x/onecloud/pkg/llm/options/option.go index b87d9c256..500680c61 100644 --- a/vendor/yunion.io/x/onecloud/pkg/llm/options/option.go +++ b/vendor/yunion.io/x/onecloud/pkg/llm/options/option.go @@ -14,7 +14,11 @@ package options -import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" +import ( + "strings" + + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" +) type LLMOptions struct { common_options.CommonOptions @@ -29,12 +33,32 @@ type LLMOptions struct { ImportTaskWorkerCount int `help:"import task worker count" default:"8"` StartTaskWorkerCount int `help:"start task worker count" default:"128"` + LLMBenchmarkWorkDir string `help:"llm benchmark working directory" default:"/opt/cloud/workspace/llm/benchmarks"` + LLMBenchmarkDefaultImage string `help:"default GuideLLM benchmark image" default:"registry.cn-beijing.aliyuncs.com/cloudpods/guidellm:v0.7.0-amd64"` + LLMBenchmarkRunnerCPU int `help:"llm benchmark runner cpu" default:"1"` + LLMBenchmarkRunnerMemoryMB int `help:"llm benchmark runner memory MB" default:"2048"` + LLMBenchmarkDefaultRequestRate int `help:"default benchmark request rate" default:"1"` + LLMBenchmarkDefaultTotalRequests int `help:"default benchmark total requests" default:"100"` + LLMBenchmarkDefaultInputTokens int `help:"default synthetic prompt tokens" default:"1024"` + LLMBenchmarkDefaultOutputTokens int `help:"default synthetic output tokens" default:"128"` + LLMBenchmarkMaxDurationSeconds int `help:"max benchmark duration seconds" default:"3600"` + LLMBenchmarkMaxRequestRate int `help:"max benchmark request rate" default:"100"` + LLMBenchmarkMaxTotalRequests int `help:"max benchmark total requests" default:"100000"` + ArtifactS3Endpoint string `help:"MinIO/S3 endpoint for benchmark artifacts; empty disables upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"` + ArtifactS3AccessKey string `help:"MinIO/S3 access key for benchmark artifacts" default:"monitor-admin"` + ArtifactS3SecretKey string `help:"MinIO/S3 secret key for benchmark artifacts" default:"monitor-admin"` + ArtifactS3Bucket string `help:"MinIO/S3 bucket for benchmark artifacts" default:"llm-benchmark"` + ArtifactS3Secure bool `help:"Use HTTPS for benchmark artifact endpoint without scheme" default:"false"` + ArtifactS3Prefix string `help:"MinIO/S3 object key prefix for benchmark artifacts" default:"llm-benchmarks"` + // MCP Agent 配置 - MCPServerURL string `help:"MCP Server URL" default:"http://default-mcp-server:30876"` - MCPAgentTimeout int `help:"MCP Agent request timeout in seconds" default:"120"` + MCPServerURL string `help:"MCP Server URL" default:"http://default-mcp-server:30876"` + // MCPAgentTimeout 单次 MCP tools/call（含 server-create 等待）超时；须大于 mcp-server 的 ServerCreateWaitSeconds + MCPAgentTimeout int `help:"MCP Agent tools/call timeout in seconds (cover public-cloud server create wait)" default:"600"` MCPAgentUserCharLimit int `help:"MCP Agent user char limit" default:"3200"` MCPAgentAssistantCharLimit int `help:"MCP Agent assistant char limit" default:"6400"` + MCPAgentMaxToolRounds int `help:"Max MCP tool-call rounds per chat request" default:"16"` // LLM model catalog (browsable curated entries). Value can be either an // http(s) URL or a local file path; sources without an http:// or https:// @@ -55,3 +79,28 @@ type LLMOptions struct { var ( Options LLMOptions ) + +const DefaultPlatformName = "Cloudpods" + +// ResolvedPlatformName 返回配置中的平台展示名，空则回退 DefaultPlatformName。 +func ResolvedPlatformName() string { + name := strings.TrimSpace(Options.PlatformName) + if name == "" { + return DefaultPlatformName + } + return name +} + +func OnOptionsChange(oldO, newO interface{}) bool { + oldOpts := oldO.(*LLMOptions) + newOpts := newO.(*LLMOptions) + + changed := false + if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) { + changed = true + } + if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) { + changed = true + } + return changed +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/aksk.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/aksk.go index b5563deb8..73046544e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/aksk.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/aksk.go @@ -16,6 +16,7 @@ package auth import ( "net/http" + "time" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/cache" @@ -24,6 +25,41 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" ) +// maxSignatureSkew is the maximal clock skew accepted for a signed request, +// the same window AWS uses: requests signed earlier can not be replayed +// indefinitely. +const maxSignatureSkew = 15 * time.Minute + +// checkRequestFreshness rejects replayed signatures: a signed request is only +// accepted within 15 minutes of its signing time. +func checkRequestFreshness(req http.Request) error { + dateStr := req.Header.Get("x-amz-date") + if len(dateStr) > 0 { + signTime, err := time.Parse("20060102T150405Z", dateStr) + if err != nil { + return errors.Wrap(err, "invalid x-amz-date") + } + return checkTimeSkew(signTime) + } + dateStr = req.Header.Get("Date") + if len(dateStr) > 0 { + signTime, err := http.ParseTime(dateStr) + if err != nil { + return errors.Wrap(err, "invalid Date header") + } + return checkTimeSkew(signTime) + } + return errors.Error("missing signing date") +} + +func checkTimeSkew(signTime time.Time) error { + skew := time.Since(signTime) + if skew > maxSignatureSkew || skew < -maxSignatureSkew { + return errors.Errorf("request signature expired, signed at %s", signTime) + } + return nil +} + type sAccessKeyCache struct { *cache.LRUCache } @@ -64,6 +100,9 @@ func (c *sAccessKeyCache) Verify(cli *mcclient.Client, req http.Request, virtual if err != nil { return nil, errors.Wrap(err, "s3auth.DecodeAccessKeyRequestV2") } + if err := checkRequestFreshness(req); err != nil { + return nil, errors.Wrap(err, "checkRequestFreshness") + } token, found := c.getToken(aksk.GetAccessKey()) if found { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/auth.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/auth.go index 47fc18ffa..ac31faeac 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/auth.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/auth/auth.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/cache" "yunion.io/x/pkg/util/httputils" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/syncman" @@ -123,7 +124,7 @@ func (c *TokenCacheVerify) Verify(ctx context.Context, cli *mcclient.Client, adm return cred, nil } else { c.DeleteToken(token) - log.Infof("Remove expired cache token: %s", token) + log.Infof("Remove expired cache token: %s", utils.TruncateString(token, 16)) } } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modulebase/modules.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modulebase/modules.go index 2e72018c5..94772cd4d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modulebase/modules.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modulebase/modules.go @@ -294,7 +294,9 @@ func registerAllJointModules() { func _getModule(session *mcclient.ClientSession, name string) (IBaseManager, error) { mods, ok := modules[name] if !ok { - return nil, fmt.Errorf("No such module %s", name) + // unknown module means the request URL does not exist: a client + // error (404), not a server error + return nil, errors.Wrapf(errors.ErrNotFound, "No such module %s", name) } if len(mods) == 1 { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_ipsets.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_ipsets.go new file mode 100644 index 000000000..f5b7c6093 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_ipsets.go @@ -0,0 +1,32 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + IpSets modulebase.ResourceManager +) + +func init() { + IpSets = modules.NewComputeManager("ipset", "ipsets", + []string{"ID", "Name", "IP_Set_Type", "Data", "Status", "Tenant", "Description"}, + []string{}) + + modules.RegisterCompute(&IpSets) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_isolatedevices.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_isolatedevices.go index 3efc8e4a2..72b11185e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_isolatedevices.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_isolatedevices.go @@ -20,15 +20,27 @@ import ( ) var ( - IsolatedDevices modulebase.ResourceManager + IsolatedDevices modulebase.ResourceManager + ServerIsolatedDevices modulebase.JointResourceManager ) func init() { IsolatedDevices = modules.NewComputeManager("isolated_device", "isolated_devices", - []string{"ID", "Dev_type", - "Model", "Addr", "Vendor_device_id", "Mdev_id", + []string{"ID", "Dev_type", "Sharing_mode", + "Model", "Vendor", "Addr", "Vendor_device_id", "Mdev_id", "Host_id", "Host", "numa_node", "Guest_id", "Guest", "Guest_status", "Device_path", "Render_path", "PCIE_Info", "Index", "Device_minor"}, []string{}) modules.RegisterCompute(&IsolatedDevices) + + ServerIsolatedDevices = modules.NewJointComputeManager( + "guestisolateddevice", + "guestisolateddevices", + []string{"Guest_ID", "Guest", "Isolated_device_ID", "Index", "Dev_type", "Sharing_mode", "Vendor", + "Device_memory_size", "Sm_util_limit", "Network_index", "Disk_index"}, + []string{}, + &Servers, + &IsolatedDevices, + ) + modules.RegisterCompute(&ServerIsolatedDevices) } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_snapshots.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_snapshots.go index 72b3c2c1a..1a9ae7b86 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_snapshots.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/compute/mod_snapshots.go @@ -27,7 +27,7 @@ func init() { Snapshots = modules.NewComputeManager("snapshot", "snapshots", []string{"ID", "Name", "Size", "Status", "Disk_id", "Guest_id", "Created_at"}, - []string{"Storage_id", "Storage_type", "Create_by", "Location", "Out_of_chain", "disk_type", "provider"}) + []string{"Storage_id", "Storage_type", "Create_by", "Location", "disk_type", "provider"}) modules.RegisterCompute(&Snapshots) } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_image.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_image.go new file mode 100644 index 000000000..6108a229b --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_image.go @@ -0,0 +1,29 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ContainerImages modulebase.ResourceManager + +func init() { + ContainerImages = modules.NewImageManager("container_image", "container_images", + []string{"ID", "Name", "Image_Name", "Image_Label", "Registry_Id", "Status"}, + []string{}) + modules.Register(&ContainerImages) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_registry.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_registry.go new file mode 100644 index 000000000..a708a6b3f --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/image/mod_container_registry.go @@ -0,0 +1,111 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strconv" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/httputils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +var ( + ContainerRegistries *ContainerRegistryManager +) + +func init() { + ContainerRegistries = NewContainerRegistryManager() + modules.Register(ContainerRegistries) +} + +type ContainerRegistryManager struct { + modulebase.ResourceManager +} + +func NewContainerRegistryManager() *ContainerRegistryManager { + man := &ContainerRegistryManager{ + ResourceManager: modules.NewImageManager("container_registry", "container_registries", + []string{"ID", "Name", "Url", "Type", "Status"}, + []string{}), + } + // Force /container_registries//images to GetSpecific, otherwise apigateway + // treats "images" as the glance images module and does ListInContext. + man.SetSpecificMethods("images") + return man +} + +func (m *ContainerRegistryManager) UploadImage(s *mcclient.ClientSession, id string, params jsonutils.JSONObject, body io.Reader, size int64) (jsonutils.JSONObject, error) { + path := fmt.Sprintf("/%s/%s/upload-image", m.URLPath(), id) + headers := http.Header{} + headers.Add("Content-Type", "application/octet-stream") + if size > 0 { + headers.Add("Content-Length", fmt.Sprintf("%d", size)) + } + resp, err := modulebase.RawRequest(m.ResourceManager, s, httputils.POST, path, headers, body) + _, json, err := s.ParseJSONResponse("", resp, err) + if err != nil { + return nil, err + } + return json, nil +} + +type DownloadImageByManagerInput struct { + Insecure bool `json:"insecure"` + Image string `json:"image"` + Username string `json:"username"` + Password string `json:"password"` +} + +func (m *ContainerRegistryManager) DownloadImageByManager(s *mcclient.ClientSession, input *DownloadImageByManagerInput) (string, io.Reader, int64, error) { + path := fmt.Sprintf("/%s/download-image", m.URLPath()) + return m.downloadImage(s, path, input) +} + +func (m *ContainerRegistryManager) DownloadImage(s *mcclient.ClientSession, id string, imageName string, imageTag string) (string, io.Reader, int64, error) { + params := map[string]string{ + "image_name": imageName, + "tag": imageTag, + } + path := fmt.Sprintf("/%s/%s/download-image", m.URLPath(), url.PathEscape(id)) + return m.downloadImage(s, path, params) +} + +func (m *ContainerRegistryManager) downloadImage(s *mcclient.ClientSession, path string, params interface{}) (string, io.Reader, int64, error) { + query := jsonutils.Marshal(params) + queryString := query.QueryString() + if len(queryString) > 0 { + path = fmt.Sprintf("%s?%s", path, queryString) + } + resp, err := modulebase.RawRequest(m.ResourceManager, s, "GET", path, nil, nil) + if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 { + sizeBytes, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + log.Errorf("Download image unknown size") + sizeBytes = -1 + } + return resp.Header.Get("Image-Filename"), resp.Body, sizeBytes, nil + } + _, _, err = s.ParseJSONResponse("", resp, err) + return "", nil, -1, err +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/k8s/container_registry.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/k8s/container_registry.go index d9ce761da..c72b3b8d5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/k8s/container_registry.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/k8s/container_registry.go @@ -27,7 +27,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modulebase" - "yunion.io/x/onecloud/pkg/mcclient/modules" ) var ( @@ -35,8 +34,11 @@ var ( ) func init() { + // Do not modules.Register: container_registries is owned by glance (image service). + // Global Register would race with image.ContainerRegistries on resourceKey2 and + // can make apigateway proxy /api/v1/container_registries to kubeserver. + // Direct callers (climc k8s shell, import-from-kubeserver) still use this manager. ContainerRegistries = NewContainerRegistryManager() - modules.Register(ContainerRegistries) } type ContainerRegistryManager struct { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark.go new file mode 100644 index 000000000..aad7f2c6f --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark.go @@ -0,0 +1,60 @@ +package llm + +import ( + "fmt" + "io" + "net/url" + "strconv" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMBenchmarkManager struct { + modulebase.ResourceManager +} + +var ( + LLMBenchmarks LLMBenchmarkManager +) + +func (m *LLMBenchmarkManager) Artifact(s *mcclient.ClientSession, id string, kind string) (io.ReadCloser, int64, error) { + path := fmt.Sprintf("/%s/%s/artifacts/%s", m.URLPath(), url.PathEscape(id), url.PathEscape(kind)) + resp, err := modulebase.RawRequest(m.ResourceManager, s, "GET", path, nil, nil) + if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 { + sizeBytes, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + log.Errorf("download benchmark artifact unknown size") + sizeBytes = -1 + } + return resp.Body, sizeBytes, nil + } + _, _, err = s.ParseJSONResponse("", resp, err) + return nil, -1, err +} + +func init() { + LLMBenchmarks = LLMBenchmarkManager{ + ResourceManager: modules.NewLLMManager("llm_benchmark", "llm_benchmarks", + []string{ + "ID", + "Name", + "LLMDeploymentId", + "LLMDeployment", + "LLMId", + "State", + "RequestRate", + "TotalRequests", + "RequestTotal", + "RequestSuccessful", + "RequestErrored", + "ErrorRate", + }, + []string{}, + ), + } + modules.Register(&LLMBenchmarks) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go new file mode 100644 index 000000000..2d6077460 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_benchmark_package.go @@ -0,0 +1,35 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMBenchmarkPackageManager struct { + modulebase.ResourceManager +} + +var ( + LLMBenchmarkPackages LLMBenchmarkPackageManager +) + +func init() { + LLMBenchmarkPackages = LLMBenchmarkPackageManager{ + ResourceManager: modules.NewLLMManager("llm_benchmark_package", "llm_benchmark_packages", + []string{ + "ID", + "Name", + "Status", + "Source", + "RepoId", + "Revision", + "FilePath", + "Format", + "ImageId", + "DatasetPath", + }, + []string{}, + ), + } + modules.Register(&LLMBenchmarkPackages) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_router_agent.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_router_agent.go new file mode 100644 index 000000000..b5998990d --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/llm/mod_llm_router_agent.go @@ -0,0 +1,24 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMRouterAgentManager struct { + modulebase.ResourceManager +} + +var ( + LLMRouterAgent LLMRouterAgentManager +) + +func init() { + LLMRouterAgent = LLMRouterAgentManager{ + ResourceManager: modules.NewLLMManager("llm_router_agent", "llm_router_agents", + []string{}, + []string{}, + ), + } + modules.Register(&LLMRouterAgent) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/monitor/helper.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/monitor/helper.go index e3e6f729c..75c01b280 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/monitor/helper.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/monitor/helper.go @@ -639,6 +639,17 @@ func (input *MetricQueryInput) To(to time.Time) *MetricQueryInput { return input } +// FromRaw 传相对时间（如 720h）或 now，与控制台 unifiedmonitor 一致。 +func (input *MetricQueryInput) FromRaw(from string) *MetricQueryInput { + input.from = from + return input +} + +func (input *MetricQueryInput) ToRaw(to string) *MetricQueryInput { + input.to = to + return input +} + func (input *MetricQueryInput) Scope(scope string) *MetricQueryInput { input.scope = scope return input diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/scheduler/mod_scheduler.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/scheduler/mod_scheduler.go index a76b0f47e..9a988d328 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/scheduler/mod_scheduler.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/modules/scheduler/mod_scheduler.go @@ -165,7 +165,13 @@ func (this *SchedulerManager) CandidateList(s *mcclient.ClientSession, params js return } free, err := resObj.Int("free") + if err != nil { + return + } reserverd, err := resObj.Int("reserverd") + if err != nil { + return + } total, err := resObj.Int("total") if err != nil { return diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/base.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/base.go index 95fdf856b..8a8aed920 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/base.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/base.go @@ -198,27 +198,29 @@ const ( ListOrderDesc = "desc" ) +// BaseListOptions 通用列表参数。 +// 字段上的 mcp:"true" 表示该参数应对 AI/MCP 暴露（见 pkg/mcp-server/climcgen）。 type BaseListOptions struct { - Limit *int `default:"20" help:"Page limit"` - Offset *int `default:"0" help:"Page offset"` - OrderBy []string `help:"Name of the field to be ordered by"` - Order string `help:"List order" choices:"desc|asc"` - Details *bool `help:"Show more details" default:"false"` + Limit *int `default:"20" help:"Page limit" mcp:"true"` + Offset *int `default:"0" help:"Page offset" mcp:"true"` + OrderBy []string `help:"Name of the field to be ordered by" mcp:"true"` + Order string `help:"List order" choices:"desc|asc" mcp:"true"` + Details *bool `help:"Show more details" default:"false" mcp:"true"` ShowFailReason *bool `help:"show fail reason fields"` - Search string `help:"Filter results by a simple keyword search"` + Search string `help:"Filter results by a simple keyword search" mcp:"true"` Meta *bool `help:"Piggyback metadata information" json:"with_meta" token:"meta"` - Filter []string `help:"Filters"` + Filter []string `help:"Filters" mcp:"true"` JointFilter []string `help:"Filters with joint table col; joint_tbl.related_key(origin_key).filter_col.filter_cond(filters)"` FilterAny *bool `help:"If true, match if any of the filters matches; otherwise, match if all of the filters match"` - Admin *bool `help:"Is an admin call?"` - Tenant string `help:"Tenant ID or Name" alias:"project"` + Admin *bool `help:"Is an admin call?" mcp:"true"` + Tenant string `help:"Tenant ID or Name" alias:"project" mcp:"true"` ProjectDomain string `help:"Project domain filter"` User string `help:"User ID or Name"` - Field []string `help:"Show only specified fields"` - Scope string `help:"resource scope" choices:"system|domain|project|user"` + Field []string `help:"Show only specified fields" mcp:"true"` + Scope string `help:"resource scope" choices:"system|domain|project|user|max" mcp:"true"` - System *bool `help:"Show system resource"` + System *bool `help:"Show system resource" mcp:"true"` PendingDelete *bool `help:"Show only pending deleted resources"` PendingDeleteAll *bool `help:"Show also pending-deleted resources" json:"-"` DeleteAll *bool `help:"Show also deleted resources" json:"-"` @@ -245,7 +247,7 @@ type BaseListOptions struct { Manager []string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"` Account string `help:"List objects belonging to the cloud account" json:"account,omitempty"` - Provider []string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|RockBase|VolcEngine|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|CephFS|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud|OracleCloud|SangFor|ZettaKit|UIS|CNWare" json:"provider,omitempty"` + Provider []string `help:"List objects from the provider；公有云/私有云创建时必传，例如 Aliyun、Aws、Huawei、CAS" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|RockBase|VolcEngine|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|CephFS|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud|OracleCloud|SangFor|ZettaKit|UIS|CAS|CNWare" json:"provider,omitempty" mcp:"true"` Brand []string `help:"List objects belonging to a special brand"` CloudEnv string `help:"Cloud environment" choices:"public|private|onpremise|private_or_onpremise" json:"cloud_env,omitempty"` PublicCloud *bool `help:"List objects belonging to public cloud" json:"public_cloud"` @@ -263,7 +265,7 @@ type BaseListOptions struct { Id []string `help:"filter by id"` // Name []string `help:"fitler by name"` - Status []string `help:"filter by status"` + Status []string `help:"filter by status" mcp:"true"` SummaryStats bool `help:"show summary stats" json:"summary_stats"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/backup.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/backup.go index 9350407b3..d95cad425 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/backup.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/backup.go @@ -66,7 +66,9 @@ type DiskBackupCreateOptions struct { AsTarIgnoreNotExistFile bool `help:"ignore not exist file when using tar"` DISKID string `help:"disk id" json:"disk_id"` - BACKUPSTORAGEID string `help:"back storage id" json:"backup_storage_id"` + BACKUPSTORAGEID string `help:"backup storage id" json:"backup_storage_id"` + + BackupPath string `help:"backup path" json:"backup_path"` } func (opts *DiskBackupCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -92,6 +94,9 @@ func (opts *DiskBackupCreateOptions) Params() (jsonutils.JSONObject, error) { if opts.AsTarIgnoreNotExistFile { input.BackupAsTar.IgnoreNotExistFile = opts.AsTarIgnoreNotExistFile } + if opts.BackupPath != "" { + input.BackupFilePath = opts.BackupPath + } return jsonutils.Marshal(input), nil } @@ -152,7 +157,7 @@ type BackupStorageCreateOptions struct { ObjectBucketUrl string `help:"object bucket url, required when storage_type is object"` ObjectAccessKey string `help:"object storage access key, required when storage_type is object"` ObjectSecret string `help:"object storage secret, required when storage_type is object"` - ObjectSignVer string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"` + ObjectSignVer string `help:"object storage signing algorithm version, optional" choices:"v2|v4"` ObjectBucketUrlExt string `help:"object storage external access url, optional"` @@ -172,7 +177,7 @@ type BackupStorageUpdateOptions struct { ObjectBucketUrl string `help:"object bucket url, required when storage_type is object"` ObjectAccessKey string `help:"object storage access key, required when storage_type is object"` ObjectSecret string `help:"object storage secret, required when storage_type is object"` - ObjectSignVer string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"` + ObjectSignVer string `help:"object storage signing algorithm version, optional" choices:"v2|v4"` ObjectBucketUrlExt string `help:"object storage external access url, optional"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/buckets.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/buckets.go index 2192e4379..19d98760f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/buckets.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/buckets.go @@ -23,6 +23,8 @@ import ( ) type BucketListOptions struct { + _ struct{} `mcp-desc:"列出对象存储 Bucket。可用 search/provider 过滤"` + options.BaseListOptions DistinctField string `help:"query specified distinct field"` } @@ -336,7 +338,7 @@ type BucketSetRefererOption struct { // 域名列表 DomainList []string // 是否允许空referer 访问 - AllowEmptyRefer bool `help:"all empty refer access"` + AllowEmptyRefer bool `help:"allow empty referer access"` Enabled bool RerererType string `help:"Referer type" choices:"Black-List|White-List"` } @@ -430,8 +432,8 @@ type BucketUploadObjectsOptions struct { KEY string `help:"Key of object to upload"` Path string `help:"Path to file to upload" required:"true"` - ContentLength int64 `help:"Content lenght (bytes)" default:"-1"` - StorageClass string `help:"storage CLass"` + ContentLength int64 `help:"Content length (bytes)" default:"-1"` + StorageClass string `help:"storage class"` Acl string `help:"object acl." choices:"private|public-read|public-read-write"` objectstore.ObjectHeaderOptions diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cachedimage.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cachedimage.go index 9704d0393..967173d2e 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cachedimage.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cachedimage.go @@ -21,14 +21,17 @@ import ( ) type CachedImageListOptions struct { + _ struct{} `mcp-desc:"【创建流程中的中间步骤】公有云/私有云选镜像。必须带 provider（如 [\"Aliyun\"]/[\"CAS\"]），region 传 cloudregion UUID。公有云系统盘用非 ISO；CAS/UIS/SangFor 选 ISO，创建时作 cdrom（系统盘不要挂 image）。查完后继续 sku，最后 climc_server_create"` + options.BaseListOptions - ImageType string `help:"image type" choices:"system|customized|shared|market"` + ImageType string `help:"image type；公有云系统盘常用 system" choices:"system|customized|shared|market" mcp:"true"` - Region string `help:"show images cached at cloud region"` - Zone string `help:"show images cached at zone"` + // Region 序列化为 cloudregion_id；优先传 cloudregion UUID + Region string `help:"cloudregion id（推荐）或 name；不要传 cn-shanghai 这类外部 region code" json:"cloudregion_id" mcp:"true"` + Zone string `help:"show images cached at zone" mcp:"true"` - HostSchedtagId string `help:"filter cached image with host schedtag"` - Valid *bool `help:"valid cachedimage"` + HostSchedtagId string `help:"filter cached image with host schedtag" mcp:"true"` + Valid *bool `help:"valid cachedimage" mcp:"true"` } func (opts *CachedImageListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cdn_domains.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cdn_domains.go index b1dada342..99ee70efa 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cdn_domains.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cdn_domains.go @@ -58,7 +58,7 @@ func (opts *CDNDomainCreateOptions) Params() (jsonutils.JSONObject, error) { type CDNDomainUpdateOptions struct { options.BaseIdOptions Description string - Delete string `help:"Lock or not lock cdn domain" choices:"enable|disable"` + Delete string `help:"Lock or unlock CDN domain deletion (disable=lock, enable=unlock)" choices:"enable|disable"` } func (opts *CDNDomainUpdateOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudaccounts.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudaccounts.go index 6b038be8d..08c90d733 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudaccounts.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudaccounts.go @@ -26,12 +26,14 @@ import ( ) type CloudaccountListOptions struct { + _ struct{} `mcp-desc:"列出云账号。可用 search/provider/status 等过滤；详情用 climc_cloud_account_show，同步用 climc_cloud_account_sync"` + baseoptions.BaseListOptions - Capability []string `help:"capability filter" choices:"project|compute|network|loadbalancer|objectstore|rds|cache|event|tablestore"` + Capability []string `help:"capability filter" choices:"project|compute|network|loadbalancer|objectstore|rds|cache|event|tablestore" mcp:"true"` - ReadOnly *bool `help:"filter read only account" negative:"no-read-only"` + ReadOnly *bool `help:"filter read only account" negative:"no-read-only" mcp:"true"` //DistinctField string `help:"distinct field"` - ProxySetting string `help:"Proxy setting id or name"` + ProxySetting string `help:"Proxy setting id or name" mcp:"true"` // 按宿主机数量排序 OrderByHostCount string // 按虚拟机数量排序 @@ -71,7 +73,7 @@ type SProxmoxCredentialWithEnvironment struct { type SAzureCredential struct { ClientID string `help:"Azure client_id" positional:"true"` - ClientSecret string `help:"Azure clinet_secret" positional:"true"` + ClientSecret string `help:"Azure client_secret" positional:"true"` } type SAzureCredentialWithEnvironment struct { @@ -79,7 +81,7 @@ type SAzureCredentialWithEnvironment struct { SAzureCredential - Environment string `help:"Cloud environment" choices:"AzureGermanCloud|AzureChinaCloud|AzurePublicCloud" default:"AzureChinaCloud"` + Environment string `help:"Cloud environment" choices:"AzureGermanCloud|AzureChinaCloud|AzureUSGovernmentCloud|AzurePublicCloud" default:"AzureChinaCloud"` } type SQcloudCredential struct { @@ -120,11 +122,11 @@ type SCloudAccountCreateBaseOptions struct { Desc string `help:"Description" token:"desc" json:"description"` Brand string `help:"Brand of cloud account"` - AutoCreateProject bool `help:"Enable the account with same name project"` - AutoCreateProjectForProvider bool `help:"Is Auto Create Project For Provider"` + AutoCreateProject bool `help:"Auto create local projects from cloud projects/subscriptions"` + AutoCreateProjectForProvider bool `help:"Auto create local project for each cloud provider"` EnableAutoSync bool `help:"Enable automatically synchronize resources of this account"` - SyncIntervalSeconds int `help:"Interval to synchronize if auto sync is enable" metavar:"SECONDS"` + SyncIntervalSeconds int `help:"Interval to synchronize if auto sync is enabled" metavar:"SECONDS"` Project string `help:"project for this account"` ProjectDomain string `help:"domain for this account"` @@ -146,7 +148,7 @@ type SCloudAccountCreateBaseOptions struct { EnableProjectSync bool EnableResourceSync bool - SkipSyncResources []string `help:"Skip sync resource, etc snapshot"` + SkipSyncResources []string `help:"Skip sync resource, e.g. snapshot"` Currency string `choices:"CNY|USD"` } @@ -509,6 +511,13 @@ func (opts *SCloudAccountIdOptions) Params() (jsonutils.JSONObject, error) { return nil, nil } +// CloudaccountShowOptions 单独包装，避免 SCloudAccountIdOptions 被 delete/enable 等复用时误注册。 +type CloudaccountShowOptions struct { + _ struct{} `mcp-desc:"查询云账号详情（含 sync_status、provider、余额等）。ID 可用 climc_cloud_account_list 返回的 id/name"` + + SCloudAccountIdOptions +} + type SVMwareCloudAccountUpdateCredentialOptions struct { SCloudAccountIdOptions SUserPasswordCredential @@ -691,7 +700,7 @@ type SCloudAccountUpdateBaseOptions struct { SCloudAccountIdOptions Name string `help:"New name to update"` - SyncIntervalSeconds *int `help:"auto synchornize interval in seconds"` + SyncIntervalSeconds *int `help:"auto synchronize interval in seconds"` AutoCreateProject *bool `help:"automatically create local project for new remote project" negative:"no_auto_create_project"` EnableAutoSyncResource *bool `help:"automatically sync resources" negative:"disable_auto_sync_resource"` ProxySetting string `help:"proxy setting name or id" json:"proxy_setting"` @@ -699,7 +708,7 @@ type SCloudAccountUpdateBaseOptions struct { ReadOnly *bool `help:"is account read only" negative:"no_read_only"` - CleanLakeOfPermissions bool `help:"clean lake of permissions"` + CleanLakeOfPermissions bool `help:"clean lack of permissions"` SkipSyncResources []string AddSkipSyncResources []string @@ -777,7 +786,7 @@ type SAzureCloudAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions OptionsBalanceKey string `help:"update cloud balance account key, such as Azure EA key" json:"-"` - RemoveOptionsBalanceKey bool `help:"remove cloud blance account key" json:"-"` + RemoveOptionsBalanceKey bool `help:"remove cloud balance account key" json:"-"` RemoveOptionsBillingScope bool OptionsBillingReportBucket string `help:"update Azure bucket that stores account billing report" json:"-"` @@ -1164,8 +1173,8 @@ func (opts *SApsaraCloudAccountCreateOptions) Params() (jsonutils.JSONObject, er type CloudaccountUpdateCredentialOptions struct { SCloudAccountIdOptions - AccessKeyID string `help:"Aiyun|HuaWei|Aws access_key_id"` - AccessKeySecret string `help:"Aiyun|HuaWei|Aws access_key_secret"` + AccessKeyID string `help:"Aliyun|HuaWei|Aws access_key_id"` + AccessKeySecret string `help:"Aliyun|HuaWei|Aws access_key_secret"` AppID string `help:"Qcloud appid"` SecretID string `help:"Qcloud secret_id"` SecretKey string `help:"Qcloud secret_key"` @@ -1174,7 +1183,7 @@ type CloudaccountUpdateCredentialOptions struct { Password string `help:"OpenStack|VMware password"` EndpointType string `help:"OpenStack endpointType"` ClientID string `help:"Azure tenant_id"` - ClientSecret string `help:"Azure clinet_secret"` + ClientSecret string `help:"Azure client_secret"` } func (opts *CloudaccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, error) { @@ -1182,9 +1191,18 @@ func (opts *CloudaccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, } type CloudaccountSyncOptions struct { + _ struct{} `mcp-desc:"同步云账号资源（异步拉取公有云/私有云库存）。ID 用 climc_cloud_account_list 返回的 id/name；常用 force=true 强制同步，可用 region/resources 限定范围。调用后可用 climc_cloud_account_show 查看 sync_status"` + SCloudAccountIdOptions - api.SyncRangeInput + Force bool `help:"Force sync" json:"force" mcp:"true"` + FullSync bool `help:"Full sync" json:"full_sync" mcp:"true"` + DeepSync bool `help:"Deep sync" json:"deep_sync" mcp:"true"` + Xor bool `help:"Incremental xor sync mode" json:"xor" mcp:"true"` + Region []string `help:"Only sync specified regions" json:"region" mcp:"true"` + Zone []string `help:"Only sync specified zones" json:"zone" mcp:"true"` + Host []string `help:"Only sync specified hosts" json:"host" mcp:"true"` + Resources []string `help:"Resource types to sync" json:"resources" mcp:"true" choices:"project|compute|network|eip|loadbalancer|objectstore|rds|cache|event|cloudid|dnszone|public_ip|intervpcnetwork|saml_auth|quota|nat|nas|waf|mongodb|es|kafka|app|cdn|container|ipv6_gateway|tablestore|modelarts|vpcpeer|misc|image"` } func (opts *CloudaccountSyncOptions) Params() (jsonutils.JSONObject, error) { @@ -1202,7 +1220,7 @@ func (opts *CloudaccountEnableAutoSyncOptions) Params() (jsonutils.JSONObject, e type CloudaccountPublicOptions struct { SCloudAccountIdOptions - Scope string `help:"public_sccope" choices:"domain|system" json:"scope"` + Scope string `help:"public scope" choices:"domain|system" json:"scope"` SharedDomains []string `help:"shared domains" json:"shared_domains"` ShareMode string `help:"share_mode" choices:"account_domain|provider_domain|system"` } @@ -1259,7 +1277,7 @@ func (opts *ClouaccountChangeOwnerOptions) Params() (jsonutils.JSONObject, error type ClouaccountChangeProjectOptions struct { SCloudAccountIdOptions - PROJECT string `json:"project" help:"target domain"` + PROJECT string `json:"project" help:"target project ID or name"` } func (opts *ClouaccountChangeProjectOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudproviders.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudproviders.go index 73466056f..1cb46f48a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudproviders.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/cloudproviders.go @@ -27,7 +27,7 @@ type CloudproviderListOptions struct { HasObjectStorage bool `help:"filter cloudproviders that has object storage" negative:"no-object-storage"` Capability []string `help:"capability filter" choices:"project|compute|network|loadbalancer|objectstore|rds|cache|event"` - Cloudregion string `help:"filter cloudproviders by cloudregion"` + Cloudregion []string `help:"filter cloudproviders by cloudregion id or name" json:"cloudregion_id"` ReadOnly *bool `help:"filter read only account" negative:"no-read-only"` @@ -75,8 +75,8 @@ type CloudproviderSyncOptions struct { FullSync bool `help:"Synchronize everything"` ProjectSync bool `help:"Auto sync project info"` Region []string `help:"region to sync"` - Zone []string `help:"region to sync"` - Host []string `help:"region to sync"` + Zone []string `help:"zone to sync"` + Host []string `help:"host to sync"` } func (opts *CloudproviderSyncOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/containers.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/containers.go index f04c05076..e645a0635 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/containers.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/containers.go @@ -48,43 +48,47 @@ type ContainerDeleteOptions struct { } type ContainerCreateCommonOptions struct { - IMAGE string `help:"Image of container" json:"image"` - ImageCredentialId string `help:"Image credential id" json:"image_credential_id"` - Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"` - Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"` - WorkingDir string `help:"Current working directory of the command" json:"working_dir"` - Env []string `help:"List of environment variable to set in the container and the format is: ="` - RootFs string `help:"Root filesystem of the container, e.g.: disk_index=,disk_id="` - VolumeMount []string `help:"Volume mount of the container and the format is: name=,mount_path=,readonly=,case_insensitive_paths=p1,p2,disk_index=,disk_id="` - Device []string `help:"Host device: ::, e.g.: /dev/snd:/dev/snd:rwm"` - Privileged bool `help:"Privileged mode"` - Caps string `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"` - DropCaps string `help:"Container dropped capabilities, split by ','"` - EnableLxcfs bool `help:"Enable lxcfs"` - PostStartExec string `help:"Post started execution command"` - CgroupDeviceAllow []string `help:"Cgroup devices.allow, e.g.: 'c 13:* rwm'"` - SimulateCpu bool `help:"Simulating /sys/devices/system/cpu files"` - ShmSizeMb int `help:"Shm size MB"` - Uid int64 `help:"UID of container" default:"0"` - Gid int64 `help:"GID of container" default:"0"` - DisableNoNewPrivs bool `help:"Disable no_new_privs flag of the container"` - Apparmor string `help:"Apparmor profile for container"` + IMAGE string `help:"Image of container" json:"image"` + ImageCredentialId string `help:"Image credential id" json:"image_credential_id"` + Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"` + Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"` + WorkingDir string `help:"Current working directory of the command" json:"working_dir"` + Env []string `help:"List of environment variable to set in the container and the format is: ="` + RootFs string `help:"Root filesystem of the container, e.g.: disk_index=,disk_id="` + VolumeMount []string `help:"Volume mount of the container and the format is: name=,mount_path=,readonly=,case_insensitive_paths=p1,p2,disk_index=,disk_id="` + Device []string `help:"Host device: ::, e.g.: /dev/snd:/dev/snd:rwm"` + Privileged bool `help:"Privileged mode"` + Caps string `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"` + DropCaps string `help:"Container dropped capabilities, split by ','"` + EnableLxcfs bool `help:"Enable lxcfs"` + PostStartExec string `help:"Post started execution command"` + CgroupDeviceAllow []string `help:"Cgroup devices.allow, e.g.: 'c 13:* rwm'"` + DisableCgroupCpuLimit bool `help:"Do not set CPU cgroup CFS quota"` + DisableCgroupMemoryLimit bool `help:"Do not set memory cgroup hard limit"` + SimulateCpu bool `help:"Simulating /sys/devices/system/cpu files"` + ShmSizeMb int `help:"Shm size MB"` + Uid int64 `help:"UID of container" default:"0"` + Gid int64 `help:"GID of container" default:"0"` + DisableNoNewPrivs bool `help:"Disable no_new_privs flag of the container"` + Apparmor string `help:"Apparmor profile for container"` } func (o ContainerCreateCommonOptions) getCreateSpec() (*computeapi.ContainerSpec, error) { req := &computeapi.ContainerSpec{ ContainerSpec: apis.ContainerSpec{ - Image: o.IMAGE, - ImageCredentialId: o.ImageCredentialId, - Command: o.Command, - Args: o.Args, - WorkingDir: o.WorkingDir, - EnableLxcfs: o.EnableLxcfs, - Privileged: o.Privileged, - Capabilities: &apis.ContainerCapability{}, - CgroupDevicesAllow: o.CgroupDeviceAllow, - SimulateCpu: o.SimulateCpu, - DisableNoNewPrivs: o.DisableNoNewPrivs, + Image: o.IMAGE, + ImageCredentialId: o.ImageCredentialId, + Command: o.Command, + Args: o.Args, + WorkingDir: o.WorkingDir, + EnableLxcfs: o.EnableLxcfs, + Privileged: o.Privileged, + Capabilities: &apis.ContainerCapability{}, + CgroupDevicesAllow: o.CgroupDeviceAllow, + DisableCgroupCpuLimit: o.DisableCgroupCpuLimit, + DisableCgroupMemoryLimit: o.DisableCgroupMemoryLimit, + SimulateCpu: o.SimulateCpu, + DisableNoNewPrivs: o.DisableNoNewPrivs, SecurityContext: &apis.ContainerSecurityContext{ RunAsUser: nil, RunAsGroup: nil, @@ -360,7 +364,7 @@ type ContainerSaveVolumeMountImage struct { INDEX int `help:"Index of volume mount"` GenerateName string `help:"Generate image name automatically"` Notes string `help:"Extra notes of the image"` - UsedByPostOverlay bool `help:"Used by voluem mount post-overlay"` + UsedByPostOverlay bool `help:"Used by volume mount post-overlay"` Dirs []string `help:"Internal directories"` } @@ -569,8 +573,8 @@ func (o *ContainerRemoveVolumeMountPostOverlayOptions) Params() (jsonutils.JSONO } type ContainerCopyOptions struct { - SRC string `help:"Local path or file name, or cotnainer:path, e.g. /etc/hots or ctr-0:/etc/hosts"` - DST string `help:"Local path or file name, or cotnainer:path, e.g. /etc/hots or ctr-0:/etc/hosts"` + SRC string `help:"Local path or file name, or container:path, e.g. /etc/hosts or ctr-0:/etc/hosts"` + DST string `help:"Local path or file name, or container:path, e.g. /etc/hosts or ctr-0:/etc/hosts"` RawFile bool `help:"copy the file as raw data, if false, requires tar in executive path in container and host"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/dbinstances.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/dbinstances.go index 173f00ce2..c1cd78efd 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/dbinstances.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/dbinstances.go @@ -27,21 +27,21 @@ import ( type DBInstanceCreateOptions struct { NAME string `help:"DBInstance Name"` InstanceType string `help:"InstanceType for DBInstance"` - VcpuCount int `help:"Core of cpu for DBInstance"` - VmemSizeMb int `help:"Memory size of DBInstance"` + VcpuCount int `help:"CPU cores for DBInstance"` + VmemSizeMb int `help:"Memory size in MB for DBInstance"` Port int `help:"Port of DBInstance"` Category string `help:"Category of DBInstance"` Network string `help:"Network of DBInstance"` Address string `help:"Address of DBInstance"` Engine string `help:"Engine of DBInstance"` EngineVersion string `help:"EngineVersion of DBInstance Engine"` - StorageType string `help:"StorageTyep of DBInstance"` + StorageType string `help:"Storage type of DBInstance"` Secgroup string `help:"Secgroup name or Id for DBInstance"` Zone string `help:"ZoneId or name for DBInstance"` - DiskSizeGB int `help:"Storage size for DBInstance"` + DiskSizeGB int `help:"Storage size in GB for DBInstance"` Duration string `help:"Duration for DBInstance"` - AllowDelete *bool `help:"not lock dbinstance" ` - Tags []string `help:"Tags info,prefix with 'user:', eg: user:project=default" json:"-"` + AllowDelete *bool `help:"Allow deleting the DBInstance (disable_delete=false)"` + Tags []string `help:"Tags info, prefix with 'user:', eg: user:project=default" json:"-"` DBInstancebackupId string `help:"create dbinstance from backup" json:"dbinstancebackup_id"` MultiAz bool `help:"deploy rds with multi az"` } @@ -73,10 +73,12 @@ func (opts *DBInstanceCreateOptions) Params() (jsonutils.JSONObject, error) { } type DBInstanceListOptions struct { + _ struct{} `mcp-desc:"列出 RDS（dbinstance）。可用 search/provider/region 等过滤；详情用 climc_dbinstance_show"` + options.BaseListOptions - BillingType string `help:"billing type" choices:"postpaid|prepaid"` - IpAddr []string - SecgroupId string + BillingType string `help:"billing type" choices:"postpaid|prepaid" mcp:"true"` + IpAddr []string `mcp:"true"` + SecgroupId string `mcp:"true"` } func (opts *DBInstanceListOptions) Params() (jsonutils.JSONObject, error) { @@ -95,6 +97,13 @@ func (opts *DBInstanceIdOptions) Params() (jsonutils.JSONObject, error) { return nil, nil } +// DBInstanceShowOptions 单独包装，避免 IdOptions 被 reboot/sync 等复用时误注册。 +type DBInstanceShowOptions struct { + _ struct{} `mcp-desc:"查询 RDS（dbinstance）详情。ID 可用 climc_dbinstance_list 返回的 id/name"` + + DBInstanceIdOptions +} + type DBInstanceRenewOptions struct { DBInstanceIdOptions DURATION string `help:"Duration of renew, ADMIN only command"` @@ -110,7 +119,7 @@ type DBInstanceUpdateOptions struct { DBInstanceIdOptions Name string Description string - Delete string `help:"Lock or not lock dbinstance" choices:"enable|disable"` + Delete string `help:"Lock or unlock deletion (disable=lock, enable=unlock)" choices:"enable|disable"` } func (opts *DBInstanceUpdateOptions) Params() (jsonutils.JSONObject, error) { @@ -152,7 +161,7 @@ func (opts *DBInstanceChangeConfigOptions) Params() (jsonutils.JSONObject, error params.Add(jsonutils.NewInt(opts.VcpuCount), "vcpu_count") } if opts.VmemSizeMb > 0 { - params.Add(jsonutils.NewInt(opts.VmemSizeMb), "vmeme_size_mb") + params.Add(jsonutils.NewInt(opts.VmemSizeMb), "vmem_size_mb") } return params, nil } @@ -209,7 +218,7 @@ func (opts *DBInstanceDeleteOptions) Params() (jsonutils.JSONObject, error) { type DBInstanceChangeOwnerOptions struct { DBInstanceIdOptions - PROJECT string `help:"Project ID or change" json:"tenant"` + PROJECT string `help:"Target project ID or name" json:"tenant"` } func (opts *DBInstanceChangeOwnerOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/disks.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/disks.go index 5f1ae5f4c..acdba68d2 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/disks.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/disks.go @@ -23,16 +23,16 @@ import ( ) type DiskCreateOptions struct { - Manager string `help:"Preferred manager where virtual server should be created" json:"prefer_manager_id"` - Region string `help:"Preferred region where virtual server should be created" json:"prefer_region_id"` - Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone_id"` - Zones []string `help:"Preferred zones where virtual server should be created" json:"prefer_zones"` - Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire_id"` - Host string `help:"Preferred host where virtual server should be created" json:"prefer_host_id"` + Manager string `help:"Preferred manager where disk should be created" json:"prefer_manager_id"` + Region string `help:"Preferred region where disk should be created" json:"prefer_region_id"` + Zone string `help:"Preferred zone where disk should be created" json:"prefer_zone_id"` + Zones []string `help:"Preferred zones where disk should be created" json:"prefer_zones"` + Wire string `help:"Preferred wire where disk should be created" json:"prefer_wire_id"` + Host string `help:"Preferred host where disk should be created" json:"prefer_host_id"` Count int `help:"Count to create" json:"count"` NAME string `help:"Name of the disk"` - DISKDESC string `help:"Image size or size of virtual disk"` + DISKDESC string `help:"Disk config, e.g. 'size=10G' or 'image=,size=40G,backend=local'"` Desc string `help:"Description" metavar:"Description"` Storage string `help:"ID or name of storage where the disk is created"` Hypervisor string `help:"Hypervisor of this disk, used by schedule"` @@ -92,6 +92,28 @@ func (o *DiskIdOptions) Params() (jsonutils.JSONObject, error) { return nil, nil } +// DiskShowOptions 单独包装，避免 DiskIdOptions 被其它 Perform 复用时误注册。 +type DiskShowOptions struct { + _ struct{} `mcp-desc:"查询单块硬盘详情。ID 可用 climc_disk_list 返回的 id/name"` + + DiskIdOptions +} + +type DiskResizeOptions struct { + _ struct{} `mcp-desc:"硬盘扩容。DISK 为硬盘 id/name，SIZE 为目标容量（如 100G）；扩容前可用 climc_disk_list / climc_disk_show 确认"` + + DISK string `help:"ID or name of disk"` + SIZE string `help:"Size of disk"` +} + +type DiskDeleteOptions struct { + _ struct{} `mcp-desc:"删除硬盘。若尚不知 id，先用 climc_disk_list 定位；删除前确认未被虚机占用或已卸载"` + + ID []string `help:"ID of disks to delete" metavar:"DISK"` + OverridePendingDelete bool `help:"Delete disk directly instead of pending delete" short-token:"f" mcp:"true"` + DeleteSnapshots bool `help:"Delete disk snapshots before delete disk" mcp:"true"` +} + type DiskMigrateOptions struct { DiskIdOptions @@ -115,7 +137,7 @@ func (o *DiskChangeStorageTypeOptions) Params() (jsonutils.JSONObject, error) { type DiskResetTemplateOptions struct { DiskIdOptions - TemplateId string `help:"reset disk tempalte id" json:"template_id"` + TemplateId string `help:"reset disk template id" json:"template_id"` } func (o *DiskResetTemplateOptions) Params() (jsonutils.JSONObject, error) { @@ -143,7 +165,7 @@ func (o *DiskRebuildOptions) Params() (jsonutils.JSONObject, error) { res.TemplateId = &o.TemplateId } if o.Size != "" { - res.TemplateId = &o.Size + res.Size = &o.Size } if o.Fs != "" { res.Fs = &o.Fs @@ -177,24 +199,26 @@ func (o *DiskRebuildOptions) Params() (jsonutils.JSONObject, error) { } type DiskListOptions struct { + _ struct{} `mcp-desc:"列出硬盘。可用 search/server-id/unused 等过滤；查详情用 climc_disk_show，扩容用 climc_disk_resize，删除用 climc_disk_delete"` + options.BaseListOptions - Unused *bool `help:"Show unused disks"` - Share *bool `help:"Show Share storage disks"` - Local *bool `help:"Show Local storage disks"` - ServerId []string `help:"Guest ID or name"` - GuestStatus string `help:"Guest Status"` + Unused *bool `help:"Show unused disks" mcp:"true"` + Share *bool `help:"Show Share storage disks" mcp:"true"` + Local *bool `help:"Show Local storage disks" mcp:"true"` + ServerId []string `help:"Guest ID or name" mcp:"true"` + GuestStatus string `help:"Guest Status" mcp:"true"` OrderByServer string `help:"Order By Server"` - Storage string `help:"Storage ID or name"` - Type string `help:"Disk type" choices:"sys|data|swap|volume"` - CloudType string `help:"Public cloud or private cloud" choices:"Public|Private"` + Storage string `help:"Storage ID or name" mcp:"true"` + Type string `help:"Disk type" choices:"sys|data|swap|volume" mcp:"true"` + CloudType string `help:"Public cloud or private cloud" choices:"Public|Private" mcp:"true"` OrderByGuestCount string `help:"Order By Guest Count"` - BillingType string `help:"billing type" choices:"postpaid|prepaid"` + BillingType string `help:"billing type" choices:"postpaid|prepaid" mcp:"true"` SnapshotpolicyId string `help:"snapshotpolicy id"` - StorageHostId string `help:"filter disk by host"` + StorageHostId string `help:"filter disk by host" mcp:"true"` BindingServerSnapshotpolicy *bool `help:"filter disk by binding server snapshotpolicy" negative:"no-binding-server-snapshotpolicy"` BindingSnapshotpolicy *bool `help:"filter disk by binding snapshotpolicy" negative:"no-binding-snapshotpolicy"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/eips.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/eips.go index 6023eabaa..adfa7e2be 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/eips.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/eips.go @@ -21,18 +21,20 @@ import ( ) type ElasticipListOptions struct { - Region string `help:"List eips in cloudregion"` + _ struct{} `mcp-desc:"列出弹性公网 IP（EIP）。可用 search/region/usable 过滤；详情 climc_eip_show，绑虚机 climc_server_associate_eip"` - Usable *bool `help:"List all zones that is usable"` - UsableEipForAssociateType string `help:"With associate id filter which eip can associate" choices:"server|natgateway|loadbalancer"` - UsableEipForAssociateId string `help:"With associate type filter which eip can associate"` + Region string `help:"List eips in cloudregion" mcp:"true"` + + Usable *bool `help:"List usable EIPs" mcp:"true"` + UsableEipForAssociateType string `help:"Filter usable EIPs for given associate type" choices:"server|natgateway|loadbalancer" mcp:"true"` + UsableEipForAssociateId string `help:"Filter usable EIPs for given associate id" mcp:"true"` OrderByIp string - AssociateId []string - AssociateType []string + AssociateId []string `mcp:"true"` + AssociateType []string `mcp:"true"` AssociateName []string - IsAssociated *bool + IsAssociated *bool `mcp:"true"` - IpAddr []string + IpAddr []string `mcp:"true"` options.BaseListOptions } @@ -40,6 +42,13 @@ func (opts *ElasticipListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(opts) } +// EipShowOptions 单独包装，避免 BaseShowOptions 被其它资源复用时误注册。 +type EipShowOptions struct { + _ struct{} `mcp-desc:"查询 EIP 详情。ID 可用 climc_eip_list 返回的 id/name"` + + options.BaseShowOptions +} + type EipCreateOptions struct { options.BaseCreateOptions Manager *string `help:"cloud provider"` @@ -58,7 +67,7 @@ func (opts *EipCreateOptions) Params() (jsonutils.JSONObject, error) { type EipUpdateOptions struct { options.BaseUpdateOptions - AutoDellocate *string `help:"enable or disable automatically dellocate when dissociate from instance" choices:"true|false"` + AutoDellocate *string `help:"enable or disable automatically deallocate when dissociate from instance" choices:"true|false"` IpAddr string AssociateId string AssociateType string @@ -98,7 +107,7 @@ func (opts *EipChangeBandwidthOptions) Params() (jsonutils.JSONObject, error) { type EipChangeOwnerOptions struct { options.BaseIdOptions - PROJECT string `help:"Project ID or change"` + PROJECT string `help:"Target project ID or name"` //RawId bool `help:"User raw ID, instead of name"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elastic_search.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elastic_search.go index 7641ef982..337dd682a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elastic_search.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elastic_search.go @@ -44,7 +44,7 @@ type ElasticSearchUpdateOptions struct { ElasticSearchIdOption Name string Description string - Delete string `help:"Lock or not lock dbinstance" choices:"enable|disable"` + Delete string `help:"Lock or unlock Elasticsearch deletion (disable=lock, enable=unlock)" choices:"enable|disable"` } func (opts *ElasticSearchUpdateOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elasticache.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elasticache.go index 222213fdb..281d1d9e6 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elasticache.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/elasticache.go @@ -24,9 +24,11 @@ import ( ) type ElasticCacheListOptions struct { + _ struct{} `mcp-desc:"列出 Redis/弹性缓存（elastic-cache）。可用 search 等过滤；详情用 climc_elastic_cache_show"` + options.BaseListOptions - SecgroupId string + SecgroupId string `mcp:"true"` } func (opts *ElasticCacheListOptions) Params() (jsonutils.JSONObject, error) { @@ -45,6 +47,13 @@ func (opts *ElasticCacheIdOption) Params() (jsonutils.JSONObject, error) { return nil, nil } +// ElasticCacheShowOptions 单独包装，避免 IdOption 被 restart/delete 等复用时误注册。 +type ElasticCacheShowOptions struct { + _ struct{} `mcp-desc:"查询 Redis/弹性缓存详情。ID 可用 climc_elastic_cache_list 返回的 id/name"` + + ElasticCacheIdOption +} + type ElasticCacheCreateOptions struct { NAME string Manager string @@ -56,7 +65,7 @@ type ElasticCacheCreateOptions struct { SecgroupIds []string `help:"elastic cache security group. required by qcloud."` Engine string `choices:"redis"` EngineVersion string `choices:"2.8|3.0|3.2|4.0|5.0"` - PrivateIP string `help:"private ip address in specificated network"` + PrivateIP string `help:"private ip address in specified network"` Password string `help:"set auth password"` InstanceType string CapacityMB string `help:"elastic cache capacity. required by huawei."` diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/host.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/host.go index 878bd438f..447e915fb 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/host.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/host.go @@ -22,29 +22,29 @@ import ( ) type HostListOptions struct { - Schedtag string `help:"List hosts in schedtag"` - Zone string `help:"List hosts in zone"` - Region string `help:"List hosts in region"` - Wire string `help:"List hosts in wire"` - Image string `help:"List hosts cached images" json:"cachedimage"` - Storage string `help:"List hosts attached to storages"` - Baremetal string `help:"List hosts that is managed by baremetal system" choices:"true|false"` + Schedtag string `help:"List hosts in schedtag" mcp:"true"` + Zone string `help:"List hosts in zone" mcp:"true"` + Region string `help:"List hosts in region" mcp:"true"` + Wire string `help:"List hosts in wire" mcp:"true"` + Image string `help:"List hosts cached images" json:"cachedimage" mcp:"true"` + Storage string `help:"List hosts attached to storages" mcp:"true"` + Baremetal string `help:"List hosts that is managed by baremetal system" choices:"true|false" mcp:"true"` Empty bool `help:"show empty host" json:"-"` - Occupied bool `help:"show occupid host" json:"-"` + Occupied bool `help:"show occupied host" json:"-"` Enabled bool `help:"Show enabled host only" json:"-"` Disabled bool `help:"Show disabled host only" json:"-"` - HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|container|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud|google|ctyun"` + HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|container|hyperv|aliyun|apsara|azure|qcloud|aws|huawei|ucloud|google|ctyun|dedicated|openstack|volcengine|zstack|ecloud|jdcloud|cloudpods|nutanix|bingocloud|incloudsphere|proxmox|remotefile|h3c|ksyun|baidu|cucloud|qingcloud|oracle|sangfor|zettakit|uis|cas|cnware|rockbase" mcp:"true"` AnyMac string `help:"Mac matches one of the host's interface"` - AnyIp []string `help:"IP matches one of the host's interface"` + AnyIp []string `help:"IP matches one of the host's interface" mcp:"true"` HostStorageType []string `help:"List host in host_storage_type"` - IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"` + IsBaremetal *bool `help:"filter host list by is_baremetal=true|false" mcp:"true"` - ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` + ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated" mcp:"true"` - Usable *bool `help:"List all zones that is usable"` + Usable *bool `help:"List usable hosts" mcp:"true"` - Hypervisor string `help:"filter hosts by hypervisor"` + Hypervisor string `help:"filter hosts by hypervisor" mcp:"true"` StorageNotAttached bool `help:"List hosts not attach specified storage"` @@ -56,9 +56,9 @@ type HostListOptions struct { OrderByServerCount string `help:"Order by server count" choices:"desc|asc"` OrderByStorage string `help:"Order by host storage" choices:"desc|asc"` - OrderByStorageCommitRate string `help:"Order by host storage commite rate" choices:"desc|asc"` + OrderByStorageCommitRate string `help:"Order by host storage commit rate" choices:"desc|asc"` OrderByCpuCommitRate string `help:"Order by host cpu commit rate" choices:"desc|asc"` - OrderByMemCommitRate string `help:"Order by host meme commit rate" choices:"desc|asc"` + OrderByMemCommitRate string `help:"Order by host mem commit rate" choices:"desc|asc"` OrderByStorageUsed string `help:"Order by storage used" choices:"desc|asc"` OrderByCpuCommit string `help:"Order by cpu commit" choices:"desc|asc"` @@ -105,12 +105,21 @@ func (opts *HostListOptions) Params() (jsonutils.JSONObject, error) { return params, nil } +// HostListForMcpOptions 单独包装，避免 HostListOptions 被 host-node-count 等复用时误注册。 +type HostListForMcpOptions struct { + _ struct{} `mcp-desc:"列出宿主机。可用 zone/region/host-type/search 过滤；详情用 climc_host_show"` + + HostListOptions +} + type HostShowOptions struct { + _ struct{} `mcp-desc:"查询宿主机详情。ID 可用 climc_host_list 返回的 id/name"` + options.BaseShowOptions - ShowMetadata bool `help:"Show host metadata in details"` - ShowNicInfo bool `help:"Show host nic_info in details"` - ShowSysInfo bool `help:"Show host sys_info in details"` - ShowAll bool `help:"Show all of host details" short-token:"a"` + ShowMetadata bool `help:"Show host metadata in details" mcp:"true"` + ShowNicInfo bool `help:"Show host nic_info in details" mcp:"true"` + ShowSysInfo bool `help:"Show host sys_info in details" mcp:"true"` + ShowAll bool `help:"Show all of host details" short-token:"a" mcp:"true"` } func (o *HostShowOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/instancegroup_guest.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/instancegroup_guest.go index 8e4f387c2..77e0eabc5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/instancegroup_guest.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/instancegroup_guest.go @@ -24,7 +24,7 @@ type InstanceGroupGuestListOptions struct { options.BaseListOptions Group string `help:"Group ID or Name"` - Server string `help:"Gest ID or Name"` + Server string `help:"Guest ID or Name"` } func (opts *InstanceGroupGuestListOptions) GetMasterOpt() string { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/ipset.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/ipset.go new file mode 100644 index 000000000..2a88db86f --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/ipset.go @@ -0,0 +1,77 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type IpSetListOptions struct { + options.BaseListOptions + + IPSetType []string `help:"filter by ip set type" choices:"ipv4_cidr_list|ipv6_cidr_list" json:"ip_set_type"` + Ip string `help:"filter by ip or cidr" json:"ip"` + Cloudregion string `help:"filter by cloudregion" json:"cloudregion_id"` + ExternalId string `help:"filter by external id" json:"external_id"` +} + +func (opts *IpSetListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(opts) +} + +type IpSetCreateOptions struct { + options.BaseCreateOptions + + IPSetType string `help:"ip set type" choices:"ipv4_cidr_list|ipv6_cidr_list" json:"ip_set_type" required:"true"` + Data string `help:"comma separated ip/cidr list, e.g. 192.168.1.0/24,10.0.0.1" json:"data" required:"true"` + CloudregionId string `help:"cloudregion id" json:"cloudregion_id"` + ManagerId string `help:"cloudprovider id" json:"manager_id"` +} + +func (opts *IpSetCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + +type IpSetIdOptions struct { + options.BaseIdOptions +} + +type IpSetUpdateOptions struct { + options.BaseIdOptions + + Name string `help:"new name of ipset"` + Desc string `help:"description" json:"description"` + IPSetType string `help:"ip set type" choices:"ipv4_cidr_list|ipv6_cidr_list" json:"ip_set_type"` + Data string `help:"comma separated ip/cidr list, e.g. 192.168.1.0/24,10.0.0.1" json:"data"` +} + +func (opts *IpSetUpdateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.NewDict() + if len(opts.Name) > 0 { + params.Add(jsonutils.NewString(opts.Name), "name") + } + if len(opts.Desc) > 0 { + params.Add(jsonutils.NewString(opts.Desc), "description") + } + if len(opts.IPSetType) > 0 { + params.Add(jsonutils.NewString(opts.IPSetType), "ip_set_type") + } + if len(opts.Data) > 0 { + params.Add(jsonutils.NewString(opts.Data), "data") + } + return params, nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/isolated_devices.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/isolated_devices.go index 76d683682..42f3251b1 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/isolated_devices.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/isolated_devices.go @@ -31,10 +31,11 @@ type DeviceListOptions struct { DevType []string `help:"filter by dev_type"` Model []string `help:"filter by model"` + Vendor []string `help:"filter by vendor, e.g. HYGON, NVIDIA"` Addr []string `help:"filter by addr"` DevicePath []string `help:"filter by device path"` VendorDeviceId []string `help:"filter by vendor device id(PCIID)"` - NumaNode []uint8 `help:"fitler by numa node index"` + NumaNode []uint8 `help:"filter by numa node index"` Index *int `help:"filter by index"` DeviceMinor *int `help:"filter by device minor"` } @@ -43,6 +44,35 @@ func (o *DeviceListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(o) } +type ServerDeviceListOptions struct { + DeviceListOptions + IsolateDeviceIds []string `json:"isolate_device_ids"` +} + +func (o *ServerDeviceListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type DeviceCreateOptions struct { + options.BaseCreateOptions + + HostId string + DevType string + SharingMode string + Model string + HotPluggable bool + Addr string + DevicePath string + VendorDeviceId string + MemorySize int + VirtualNum int + NumaNode int +} + +func (o *DeviceCreateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + type DeviceShowOptions struct { options.BaseIdOptions } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/kafka.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/kafka.go index 1a6828fc9..a15e44c41 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/kafka.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/kafka.go @@ -21,9 +21,18 @@ import ( ) type KafkaListOptions struct { + _ struct{} `mcp-desc:"列出 Kafka 实例。可用 search 等过滤；详情用 climc_kafka_show"` + options.BaseListOptions } +// KafkaShowOptions 单独包装，避免 IdOption 被 delete/syncstatus 复用时误注册。 +type KafkaShowOptions struct { + _ struct{} `mcp-desc:"查询 Kafka 实例详情。ID 可用 climc_kafka_list 返回的 id/name"` + + KafkaIdOption +} + func (opts *KafkaListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(opts) } @@ -44,7 +53,7 @@ type KafkaUpdateOptions struct { KafkaIdOption Name string Description string - Delete string `help:"Lock or not lock dbinstance" choices:"enable|disable"` + Delete string `help:"Lock or unlock Kafka deletion (disable=lock, enable=unlock)" choices:"enable|disable"` } func (opts *KafkaUpdateOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/keypairs.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/keypairs.go index 335b0bcb6..f04c8cce4 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/keypairs.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/keypairs.go @@ -21,8 +21,10 @@ import ( ) type KeypairList struct { + _ struct{} `mcp-desc:"列出 SSH 密钥对（keypair）。可用 search/scheme 过滤"` + options.BaseListOptions - Scheme string `help:"Scheme of keypair, default is RSA" choices:"RSA|DSA|ECDSA|ED25519"` + Scheme string `help:"Scheme of keypair, default is RSA" choices:"RSA|DSA|ECDSA|ED25519" mcp:"true"` } func (self *KeypairList) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/loadbalancers.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/loadbalancers.go index 1919285eb..25feebdc3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/loadbalancers.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/loadbalancers.go @@ -111,17 +111,19 @@ type LoadbalancerPurgeOptions struct { } type LoadbalancerListOptions struct { + _ struct{} `mcp-desc:"列出负载均衡（lb）。可用 search/cloudregion/address-type 过滤"` + options.BaseListOptions - Address string - AddressType string `choices:"intranet|internet"` - NetworkType string `choices:"classic|vpc"` - Network string + Address string `mcp:"true"` + AddressType string `choices:"intranet|internet" mcp:"true"` + NetworkType string `choices:"classic|vpc" mcp:"true"` + Network string `mcp:"true"` BackendGroup string - Cloudregion string - Zone string + Cloudregion string `mcp:"true"` + Zone string `mcp:"true"` Cluster string `json:"cluster_id"` - SecgroupId string + SecgroupId string `mcp:"true"` } func (opts *LoadbalancerListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/mongodb.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/mongodb.go index c638dd707..b34309f26 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/mongodb.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/mongodb.go @@ -21,9 +21,18 @@ import ( ) type MongoDBListOptions struct { + _ struct{} `mcp-desc:"列出 MongoDB 实例。可用 search 等过滤；详情用 climc_mongodb_show"` + options.BaseListOptions } +// MongoDBShowOptions 单独包装，避免复用 BaseIdOptions 误注册。 +type MongoDBShowOptions struct { + _ struct{} `mcp-desc:"查询 MongoDB 实例详情。ID 可用 climc_mongodb_list 返回的 id/name"` + + options.BaseIdOptions +} + func (opts *MongoDBListOptions) Params() (jsonutils.JSONObject, error) { return options.ListStructToParams(opts) } @@ -32,7 +41,7 @@ type MongoDBUpdateOptions struct { options.BaseIdOptions Name string Description string - Delete string `help:"Lock or not lock dbinstance" choices:"enable|disable"` + Delete string `help:"Lock or unlock MongoDB deletion (disable=lock, enable=unlock)" choices:"enable|disable"` } func (opts *MongoDBUpdateOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/natgateways.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/natgateways.go index 6528d9daf..7d06698d7 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/natgateways.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/natgateways.go @@ -24,7 +24,7 @@ import ( type NatGatewayListOptions struct { options.BaseListOptions Vpc string `help:"vpc id or name"` - Cloudregion string `help:"cloudreigon id or name"` + Cloudregion string `help:"cloudregion id or name"` } func (opts *NatGatewayListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/network.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/network.go index 8c8b2e966..353e71487 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/network.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/network.go @@ -24,22 +24,24 @@ import ( ) type NetworkListOptions struct { + _ struct{} `mcp-desc:"【创建流程中的中间步骤】本工具不能完成创建。公有云须带 provider（如 [\"Aliyun\"]）及 region；查完网络后凑齐同云镜像与规格，立刻 climc_server_create。严禁只调用本工具后就停止"` + options.BaseListOptions - Ip string `help:"search networks that contain this IP"` - ZoneIds []string `help:"search networks in zones"` + Ip string `help:"search networks that contain this IP" mcp:"true"` + ZoneIds []string `help:"search networks in zones" mcp:"true"` Wire string `help:"search networks belongs to a wire" json:"-"` - Host string `help:"search networks attached to a host"` - Vpc string `help:"search networks belongs to a VPC"` - Region string `help:"search networks belongs to a CloudRegion" json:"cloudregion"` + Host string `help:"search networks attached to a host" mcp:"true"` + Vpc string `help:"search networks belongs to a VPC" mcp:"true"` + Region string `help:"search networks belongs to a CloudRegion" json:"cloudregion" mcp:"true"` City string `help:"search networks belongs to a city"` - Usable *bool `help:"search usable networks"` - ServerType string `help:"search networks belongs to a ServerType" choices:"baremetal|container|eip|guest|ipmi|pxe|hostlocal"` + Usable *bool `help:"search usable networks" mcp:"true"` + ServerType string `help:"search networks belongs to a ServerType" choices:"baremetal|container|eip|guest|ipmi|pxe|hostlocal" mcp:"true"` Schedtag string `help:"filter networks by schedtag"` HostSchedtagId string `help:"filter by host schedtag"` - IsAutoAlloc *bool `help:"search network with is_auto_alloc"` + IsAutoAlloc *bool `help:"search network with is_auto_alloc" mcp:"true"` IsClassic *bool `help:"search classic on-premise network"` // Status string `help:"filter by network status"` @@ -68,7 +70,7 @@ type NetworkCreateOptions struct { WIRE string `help:"ID or Name of wire in which the network is created"` NETWORK string `help:"Name of new network"` StartIp string `help:"Start of IPv4 address range" positional:"true" json:"start_ip"` - EndIp string `help:"End of IPv4 address rnage" positional:"true" json:"end_ip"` + EndIp string `help:"End of IPv4 address range" positional:"true" json:"end_ip"` NetMask int64 `help:"Length of network mask" positional:"true" json:"net_mask"` Gateway string `help:"Default gateway"` @@ -82,7 +84,7 @@ type NetworkCreateOptions struct { AllocPolicy string `help:"Address allocation policy" choices:"none|stepdown|stepup|random"` ServerType string `help:"Server type" choices:"baremetal|container|eip|guest|ipmi|pxe|hostlocal"` IsAutoAlloc *bool `help:"Auto allocation IP pool"` - BgpType string `help:"Internet service provider name" positional:"false"` + BgpType string `help:"BGP/line type (e.g. BGP, ChinaTelecom)" positional:"false"` Desc string `help:"Description" metavar:"DESCRIPTION"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedtags.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedtags.go index 7601e4624..da64727c3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedtags.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedtags.go @@ -119,7 +119,7 @@ func (o SchedtagCreateOptions) Params() (jsonutils.JSONObject, error) { } type SchedtagUpdateOptions struct { - ID string `help:"ID or Name of schetag"` + ID string `help:"ID or Name of schedtag"` Name string `help:"New name of schedtag"` Strategy string `help:"Policy" choices:"require|exclude|prefer|avoid"` Desc string `help:"Description"` @@ -152,7 +152,7 @@ func (o SchedtagUpdateOptions) Params() (jsonutils.JSONObject, error) { } type SchedtagSetScopeOptions struct { - ID []string `help:"ID or Name of schetag"` + ID []string `help:"ID or Name of schedtag"` Project string `help:"ID or Name of project"` Domain string `help:"ID or Name of domain"` System bool `help:"Set to system scope"` @@ -176,7 +176,7 @@ func (o SchedtagSetScopeOptions) Params() (jsonutils.JSONObject, error) { } type SchedtagSetResource struct { - ID string `help:"ID or Name of schetag"` + ID string `help:"ID or Name of schedtag"` Resource []string `help:"Resource id or name"` UnbindAll bool `help:"Unbind all attached resources"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedulers.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedulers.go index 7f03bf660..11535dd31 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedulers.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/schedulers.go @@ -63,8 +63,8 @@ func (o SchedulerTestBaseOptions) options() *scheduler.ScheduleBaseConfig { type SchedulerTestOptions struct { SchedulerTestBaseOptions - SuggestionLimit int64 `help:"Number of schedule candidate informations" default:"50"` - SuggestionAll bool `help:"Show all schedule candidate informations"` + SuggestionLimit int64 `help:"Number of schedule candidate information" default:"50"` + SuggestionAll bool `help:"Show all schedule candidate information"` Details bool `help:"Show suggestion details"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgroup.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgroup.go index 6bb382415..cc19d2957 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgroup.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgroup.go @@ -28,18 +28,22 @@ import ( ) type SecgroupListOptions struct { + _ struct{} `mcp-desc:"列出安全组。可用 search/server/vpc-id 等过滤；详情 climc_secgroup_show，规则 climc_secgroup_rule_list"` + baseoptions.BaseListOptions - Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one"` - Server string `help:"Filter secgroups bound to specified server"` - Ip string `help:"Filter secgroup by ip"` - Ports string `help:"Filter secgroup by ports"` - Direction string `help:"Filter secgroup by ports" choices:"all|in|out"` - DBInstance string `help:"Filter secgroups bound to specified rds" json:"dbinstance"` - Cloudregion string `help:"Filter secgroups by region"` - VpcId string - Cloudaccount string `help:"Filter secgroups by account"` - LoadbalancerId string + Equals string `help:"Secgroup ID or Name, filter secgroups whose rules equals the specified one" mcp:"true"` + Server string `help:"Filter secgroups bound to specified server" mcp:"true"` + Ip string `help:"Filter secgroup by ip" mcp:"true"` + Ports string `help:"Filter secgroup by ports" mcp:"true"` + Direction string `help:"Filter secgroup by direction" choices:"all|in|out" mcp:"true"` + DBInstance string `help:"Filter secgroups bound to specified rds" json:"dbinstance" mcp:"true"` + Cloudregion string `help:"Filter secgroups by region" mcp:"true"` + VpcId string `mcp:"true"` + Cloudaccount string `help:"Filter secgroups by account" mcp:"true"` + LoadbalancerId string `mcp:"true"` + + IpSetId []string `help:"Filter secgroups by ip set" mcp:"true"` } func (opts *SecgroupListOptions) Params() (jsonutils.JSONObject, error) { @@ -47,10 +51,12 @@ func (opts *SecgroupListOptions) Params() (jsonutils.JSONObject, error) { } type SecgroupCreateOptions struct { + _ struct{} `mcp-desc:"创建安全组。NAME 必填；可传 rules（安全规则字符串数组）与 vpc-id。创建后可用 climc_secgroup_rule_create 继续加规则"` + baseoptions.BaseCreateOptions - VpcId string - Tags []string - Rules []string `help:"security rule to create"` + VpcId string `mcp:"true"` + Tags []string `mcp:"true"` + Rules []string `help:"security rule to create" mcp:"true"` } func (opts *SecgroupCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -84,7 +90,7 @@ func (opts *SecgroupCreateOptions) Params() (jsonutils.JSONObject, error) { } type SecgroupIdOptions struct { - ID string `help:"ID or Name of security group destination"` + ID string `help:"ID or Name of security group"` } func (opts *SecgroupIdOptions) GetId() string { @@ -95,6 +101,19 @@ func (opts *SecgroupIdOptions) Params() (jsonutils.JSONObject, error) { return nil, nil } +// SecgroupShowOptions / SecgroupDeleteOptions 单独包装，避免与其它 Perform 复用 IdOptions 时注册。 +type SecgroupShowOptions struct { + _ struct{} `mcp-desc:"查询安全组详情。ID 可用 climc_secgroup_list 返回的 id/name"` + + SecgroupIdOptions +} + +type SecgroupDeleteOptions struct { + _ struct{} `mcp-desc:"删除安全组。若尚不知 id，先用 climc_secgroup_list 定位；确认无虚机绑定后再删"` + + SecgroupIdOptions +} + type SecgroupMergeOptions struct { SecgroupIdOptions SECGROUPS []string `help:"source IDs or Names of secgroup"` @@ -108,10 +127,10 @@ type SecgroupsAddRuleOptions struct { SecgroupIdOptions DIRECTION string `help:"Direction of rule" choices:"in|out"` PROTOCOL string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"` - ACTION string `help:"Actin of rule" choices:"allow|deny"` + ACTION string `help:"Action of rule" choices:"allow|deny"` PRIORITY int `help:"Priority for rule, range 1 ~ 100"` - Cidr string `help:"IP or CIRD for rule"` - Description string `help:"Desciption for rule"` + Cidr string `help:"IP or CIDR for rule"` + Description string `help:"Description for rule"` Ports string `help:"Port for rule"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgrouprule.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgrouprule.go index 1cde0e9f6..fce070235 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgrouprule.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/secgrouprule.go @@ -23,15 +23,17 @@ import ( ) type SecGroupRulesListOptions struct { + _ struct{} `mcp-desc:"列出安全组规则。建议传 secgroup（安全组 id/name）；详情 climc_secgroup_rule_show，创建 climc_secgroup_rule_create，删除 climc_secgroup_rule_delete"` + options.BaseListOptions - Secgroup string `help:"Secgroup ID or Name"` - SecgroupName string `help:"Search rules by fuzzy secgroup name"` - Projects []string `help:"Filter rules by project"` - Direction string `help:"filter Direction of rule" choices:"in|out"` - Protocol string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp"` - Action string `help:"filter Actin of rule" choices:"allow|deny"` - Ports string `help:"filter Ports of rule"` - Ip string `help:"filter cidr of rule"` + Secgroup string `help:"Secgroup ID or Name" mcp:"true"` + SecgroupName string `help:"Search rules by fuzzy secgroup name" mcp:"true"` + Projects []string `help:"Filter rules by project" mcp:"true"` + Direction string `help:"filter Direction of rule" choices:"in|out" mcp:"true"` + Protocol string `help:"filter Protocol of rule" choices:"any|tcp|udp|icmp" mcp:"true"` + Action string `help:"filter Action of rule" choices:"allow|deny" mcp:"true"` + Ports string `help:"filter Ports of rule" mcp:"true"` + Ip string `help:"filter cidr of rule" mcp:"true"` } func (opts *SecGroupRulesListOptions) Params() (jsonutils.JSONObject, error) { @@ -39,10 +41,27 @@ func (opts *SecGroupRulesListOptions) Params() (jsonutils.JSONObject, error) { } type SecGroupRulesCreateOptions struct { - SECGROUP string `help:"Secgroup ID or Name" metavar:"Secgroup"` - RULE string `json:"-"` - Priority int64 `help:"priority of Rule" default:"50"` - Desc string `help:"Description" json:"description"` + _ struct{} `mcp-desc:"创建安全组规则。SECGROUP 为安全组 id/name，RULE 为规则字符串（如 in:allow tcp 22）；腾讯云可用 --ip-set 指定地址模板；可用 climc_secgroup_list 定位安全组"` + + SECGROUP string `help:"Secgroup ID or Name" metavar:"Secgroup"` + RULE string `json:"-"` + Priority int64 `help:"priority of Rule" default:"50" mcp:"true"` + Desc string `help:"Description" json:"description" mcp:"true"` + TargetType string `help:"Target type" choices:"cidr|ip_set" json:"target_type"` + IpSet string `help:"Ip set ID or name when target type is ip_set"` +} + +// SecGroupRuleShowOptions / SecGroupRuleDeleteOptions 单独包装，避免复用 BaseShow/BaseId 误注册。 +type SecGroupRuleShowOptions struct { + _ struct{} `mcp-desc:"查询单条安全组规则详情。ID 可用 climc_secgroup_rule_list 返回的 id"` + + options.BaseShowOptions +} + +type SecGroupRuleDeleteOptions struct { + _ struct{} `mcp-desc:"删除安全组规则。若尚不知 id，先用 climc_secgroup_rule_list（建议带 secgroup）定位"` + + options.BaseIdOptions } func (opts *SecGroupRulesCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -50,27 +69,42 @@ func (opts *SecGroupRulesCreateOptions) Params() (jsonutils.JSONObject, error) { if err != nil { return nil, errors.Wrapf(err, "invalid rule %s", opts.RULE) } + targetType := opts.TargetType + cidr := "" + if rule.IPNet != nil { + cidr = rule.IPNet.String() + } + if len(opts.IpSet) > 0 { + targetType = "ip_set" + cidr = opts.IpSet + } + if len(targetType) == 0 { + targetType = "cidr" + } return jsonutils.Marshal(map[string]interface{}{ "direction": rule.Direction, "action": rule.Action, "protocol": rule.Protocol, - "cidr": rule.IPNet.String(), + "cidr": cidr, "ports": rule.GetPortsString(), "priority": opts.Priority, "description": opts.Desc, "secgroup_id": opts.SECGROUP, + "target_type": targetType, }), nil } type SecGroupRulesUpdateOptions struct { options.BaseIdOptions - Name string `help:"New name of rule"` - Priority int64 `help:"priority of Rule"` - Protocol string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"` - Ports string `help:"Ports of rule"` - Cidr string `help:"Cidr of rule"` - Action string `help:"filter Actin of rule" choices:"allow|deny"` - Desc string `help:"Description" metavar:"Description"` + Name string `help:"New name of rule"` + Priority int64 `help:"priority of Rule"` + Protocol string `help:"Protocol of rule" choices:"any|tcp|udp|icmp"` + Ports string `help:"Ports of rule"` + Cidr string `help:"Cidr of rule"` + IpSet string `help:"Ip set ID or name when target type is ip_set"` + TargetType string `help:"Target type" choices:"cidr|ip_set"` + Action string `help:"filter Action of rule" choices:"allow|deny"` + Desc string `help:"Description" metavar:"Description"` } func (opts *SecGroupRulesUpdateOptions) Params() (jsonutils.JSONObject, error) { @@ -93,6 +127,12 @@ func (opts *SecGroupRulesUpdateOptions) Params() (jsonutils.JSONObject, error) { if len(opts.Cidr) > 0 { params.Add(jsonutils.NewString(opts.Cidr), "cidr") } + if len(opts.IpSet) > 0 { + params.Add(jsonutils.NewString(opts.IpSet), "cidr") + params.Add(jsonutils.NewString("ip_set"), "target_type") + } else if len(opts.TargetType) > 0 { + params.Add(jsonutils.NewString(opts.TargetType), "target_type") + } if len(opts.Action) > 0 { params.Add(jsonutils.NewString(opts.Action), "action") } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/server_pod.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/server_pod.go index 50c7bf44b..2f037498d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/server_pod.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/server_pod.go @@ -34,7 +34,7 @@ type PodCreateOptions struct { ServerCreateCommonConfig `"disk->nargs":"*"` MEM string `help:"Memory size MB" metavar:"MEM" json:"-"` VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` - AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` + AllowDelete *bool `help:"Allow deleting the server (disable_delete=false)" json:"-"` //PortMapping []string `help:"Port mapping of the pod and the format is: host_port=8080,port=80,protocol=,host_port_range=-" short-token:"p"` Arch string `help:"image arch" choices:"aarch64|x86_64"` AutoStart bool `help:"Auto start server after it is created"` diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/servers.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/servers.go index bd0cb633d..7a613e90d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/servers.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/servers.go @@ -38,27 +38,29 @@ import ( var ErrEmtptyUpdate = errors.Error("No valid update data") type ServerListOptions struct { - Zone string `help:"Zone ID or Name"` + _ struct{} `mcp-desc:"列出虚机。操作（启停/重启/删除/重置密码）前用本工具取 id，取到后立刻调用对应操作工具，不要只查询就结束"` + + Zone string `help:"Zone ID or Name" mcp:"true"` Wire string `help:"Wire ID or Name"` - Network string `help:"Network ID or Name"` + Network string `help:"Network ID or Name" mcp:"true"` Disk string `help:"Disk ID or Name"` - Host string `help:"Host ID or Name"` + Host string `help:"Host ID or Name" mcp:"true"` Baremetal *bool `help:"Show baremetal servers"` Gpu *bool `help:"Show gpu servers"` Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` - Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|pod|baremetal|aliyun|azure|aws|huawei|ucloud|volcengine|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud|sangfor|zettakit|uis|cnware"` - Region string `help:"Show servers in cloudregion"` + Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|pod|baremetal|aliyun|apsara|azure|aws|huawei|ucloud|volcengine|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud|oracle|sangfor|zettakit|uis|cas|cnware|rockbase" mcp:"true"` + Region string `help:"Show servers in cloudregion" mcp:"true"` WithEip *bool `help:"Show Servers with EIP"` WithoutEip *bool `help:"Show Servers without EIP"` - OsType string `help:"OS Type" choices:"linux|windows|vmware"` - Vpc []string `help:"Vpc id or name"` + OsType string `help:"OS Type" choices:"linux|windows|vmware" mcp:"true"` + Vpc []string `help:"Vpc id or name" mcp:"true"` UsableServerForEip string `help:"Eip id or name"` WithoutUserMeta *bool `help:"Show Servers without user metadata"` EipAssociable *bool `help:"Show Servers can associate with eip"` HostSn string `help:"Host SN"` - IpAddr string `help:"Fileter by ip"` - IpAddrs []string `help:"Fileter by ips"` + IpAddr string `help:"Filter by ip" mcp:"true"` + IpAddrs []string `help:"Filter by ips"` OrderByDisk string `help:"Order by disk size" choices:"asc|desc"` OrderByOsDist string `help:"Order by os distribution" choices:"asc|desc"` @@ -81,6 +83,7 @@ type ServerListOptions struct { WithUserMeta *bool `help:"filter by user metadata" negative:"without_user_meta"` + IsolateDeviceId string `help:"filter guest with isolated device id" json:"isolate_device_id"` WithHost *bool `help:"filter guest with host or not" negative:"without_host"` SnapshotpolicyId string `help:"filter guest with snapshotpolicy or not" json:"snapshotpolicy_id"` BindingDisksSnapshotpolicy *bool `help:"filter guest with disks binding snapshotpolicy or not" negative:"no-binding-disks-snapshotpolicy" json:"binding_disks_snapshotpolicy"` @@ -120,7 +123,13 @@ type ServerSSHLoginOptions struct { type ServerConvertToKvmOptions struct { ServerIdsOptions - PreferHost string `help:"Perfer host id or name" json:"prefer_host"` + PreferHost string `help:"Prefer host id or name" json:"prefer_host"` + SysDiskBackend string `help:"Prefer disk backend for system disk, e.g. local/lvm/slvm/nfs/rbd" json:"sys_disk_backend"` + SysPreferStorage string `help:"Prefer storage id or name for system disk" json:"sys_prefer_storage"` + SysDiskMedium string `help:"Prefer medium for system disk, e.g. rotate/ssd/hybrid" json:"sys_disk_medium"` + DataDiskBackend string `help:"Prefer disk backend for data disks, e.g. local/lvm/slvm/nfs/rbd" json:"data_disk_backend"` + DataPreferStorage string `help:"Prefer storage id or name for data disks" json:"data_prefer_storage"` + DataDiskMedium string `help:"Prefer medium for data disks, e.g. rotate/ssd/hybrid" json:"data_disk_medium"` } func (o *ServerConvertToKvmOptions) Params() (jsonutils.JSONObject, error) { @@ -128,6 +137,8 @@ func (o *ServerConvertToKvmOptions) Params() (jsonutils.JSONObject, error) { } type ServerStartOptions struct { + _ struct{} `mcp-desc:"启动虚机（真正执行）。未知 id 时先 climc_server_list；拿到 id 后立刻调用，仅查询不算完成"` + ServerIdsOptions QemuVersion string `help:"prefer qemu version" json:"qemu_version"` @@ -138,6 +149,16 @@ func (o *ServerStartOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(o), nil } +// ServerStartRescueOptions 与启动参数相同，单独包装以免继承 server-start 的 mcp-desc。 +type ServerStartRescueOptions struct { + ServerStartOptions +} + +// ServerStopRescueOptions 与启动参数相同，单独包装以免继承 server-start 的 mcp-desc。 +type ServerStopRescueOptions struct { + ServerStartOptions +} + type ServerIdsOptions struct { ID []string `help:"ID of servers to operate" metavar:"SERVER" json:"-"` } @@ -194,6 +215,8 @@ func (o *ServerCreateBackupOptions) Params() (jsonutils.JSONObject, error) { } type ServerShowOptions struct { + _ struct{} `mcp-desc:"查询单台虚拟机详情。ID 可用 climc_server_list 返回的 id/name；查看状态、配置、IP 等信息时使用"` + options.BaseShowOptions `id->help:"ID or name of the server"` } @@ -249,22 +272,22 @@ func ParseServerDeployInfoList(list []string) ([]*computeapi.DeployConfig, error } type ServerCreateCommonConfig struct { - Manager string `help:"Preferred cloudprovider where virtual server should bd created" json:"prefer_manager"` - Region string `help:"Preferred region where virtual server should be created" json:"prefer_region"` - Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone"` + Manager string `help:"Preferred cloudprovider where virtual server should be created" json:"prefer_manager" mcp:"true"` + Region string `help:"Preferred region where virtual server should be created" json:"prefer_region" mcp:"true"` + Zone string `help:"Preferred zone where virtual server should be created" json:"prefer_zone" mcp:"true"` Zones []string `help:"Preferred zones where virtual server should be created" json:"prefer_zones"` Wire string `help:"Preferred wire where virtual server should be created" json:"prefer_wire"` - Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` + Host string `help:"Preferred host where virtual server should be created" json:"prefer_host" mcp:"true"` ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` Schedtag []string `help:"Schedule policy, key = aggregate name, value = require|exclude|prefer|avoid" metavar:""` - Net []string `help:"Network descriptions" metavar:"NETWORK"` + Net []string `help:"Network descriptions；可省略，省略时 MCP 自动用 random（等价 nets:[{exit:false}]）走自动调度" metavar:"NETWORK" mcp:"true"` NetPortMapping []string `help:"Network port mapping, e.g. 'index=0,port=80,host_port=8080,protocol=,host_port_range=-,remote_ips=x.x.x.x|y.y.y.y'" short-token:"p"` NetSchedtag []string `help:"Network schedtag description, e.g. '0::'"` IsolatedDevice []string `help:"Isolated device model or ID" metavar:"ISOLATED_DEVICE"` - Project string `help:"'Owner project ID or Name" json:"tenant"` + Project string `help:"Owner project ID or Name" json:"tenant" mcp:"true"` User string `help:"Owner user ID or Name"` - Count int `help:"Create multiple simultaneously" default:"1"` + Count int `help:"Create multiple simultaneously" default:"1" mcp:"true"` Disk []string `help:" Disk descriptions size: 500M, 10G @@ -273,20 +296,21 @@ type ServerCreateCommonConfig struct { fs_features: casefold format: qcow2, raw, docker, iso, vmdk, vmdkflatver1, vmdkflatver2, vmdkflat, vmdksparse, vmdksparsever1, vmdksparsever2, vmdksesparse, vhd driver: virtio, ide, scsi, sata, pvscsi - cache_mod: writeback, none, writethrough + cache_mode: writeback, none, writethrough medium: rotate, ssd, hybrid disk_type: sys, data mountpoint: /, /opt - storage_type: local, rbd, nas, nfs + storage_type/backend: local, rbd, nas, nfs；公有云用 cloud-region-capability 的 storage_types2（如 cloud_essd） snapshot_id: use snapshot-list get snapshot id disk_id: use disk-list get disk id storage_id: use storage-list get storage id - image_id: use image-list get image id + image_id: use image-list/cached-image-list get image id for example: --disk 'image_id=c2be02a4-7ff2-43e6-8a00-a489e04d2d6f,size=10G,driver=ide,storage_type=rbd,auto_delete=true' + --disk 'size=40g,image=,backend=cloud_essd' --disk 'size=500M' - --disk 'snpahost_id=1ceb8c6d-6571-451d-8957-4bd3a871af85' - " nargs:"+"` + --disk 'snapshot_id=1ceb8c6d-6571-451d-8957-4bd3a871af85' + " mcp:"true"` DiskSchedtag []string `help:"Disk schedtag description, e.g. '0::'"` } @@ -371,11 +395,12 @@ func (o ServerCreateCommonConfig) Data() (*computeapi.ServerConfigs, error) { type ServerConfigs struct { ServerCreateCommonConfig - Hypervisor string `help:"Hypervisor type" choices:"kvm|pod|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|volcengine|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|sangfor|zettakit|uis"` + Hypervisor string `help:"Hypervisor type" choices:"kvm|pod|esxi|baremetal|container|aliyun|apsara|azure|qcloud|aws|huawei|openstack|ucloud|volcengine|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud|oracle|sangfor|zettakit|uis|cas|cnware|rockbase" mcp:"true"` Backup bool `help:"Create server with backup server"` - BackupHost string `help:"Perfered host where virtual backup server should be created"` + BackupHost string `help:"Preferred host where virtual backup server should be created"` AutoSwitchToBackupOnHostDown bool `help:"Auto switch to backup server on host down"` Daemon *bool `help:"Set as a daemon server" json:"is_daemon"` + QemuVersion string `help:"specific server guest start version" json:"qemu_version"` RaidConfig []string `help:"Baremetal raid config" json:"-"` RootDiskMatcher string `help:"Baremetal root disk matcher, e.g. 'device=/dev/sdb' 'size=900G' 'size_start=800G,size_end=900G'" json:"-"` @@ -390,6 +415,7 @@ func (o ServerConfigs) Data() (*computeapi.ServerConfigs, error) { data.PreferBackupHost = o.BackupHost data.IsDaemon = o.Daemon data.Hypervisor = o.Hypervisor + data.QemuVersion = o.QemuVersion if len(o.RaidConfig) > 0 { // if data.Hypervisor != "baremetal" { // return nil, fmt.Errorf("RaidConfig is applicable to baremetal ONLY") @@ -414,10 +440,10 @@ func (o ServerConfigs) Data() (*computeapi.ServerConfigs, error) { type ServerCloneOptions struct { SOURCE string `help:"Source server id or name" json:"-"` - TARGET_NAME string `help:"Name of newly server" json:"name"` + TARGET_NAME string `help:"Name of the new server" json:"name"` AutoStart bool `help:"Auto start server after it is created"` - EipBw int `help:"allocate EIP with bandwidth in MB when server is created" json:"eip_bw,omitzero"` + EipBw int `help:"allocate EIP with bandwidth in Mbps when server is created" json:"eip_bw,omitzero"` EipChargeType string `help:"newly allocated EIP charge type" choices:"traffic|bandwidth" json:"eip_charge_type,omitempty"` Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` } @@ -435,64 +461,68 @@ func (o *ServerCloneOptions) Description() string { } type ServerCreateFromInstanceSnapshot struct { - InstaceSnapshotId string `help:"Instace snapshot id or name"` - NAME string `help:"Name of newly server" json:"name"` + InstaceSnapshotId string `help:"Instance snapshot id or name"` + NAME string `help:"Name of the new server" json:"name"` AutoStart bool `help:"Auto start server after it is created"` - AllowDelete bool `help:"Unlock server to allow deleting"` + AllowDelete bool `help:"Allow deleting the server (disable_delete=false)"` - EipBw int `help:"allocate EIP with bandwidth in MB when server is created" json:"eip_bw,omitzero"` - EipTxBw int `help:"allocate EIP with上行带宽 in MB when server is created" json:"eip_tx_bw,omitzero"` - EipRxBw int `help:"allocate EIP with下行带宽 in MB when server is created" json:"eip_rx_bw,omitzero"` + EipBw int `help:"allocate EIP with bandwidth in Mbps when server is created" json:"eip_bw,omitzero"` + EipTxBw int `help:"allocate EIP with outbound bandwidth in Mbps when server is created" json:"eip_tx_bw,omitzero"` + EipRxBw int `help:"allocate EIP with inbound bandwidth in Mbps when server is created" json:"eip_rx_bw,omitzero"` EipChargeType string `help:"newly allocated EIP charge type" choices:"traffic|bandwidth" json:"eip_charge_type,omitempty"` Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` } type ServerCreateOptions struct { + _ struct{} `mcp-desc:"【创建虚拟机的最终动作】自动 forecast→创建→等待 running/ready。最少：name、规格。KVM/公有云：disk 含 image+backend；CAS/UIS/SangFor：hypervisor=cas 等，系统盘仅 size+backend（dir/fs），镜像用 cdrom=ISO(cached-image id)，并传 prefer-region。超时返回 wait_pending+server_id，用 climc_server_show 续查，勿重复创建"` + ServerCreateOptionalOptions - NAME string `help:"Name of server" json:"-"` + NAME string `help:"虚拟机名称模板；默认配合 generate-name 自动去重" json:"-" mcp:"required"` } type ServerCreateOptionalOptions struct { ServerConfigs - MemSpec string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"` + MemSpec string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-" mcp:"true"` CpuSockets int `help:"Cpu sockets"` EnableMemclean bool `help:"clean guest memory after guest exit" json:"enable_memclean"` EnableTpm bool `help:"enable tpm device" json:"enable_tpm"` - Keypair string `help:"SSH Keypair"` - Password string `help:"Default user password"` - LoginAccount string `help:"Guest login account"` - Iso string `help:"ISO image ID" metavar:"IMAGE_ID" json:"cdrom"` - IsoBootIndex *int8 `help:"Iso bootindex" metavar:"IMAGE_BOOT_INDEX" json:"cdrom_boot_index"` - VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu"` - ExtraCpuCount int `help:"Extra allocate cpu count" json:"extra_cpu_count"` - InstanceType string `help:"instance flavor"` - Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl|virtio"` - Vdi string `help:"VDI protocool" choices:"vnc|spice"` - Bios string `help:"BIOS" choices:"BIOS|UEFI"` - Machine string `help:"Machine type" choices:"pc|q35"` - Desc string `help:"Description" metavar:"" json:"description"` - Boot string `help:"Boot device" metavar:"" choices:"disk|cdrom" json:"-"` - EnableCloudInit bool `help:"Enable cloud-init service"` - NoAccountInit *bool `help:"Not reset account password"` - AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` - ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"" choices:"stop|terminate|stop_release_gpu"` - AutoStart bool `help:"Auto start server after it is created"` - Deploy []string `help:"Specify deploy files in virtual server file system" json:"-"` - DeployTelegraf bool `help:"Deploy telegraf agent if guest os is supported"` - Group []string `help:"Group ID or Name of virtual server"` - System bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` - TaskNotify *bool `help:"Setup task notify" json:"-"` - FakeCreate *bool `help:"Fake create server"` - DryRun *bool `help:"Dry run to test scheduler" json:"-"` - UserDataFile string `help:"user_data file path" json:"-"` - InstanceSnapshot string `help:"instance snapshot" json:"instance_snapshot"` - Secgroups []string `help:"secgroups" json:"secgroups"` - NetworkTags []string `help:"GCP network tags, google only; when set, secgroups can be omitted" json:"network_tags"` - - OsType string `help:"os type, e.g. Linux, Windows, etc."` + Keypair string `help:"SSH Keypair" mcp:"true"` + Password string `help:"Default user password" mcp:"true"` + LoginAccount string `help:"Guest login account" mcp:"true"` + Iso string `help:"ISO image ID" metavar:"IMAGE_ID" json:"cdrom" mcp:"true"` + IsoBootIndex *int8 `help:"Iso bootindex" metavar:"IMAGE_BOOT_INDEX" json:"cdrom_boot_index"` + VcpuCount int `help:"#CPU cores of VM server, default 1" default:"1" metavar:"" json:"vcpu_count" token:"ncpu" mcp:"true"` + ExtraCpuCount int `help:"Extra allocate cpu count" json:"extra_cpu_count"` + InstanceType string `help:"instance flavor" mcp:"true"` + Vga string `help:"VGA driver" choices:"std|vmware|cirrus|qxl|virtio"` + Vdi string `help:"VDI protocol" choices:"vnc|spice"` + Bios string `help:"BIOS" choices:"BIOS|UEFI" mcp:"true"` + Machine string `help:"Machine type" choices:"pc|q35"` + Desc string `help:"Description" metavar:"" json:"description"` + Boot string `help:"Boot device" metavar:"" choices:"disk|cdrom" json:"-"` + EnableCloudInit bool `help:"Enable cloud-init service"` + NoAccountInit *bool `help:"Not reset account password"` + AllowDelete *bool `help:"Allow deleting the server (disable_delete=false)" json:"-"` + ShutdownBehavior string `help:"Behavior after VM server shutdown" metavar:"" choices:"stop|terminate|stop_release_gpu"` + AutoStart bool `help:"Auto start server after it is created" mcp:"true"` + Deploy []string `help:"Specify deploy files in virtual server file system" json:"-"` + DeployTelegraf bool `help:"Deploy telegraf agent if guest os is supported"` + Group []string `help:"Group ID or Name of virtual server"` + System bool `help:"Create a system VM, sysadmin ONLY option" json:"is_system"` + TaskNotify *bool `help:"Setup task notify" json:"-"` + FakeCreate *bool `help:"Fake create server"` + DryRun *bool `help:"Dry run to validate create params (not preschedule)；MCP 创建会自动调 scheduler-forecast 预调度，一般无需手动传" json:"-" mcp:"true"` + UserDataFile string `help:"user_data file path" json:"-"` + InstanceSnapshot string `help:"instance snapshot" json:"instance_snapshot"` + Secgroups []string `help:"Security group IDs or names" json:"secgroups"` + NetworkTags []string `help:"GCP network tags, google only; when set, secgroups can be omitted" json:"network_tags"` + DisableSrcIpCheck *bool `help:"Disable source IP check" json:"-"` + DisableSrcMacCheck *bool `help:"Disable source MAC check" json:"-"` + + OsType string `help:"os type, e.g. Linux, Windows, etc." mcp:"true"` Duration string `help:"valid duration of the server, e.g. 1H, 1D, 1W, 1M, 1Y, ADMIN ONLY option"` AutoRenew bool `help:"auto renew for prepaid server"` @@ -507,19 +537,19 @@ type ServerCreateOptionalOptions struct { KickstartMaxRetries int `help:"Kickstart max retries" default:"3" json:"-"` KickstartTimeoutMinutes int `help:"Kickstart timeout in minutes" default:"60" json:"-"` - GenerateName bool `help:"name is generated by pattern" json:"-"` + GenerateName bool `help:"name is generated by pattern" json:"-" mcp:"true"` - EipBw int `help:"allocate EIP with bandwidth in MB when server is created" json:"eip_bw,omitzero"` - EipTxBw int `help:"allocate EIP with上行带宽 in MB when server is created" json:"eip_tx_bw,omitzero"` - EipRxBw int `help:"allocate EIP with下行带宽 in MB when server is created" json:"eip_rx_bw,omitzero"` - EipBgpType string `help:"desired BGP type of newly alloated EIP" json:"eip_bgp_type,omitzero"` + EipBw int `help:"allocate EIP with bandwidth in Mbps when server is created" json:"eip_bw,omitzero"` + EipTxBw int `help:"allocate EIP with outbound bandwidth in Mbps when server is created" json:"eip_tx_bw,omitzero"` + EipRxBw int `help:"allocate EIP with inbound bandwidth in Mbps when server is created" json:"eip_rx_bw,omitzero"` + EipBgpType string `help:"desired BGP type of newly allocated EIP" json:"eip_bgp_type,omitzero"` EipChargeType string `help:"newly allocated EIP charge type" choices:"traffic|bandwidth" json:"eip_charge_type,omitempty"` Eip string `help:"associate with an existing EIP when server is created" json:"eip,omitempty"` - PublicIpBw int `help:"associate public ip with bandwidth in MB where server is created" json:"public_ip_bw,omitzero"` + PublicIpBw int `help:"associate public ip with bandwidth in Mbps when server is created" json:"public_ip_bw,omitzero"` PublicIpChargeType string `help:"newly allocated public ip charge type" choices:"traffic|bandwidth" json:"public_ip_charge_type,omitempty"` - GuestImageID string `help:"create from guest image, need to specify the guest image id"` + GuestImageID string `help:"create from guest image, need to specify the guest image id" mcp:"true"` EncryptKey string `help:"encryption key"` Tags []string `help:"tags in the form of key=value"` @@ -625,6 +655,14 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre if len(opts.EncryptKey) > 0 { params.EncryptKeyId = &opts.EncryptKey } + if opts.DisableSrcMacCheck != nil && *opts.DisableSrcMacCheck { + var srcMacCheck = false + params.SrcMacCheck = &srcMacCheck + } + if opts.DisableSrcIpCheck != nil && *opts.DisableSrcIpCheck { + var srcIpCheck = false + params.SrcIpCheck = &srcIpCheck + } if regutils.MatchSize(opts.MemSpec) { memSize, err := fileutils.GetSizeMb(opts.MemSpec, 'M', 1024) @@ -759,9 +797,12 @@ func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error) } type ServerStopOptions struct { + _ struct{} `mcp-desc:"停止虚机（真正执行）。未知 id 时先 climc_server_list；拿到 id 后立刻调用，仅查询不算完成"` + ID []string `help:"ID or Name of server" json:"-"` - Force *bool `help:"Stop server forcefully" json:"is_force"` - StopCharging *bool `help:"Stop charging when server stop"` + Force *bool `help:"Stop server forcefully" json:"is_force" mcp:"true"` + StopCharging *bool `help:"Stop charging when server stop" mcp:"true"` + TimeoutSecs *int `help:"Guest stop timeout seconds" json:"timeout_secs" mcp:"true"` } func (o *ServerStopOptions) GetIds() []string { @@ -820,12 +861,14 @@ func (opts *ServerUpdateOptions) Params() (jsonutils.JSONObject, error) { } type ServerDeleteOptions struct { + _ struct{} `mcp-desc:"删除虚机（真正执行；若 disable_delete 会先自动解锁）。未知 id 时先 climc_server_list；拿到 id 后立刻调用，仅查询不算完成"` + ServerIdsOptions - OverridePendingDelete *bool `help:"Delete server directly instead of pending delete" short-token:"f"` - DeleteSnapshots *bool `help:"Delete server snapshots"` - DeleteDisks *bool `help:"Delete server disks"` - DeleteEip *bool `help:"Delete eip"` - DeleteBastionServer *bool `help:"Remove from bastion host"` + OverridePendingDelete *bool `help:"Delete server directly instead of pending delete" short-token:"f" mcp:"true"` + DeleteSnapshots *bool `help:"Delete server snapshots" mcp:"true"` + DeleteDisks *bool `help:"Delete server disks" mcp:"true"` + DeleteEip *bool `help:"Delete eip" mcp:"true"` + DeleteBastionServer *bool `help:"Remove from bastion host" mcp:"true"` } func (o *ServerDeleteOptions) QueryParams() (jsonutils.JSONObject, error) { @@ -997,6 +1040,9 @@ func (o *ServerKickstartCompleteOptions) Params() (jsonutils.JSONObject, error) return options.StructToParams(o) } +// ServerMonitorOptions 不注册为 MCP tool（无 mcp-desc tag）：向虚机发送任意 +// HMP/QMP 命令（如 pmemsave/migrate）超出"监控"语义，经 LLM 通道可被提示注入 +// 无感知触发，禁止 AI 调用。仅保留 climc 命令行与受控 API 使用。 type ServerMonitorOptions struct { ServerIdOptions @@ -1057,12 +1103,14 @@ func (o *ServerQgaGetNetwork) Params() (jsonutils.JSONObject, error) { } type ServerSetPasswordOptions struct { + _ struct{} `mcp-desc:"重置虚机密码（真正执行）。未知 id 时先 climc_server_list；拿到 id 后立刻调用，仅查询不算完成"` + ServerIdOptions - Username string `help:"Which user to set password" json:"username"` - Password string `help:"Password content" json:"password"` - ResetPassword bool `help:"Force reset password"` - AutoStart bool `help:"Auto start server after reset password"` + Username string `help:"Which user to set password" json:"username" mcp:"true"` + Password string `help:"Password content" json:"password" mcp:"required"` + ResetPassword bool `help:"Force reset password" mcp:"true"` + AutoStart bool `help:"Auto start server after reset password" mcp:"true"` } func (o *ServerSetPasswordOptions) Params() (jsonutils.JSONObject, error) { @@ -1138,7 +1186,7 @@ func (o *ServerSaveGuestImageOptions) Description() string { type ServerChangeOwnerOptions struct { ID string `help:"Server to change owner" json:"-"` - PROJECT string `help:"Project ID or change" json:"tenant"` + PROJECT string `help:"Target project ID or name" json:"tenant"` } func (o *ServerChangeOwnerOptions) GetId() string { @@ -1150,14 +1198,16 @@ func (o *ServerChangeOwnerOptions) Params() (jsonutils.JSONObject, error) { } type ServerRebuildRootOptions struct { + _ struct{} `mcp-desc:"重装系统盘。ID 为虚机 id；可选 image、password、auto-start。先 climc_server_list / climc_image_list 定位"` + ID string `help:"Server to rebuild root" json:"-"` - ImageId string `help:"New root Image template ID" json:"image_id" token:"image"` - Keypair string `help:"ssh Keypair used for login"` + ImageId string `help:"New root Image template ID" json:"image_id" token:"image" mcp:"true"` + Keypair string `help:"ssh Keypair used for login" mcp:"true"` Password string `help:"Default user password"` - LoginAccount string `help:"Guest login account"` - NoAccountInit *bool `help:"Not reset account password"` - AutoStart *bool `help:"Auto start server after it is created"` - AllDisks *bool `help:"Rebuild all disks including data disks"` + LoginAccount string `help:"Guest login account" mcp:"true"` + NoAccountInit *bool `help:"Not reset account password" mcp:"true"` + AutoStart *bool `help:"Auto start server after it is created" mcp:"true"` + AllDisks *bool `help:"Rebuild all disks including data disks" mcp:"true"` UserData string `hlep:"user data scripts"` } @@ -1170,12 +1220,10 @@ func (o *ServerRebuildRootOptions) Params() (jsonutils.JSONObject, error) { if err != nil { return nil, err } + params.Set("reset_password", jsonutils.JSONTrue) if o.NoAccountInit != nil && *o.NoAccountInit { params.Add(jsonutils.JSONFalse, "reset_password") } - if o.Password != "" { - params.Set("reset_password", jsonutils.JSONTrue) - } return params, nil } @@ -1184,16 +1232,18 @@ func (o *ServerRebuildRootOptions) Description() string { } type ServerChangeConfigOptions struct { + _ struct{} `mcp-desc:"调整虚机配置（CPU/内存/套餐）。ID 用 climc_server_list；传 ncpu/vmem 或 instance-type。改配前确认状态允许"` + ServerIdOptions - VcpuCount *int `help:"New number of Virtual CPU cores" json:"vcpu_count" token:"ncpu"` - ExtraCpuCount *int `help:"Extra allocate cpu count" json:"extra_cpu_count"` - CpuSockets *int `help:"Cpu sockets"` - VmemSize string `help:"New memory size" json:"vmem_size" token:"vmem"` - Disk []string `help:"Data disk description, from the 1st data disk to the last one, empty string if no change for this data disk"` + VcpuCount *int `help:"New number of Virtual CPU cores" json:"vcpu_count" token:"ncpu" mcp:"true"` + ExtraCpuCount *int `help:"Extra allocate cpu count" json:"extra_cpu_count" mcp:"true"` + CpuSockets *int `help:"Cpu sockets" mcp:"true"` + VmemSize string `help:"New memory size" json:"vmem_size" token:"vmem" mcp:"true"` + Disk []string `help:"Data disk description, from the 1st data disk to the last one, empty string if no change for this data disk" mcp:"true"` - InstanceType string `help:"Instance Type, e.g. S2.SMALL2 for qcloud"` + InstanceType string `help:"Instance Type, e.g. S2.SMALL2 for qcloud" mcp:"true"` - ForceStop *bool `help:"Force stop the server before changing config" json:"force_stop"` + ForceStop *bool `help:"Force stop the server before changing config" json:"force_stop" mcp:"true"` ResetTrafficLimits []string `help:"reset traffic limits, mac,rx,tx"` SetTrafficLimits []string `help:"set traffic limits, mac,rx,tx"` @@ -1300,8 +1350,11 @@ func (o *ServerResetOptions) Params() (jsonutils.JSONObject, error) { } type ServerRestartOptions struct { - ID []string `help:"ID of servers to operate" metavar:"SERVER" json:"-"` - IsForce *bool `help:"Force reset or not; default false" json:"is_force"` + _ struct{} `mcp-desc:"重启虚机（真正执行）。未知 id 时先 climc_server_list；拿到 id 后立刻调用，仅查询不算完成"` + + ID []string `help:"ID of servers to operate" metavar:"SERVER" json:"-"` + IsForce *bool `help:"Force reset or not; default false" json:"is_force" mcp:"true"` + TimeoutSecs *int `help:"Guest restart stop guest timeout" json:"timeout_secs" mcp:"true"` } func (o *ServerRestartOptions) GetIds() []string { @@ -1350,7 +1403,7 @@ type ServerLiveMigrateOptions struct { SkipKernelCheck *bool `help:"Skip target kernel version check" json:"skip_kernel_check"` EnableTLS *bool `help:"Enable tls migration" json:"enable_tls"` QuicklyFinish *bool `help:"quickly finish, fix downtime after a few rounds of memory synchronization"` - MaxBandwidthMb *int64 `help:"live migrate downtime, unit MB"` + MaxBandwidthMb *int64 `help:"live migrate max bandwidth, unit MB"` KeepDestGuestOnFailed *bool `help:"do not delete dest guest on migrate failed, for debug"` } @@ -1365,7 +1418,7 @@ func (o *ServerLiveMigrateOptions) Params() (jsonutils.JSONObject, error) { type ServerSetLiveMigrateParamsOptions struct { ID string `help:"ID of server" json:"-"` - MaxBandwidthMB *int64 `help:"live migrate downtime, unit MB"` + MaxBandwidthMB *int64 `help:"live migrate max bandwidth, unit MB"` DowntimeLimitMS *int64 `help:"live migrate downtime limit"` } @@ -1407,8 +1460,10 @@ func (opts *ServerBatchMetadataOptions) Params() (jsonutils.JSONObject, error) { } type ServerAssociateEipOptions struct { + _ struct{} `mcp-desc:"将 EIP 绑定到虚机。需虚机 ID 与 EIP（climc_eip_list 的 id/name）"` + ServerIdOptions - EIP string `help:"ID or name of EIP to associate"` + EIP string `help:"ID or name of EIP to associate" mcp:"required"` } func (o *ServerAssociateEipOptions) Params() (jsonutils.JSONObject, error) { @@ -1656,7 +1711,9 @@ func (o *ServerScreenDumpOptions) Params() (jsonutils.JSONObject, error) { type ServerChangeDiskDriverOptions struct { ServerIdOptions DISK_ID string - DRIVER string `help:"Driver of vDisk" choices:"virtio|ide|sata|scsi|pvscsi"` + Driver string `help:"Driver of vDisk" choices:"virtio|ide|sata|scsi|pvscsi"` + Cache string `help:"Cache mode of vDisk" choices:"writethrough|none|writeback|directsync"` + Aio string `help:"Asynchronous IO mode of vDisk" choices:"native|threads"` } func (o *ServerChangeDiskDriverOptions) Params() (jsonutils.JSONObject, error) { @@ -1673,6 +1730,18 @@ func (o *ServerSetNetworkNumQueues) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(o), nil } +type ServerSetIsoOptions struct { + ServerIdOptions + + CDROM_ORDINAL int64 `json:"cdrom_ordinal" help:"cdrom ordinal"` + ImageId string `help:"Iso image id, eject on image id empty"` + BootIndex *int8 `help:"Iso boot index"` +} + +func (o *ServerSetIsoOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(o), nil +} + type ServerIsoOptions struct { ServerIdOptions Ordinal int `help:"server iso ordinal, default 0"` diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/serverskus.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/serverskus.go index bd990beba..6a9c84c8d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/serverskus.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/serverskus.go @@ -15,25 +15,34 @@ package compute import ( + "fmt" + "regexp" + "strconv" + "strings" + "yunion.io/x/jsonutils" baseoptions "yunion.io/x/onecloud/pkg/mcclient/options" ) type ServerSkusListOptions struct { + _ struct{} `mcp-desc:"【创建流程中的中间步骤】查规格。口语 2c2g 用 spec；公有云须 provider+cloudregion。取 name 作 instance-type 后立刻 climc_server_create"` + baseoptions.BaseListOptions - Cloudregion string `help:"region Id or name"` - Usable bool `help:"Filter usable sku"` - Zone string `help:"zone Id or name"` - City *string `help:"city name,eg. BeiJing"` - Cpu *int `help:"Cpu core count" json:"cpu_core_count"` - Mem *int `help:"Memory size in MB" json:"memory_size_mb"` - Name string `help:"Name of Sku"` - PostpaidStatus string `help:"Postpaid status" choices:"soldout|available"` - PrepaidStatus string `help:"Prepaid status" choices:"soldout|available"` - CpuArch string `help:"Cpu Arch" choices:"x86|arm"` - Enabled *bool `help:"Filter enabled skus"` - Distinct bool `help:"distinct sku by name"` + Cloudregion string `help:"region Id or name" mcp:"true"` + Usable bool `help:"Filter usable sku" mcp:"true"` + Zone string `help:"zone Id or name" mcp:"true"` + City *string `help:"city name,eg. BeiJing"` + // Spec 口语规格，如 2c2g / 2核2G / 4C8G；会解析为 cpu_core_count + memory_size_mb + Spec string `help:"Human spec like 2c2g / 2核2G / 4C8G; expands to cpu+mem(MB)" json:"-" mcp:"true"` + Cpu *int `help:"Cpu core count；用户说2核时传2。也可改用 --spec 2c2g" json:"cpu_core_count" mcp:"true"` + Mem *int `help:"Memory size in MB；2G=2048。也可改用 --spec 2c2g" json:"memory_size_mb" mcp:"true"` + Name string `help:"Name of Sku" mcp:"true"` + PostpaidStatus string `help:"Postpaid status；创建优先 available" choices:"soldout|available" mcp:"true"` + PrepaidStatus string `help:"Prepaid status" choices:"soldout|available"` + CpuArch string `help:"Cpu Arch" choices:"x86|aarch64" mcp:"true"` + Enabled *bool `help:"Filter enabled skus" mcp:"true"` + Distinct bool `help:"distinct sku by name"` OrderByTotalGuestCount string } @@ -42,9 +51,70 @@ func (opts *ServerSkusListOptions) GetId() string { } func (opts *ServerSkusListOptions) Params() (jsonutils.JSONObject, error) { + if err := opts.applySpec(); err != nil { + return nil, err + } return baseoptions.ListStructToParams(opts) } +func (opts *ServerSkusListOptions) applySpec() error { + spec := strings.TrimSpace(opts.Spec) + if spec == "" { + return nil + } + cpu, memMB, err := ParseSkuSpec(spec) + if err != nil { + return err + } + if opts.Cpu == nil { + opts.Cpu = &cpu + } + if opts.Mem == nil { + opts.Mem = &memMB + } + return nil +} + +// skuSpecPatterns 支持：2c2g、2C2G、4c8g、2核2G、2核2g、2vcpu2gb、2c/2g、2x2g +var skuSpecPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)^\s*(\d+)\s*[cC]\s*[xX/]?\s*(\d+)\s*([gGmM])b?\s*$`), + regexp.MustCompile(`(?i)^\s*(\d+)\s*[xX/]\s*(\d+)\s*([gGmM])b?\s*$`), + regexp.MustCompile(`(?i)^\s*(\d+)\s*核\s*(\d+)\s*([gGmM])b?\s*$`), + regexp.MustCompile(`(?i)^\s*(\d+)\s*v?cpu\s*[xX/]?\s*(\d+)\s*([gGmM])b?\s*$`), +} + +// ParseSkuSpec 将口语规格解析为 CPU 核数与内存 MB。 +func ParseSkuSpec(spec string) (cpu int, memMB int, err error) { + s := strings.TrimSpace(spec) + if s == "" { + return 0, 0, fmt.Errorf("empty sku spec") + } + for _, re := range skuSpecPatterns { + m := re.FindStringSubmatch(s) + if m == nil { + continue + } + cpu, err = strconv.Atoi(m[1]) + if err != nil || cpu <= 0 { + return 0, 0, fmt.Errorf("invalid cpu in spec %q", spec) + } + mem, err := strconv.Atoi(m[2]) + if err != nil || mem <= 0 { + return 0, 0, fmt.Errorf("invalid memory in spec %q", spec) + } + switch strings.ToLower(m[3]) { + case "g": + memMB = mem * 1024 + case "m": + memMB = mem + default: + return 0, 0, fmt.Errorf("unsupported memory unit in spec %q", spec) + } + return cpu, memMB, nil + } + return 0, 0, fmt.Errorf("unrecognized sku spec %q, expect like 2c2g or 2核2G", spec) +} + type ServerSkusIdOptions struct { ID string `help:"ID or Name of SKU to show"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storage.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storage.go index 486f4141f..86367e1e3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storage.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storage.go @@ -24,18 +24,20 @@ import ( ) type StorageListOptions struct { + _ struct{} `mcp-desc:"【创建流程中的中间步骤】本工具不能完成创建。创建场景下查完存储后立刻 climc_server_create。严禁只调用本工具后就停止"` + options.BaseListOptions Share *bool `help:"Share storage list"` Local *bool `help:"Local storage list"` - Usable *bool `help:"Usable storage list"` - Zone string `help:"List storages in zone" json:"-"` - Region string `help:"List storages in region"` + Usable *bool `help:"Usable storage list" mcp:"true"` + Zone string `help:"List storages in zone" json:"-" mcp:"true"` + Region string `help:"List storages in region" mcp:"true"` Schedtag string `help:"filter storage by schedtag"` - HostId string `help:"filter storages which attached the specified host"` + HostId string `help:"filter storages which attached the specified host" mcp:"true"` HostSchedtagId string `help:"filter storage by host schedtag"` - ImageId string `help:"filter storage by image"` + ImageId string `help:"filter storage by image" mcp:"true"` IsBaremetal *bool `help:"Baremetal storage list"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storagecache.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storagecache.go index 7810d87db..f68c506b5 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storagecache.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/storagecache.go @@ -34,7 +34,7 @@ type StorageCacheImageActionOptions struct { options.BaseIdOptions IMAGE string `help:"ID or name of image"` Force bool `help:"Force refresh cache, even if the image exists in cache"` - Format string `help:"Image force" choices:"iso|vmdk|qcow2|vhd|tgz"` + Format string `help:"Image format" choices:"iso|vmdk|qcow2|vhd|tgz"` } func (opts *StorageCacheImageActionOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/vpc.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/vpc.go index e3f32992e..c4a130b3d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/vpc.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/compute/vpc.go @@ -23,15 +23,17 @@ import ( ) type VpcListOptions struct { + _ struct{} `mcp-desc:"【创建流程中的中间步骤】本工具不能完成创建。创建场景下查完 VPC 后继续 climc_network_list，最后 climc_server_create。严禁只调用本工具后就停止"` + baseoptions.BaseListOptions - Usable *bool `help:"Filter usable vpcs"` - Region string `help:"ID or Name of region" json:"-"` + Usable *bool `help:"Filter usable vpcs" mcp:"true"` + Region string `help:"ID or Name of region" json:"-" mcp:"true"` Globalvpc string `help:"Filter by globalvpc"` DnsZoneId string `help:"Filter by DnsZone"` InterVpcNetworkId string `help:"Filter by InterVpcNetwork"` ExternalAccessMode string `help:"Filter by external access mode" choices:"distgw|eip|eip-distgw"` - ZoneId string `help:"Filter by zone which has networks"` + ZoneId string `help:"Filter by zone which has networks" mcp:"true"` UsableForInterVpcNetworkId string `help:"Filter usable vpcs for inter vpc network"` OrderByWireCount string CidrBlock string `help:"IPv4 cidr block"` diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/domain.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/domain.go index c0d0897fa..c01f05bbc 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/domain.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/domain.go @@ -22,9 +22,11 @@ import ( ) type DomainListOptions struct { + _ struct{} `mcp-desc:"列出域（domain）。可用 search 等过滤；详情 climc_domain_show，创建 climc_domain_create，删除 climc_domain_delete"` + options.BaseListOptions - IdpId string `help:"filter by idp_id"` - IdpEntityId string `help:"filter by idp_entity_id"` + IdpId string `help:"filter by idp_id" mcp:"true"` + IdpEntityId string `help:"filter by idp_entity_id" mcp:"true"` } func (opts *DomainListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/project.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/project.go index 645a78701..17b42753f 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/project.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/project.go @@ -23,14 +23,16 @@ import ( ) type ProjectListOptions struct { + _ struct{} `mcp-desc:"列出项目（project/tenant）。可用 search/domain 等过滤；详情 climc_project_show，创建 climc_project_create，删除 climc_project_delete"` + options.BaseListOptions - UserId string `help:"filter by user id"` - GroupId string `help:"filter by group id"` - IdpId string `help:"filter by idp id"` - AdminId []string + UserId string `help:"filter by user id" mcp:"true"` + GroupId string `help:"filter by group id" mcp:"true"` + IdpId string `help:"filter by idp id" mcp:"true"` + AdminId []string `mcp:"true"` - OrderByDomain string `help:"order by domain name" choices:"asc|desc"` + OrderByDomain string `help:"order by domain name" choices:"asc|desc" mcp:"true"` } func (opts *ProjectListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/users.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/users.go index 3755e3a4c..16027204a 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/users.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/identity/users.go @@ -21,14 +21,16 @@ import ( ) type UserListOptions struct { + _ struct{} `mcp-desc:"列出用户。可用 search/name/domain 等过滤；详情 climc_user_show，创建 climc_user_create，删除 climc_user_delete"` + options.BaseListOptions - Name string `help:"Filter by name"` - OrderByDomain string `help:"order by domain name" choices:"asc|desc"` - Role string `help:"Filter by role"` - RoleAssignmentDomainId string `help:"filter role assignment domain"` - RoleAssignmentProjectId string `help:"filter role assignment project"` - IdpId string `help:"filter by idp_id"` - IdpEntityId string `help:"filter by idp_entity_id"` + Name string `help:"Filter by name" mcp:"true"` + OrderByDomain string `help:"order by domain name" choices:"asc|desc" mcp:"true"` + Role string `help:"Filter by role" mcp:"true"` + RoleAssignmentDomainId string `help:"filter role assignment domain" mcp:"true"` + RoleAssignmentProjectId string `help:"filter role assignment project" mcp:"true"` + IdpId string `help:"filter by idp_id" mcp:"true"` + IdpEntityId string `help:"filter by idp_entity_id" mcp:"true"` } func (opts *UserListOptions) Params() (jsonutils.JSONObject, error) { diff --git a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/k8s/cluster.go b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/k8s/cluster.go index d6933a2f5..c83842e76 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcclient/options/k8s/cluster.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcclient/options/k8s/cluster.go @@ -262,6 +262,18 @@ func (o ClusterGCOpts) Params() (jsonutils.JSONObject, error) { return nil, nil } +type ClusterHistoryDataCleanOpts struct { + Day *int `help:"days to keep soft-deleted k8s resources before hard delete" default:"30"` +} + +func (o ClusterHistoryDataCleanOpts) Params() (jsonutils.JSONObject, error) { + params := jsonutils.NewDict() + if o.Day != nil { + params.Set("day", jsonutils.NewInt(int64(*o.Day))) + } + return params, nil +} + type IdentOptions struct { ID string `help:"ID or name of the model"` } diff --git a/vendor/yunion.io/x/onecloud/pkg/mcp-server/options/options.go b/vendor/yunion.io/x/onecloud/pkg/mcp-server/options/options.go index 84e37e986..dfa711f4c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/mcp-server/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/mcp-server/options/options.go @@ -15,9 +15,14 @@ package options import ( + "strings" + "time" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" ) +const DefaultPlatformName = "Cloudpods" + type MCPServerOptions struct { common_options.CommonOptions // 服务基础信息 @@ -26,9 +31,45 @@ type MCPServerOptions struct { MCPServerDescription string `help:"MCP service description"` // 连接超时配置 - Timeout int `help:"SDK connection timeout to cloudpods service (seconds)" default:"30"` + Timeout int `help:"SDK connection timeout to platform API (seconds)" default:"30"` + // ServerCreateWaitSeconds 创建后等待 running/ready 的最长时间；超时仍返回 server_id，便于 agent 用 server-show 继续轮询（须小于 LLM MCPAgentTimeout） + ServerCreateWaitSeconds int `help:"max seconds to wait for server running/ready after create; on timeout still return server_id" default:"180"` } var ( Options MCPServerOptions ) + +// ResolvedPlatformName 返回配置中的平台展示名，空则回退 DefaultPlatformName。 +func ResolvedPlatformName() string { + name := strings.TrimSpace(Options.PlatformName) + if name == "" { + return DefaultPlatformName + } + return name +} + +// ServerCreateWaitDuration 创建等待时长。 +func ServerCreateWaitDuration() time.Duration { + sec := Options.ServerCreateWaitSeconds + if sec <= 0 { + sec = 180 + } + return time.Duration(sec) * time.Second +} + +func OnOptionsChange(oldO, newO interface{}) bool { + oldOpts := oldO.(*MCPServerOptions) + newOpts := newO.(*MCPServerOptions) + + changed := false + if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) { + changed = true + } + // PlatformName 写入 ServerInstructions 需重启进程 + if oldOpts.PlatformName != newOpts.PlatformName { + changed = true + } + // ServerCreateWaitSeconds / Timeout 热更新即可（OptionManager 会拷贝到 Options） + return changed +} diff --git a/vendor/yunion.io/x/onecloud/pkg/notify/options/options.go b/vendor/yunion.io/x/onecloud/pkg/notify/options/options.go index 4cec0e752..4c99d7a85 100644 --- a/vendor/yunion.io/x/onecloud/pkg/notify/options/options.go +++ b/vendor/yunion.io/x/onecloud/pkg/notify/options/options.go @@ -34,6 +34,7 @@ type NotifyOption struct { VerifyExpireInterval int `help:"expire interval of verify message; minutes" default:"2"` VerifyValidInterval int `help:"valid interval of verify message; miniutes" default:"20"` + VerifyMaxAttempts int `help:"maximal failed attempts before a verification code is invalidated" default:"5"` SyncReceiverIntervalMinutes int `help:"interval to sync receivers from keystone, in minutes" default:"30"` EnableWatchUser bool `help:"use etcd to watch user" default:"false"` diff --git a/vendor/yunion.io/x/onecloud/pkg/proxy/http.go b/vendor/yunion.io/x/onecloud/pkg/proxy/http.go index 69072201e..615b0ac11 100644 --- a/vendor/yunion.io/x/onecloud/pkg/proxy/http.go +++ b/vendor/yunion.io/x/onecloud/pkg/proxy/http.go @@ -20,6 +20,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "time" "yunion.io/x/log" "yunion.io/x/pkg/appctx" @@ -68,15 +69,24 @@ func (p *SReverseProxy) ServeHTTP(ctx context.Context, w http.ResponseWriter, r } log.Debugf("Forwarding to servie: %q, url: %q", p.serviceName, remoteUrl.String()) proxy := httputil.NewSingleHostReverseProxy(remoteUrl) + // SSE / long-polling: flush immediately; never ask upstream for gzip (buffers whole stream). + proxy.FlushInterval = -1 proxy.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, + DisableCompression: true, + IdleConnTimeout: 90 * time.Second, + } + proxy.ModifyResponse = func(resp *http.Response) error { + resp.Header.Set("X-Accel-Buffering", "no") + return nil } r, err = p.manipulator(ctx, r) if err != nil { httperrors.InternalServerError(ctx, w, "%v", err) return } + r.Header.Del("Accept-Encoding") proxy.ServeHTTP(w, r) } diff --git a/vendor/yunion.io/x/onecloud/pkg/util/ansible/module.go b/vendor/yunion.io/x/onecloud/pkg/util/ansible/module.go index 6cab30cf1..0ad22b0ac 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/ansible/module.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/ansible/module.go @@ -16,6 +16,7 @@ package ansible import ( "bytes" + "strings" ) // Module represents name and args of ansible module to execute @@ -74,9 +75,24 @@ func (i *Inventory) Data() []byte { b.WriteRune(' ') b.WriteString(k) b.WriteRune('=') - b.WriteString(v) + b.WriteString(quoteInventoryValue(v)) } b.WriteRune('\n') } return b.Bytes() } + +func quoteInventoryValue(v string) string { + var b strings.Builder + b.Grow(len(v) + 2) + b.WriteByte('"') + for i := 0; i < len(v); i++ { + c := v[i] + if c == '\\' || c == '"' { + b.WriteByte('\\') + } + b.WriteByte(c) + } + b.WriteByte('"') + return b.String() +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/ansible/playbook.go b/vendor/yunion.io/x/onecloud/pkg/util/ansible/playbook.go index c82e87fbe..65ee6952c 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/ansible/playbook.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/ansible/playbook.go @@ -26,6 +26,8 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/util/fileutils2" ) type pbState int @@ -160,7 +162,11 @@ func (pb *Playbook) Run(ctx context.Context) (err error) { // write out files for name, content := range pb.Files { - path := filepath.Join(tmpdir, name) + path, err2 := fileutils2.JoinInside(tmpdir, name) + if err2 != nil { + err = errors.Wrapf(err2, "playbook file %s", name) + return + } dir := filepath.Dir(path) err = os.MkdirAll(dir, os.FileMode(0700)) if err != nil { diff --git a/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate.go b/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate.go new file mode 100644 index 000000000..f5c3692eb --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate.go @@ -0,0 +1,175 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ansible + +import ( + "net" + "path" + "regexp" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/util/fileutils2" +) + +var ( + ansibleUserRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + ansibleVarRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) +) + +var allowedConnections = map[string]struct{}{ + "": {}, + "ssh": {}, + "smart": {}, + "paramiko": {}, + "winrm": {}, + "psrp": {}, +} + +var deniedAnsibleVars = map[string]struct{}{ + "ansible_shell_executable": {}, + "ansible_python_interpreter": {}, + "ansible_ssh_executable": {}, + "ansible_remote_tmp": {}, + "ansible_executable": {}, + "ansible_shell_type": {}, + "ansible_become_exe": {}, + "ansible_become_flags": {}, + "ansible_ssh_args": {}, + "ansible_async_dir": {}, +} + +func IsValidAnsibleUser(user string) bool { + return ansibleUserRe.MatchString(user) +} + +func ValidateInventoryHostName(name string) error { + name = strings.TrimSpace(name) + if name == "" { + return errors.Errorf("empty host name") + } + if strings.ContainsAny(name, " \t\r\n=") { + return errors.Errorf("invalid host name") + } + if isLocalTarget(name) { + return errors.Errorf("host %q is not allowed", name) + } + return nil +} + +func ValidateInventoryVar(key, value string) error { + if !ansibleVarRe.MatchString(key) { + return errors.Errorf("invalid inventory variable %q", key) + } + if strings.ContainsAny(value, "\x00\r\n") { + return errors.Errorf("invalid inventory variable %s", key) + } + lk := strings.ToLower(key) + if isDeniedAnsibleVar(lk) { + return errors.Errorf("inventory variable %s is not allowed", key) + } + switch lk { + case "ansible_connection": + if _, ok := allowedConnections[strings.ToLower(strings.TrimSpace(value))]; !ok { + return errors.Errorf("ansible_connection %q is not allowed", value) + } + case "ansible_host": + if isLocalTarget(value) { + return errors.Errorf("ansible_host %q is not allowed", value) + } + case "ansible_user": + if value != "" && !IsValidAnsibleUser(value) { + return errors.Errorf("invalid ansible_user") + } + case "ansible_ssh_private_key_file": + if _, err := fileutils2.CleanRelSubpath(value); err != nil { + return errors.Wrap(err, "ansible_ssh_private_key_file") + } + } + return nil +} + +func ValidateInventoryHost(h Host) error { + if err := ValidateInventoryHostName(h.Name); err != nil { + return err + } + for k, v := range h.Vars { + if err := ValidateInventoryVar(k, v); err != nil { + return err + } + } + return nil +} + +func ValidatePlaybookFileName(name string) error { + rel, err := fileutils2.CleanRelSubpath(name) + if err != nil { + return errors.Wrapf(err, "playbook file %q", name) + } + base := path.Base(rel) + if base == "ansible.cfg" || base == ".ansible.cfg" { + return errors.Errorf("playbook file %q is not allowed", name) + } + return nil +} + +func ValidatePlaybook(pb *Playbook) error { + if pb == nil { + return errors.Errorf("empty playbook") + } + for i := range pb.Inventory.Hosts { + if err := ValidateInventoryHost(pb.Inventory.Hosts[i]); err != nil { + return err + } + } + for name := range pb.Files { + if err := ValidatePlaybookFileName(name); err != nil { + return err + } + } + return nil +} + +func isDeniedAnsibleVar(key string) bool { + if _, ok := deniedAnsibleVars[key]; ok { + return true + } + if !strings.HasPrefix(key, "ansible_") { + return false + } + for _, suffix := range []string{"_executable", "_interpreter", "_extra_args", "_common_args"} { + if strings.HasSuffix(key, suffix) { + return true + } + } + return false +} + +func isLocalTarget(s string) bool { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + return false + } + switch s { + case "localhost", "localhost.localdomain", "ip6-localhost", "ip6-loopback": + return true + } + ip := net.ParseIP(s) + if ip == nil { + return false + } + return ip.IsLoopback() || ip.IsUnspecified() +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate_yaml.go b/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate_yaml.go new file mode 100644 index 000000000..2777e01b3 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/util/ansible/validate_yaml.go @@ -0,0 +1,217 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ansible + +import ( + "fmt" + "net" + "path/filepath" + "regexp" + "strings" + + "github.com/go-yaml/yaml" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/util/fileutils2" +) + +var controllerCallRe = regexp.MustCompile(`(?i)(^|[^A-Za-z0-9_])(lookup|query)\s*\(`) + +func ValidatePlaybookYAML(doc string) error { + doc = strings.TrimSpace(doc) + if doc == "" { + return errors.Errorf("empty playbook") + } + if err := rejectControllerCalls(doc); err != nil { + return err + } + var parsed interface{} + if err := yaml.Unmarshal([]byte(doc), &parsed); err != nil { + return errors.Wrap(err, "parse playbook") + } + return walkYAML(parsed, "") +} + +func ValidateInventoryYAML(doc string) error { + doc = strings.TrimSpace(doc) + if doc == "" { + return errors.Errorf("empty inventory") + } + if err := rejectControllerCalls(doc); err != nil { + return err + } + var parsed interface{} + if err := yaml.Unmarshal([]byte(doc), &parsed); err != nil { + return errors.Wrap(err, "parse inventory") + } + return walkYAML(parsed, "") +} + +func ValidateFilesJSON(files string) error { + files = strings.TrimSpace(files) + if files == "" { + return nil + } + obj, err := jsonutils.ParseString(files) + if err != nil { + return errors.Wrap(err, "parse files") + } + m, err := obj.GetMap() + if err != nil { + return errors.Wrap(err, "files must be a json object") + } + for name := range m { + if err := ValidatePlaybookFileName(name); err != nil { + return err + } + } + return nil +} + +func rejectControllerCalls(doc string) error { + if controllerCallRe.MatchString(doc) { + return errors.Errorf("lookup/query is not allowed") + } + return nil +} + +func walkYAML(v interface{}, parent string) error { + switch t := v.(type) { + case map[interface{}]interface{}: + for k, val := range t { + key := fmt.Sprint(k) + if err := checkYAMLPair(parent, key, val); err != nil { + return err + } + if err := walkYAML(val, key); err != nil { + return err + } + } + case map[string]interface{}: + for key, val := range t { + if err := checkYAMLPair(parent, key, val); err != nil { + return err + } + if err := walkYAML(val, key); err != nil { + return err + } + } + case []interface{}: + for _, item := range t { + if s, ok := yamlString(item); ok && (parent == "hosts" || parent == "delegate_to") { + if hostsContainLocal(s) { + return errors.Errorf("%s %q is not allowed", parent, s) + } + } + if err := walkYAML(item, parent); err != nil { + return err + } + } + } + return nil +} + +func checkYAMLPair(parent, key string, val interface{}) error { + lk := strings.ToLower(strings.TrimSpace(key)) + s, isStr := yamlString(val) + switch lk { + case "connection", "ansible_connection": + if !isStr { + return errors.Errorf("%s must be a string", key) + } + if _, ok := allowedConnections[strings.ToLower(strings.TrimSpace(s))]; !ok { + return errors.Errorf("%s %q is not allowed", key, s) + } + case "delegate_to", "ansible_host": + if isStr && hostsContainLocal(s) { + return errors.Errorf("%s %q is not allowed", key, s) + } + case "hosts": + if isStr && hostsContainLocal(s) { + return errors.Errorf("hosts %q is not allowed", s) + } + case "local_action": + return errors.Errorf("local_action is not allowed") + case "ansible_user": + if isStr && s != "" && !IsValidAnsibleUser(s) { + return errors.Errorf("invalid ansible_user") + } + case "ansible_ssh_private_key_file": + if isStr { + if _, err := fileutils2.CleanRelSubpath(s); err != nil { + return errors.Wrap(err, "ansible_ssh_private_key_file") + } + } + case "import_playbook", "include", "include_playbook", "import_tasks", "include_tasks", "include_vars": + if isStr { + if strings.Contains(s, "://") || filepath.IsAbs(s) || strings.Contains(s, "{{") { + return errors.Errorf("%s %q is not allowed", key, s) + } + if _, err := fileutils2.CleanRelSubpath(s); err != nil { + return errors.Wrapf(err, "%s", key) + } + } + } + if isDeniedAnsibleVar(lk) { + return errors.Errorf("%s is not allowed", key) + } + if parent == "hosts" && isLocalTarget(key) { + return errors.Errorf("host %q is not allowed", key) + } + return nil +} + +func hostsContainLocal(s string) bool { + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + host := stripHostPort(part) + if isLocalTarget(host) { + return true + } + } + return false +} + +func stripHostPort(part string) string { + if strings.HasPrefix(part, "[") { + end := strings.Index(part, "]") + if end > 1 { + return part[1:end] + } + return part + } + if i := strings.LastIndex(part, ":"); i > 0 { + if net.ParseIP(part) == nil { + return part[:i] + } + } + return part +} + +func yamlString(v interface{}) (string, bool) { + switch t := v.(type) { + case string: + return t, true + case []byte: + return string(t), true + default: + return "", false + } +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/fileutils.go b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/fileutils.go index cf551d733..15701958d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/fileutils.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/fileutils.go @@ -315,10 +315,15 @@ func CleanFailedMountpoints() { f, err := os.Open(mtfile) if err != nil { log.Errorf("CleanFailedMountpoints error: %s", err) + return } + defer f.Close() reader := bufio.NewReader(f) - line, _, err := reader.ReadLine() - for err != nil { + for { + line, _, err := reader.ReadLine() + if err != nil { + break + } m := strings.Split(string(line), " ") if len(m) > 1 { mp := m[1] diff --git a/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/hostpath.go b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/hostpath.go new file mode 100644 index 000000000..154942a1d --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/hostpath.go @@ -0,0 +1,153 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutils2 + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "yunion.io/x/pkg/errors" +) + +var unixFileModeRe = regexp.MustCompile(`^[0-7]{3,4}$`) + +var restrictedHostBindPrefixes = []string{ + "/etc", + "/root", + "/proc", + "/sys", + "/boot", + "/dev", + "/run", + "/var/run", + "/var/lib", + "/usr", + "/bin", + "/sbin", + "/lib", + "/lib64", +} + +// CleanHostBindPath returns a cleaned absolute host path suitable for bind-mount. +// The root filesystem path "/" is rejected. +func CleanHostBindPath(p string) (string, error) { + p = strings.TrimSpace(p) + if p == "" { + return "", errors.Errorf("path is empty") + } + if !filepath.IsAbs(p) { + return "", errors.Errorf("path %q must be absolute", p) + } + clean := filepath.Clean(p) + if !filepath.IsAbs(clean) { + return "", errors.Errorf("path %q is not absolute after cleaning", p) + } + if clean == string(filepath.Separator) { + return "", errors.Errorf("path %q is not allowed", p) + } + return clean, nil +} + +// IsRestrictedHostBindPath reports whether a cleaned absolute path is under a +// host location that ordinary users must not bind into a container. +func IsRestrictedHostBindPath(p string) bool { + p = filepath.Clean(p) + for _, prefix := range restrictedHostBindPrefixes { + if p == prefix || strings.HasPrefix(p, prefix+string(filepath.Separator)) { + return true + } + } + return false +} + +// ParseUnixFileMode parses a chmod-style octal mode (e.g. "755" or "0755"). +// The returned string is the canonical octal form without a leading zero. +func ParseUnixFileMode(perm string) (os.FileMode, string, error) { + perm = strings.TrimSpace(perm) + if perm == "" { + return 0, "", errors.Errorf("file mode is empty") + } + if !unixFileModeRe.MatchString(perm) { + return 0, "", errors.Errorf("invalid file mode %q", perm) + } + v, err := strconv.ParseUint(perm, 8, 32) + if err != nil { + return 0, "", errors.Wrapf(err, "invalid file mode %q", perm) + } + if v > 0o777 { + return 0, "", errors.Errorf("invalid file mode %q", perm) + } + return os.FileMode(v), strconv.FormatUint(v, 8), nil +} + +// CleanRelSubpath cleans a relative path and rejects absolute paths or parent +// directory traversal. +func CleanRelSubpath(p string) (string, error) { + p = strings.TrimSpace(p) + if p == "" { + return "", errors.Errorf("path is empty") + } + if strings.ContainsRune(p, 0) { + return "", errors.Errorf("path contains NUL") + } + if filepath.IsAbs(p) { + return "", errors.Errorf("path %q must be relative", p) + } + clean := filepath.Clean(p) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", errors.Errorf("path %q is not allowed", p) + } + if filepath.IsAbs(clean) { + return "", errors.Errorf("path %q must be relative", p) + } + return clean, nil +} + +// JoinInside joins elem onto base and requires the result to stay inside base. +func JoinInside(base, elem string) (string, error) { + base = filepath.Clean(base) + if elem == "" { + return base, nil + } + rel, err := CleanRelSubpath(elem) + if err != nil { + return "", err + } + joined := filepath.Join(base, rel) + if !IsPathInside(base, joined) { + return "", errors.Errorf("path %q escapes %q", elem, base) + } + return joined, nil +} + +// JoinInsideAll joins elems onto base in order and requires each step to stay +// inside the previous result. +func JoinInsideAll(base string, elems ...string) (string, error) { + cur := filepath.Clean(base) + for _, e := range elems { + if e == "" { + continue + } + next, err := JoinInside(cur, e) + if err != nil { + return "", err + } + cur = next + } + return cur, nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/nofollow.go b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/nofollow.go new file mode 100644 index 000000000..1c5d98fdf --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/util/fileutils2/nofollow.go @@ -0,0 +1,103 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or authorized to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutils2 + +import ( + "io" + "os" + "path" + "path/filepath" + "strings" + "syscall" + + "yunion.io/x/pkg/errors" +) + +// IsPathInside reports whether target is the same as base or a descendant of it. +func IsPathInside(base, target string) bool { + base = filepath.Clean(base) + target = filepath.Clean(target) + rel, err := filepath.Rel(base, target) + if err != nil { + return false + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return false + } + return !filepath.IsAbs(rel) +} + +// CleanGuestDeployPath requires an absolute guest path and rejects parent-dir +// traversal after cleaning. +func CleanGuestDeployPath(p string) (string, error) { + p = strings.TrimSpace(p) + if p == "" { + return "", errors.Errorf("path is empty") + } + if strings.ContainsRune(p, 0) { + return "", errors.Errorf("path contains NUL") + } + if !path.IsAbs(p) && !filepath.IsAbs(p) { + return "", errors.Errorf("path %q must be absolute", p) + } + clean := path.Clean("/" + strings.TrimPrefix(filepath.ToSlash(p), "/")) + if !path.IsAbs(clean) { + return "", errors.Errorf("path %q must be absolute", p) + } + if clean == "/" { + return "", errors.Errorf("path %q is not allowed", p) + } + if clean == ".." || strings.HasPrefix(clean, "../") { + return "", errors.Errorf("path %q is not allowed", p) + } + return clean, nil +} + +func IsSymlink(path string) bool { + fi, err := os.Lstat(path) + if err != nil { + return false + } + return fi.Mode()&os.ModeSymlink != 0 +} + +func OpenFileNoFollow(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, flag|syscall.O_NOFOLLOW, perm) +} + +func FilePutContentsNoFollow(filename string, content string, modAppend bool) error { + mode := os.O_WRONLY | os.O_CREATE + if modAppend { + mode |= os.O_APPEND + } else { + mode |= os.O_TRUNC + } + fd, err := OpenFileNoFollow(filename, mode, 0644) + if err != nil { + return err + } + defer fd.Close() + _, err = fd.WriteString(content) + return err +} + +func FileGetContentsNoFollow(filename string) ([]byte, error) { + fd, err := OpenFileNoFollow(filename, os.O_RDONLY, 0) + if err != nil { + return nil, err + } + defer fd.Close() + return io.ReadAll(fd) +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts.go b/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts.go index 3d0dfbfaa..65a35d124 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts.go @@ -321,4 +321,6 @@ const ( ACT_REGISTER_AIPROXY = "register_aiproxy" ACT_UNREGISTER_AIPROXY = "unregister_aiproxy" + + ACT_RESUME_IMPORT = "resume_import" ) diff --git a/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts_i18n.go b/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts_i18n.go index 937c17d58..3d70cff37 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts_i18n.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/logclient/consts_i18n.go @@ -1477,6 +1477,11 @@ func init() { CN("保存镜像"), ) + o.Set(ACT_RESUME_IMPORT, i18n.NewTableEntry(). + EN("Resume Import"). + CN("继续导入"), + ) + o.Set(ACT_CLOUD_SYNC, i18n.NewTableEntry(). EN("Sync Cloud Resource"). CN("同步云资源"), diff --git a/vendor/yunion.io/x/onecloud/pkg/util/netutils2/netutils.go b/vendor/yunion.io/x/onecloud/pkg/util/netutils2/netutils.go index 7ebb3540d..2ecafe160 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/netutils2/netutils.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/netutils2/netutils.go @@ -335,12 +335,37 @@ func addRoute(routes []SRouteInfo, net, gw string) []SRouteInfo { return routes } -func extendRoutes(routes4, routes6 []SRouteInfo, nicRoutes []types.SRoute) ([]SRouteInfo, []SRouteInfo) { +func extendRoutes(routes4, routes6 []SRouteInfo, nicRoutes []types.SRoute, nicV4Addr string, nicV6Addr string) ([]SRouteInfo, []SRouteInfo) { for i := 0; i < len(nicRoutes); i++ { - if regutils.MatchCIDR6(nicRoutes[i][0]) { - routes6 = addRoute(routes6, nicRoutes[i][0], nicRoutes[i][1]) + prefix := nicRoutes[i][0] + if regutils.MatchCIDR6(prefix) { + if utils.IsInArray(prefix, Ip4MetadataServers) || prefix == "0.0.0.0/0" { + continue + } + var nextHop string + if len(nicRoutes[i]) > 1 { + nextHop = nicRoutes[i][1] + } else { + nextHop = "0.0.0.0" + } + if nextHop == nicV4Addr { + continue + } + routes6 = addRoute(routes6, prefix, nextHop) } else { - routes4 = addRoute(routes4, nicRoutes[i][0], nicRoutes[i][1]) + if utils.IsInArray(prefix, Ip6MetadataServers) || prefix == "::/0" { + continue + } + var nextHop string + if len(nicRoutes[i]) > 1 { + nextHop = nicRoutes[i][1] + } else { + nextHop = "::" + } + if nextHop == nicV6Addr { + continue + } + routes4 = addRoute(routes4, prefix, nextHop) } } return routes4, routes6 @@ -388,7 +413,7 @@ func AddNicRoutes(routes4 []SRouteInfo, routes6 []SRouteInfo, nicDesc *types.SSe // return routes // } if len(nicDesc.Routes) > 0 { - routes4, routes6 = extendRoutes(routes4, routes6, nicDesc.Routes) + routes4, routes6 = extendRoutes(routes4, routes6, nicDesc.Routes, nicDesc.Ip, nicDesc.Ip6) } else if len(nicDesc.Gateway) > 0 && !isExitAddress(nicDesc.Ip) && nicCnt == 2 && nicDesc.Ip != mainIp && isExitAddress(mainIp) { for _, pref := range netutils.GetPrivateIPRanges() { diff --git a/vendor/yunion.io/x/onecloud/pkg/util/procutils/remote_path.go b/vendor/yunion.io/x/onecloud/pkg/util/procutils/remote_path.go new file mode 100644 index 000000000..eb2aa2615 --- /dev/null +++ b/vendor/yunion.io/x/onecloud/pkg/util/procutils/remote_path.go @@ -0,0 +1,36 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procutils + +import ( + "strings" + + "yunion.io/x/pkg/errors" +) + +// RemotePathExists checks whether path exists on the host via executor. +func RemotePathExists(path string) bool { + _, err := RemoteStat(path) + return err == nil +} + +// RemoteReadlink resolves a symlink on the host via executor. +func RemoteReadlink(path string) (string, error) { + out, err := NewRemoteCommandAsFarAsPossible("readlink", "-f", path).Output() + if err != nil { + return "", errors.Wrapf(err, "readlink -f %s: %s", path, out) + } + return strings.TrimSpace(string(out)), nil +} diff --git a/vendor/yunion.io/x/onecloud/pkg/util/seclib2/crypto.go b/vendor/yunion.io/x/onecloud/pkg/util/seclib2/crypto.go index 16fb0a5c9..4d513df7d 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/seclib2/crypto.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/seclib2/crypto.go @@ -18,6 +18,7 @@ import ( "crypto" "crypto/dsa" "crypto/ecdsa" + "crypto/ed25519" "crypto/rand" "crypto/rsa" "crypto/sha1" @@ -72,6 +73,8 @@ func Encrypt(publicKey, origData []byte) ([]byte, error) { pubInf = ssh2dsaPublicKey(pub) case ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521: pubInf = ssh2ecdsaPublicKey(pub) + case ssh.KeyAlgoED25519: + pubInf = ssh2CryptoPublicKey(pub) default: return nil, fmt.Errorf("unsupported key type %s", pub.Type()) } @@ -106,6 +109,12 @@ func Decrypt(privateKey, secret []byte) ([]byte, error) { return nil, err } return decryptAES(ecdsaPub, secret) + case ed25519.PrivateKey, *ed25519.PrivateKey: + ed25519Pub, err := exportSshPublicKey(priv.(crypto.Signer).Public()) + if err != nil { + return nil, err + } + return decryptAES(ed25519Pub, secret) } return nil, fmt.Errorf("unsupported") } diff --git a/vendor/yunion.io/x/onecloud/pkg/util/ssh/ssh.go b/vendor/yunion.io/x/onecloud/pkg/util/ssh/ssh.go index 9d350b4d5..8a6f05f57 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/ssh/ssh.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/ssh/ssh.go @@ -144,6 +144,10 @@ func (s *Client) GetConfig() ClientConfig { return s.config } +func (s *Client) GetSshClient() *ssh.Client { + return s.client +} + func (s *Client) RawRun(cmds ...string) ([]string, error) { return s.run(false, cmds, nil, false) } diff --git a/vendor/yunion.io/x/onecloud/pkg/util/sysutils/kvm.go b/vendor/yunion.io/x/onecloud/pkg/util/sysutils/kvm.go index 7f8daba8b..96fc3bc01 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/sysutils/kvm.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/sysutils/kvm.go @@ -42,7 +42,6 @@ const ( var ( kvmModuleSupport string - nestStatus string ) func GetKVMModuleSupport() string { @@ -93,11 +92,11 @@ func detectKVMModuleSupport() string { km = KVM_MODULE_BUILDIN } } else { - if ModprobeKvmModule(KVM_MODULE_INTEL, false, false) { + if ModprobeKvmModule(KVM_MODULE_INTEL, false, nil) { km = KVM_MODULE_INTEL - } else if ModprobeKvmModule(KVM_MODULE_AMD, false, false) { + } else if ModprobeKvmModule(KVM_MODULE_AMD, false, nil) { km = KVM_MODULE_AMD - } else if ModprobeKvmModule(KVM_MODULE, false, false) { + } else if ModprobeKvmModule(KVM_MODULE, false, nil) { if isDevKVMExists() { km = KVM_MODULE } @@ -111,15 +110,20 @@ func detectKVMModuleSupport() string { return km } -func ModprobeKvmModule(name string, remove, nest bool) bool { +func ModprobeKvmModule(name string, remove bool, nest *bool) bool { var params = []string{"modprobe"} if remove { params = append(params, "-r") } params = append(params, name) - if nest { - params = append(params, "nested=1") + if nest != nil { + if *nest { + params = append(params, "nested=1") + } else { + params = append(params, "nested=0") + } } + if err := procutils.NewRemoteCommandAsFarAsPossible(params[0], params[1:]...).Run(); err != nil { log.Errorf("Modprobe kvm %v failed: %s", params, err) return false @@ -127,31 +131,18 @@ func ModprobeKvmModule(name string, remove, nest bool) bool { return true } -func IsNestEnabled() bool { - return GetNestSupport() == HOST_NEST_ENABLE -} - -func GetNestSupport() string { - if len(nestStatus) == 0 { - nestStatus = detectNestSupport() - } - return nestStatus -} - -func detectNestSupport() string { +func DetectNestSupport(enableNest bool) string { moduleName := GetKVMModuleSupport() - nestStatus := HOST_NEST_UNSUPPORT - - if moduleName != KVM_MODULE_UNSUPPORT && isNestSupport(moduleName) { - log.Infof("Host is support kvm nest ...") - nestStatus = HOST_NEST_SUPPORT + if moduleName == KVM_MODULE_UNSUPPORT { + return HOST_NEST_UNSUPPORT } - - if nestStatus == HOST_NEST_SUPPORT && loadKvmModuleWithNest(moduleName) { - log.Infof("Host kvm nest is enabled ...") - nestStatus = HOST_NEST_ENABLE + if !isNestSupport(moduleName) { + return HOST_NEST_UNSUPPORT } - return nestStatus + if loadKvmModuleWithNest(moduleName, enableNest) && enableNest { + return HOST_NEST_ENABLE + } + return HOST_NEST_SUPPORT } func isNestSupport(name string) bool { @@ -171,17 +162,22 @@ func isNestSupport(name string) bool { return false } -func loadKvmModuleWithNest(name string) bool { +func loadKvmModuleWithNest(name string, enableNest bool) bool { var notload = true + var enabledNest = false if IsKernelModuleLoaded(name) { nest := GetKernelModuleParameter(name, "nested") - if nest == "Y" { + if strings.TrimSpace(nest) == "Y" { + enabledNest = true + } + if enableNest == enabledNest { return true } notload = unloadKvmModule(name) } + if notload { - if ModprobeKvmModule(name, false, true) { + if ModprobeKvmModule(name, false, &enableNest) { return true } } @@ -189,7 +185,7 @@ func loadKvmModuleWithNest(name string) bool { } func unloadKvmModule(name string) bool { - return ModprobeKvmModule(name, true, false) + return ModprobeKvmModule(name, true, nil) } func GetKernelModuleParameter(name, moduel string) string { diff --git a/vendor/yunion.io/x/onecloud/pkg/util/winutils/winutils.go b/vendor/yunion.io/x/onecloud/pkg/util/winutils/winutils.go index 55d5432ed..6497222b3 100644 --- a/vendor/yunion.io/x/onecloud/pkg/util/winutils/winutils.go +++ b/vendor/yunion.io/x/onecloud/pkg/util/winutils/winutils.go @@ -774,3 +774,20 @@ func (w *SWinRegTool) ResetUSBProfile() { key = w.GetCcsKeyPath() + `\Enum\USB` w.DelRegistry(key) } + +func (w *SWinRegTool) IsNetKVMInstalled() bool { + key := w.GetCcsKeyPath() + `\Services\netkvm` + + if !w.KeyExists(key) { + log.Errorf("key %s not exist", key) + return false + } + + imagePath := w.GetRegistry(key + `\ImagePath`) + if imagePath == "" { + log.Errorf("image path is empty") + return false + } + + return true +} diff --git a/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go b/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go index c14407458..4bd64a8d6 100644 --- a/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go +++ b/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/base64" "fmt" + "path" "strings" "yunion.io/x/jsonutils" @@ -99,32 +100,127 @@ func NewWriteFile(path string, content string, perm string, owner string, isBase return f } +// shellQuote wraps s so that a POSIX shell treats it as one literal word. +// A single quote inside s is closed, escaped and reopened. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// validUserName reports whether name can be used as a user name. +// +// The set is deliberately narrow. A name becomes a directory under /home, a +// file name under /etc/sudoers.d and an argument to useradd, so a slash or a +// ".." would move the files that are written, and a control character would +// split a generated line. Leading "-" is refused so the name cannot be read as +// an option. +func validUserName(name string) bool { + if len(name) == 0 || name == "." || name == ".." || name[0] == '-' { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + case c == '_' || c == '-' || c == '.': + default: + return false + } + } + return true +} + +// validWritePath reports whether path may be written by a generated script. +// +// Quoting keeps a path from running as a command, but a path that walks up out +// of its directory would still write somewhere else. A path carrying a ".." +// element is refused rather than normalised, so that what is written is what +// was asked for. +func validWritePath(filePath string) bool { + if len(filePath) == 0 { + return false + } + for _, elem := range strings.Split(filePath, "/") { + if elem == ".." { + return false + } + } + return true +} + +// escapePowerShell escapes a value for use inside a PowerShell double quoted +// string, where a backtick introduces an escape, $ introduces a variable, and +// a control character written literally would break the line it sits on. +func escapePowerShell(s string) string { + return strings.NewReplacer( + "`", "``", + `"`, "`\"", + "$", "`$", + "\r", "`r", + "\n", "`n", + "\t", "`t", + "\x00", "`0", + ).Replace(s) +} + +const heredocPrefix = "_YUNION_EOF_" + +// heredocTerminator returns a terminator that does not occur in content, so +// that a line of the content cannot end the heredoc early. Quote it at the use +// site so the shell does not expand the content. +// +// The generated value is checked against the content rather than assumed to be +// unique: utils.GenRequestId returns the empty string if the random source is +// unavailable, which would otherwise leave a fixed, guessable terminator. +func heredocTerminator(content string) string { + return uniqueTerminator(heredocPrefix+utils.GenRequestId(8), content) +} + +// uniqueTerminator returns term with underscores appended until it does not +// occur in content. Each pass lengthens it, so this ends once it is longer +// than the content and can no longer occur in it. +func uniqueTerminator(term, content string) string { + for strings.Contains(content, term) { + term += "_" + } + return term +} + func setFilePermission(path, permission, owner string) []string { cmds := []string{} if len(permission) > 0 { - cmds = append(cmds, fmt.Sprintf("chmod %s %s", permission, path)) + cmds = append(cmds, fmt.Sprintf("chmod %s %s", shellQuote(permission), shellQuote(path))) } if len(owner) > 0 { - cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", owner, owner, path)) + cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", shellQuote(owner), shellQuote(owner), shellQuote(path))) } return cmds } +// mkWriteFileCmd builds the commands that create path with content. The +// redirection and the path are quoted, and the heredoc is quoted with a +// terminator that is not expected to occur in the content. +func mkWriteFileCmd(redirect, filePath, content, permission, owner string) []string { + terminator := heredocTerminator(content) + cmds := []string{ + fmt.Sprintf("mkdir -p %s", shellQuote(path.Dir(filePath))), + fmt.Sprintf("cat %s %s <<'%s'\n%s\n%s", redirect, shellQuote(filePath), terminator, content, terminator), + } + return append(cmds, setFilePermission(filePath, permission, owner)...) +} + func mkPutFileCmd(path string, content string, permission string, owner string) []string { - cmds := []string{} - cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path)) - cmds = append(cmds, fmt.Sprintf("cat > %s <<_END\n%s\n_END", path, content)) - return append(cmds, setFilePermission(path, permission, owner)...) + return mkWriteFileCmd(">", path, content, permission, owner) } func mkAppendFileCmd(path string, content string, permission string, owner string) []string { - cmds := []string{} - cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path)) - cmds = append(cmds, fmt.Sprintf("cat >> %s <<_END\n%s\n_END", path, content)) - return append(cmds, setFilePermission(path, permission, owner)...) + return mkWriteFileCmd(">>", path, content, permission, owner) } func (wf *SWriteFile) ShellScripts() []string { + if !validWritePath(wf.Path) { + log.Errorf("cloudinit: skipping write_file with a path that walks up out of its directory: %q", wf.Path) + return nil + } content := wf.Content if wf.Encoding == "b64" { _content, _ := base64.StdEncoding.DecodeString(wf.Content) @@ -176,23 +272,36 @@ func (u *SUser) Password(passwd string) *SUser { } func (u *SUser) PowerShellScripts() []string { + if !validUserName(u.Name) { + log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name) + return nil + } + // Every line below is parsed by PowerShell first, so the name and the + // password are escaped the same way. Using the unescaped name for one of + // the lines would have them name different accounts. + name := escapePowerShell(u.Name) shells := []string{} - shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, u.Name)) - shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, u.Name)) + shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, name)) + shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, name)) if len(u.PlainTextPasswd) > 0 { - shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, u.Name, u.PlainTextPasswd)) + shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, name, escapePowerShell(u.PlainTextPasswd))) } // enable需要再设置密码之后，否则会出现Enable-LocalUser : Unable to update the password. The value provided for the new password does not meet the length, complexity, or history requirements of the domain - shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, u.Name)) + shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, name)) return shells } func (u *SUser) ShellScripts() []string { + if !validUserName(u.Name) { + log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name) + return nil + } + name := shellQuote(u.Name) shells := []string{} - shells = append(shells, fmt.Sprintf("useradd -m %s || true", u.Name)) + shells = append(shells, fmt.Sprintf("useradd -m %s || true", name)) if len(u.HashedPasswd) > 0 { - shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.HashedPasswd, u.Name)) + shells = append(shells, fmt.Sprintf("usermod -p %s %s", shellQuote(u.HashedPasswd), name)) } home := "/" + u.Name @@ -202,7 +311,7 @@ func (u *SUser) ShellScripts() []string { keyPath := fmt.Sprintf("%s/.ssh/authorized_keys", home) shells = append(shells, mkAppendFileCmd(keyPath, strings.Join(u.SshAuthorizedKeys, "\n"), "600", u.Name)...) - shells = append(shells, fmt.Sprintf("chown -R %s:%s %s/.ssh", u.Name, u.Name, home)) + shells = append(shells, fmt.Sprintf("chown -R %s:%s %s", name, name, shellQuote(home+"/.ssh"))) if !utils.IsInStringArray(u.Sudo, []string{"", "False"}) { shells = append(shells, mkPutFileCmd("/etc/sudoers.d/"+u.Name, fmt.Sprintf("%s %s", u.Name, u.Sudo), "", "")...) @@ -212,6 +321,9 @@ func (u *SUser) ShellScripts() []string { } func (conf *SCloudConfig) UserData() string { + if conf == nil { + return "" + } var buf bytes.Buffer jsonConf := jsonutils.Marshal(conf).(*jsonutils.JSONDict) if jsonConf.Contains("users") { @@ -228,6 +340,9 @@ func (conf *SCloudConfig) UserData() string { } func (conf *SCloudConfig) UserDataScript() string { + if conf == nil { + return "" + } shells := []string{} for _, u := range conf.Users { shells = append(shells, u.ShellScripts()...) @@ -248,8 +363,11 @@ func (conf *SCloudConfig) UserDataScript() string { } for _, pkg := range conf.Packages { - shells = append(shells, "which yum &>/dev/null && yum install -y "+pkg) - shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+pkg) + // A package name is an argument, not part of the command line, so it + // is quoted rather than pasted in. + quoted := shellQuote(pkg) + shells = append(shells, "which yum &>/dev/null && yum install -y "+quoted) + shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+quoted) } for _, wf := range conf.WriteFiles { shells = append(shells, wf.ShellScripts()...) @@ -317,7 +435,13 @@ func ParseUserData(data string) (*SCloudConfig, error) { if err != nil { return nil, errors.Wrapf(err, "ParseYAML") } - jsonDict := jsonConf.(*jsonutils.JSONDict) + jsonDict, ok := jsonConf.(*jsonutils.JSONDict) + if !ok { + // Anything that is valid YAML but not a mapping, e.g. a list or a + // scalar, is not a usable cloud-config document. + return nil, errors.Wrapf(errors.ErrInvalidFormat, + "cloud-config must be a YAML mapping, got %s", jsonConf.String()) + } if jsonDict.Contains("users") { userArray := jsonutils.NewArray() users, _ := jsonConf.GetArray("users") diff --git a/vendor/yunion.io/x/pkg/util/httputils/fs.go b/vendor/yunion.io/x/pkg/util/httputils/fs.go index ca6445d95..4d92114ec 100644 --- a/vendor/yunion.io/x/pkg/util/httputils/fs.go +++ b/vendor/yunion.io/x/pkg/util/httputils/fs.go @@ -35,11 +35,20 @@ func (fs FileSystem) Open(path string) (http.File, error) { } s, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } if s.IsDir() { index := strings.TrimSuffix(path, "/") + "/index.html" - if _, err := fs.fs.Open(index); err != nil { + indexFile, err := fs.fs.Open(index) + if err != nil { + // Without an index file a directory must not be listed. + f.Close() return nil, err } + // The index file is only checked for; it is not the file returned. + indexFile.Close() } return f, nil diff --git a/vendor/yunion.io/x/pkg/util/httputils/httputils.go b/vendor/yunion.io/x/pkg/util/httputils/httputils.go index 87c9e86fe..75dd553ac 100644 --- a/vendor/yunion.io/x/pkg/util/httputils/httputils.go +++ b/vendor/yunion.io/x/pkg/util/httputils/httputils.go @@ -28,6 +28,7 @@ import ( "net/http/httputil" "net/url" "os" + "regexp" "strconv" "strings" "syscall" @@ -93,6 +94,173 @@ type sClient interface { Do(req *http.Request) (*http.Response, error) } +// Field and parameter names that carry a credential on their own, and +// fragments that mark a name as carrying one. The same lists are used for the +// fields of a request body and the parameters of a request URL, so that the +// two cannot drift apart. +var sensitiveBodyKeyNames = []string{ + "key", "code", "sig", "pwd", "auth", "ticket", "session", "sessionid", + // The plain name is a credential; "access_key_id" is an identifier and + // is deliberately not matched, so it stays readable in the message. + "access_key", +} + +var sensitiveBodyKeyParts = []string{ + "token", "secret", "password", "passwd", "signature", "credential", + "apikey", "api_key", "private_key", +} + +func isSensitiveBodyKey(name string) bool { + lower := strings.ToLower(name) + if utils.IsInStringArray(lower, sensitiveBodyKeyNames) { + return true + } + for _, part := range sensitiveBodyKeyParts { + if strings.Contains(lower, part) { + return true + } + } + return false +} + +// redactURL masks the values of query parameters that commonly carry a +// credential, so that an error message can name the request without +// reproducing the secret. A URL with nothing to mask is returned unchanged, +// byte for byte, so no existing message is reformatted. +func redactURL(urlStr string) string { + parsed, err := url.Parse(urlStr) + if err != nil || len(parsed.RawQuery) == 0 { + return urlStr + } + query := parsed.Query() + changed := false + for name := range query { + if isSensitiveBodyKey(name) { + query.Set(name, "*") + changed = true + } + } + if !changed { + return urlStr + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// redactJSONObject returns obj with the values of credential carrying fields +// replaced, at any depth. +func redactJSONObject(obj jsonutils.JSONObject) jsonutils.JSONObject { + switch v := obj.(type) { + case *jsonutils.JSONDict: + redacted := jsonutils.NewDict() + for name, value := range v.Value() { + if isSensitiveBodyKey(name) { + redacted.Set(name, jsonutils.NewString("*")) + } else { + redacted.Set(name, redactJSONObject(value)) + } + } + return redacted + case *jsonutils.JSONArray: + redacted := jsonutils.NewArray() + for _, value := range v.Value() { + redacted.Add(redactJSONObject(value)) + } + return redacted + default: + return obj + } +} + +// redactFormBody masks the credential carrying fields of an urlencoded body. +// +// The body is walked pair by pair rather than through url.ParseQuery, which +// stops at the first pair it cannot decode: the pairs it did decode are still +// returned, but taking only the value it returns would either drop the rest of +// the body or, if the raw body were used instead, leave the credential in it. +func redactFormBody(body string) string { + parts := strings.Split(body, "&") + changed := false + for i, part := range parts { + eq := strings.IndexByte(part, '=') + if eq < 0 { + continue + } + name := part[:eq] + if decoded, err := url.QueryUnescape(name); err == nil { + name = decoded + } + if isSensitiveBodyKey(name) { + parts[i] = part[:eq+1] + "*" + changed = true + } + } + if !changed { + return body + } + return strings.Join(parts, "&") +} + +// A credential carrying name as it appears in a body that cannot be walked +// field by field, e.g. XML, or a JSON document that did not parse. +const sensitiveBodyNamePattern = `(?i)[A-Za-z0-9_.:-]*(?:token|secret|password|passwd|signature|credential|apikey|api_key|private_key)[A-Za-z0-9_.:-]*` + +var ( + // value, where the value may be a CDATA section + xmlElementSensitiveRe = regexp.MustCompile( + `(?is)(<` + sensitiveBodyNamePattern + `\b[^>]*>)(?:\s*|[^<]*)( maxRedactedBodyBytes { + return nil + } + switch { + case strings.Contains(contType, "json"): + if parsed, err := jsonutils.ParseString(body); err == nil { + return redactJSONObject(parsed) + } + // A JSON body that does not parse is still masked by field name, + // since the names survive in the text. + return jsonutils.NewString(redactBodyText(body)) + case strings.Contains(contType, "x-www-form-urlencoded"): + return jsonutils.NewString(redactFormBody(body)) + case strings.Contains(contType, "xml"): + return jsonutils.NewString(redactBodyText(body)) + } + return nil +} + // body might have been consumed, so body is provided separately func newJsonClientErrorFromRequest(req *http.Request, body string) *JSONClientError { return newJsonClientErrorFromRequest2(req.Method, req.URL.String(), req.Header, body) @@ -102,7 +270,7 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head jce := &JSONClientError{} jce.Request.Method = strings.ToUpper(method) - jce.Request.Url = urlStr + jce.Request.Url = redactURL(urlStr) jce.Request.Headers = make(map[string]string) excludeHdrs := []string{ "Accept", @@ -112,22 +280,13 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head http.CanonicalHeaderKey("authorization"), http.CanonicalHeaderKey("x-auth-token"), http.CanonicalHeaderKey("x-subject-token"), + http.CanonicalHeaderKey("cookie"), + http.CanonicalHeaderKey("set-cookie"), } - const ( - MAX_BODY = 128 - FIRST_PART = 100 - ) switch jce.Request.Method { case "PUT", "POST", "PATCH": contType := hdrs.Get(http.CanonicalHeaderKey("content-type")) - if len(body) > MAX_BODY { - jce.Request.Body = jsonutils.NewString(body[:FIRST_PART] + "..." + body[len(body)-MAX_BODY+FIRST_PART+3:]) - } else if strings.Contains(contType, "json") { - jce.Request.Body, _ = jsonutils.ParseString(body) - } else if strings.Contains(contType, "xml") || - strings.Contains(contType, "x-www-form-urlencoded") { - jce.Request.Body = jsonutils.NewString(body) - } + jce.Request.Body = redactRequestBody(body, contType) default: excludeHdrs = append(excludeHdrs, http.CanonicalHeaderKey("content-type"), http.CanonicalHeaderKey("content-length")) } @@ -233,13 +392,10 @@ func NewJsonClient(client sClient) *JsonClient { return &JsonClient{client: client} } +// Error renders the error as JSON. The request it carries is already masked +// when the error is built, so this has no side effects and repeating it gives +// the same message. func (e *JSONClientError) Error() string { - if !gotypes.IsNil(e.Request.Body) { - if body, ok := e.Request.Body.(*jsonutils.JSONDict); ok && body.Contains("password") { - body.Set("password", jsonutils.NewString("***")) - e.Request.Body = body - } - } errMsg := JSONClientErrorMsg{Error: e} return jsonutils.Marshal(errMsg).String() } @@ -302,11 +458,11 @@ func GetAddrPort(urlStr string) (string, int, error) { } func GetTransport(insecure bool) *http.Transport { - return getTransport(insecure, false, 0) + return getTransport(insecure, false, false, 0) } func GetAdaptiveTransport(insecure bool) *http.Transport { - return getTransport(insecure, true, 0) + return getTransport(insecure, false, true, 0) } func adptiveDial(ctx context.Context, network, addr string) (net.Conn, error) { @@ -317,7 +473,44 @@ func adptiveDial(ctx context.Context, network, addr string) (net.Conn, error) { return getConnDelegate(conn, 10*time.Second, 20*time.Second), nil } -func getTransport(insecure bool, adaptive bool, timeout time.Duration) *http.Transport { +func getTransport(insecure bool, legacyTLS bool, adaptive bool, timeout time.Duration) *http.Transport { + tlsConfig := &tls.Config{} + if insecure { + tlsConfig.InsecureSkipVerify = true + } + if legacyTLS { + tlsConfig.MinVersion = tls.VersionTLS10 + tlsConfig.CipherSuites = []uint16{ + // https://github.com/golang/go/issues/66512 + // TLS 1.0 - 1.2 cipher suites. + tls.TLS_RSA_WITH_RC4_128_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA256, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + // TLS 1.3 cipher suites. + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + } + } tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, // 一个空闲连接保持连接的时间 @@ -345,7 +538,7 @@ func getTransport(insecure bool, adaptive bool, timeout time.Duration) *http.Tra // waiting for the server to approve. // This time does not include the time to send the request header. ExpectContinueTimeout: 5 * time.Second, - TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, + TLSClientConfig: tlsConfig, } if adaptive { tr.DialContext = adptiveDial @@ -382,14 +575,59 @@ func getTransport(insecure bool, adaptive bool, timeout time.Duration) *http.Tra return tr } +func GetLegacyTLSClient(timeout time.Duration) *http.Client { + return getClient(true, true, timeout) +} + func GetClient(insecure bool, timeout time.Duration) *http.Client { + return getClient(insecure, false, timeout) +} + +// Session headers that a caller of this package sets, and that must not +// travel to a host the request was not made to. net/http already drops +// Authorization, Cookie and the proxy headers in that case, but does not know +// about these. +var redirectSensitiveHeaders = []string{ + "X-Auth-Token", + "X-Subject-Token", +} + +// isSameOrSubdomain reports whether dest is initial, or a subdomain of it. +// A redirect to a parent domain is not covered, matching the rule net/http +// uses for the headers it strips. +func isSameOrSubdomain(initial, dest *url.URL) bool { + i := strings.ToLower(initial.Hostname()) + d := strings.ToLower(dest.Hostname()) + if i == d { + return true + } + return strings.HasSuffix(d, "."+i) +} + +// checkRedirect drops the session headers above when a redirect leaves the +// host the request was made to, and keeps the usual limit on how many +// redirects are followed. +func checkRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.Errorf("stopped after 10 redirects") + } + if len(via) > 0 && !isSameOrSubdomain(via[0].URL, req.URL) { + for _, h := range redirectSensitiveHeaders { + req.Header.Del(h) + } + } + return nil +} + +func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Client { adaptive := false if timeout == 0 { adaptive = true } - tr := getTransport(insecure, adaptive, timeout) + tr := getTransport(insecure, legacyTLS, adaptive, timeout) return &http.Client{ - Transport: tr, + Transport: tr, + CheckRedirect: checkRedirect, // 一个完整http request的超时时间 // Timeout specifies a time limit for requests made by this // Client. The timeout includes connection time, any diff --git a/vendor/yunion.io/x/pkg/util/imagetools/imagetools.go b/vendor/yunion.io/x/pkg/util/imagetools/imagetools.go index 82b16c5bf..ebe1d3925 100644 --- a/vendor/yunion.io/x/pkg/util/imagetools/imagetools.go +++ b/vendor/yunion.io/x/pkg/util/imagetools/imagetools.go @@ -156,8 +156,11 @@ func normalizeOsDistribution(osDist string, imageName string) string { return OS_DIST_KYLIN } else if strings.Contains(osDist, "uos") { return OS_DIST_UOS - } else if strings.Contains(osDist, "windows") || regexp.MustCompile("(^|[^a-z0-9])win(xp|7|8|10|11|2003|2008|2012|2016|2019|2022)([^a-z0-9]|$)").MatchString(osDist) { - for _, ver := range []string{"2003", "2008", "2012", "2016", "2019", "2022"} { + } else if strings.Contains(osDist, "windows") || strings.Contains(osDist, "winserver") || regexp.MustCompile("(^|[^a-z0-9])win(xp|7|8|10|11|2003|2008|2012|2016|2019|2022|2025)([^a-z0-9]|$)").MatchString(osDist) { + if strings.Contains(osDist, "winserver") || strings.Contains(osDist, "windows server") { + return OS_DIST_WINDOWS_SERVER + } + for _, ver := range []string{"2003", "2008", "2012", "2016", "2019", "2022", "2025"} { if strings.Contains(osDist, ver) { return OS_DIST_WINDOWS_SERVER } @@ -180,40 +183,40 @@ var imageVersions = map[string][]string{ // FreeBSD：补充最新稳定版，覆盖10到15 OS_DIST_FREE_BSD: {"10", "11", "12", "13", "14", "15"}, - // Ubuntu：Server版补充LTS版本（每2年一个），Desktop版补充所有主要版本 - OS_DIST_UBUNTU_SERVER: {"10.04", "12.04", "14.04", "16.04", "18.04", "20.04", "22.04", "24.04", - "10", "12", "14", "16", "18", "20", "22", "24"}, - OS_DIST_UBUNTU: {"10.04", "12.04", "14.04", "15.04", "16.04", "17.04", "18.04", "19.04", "20.04", "21.04", "22.04", "23.04", "24.04", - "10", "12", "14", "16", "17", "18", "19", "20", "21", "22", "23", "24"}, + // Ubuntu：Server版补充LTS版本（每2年一个），Desktop版补充主要.04版本 + OS_DIST_UBUNTU_SERVER: {"10.04", "12.04", "14.04", "16.04", "18.04", "20.04", "22.04", "24.04", "26.04", + "10", "12", "14", "16", "18", "20", "22", "24", "26"}, + OS_DIST_UBUNTU: {"10.04", "12.04", "14.04", "15.04", "16.04", "17.04", "18.04", "19.04", "20.04", "21.04", "22.04", "23.04", "24.04", "25.04", "26.04", + "10", "12", "14", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26"}, // OpenSUSE：补充Leap版本，SUSE补充SLES主版本 - OS_DIST_OPEN_SUSE: {"11", "12", "13", "42", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6"}, - OS_DIST_SUSE: {"10", "11", "12", "15", "15 SP1", "15 SP2", "15 SP3", "15 SP4", "15 SP5"}, + OS_DIST_OPEN_SUSE: {"11", "12", "13", "42", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "16.0", "16"}, + OS_DIST_SUSE: {"10", "11", "12", "15", "15 SP1", "15 SP2", "15 SP3", "15 SP4", "15 SP5", "15 SP6", "15 SP7", "16.0", "16.1", "16"}, // Debian：补充从6到最新的13，覆盖所有稳定版 OS_DIST_DEBIAN: {"6", "7", "8", "9", "10", "11", "12", "13"}, // CoreOS：补充Container Linux和Fedora CoreOS的主要版本 OS_DIST_CORE_OS: {"7", "200", "213", "224", "234", "246", "251", "3033"}, // 欧拉OS：补充openEuler和EulerOS完整版本 - OS_DIST_OPEN_EULER: {"2.0 SP1", "2.0 SP2", "2.0 SP3", "2.0 SP8", "3.0", "22.03", "23.09"}, + OS_DIST_OPEN_EULER: {"2.0 SP1", "2.0 SP2", "2.0 SP3", "2.0 SP8", "3.0", "20.03", "22.03", "23.09", "24.03", "25.03"}, OS_DIST_EULER_OS: {"2"}, - // 阿里云Linux：补充1代和2/3代版本 - OS_DIST_ALIYUN: {"1", "2.1903", "2", "3.2104", "3.2304", "3"}, + // 阿里云Linux：补充1代和2/3/4代版本 + OS_DIST_ALIYUN: {"1", "2.1903", "2", "3.2104", "3.2304", "3", "4"}, // 阿里云轻量版：补充完整版本 - OS_DIST_ALIBABA_CLOUD_LINUX: {"2.1903", "2", "3.2104", "3.2304", "3.2404", "3"}, + OS_DIST_ALIBABA_CLOUD_LINUX: {"2.1903", "2", "3.2104", "3.2304", "3.2404", "3", "4"}, // 龙蜥OS：补充7/8系列完整小版本 OS_DIST_ANOLIS: {"7.6", "7.9", "7", "8.2", "8.4", "8.6", "8.8", "8", "9.0", "9.2", "9"}, - // Rocky Linux：补充8/9全系列小版本 - OS_DIST_ROCKY_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9"}, - // Fedora：补充近年主流版本（33到40） - OS_DIST_FEDORA: {"33", "34", "35", "36", "37", "38", "39", "40"}, - // AlmaLinux：补充8/9全系列 - OS_DIST_ALMA_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9"}, + // Rocky Linux：补充8/9/10全系列小版本 + OS_DIST_ROCKY_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9.6", "9.7", "9.8", "9", "10.0", "10.1", "10.2", "10"}, + // Fedora：补充近年主流版本（33到44） + OS_DIST_FEDORA: {"33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44"}, + // AlmaLinux：补充8/9/10全系列 + OS_DIST_ALMA_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9.6", "9.7", "9.8", "9", "10.0", "10.1", "10.2", "10"}, // Amazon Linux：补充1/2/2023版本 OS_DIST_AMAZON_LINUX: {"2022", "2023", "1", "2"}, // Windows Server：补充完整服务器版本 - OS_DIST_WINDOWS_SERVER: {"2003", "2008", "2008 R2", "2012", "2012 R2", "2016", "2019", "2022"}, + OS_DIST_WINDOWS_SERVER: {"2003", "2008", "2008 R2", "2012", "2012 R2", "2016", "2019", "2022", "2025", "25"}, // Windows 桌面版：补充完整版本 OS_DIST_WINDOWS: {"XP", "Vista", "7", "8", "8.1", "10", "11"}, @@ -226,7 +229,7 @@ var imageVersions = map[string][]string{ OS_DIST_TENCENTOS_SERVER: {"2.4", "3.1", "3.2", "3.3", "4.0", "4"}, // 其他Linux：预留空列表，可根据实际场景补充 - OS_DIST_DEEPIN: {"20", "20.9", "21", "21.9", "22", "22.9", "23", "23.9", "Crimson"}, + OS_DIST_DEEPIN: {"20", "20.9", "21", "21.9", "22", "22.9", "23", "23.9", "25.2", "25", "Crimson"}, OS_DIST_OTHER_LINUX: {}, } @@ -261,6 +264,10 @@ func normalizeOsVersion(imageName string, osDist string, osVersion string) strin } } if len(bestMatch) > 0 { + // winserver25 -> Windows Server 2025 + if osDist == OS_DIST_WINDOWS_SERVER && len(bestMatch) == 2 { + return "20" + bestMatch + } return bestMatch } } diff --git a/vendor/yunion.io/x/pkg/util/netutils/http.go b/vendor/yunion.io/x/pkg/util/netutils/http.go index cc9feb643..6d3a75ebd 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/http.go +++ b/vendor/yunion.io/x/pkg/util/netutils/http.go @@ -15,26 +15,47 @@ package netutils import ( + "net" "net/http" "strings" ) +// parseIPLiteral returns s as a plain address if it is one, and "" otherwise. +func parseIPLiteral(s string) string { + s = strings.TrimSpace(s) + if len(s) == 0 { + return "" + } + if ip := net.ParseIP(s); ip != nil { + return ip.String() + } + return "" +} + +// GetHttpRequestIp returns the address of the client that made the request. +// +// The forwarded headers are consulted first, since a proxy in front of the +// service rewrites them. When no proxy is in place they are whatever the +// client sent, so each candidate is parsed and only an actual address is +// returned; a value that is not an address is skipped rather than passed on +// as though it were one. +// +// The connection address is used as the last resort, split with +// net.SplitHostPort so that an IPv6 peer is returned without its brackets and +// without its port. func GetHttpRequestIp(r *http.Request) string { - ipStr := r.Header.Get("X-Forwarded-For") - if len(ipStr) > 0 { - ipList := strings.Split(ipStr, ",") - if len(ipList) > 0 { - return ipList[0] + if forwarded := r.Header.Get("X-Forwarded-For"); len(forwarded) > 0 { + for _, part := range strings.Split(forwarded, ",") { + if ip := parseIPLiteral(part); len(ip) > 0 { + return ip + } } } - ipStr = r.Header.Get("X-Real-Ip") - if len(ipStr) > 0 { - return ipStr + if ip := parseIPLiteral(r.Header.Get("X-Real-Ip")); len(ip) > 0 { + return ip } - ipStr = r.RemoteAddr - colonPos := strings.Index(ipStr, ":") - if colonPos > 0 { - ipStr = ipStr[:colonPos] + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + return host } - return ipStr + return r.RemoteAddr } diff --git a/vendor/yunion.io/x/pkg/util/netutils/ipv6.go b/vendor/yunion.io/x/pkg/util/netutils/ipv6.go index 4c20786f8..161717113 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/ipv6.go +++ b/vendor/yunion.io/x/pkg/util/netutils/ipv6.go @@ -154,6 +154,10 @@ func normalizeIpv6Addr(addrStr string) ([8]uint16, error) { } } } + } else if len(parts) > expectPartCnt { + // More groups than the address can hold. A "::" run can only reduce + // the number of groups, so there is no valid reading of this input. + return addr, errors.Wrapf(errors.ErrInvalidFormat, "%s too many groups", addrStr) } else { for i := 0; i < len(parts); i++ { partStr := strings.TrimSpace(parts[i]) diff --git a/vendor/yunion.io/x/pkg/util/netutils/netutils.go b/vendor/yunion.io/x/pkg/util/netutils/netutils.go index 929520a1b..97480dcf2 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/netutils.go +++ b/vendor/yunion.io/x/pkg/util/netutils/netutils.go @@ -181,12 +181,21 @@ func (ar IPV4AddrRange) ContainsRange(ar2 IPV4AddrRange) bool { return ar.start <= ar2.start && ar.end >= ar2.end } +// Random returns an address from the range, excluding end. A range that +// covers a single address returns that address. func (ar IPV4AddrRange) Random() IPV4Addr { - return IPV4Addr(uint32(ar.start) + uint32(rand.Intn(int(uint32(ar.end)-uint32(ar.start))))) + if ar.start >= ar.end { + return ar.start + } + // int64 so the span of a very large range cannot overflow the argument + // to the random source. + span := int64(uint32(ar.end) - uint32(ar.start)) + return IPV4Addr(uint32(ar.start) + uint32(rand.Int63n(span))) } func (ar IPV4AddrRange) AddressCount() int { - return int(uint32(ar.end) - uint32(ar.start) + 1) + // 64 bit arithmetic so a full range does not wrap around to zero. + return int(uint64(uint32(ar.end)) - uint64(uint32(ar.start)) + 1) } func (ar IPV4AddrRange) String() string { @@ -360,10 +369,18 @@ func (ar IPV4AddrRange) equals(ar2 IPV4AddrRange) bool { return ar.start == ar2.start && ar.end == ar2.end } +// Masklen2Mask returns the network mask for a prefix length. +// +// A length above 32 cannot be represented and is treated as 32, i.e. a single +// host, rather than shifting past the width of the mask and wrapping around to +// a match-all mask. func Masklen2Mask(maskLen int8) IPV4Addr { if maskLen < 0 { panic("negative masklen") } + if maskLen > 32 { + maskLen = 32 + } return IPV4Addr(^(uint32(1<<(32-uint8(maskLen))) - 1)) } @@ -392,6 +409,25 @@ func Mask2Len(mask IPV4Addr) int8 { return int8(bits.LeadingZeros32(^uint32(mask))) } +// ParsePrefix parses an IPv4 prefix written as "address/masklen", or as a +// bare address, which is taken to be a /32. An empty string parses as +// 0.0.0.0/32 rather than being rejected. +// +// The prefix length may also be written as a dotted-decimal mask, e.g. +// "10.0.0.0/255.0.0.0". A dotted mask is converted by counting its leading +// one bits, so the result always has a contiguous mask: +// "1.2.3.4/255.0.255.0" yields 1.0.0.0/8 rather than being rejected. +// +// The numeric form is read with strconv.Atoi, so it also accepts a leading +// sign and leading zeros: "+8" and "024" parse as 8 and 24. +// +// These accepted forms are more permissive than net.ParseCIDR. Note that +// regutils.MatchCIDR and regutils.MatchIP4Addr reject the dotted-mask and +// leading-zero spellings that this function accepts, so a value checked with +// one of those and then normalised here can end up covering something other +// than what was checked. Prefer net.ParseCIDR where the stricter grammar is +// wanted; the lenient spellings are kept for compatibility with prefixes +// already stored by callers. func ParsePrefix(prefix string) (IPV4Addr, int8, error) { slash := strings.IndexByte(prefix, '/') if slash > 0 { @@ -438,7 +474,15 @@ func NewIPV4Prefix(prefix string) (IPV4Prefix, error) { return pref, nil } +// NewIPV4PrefixFromAddr builds a prefix from an address and a prefix length. +// +// A length above 32 is clamped to 32 so that the stored MaskLen always agrees +// with the address and range it describes. A negative length keeps its +// existing behaviour of panicking. func NewIPV4PrefixFromAddr(addr IPV4Addr, masklen int8) IPV4Prefix { + if masklen > 32 { + masklen = 32 + } pref := IPV4Prefix{ Address: addr.NetAddr(masklen), MaskLen: masklen, @@ -465,14 +509,28 @@ const ( multicastPrefix = "224.0.0.0/4" ) +// Ranges that cannot be a public address but are not part of the private +// ranges used to classify guest addresses. They are consulted by +// IsExitAddress only, so that IsPrivate keeps its current meaning for the +// callers that use it for address allocation and DHCP. +var reservedPrefixes = []string{ + "0.0.0.0/8", // "this network" + "192.0.2.0/24", // TEST-NET-1 + "198.51.100.0/24", // TEST-NET-2 + "203.0.113.0/24", // TEST-NET-3 + "240.0.0.0/4", // reserved, includes the limited broadcast address +} + var privateIPRanges []IPV4AddrRange var customizedPrivateIPRanges []IPV4AddrRange +var reservedIPRanges []IPV4AddrRange var hostLocalIPRange IPV4AddrRange var linkLocalIPRange IPV4AddrRange var multicastIPRange IPV4AddrRange func init() { initPrivateIPRanges() + initReservedIPRanges() prefix, _ := NewIPV4Prefix(hostlocalPrefix) hostLocalIPRange = prefix.ToIPRange() @@ -502,6 +560,22 @@ func initPrivateIPRanges() { } } +func initReservedIPRanges() { + reservedIPRanges = make([]IPV4AddrRange, 0, len(reservedPrefixes)) + for _, prefix := range reservedPrefixes { + p, err := NewIPV4Prefix(prefix) + if err != nil { + continue + } + reservedIPRanges = append(reservedIPRanges, p.ToIPRange()) + } +} + +// GetReservedIPRanges returns the ranges that IsReserved consults. +func GetReservedIPRanges() []IPV4AddrRange { + return append([]IPV4AddrRange(nil), reservedIPRanges...) +} + func SetPrivatePrefixes(pref []string) { customizedPrivateIPRanges = make([]IPV4AddrRange, 0) for _, prefix := range pref { @@ -538,8 +612,23 @@ func IsMulticast(addr IPV4Addr) bool { return multicastIPRange.Contains(addr) } +// IsReserved reports whether addr is in a range that is reserved and can +// never be a public address. +func IsReserved(addr IPV4Addr) bool { + for _, ipRange := range reservedIPRanges { + if ipRange.Contains(addr) { + return true + } + } + return false +} + +// IsExitAddress reports whether addr can be a public address, i.e. one that a +// guest can reach directly. Reserved ranges are excluded along with the +// private, host local, link local and multicast ones. func IsExitAddress(addr IPV4Addr) bool { - return !IsPrivate(addr) && !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr) + return !IsPrivate(addr) && !IsReserved(addr) && + !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr) } func MacUnpackHex(mac string) string { diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go b/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go index 7d422ecf6..ec0afc83b 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go @@ -24,34 +24,99 @@ const ( TAG_OLD_DEPRECATED_BY = "deprecated-by" ) +// expandAmbiguousPrefix prepends the prefix given by the +// TAG_AMBIGUOUS_PREFIX tag to the fields sharing a name with another field. +// +// Expanding a prefix can itself collide with a name that is already in use, so +// the expansion is repeated until the names settle. A field whose expanded +// name would take over a name owned by a field outside its own ambiguous +// group keeps its original name, so that the expansion never introduces a new +// ambiguity. func expandAmbiguousPrefix(fields SStructFieldValueSet) SStructFieldValueSet { - keyIndexMap := make(map[string][]int) - for i := range fields { - if fields[i].Info.Ignore { - continue - } - key := fields[i].Info.MarshalName() - values, ok := keyIndexMap[key] - if !ok { - values = make([]int, 0, 2) + prefixed := make(map[int]bool) + for { + keyIndexMap := make(map[string][]int) + for i := range fields { + if fields[i].Info.Ignore { + continue + } + key := fields[i].Info.MarshalName() + values, ok := keyIndexMap[key] + if !ok { + values = make([]int, 0, 2) + } + keyIndexMap[key] = append(values, i) } - keyIndexMap[key] = append(values, i) - } - for _, indexes := range keyIndexMap { - if len(indexes) > 1 { + changed := false + for _, indexes := range keyIndexMap { + if len(indexes) < 2 { + continue + } // ambiguous found for _, idx := range indexes { - if amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]; ok { - fields[idx].Info.Name = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name) - if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok { - fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) - } - if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok { - fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) + if prefixed[idx] { + continue + } + amPrefix, ok := fields[idx].Info.Tag(TAG_AMBIGUOUS_PREFIX) + if !ok { + continue + } + expanded := fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name) + if takenByOther(fields, expanded, indexes) { + continue + } + info := fields[idx].Info + info.Name = expanded + _, newDepBy := info.tags[TAG_DEPRECATED_BY] + _, oldDepBy := info.tags[TAG_OLD_DEPRECATED_BY] + if newDepBy || oldDepBy { + // the tags may still be shared with other callers + info.copyTags() + } + if newDepBy { + info.tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, info.tags[TAG_DEPRECATED_BY]) + } + if oldDepBy { + info.tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, info.tags[TAG_OLD_DEPRECATED_BY]) + } + if len(info.aliases) > 0 { + info.copyAliases() + for i := range info.aliases { + info.aliases[i] = fmt.Sprintf("%s%s", amPrefix, info.aliases[i]) } } + prefixed[idx] = true + changed = true } } + if !changed { + break + } } return fields } + +// takenByOther reports whether name is already used by a field outside group +func takenByOther(fields SStructFieldValueSet, name string, group []int) bool { + for i := range fields { + if fields[i].Info.Ignore { + continue + } + if fields[i].Info.MarshalName() != name { + continue + } + if !containsIndex(group, i) { + return true + } + } + return false +} + +func containsIndex(indexes []int, idx int) bool { + for _, i := range indexes { + if i == idx { + return true + } + } + return false +} diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/doc.go b/vendor/yunion.io/x/pkg/util/reflectutils/doc.go index 8c0a81831..55acd8687 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/doc.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/doc.go @@ -12,4 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package reflectutils walks struct values by reflection, mainly to give the +// json encoder and decoder in yunion.io/x/jsonutils the field names and the +// tags to honour. +// +// # Field naming +// +// Fields are collected in declaration order, with the fields of an embedded +// struct expanded in place. A name is taken from the "name" tag, then from +// the "json" tag, and finally from the kebab form of the field name, so it can +// be looked up by any of those. +// +// Two collected fields can carry the same name. An embedded struct may bring +// in a field that another embedded struct also contributes, and unlike +// encoding/json an outer field does not hide a field of an embedded struct. +// All the fields sharing a name are reported, and a value decoded into the +// struct is written to every one of them, so that code reaching the field +// through any of the embedded paths sees it. Shadowing is deliberately not +// implemented: the list inputs of the resource APIs rely on this, e.g. +// ServerListInput carries several fields named "domain" coming from different +// embedded structs. +// +// To set such fields apart, tag an embedded struct with +// TAG_AMBIGUOUS_PREFIX, whose value is prepended to the names of the fields +// that embedded struct contributes. package reflectutils // import "yunion.io/x/pkg/util/reflectutils" diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go b/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go index 2f994669e..a42aa439e 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go @@ -29,12 +29,32 @@ import ( // This struct has unexported fields initialized by exported functions in this // package. Do not construct a literal or modify the exported fields in an // unmanaged way +// +// The tags of a field are read through Tag and TagMap. The map behind them +// is shared with the other callers of the fetch functions, which is what +// makes reading a field several times cheap, so a caller may only modify what +// TagMap hands out, which is a copy of its own. type SStructFieldInfo struct { // True if the field has json tag `json:"-"` - Ignore bool + Ignore bool + + // True if an empty string, slice, map, struct or dict is left out of + // the json object. A number and a boolean are not governed by this, + // see OmitZero and OmitFalse. + // + // An empty value is left out by default, whether or not the json tag + // asks for it; `allowempty` turns that off and `omitempty` states it. OmitEmpty bool + + // True if a false boolean is left out of the json object. A false is + // written by default; `omitfalse` turns that off and `allowfalse` + // states it. OmitFalse bool - OmitZero bool + + // True if a zero number is left out of the json object. A zero is + // written by default; `omitzero` turns that off and `allowzero` states + // it. + OmitZero bool // Name can take the following values, in descreasing preference // @@ -42,39 +62,45 @@ type SStructFieldInfo struct { // 2. name of "json" tag, when it's not for ignoration // 3. kebab form of FieldName concatenated with "_" when Ignore is false // 4. empty string - Name string - FieldName string + Name string + + // FieldName is the name of the go struct field + FieldName string + kebabFieldName string - ForceString bool - Tags map[string]string - Aliases []string -} + // True if the field has the "string" json tag option, which writes the + // value as a json string + ForceString bool -func (s *SStructFieldInfo) updateTags(k, v string) { - s.Tags[k] = v + // tags holds the tags of the field keyed by tag name, a tag without a + // value being mapped to the empty string. The map is shared with the + // other callers of the fetch functions, so it has to be copied with + // copyTags before being written to; read it through Tag or TagMap. + tags map[string]string + + // aliases are the other names the field is looked up by, taken from + // the "alias" tag. Like tags it is shared, and has to be copied with + // copyAliases before being written to. + aliases []string } -func (s SStructFieldInfo) deepCopy() *SStructFieldInfo { - scopy := SStructFieldInfo{ - Ignore: s.Ignore, - OmitEmpty: s.OmitEmpty, - OmitFalse: s.OmitFalse, - OmitZero: s.OmitZero, - Name: s.Name, - FieldName: s.FieldName, - ForceString: s.ForceString, - kebabFieldName: s.kebabFieldName, - } - tags := make(map[string]string, len(s.Tags)) - for k, v := range s.Tags { +// copyTags takes a private copy of the tags, so that they can be written to +// without touching the ones this info was read out of. +func (s *SStructFieldInfo) copyTags() { + tags := make(map[string]string, len(s.tags)+1) + for k, v := range s.tags { tags[k] = v } - scopy.Tags = tags - aliases := make([]string, len(s.Aliases)) - copy(aliases, s.Aliases) - scopy.Aliases = aliases - return &scopy + s.tags = tags +} + +// copyAliases takes a private copy of the aliases, so that they can be +// written to without touching the ones this info was read out of. +func (s *SStructFieldInfo) copyAliases() { + aliases := make([]string, len(s.aliases)) + copy(aliases, s.aliases) + s.aliases = aliases } func ParseStructFieldJsonInfo(sf reflect.StructField) SStructFieldInfo { @@ -89,8 +115,8 @@ func ParseFieldJsonInfo(name string, tag reflect.StructTag) SStructFieldInfo { info.OmitZero = false info.OmitFalse = false - info.Tags = utils.TagMap(tag) - if val, ok := info.Tags["json"]; ok { + info.tags = utils.TagMap(tag) + if val, ok := info.tags["json"]; ok { keys := strings.Split(val, ",") if len(keys) > 0 { if keys[0] == "-" { @@ -124,14 +150,14 @@ func ParseFieldJsonInfo(name string, tag reflect.StructTag) SStructFieldInfo { } } } - if val, ok := info.Tags["name"]; ok { + if val, ok := info.tags["name"]; ok { info.Name = val } if !info.Ignore && len(info.Name) == 0 { info.Name = info.kebabFieldName } - if val, ok := info.Tags["alias"]; !info.Ignore && ok { - info.Aliases = strings.Split(val, ",") + if val, ok := info.tags["alias"]; !info.Ignore && ok { + info.aliases = strings.Split(val, ",") } return info } @@ -145,6 +171,23 @@ func (info *SStructFieldInfo) MarshalName() string { return info.kebabFieldName } +// Tag returns the value of the tag named name and whether the field has it. +// A tag without a value is reported as present with an empty value. +func (info *SStructFieldInfo) Tag(name string) (string, bool) { + val, ok := info.tags[name] + return val, ok +} + +// TagMap returns a copy of the tags of the field, which the caller owns and +// is free to modify. +func (info *SStructFieldInfo) TagMap() map[string]string { + tags := make(map[string]string, len(info.tags)) + for k, v := range info.tags { + tags[k] = v + } + return tags +} + type SStructFieldValue struct { Info *SStructFieldInfo Value reflect.Value @@ -152,9 +195,41 @@ type SStructFieldValue struct { Parent *SEmbedStructFieldValue } +// SEmbedStructFieldValue ties the fields enumerated for a nil embedded struct +// to the struct they belong to. Field is the embedded pointer on the struct, +// Value is the value the fields were enumerated out of, and Parent is the +// entry of the enclosing embedded struct, if any. type SEmbedStructFieldValue struct { Field reflect.Value Value reflect.Value + + Parent *SEmbedStructFieldValue +} + +// adoptEmbeddedStruct assigns Value to Field for every nil embedded struct +// the field was enumerated through, which puts the enumerated fields back on +// the struct. The assignments are made from the outermost embedded struct +// inwards, so that each one lands on a struct that is already part of the +// real one. It reports whether the field is backed by the struct afterwards, +// which is what writing to it requires. +func (v *SStructFieldValue) adoptEmbeddedStruct() bool { + chain := make([]*SEmbedStructFieldValue, 0, 2) + for p := v.Parent; p != nil; p = p.Parent { + chain = append(chain, p) + } + for i := len(chain) - 1; i >= 0; i -= 1 { + p := chain[i] + if !p.Field.IsValid() || p.Field.Kind() != reflect.Ptr || !p.Field.IsNil() { + // nothing to adopt, the pointer is already there + continue + } + if !p.Field.CanSet() { + // the struct can not hold the embedded pointer + return false + } + p.Field.Set(p.Value) + } + return true } type SStructFieldValueSet []SStructFieldValue @@ -225,6 +300,19 @@ func fetchStructFieldValueSet2(dataValue reflect.Value, allocatePtr bool, tags m } func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags map[string]string, includeIgnore bool, parent *SEmbedStructFieldValue) SStructFieldValueSet { + if !dataValue.IsValid() || dataValue.Kind() != reflect.Struct { + // A zero Value, a nil or a non struct value has no field to + // enumerate. Report it as such rather than through a panic. + return SStructFieldValueSet{} + } + if allocatePtr && !dataValue.CanAddr() { + // The value can not be modified in place, so a nil embedded + // pointer can not be allocated on it. The fields of such a + // pointer are enumerated out of a value allocated on the side + // instead, the same way as when the caller does not ask for + // allocation; see the note below for how to write to them. + allocatePtr = false + } fields := SStructFieldValueSet{} dataType := dataValue.Type() fieldInfos := fetchCacheStructFieldInfos(dataType) @@ -252,9 +340,17 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m if fv.Kind() == reflect.Ptr && allocatePtr { fv.Set(reflect.New(fv.Type().Elem())) } else if fv.Kind() == reflect.Ptr && !allocatePtr { + // The embedded pointer is nil, so the fields are + // enumerated out of a value allocated on the side. + // Value is a pointer to that value and the fields + // below are the fields of the value it points at, + // so assigning Value to Field puts them back on + // the struct. Callers writing to those fields + // must do so first. efv = &SEmbedStructFieldValue{ - Field: fv, - Value: reflect.New(fv.Type().Elem()), + Field: fv, + Value: reflect.New(fv.Type().Elem()), + Parent: parent, } fv = efv.Value } else { @@ -274,10 +370,10 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m continue } } - fieldInfo := fieldInfos[sf.Name].deepCopy() + fieldInfo := fieldInfos[sf.Name] if !fieldInfo.Ignore || includeIgnore { structFieldVaule := SStructFieldValue{ - Info: fieldInfo, + Info: &fieldInfo, Value: fv, } if parent != nil { @@ -289,6 +385,7 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m if len(tags) > 0 { for i := range fields { fieldName := fields[i].Info.MarshalName() + owned := false for k, v := range tags { target := "" pos := strings.Index(k, "->") @@ -299,7 +396,12 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m if len(target) > 0 && target != fieldName { continue } - fields[i].Info.updateTags(k, v) + if !owned { + // the tags may still be shared with other callers + fields[i].Info.copyTags() + owned = true + } + fields[i].Info.tags[k] = v } } } @@ -344,7 +446,7 @@ func (fields SStructFieldValueSet) GetStructFieldIndexes2(name string, strictMod ret = append(ret, i) } else if info.FieldName == capName { ret = append(ret, i) - } else if len(info.Aliases) > 0 && utils.IsInArray(name, info.Aliases) { + } else if len(info.aliases) > 0 && utils.IsInArray(name, info.aliases) { ret = append(ret, i) } } diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go index 4d61d9d46..7f3370bd9 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go @@ -19,6 +19,7 @@ import ( "reflect" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" ) /* @@ -96,13 +97,19 @@ func fetchStructFieldNameValues(dataType reflect.Type, dataValue reflect.Value, } */ +// FindStructFieldValue returns the field of dataValue named name. The field +// has to be writable, so a field reached through a nil embedded pointer is +// only returned once that pointer has been put in place, which this does. func FindStructFieldValue(dataValue reflect.Value, name string) (reflect.Value, bool) { set := FetchStructFieldValueSet(dataValue) - val, find := set.GetValue(name) - if find && val.CanSet() { - return val, true + idx := set.GetStructFieldIndex(name) + if idx < 0 { + return reflect.Value{}, false } - return reflect.Value{}, false + if !set[idx].adoptEmbeddedStruct() || !set[idx].Value.CanSet() { + return reflect.Value{}, false + } + return set[idx].Value, true } func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}, bool) { @@ -111,35 +118,53 @@ func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{} } func FillEmbededStructValue(container reflect.Value, embed reflect.Value) bool { + if !container.IsValid() || container.Kind() != reflect.Struct || !embed.IsValid() { + return false + } containerType := container.Type() + embedType := embed.Type() for i := 0; i < containerType.NumField(); i += 1 { fieldType := containerType.Field(i) + if fieldType.Type.Kind() != reflect.Struct || !fieldType.Anonymous { + continue + } fieldValue := container.Field(i) - if fieldType.Type.Kind() == reflect.Struct && fieldType.Anonymous { - if fieldType.Type == embed.Type() { - fieldValue.Set(embed) - return true - } else { - filled := FillEmbededStructValue(fieldValue, embed) - if filled { - return true - } - } + if !fieldValue.CanSet() { + // an unexported embedded struct can not be assigned to, and + // neither can anything inside it + continue + } + if fieldType.Type == embedType { + fieldValue.Set(embed) + return true + } + if FillEmbededStructValue(fieldValue, embed) { + return true } - } return false } +// SetStructFieldValue sets the field of structValue named fieldName to val. +// A field reached through a nil embedded pointer is written only once that +// pointer has been put in place, which this does. func SetStructFieldValue(structValue reflect.Value, fieldName string, val reflect.Value) bool { set := FetchStructFieldValueSet(structValue) - target, find := set.GetValue(fieldName) - if !find { + idx := set.GetStructFieldIndex(fieldName) + if idx < 0 { + return false + } + if !set[idx].adoptEmbeddedStruct() { return false } + target := set[idx].Value if !target.CanSet() { return false } + if !val.IsValid() || !val.Type().AssignableTo(target.Type()) { + // report a failure instead of letting reflect.Value.Set panic + return false + } target.Set(val) return true } @@ -161,15 +186,34 @@ func ExpandInterface(val interface{}) []interface{} { func getAnonymouStructPointer(structValue reflect.Value, targetType reflect.Type) interface{} { structType := structValue.Type() if structType == targetType { + if !structValue.CanInterface() { + // the value was reached through an unexported field + return nil + } return structValue.Addr().Interface() } for i := 0; i < structValue.NumField(); i += 1 { fieldType := structType.Field(i) - if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct { - ptr := getAnonymouStructPointer(structValue.Field(i), targetType) - if ptr != nil { - return ptr + if !fieldType.Anonymous || !gotypes.IsFieldExportable(fieldType.Name) { + // an unexported embedded struct can not be pointed at + continue + } + fieldValue := structValue.Field(i) + fieldT := fieldType.Type + if fieldT.Kind() == reflect.Ptr { + // an embedded pointer that is nil has nothing to point at + if fieldValue.IsNil() { + continue } + fieldValue = fieldValue.Elem() + fieldT = fieldT.Elem() + } + if fieldT.Kind() != reflect.Struct { + continue + } + ptr := getAnonymouStructPointer(fieldValue, targetType) + if ptr != nil { + return ptr } } return nil @@ -214,11 +258,18 @@ func StructContains(type1 reflect.Type, type2 reflect.Type) bool { } for i := 0; i < type1.NumField(); i += 1 { field := type1.Field(i) - if field.Anonymous && field.Type.Kind() == reflect.Struct { - contains := StructContains(field.Type, type2) - if contains { - return true - } + if !field.Anonymous { + continue + } + fieldType := field.Type + if fieldType.Kind() == reflect.Ptr { + fieldType = fieldType.Elem() + } + if fieldType.Kind() != reflect.Struct { + continue + } + if StructContains(fieldType, type2) { + return true } } return false diff --git a/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go b/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go index 7a6e1d7a5..37de0faaf 100644 --- a/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go +++ b/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go @@ -76,6 +76,9 @@ func (key EncryptedKey) decryptKey(privateKey *rsa.PrivateKey) ([]byte, error) { encAlg := key.EncryptionMethod.Algorithm switch encAlg { case "http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p": + if key.EncryptionMethod.DigestMethod == nil { + return nil, errors.Wrap(errors.ErrInvalidFormat, "missing DigestMethod") + } var shaAlg hash.Hash hashAlg := key.EncryptionMethod.DigestMethod.Algorithm switch hashAlg { @@ -99,6 +102,9 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error if err != nil { return nil, errors.Wrap(err, "base64.StdEncoding.DecodeString") } + if data.KeyInfo.EncryptedKey == nil { + return nil, errors.Wrap(errors.ErrInvalidFormat, "missing KeyInfo.EncryptedKey") + } key, err := data.KeyInfo.EncryptedKey.decryptKey(privateKey) if err != nil { return nil, errors.Wrap(err, "KeyInfo.EncryptedKey.decryptKey") @@ -112,7 +118,36 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error } } +// stripPKCS7Padding removes the padding that XML Encryption appends to the +// last block of a CBC payload. +// +// Data that does not carry valid padding is returned unchanged rather than +// reported, so that this cannot be used to tell one payload from another. +func stripPKCS7Padding(data []byte, blockSize int) []byte { + if len(data) == 0 || len(data)%blockSize != 0 { + return data + } + n := int(data[len(data)-1]) + if n == 0 || n > blockSize || n > len(data) { + return data + } + for _, b := range data[len(data)-n:] { + if int(b) != n { + return data + } + } + return data[:len(data)-n] +} + func decryptAesCbc(key []byte, secret []byte) ([]byte, error) { + // The payload is an IV followed by whole ciphertext blocks. Anything + // shorter, or not a whole number of blocks, cannot be decrypted. + if len(secret) < 2*aes.BlockSize || len(secret)%aes.BlockSize != 0 { + return nil, errors.Wrapf(errors.ErrInvalidFormat, + "ciphertext of %d bytes is not a whole number of %d byte blocks preceded by an IV", + len(secret), aes.BlockSize) + } + c, err := aes.NewCipher(key) if err != nil { return nil, errors.Wrap(err, "aes.NewCipher") @@ -125,5 +160,5 @@ func decryptAesCbc(key []byte, secret []byte) ([]byte, error) { decrypter.CryptBlocks(data, data) - return data, nil + return stripPKCS7Padding(data, aes.BlockSize), nil } diff --git a/vendor/yunion.io/x/pkg/util/samlutils/util.go b/vendor/yunion.io/x/pkg/util/samlutils/util.go index 0e934ee0d..dee143c4d 100644 --- a/vendor/yunion.io/x/pkg/util/samlutils/util.go +++ b/vendor/yunion.io/x/pkg/util/samlutils/util.go @@ -20,7 +20,9 @@ import ( "crypto/rsa" "encoding/base64" "fmt" + "html" "io" + "sort" "strings" "github.com/ma314smith/signedxml" @@ -92,14 +94,23 @@ func SAMLEncode(input []byte) (string, error) { return base64.StdEncoding.EncodeToString(comp), nil } +// SAMLForm renders an auto-submitting HTML form carrying the given +// parameters. Both the action and the values are escaped, and the keys are +// sorted so the output is stable for a given input. func SAMLForm(action string, attrs map[string]string) string { form := strings.Builder{} // form.WriteString(``) form.WriteString(`
`) - for k, v := range attrs { - form.WriteString(fmt.Sprintf("", k, v)) + keys := make([]string, 0, len(attrs)) + for k := range attrs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + form.WriteString(fmt.Sprintf("", + html.EscapeString(k), html.EscapeString(attrs[k]))) } form.WriteString(``) form.WriteString("