From d51d174584188d4a8020e802fdbd2a10a936ee69 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 9 Apr 2026 12:34:28 +0200 Subject: [PATCH] STAC-23457 Make sure stackpacks are always downloaded from within the stackgraph or settings buckets --- cmd/settings/restore.go | 4 +- cmd/settings/restore_test.go | 72 ++++++++++ cmd/stackgraph/restore.go | 4 +- cmd/stackgraph/restore_test.go | 64 +++++++++ internal/foundation/config/config.go | 19 ++- internal/foundation/config/config_test.go | 33 +++++ .../config/testdata/validConfigMapConfig.yaml | 3 +- .../config/testdata/validConfigMapOnly.yaml | 1 + .../testdata/validStorageConfigMapConfig.yaml | 1 + .../validStorageConfigMapNoStackpacks.yaml | 134 ++++++++++++++++++ .../testdata/validStorageConfigMapOnly.yaml | 1 + .../scripts/restore-settings-backup.sh | 4 +- .../scripts/restore-stackgraph-backup.sh | 4 +- 13 files changed, 324 insertions(+), 20 deletions(-) create mode 100644 internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml diff --git a/cmd/settings/restore.go b/cmd/settings/restore.go index 5a8ea32..05e09c8 100644 --- a/cmd/settings/restore.go +++ b/cmd/settings/restore.go @@ -197,14 +197,13 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En commonVar := []corev1.EnvVar{ {Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket}, {Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix}, - {Name: "BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX", Value: config.Settings.StackpacksS3Prefix}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, {Name: "ZOOKEEPER_QUORUM", Value: config.Settings.Restore.ZookeeperQuorum}, {Name: "BACKUP_CONFIGURATION_UPLOAD_REMOTE", Value: strconv.FormatBool(config.GlobalBackupEnabled())}, - {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks || config.Stackpacks == nil)}, } if fromPVC { // Force PVC mode in the shell script, suppress local bucket @@ -214,6 +213,7 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En } if config.Stackpacks != nil { commonVar = append(commonVar, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksURI}) + commonVar = append(commonVar, corev1.EnvVar{Name: "BACKUP_CONFIGURATION_STACKPACKS_DIR", Value: config.Stackpacks.BackupDirectory}) } commonVar = append(commonVar, extraEnvVar...) return commonVar diff --git a/cmd/settings/restore_test.go b/cmd/settings/restore_test.go index 30233d1..e4e9c67 100644 --- a/cmd/settings/restore_test.go +++ b/cmd/settings/restore_test.go @@ -7,6 +7,78 @@ import ( "github.com/stretchr/testify/assert" ) +func TestBuildEnvVar_SkipStackpacksWhenMissing(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + skipStackpacksFlag bool + expectedValue string + }{ + { + name: "stackpacks nil and flag false", + stackpacks: nil, + skipStackpacksFlag: false, + expectedValue: "true", + }, + { + name: "stackpacks nil and flag true", + stackpacks: nil, + skipStackpacksFlag: true, + expectedValue: "true", + }, + { + name: "stackpacks present and flag false", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: false, + expectedValue: "false", + }, + { + name: "stackpacks present and flag true", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: true, + expectedValue: "true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set the package-level flags + skipStackpacks = tt.skipStackpacksFlag + fromPVC = false + + cfg := &config.Config{ + Stackpacks: tt.stackpacks, + Storage: config.StorageConfig{ + GlobalBackupEnabled: true, + Service: config.ServiceConfig{Name: "storage", Port: 9000}, + }, + Settings: config.SettingsConfig{ + Bucket: "settings-backup", + Restore: config.SettingsRestoreConfig{ + ZookeeperQuorum: "zk:2181", + }, + }, + } + envVars := buildEnvVar(nil, cfg) + + var skipValue string + for _, env := range envVars { + if env.Name == "SKIP_STACKPACKS" { + skipValue = env.Value + break + } + } + assert.Equal(t, tt.expectedValue, skipValue) + }) + } +} + func TestBuildVolumeMounts_StackpacksLocalFileURI(t *testing.T) { tests := []struct { name string diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index ee620da..3226831 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -278,17 +278,17 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, - {Name: "BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX", Value: config.Stackgraph.StackpacksS3Prefix}, {Name: "BACKUP_STACKGRAPH_MULTIPART_ARCHIVE", Value: strconv.FormatBool(config.Stackgraph.MultipartArchive)}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, {Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()}, {Name: "ZOOKEEPER_QUORUM", Value: config.Stackgraph.Restore.ZookeeperQuorum}, - {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks)}, + {Name: "SKIP_STACKPACKS", Value: strconv.FormatBool(skipStackpacks || config.Stackpacks == nil)}, } if config.Stackpacks != nil { env = append(env, corev1.EnvVar{Name: "CONFIG_FORCE_stackstate_stackPacks_localStackPacksUri", Value: config.Stackpacks.LocalStackPacksURI}) + env = append(env, corev1.EnvVar{Name: "BACKUP_STACKGRAPH_STACKPACKS_DIR", Value: config.Stackpacks.BackupDirectory}) } return env } diff --git a/cmd/stackgraph/restore_test.go b/cmd/stackgraph/restore_test.go index 77de1a1..7278244 100644 --- a/cmd/stackgraph/restore_test.go +++ b/cmd/stackgraph/restore_test.go @@ -7,6 +7,70 @@ import ( "github.com/stretchr/testify/assert" ) +func TestBuildRestoreEnvVars_SkipStackpacksWhenMissing(t *testing.T) { + tests := []struct { + name string + stackpacks *config.StackpacksConfig + skipStackpacksFlag bool + expectedValue string + }{ + { + name: "stackpacks nil and flag false", + stackpacks: nil, + skipStackpacksFlag: false, + expectedValue: "true", + }, + { + name: "stackpacks nil and flag true", + stackpacks: nil, + skipStackpacksFlag: true, + expectedValue: "true", + }, + { + name: "stackpacks present and flag false", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: false, + expectedValue: "false", + }, + { + name: "stackpacks present and flag true", + stackpacks: &config.StackpacksConfig{ + LocalStackPacksURI: "/var/stackpacks_local", + BackupDirectory: "stackpacks/", + }, + skipStackpacksFlag: true, + expectedValue: "true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set the package-level flag + skipStackpacks = tt.skipStackpacksFlag + + cfg := &config.Config{ + Stackpacks: tt.stackpacks, + Storage: config.StorageConfig{ + Service: config.ServiceConfig{Name: "storage", Port: 9000}, + }, + } + envVars := buildRestoreEnvVars("backup.graph", cfg) + + var skipValue string + for _, env := range envVars { + if env.Name == "SKIP_STACKPACKS" { + skipValue = env.Value + break + } + } + assert.Equal(t, tt.expectedValue, skipValue) + }) + } +} + func TestBuildRestoreVolumeMounts_StackpacksLocalFileURI(t *testing.T) { tests := []struct { name string diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 61d652b..07d8a90 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -37,6 +37,7 @@ type StackpacksConfig struct { PlatformVersion string `yaml:"platformVersion"` LocalStackPacksURI string `yaml:"localStackPacksUri" validate:"required"` PVC string `yaml:"pvc"` + BackupDirectory string `yaml:"backupDir" validate:"required"` } // GetBaseURL returns the StackState base URL, preferring the top-level stackpacks section @@ -175,11 +176,10 @@ type StorageConfig struct { // StackgraphConfig holds Stackgraph backup-specific configuration type StackgraphConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` - MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` - Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` + Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` } type VictoriaMetricsConfig struct { @@ -210,11 +210,10 @@ type StackgraphRestoreConfig struct { } type SettingsConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - StackpacksS3Prefix string `yaml:"stackpacksS3Prefix"` - LocalBucket string `yaml:"localBucket"` - Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + LocalBucket string `yaml:"localBucket"` + Restore SettingsRestoreConfig `yaml:"restore" validate:"required"` } type SettingsRestoreConfig struct { diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 2719e87..07fecc9 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -407,6 +407,39 @@ func TestLoadConfig_Storage_WithSecretOverride(t *testing.T) { assert.Equal(t, "secret-storage-secret-key", config.GetStorageSecretKey()) } +func TestLoadConfig_MissingStackpacksSection(t *testing.T) { + fakeClient := fake.NewClientset() + validConfigYAML := loadTestData(t, "validStorageConfigMapNoStackpacks.yaml") + + // Create ConfigMap without stackpacks section + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "backup-config", + Namespace: "test-ns", + }, + Data: map[string]string{ + "config": validConfigYAML, + }, + } + _, err := fakeClient.CoreV1().ConfigMaps("test-ns").Create( + context.Background(), cm, metav1.CreateOptions{}, + ) + require.NoError(t, err) + + // Load config + config, err := LoadConfig(fakeClient, "test-ns", "backup-config", "") + + // Assertions + require.NoError(t, err) + assert.NotNil(t, config) + // When stackpacks section is missing, Stackpacks should be nil + assert.Nil(t, config.Stackpacks) + // baseUrl/receiverBaseUrl/platformVersion should fall back to settings.restore values + assert.Equal(t, "http://suse-observability-server:7070", config.GetBaseURL()) + assert.Equal(t, "http://suse-observability-receiver:7077", config.GetReceiverBaseURL()) + assert.Equal(t, "5.2.0", config.GetPlatformVersion()) +} + func TestLoadConfig_ConfigMapNotFound(t *testing.T) { fakeClient := fake.NewClientset() diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index 60efe2a..4437c07 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -65,6 +65,7 @@ stackpacks: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" localStackPacksUri: "/var/stackpacks_local" + backupDir: stackpacks/ # Minio configuration for S3-compatible storage minio: @@ -84,7 +85,6 @@ stackgraph: # S3 prefix path for backups s3Prefix: "" # S3 prefix path for stackpacks backups - stackpacksS3Prefix: "stackpacks/" # Archive split to multiple parts multipartArchive: true # Restore configuration @@ -148,7 +148,6 @@ victoriaMetrics: settings: bucket: sts-settings-backup s3Prefix: "" - stackpacksS3Prefix: "stackpacks/" restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index 323993e..17ded94 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -71,6 +71,7 @@ stackpacks: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" localStackPacksUri: "/var/stackpacks_local" + backupDir: "stackpacks/" # Minio configuration for S3-compatible storage minio: diff --git a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml index 227d8a3..3affec5 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml @@ -66,6 +66,7 @@ stackpacks: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" localStackPacksUri: "/var/stackpacks_local" + backupDir: "stackpacks/" # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: diff --git a/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml b/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml new file mode 100644 index 0000000..0c058f0 --- /dev/null +++ b/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml @@ -0,0 +1,134 @@ +# Valid ConfigMap Configuration for SUSE Observability Backup CLI (Storage mode) +# This file contains a complete configuration WITHOUT the stackpacks section. +# Used to test that skipStackpacks is true when stackpacks config is missing. + +elasticsearch: + snapshotRepository: + name: sts-backup + bucket: sts-elasticsearch-backup + endpoint: suse-observability-storage:9000 + basepath: "" + accessKey: configmap-access-key + secretKey: configmap-secret-key + + slm: + name: auto-sts-backup + schedule: "0 0 3 * * ?" + snapshotTemplateName: "" + repository: sts-backup + indices: "sts*" + retentionExpireAfter: 30d + retentionMinCount: 5 + retentionMaxCount: 30 + + service: + name: suse-observability-elasticsearch-master-headless + port: 9200 + localPortForwardPort: 9200 + + restore: + repository: sts-backup + scaleDownLabelSelector: "observability.suse.com/scalable-during-es-restore=true" + indexPrefix: sts + datastreamIndexPrefix: .ds-sts_k8s_logs + datastreamName: sts_k8s_logs + indicesPattern: sts*,.ds-sts_k8s_logs* + +# No stackpacks section - stackpacks should be skipped + +storage: + globalBackupEnabled: true + service: + name: suse-observability-storage + port: 9000 + localPortForwardPort: 9000 + accessKey: storageadmin + secretKey: storageadmin + +stackgraph: + bucket: sts-stackgraph-backup + s3Prefix: "" + multipartArchive: true + restore: + scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" + loggingConfigConfigMap: suse-observability-logging + zookeeperQuorum: "suse-observability-zookeeper:2181" + job: + labels: + app: stackgraph-restore + image: quay.io/stackstate/stackstate-backup:latest + waitImage: quay.io/stackstate/wait:latest + resources: + limits: + cpu: "2" + memory: "4Gi" + requests: + cpu: "1" + memory: "2Gi" + pvc: + size: "10Gi" + accessModes: + - ReadWriteOnce + +victoriaMetrics: + S3Locations: + - bucket: sts-victoria-metrics-backup + prefix: victoria-metrics-0 + - bucket: sts-victoria-metrics-backup + prefix: victoria-metrics-1 + restore: + haMode: "mirror" + persistentVolumeClaimPrefix: "database-victoria-metrics-" + scaleDownLabelSelector: "observability.suse.com/scalable-during-vm-restore=true" + job: + labels: + app: victoria-metrics-restore + image: quay.io/stackstate/victoria-metrics-backup:latest + waitImage: quay.io/stackstate/wait:latest + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + +# baseUrl, receiverBaseUrl, platformVersion are set here since there is no stackpacks section +settings: + bucket: sts-settings-backup + s3Prefix: "" + localBucket: sts-settings-local-backup + restore: + scaleDownLabelSelector: "observability.suse.com/scalable-during-settings-restore=true" + loggingConfigConfigMap: suse-observability-logging + baseUrl: "http://suse-observability-server:7070" + receiverBaseUrl: "http://suse-observability-receiver:7077" + platformVersion: "5.2.0" + zookeeperQuorum: "suse-observability-zookeeper:2181" + job: + labels: + app: settings-restore + image: quay.io/stackstate/settings-backup:latest + waitImage: quay.io/stackstate/wait:latest + resources: + limits: + cpu: "1" + memory: "2Gi" + requests: + cpu: "500m" + memory: "1Gi" + +clickhouse: + service: + name: "suse-observability-clickhouse-shard0-0" + port: 9000 + localPortForwardPort: 9000 + backupService: + name: "suse-observability-clickhouse-shard0-0" + port: 7171 + localPortForwardPort: 7171 + database: "default" + username: "default" + password: "password" + restore: + scaleDownLabelSelector: "observability.suse.com/scalable-during-clickhouse-restore=true" diff --git a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml index d08832d..036f06f 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml @@ -73,6 +73,7 @@ stackpacks: receiverBaseUrl: "http://suse-observability-receiver:7077" platformVersion: "5.2.0" localStackPacksUri: "/var/stackpacks_local" + backupDir: "stackpacks/" # Storage configuration for S3-compatible storage (new mode, replaces Minio) storage: diff --git a/internal/scripts/scripts/restore-settings-backup.sh b/internal/scripts/scripts/restore-settings-backup.sh index 0b42ffa..2dde3db 100644 --- a/internal/scripts/scripts/restore-settings-backup.sh +++ b/internal/scripts/scripts/restore-settings-backup.sh @@ -70,11 +70,11 @@ else echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_CONFIGURATION_LOCAL_BUCKET}\"..." setup_aws_credentials - if download_from_s3 "${BACKUP_CONFIGURATION_LOCAL_BUCKET}" "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then + if download_from_s3 "${BACKUP_CONFIGURATION_LOCAL_BUCKET}" "${BACKUP_CONFIGURATION_S3_PREFIX}${BACKUP_CONFIGURATION_STACKPACKS_DIR}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" elif [ "${BACKUP_CONFIGURATION_UPLOAD_REMOTE}" == "true" ]; then echo "=== StackPacks backup not found in kubernetes settings storage, trying main backups storage..." - if download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_STACKPACKS_S3_PREFIX}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then + if download_from_s3 "${BACKUP_CONFIGURATION_BUCKET_NAME}" "${BACKUP_CONFIGURATION_S3_PREFIX}${BACKUP_CONFIGURATION_STACKPACKS_DIR}" "${TMP_DIR}" "${STACKPACKS_FILE}"; then STACKPACKS_RESTORE_FILE="${TMP_DIR}/${STACKPACKS_FILE}" fi fi diff --git a/internal/scripts/scripts/restore-stackgraph-backup.sh b/internal/scripts/scripts/restore-stackgraph-backup.sh index 0f3a9c4..db11cef 100644 --- a/internal/scripts/scripts/restore-stackgraph-backup.sh +++ b/internal/scripts/scripts/restore-stackgraph-backup.sh @@ -42,9 +42,9 @@ else echo "=== Checking for StackPacks backup \"${STACKPACKS_FILE}\" in bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." # Check if stackpacks backup exists in S3 - if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then + if sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" 2>/dev/null | grep -q "${STACKPACKS_FILE}"; then echo "=== Downloading StackPacks backup..." - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_STACKPACKS_S3_PREFIX}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" + sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_STACKGRAPH_STACKPACKS_DIR}${STACKPACKS_FILE}" "${TMP_DIR}/${STACKPACKS_FILE}" echo "=== Restoring StackPacks from \"${STACKPACKS_FILE}\"..." /opt/docker/bin/stack-packs-backup -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -restore "${TMP_DIR}/${STACKPACKS_FILE}"