feat: Add upgrade_yml automation for per-component upgrade testing

automation_library/upgrade_and_rollback/__init__.py

@@ -17,6 +17,7 @@
 
 Combined module for Omnia upgrade and rollback workflows.
 """
+# pylint: disable=duplicate-code
 
 from .functions import (
     # Common
@@ -43,6 +44,14 @@
     run_omnia_rollback,
     verify_rollback_container,
     verify_rollback_backup_md5sum,
+)
+from .functions import (
+    # upgrade_yml
+    check_upgrade_yml_exists,
+    run_upgrade_yml,
+    verify_upgrade_manifest,
+    verify_manifest_component_status,
+    check_software_component_enabled,
     # K8s & Telemetry pre-check collectors
     collect_k8s_node_versions,
     collect_node_readiness,
@@ -104,7 +113,6 @@
     verify_idrac_telemetry_running,
     verify_ldms_collecting,
     verify_new_telemetry_components,
-    verify_upgrade_manifest,
     verify_cps_at_target,
     verify_workers_at_target,
     verify_etcd_backup_exists,
@@ -162,6 +170,7 @@
     PREPARE_UPGRADE_VARS,
     BACKUP_VERIFY_VARS,
     ROLLBACK_VARS,
+    UPGRADE_YML_VARS,
     SLURM_UPGRADE_VARS,
     ROLLBACK_YML_VARS,
     # K8s & Telemetry upgrade
@@ -188,6 +197,10 @@
     ROLLBACK_LOG_MSGS,
     ROLLBACK_ASSERT_MSGS,
     ROLLBACK_SKIP_MSGS,
+    UPGRADE_YML_TEST_NAMES,
+    UPGRADE_YML_LOG_MSGS,
+    UPGRADE_YML_ASSERT_MSGS,
+    UPGRADE_YML_SKIP_MSGS,
     # K8s & Telemetry upgrade messages
     PRECHECK_TEST_NAMES,
     PRECHECK_LOG_MSGS,
@@ -233,6 +246,12 @@
     "run_omnia_rollback",
     "verify_rollback_container",
     "verify_rollback_backup_md5sum",
+    # upgrade_yml functions
+    "check_upgrade_yml_exists",
+    "run_upgrade_yml",
+    "verify_upgrade_manifest",
+    "verify_manifest_component_status",
+    "check_software_component_enabled",
     # Vars
     "UPGRADE_VARS",
     "SUPPORTED_VERSIONS",
@@ -241,6 +260,7 @@
     "PREPARE_UPGRADE_VARS",
     "BACKUP_VERIFY_VARS",
     "ROLLBACK_VARS",
+    "UPGRADE_YML_VARS",
     # Messages
     "TEST_NAMES",
     "TEST_LOG_MSGS",
@@ -258,6 +278,10 @@
     "ROLLBACK_LOG_MSGS",
     "ROLLBACK_ASSERT_MSGS",
     "ROLLBACK_SKIP_MSGS",
+    "UPGRADE_YML_TEST_NAMES",
+    "UPGRADE_YML_LOG_MSGS",
+    "UPGRADE_YML_ASSERT_MSGS",
+    "UPGRADE_YML_SKIP_MSGS",
     # K8s & Telemetry pre-check collectors
     "collect_k8s_node_versions",
     "collect_node_readiness",
@@ -319,7 +343,6 @@
     "verify_idrac_telemetry_running",
     "verify_ldms_collecting",
     "verify_new_telemetry_components",
-    "verify_upgrade_manifest",
     "verify_cps_at_target",
     "verify_workers_at_target",
     "verify_etcd_backup_exists",

automation_library/upgrade_and_rollback/functions/__init__.py

@@ -44,8 +44,13 @@
     verify_rollback_container,
     verify_rollback_backup_md5sum,
 )
-
-# K8s & Telemetry upgrade pre/post check
+from .upgrade_yml_func import (
+    check_upgrade_yml_exists,
+    run_upgrade_yml,
+    verify_upgrade_manifest,
+    verify_manifest_component_status,
+    check_software_component_enabled,
+)
 from .precheck_func import (
     collect_k8s_node_versions,
     collect_node_readiness,
@@ -107,14 +112,7 @@
     verify_nfs_provisioner_running,
     verify_idrac_telemetry_running,
     verify_ldms_collecting,
-    verify_dcgm_running,
-    verify_powerscale_telemetry_running,
-    verify_vast_telemetry_running,
-    verify_ufm_telemetry_running,
-    verify_vector_running,
-    verify_victorialogs_running,
     verify_new_telemetry_components,
-    verify_upgrade_manifest,
     verify_cps_at_target,
     verify_workers_at_target,
     verify_etcd_backup_exists,
@@ -167,9 +165,11 @@
 )
 
 __all__ = [
+    # Common
     "compare_versions",
     "get_oim_metadata",
     "check_container_service_status",
+    # Upgrade core
     "validate_upgrade_versions",
     "validate_versions",
     "validate_config",
@@ -185,12 +185,19 @@
     "verify_post_upgrade_state",
     "run_prepare_upgrade",
     "verify_backup_md5sum",
+    # Rollback core
     "verify_rollback_precondition",
     "check_rollback_image",
     "download_omnia_sh_for_rollback",
     "run_omnia_rollback",
     "verify_rollback_container",
     "verify_rollback_backup_md5sum",
+    # upgrade_yml
+    "check_upgrade_yml_exists",
+    "run_upgrade_yml",
+    "verify_upgrade_manifest",
+    "verify_manifest_component_status",
+    "check_software_component_enabled",
     # K8s & Telemetry pre-check collectors
     "collect_k8s_node_versions",
     "collect_node_readiness",
@@ -251,14 +258,7 @@
     "verify_nfs_provisioner_running",
     "verify_idrac_telemetry_running",
     "verify_ldms_collecting",
-    "verify_dcgm_running",
-    "verify_powerscale_telemetry_running",
-    "verify_vast_telemetry_running",
-    "verify_ufm_telemetry_running",
-    "verify_vector_running",
-    "verify_victorialogs_running",
     "verify_new_telemetry_components",
-    "verify_upgrade_manifest",
     "verify_cps_at_target",
     "verify_workers_at_target",
     "verify_etcd_backup_exists",
@@ -277,6 +277,23 @@
     "verify_rollback_telemetry_healthy",
     "verify_rollback_metallb_cleaned",
     "verify_rollback_csi_cleaned",
+    # Slurm upgrade
+    "check_slurm_upgrade_state",
+    "run_slurm_upgrade",
+    "verify_slurm_pre_upgrade",
+    "capture_slurm_pre_upgrade_state",
+    "save_slurm_pre_upgrade_state",
+    "verify_slurm_nfs_mount",
+    "verify_slurm_conf_backup",
+    "verify_mysql_datadir_backup",
+    "verify_hpc_tracking_cleanup",
+    "verify_no_running_jobs",
+    "verify_all_nodes_idle",
+    "verify_slurmctld_post_upgrade",
+    "verify_slurmd_post_upgrade",
+    "verify_munge_post_upgrade",
+    "verify_sbatch_post_upgrade",
+    "verify_srun_post_upgrade",
     # Snapshot persistence
     "save_precheck_snapshot",
     "load_precheck_snapshot",

automation_library/upgrade_and_rollback/functions/slurm_upgrade_func.py

@@ -350,7 +350,7 @@ def verify_slurm_nfs_mount(host) -> Dict[str, Any]:
     Returns:
         Dict with success, message, mount_point, source, error.
     """
-    result = {"success": False, 
+    result = {"success": False,
         "message": "",
         "mount_point": "",
         "source": "",
@@ -399,18 +399,19 @@ def verify_slurm_nfs_mount(host) -> Dict[str, Any]:
             )
         result["success"] = False
         return result
-
-    mount_check = host.run(f"findmnt -rn -o TARGET {slurm_mount["mount_point"]}")
-    if (mount_check.rc != 0):
+
+    mount_point = slurm_mount["mount_point"]
+    mount_check = host.run(f"findmnt -rn -o TARGET {mount_point}")
+    if mount_check.rc != 0:
         result["message"] = LOG["nfs_config_error"].format(
-                error=f"'{slurm_mount_name}' with mount point {slurm_mount["mount_point"]} not mounted on OIM",
+                error=f"'{slurm_mount_name}' with mount point {mount_point} not mounted on OIM",
             )
         result["success"] = False
         return result
 
     result["success"] = True
     result["message"] = "Slurm mount is successfully mounted on OIM"
-    result["mount_point"] = slurm_mount["mount_point"]
+    result["mount_point"] = mount_point
     return result