diff --git a/manifests/core_host_inventory.pp b/manifests/core_host_inventory.pp index 67faaf3..a71033d 100644 --- a/manifests/core_host_inventory.pp +++ b/manifests/core_host_inventory.pp @@ -209,6 +209,9 @@ 'Image' => $image, 'ContainerName' => 'iop-core-host-inventory', 'Network' => 'iop-core-network', + # Allow the worker to finish in-flight message processing and close its Kafka + # client before Podman escalates to SIGKILL (default container stop timeout is 10s). + 'StopTimeout' => '45', 'Exec' => './inv_mq_service.py', 'Environment' => [ 'KAFKA_BOOTSTRAP_SERVERS=PLAINTEXT://iop-core-kafka:9092', @@ -225,8 +228,9 @@ ], }, 'Service' => { - 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', - 'Restart' => 'on-failure', + 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', + 'Restart' => 'on-failure', + 'SuccessExitStatus' => '143', }, 'Install' => { 'WantedBy' => ['multi-user.target', 'default.target'], @@ -257,6 +261,9 @@ 'Image' => $image, 'ContainerName' => 'iop-core-host-inventory-api', 'Network' => 'iop-core-network', + # Must exceed gunicorn's graceful_timeout (30s) so workers drain before Podman + # escalates to SIGKILL (default container stop timeout is 10s). + 'StopTimeout' => '45', 'Exec' => 'python run_gunicorn.py', 'Environment' => [ 'KAFKA_BOOTSTRAP_SERVERS=iop-core-kafka:9092', @@ -275,8 +282,9 @@ ], }, 'Service' => { - 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', - 'Restart' => 'on-failure', + 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', + 'Restart' => 'on-failure', + 'SuccessExitStatus' => '143', }, 'Install' => { 'WantedBy' => ['multi-user.target', 'default.target'], diff --git a/manifests/core_ingress.pp b/manifests/core_ingress.pp index 264d6c8..58d3c4b 100644 --- a/manifests/core_ingress.pp +++ b/manifests/core_ingress.pp @@ -45,6 +45,9 @@ 'Image' => $image, 'ContainerName' => 'iop-core-ingress', 'Network' => 'iop-core-network', + # Give the bounded HTTP/telemetry shutdown (~10s in-app) headroom before Podman + # escalates to SIGKILL (default container stop timeout is 10s). + 'StopTimeout' => '20', 'Volume' => '/var/tmp/insights-archives:/var/tmp/insights-archives:rw,Z', 'Environment' => [ 'INGRESS_VALID_UPLOAD_TYPES=advisor,compliance,qpc,rhv,tower,leapp-reporting,xavier,playbook,playbook-sat,malware-detection,tasks', diff --git a/manifests/core_kafka.pp b/manifests/core_kafka.pp index 42aa70b..5614315 100644 --- a/manifests/core_kafka.pp +++ b/manifests/core_kafka.pp @@ -62,8 +62,9 @@ ], }, 'Service' => { - 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', - 'Restart' => 'on-failure', + 'Environment' => 'REGISTRY_AUTH_FILE=/etc/foreman/registry-auth.json', + 'Restart' => 'on-failure', + 'SuccessExitStatus' => '143', }, 'Install' => { 'WantedBy' => ['multi-user.target', 'default.target'], diff --git a/manifests/service_vulnerability.pp b/manifests/service_vulnerability.pp index bca4b21..233f8a8 100644 --- a/manifests/service_vulnerability.pp +++ b/manifests/service_vulnerability.pp @@ -223,6 +223,9 @@ 'Image' => $image, 'ContainerName' => 'iop-service-vuln-taskomatic', 'Network' => 'iop-core-network', + # The scheduler stops with wait=True, blocking until an in-progress job finishes. + # Give a running job room to complete before Podman escalates to SIGKILL (default 10s). + 'StopTimeout' => '30', 'Exec' => '/engine/entrypoint.sh taskomatic', 'Volume' => $socket_volume, 'Environment' => [ @@ -271,6 +274,9 @@ 'Image' => $image, 'ContainerName' => 'iop-service-vuln-grouper', 'Network' => 'iop-core-network', + # Give the service time to finish in-flight Kafka message handling and close its + # aiokafka clients before Podman escalates to SIGKILL (default stop timeout is 10s). + 'StopTimeout' => '30', 'Exec' => '/engine/entrypoint.sh grouper', 'Volume' => $socket_volume, 'Environment' => [ @@ -323,6 +329,9 @@ 'Image' => $image, 'ContainerName' => 'iop-service-vuln-listener', 'Network' => 'iop-core-network', + # Give the service time to finish in-flight Kafka message handling and close its + # aiokafka clients before Podman escalates to SIGKILL (default stop timeout is 10s). + 'StopTimeout' => '30', 'Exec' => '/engine/entrypoint.sh listener', 'Volume' => $socket_volume, 'Environment' => [ @@ -376,6 +385,9 @@ 'Image' => $image, 'ContainerName' => 'iop-service-vuln-evaluator-recalc', 'Network' => 'iop-core-network', + # Give the service time to finish in-flight Kafka message handling and close its + # aiokafka clients before Podman escalates to SIGKILL (default stop timeout is 10s). + 'StopTimeout' => '30', 'Exec' => '/engine/entrypoint.sh evaluator', 'Volume' => $socket_volume, 'Environment' => [ @@ -429,6 +441,9 @@ 'Image' => $image, 'ContainerName' => 'iop-service-vuln-evaluator-upload', 'Network' => 'iop-core-network', + # Give the service time to finish in-flight Kafka message handling and close its + # aiokafka clients before Podman escalates to SIGKILL (default stop timeout is 10s). + 'StopTimeout' => '30', 'Exec' => '/engine/entrypoint.sh evaluator', 'Volume' => $socket_volume, 'Environment' => [