11#! /usr/bin/env bash
2- # Run checks for the provided service(s).
2+ # Run health checks for the provided service(s).
33# To specify multiple services, separate them with spaces or plus signs (+).
4- # To specify all services, just pass in "all".
4+ # To specify all running services, just pass in "all".
55#
66# Examples:
77# ./check.sh lms
88# ./check.sh lms+forum
99# ./check.sh lms+forum discovery
1010# ./check.sh all
1111#
12- # Exists 0 if successful; non-zero otherwise.
12+ # Exits 0 if successful; non-zero otherwise.
1313#
1414# Fails if no services specified.
1515#
1818
1919set -eu -o pipefail
2020
21- # Grab all arguments into one string, replacing plus signs with spaces.
22- # Pad on either side with spaces so that the regex in `should_check` works correctly.
23- services=" ${*// +/ } "
24-
2521# Which checks succeeded and failed.
2622succeeded=" "
2723failed=" "
2824
29- # Returns whether service in first arg should be checked.
30- should_check () {
31- local service=" $1 "
32- if [[ " $services " == * " all " * ]] || [[ " $services " == * " $service " * ]]; then
33- return 0 # Note that '0' means 'success' (i.e., true) in bash.
34- else
35- return 1
36- fi
37- }
38-
39- # Runs a check named $1 on service $2 using the command $3.
25+ # Runs a check named $1 on service $2 using the host-side command $3.
4026run_check () {
4127 local check_name=" $1 "
4228 local service=" $2 "
@@ -53,133 +39,75 @@ run_check() {
5339 echo # Newline
5440}
5541
56- mysql_run_check () {
57- container_name=" $1 "
58- mysql_probe=" SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')"
59- # The use of `--protocol tcp` forces MySQL to connect over TCP rather than
60- # via a UNIX socket. This is needed because when MySQL starts for the first
61- # time in a new container, it starts a "temporary server" that runs for a
62- # few seconds and then shuts down before the "real" server starts up. The
63- # temporary server does not listen on the TCP port, but if the mysql
64- # command is not told which server to use, it will first try the UNIX
65- # socket and only after that will it try the default TCP port.
66- #
67- # By specifying that mysql should use TCP, we won't get an early false
68- # positive "ready" response while the temporary server is running.
69- run_check " ${container_name} _query" " $container_name "
70- " docker compose exec -T $( printf %q " $container_name " ) $( printf %q " $mysql_probe " ) "
42+ # Print a service container's Docker healthcheck status, one of:
43+ # healthy | unhealthy | starting | none | missing
44+ # "none" means the container exists but declares no healthcheck; "missing"
45+ # means no container is running for the service.
46+ container_health () {
47+ local service=" $1 "
48+ cid=" $( docker compose ps -q " $service " 2> /dev/null || true) "
49+ if [[ -z " $cid " ; then
50+ echo " missing"
51+ return
52+ fi
53+ docker inspect \
54+ --format ' {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'
55+ " $cid " 2> /dev/null || echo " missing"
7156}
7257
73- if should_check mysql80; then
74- echo " Checking MySQL 8.0 query endpoint:"
75- mysql_run_check mysql80
76- fi
77-
78- if should_check mongo; then
79- echo " Checking MongoDB status:"
80- run_check mongo_status mongo \
81- " docker compose exec -T mongo mongo --eval \" db.serverStatus()\" "
82- fi
83-
84- if should_check registrar; then
85- echo " Checking Registrar heartbeat:"
86- run_check registrar_heartbeat registrar \
87- " curl --fail -L http://localhost:18734/health"
88- fi
89-
90- if should_check lms; then
91- echo " Checking LMS heartbeat:"
92- run_check lms_heartbeat lms \
93- " curl --fail -L http://localhost:18000/heartbeat"
94-
95- echo " Validating LMS volume:"
96- run_check lms_volume lms \
97- " make validate-lms-volume"
98- fi
99-
100- if should_check cms; then
101- echo " Checking CMS heartbeat:"
102- run_check cms_heartbeat cms \
103- " curl --fail -L http://localhost:18010/heartbeat"
104- fi
105-
106- if should_check ecommerce; then
107- echo " Checking ecommerce health:"
108- run_check ecommerce_heartbeat ecommerce \
109- " curl --fail -L http://localhost:18130/health/"
110- fi
111-
112- if should_check enterprise_access; then
113- echo " Checking enterprise-access health:"
114- run_check enterprise_access_heartbeat enterprise-access \
115- " curl --fail -L http://localhost:18130/health/"
116- fi
117-
118- if should_check enterprise-subsidy; then
119- echo " Checking enterprise_subsidy health:"
120- run_check enterprise-subsidy_heartbeat enterprise-subsidy \
121- " curl --fail -L http://localhost:18280/health/"
122- fi
123-
124- if should_check discovery; then
125- echo " Checking discovery health:"
126- run_check discovery_heartbeat discovery \
127- " curl --fail -L http://localhost:18381/health/"
128- fi
129-
130- if should_check forum; then
131- echo " Checking forum heartbeat:"
132- run_check forum_heartbeat forum \
133- " curl --fail -L http://localhost:44567/heartbeat"
134- fi
135-
136- if should_check edx_notes_api; then
137- echo " Checking edx_notes_api heartbeat:"
138- run_check edx_notes_api_heartbeat edx_notes_api \
139- " curl --fail -L http://localhost:18120/heartbeat"
140- fi
141-
142- if should_check designer; then
143- echo " Checking designer health:"
144- run_check designer_heartbeat designer \
145- " curl --fail -L http://localhost:18808/health/"
146- fi
147-
148- if should_check credentials; then
149- echo " Checking credentials heartbeat:"
150- run_check credentials_heartbeat credentials \
151- " curl --fail -L http://localhost:18150/health"
152- fi
153-
154- if should_check xqueue; then
155- echo " Checking xqueue status:"
156- run_check xqueue_heartbeat xqueue \
157- " curl --fail -L http://localhost:18040/xqueue/status"
158- fi
159-
160- if should_check insights; then
161- echo " Running Analytics Dashboard Devstack tests: "
162- run_check insights_heartbeat insights \
163- " curl --fail -L http://localhost:18110/health/"
164- fi
58+ # Default check: pass iff the service's container reports itself healthy.
59+ # Services with no healthcheck (and no extra check) simply don't contribute a
60+ # check, matching the old behavior for unrecognized services.
61+ run_health_check () {
62+ local service=" $1 "
63+ local status
64+ status=" $( container_health " $service " ) "
65+ case " $status " in
66+ healthy)
67+ echo " Checking $service : healthy"
68+ succeeded=" $succeeded ${service} _health"
69+ echo
70+ ;;
71+ starting|unhealthy)
72+ echo " Checking $service : $status "
73+ docker compose logs --tail 500 " $service "
74+ failed=" $failed ${service} _health"
75+ echo
76+ ;;
77+ none|missing)
78+ # No container healthcheck to consult; rely on any extra checks.
79+ :
80+ ;;
81+ esac
82+ }
16583
166- if should_check analyticsapi; then
167- echo " Running Analytics Data API Devstack tests: "
168- run_check analyticsapi_heartbeat analyticsapi \
169- " curl --fail -L http://localhost:19001/health/"
170- fi
84+ # Extra/override checks for services that need more than their container
85+ # healthcheck can express. Keep this SMALL -- it is the only place service
86+ # names should be enumerated.
87+ run_extra_checks () {
88+ local service=" $1 "
89+ case " $service " in
90+ lms)
91+ echo " Validating LMS volume:"
92+ run_check lms_volume lms " make validate-lms-volume"
93+ ;;
94+ esac
95+ }
17196
172- if should_check license-manager; then
173- echo " Running License Manager Devstack tests: "
174- run_check license_manager_heartbeat license-manager \
175- " curl --fail -L http://localhost:18170/health/"
97+ # Expand the requested services into a plain, space-separated list. "all"
98+ # means every service with a running container (word-splitting is safe here
99+ # because compose service names contain no whitespace).
100+ requested=" ${*// +/ } "
101+ if [[ " $requested " == * " all " * ]]; then
102+ service_list=" $( docker compose ps --services) "
103+ else
104+ service_list=" ${*// +/ } "
176105fi
177106
178- if should_check edx-exams; then
179- echo " Running edX Exam Devstack tests: "
180- run_check edx-exams_heartbeat edx-exams \
181- " curl --fail -L http://localhost:18740/health/"
182- fi
107+ for service in $service_list ; do
108+ run_health_check " $service "
109+ run_extra_checks " $service "
110+ done
183111
184112echo " Successful checks:${succeeded:- NONE} "
185113echo " Failed checks:${failed:- NONE} "
0 commit comments