docker health checks

.github/workflows/docker.yaml

@@ -52,6 +52,7 @@ jobs:
               - src/backend/InvenTree/InvenTree/settings.py
               - src/backend/requirements.txt
               - tasks.py
+              - test_tasks.py
 
   # Build the docker image
   build:
@@ -94,6 +95,14 @@ jobs:
           docker run --rm inventree-test grep -q '"src/locales/fr/messages.ts"' /home/inventree/src/backend/InvenTree/web/static/web/.vite/manifest.json
           docker run --rm inventree-test grep -q '"src/locales/ru/messages.ts"' /home/inventree/src/backend/InvenTree/web/static/web/.vite/manifest.json
           docker run --rm inventree-test grep -q '"src/locales/zh_Hans/messages.ts"' /home/inventree/src/backend/InvenTree/web/static/web/.vite/manifest.json
+      - name: Test 'server-health' / 'worker-health' Invoke Tasks
+        # Run the unit tests for the invoke tasks used by the docker-compose healthchecks,
+        # against the actual built image (real 'invoke' install, real tasks.py).
+        # The test file is not baked into the image, so it is mounted in at runtime.
+        run: |
+          docker run --rm \
+            -v "${{ github.workspace }}/test_tasks.py:/home/inventree/test_tasks.py:ro" \
+            inventree-test python -m unittest test_tasks -v
       - name: Build Docker Image
         # Build the development docker image (using docker-compose.yml)
         run: docker compose --project-directory . -f contrib/container/dev-docker-compose.yml build --no-cache
@@ -124,6 +133,44 @@ jobs:
         run: |
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> contrib/container/docker.dev.env
           docker compose --project-directory . -f contrib/container/dev-docker-compose.yml run --rm inventree-dev-server invoke dev.test --disable-pty --translations
+      - name: Start Production Stack (docker-compose.yml)
+        # Bring up the production docker-compose.yml stack, using the image built above.
+        # This exercises the full healthcheck chain (db, cache, server, worker, proxy),
+        # including the 'invoke server-health' / 'invoke worker-health' checks.
+        working-directory: contrib/container
+        env:
+          INVENTREE_TAG: test
+        run: |
+          docker tag inventree-test inventree/inventree:test
+          docker compose -f docker-compose.yml --env-file .env up -d
+      - name: Wait for Production Stack to become healthy
+        working-directory: contrib/container
+        run: |
+          services="inventree-db inventree-cache inventree-server inventree-worker inventree-proxy"
+          deadline=$((SECONDS + 360))
+          for service in $services; do
+            while true; do
+              status="$(docker inspect -f '{{.State.Health.Status}}' "$service" 2>/dev/null || echo missing)"
+              if [ "$status" = "healthy" ]; then
+                echo "Service '$service' is healthy"
+                break
+              fi
+              if [ "$status" = "unhealthy" ] || [ $SECONDS -ge $deadline ]; then
+                echo "::error::Service '$service' did not become healthy (status: $status)"
+                docker compose -f docker-compose.yml ps
+                docker compose -f docker-compose.yml logs "$service"
+                exit 1
+              fi
+              sleep 5
+            done
+          done
+      - name: Verify End-to-End Request Through Proxy
+        working-directory: contrib/container
+        run: curl --silent --show-error --fail http://localhost/api/system/health/
+      - name: Stop Production Stack
+        if: always()
+        working-directory: contrib/container
+        run: docker compose -f docker-compose.yml down -v
 
   # Run migration test
   migration_test:

contrib/container/docker-compose.yml

@@ -54,6 +54,12 @@ services:
         volumes:
             # Map 'data' volume such that postgres database is stored externally
             - ${INVENTREE_EXT_VOLUME:?You must specify the 'INVENTREE_EXT_VOLUME' variable in the .env file!}:/var/lib/postgresql/data/:z
+        healthcheck:
+            test: ['CMD-SHELL', 'pg_isready -U "$${POSTGRES_USER}" -d "$${POSTGRES_DB}"']
+            interval: 15s
+            timeout: 5s
+            retries: 10
+            start_period: 20s
         restart: unless-stopped
 
     # redis acts as database cache manager
@@ -69,6 +75,12 @@ services:
             - ${INVENTREE_CACHE_PORT:-6379}
         volumes:
             - ${INVENTREE_EXT_VOLUME}/redis:/data
+        healthcheck:
+            test: ['CMD', 'redis-cli', 'ping']
+            interval: 15s
+            timeout: 5s
+            retries: 10
+            start_period: 10s
         restart: always
 
     # InvenTree web server service
@@ -81,15 +93,32 @@ services:
         expose:
             - ${INVENTREE_WEB_PORT:-8000}
         depends_on:
-            - inventree-db
-            - inventree-cache
+            inventree-db:
+                condition: service_healthy
+            inventree-cache:
+                condition: service_healthy
         env_file:
             - .env
         environment:
           INVENTREE_SERVER: http://inventree-server:${INVENTREE_WEB_PORT}
         volumes:
             # Data volume must map to /home/inventree/data
             - ${INVENTREE_EXT_VOLUME}:/home/inventree/data:z
+        # Delegate to the 'invoke server-health' task (contrib/container/../tasks.py)
+        # This hits the /api/system/health/ endpoint without needing curl/wget in the image
+        healthcheck:
+            test:
+                [
+                    'CMD',
+                    'invoke',
+                    'server-health',
+                    '--address',
+                    'http://localhost:${INVENTREE_WEB_PORT:-8000}',
+                ]
+            interval: 20s
+            timeout: 10s
+            retries: 10
+            start_period: 60s
         restart: unless-stopped
 
     # Background worker process handles long-running or periodic tasks
@@ -99,12 +128,21 @@ services:
         container_name: inventree-worker
         command: invoke worker
         depends_on:
-            - inventree-server
+            inventree-server:
+                condition: service_healthy
         env_file:
             - .env
         volumes:
             # Data volume must map to /home/inventree/data
             - ${INVENTREE_EXT_VOLUME}:/home/inventree/data:z
+        # Delegate to the 'invoke worker-health' task, which checks the heartbeat
+        # file written by the background worker (no Django startup required)
+        healthcheck:
+            test: ['CMD', 'invoke', 'worker-health']
+            interval: 30s
+            timeout: 10s
+            retries: 5
+            start_period: 60s
         restart: unless-stopped
 
     # caddy acts as reverse proxy and static file server
@@ -115,12 +153,28 @@ services:
         image: caddy:alpine
         restart: always
         depends_on:
-            - inventree-server
+            inventree-server:
+                condition: service_healthy
+            inventree-worker:
+                condition: service_healthy
         ports:
             - ${INVENTREE_HTTP_PORT:-80}:80
             - ${INVENTREE_HTTPS_PORT:-443}:443
         env_file:
             - .env
+        # Caddy has no 'invoke' available, so hit the health endpoint directly.
+        # Caddy routes on the Host header (see INVENTREE_SITE_URL in the Caddyfile),
+        # so the request must present a matching Host to reach the InvenTree backend.
+        healthcheck:
+            test:
+                [
+                    'CMD-SHELL',
+                    'host=$$(printf "%s" "$${INVENTREE_SITE_URL:-}" | cut -d"," -f1 | sed -E "s#https?://##; s#/.*##"); if [ -n "$${host}" ]; then wget -qO- --header="Host: $${host}" http://127.0.0.1/api/system/health/; else wget -qO- http://127.0.0.1/api/system/health/; fi | grep -Eq "\"status\"[[:space:]]*:[[:space:]]*\"ok\""',
+                ]
+            interval: 20s
+            timeout: 5s
+            retries: 10
+            start_period: 20s
         volumes:
             - ./Caddyfile:/etc/caddy/Caddyfile:ro,z
             - ${INVENTREE_EXT_VOLUME}/static:/var/www/static:z