Add health checks for Docker services, ACME environment readiness validation, and Certbot unit tests

- Introduced health checks to Docker Compose files for better service reliability.
- Added ACME environment readiness validation to ensure proper configuration for Certbot.
- Implemented unit tests for Certbot's `check_acme_environment_ready` method to cover edge cases.
- Improved E2E tests with startup wait adjustments and dynamic certificate issuance verification.
- Enhanced Kubernetes test cleanup with forced resource deletion for faster termination.

tests_e2e/docker/docker-compose-acme-e2e.yml

@@ -64,6 +64,22 @@ services:
     networks:
       - acme-test
 
+  # Pebble health check sidecar
+  pebble_health:
+    image: curlimages/curl:8.6.0
+    depends_on:
+      - pebble
+    healthcheck:
+      test: ["CMD", "curl", "-skf", "https://pebble:14000/dir"]
+      interval: 5s
+      timeout: 3s
+      start_period: 5s
+      retries: 3
+    networks:
+      - acme-test
+    restart: "no"
+    command: ["tail", "-f", "/dev/null"]
+
   # Backend web server
   backend:
     image: byjg/static-httpserver
@@ -82,8 +98,16 @@ services:
       context: ../..
       dockerfile: build/Dockerfile
     depends_on:
-      - pebble
-      - backend
+      pebble_health:
+        condition: service_healthy
+      backend:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-acme.yml

@@ -65,6 +65,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Persist the CERTBOT to avoid re-challenge when the server restarts
       - ./certs:/etc/easyhaproxy/certs
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-changed-label.yml

@@ -39,6 +39,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_LABEL_PREFIX: haproxy

tests_e2e/docker/docker-compose-cloudflare.yml

@@ -54,6 +54,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Mount Cloudflare IP list
       - ./cloudflare_ips.lst:/etc/easyhaproxy/cloudflare_ips.lst:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-ip-whitelist.yml

@@ -48,6 +48,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-jwt-validator.yml

@@ -58,6 +58,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Mount the public key for JWT verification
       - ./jwt_pubkey.pem:/etc/easyhaproxy/jwt_keys/api_pubkey.pem:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-multi-containers.yml

@@ -52,6 +52,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-php-fpm.yml

@@ -50,6 +50,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-plugins-combined.yml

@@ -70,6 +70,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./cloudflare_ips.lst:/etc/easyhaproxy/cloudflare_ips.lst:ro
       - ./jwt_pubkey.pem:/etc/easyhaproxy/jwt_keys/api_pubkey.pem:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-portainer.yml

@@ -64,7 +64,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - certs_certbot:/etc/easyhaproxy/certs/certbot
 #      - certs_haproxy:/etc/easyhaproxy/certs/haproxy
-
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_LABEL_PREFIX: easyhaproxy

tests_e2e/docker/docker-compose.yml

@@ -55,6 +55,12 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./host2.local.pem:/etc/easyhaproxy/certs/haproxy/host2.local.pem
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_SSL_MODE: "loose"

tests_e2e/test_docker_compose.py

@@ -858,7 +858,7 @@ def docker_compose_acme() -> Generator[None, None, None]:
         stderr=subprocess.DEVNULL  # Ignore error if volume doesn't exist
     )
 
-    fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=15)
+    fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=0)
     fixture.up()
     yield
     fixture.down()
@@ -930,26 +930,38 @@ class TestACME:
 
     def test_certificate_issuance(self, docker_compose_acme):
         """Test that Pebble successfully issues a certificate"""
-        # Check HAProxy logs for certificate issuance
-        result = subprocess.run(
-            ["docker", "logs", "docker-haproxy-1"],
-            capture_output=True,
-            text=True
-        )
-        logs = result.stdout + result.stderr
+        # Wait for certificate issuance (Certbot runs in background loop)
+        # Typical time: 10-15 seconds from container start
+        max_wait = 30
+        check_interval = 2
+        has_success = False
 
-        # Look for certbot success messages
-        # Certbot outputs: "Successfully received certificate"
-        has_success = "Successfully received certificate" in logs or \
-                     "Certificate not yet due for renewal" in logs or \
-                     "Cert not yet due for renewal" in logs
+        for attempt in range(max_wait // check_interval):
+            result = subprocess.run(
+                ["docker", "logs", "docker-haproxy-1"],
+                capture_output=True,
+                text=True
+            )
+            logs = result.stdout + result.stderr
 
-        # If not successful, check for Pebble connection
+            # Look for certbot success messages
+            # Certbot outputs: "Successfully received certificate"
+            has_success = "Successfully received certificate" in logs or \
+                         "Certificate not yet due for renewal" in logs or \
+                         "Cert not yet due for renewal" in logs
+
+            if has_success:
+                break
+
+            # Wait before next check
+            time.sleep(check_interval)
+
+        # If not successful after waiting, check for Pebble connection
         if not has_success:
             # Check if we can at least connect to Pebble
             has_pebble_connection = "pebble:14000/dir" in logs or "pebble:14000" in logs
             assert has_pebble_connection, \
-                f"HAProxy cannot connect to Pebble ACME server. Check docker network.\nLogs:\n{logs[-2000:]}"
+                f"HAProxy cannot connect to Pebble ACME server after {max_wait}s. Check docker network.\nLogs:\n{logs[-2000:]}"
 
         # Verify merged certificate file exists
         # EasyHAProxy merges cert+key from /etc/easyhaproxy/certs/live/ to /etc/easyhaproxy/certs/certbot/{domain}.pem

tests_e2e/test_kubernetes.py

@@ -462,7 +462,7 @@ class KubernetesFixture:
         """Delete Kubernetes resources"""
         subprocess.run(
             [self.kubectl, "delete", "-f", self.manifest_file, "-n", self.namespace,
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )
@@ -471,7 +471,7 @@ class KubernetesFixture:
         if self.namespace != "default":
             subprocess.run(
                 [self.kubectl, "delete", "namespace", self.namespace,
-                 "--ignore-not-found=true"],
+                 "--ignore-not-found=true", "--force", "--grace-period=0"],
                 capture_output=True
             )
 
@@ -556,7 +556,7 @@ def k8s_service_tls(kind_cluster) -> Generator[str, None, None]:
         # Cleanup
         subprocess.run(
             [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default",
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )
@@ -597,7 +597,7 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     print("  → Creating JWT secret 'jwt-pubkey-secret'...")
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
@@ -611,7 +611,7 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     print("  → Creating JWT secret 'jwt-custom-secret'...")
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
@@ -655,12 +655,12 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     # Delete the JWT secrets
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
 
@@ -740,7 +740,7 @@ def k8s_cloudflare(kind_cluster, kind_cmd) -> Generator[str, None, None]:
         # Cleanup
         subprocess.run(
             [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default",
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )

tests_e2e/utils.py

@@ -48,6 +48,9 @@ class DockerComposeFixture:
         if self.build:
             cmd.append("--build")
 
+        # Use native Docker healthcheck waiting
+        cmd.append("--wait")
+
         result = subprocess.run(
             cmd,
             capture_output=True,
@@ -96,7 +99,7 @@ class DockerComposeFixture:
         print(f"  → Stopping services from {compose_name}...")
 
         result = subprocess.run(
-            ["docker", "compose", "-f", self.compose_file, "down", "--remove-orphans"],
+            ["docker", "compose", "-f", self.compose_file, "down", "--remove-orphans", "-t", "0"],
             capture_output=True,
             text=True
         )