Add health checks for Docker services, ACME environment readiness validation, and Certbot unit tests

- Introduced health checks to Docker Compose files for better service reliability.
- Added ACME environment readiness validation to ensure proper configuration for Certbot.
- Implemented unit tests for Certbot's `check_acme_environment_ready` method to cover edge cases.
- Improved E2E tests with startup wait adjustments and dynamic certificate issuance verification.
- Enhanced Kubernetes test cleanup with forced resource deletion for faster termination.

deploy/docker/docker-compose.yml

@@ -5,7 +5,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - certs_certbot:/etc/easyhaproxy/certs/certbot
       - certs_haproxy:/etc/easyhaproxy/certs/haproxy
-
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_LABEL_PREFIX: easyhaproxy

src/functions/__init__.py

@@ -458,6 +458,46 @@ class Certbot:
         else:
             return ""
 
+    @staticmethod
+    def check_acme_environment_ready(email, acme_server):
+        """
+        Check if ACME environment is ready for certificate operations.
+
+        Args:
+            email: EASYHAPROXY_CERTBOT_EMAIL value
+            acme_server: Processed ACME server string from set_acme_server()
+
+        Returns:
+            tuple: (is_ready: bool, error_message: str)
+        """
+        # Check 1: Email configured
+        if not email or email == "":
+            return False, "ACME email not configured (EASYHAPROXY_CERTBOT_EMAIL)"
+
+        # Check 2: ACME server configured
+        if not acme_server or acme_server == "":
+            return False, "ACME server not configured (EASYHAPROXY_CERTBOT_SERVER)"
+
+        # Check 3: ACME server reachability (if URL provided)
+        if "--server " in acme_server:
+            server_url = acme_server.replace("--server ", "")
+            try:
+                # Use 10s timeout, respect REQUESTS_CA_BUNDLE for Pebble CA
+                response = requests.get(server_url, timeout=10, verify=os.getenv("REQUESTS_CA_BUNDLE", True))
+                if response.status_code != 200:
+                    return False, f"ACME server {server_url} returned HTTP {response.status_code}"
+
+                # Validate ACME directory structure (RFC 8555)
+                data = response.json()
+                if "newAccount" not in data:
+                    return False, f"ACME server {server_url} returned invalid ACME directory"
+            except requests.exceptions.RequestException as e:
+                return False, f"ACME server {server_url} not reachable: {str(e)}"
+            except Exception as e:
+                return False, f"ACME server validation failed: {str(e)}"
+
+        return True, ""
+
     def check_certificates(self, hosts):
         if self.email == "" or len(hosts) == 0:
             return False

src/main.py

@@ -35,6 +35,15 @@ def start():
 
     certbot = Certbot(Consts.certs_certbot)
 
+    # Check ACME environment readiness if Certbot is configured
+    if certbot.email != "":
+        is_ready, error_msg = Certbot.check_acme_environment_ready(certbot.email, certbot.acme_server)
+        if not is_ready:
+            logger_easyhaproxy.warning(f"ACME environment not ready: {error_msg}")
+            logger_easyhaproxy.warning("Certificate auto-renewal may fail. Verify ACME server configuration.")
+        else:
+            logger_easyhaproxy.info("ACME environment validated and ready")
+
     while True:
         if old_haproxy is not None:
             old_haproxy.kill()

tests/test_certbot.py

@@ -72,6 +72,129 @@ class TestCertbotStaticMethods:
         hmac = "test-hmac-key-abcdef"
         assert Certbot.set_eab_hmac_key(hmac) == f'--eab-hmac-key "{hmac}"'
 
+    def test_check_acme_environment_ready_missing_email(self):
+        """Test ACME environment check with missing email"""
+        is_ready, error_msg = Certbot.check_acme_environment_ready("", "--staging")
+        assert is_ready is False
+        assert "ACME email not configured" in error_msg
+
+    def test_check_acme_environment_ready_missing_server(self):
+        """Test ACME environment check with missing server"""
+        is_ready, error_msg = Certbot.check_acme_environment_ready("test@example.com", "")
+        assert is_ready is False
+        assert "ACME server not configured" in error_msg
+
+    def test_check_acme_environment_ready_staging(self):
+        """Test ACME environment check with staging server (no URL to check)"""
+        is_ready, error_msg = Certbot.check_acme_environment_ready("test@example.com", "--staging")
+        assert is_ready is True
+        assert error_msg == ""
+
+    @patch('requests.get')
+    def test_check_acme_environment_ready_server_reachable(self, mock_get):
+        """Test ACME environment check with reachable server"""
+        # Mock successful response with valid ACME directory
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "newAccount": "https://acme.example.com/new-account",
+            "newNonce": "https://acme.example.com/new-nonce",
+            "newOrder": "https://acme.example.com/new-order"
+        }
+        mock_get.return_value = mock_response
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://acme.example.com/directory"
+        )
+
+        assert is_ready is True
+        assert error_msg == ""
+        mock_get.assert_called_once_with("https://acme.example.com/directory", timeout=10, verify=True)
+
+    @patch('requests.get')
+    def test_check_acme_environment_ready_server_unreachable(self, mock_get):
+        """Test ACME environment check with unreachable server"""
+        import requests
+        mock_get.side_effect = requests.exceptions.ConnectionError("Connection refused")
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://acme.example.com/directory"
+        )
+
+        assert is_ready is False
+        assert "not reachable" in error_msg
+        assert "Connection refused" in error_msg
+
+    @patch('requests.get')
+    def test_check_acme_environment_ready_server_timeout(self, mock_get):
+        """Test ACME environment check with timeout"""
+        import requests
+        mock_get.side_effect = requests.exceptions.Timeout("Request timed out")
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://acme.example.com/directory"
+        )
+
+        assert is_ready is False
+        assert "not reachable" in error_msg
+        assert "timed out" in error_msg
+
+    @patch('requests.get')
+    def test_check_acme_environment_ready_server_http_error(self, mock_get):
+        """Test ACME environment check with HTTP error status"""
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_get.return_value = mock_response
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://acme.example.com/directory"
+        )
+
+        assert is_ready is False
+        assert "returned HTTP 404" in error_msg
+
+    @patch('requests.get')
+    def test_check_acme_environment_ready_invalid_acme_directory(self, mock_get):
+        """Test ACME environment check with invalid ACME directory"""
+        # Mock response without required "newAccount" key
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "status": "ok",
+            "message": "Not an ACME directory"
+        }
+        mock_get.return_value = mock_response
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://acme.example.com/directory"
+        )
+
+        assert is_ready is False
+        assert "invalid ACME directory" in error_msg
+
+    @patch.dict(os.environ, {'REQUESTS_CA_BUNDLE': '/path/to/pebble-ca.pem'})
+    @patch('requests.get')
+    def test_check_acme_environment_ready_respects_ca_bundle(self, mock_get):
+        """Test that REQUESTS_CA_BUNDLE environment variable is respected"""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {"newAccount": "https://pebble:14000/new-account"}
+        mock_get.return_value = mock_response
+
+        is_ready, error_msg = Certbot.check_acme_environment_ready(
+            "test@example.com",
+            "--server https://pebble:14000/dir"
+        )
+
+        assert is_ready is True
+        # Verify verify parameter uses REQUESTS_CA_BUNDLE
+        mock_get.assert_called_once_with("https://pebble:14000/dir", timeout=10, verify='/path/to/pebble-ca.pem')
+
 
 class TestCertbotInitialization:
     """Test Certbot class initialization"""

tests_e2e/docker/docker-compose-acme-e2e.yml

@@ -64,6 +64,22 @@ services:
     networks:
       - acme-test
 
+  # Pebble health check sidecar
+  pebble_health:
+    image: curlimages/curl:8.6.0
+    depends_on:
+      - pebble
+    healthcheck:
+      test: ["CMD", "curl", "-skf", "https://pebble:14000/dir"]
+      interval: 5s
+      timeout: 3s
+      start_period: 5s
+      retries: 3
+    networks:
+      - acme-test
+    restart: "no"
+    command: ["tail", "-f", "/dev/null"]
+
   # Backend web server
   backend:
     image: byjg/static-httpserver
@@ -82,8 +98,16 @@ services:
       context: ../..
       dockerfile: build/Dockerfile
     depends_on:
-      - pebble
-      - backend
+      pebble_health:
+        condition: service_healthy
+      backend:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-acme.yml

@@ -65,6 +65,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Persist the CERTBOT to avoid re-challenge when the server restarts
       - ./certs:/etc/easyhaproxy/certs
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-changed-label.yml

@@ -39,6 +39,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_LABEL_PREFIX: haproxy

tests_e2e/docker/docker-compose-cloudflare.yml

@@ -54,6 +54,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Mount Cloudflare IP list
       - ./cloudflare_ips.lst:/etc/easyhaproxy/cloudflare_ips.lst:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-ip-whitelist.yml

@@ -48,6 +48,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-jwt-validator.yml

@@ -58,6 +58,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       # Mount the public key for JWT verification
       - ./jwt_pubkey.pem:/etc/easyhaproxy/jwt_keys/api_pubkey.pem:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-multi-containers.yml

@@ -52,6 +52,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-php-fpm.yml

@@ -50,6 +50,12 @@ services:
     image: byjg/easy-haproxy:local
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-plugins-combined.yml

@@ -70,6 +70,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./cloudflare_ips.lst:/etc/easyhaproxy/cloudflare_ips.lst:ro
       - ./jwt_pubkey.pem:/etc/easyhaproxy/jwt_keys/api_pubkey.pem:ro
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       HAPROXY_CUSTOMERRORS: "true"

tests_e2e/docker/docker-compose-portainer.yml

@@ -64,7 +64,12 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - certs_certbot:/etc/easyhaproxy/certs/certbot
 #      - certs_haproxy:/etc/easyhaproxy/certs/haproxy
-
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_LABEL_PREFIX: easyhaproxy

tests_e2e/docker/docker-compose.yml

@@ -55,6 +55,12 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./host2.local.pem:/etc/easyhaproxy/certs/haproxy/host2.local.pem
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:password", "http://localhost:1936"]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     environment:
       EASYHAPROXY_DISCOVER: docker
       EASYHAPROXY_SSL_MODE: "loose"

tests_e2e/test_docker_compose.py

@@ -858,7 +858,7 @@ def docker_compose_acme() -> Generator[None, None, None]:
         stderr=subprocess.DEVNULL  # Ignore error if volume doesn't exist
     )
 
-    fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=15)
+    fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=0)
     fixture.up()
     yield
     fixture.down()
@@ -930,7 +930,13 @@ class TestACME:
 
     def test_certificate_issuance(self, docker_compose_acme):
         """Test that Pebble successfully issues a certificate"""
-        # Check HAProxy logs for certificate issuance
+        # Wait for certificate issuance (Certbot runs in background loop)
+        # Typical time: 10-15 seconds from container start
+        max_wait = 30
+        check_interval = 2
+        has_success = False
+
+        for attempt in range(max_wait // check_interval):
             result = subprocess.run(
                 ["docker", "logs", "docker-haproxy-1"],
                 capture_output=True,
@@ -944,12 +950,18 @@ class TestACME:
                          "Certificate not yet due for renewal" in logs or \
                          "Cert not yet due for renewal" in logs
 
-        # If not successful, check for Pebble connection
+            if has_success:
+                break
+
+            # Wait before next check
+            time.sleep(check_interval)
+
+        # If not successful after waiting, check for Pebble connection
         if not has_success:
             # Check if we can at least connect to Pebble
             has_pebble_connection = "pebble:14000/dir" in logs or "pebble:14000" in logs
             assert has_pebble_connection, \
-                f"HAProxy cannot connect to Pebble ACME server. Check docker network.\nLogs:\n{logs[-2000:]}"
+                f"HAProxy cannot connect to Pebble ACME server after {max_wait}s. Check docker network.\nLogs:\n{logs[-2000:]}"
 
         # Verify merged certificate file exists
         # EasyHAProxy merges cert+key from /etc/easyhaproxy/certs/live/ to /etc/easyhaproxy/certs/certbot/{domain}.pem

tests_e2e/test_kubernetes.py

@@ -462,7 +462,7 @@ class KubernetesFixture:
         """Delete Kubernetes resources"""
         subprocess.run(
             [self.kubectl, "delete", "-f", self.manifest_file, "-n", self.namespace,
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )
@@ -471,7 +471,7 @@ class KubernetesFixture:
         if self.namespace != "default":
             subprocess.run(
                 [self.kubectl, "delete", "namespace", self.namespace,
-                 "--ignore-not-found=true"],
+                 "--ignore-not-found=true", "--force", "--grace-period=0"],
                 capture_output=True
             )
 
@@ -556,7 +556,7 @@ def k8s_service_tls(kind_cluster) -> Generator[str, None, None]:
         # Cleanup
         subprocess.run(
             [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default",
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )
@@ -597,7 +597,7 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     print("  → Creating JWT secret 'jwt-pubkey-secret'...")
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
@@ -611,7 +611,7 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     print("  → Creating JWT secret 'jwt-custom-secret'...")
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
@@ -655,12 +655,12 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]:
     # Delete the JWT secrets
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
     subprocess.run(
         [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default",
-         "--ignore-not-found=true"],
+         "--ignore-not-found=true", "--force", "--grace-period=0"],
         capture_output=True
     )
 
@@ -740,7 +740,7 @@ def k8s_cloudflare(kind_cluster, kind_cmd) -> Generator[str, None, None]:
         # Cleanup
         subprocess.run(
             [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default",
-             "--ignore-not-found=true"],
+             "--ignore-not-found=true", "--force", "--grace-period=0"],
             check=True,
             capture_output=True
         )

tests_e2e/utils.py

@@ -48,6 +48,9 @@ class DockerComposeFixture:
         if self.build:
             cmd.append("--build")
 
+        # Use native Docker healthcheck waiting
+        cmd.append("--wait")
+
         result = subprocess.run(
             cmd,
             capture_output=True,
@@ -96,7 +99,7 @@ class DockerComposeFixture:
         print(f"  → Stopping services from {compose_name}...")
 
         result = subprocess.run(
-            ["docker", "compose", "-f", self.compose_file, "down", "--remove-orphans"],
+            ["docker", "compose", "-f", self.compose_file, "down", "--remove-orphans", "-t", "0"],
             capture_output=True,
             text=True
         )