Add ACME/Certbot E2E tests with Pebble integration and update dependencies in CI workflows

- Introduced `docker-compose-acme-e2e.yml` for end-to-end testing with Pebble test server.
- Added tests to validate ACME challenge routing, certificate issuance, HTTPS functionality, and HAProxy configuration.
- Implemented CA certificate download fixture (`create_pebble_ca_file`) for test session initialization.
- Updated `.gitignore` to exclude Pebble-related files.
- Modified CI workflows to include `needs: [Test]` dependencies for E2E jobs, ensuring proper sequencing.

.github/workflows/build.yml

@@ -43,6 +43,7 @@ jobs:
   Tests-E2E-Docker:
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    needs: [Test]
     permissions:
       contents: read
 
@@ -66,6 +67,7 @@ jobs:
   Tests-E2E-Kubernetes:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    needs: [Test]
     permissions:
       contents: read
 
@@ -88,6 +90,7 @@ jobs:
 
   Tests-E2E-Static:
     runs-on: ubuntu-latest
+    needs: [Test]
     timeout-minutes: 20
     permissions:
       contents: read

.gitignore

@@ -24,3 +24,4 @@ build/
 /tests_e2e/docker/jwt_private.pem
 /tests_e2e/docker/jwt_pubkey.pem
 /tests_e2e/docker/cloudflare_ips.lst
+/tests_e2e/docker/pebble-*

tests_e2e/docker/docker-compose-acme-e2e.yml

@@ -0,0 +1,124 @@
+# ==============================================================================
+# E2E Test: ACME/Certbot with Pebble Test Server
+# ==============================================================================
+#
+# WHAT THIS TESTS:
+# - HAProxy routing of /.well-known/acme-challenge/ to certbot backend
+# - Certbot HTTP-01 challenge completion with Pebble ACME server
+# - Certificate issuance and storage in /certs/certbot/live/{domain}/
+# - HTTPS serving with issued certificate
+# - Full end-to-end ACME protocol flow
+#
+# ABOUT PEBBLE:
+# Pebble is Let's Encrypt's official ACME test server (RFC 8555 compliant)
+# - Runs locally without internet access
+# - No rate limits or DNS requirements
+# - Issues test certificates (not trusted by browsers)
+# - Perfect for integration testing
+#
+# HOW TO RUN (via pytest):
+# ```bash
+# cd tests_e2e
+# pytest test_docker_compose.py::TestACME -v
+# ```
+#
+# MANUAL TESTING:
+# ```bash
+# cd tests_e2e/docker
+# docker compose -f docker-compose-acme-e2e.yml up --build
+#
+# # Wait 10-15 seconds for certificate issuance
+# # Check logs
+# docker compose -f docker-compose-acme-e2e.yml logs haproxy
+#
+# # Verify certificate was issued
+# ls -la ../../certs/certbot/live/test.local/
+#
+# # Test HTTPS (will show certificate warning - expected for test certs)
+# curl -k https://localhost/ -H "Host: test.local"
+#
+# # Cleanup
+# docker compose -f docker-compose-acme-e2e.yml down
+# ```
+#
+# ==============================================================================
+
+services:
+  # Pebble ACME Server - Let's Encrypt test environment
+  pebble:
+    image: ghcr.io/letsencrypt/pebble:latest
+    command: -config /test/my-pebble-config.json
+    environment:
+      # Speed up validation (no artificial delays)
+      PEBBLE_VA_NOSLEEP: 1
+      # Actually perform challenge validation (not always valid)
+      PEBBLE_VA_ALWAYS_VALID: 0
+    volumes:
+      # Custom config to use port 80 for validation
+      - ./pebble-config.json:/test/my-pebble-config.json:ro
+    ports:
+      # ACME API endpoint
+      - "14000:14000"
+      # Management API (optional)
+      - "15000:15000"
+    networks:
+      - acme-test
+
+  # Backend web server
+  backend:
+    image: byjg/static-httpserver
+    labels:
+      easyhaproxy.http.host: test.local
+      easyhaproxy.http.localport: 8080
+      easyhaproxy.http.certbot: "true"
+      easyhaproxy.http.clone_to_ssl: "true"
+      easyhaproxy.http.redirect_ssl: "true"
+    networks:
+      - acme-test
+
+  # EasyHAProxy with Certbot
+  haproxy:
+    build:
+      context: ../..
+      dockerfile: build/Dockerfile
+    depends_on:
+      - pebble
+      - backend
+    environment:
+      EASYHAPROXY_DISCOVER: docker
+      HAPROXY_CUSTOMERRORS: "true"
+
+      # Certbot configuration pointing to Pebble
+      EASYHAPROXY_CERTBOT_EMAIL: test@example.com
+      EASYHAPROXY_CERTBOT_SERVER: https://pebble:14000/dir
+
+      # Trust Pebble's CA certificate
+      REQUESTS_CA_BUNDLE: /etc/ssl/certs/pebble-ca.pem
+
+      # Reduce certbot timeout for faster tests
+      EASYHAPROXY_CERTBOT_TIMEOUT: 30
+
+      # Enable debug logging for troubleshooting
+      EASYHAPROXY_DEBUG: "false"
+
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Certificate storage (Docker volume for clean test isolation)
+      - certbot-certs:/certs/certbot
+      # Pebble CA certificate (downloaded during test session)
+      - ./pebble-ca.pem:/etc/ssl/certs/pebble-ca.pem:ro
+    ports:
+      - "80:80/tcp"
+      - "443:443/tcp"
+    networks:
+      acme-test:
+        aliases:
+          # Allow Pebble to reach HAProxy via test.local for challenge validation
+          - test.local
+
+networks:
+  acme-test:
+    driver: bridge
+
+volumes:
+  certbot-certs:

tests_e2e/test_docker_compose.py

@@ -41,6 +41,40 @@ DOCKER_DIR = BASE_DIR / "docker"
 
 # Track if cloudflare_ips.lst has been created in this test session
 _cloudflare_ips_created = False
+# Track if pebble CA cert has been downloaded in this test session
+_pebble_ca_downloaded = False
+
+
+def create_pebble_ca_file():
+    """
+    Download Pebble's test CA certificate.
+
+    This file is required for docker-compose-acme-e2e.yml to trust Pebble's HTTPS endpoint.
+    Downloads from Pebble's GitHub repository.
+
+    Strategy:
+    - First call: Always download (fresh certificate)
+    - Subsequent calls: Skip if file exists (reuse from first call)
+    """
+    global _pebble_ca_downloaded
+
+    pebble_ca_path = DOCKER_DIR / "pebble-ca.pem"
+
+    # On subsequent calls, skip if file exists
+    if _pebble_ca_downloaded and pebble_ca_path.exists() and pebble_ca_path.is_file():
+        return
+
+    # Download Pebble's test CA certificate
+    subprocess.run(
+        [
+            "curl", "-sL", "-o", str(pebble_ca_path),
+            "https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem"
+        ],
+        check=True
+    )
+
+    # Mark as downloaded for this test session
+    _pebble_ca_downloaded = True
 
 
 def create_cloudflare_ips_file():
@@ -805,6 +839,179 @@ class TestChangedLabel:
         verify_haproxy_stats()
 
 
+# =============================================================================
+# Test: docker-compose-acme-e2e.yml - ACME/Certbot with Pebble
+# =============================================================================
+
+@pytest.fixture
+def docker_compose_acme() -> Generator[None, None, None]:
+    """Fixture for docker-compose-acme-e2e.yml - ACME/Certbot E2E test"""
+    volume_name = "docker_certbot-certs"
+
+    # Download Pebble CA certificate (only once per test session)
+    create_pebble_ca_file()
+
+    # Clean up volume from previous test runs (ensures fresh start)
+    subprocess.run(
+        ["docker", "volume", "rm", volume_name],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL  # Ignore error if volume doesn't exist
+    )
+
+    fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=15)
+    fixture.up()
+    yield
+    fixture.down()
+
+    # Clean up volume after test
+    subprocess.run(
+        ["docker", "volume", "rm", volume_name],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL
+    )
+
+
+@pytest.mark.acme
+class TestACME:
+    """Tests for docker-compose-acme-e2e.yml - ACME/Certbot with Pebble test server"""
+
+    def test_haproxy_config(self, docker_compose_acme):
+        """Test HAProxy configuration has ACME challenge routing"""
+        result = subprocess.run(
+            ["docker", "exec", "docker-haproxy-1", "cat", "/etc/haproxy/haproxy.cfg"],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        config = result.stdout
+
+        # Verify ACME challenge ACL exists
+        assert 'acl is_certbot_test_local_80 path_beg /.well-known/acme-challenge/' in config, \
+            "ACME challenge ACL not found in HAProxy config"
+
+        # Verify routing to certbot_backend
+        assert 'use_backend certbot_backend if is_certbot_test_local_80' in config, \
+            "ACME challenge routing rule not found"
+
+        # Verify certbot_backend definition
+        assert 'backend certbot_backend' in config, \
+            "certbot_backend not defined"
+        assert 'server certbot 127.0.0.1:2080' in config, \
+            "certbot backend server not configured correctly"
+
+        # Verify SSL redirect bypasses ACME challenges
+        # Find redirect rule and verify it excludes certbot ACL
+        lines = config.split('\n')
+        for line in lines:
+            if 'http-request redirect scheme https' in line and 'test_local' in line:
+                assert '!is_certbot_test_local_80' in line, \
+                    "SSL redirect should bypass ACME challenges"
+                break
+
+    def test_acme_challenge_routing(self, docker_compose_acme):
+        """Test that HTTP requests to /.well-known/acme-challenge/ route to certbot backend"""
+        # Request to ACME challenge path
+        # We expect a 404 from certbot standalone server (no actual challenge file)
+        # This confirms routing works - backend server would return different response
+        response = requests.get(
+            'http://localhost/.well-known/acme-challenge/test-token-12345',
+            headers={'Host': 'test.local'},
+            allow_redirects=False
+        )
+
+        # Should NOT redirect to HTTPS (ACME challenges must be HTTP)
+        assert response.status_code != 301 and response.status_code != 302, \
+            "ACME challenge path should not redirect to HTTPS"
+
+        # Expected: 404 or connection error from certbot (not running during challenge)
+        # What we're verifying is that it doesn't return backend's response
+        assert response.status_code in [404, 502, 503], \
+            f"Expected 404/502/503 from certbot backend, got {response.status_code}"
+
+    def test_certificate_issuance(self, docker_compose_acme):
+        """Test that Pebble successfully issues a certificate"""
+        # Check HAProxy logs for certificate issuance
+        result = subprocess.run(
+            ["docker", "logs", "docker-haproxy-1"],
+            capture_output=True,
+            text=True
+        )
+        logs = result.stdout + result.stderr
+
+        # Look for certbot success messages
+        # Certbot outputs: "Successfully received certificate"
+        has_success = "Successfully received certificate" in logs or \
+                     "Certificate not yet due for renewal" in logs or \
+                     "Cert not yet due for renewal" in logs
+
+        # If not successful, check for Pebble connection
+        if not has_success:
+            # Check if we can at least connect to Pebble
+            has_pebble_connection = "pebble:14000/dir" in logs or "pebble:14000" in logs
+            assert has_pebble_connection, \
+                f"HAProxy cannot connect to Pebble ACME server. Check docker network.\nLogs:\n{logs[-2000:]}"
+
+        # Verify merged certificate file exists
+        # EasyHAProxy merges cert+key from /etc/letsencrypt/live/ to /certs/certbot/{domain}.pem
+        merged_cert_path = "/certs/certbot/test.local.pem"
+        result = subprocess.run(
+            ["docker", "exec", "docker-haproxy-1", "test", "-f", merged_cert_path],
+            capture_output=True
+        )
+        assert result.returncode == 0, \
+            f"Merged certificate file not found at {merged_cert_path}. " \
+            f"Certificate issuance or merging may have failed. Check logs: docker logs docker-haproxy-1"
+
+        # Verify merged certificate is valid (contains both cert and key)
+        result = subprocess.run(
+            ["docker", "exec", "docker-haproxy-1", "cat", merged_cert_path],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        cert_content = result.stdout
+        assert '-----BEGIN CERTIFICATE-----' in cert_content, \
+            f"{merged_cert_path} does not contain a certificate"
+        assert '-----END CERTIFICATE-----' in cert_content, \
+            f"{merged_cert_path} certificate is incomplete"
+        assert '-----BEGIN PRIVATE KEY-----' in cert_content or '-----BEGIN RSA PRIVATE KEY-----' in cert_content, \
+            f"{merged_cert_path} does not contain a private key"
+
+    def test_https_with_issued_cert(self, docker_compose_acme):
+        """Test HTTPS works with Pebble-issued certificate"""
+        # Pebble issues real certificates, but from a test CA
+        # Browsers won't trust them, but the TLS handshake should work
+        response = requests.get(
+            'https://localhost/',
+            headers={'Host': 'test.local'},
+            verify=False  # Pebble uses test CA not trusted by system
+        )
+
+        # Should get 200 from backend server
+        assert response.status_code == 200, \
+            f"Expected 200 OK, got {response.status_code}"
+
+        # Verify it's the backend server responding (static-httpserver)
+        assert "soon" in response.text.lower() or "coming" in response.text.lower(), \
+            "Response doesn't match expected backend server content"
+
+    def test_http_to_https_redirect_with_acme_bypass(self, docker_compose_acme):
+        """Test HTTP redirects to HTTPS but ACME challenges bypass redirect"""
+        # Regular HTTP request (not ACME challenge) should redirect
+        response = requests.get(
+            'http://localhost/',
+            headers={'Host': 'test.local'},
+            allow_redirects=False
+        )
+
+        assert response.status_code == 301, \
+            f"Expected HTTP 301 redirect, got {response.status_code}"
+        assert response.headers['Location'].startswith('https://'), \
+            f"Expected redirect to HTTPS, got {response.headers['Location']}"
+
+        # ACME challenge path should NOT redirect (tested in test_acme_challenge_routing)
+
+
 # =============================================================================
 # Helper functions for manual testing
 # =============================================================================
@@ -842,3 +1049,4 @@ if __name__ == "__main__":
     print("  - TestIPWhitelist: IP whitelist plugin tests")
     print("  - TestCloudflare: Cloudflare IP restoration plugin tests")
     print("  - TestChangedLabel: Custom label prefix tests")
+    print("  - TestACME: ACME/Certbot certificate issuance with Pebble test server")