From 3e963228f31a9b6246730bdba43ce43758c4a987 Mon Sep 17 00:00:00 2001 From: Joao Gilberto Magalhaes Date: Sat, 14 Feb 2026 17:40:40 -0500 Subject: [PATCH] Add ACME/Certbot E2E tests with Pebble integration and update dependencies in CI workflows - Introduced `docker-compose-acme-e2e.yml` for end-to-end testing with Pebble test server. - Added tests to validate ACME challenge routing, certificate issuance, HTTPS functionality, and HAProxy configuration. - Implemented CA certificate download fixture (`create_pebble_ca_file`) for test session initialization. - Updated `.gitignore` to exclude Pebble-related files. - Modified CI workflows to include `needs: [Test]` dependencies for E2E jobs, ensuring proper sequencing. --- .github/workflows/build.yml | 3 + .gitignore | 1 + tests_e2e/docker/docker-compose-acme-e2e.yml | 124 +++++++++++ tests_e2e/test_docker_compose.py | 210 ++++++++++++++++++- 4 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 tests_e2e/docker/docker-compose-acme-e2e.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1b3f055..3fbb6a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: Tests-E2E-Docker: runs-on: ubuntu-latest timeout-minutes: 20 + needs: [Test] permissions: contents: read @@ -66,6 +67,7 @@ jobs: Tests-E2E-Kubernetes: runs-on: ubuntu-latest timeout-minutes: 30 + needs: [Test] permissions: contents: read @@ -88,6 +90,7 @@ jobs: Tests-E2E-Static: runs-on: ubuntu-latest + needs: [Test] timeout-minutes: 20 permissions: contents: read diff --git a/.gitignore b/.gitignore index fc6af0b..41c15f8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ build/ /tests_e2e/docker/jwt_private.pem /tests_e2e/docker/jwt_pubkey.pem /tests_e2e/docker/cloudflare_ips.lst +/tests_e2e/docker/pebble-* diff --git a/tests_e2e/docker/docker-compose-acme-e2e.yml b/tests_e2e/docker/docker-compose-acme-e2e.yml new file mode 100644 index 0000000..88c2686 --- /dev/null +++ b/tests_e2e/docker/docker-compose-acme-e2e.yml @@ -0,0 +1,124 @@ +# ============================================================================== +# E2E Test: ACME/Certbot with Pebble Test Server +# ============================================================================== +# +# WHAT THIS TESTS: +# - HAProxy routing of /.well-known/acme-challenge/ to certbot backend +# - Certbot HTTP-01 challenge completion with Pebble ACME server +# - Certificate issuance and storage in /certs/certbot/live/{domain}/ +# - HTTPS serving with issued certificate +# - Full end-to-end ACME protocol flow +# +# ABOUT PEBBLE: +# Pebble is Let's Encrypt's official ACME test server (RFC 8555 compliant) +# - Runs locally without internet access +# - No rate limits or DNS requirements +# - Issues test certificates (not trusted by browsers) +# - Perfect for integration testing +# +# HOW TO RUN (via pytest): +# ```bash +# cd tests_e2e +# pytest test_docker_compose.py::TestACME -v +# ``` +# +# MANUAL TESTING: +# ```bash +# cd tests_e2e/docker +# docker compose -f docker-compose-acme-e2e.yml up --build +# +# # Wait 10-15 seconds for certificate issuance +# # Check logs +# docker compose -f docker-compose-acme-e2e.yml logs haproxy +# +# # Verify certificate was issued +# ls -la ../../certs/certbot/live/test.local/ +# +# # Test HTTPS (will show certificate warning - expected for test certs) +# curl -k https://localhost/ -H "Host: test.local" +# +# # Cleanup +# docker compose -f docker-compose-acme-e2e.yml down +# ``` +# +# ============================================================================== + +services: + # Pebble ACME Server - Let's Encrypt test environment + pebble: + image: ghcr.io/letsencrypt/pebble:latest + command: -config /test/my-pebble-config.json + environment: + # Speed up validation (no artificial delays) + PEBBLE_VA_NOSLEEP: 1 + # Actually perform challenge validation (not always valid) + PEBBLE_VA_ALWAYS_VALID: 0 + volumes: + # Custom config to use port 80 for validation + - ./pebble-config.json:/test/my-pebble-config.json:ro + ports: + # ACME API endpoint + - "14000:14000" + # Management API (optional) + - "15000:15000" + networks: + - acme-test + + # Backend web server + backend: + image: byjg/static-httpserver + labels: + easyhaproxy.http.host: test.local + easyhaproxy.http.localport: 8080 + easyhaproxy.http.certbot: "true" + easyhaproxy.http.clone_to_ssl: "true" + easyhaproxy.http.redirect_ssl: "true" + networks: + - acme-test + + # EasyHAProxy with Certbot + haproxy: + build: + context: ../.. + dockerfile: build/Dockerfile + depends_on: + - pebble + - backend + environment: + EASYHAPROXY_DISCOVER: docker + HAPROXY_CUSTOMERRORS: "true" + + # Certbot configuration pointing to Pebble + EASYHAPROXY_CERTBOT_EMAIL: test@example.com + EASYHAPROXY_CERTBOT_SERVER: https://pebble:14000/dir + + # Trust Pebble's CA certificate + REQUESTS_CA_BUNDLE: /etc/ssl/certs/pebble-ca.pem + + # Reduce certbot timeout for faster tests + EASYHAPROXY_CERTBOT_TIMEOUT: 30 + + # Enable debug logging for troubleshooting + EASYHAPROXY_DEBUG: "false" + + volumes: + - /var/run/docker.sock:/var/run/docker.sock + # Certificate storage (Docker volume for clean test isolation) + - certbot-certs:/certs/certbot + # Pebble CA certificate (downloaded during test session) + - ./pebble-ca.pem:/etc/ssl/certs/pebble-ca.pem:ro + ports: + - "80:80/tcp" + - "443:443/tcp" + networks: + acme-test: + aliases: + # Allow Pebble to reach HAProxy via test.local for challenge validation + - test.local + +networks: + acme-test: + driver: bridge + +volumes: + certbot-certs: \ No newline at end of file diff --git a/tests_e2e/test_docker_compose.py b/tests_e2e/test_docker_compose.py index 449855a..3a2bd4c 100644 --- a/tests_e2e/test_docker_compose.py +++ b/tests_e2e/test_docker_compose.py @@ -41,6 +41,40 @@ DOCKER_DIR = BASE_DIR / "docker" # Track if cloudflare_ips.lst has been created in this test session _cloudflare_ips_created = False +# Track if pebble CA cert has been downloaded in this test session +_pebble_ca_downloaded = False + + +def create_pebble_ca_file(): + """ + Download Pebble's test CA certificate. + + This file is required for docker-compose-acme-e2e.yml to trust Pebble's HTTPS endpoint. + Downloads from Pebble's GitHub repository. + + Strategy: + - First call: Always download (fresh certificate) + - Subsequent calls: Skip if file exists (reuse from first call) + """ + global _pebble_ca_downloaded + + pebble_ca_path = DOCKER_DIR / "pebble-ca.pem" + + # On subsequent calls, skip if file exists + if _pebble_ca_downloaded and pebble_ca_path.exists() and pebble_ca_path.is_file(): + return + + # Download Pebble's test CA certificate + subprocess.run( + [ + "curl", "-sL", "-o", str(pebble_ca_path), + "https://raw.githubusercontent.com/letsencrypt/pebble/main/test/certs/pebble.minica.pem" + ], + check=True + ) + + # Mark as downloaded for this test session + _pebble_ca_downloaded = True def create_cloudflare_ips_file(): @@ -805,6 +839,179 @@ class TestChangedLabel: verify_haproxy_stats() +# ============================================================================= +# Test: docker-compose-acme-e2e.yml - ACME/Certbot with Pebble +# ============================================================================= + +@pytest.fixture +def docker_compose_acme() -> Generator[None, None, None]: + """Fixture for docker-compose-acme-e2e.yml - ACME/Certbot E2E test""" + volume_name = "docker_certbot-certs" + + # Download Pebble CA certificate (only once per test session) + create_pebble_ca_file() + + # Clean up volume from previous test runs (ensures fresh start) + subprocess.run( + ["docker", "volume", "rm", volume_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL # Ignore error if volume doesn't exist + ) + + fixture = DockerComposeFixture(str(DOCKER_DIR / "docker-compose-acme-e2e.yml"), startup_wait=15) + fixture.up() + yield + fixture.down() + + # Clean up volume after test + subprocess.run( + ["docker", "volume", "rm", volume_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + + +@pytest.mark.acme +class TestACME: + """Tests for docker-compose-acme-e2e.yml - ACME/Certbot with Pebble test server""" + + def test_haproxy_config(self, docker_compose_acme): + """Test HAProxy configuration has ACME challenge routing""" + result = subprocess.run( + ["docker", "exec", "docker-haproxy-1", "cat", "/etc/haproxy/haproxy.cfg"], + capture_output=True, + text=True, + check=True + ) + config = result.stdout + + # Verify ACME challenge ACL exists + assert 'acl is_certbot_test_local_80 path_beg /.well-known/acme-challenge/' in config, \ + "ACME challenge ACL not found in HAProxy config" + + # Verify routing to certbot_backend + assert 'use_backend certbot_backend if is_certbot_test_local_80' in config, \ + "ACME challenge routing rule not found" + + # Verify certbot_backend definition + assert 'backend certbot_backend' in config, \ + "certbot_backend not defined" + assert 'server certbot 127.0.0.1:2080' in config, \ + "certbot backend server not configured correctly" + + # Verify SSL redirect bypasses ACME challenges + # Find redirect rule and verify it excludes certbot ACL + lines = config.split('\n') + for line in lines: + if 'http-request redirect scheme https' in line and 'test_local' in line: + assert '!is_certbot_test_local_80' in line, \ + "SSL redirect should bypass ACME challenges" + break + + def test_acme_challenge_routing(self, docker_compose_acme): + """Test that HTTP requests to /.well-known/acme-challenge/ route to certbot backend""" + # Request to ACME challenge path + # We expect a 404 from certbot standalone server (no actual challenge file) + # This confirms routing works - backend server would return different response + response = requests.get( + 'http://localhost/.well-known/acme-challenge/test-token-12345', + headers={'Host': 'test.local'}, + allow_redirects=False + ) + + # Should NOT redirect to HTTPS (ACME challenges must be HTTP) + assert response.status_code != 301 and response.status_code != 302, \ + "ACME challenge path should not redirect to HTTPS" + + # Expected: 404 or connection error from certbot (not running during challenge) + # What we're verifying is that it doesn't return backend's response + assert response.status_code in [404, 502, 503], \ + f"Expected 404/502/503 from certbot backend, got {response.status_code}" + + def test_certificate_issuance(self, docker_compose_acme): + """Test that Pebble successfully issues a certificate""" + # Check HAProxy logs for certificate issuance + result = subprocess.run( + ["docker", "logs", "docker-haproxy-1"], + capture_output=True, + text=True + ) + logs = result.stdout + result.stderr + + # Look for certbot success messages + # Certbot outputs: "Successfully received certificate" + has_success = "Successfully received certificate" in logs or \ + "Certificate not yet due for renewal" in logs or \ + "Cert not yet due for renewal" in logs + + # If not successful, check for Pebble connection + if not has_success: + # Check if we can at least connect to Pebble + has_pebble_connection = "pebble:14000/dir" in logs or "pebble:14000" in logs + assert has_pebble_connection, \ + f"HAProxy cannot connect to Pebble ACME server. Check docker network.\nLogs:\n{logs[-2000:]}" + + # Verify merged certificate file exists + # EasyHAProxy merges cert+key from /etc/letsencrypt/live/ to /certs/certbot/{domain}.pem + merged_cert_path = "/certs/certbot/test.local.pem" + result = subprocess.run( + ["docker", "exec", "docker-haproxy-1", "test", "-f", merged_cert_path], + capture_output=True + ) + assert result.returncode == 0, \ + f"Merged certificate file not found at {merged_cert_path}. " \ + f"Certificate issuance or merging may have failed. Check logs: docker logs docker-haproxy-1" + + # Verify merged certificate is valid (contains both cert and key) + result = subprocess.run( + ["docker", "exec", "docker-haproxy-1", "cat", merged_cert_path], + capture_output=True, + text=True, + check=True + ) + cert_content = result.stdout + assert '-----BEGIN CERTIFICATE-----' in cert_content, \ + f"{merged_cert_path} does not contain a certificate" + assert '-----END CERTIFICATE-----' in cert_content, \ + f"{merged_cert_path} certificate is incomplete" + assert '-----BEGIN PRIVATE KEY-----' in cert_content or '-----BEGIN RSA PRIVATE KEY-----' in cert_content, \ + f"{merged_cert_path} does not contain a private key" + + def test_https_with_issued_cert(self, docker_compose_acme): + """Test HTTPS works with Pebble-issued certificate""" + # Pebble issues real certificates, but from a test CA + # Browsers won't trust them, but the TLS handshake should work + response = requests.get( + 'https://localhost/', + headers={'Host': 'test.local'}, + verify=False # Pebble uses test CA not trusted by system + ) + + # Should get 200 from backend server + assert response.status_code == 200, \ + f"Expected 200 OK, got {response.status_code}" + + # Verify it's the backend server responding (static-httpserver) + assert "soon" in response.text.lower() or "coming" in response.text.lower(), \ + "Response doesn't match expected backend server content" + + def test_http_to_https_redirect_with_acme_bypass(self, docker_compose_acme): + """Test HTTP redirects to HTTPS but ACME challenges bypass redirect""" + # Regular HTTP request (not ACME challenge) should redirect + response = requests.get( + 'http://localhost/', + headers={'Host': 'test.local'}, + allow_redirects=False + ) + + assert response.status_code == 301, \ + f"Expected HTTP 301 redirect, got {response.status_code}" + assert response.headers['Location'].startswith('https://'), \ + f"Expected redirect to HTTPS, got {response.headers['Location']}" + + # ACME challenge path should NOT redirect (tested in test_acme_challenge_routing) + + # ============================================================================= # Helper functions for manual testing # ============================================================================= @@ -841,4 +1048,5 @@ if __name__ == "__main__": print(" - TestPluginsCombined: Combined plugins tests") print(" - TestIPWhitelist: IP whitelist plugin tests") print(" - TestCloudflare: Cloudflare IP restoration plugin tests") - print(" - TestChangedLabel: Custom label prefix tests") \ No newline at end of file + print(" - TestChangedLabel: Custom label prefix tests") + print(" - TestACME: ACME/Certbot certificate issuance with Pebble test server") \ No newline at end of file