""" Kubernetes Integration Tests for EasyHAProxy This test suite validates EasyHAProxy Kubernetes examples using kind (Kubernetes IN Docker). Requirements: - kind (Kubernetes IN Docker) installed locally in .kind/ - kubectl installed - Docker running - PyJWT and cryptography libraries (for JWT tests): pip install pyjwt cryptography Usage: pytest test_kubernetes.py -v pytest test_kubernetes.py::TestBasicService -v pytest test_kubernetes.py::TestJWTValidatorSecret -v """ import base64 import json import os import re import subprocess import tempfile import time from pathlib import Path from typing import Generator import urllib.request import pytest import requests # Import JWT libraries for token generation try: import jwt from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend JWT_AVAILABLE = True except ImportError: JWT_AVAILABLE = False # Base directory for Kubernetes manifests BASE_DIR = Path(__file__).parent.absolute() BIN_DIR = BASE_DIR / "kubernetes" / ".kind" KIND_BIN = BIN_DIR / "kind" KUBECTL_BIN = BIN_DIR / "kubectl" HELM_BIN = BIN_DIR / "helm" # Port configuration for kind cluster # These ports map from localhost to the kind cluster HTTP_PORT = 10080 # HTTP traffic (localhost:10080 -> cluster:80) HTTPS_PORT = 10443 # HTTPS traffic (localhost:10443 -> cluster:443) STATS_PORT = 11936 # HAProxy stats (localhost:11936 -> cluster:1936) # ============================================================================= # Helper Functions # ============================================================================= def get_latest_kind_version(): url = "https://api.github.com/repos/kubernetes-sigs/kind/releases/latest" req = urllib.request.Request(url, headers={"Accept": "application/vnd.github.v3+json"}) with urllib.request.urlopen(req) as resp: data = json.loads(resp.read()) return data["tag_name"] # e.g. "v0.27.0" def get_latest_helm_version(): url = "https://api.github.com/repos/helm/helm/releases/latest" req = urllib.request.Request(url, headers={"Accept": "application/vnd.github.v3+json"}) with urllib.request.urlopen(req) as resp: data = json.loads(resp.read()) return data["tag_name"] # e.g. "v3.17.3" def ensure_kind_installed(): """Ensure kind is installed locally in .kind/""" if KIND_BIN.exists(): return str(KIND_BIN) print("Installing kind locally...") BIN_DIR.mkdir(exist_ok=True) # Download kind version = get_latest_kind_version() subprocess.run( ["curl", "-Lo", str(KIND_BIN), f"https://kind.sigs.k8s.io/dl/{version}/kind-linux-amd64"], check=True, capture_output=True ) # Make executable KIND_BIN.chmod(0o755) print(f"✓ kind {version} installed to {KIND_BIN}") return str(KIND_BIN) def ensure_kubectl_installed(): """Ensure kubectl is installed locally in .kind/""" # Check if kubectl exists globally first try: subprocess.run( ["kubectl", "version", "--client"], check=True, capture_output=True, timeout=5 ) return "kubectl" # Use global kubectl except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): pass # Use local kubectl if it exists if KUBECTL_BIN.exists(): return str(KUBECTL_BIN) print("Installing kubectl locally...") BIN_DIR.mkdir(exist_ok=True) # Get latest stable version result = subprocess.run( ["curl", "-L", "-s", "https://dl.k8s.io/release/stable.txt"], check=True, capture_output=True, text=True ) version = result.stdout.strip() # Download kubectl subprocess.run( ["curl", "-Lo", str(KUBECTL_BIN), f"https://dl.k8s.io/release/{version}/bin/linux/amd64/kubectl"], check=True, capture_output=True ) # Make executable KUBECTL_BIN.chmod(0o755) print(f"✓ kubectl {version} installed to {KUBECTL_BIN}") return str(KUBECTL_BIN) def ensure_helm_installed(): """Ensure helm is installed locally in .kind/""" # Check if helm exists globally first try: subprocess.run( ["helm", "version"], check=True, capture_output=True, timeout=5 ) return "helm" # Use global helm except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): pass # Use local helm if it exists if HELM_BIN.exists(): return str(HELM_BIN) print("Installing helm locally...") BIN_DIR.mkdir(exist_ok=True) # Download and extract helm helm_version = get_latest_helm_version() helm_tar = BIN_DIR / "helm.tar.gz" subprocess.run( ["curl", "-Lo", str(helm_tar), f"https://get.helm.sh/helm-{helm_version}-linux-amd64.tar.gz"], check=True, capture_output=True ) # Extract helm binary subprocess.run( ["tar", "-xzf", str(helm_tar), "-C", str(BIN_DIR), "--strip-components=1", "linux-amd64/helm"], check=True, capture_output=True ) # Remove tar file helm_tar.unlink() # Make executable HELM_BIN.chmod(0o755) print(f"✓ helm {helm_version} installed to {HELM_BIN}") return str(HELM_BIN) # ============================================================================= # Session Fixtures - kind Cluster Management # ============================================================================= @pytest.fixture(scope="session") def generated_certs(): """ Generate SSL certificates and JWT keys once for the entire test session. This runs the generate-keys.sh script from the examples directory. Returns: dict: Paths to generated certificate files """ print("\n[Setup] Generating SSL certificates and JWT keys...") # Path to generate-keys.sh examples_dir = BASE_DIR.parent generate_keys_script = examples_dir / "generate-keys.sh" if not generate_keys_script.exists(): raise FileNotFoundError(f"generate-keys.sh not found at {generate_keys_script}") # Run the script result = subprocess.run( ["bash", str(generate_keys_script)], cwd=str(examples_dir), capture_output=True, text=True, timeout=30 ) if result.returncode != 0: print(f"✗ Certificate generation failed: {result.stderr}") raise RuntimeError(f"Failed to generate certificates: {result.stderr}") print("✓ SSL certificates and JWT keys generated") # Return paths to generated files return { "host1_local": examples_dir / "static" / "host1.local.pem", "host2_local": examples_dir / "docker" / "host2.local.pem", "jwt_private": examples_dir / "docker" / "jwt_private.pem", "jwt_pubkey": examples_dir / "docker" / "jwt_pubkey.pem", } @pytest.fixture(scope="session") def kubectl_cmd(): """Ensure kubectl is installed and return the command""" return ensure_kubectl_installed() @pytest.fixture(scope="session") def kind_cmd(): """Ensure kind is installed and return the command""" return ensure_kind_installed() @pytest.fixture(scope="session") def helm_cmd(): """Ensure helm is installed and return the command""" return ensure_helm_installed() @pytest.fixture(scope="session") def kind_cluster(kind_cmd, kubectl_cmd, helm_cmd, generated_certs, request): """ Create a kind cluster for the entire test session. The cluster is shared across all tests for better performance. Args: generated_certs: Fixture that ensures certificates are generated before cluster creation """ cluster_name = "easyhaproxy-test" # Register cleanup to always run, even on failure def cleanup(): print(f"\n[Cleanup] Deleting kind cluster '{cluster_name}'...") try: subprocess.run( [kind_cmd, "delete", "cluster", "--name", cluster_name], capture_output=True, timeout=10 ) print("✓ Cluster deleted") except subprocess.TimeoutExpired: print("✗ Cluster deletion timed out (may still be running)") except Exception as e: print(f"✗ Cluster deletion failed: {e}") request.addfinalizer(cleanup) print(f"\n[1/9] Creating kind cluster '{cluster_name}'...") # Check if cluster already exists print("[1/9] Checking for existing cluster...") result = subprocess.run( [kind_cmd, "get", "clusters"], capture_output=True, text=True ) if cluster_name in result.stdout: print(f"[1/9] Cluster '{cluster_name}' already exists, checking if it's healthy...") # Check if cluster is healthy by trying to get nodes result = subprocess.run( [kubectl_cmd, "--context", f"kind-{cluster_name}", "get", "nodes"], capture_output=True, text=True, timeout=5 ) if result.returncode == 0: print(f"[1/9] Cluster '{cluster_name}' is healthy, reusing it...") print("✓ Reusing existing cluster\n") # Skip to yield, return context yield {"name": cluster_name, "kubectl": kubectl_cmd, "certs": generated_certs} return else: print(f"[1/9] Cluster '{cluster_name}' is unhealthy, deleting and recreating...") subprocess.run( [kind_cmd, "delete", "cluster", "--name", cluster_name], check=True, capture_output=True ) # Create cluster with port mappings for HAProxy print("[1/9] Writing cluster config...") cluster_config = BIN_DIR / "cluster-config.yaml" cluster_config.parent.mkdir(exist_ok=True) with open(cluster_config, 'w') as f: f.write(f"""kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - containerPort: 80 hostPort: {HTTP_PORT} protocol: TCP - containerPort: 443 hostPort: {HTTPS_PORT} protocol: TCP - containerPort: 1936 hostPort: {STATS_PORT} protocol: TCP """) print("[2/9] Creating kind cluster (this may take 1-2 minutes)...") subprocess.run( [kind_cmd, "create", "cluster", "--name", cluster_name, "--config", str(cluster_config)], check=True, timeout=180 ) print("[3/9] Setting kubectl context...") subprocess.run( [kubectl_cmd, "config", "use-context", f"kind-{cluster_name}"], check=True, capture_output=True ) print("[3/9] Waiting for cluster nodes to be ready...") subprocess.run( [kubectl_cmd, "wait", "--for=condition=Ready", "nodes", "--all", "--timeout=30s"], check=True, timeout=35 ) print(f"✓ kind cluster '{cluster_name}' is ready") # Build and load local EasyHAProxy image print("[4/9] Building local EasyHAProxy image (may take 30-60s)...") project_root = BASE_DIR.parent.parent subprocess.run( ["docker", "build", "-t", "byjg/easy-haproxy:local", "-f", str(project_root / "build" / "Dockerfile"), str(project_root)], check=True, capture_output=True, timeout=120 ) print("[5/9] Loading image into kind cluster (may take 10-20s)...") subprocess.run( [kind_cmd, "load", "docker-image", "byjg/easy-haproxy:local", "--name", cluster_name], check=True, timeout=30 ) # Generate EasyHAProxy manifest using Helm print("[6/9] Generating EasyHAProxy manifest from Helm...") helm_dir = project_root / "helm" manifest_path = BIN_DIR / "easyhaproxy-local.yml" result = subprocess.run( [helm_cmd, "template", "ingress", str(helm_dir / "easyhaproxy"), "--namespace", "easyhaproxy", "--set", "service.create=false", "--set", "image.tag=local", "--set", "image.pullPolicy=Never"], check=True, capture_output=True, text=True ) # Write manifest to file print("[6/9] Writing manifest to file...") with open(manifest_path, 'w') as f: f.write(result.stdout) # Install EasyHAProxy print("[7/9] Creating easyhaproxy namespace...") subprocess.run( [kubectl_cmd, "create", "namespace", "easyhaproxy"], check=True ) # Apply manifest print("[7/9] Applying EasyHAProxy manifest...") subprocess.run( [kubectl_cmd, "apply", "-f", str(manifest_path)], check=True ) # Label the control-plane node print("[8/9] Labeling control-plane node...") subprocess.run( [kubectl_cmd, "label", "nodes", f"{cluster_name}-control-plane", "easyhaproxy/node=master", "--overwrite"], check=True ) # Wait for EasyHAProxy to be ready print("[9/9] Waiting for EasyHAProxy pods to be ready...") try: subprocess.run( [kubectl_cmd, "wait", "--for=condition=Ready", "pods", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "--timeout=10s"], check=True, capture_output=True, text=True ) print("✓ EasyHAProxy pods are ready") except subprocess.CalledProcessError as e: # Show pod status for debugging print("✗ Pods not ready within 10s. Checking status...") result = subprocess.run( [kubectl_cmd, "get", "pods", "-n", "easyhaproxy", "-o", "wide"], capture_output=True, text=True ) print(result.stdout) # Show pod events result = subprocess.run( [kubectl_cmd, "get", "events", "-n", "easyhaproxy", "--sort-by=.lastTimestamp"], capture_output=True, text=True ) print("Events:") print(result.stdout) raise print("✓ All setup complete! Cluster is ready for tests.\n") yield {"name": cluster_name, "kubectl": kubectl_cmd, "certs": generated_certs} # ============================================================================= # Test Fixtures - Kubernetes Manifest Deployment # ============================================================================= class KubernetesFixture: """Helper class to manage Kubernetes manifest lifecycle""" def __init__(self, manifest_file: str, kubectl_cmd: str, namespace: str = "default", wait_time: int = 5): self.manifest_file = str(BASE_DIR / "kubernetes" / manifest_file) self.kubectl = kubectl_cmd self.namespace = namespace self.wait_time = wait_time def apply(self): """Apply Kubernetes manifest""" # Create namespace if it doesn't exist if self.namespace != "default": subprocess.run( [self.kubectl, "create", "namespace", self.namespace], capture_output=True # Ignore if already exists ) # Apply manifest subprocess.run( [self.kubectl, "apply", "-f", self.manifest_file, "-n", self.namespace], check=True, capture_output=True ) # Wait for pods to be ready time.sleep(self.wait_time) # Wait for all pods to be running max_wait = 60 start_time = time.time() while time.time() - start_time < max_wait: result = subprocess.run( [self.kubectl, "get", "pods", "-n", self.namespace, "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) if not pods['items']: time.sleep(2) continue all_running = all( pod['status']['phase'] == 'Running' for pod in pods['items'] ) if all_running: print(f"✓ All pods running in namespace '{self.namespace}'") break time.sleep(2) def delete(self): """Delete Kubernetes resources""" subprocess.run( [self.kubectl, "delete", "-f", self.manifest_file, "-n", self.namespace, "--ignore-not-found=true"], check=True, capture_output=True ) # Delete namespace if not default if self.namespace != "default": subprocess.run( [self.kubectl, "delete", "namespace", self.namespace, "--ignore-not-found=true"], capture_output=True ) def create_tls_secret_from_pem(kubectl_cmd: str, secret_name: str, namespace: str, pem_file: Path): """ Create a Kubernetes TLS secret from a PEM file. Args: kubectl_cmd: Path to kubectl command secret_name: Name for the secret namespace: Namespace to create the secret in pem_file: Path to the PEM file containing both certificate and key """ print(f" → Creating TLS secret '{secret_name}' from {pem_file.name}...") # Read the PEM file with open(pem_file, 'r') as f: pem_content = f.read() # Split certificate and key (PEM file contains both) cert_start = pem_content.find('-----BEGIN CERTIFICATE-----') cert_end = pem_content.find('-----END CERTIFICATE-----') + len('-----END CERTIFICATE-----') key_start = pem_content.find('-----BEGIN PRIVATE KEY-----') key_end = pem_content.find('-----END PRIVATE KEY-----') + len('-----END PRIVATE KEY-----') # Handle RSA PRIVATE KEY format (openssl genrsa format) if key_start == -1: key_start = pem_content.find('-----BEGIN RSA PRIVATE KEY-----') key_end = pem_content.find('-----END RSA PRIVATE KEY-----') + len('-----END RSA PRIVATE KEY-----') if cert_start == -1 or key_start == -1: raise ValueError(f"Invalid PEM file format in {pem_file}") cert = pem_content[cert_start:cert_end] key = pem_content[key_start:key_end] # Create temp files for cert and key import tempfile with tempfile.NamedTemporaryFile(mode='w', suffix='.crt', delete=False) as cert_file: cert_file.write(cert) cert_path = cert_file.name with tempfile.NamedTemporaryFile(mode='w', suffix='.key', delete=False) as key_file: key_file.write(key) key_path = key_file.name try: # Delete secret if it exists subprocess.run( [kubectl_cmd, "delete", "secret", secret_name, "-n", namespace, "--ignore-not-found=true"], capture_output=True ) # Create secret using kubectl subprocess.run( [kubectl_cmd, "create", "secret", "tls", secret_name, f"--cert={cert_path}", f"--key={key_path}", "-n", namespace], check=True, capture_output=True ) print(f" ✓ TLS secret '{secret_name}' created") finally: # Clean up temp files os.unlink(cert_path) os.unlink(key_path) @pytest.fixture def k8s_service(kind_cluster) -> Generator[str, None, None]: """Fixture for service.yml""" kubectl_cmd = kind_cluster["kubectl"] fixture = KubernetesFixture("service.yml", kubectl_cmd, namespace="default") fixture.apply() yield kubectl_cmd fixture.delete() @pytest.fixture def k8s_ip_whitelist(kind_cluster) -> Generator[str, None, None]: """Fixture for ip-whitelist.yml""" kubectl_cmd = kind_cluster["kubectl"] fixture = KubernetesFixture("ip-whitelist.yml", kubectl_cmd, namespace="default") fixture.apply() yield kubectl_cmd fixture.delete() @pytest.fixture def k8s_service_tls(kind_cluster) -> Generator[str, None, None]: """Fixture for service_tls.yml with generated certificates""" kubectl_cmd = kind_cluster["kubectl"] generated_certs = kind_cluster["certs"] # Create namespace if it doesn't exist subprocess.run( [kubectl_cmd, "create", "namespace", "default"], capture_output=True # Ignore if already exists ) # Create TLS secret from generated certificate create_tls_secret_from_pem( kubectl_cmd, secret_name="host2-tls", namespace="default", pem_file=generated_certs["host2_local"] ) # Apply the manifest (without the embedded secret, we'll use ours) # We need to filter out the Secret from service_tls.yml manifest_path = BASE_DIR / "service_tls.yml" with open(manifest_path, 'r') as f: manifest_content = f.read() # Remove the Secret section from the manifest # Remove everything between "kind: Secret" and the next "---" or end of file manifest_filtered = re.sub( r'^---\s*\napiVersion: v1\s*\nkind: Secret\s*\n.*?(?=^---|\Z)', '', manifest_content, flags=re.MULTILINE | re.DOTALL ) # Write filtered manifest to temp file in the same directory temp_manifest_path = BASE_DIR / "service_tls_generated.yml" with open(temp_manifest_path, 'w') as f: f.write(manifest_filtered) try: # Apply manifest using kubectl directly since we have a custom path subprocess.run( [kubectl_cmd, "apply", "-f", str(temp_manifest_path), "-n", "default"], check=True, capture_output=True ) # Wait for pods to be ready time.sleep(5) # Wait for all pods to be running max_wait = 60 start_time = time.time() while time.time() - start_time < max_wait: result = subprocess.run( [kubectl_cmd, "get", "pods", "-n", "default", "-l", "app=tls-example", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) if not pods['items']: time.sleep(2) continue all_running = all( pod['status']['phase'] == 'Running' for pod in pods['items'] ) if all_running: print("✓ All TLS example pods running") break time.sleep(2) yield kubectl_cmd # Cleanup subprocess.run( [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default", "--ignore-not-found=true"], check=True, capture_output=True ) finally: # Clean up temp manifest if temp_manifest_path.exists(): os.unlink(temp_manifest_path) # Delete the TLS secret subprocess.run( [kubectl_cmd, "delete", "secret", "host2-tls", "-n", "default", "--ignore-not-found=true"], capture_output=True ) @pytest.fixture def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]: """Fixture for jwt-validator-secret-example.yml with generated JWT keys""" if not JWT_AVAILABLE: pytest.skip("PyJWT not available - install with: pip install pyjwt cryptography") kubectl_cmd = kind_cluster["kubectl"] generated_certs = kind_cluster["certs"] # Create namespace if it doesn't exist subprocess.run( [kubectl_cmd, "create", "namespace", "default"], capture_output=True # Ignore if already exists ) # Read the generated JWT public key with open(generated_certs["jwt_pubkey"], 'r') as f: jwt_pubkey_content = f.read() # Create JWT secrets using kubectl print(" → Creating JWT secret 'jwt-pubkey-secret'...") subprocess.run( [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default", "--ignore-not-found=true"], capture_output=True ) subprocess.run( [kubectl_cmd, "create", "secret", "generic", "jwt-pubkey-secret", f"--from-literal=pubkey={jwt_pubkey_content}", "-n", "default"], check=True, capture_output=True ) print(" → Creating JWT secret 'jwt-custom-secret'...") subprocess.run( [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default", "--ignore-not-found=true"], capture_output=True ) subprocess.run( [kubectl_cmd, "create", "secret", "generic", "jwt-custom-secret", f"--from-literal=rsa-public-key={jwt_pubkey_content}", "-n", "default"], check=True, capture_output=True ) # Apply manifest manifest_path = BASE_DIR / "jwt-validator-secret-example.yml" subprocess.run( [kubectl_cmd, "apply", "-f", str(manifest_path), "-n", "default"], check=True, capture_output=True ) # Wait for pods to be ready time.sleep(5) # Wait for all pods to be running max_wait = 60 start_time = time.time() while time.time() - start_time < max_wait: result = subprocess.run( [kubectl_cmd, "get", "pods", "-n", "default", "-l", "app=api", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) if not pods['items']: time.sleep(2) continue all_running = all( pod['status']['phase'] == 'Running' for pod in pods['items'] ) if all_running: print("✓ All JWT API example pods running") break time.sleep(2) # Return context with paths to JWT keys yield { "kubectl": kubectl_cmd, "jwt_private_key": generated_certs["jwt_private"], "jwt_public_key": generated_certs["jwt_pubkey"] } # Cleanup subprocess.run( [kubectl_cmd, "delete", "-f", str(manifest_path), "-n", "default", "--ignore-not-found=true"], check=True, capture_output=True ) # Delete the JWT secrets subprocess.run( [kubectl_cmd, "delete", "secret", "jwt-pubkey-secret", "-n", "default", "--ignore-not-found=true"], capture_output=True ) subprocess.run( [kubectl_cmd, "delete", "secret", "jwt-custom-secret", "-n", "default", "--ignore-not-found=true"], capture_output=True ) @pytest.fixture def k8s_cloudflare(kind_cluster, kind_cmd) -> Generator[str, None, None]: """Fixture for cloudflare.yml with base64-encoded IP list""" kubectl_cmd = kind_cluster["kubectl"] cluster_name = kind_cluster["name"] # Build header-echo server image locally header_echo_dir = BASE_DIR / "fixtures" / "header-echo" print(" → Building header-echo-server:test image...") subprocess.run( ["docker", "build", "-t", "header-echo-server:test", str(header_echo_dir)], check=True, capture_output=True ) # Load image into kind cluster print(" → Loading header-echo-server:test into kind cluster...") subprocess.run( [kind_cmd, "load", "docker-image", "header-echo-server:test", "--name", cluster_name], check=True, capture_output=True ) # Create a modified cloudflare manifest with base64-encoded test IPs # Include 127.0.0.1 and Docker/kind network ranges so test requests work test_ips = [ "127.0.0.1", # localhost for testing "10.0.0.0/8", # Private network "172.16.0.0/12", # Docker default network "192.168.0.0/16", # Private network ] # Base64 encode the IP list ip_list_content = "\n".join(test_ips) ip_list_base64 = base64.b64encode(ip_list_content.encode('utf-8')).decode('ascii') # Read the original manifest manifest_path = BASE_DIR / "kubernetes" / "cloudflare.yml" with open(manifest_path, 'r') as f: manifest_content = f.read() # Add the ip_list annotation # Find the annotations section and add our base64 IP list manifest_modified = manifest_content.replace( 'easyhaproxy.plugins: "cloudflare"', f'easyhaproxy.plugins: "cloudflare"\n easyhaproxy.plugin.cloudflare.ip_list: "{ip_list_base64}"' ) # Write modified manifest to temp file temp_manifest_path = BASE_DIR / "kubernetes" / "cloudflare_test.yml" with open(temp_manifest_path, 'w') as f: f.write(manifest_modified) try: # Apply manifest subprocess.run( [kubectl_cmd, "apply", "-f", str(temp_manifest_path), "-n", "default"], check=True, capture_output=True ) # Wait for pods to be ready time.sleep(5) # Wait for all pods to be running max_wait = 60 start_time = time.time() while time.time() - start_time < max_wait: result = subprocess.run( [kubectl_cmd, "get", "pods", "-n", "default", "-l", "app=webapp", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) if not pods['items']: time.sleep(2) continue all_running = all( pod['status']['phase'] == 'Running' for pod in pods['items'] ) if all_running: print("✓ All cloudflare webapp pods running") break time.sleep(2) yield kubectl_cmd # Cleanup subprocess.run( [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default", "--ignore-not-found=true"], check=True, capture_output=True ) finally: # Clean up temp manifest if temp_manifest_path.exists(): os.unlink(temp_manifest_path) # ============================================================================= # Helper Functions for Tests # ============================================================================= def wait_for_easyhaproxy_discovery(kubectl_cmd: str, expected_host: str, timeout: int = 10) -> bool: """ Wait for EasyHAProxy to discover and configure the ingress host. This function performs multiple checks to ensure the ingress is fully ready: 1. Backend pods are Running 2. Ingress has an ADDRESS assigned 3. EasyHAProxy logs show the host was discovered 4. Simple connectivity test to HAProxy Args: kubectl_cmd: Path to kubectl command expected_host: The hostname to look for in logs (e.g., "example.org") timeout: Maximum seconds to wait (default 10) Returns: True if host is discovered and ready, False if timeout """ start_time = time.time() # Step 1: Wait for backend pods to be Running (with ingress selector) print(f" → Waiting for backend pods with host '{expected_host}' to be ready...") while time.time() - start_time < timeout: try: # Get all ingresses result = subprocess.run( [kubectl_cmd, "get", "ingress", "-A", "-o", "json"], capture_output=True, text=True, timeout=5, check=True ) ingresses = json.loads(result.stdout) # Find ingress with our host ingress_namespace = None for ing in ingresses.get('items', []): for rule in ing.get('spec', {}).get('rules', []): if rule.get('host') == expected_host: ingress_namespace = ing.get('metadata', {}).get('namespace') break if ingress_namespace: break if ingress_namespace: # Check if pods in that namespace are running result = subprocess.run( [kubectl_cmd, "get", "pods", "-n", ingress_namespace, "-o", "json"], capture_output=True, text=True, timeout=5, check=True ) pods = json.loads(result.stdout) all_running = all( pod['status']['phase'] == 'Running' for pod in pods.get('items', []) ) if all_running and pods.get('items'): print(f" ✓ Backend pods are Running") break except Exception: pass time.sleep(1) # Step 2: Wait for ingress to have an ADDRESS assigned print(f" → Waiting for ingress ADDRESS to be assigned...") address_found = False while time.time() - start_time < timeout: try: result = subprocess.run( [kubectl_cmd, "get", "ingress", "-A", "-o", "json"], capture_output=True, text=True, timeout=5, check=True ) ingresses = json.loads(result.stdout) for ing in ingresses.get('items', []): for rule in ing.get('spec', {}).get('rules', []): if rule.get('host') == expected_host: # Check if ingress has loadBalancer status lb_ingress = ing.get('status', {}).get('loadBalancer', {}).get('ingress', []) if lb_ingress: print(f" ✓ Ingress has ADDRESS assigned") address_found = True break if address_found: break if address_found: break except Exception: pass time.sleep(1) # Step 3: Wait for EasyHAProxy to discover the host in logs print(f" → Waiting for EasyHAProxy to discover '{expected_host}'...") while time.time() - start_time < timeout: # Get EasyHAProxy pod logs result = subprocess.run( [kubectl_cmd, "logs", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "--tail=100"], capture_output=True, text=True, timeout=5 ) # Check if "Found hosts:" appears in logs with our expected host if "Found hosts:" in result.stdout and expected_host in result.stdout: print(f" ✓ EasyHAProxy discovered '{expected_host}' in logs") break time.sleep(1) # Step 4: Simple connectivity check to HAProxy print(f" → Testing connectivity to HAProxy...") retries = 3 for attempt in range(retries): try: result = subprocess.run( ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", "-H", f"Host: {expected_host}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=5 ) http_code = result.stdout.strip() # Accept 200, 503 (backend may not be ready yet), or any response that proves HAProxy is responding if http_code and http_code != "000": print(f" ✓ HAProxy is responding (HTTP {http_code})") # Give HAProxy a moment to stabilize after configuration reload time.sleep(2) return True except Exception: pass if attempt < retries - 1: time.sleep(1) # Check if we timed out if time.time() - start_time >= timeout: print(f" ✗ Timeout waiting for '{expected_host}' to be ready") return False return True # ============================================================================= # Test: service.yml - Basic Service # ============================================================================= @pytest.mark.kubernetes class TestBasicService: """Tests for service.yml - Basic Kubernetes service""" def test_resources_created(self, k8s_service): """Test that deployment, service, and ingress are created""" kubectl = k8s_service # Check all resources exist result = subprocess.run( [kubectl, "get", "deployment,service,ingress", "container-example", "-n", "default"], check=True, capture_output=True, text=True ) assert "container-example" in result.stdout def test_pods_running(self, k8s_service): """Test that all pods are running""" kubectl = k8s_service # Wait for deployment to be ready subprocess.run( [kubectl, "wait", "--for=condition=Available", "deployment/container-example", "-n", "default", "--timeout=30s"], check=True ) result = subprocess.run( [kubectl, "get", "pods", "-n", "default", "-l", "app=container-example", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) assert len(pods['items']) > 0, "No container-example pods found" for pod in pods['items']: assert pod['status']['phase'] == 'Running', \ f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" def test_http_request_example_org(self, k8s_service): """Test HTTP request via ingress with example.org""" kubectl = k8s_service # Wait for EasyHAProxy to discover and fully configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "example.org", timeout=30), \ "EasyHAProxy did not become ready for example.org within 30 seconds" # Test HTTP request result = subprocess.run( ["curl", "-s", "-H", "Host: example.org", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert "My Host Example" in result.stdout, \ f"Expected 'My Host Example' in response, got: {result.stdout}" def test_http_request_www_example_org(self, k8s_service): """Test HTTP request via ingress with www.example.org""" kubectl = k8s_service # Wait for EasyHAProxy to discover and fully configure the ingress # (Even though example.org was checked in the previous test, we should verify www.example.org too) assert wait_for_easyhaproxy_discovery(kubectl, "www.example.org", timeout=30), \ "EasyHAProxy did not become ready for www.example.org within 30 seconds" # Test HTTP request result = subprocess.run( ["curl", "-s", "-H", "Host: www.example.org", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert "My Host Example" in result.stdout, \ f"Expected 'My Host Example' in response for www.example.org, got: {result.stdout}" # ============================================================================= # Test: service_tls.yml - TLS/SSL Service # ============================================================================= @pytest.mark.kubernetes class TestTLSService: """Tests for service_tls.yml - TLS/SSL ingress with custom certificates""" def test_resources_created(self, k8s_service_tls): """Test that deployment, service, ingress, and secret are created""" kubectl = k8s_service_tls # Check deployment, service, and ingress exist result = subprocess.run( [kubectl, "get", "deployment,service,ingress", "tls-example", "-n", "default"], check=True, capture_output=True, text=True ) assert "tls-example" in result.stdout # Verify TLS secret exists (separate check since it has different name) result = subprocess.run( [kubectl, "get", "secret", "host2-tls", "-n", "default"], check=True, capture_output=True, text=True ) assert "host2-tls" in result.stdout assert "kubernetes.io/tls" in result.stdout def test_pods_running(self, k8s_service_tls): """Test that all pods are running""" kubectl = k8s_service_tls # Wait for deployment to be ready subprocess.run( [kubectl, "wait", "--for=condition=Available", "deployment/tls-example", "-n", "default", "--timeout=30s"], check=True ) result = subprocess.run( [kubectl, "get", "pods", "-n", "default", "-l", "app=tls-example", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) assert len(pods['items']) > 0, "No tls-example pods found" for pod in pods['items']: assert pod['status']['phase'] == 'Running', \ f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" def test_https_request_host2_local(self, k8s_service_tls): """Test HTTPS request via ingress with host2.local""" kubectl = k8s_service_tls # Wait for EasyHAProxy to discover and fully configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "host2.local", timeout=30), \ "EasyHAProxy did not become ready for host2.local within 30 seconds" # Test HTTPS request (using -k to allow self-signed certificate) result = subprocess.run( ["curl", "-k", "-s", "-H", "Host: host2.local", f"https://localhost:{HTTPS_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert "My Host Example" in result.stdout, \ f"Expected 'My Host Example' in response, got: {result.stdout}" # ============================================================================= # Test: ip-whitelist.yml - IP Whitelist Plugin # ============================================================================= @pytest.mark.kubernetes class TestIPWhitelist: """Tests for ip-whitelist.yml - IP whitelist plugin""" def test_resources_created(self, k8s_ip_whitelist): """Test that deployment, service, and ingress are created""" kubectl = k8s_ip_whitelist # Check deployment exists result = subprocess.run( [kubectl, "get", "deployment", "admin", "-n", "default"], check=True, capture_output=True, text=True ) assert "admin" in result.stdout # Check service exists result = subprocess.run( [kubectl, "get", "service", "admin-service", "-n", "default"], check=True, capture_output=True, text=True ) assert "admin-service" in result.stdout # Check ingress exists result = subprocess.run( [kubectl, "get", "ingress", "admin-ingress-whitelist", "-n", "default"], check=True, capture_output=True, text=True ) assert "admin-ingress-whitelist" in result.stdout def test_pods_running(self, k8s_ip_whitelist): """Test that all admin pods are running""" kubectl = k8s_ip_whitelist # Wait for deployment to be ready subprocess.run( [kubectl, "wait", "--for=condition=Available", "deployment/admin", "-n", "default", "--timeout=30s"], check=True ) result = subprocess.run( [kubectl, "get", "pods", "-n", "default", "-l", "app=admin", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) assert len(pods['items']) > 0, "No admin pods found" for pod in pods['items']: assert pod['status']['phase'] == 'Running', \ f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" def test_haproxy_config_has_ip_whitelist(self, k8s_ip_whitelist): """Test that HAProxy configuration contains IP whitelist rules""" kubectl = k8s_ip_whitelist # Wait for EasyHAProxy to discover the ingress assert wait_for_easyhaproxy_discovery(kubectl, "admin.example.local", timeout=30), \ "EasyHAProxy did not discover admin.example.local within 30 seconds" # Get the EasyHAProxy pod name result = subprocess.run( [kubectl, "get", "pods", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "-o", "jsonpath={.items[0].metadata.name}"], check=True, capture_output=True, text=True ) pod_name = result.stdout.strip() assert pod_name, "EasyHAProxy pod not found" # Get HAProxy configuration result = subprocess.run( [kubectl, "exec", "-n", "easyhaproxy", pod_name, "--", "cat", "/etc/haproxy/haproxy.cfg"], check=True, capture_output=True, text=True ) config = result.stdout # Find the backend for admin service # The backend name should be something like srv_admin_example_local_80 or similar assert "admin" in config.lower(), "Admin service backend not found in HAProxy config" # Verify IP whitelist plugin comment assert "# IP Whitelist - Only allow specific IPs" in config, \ "IP Whitelist plugin comment not found" # Verify ACL for whitelisted IPs assert "acl whitelisted_ip src" in config, \ "IP whitelist ACL not found" # Verify the IPs are in the configuration assert "127.0.0.1" in config, "Localhost not in allowed IPs" assert "10.0.0.0/8" in config, "10.0.0.0/8 network not in allowed IPs" assert "172.16.0.0/12" in config, "172.16.0.0/12 network not in allowed IPs" # Verify deny rule for non-whitelisted IPs assert "http-request deny" in config and "!whitelisted_ip" in config, \ "Deny rule for non-whitelisted IPs not found" # Verify status code 403 assert "deny_status 403" in config, \ "Status code 403 not configured for blocked IPs" def test_access_from_localhost(self, k8s_ip_whitelist): """Test that access from localhost is allowed""" kubectl = k8s_ip_whitelist # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "admin.example.local", timeout=30), \ "EasyHAProxy did not become ready for admin.example.local within 30 seconds" # Test HTTP request (localhost should be in allowed IPs) result = subprocess.run( ["curl", "-s", "-H", "Host: admin.example.local", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert "Admin Panel - IP Restricted" in result.stdout, \ f"Expected 'Admin Panel - IP Restricted' in response, got: {result.stdout}" # ============================================================================= # Test: jwt-validator-secret-example.yml - JWT Validator with Kubernetes Secrets # ============================================================================= @pytest.mark.kubernetes class TestJWTValidatorSecret: """Tests for jwt-validator-secret-example.yml - JWT validation using Kubernetes secrets""" def _generate_jwt_token(self, private_key_path: Path, issuer: str, audience: str, expired: bool = False) -> str: """ Generate a JWT token for testing Args: private_key_path: Path to RSA private key issuer: JWT issuer audience: JWT audience expired: If True, generate an expired token Returns: JWT token string """ # Read private key with open(private_key_path, 'rb') as f: private_key = serialization.load_pem_private_key( f.read(), password=None, backend=default_backend() ) # Set expiration time if expired: exp = int(time.time()) - 3600 # Expired 1 hour ago else: exp = int(time.time()) + 3600 # Valid for 1 hour # Create JWT payload payload = { 'iss': issuer, 'aud': audience, 'exp': exp, 'sub': 'test-user', 'iat': int(time.time()) } # Generate token token = jwt.encode(payload, private_key, algorithm='RS256') return token def test_resources_created(self, k8s_jwt_validator_secret): """Test that secrets, service, and ingresses are created""" kubectl = k8s_jwt_validator_secret["kubectl"] # Check secrets exist result = subprocess.run( [kubectl, "get", "secret", "jwt-pubkey-secret", "-n", "default"], check=True, capture_output=True, text=True ) assert "jwt-pubkey-secret" in result.stdout result = subprocess.run( [kubectl, "get", "secret", "jwt-custom-secret", "-n", "default"], check=True, capture_output=True, text=True ) assert "jwt-custom-secret" in result.stdout # Check service exists result = subprocess.run( [kubectl, "get", "service", "api-service", "-n", "default"], check=True, capture_output=True, text=True ) assert "api-service" in result.stdout # Check both ingresses exist result = subprocess.run( [kubectl, "get", "ingress", "api-ingress-jwt-auto", "-n", "default"], check=True, capture_output=True, text=True ) assert "api-ingress-jwt-auto" in result.stdout result = subprocess.run( [kubectl, "get", "ingress", "api-ingress-jwt-explicit", "-n", "default"], check=True, capture_output=True, text=True ) assert "api-ingress-jwt-explicit" in result.stdout def test_pods_running(self, k8s_jwt_validator_secret): """Test that all API pods are running""" kubectl = k8s_jwt_validator_secret["kubectl"] result = subprocess.run( [kubectl, "get", "pods", "-n", "default", "-l", "app=api", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) assert len(pods['items']) > 0, "No API pods found" for pod in pods['items']: assert pod['status']['phase'] == 'Running', \ f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" def test_haproxy_config_has_jwt_validation(self, k8s_jwt_validator_secret): """Test that HAProxy configuration contains JWT validation rules""" kubectl = k8s_jwt_validator_secret["kubectl"] # Wait for EasyHAProxy to discover the ingress assert wait_for_easyhaproxy_discovery(kubectl, "api.example.local", timeout=30), \ "EasyHAProxy did not discover api.example.local within 30 seconds" # Get the EasyHAProxy pod name result = subprocess.run( [kubectl, "get", "pods", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "-o", "jsonpath={.items[0].metadata.name}"], check=True, capture_output=True, text=True ) pod_name = result.stdout.strip() assert pod_name, "EasyHAProxy pod not found" # Get HAProxy configuration result = subprocess.run( [kubectl, "exec", "-n", "easyhaproxy", pod_name, "--", "cat", "/etc/haproxy/haproxy.cfg"], check=True, capture_output=True, text=True ) config = result.stdout # Verify JWT Validator plugin comments assert "# JWT Validator - Validate JWT tokens" in config, \ "JWT Validator plugin comment not found" # Verify JWT extraction assert "http_auth_bearer,jwt_header_query" in config, \ "JWT header extraction not found" assert "http_auth_bearer,jwt_payload_query" in config, \ "JWT payload extraction not found" # Verify JWT validation rules assert "jwt_verify" in config, \ "JWT signature verification not found" # Verify issuer validation assert "https://auth.example.com/" in config, \ "JWT issuer validation not found" # Verify audience validation assert "https://api.example.com" in config, \ "JWT audience validation not found" # Verify JWT keys directory is used assert "/etc/haproxy/jwt_keys/" in config, \ "JWT keys directory not found in config" def test_access_without_token_denied(self, k8s_jwt_validator_secret): """Test that access without Authorization header is denied""" kubectl = k8s_jwt_validator_secret["kubectl"] # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "api.example.local", timeout=30), \ "EasyHAProxy did not become ready for api.example.local within 30 seconds" # Test HTTP request without Authorization header (should be denied) result = subprocess.run( ["curl", "-s", "-w", "\n%{http_code}", "-H", "Host: api.example.local", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) # Extract HTTP status code from last line lines = result.stdout.strip().split('\n') http_code = lines[-1] assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert http_code == "403", \ f"Expected HTTP 403 for missing Authorization header, got: {http_code}" assert "Missing Authorization HTTP header" in result.stdout, \ f"Expected 'Missing Authorization HTTP header' in response, got: {result.stdout}" def test_access_with_valid_token_allowed(self, k8s_jwt_validator_secret): """Test that access with valid JWT token is allowed""" kubectl = k8s_jwt_validator_secret["kubectl"] jwt_private_key = k8s_jwt_validator_secret["jwt_private_key"] # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "api.example.local", timeout=30), \ "EasyHAProxy did not become ready for api.example.local within 30 seconds" # Generate valid JWT token token = self._generate_jwt_token( jwt_private_key, issuer="https://auth.example.com/", audience="https://api.example.com", expired=False ) # Test HTTP request with valid token (should succeed) result = subprocess.run( ["curl", "-s", "-w", "\n%{http_code}", "-H", "Host: api.example.local", "-H", f"Authorization: Bearer {token}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) # Extract HTTP status code from last line lines = result.stdout.strip().split('\n') http_code = lines[-1] assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert http_code == "200", \ f"Expected HTTP 200 for valid JWT token, got: {http_code}\nResponse: {result.stdout}" def test_access_with_expired_token_denied(self, k8s_jwt_validator_secret): """Test that access with expired JWT token is denied""" kubectl = k8s_jwt_validator_secret["kubectl"] jwt_private_key = k8s_jwt_validator_secret["jwt_private_key"] # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "api.example.local", timeout=30), \ "EasyHAProxy did not become ready for api.example.local within 30 seconds" # Generate expired JWT token token = self._generate_jwt_token( jwt_private_key, issuer="https://auth.example.com/", audience="https://api.example.com", expired=True ) # Test HTTP request with expired token (should be denied) result = subprocess.run( ["curl", "-s", "-w", "\n%{http_code}", "-H", "Host: api.example.local", "-H", f"Authorization: Bearer {token}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) # Extract HTTP status code from last line lines = result.stdout.strip().split('\n') http_code = lines[-1] assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert http_code == "403", \ f"Expected HTTP 403 for expired JWT token, got: {http_code}" assert "JWT has expired" in result.stdout, \ f"Expected 'JWT has expired' in response, got: {result.stdout}" def test_access_with_wrong_issuer_denied(self, k8s_jwt_validator_secret): """Test that access with wrong issuer is denied""" kubectl = k8s_jwt_validator_secret["kubectl"] jwt_private_key = k8s_jwt_validator_secret["jwt_private_key"] # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "api.example.local", timeout=30), \ "EasyHAProxy did not become ready for api.example.local within 30 seconds" # Generate JWT token with wrong issuer token = self._generate_jwt_token( jwt_private_key, issuer="https://wrong-issuer.example.com/", # Wrong issuer audience="https://api.example.com", expired=False ) # Test HTTP request with wrong issuer (should be denied) result = subprocess.run( ["curl", "-s", "-w", "\n%{http_code}", "-H", "Host: api.example.local", "-H", f"Authorization: Bearer {token}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) # Extract HTTP status code from last line lines = result.stdout.strip().split('\n') http_code = lines[-1] assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert http_code == "403", \ f"Expected HTTP 403 for wrong JWT issuer, got: {http_code}" assert "Invalid JWT issuer" in result.stdout, \ f"Expected 'Invalid JWT issuer' in response, got: {result.stdout}" def test_explicit_key_ingress(self, k8s_jwt_validator_secret): """Test that the explicit key format ingress also works""" kubectl = k8s_jwt_validator_secret["kubectl"] jwt_private_key = k8s_jwt_validator_secret["jwt_private_key"] # Wait for EasyHAProxy to discover the explicit key ingress assert wait_for_easyhaproxy_discovery(kubectl, "api-custom.example.local", timeout=30), \ "EasyHAProxy did not discover api-custom.example.local within 30 seconds" # Generate valid JWT token token = self._generate_jwt_token( jwt_private_key, issuer="https://auth.example.com/", audience="https://api.example.com", expired=False ) # Test HTTP request with valid token on explicit key ingress result = subprocess.run( ["curl", "-s", "-w", "\n%{http_code}", "-H", "Host: api-custom.example.local", "-H", f"Authorization: Bearer {token}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) # Extract HTTP status code from last line lines = result.stdout.strip().split('\n') http_code = lines[-1] assert result.returncode == 0, f"Curl failed with return code {result.returncode}" assert http_code == "200", \ f"Expected HTTP 200 for valid JWT token on explicit key ingress, got: {http_code}\nResponse: {result.stdout}" # ============================================================================= # Test: cloudflare.yml - Cloudflare IP Restoration Plugin # ============================================================================= @pytest.mark.kubernetes class TestCloudflare: """Tests for cloudflare.yml - Cloudflare IP restoration from CDN""" def test_resources_created(self, k8s_cloudflare): """Test that deployment, service, and ingress are created""" kubectl = k8s_cloudflare # Check deployment exists result = subprocess.run( [kubectl, "get", "deployment", "webapp", "-n", "default"], check=True, capture_output=True, text=True ) assert "webapp" in result.stdout # Check service exists result = subprocess.run( [kubectl, "get", "service", "webapp-service", "-n", "default"], check=True, capture_output=True, text=True ) assert "webapp-service" in result.stdout # Check ingress exists result = subprocess.run( [kubectl, "get", "ingress", "webapp-ingress-cloudflare", "-n", "default"], check=True, capture_output=True, text=True ) assert "webapp-ingress-cloudflare" in result.stdout def test_pods_running(self, k8s_cloudflare): """Test that all webapp pods are running""" kubectl = k8s_cloudflare # Wait for deployment to be ready subprocess.run( [kubectl, "wait", "--for=condition=Available", "deployment/webapp", "-n", "default", "--timeout=30s"], check=True ) result = subprocess.run( [kubectl, "get", "pods", "-n", "default", "-l", "app=webapp", "-o", "json"], check=True, capture_output=True, text=True ) pods = json.loads(result.stdout) assert len(pods['items']) > 0, "No webapp pods found" for pod in pods['items']: assert pod['status']['phase'] == 'Running', \ f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" def test_haproxy_config_has_cloudflare_plugin(self, k8s_cloudflare): """Test that HAProxy configuration contains Cloudflare plugin rules""" kubectl = k8s_cloudflare # Wait for EasyHAProxy to discover the ingress assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ "EasyHAProxy did not discover myapp.example.local within 30 seconds" # Get the EasyHAProxy pod name result = subprocess.run( [kubectl, "get", "pods", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "-o", "jsonpath={.items[0].metadata.name}"], check=True, capture_output=True, text=True ) pod_name = result.stdout.strip() assert pod_name, "EasyHAProxy pod not found" # Get HAProxy configuration result = subprocess.run( [kubectl, "exec", "-n", "easyhaproxy", pod_name, "--", "cat", "/etc/haproxy/haproxy.cfg"], check=True, capture_output=True, text=True ) config = result.stdout # Verify Cloudflare plugin comment assert "# Cloudflare - Restore original visitor IP" in config, \ "Cloudflare plugin comment not found in HAProxy config" # Verify ACL for Cloudflare IPs assert "acl from_cloudflare src -f /etc/haproxy/cloudflare_ips.lst" in config, \ "Cloudflare IP ACL not found in HAProxy config" # Verify real IP extraction from CF-Connecting-IP header assert "http-request set-var(txn.real_ip) req.hdr(CF-Connecting-IP) if from_cloudflare" in config, \ "CF-Connecting-IP header extraction not found" # Verify X-Forwarded-For header update assert "http-request set-header X-Forwarded-For %[var(txn.real_ip)] if from_cloudflare" in config, \ "X-Forwarded-For header update not found" def test_cloudflare_ip_file_contains_custom_ips(self, k8s_cloudflare): """Test that custom base64-encoded IP list was written to the IP file""" kubectl = k8s_cloudflare # Wait for EasyHAProxy to discover the ingress assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ "EasyHAProxy did not discover myapp.example.local within 30 seconds" # Get the EasyHAProxy pod name result = subprocess.run( [kubectl, "get", "pods", "-n", "easyhaproxy", "-l", "app.kubernetes.io/name=easyhaproxy", "-o", "jsonpath={.items[0].metadata.name}"], check=True, capture_output=True, text=True ) pod_name = result.stdout.strip() # Read the Cloudflare IP list file result = subprocess.run( [kubectl, "exec", "-n", "easyhaproxy", pod_name, "--", "cat", "/etc/haproxy/cloudflare_ips.lst"], check=True, capture_output=True, text=True ) ip_file_content = result.stdout # Verify our custom IPs are in the file (from fixture) assert "127.0.0.1" in ip_file_content, "127.0.0.1 not found in IP list" assert "10.0.0.0/8" in ip_file_content, "10.0.0.0/8 not found in IP list" assert "172.16.0.0/12" in ip_file_content, "172.16.0.0/12 not found in IP list" assert "192.168.0.0/16" in ip_file_content, "192.168.0.0/16 not found in IP list" # Verify it DOESN'T contain built-in Cloudflare IPs # (proves that ip_list took precedence over use_builtin_ips) assert "173.245.48.0/20" not in ip_file_content, \ "Built-in Cloudflare IP found (ip_list should take precedence)" def test_access_to_webapp(self, k8s_cloudflare): """Test that the webapp is accessible and returns JSON""" kubectl = k8s_cloudflare # Wait for EasyHAProxy to discover and configure the ingress assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ "EasyHAProxy did not become ready for myapp.example.local within 30 seconds" # Test HTTP request result = subprocess.run( ["curl", "-s", "-H", "Host: myapp.example.local", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed with return code {result.returncode}" # Parse JSON response data = json.loads(result.stdout) # Verify JSON structure assert "headers" in data, "Response should contain 'headers' field" assert "client_ip" in data, "Response should contain 'client_ip' field" assert "x_forwarded_for" in data, "Response should contain 'x_forwarded_for' field" def test_cloudflare_ip_translation_works(self, k8s_cloudflare): """Test that Cloudflare plugin actually translates CF-Connecting-IP to X-Forwarded-For""" kubectl = k8s_cloudflare # Wait for EasyHAProxy to be ready assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ "EasyHAProxy did not become ready within 30 seconds" # Send request with CF-Connecting-IP header test_ip = "203.0.113.50" result = subprocess.run( ["curl", "-s", "-H", "Host: myapp.example.local", "-H", f"CF-Connecting-IP: {test_ip}", f"http://localhost:{HTTP_PORT}"], capture_output=True, text=True, timeout=10 ) assert result.returncode == 0, f"Curl failed" # Parse JSON response from header-echo server data = json.loads(result.stdout) # VERIFY: X-Forwarded-For was set to the CF-Connecting-IP value # This proves the Cloudflare plugin actually works, not just that config exists assert data['x_forwarded_for'] == test_ip, \ f"Expected X-Forwarded-For to be '{test_ip}' (from CF-Connecting-IP), " \ f"got '{data['x_forwarded_for']}'. Cloudflare IP translation NOT working!" # Verify client_ip is still the HAProxy/ingress IP (connection doesn't change) assert data['client_ip'] != test_ip, \ f"client_ip should be HAProxy pod IP, not the translated IP" # ============================================================================= # Helper functions for manual testing # ============================================================================= def run_manual_test(manifest_file: str): """ Helper function to run a test manually without pytest Example: run_manual_test("service.yml") """ kind_bin = ensure_kind_installed() kubectl_bin = ensure_kubectl_installed() cluster_name = "easyhaproxy-manual-test" print(f"Creating cluster '{cluster_name}'...") subprocess.run( [kind_bin, "create", "cluster", "--name", cluster_name], check=True ) # Wait for cluster to be ready subprocess.run( [kubectl_bin, "wait", "--for=condition=Ready", "nodes", "--all", "--timeout=120s"], check=True ) fixture = None try: fixture = KubernetesFixture(manifest_file, kubectl_bin) fixture.apply() print("✅ Resources deployed successfully!") print("\nPress Enter to cleanup...") input() finally: if fixture: fixture.delete() subprocess.run( [kind_bin, "delete", "cluster", "--name", cluster_name], check=True ) print("✅ Cleanup complete!") if __name__ == "__main__": print("This is a pytest test suite. Run with: pytest test_kubernetes.py -v") print("\nAvailable test classes:") print(" - TestBasicService: Basic Kubernetes service tests") print(" - TestTLSService: TLS/SSL ingress with custom certificates") print(" - TestIPWhitelist: IP whitelist plugin tests") print(" - TestJWTValidatorSecret: JWT validation with Kubernetes secrets")