From 1fd5873dc9fc13523509e35b972da8e855e3eda7 Mon Sep 17 00:00:00 2001 From: Joao Gilberto Magalhaes Date: Thu, 12 Feb 2026 16:03:50 -0500 Subject: [PATCH] Add Cloudflare plugin tests with base64-encoded IP list support - Enhanced the Cloudflare plugin by adding support for base64-encoded IP lists, prioritizing them over built-in lists. - Introduced Kubernetes integration tests for the Cloudflare IP restoration feature. - Validated plugin behavior for resource creation, pod readiness, ingress access, and HAProxy configurations. - Improved annotations in `cloudflare.yml` for Kubernetes-native compatibility. - Updated unit tests to cover IP list precedence, invalid base64 handling, and metadata verification. --- examples/kubernetes/cloudflare.yml | 10 +- examples/kubernetes/test_kubernetes.py | 260 +++++++++++++++++++++++++ src/plugins/builtin/cloudflare.py | 54 ++++- tests/test_plugins.py | 125 ++++++++++++ 4 files changed, 439 insertions(+), 10 deletions(-) diff --git a/examples/kubernetes/cloudflare.yml b/examples/kubernetes/cloudflare.yml index 106a8ad..84f8c32 100644 --- a/examples/kubernetes/cloudflare.yml +++ b/examples/kubernetes/cloudflare.yml @@ -112,10 +112,16 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - # Enable Cloudflare plugin + # Enable Cloudflare plugin with built-in IPs easyhaproxy.plugins: "cloudflare" - # Optional: Specify custom IP list path + # Optional: Provide custom IP list as base64-encoded text (takes precedence over built-in IPs) + # This is more Kubernetes-native than mounting ConfigMaps/files + # Example IPs: 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 127.0.0.1 + # How to create: printf "10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16\n127.0.0.1" | base64 -w 0 + # easyhaproxy.plugin.cloudflare.ip_list: "MTAuMC4wLjAvOAoxNzIuMTYuMC4wLzEyCjE5Mi4xNjguMC4wLzE2CjEyNy4wLjAuMQ==" + + # Optional: Specify custom IP list file path (only used if ip_list is not provided) # easyhaproxy.plugin.cloudflare.ip_list_path: "/etc/haproxy/cloudflare_ips.lst" name: webapp-ingress-cloudflare namespace: default diff --git a/examples/kubernetes/test_kubernetes.py b/examples/kubernetes/test_kubernetes.py index eece6de..a9a0c47 100644 --- a/examples/kubernetes/test_kubernetes.py +++ b/examples/kubernetes/test_kubernetes.py @@ -839,6 +839,94 @@ def k8s_jwt_validator_secret(kind_cluster) -> Generator[dict, None, None]: ) +@pytest.fixture +def k8s_cloudflare(kind_cluster) -> Generator[str, None, None]: + """Fixture for cloudflare.yml with base64-encoded IP list""" + kubectl_cmd = kind_cluster["kubectl"] + + # Create a modified cloudflare manifest with base64-encoded test IPs + # Include 127.0.0.1 and Docker/kind network ranges so test requests work + test_ips = [ + "127.0.0.1", # localhost for testing + "10.0.0.0/8", # Private network + "172.16.0.0/12", # Docker default network + "192.168.0.0/16", # Private network + ] + + # Base64 encode the IP list + ip_list_content = "\n".join(test_ips) + ip_list_base64 = base64.b64encode(ip_list_content.encode('utf-8')).decode('ascii') + + # Read the original manifest + manifest_path = BASE_DIR / "cloudflare.yml" + with open(manifest_path, 'r') as f: + manifest_content = f.read() + + # Add the ip_list annotation + # Find the annotations section and add our base64 IP list + manifest_modified = manifest_content.replace( + 'easyhaproxy.plugins: "cloudflare"', + f'easyhaproxy.plugins: "cloudflare"\n easyhaproxy.plugin.cloudflare.ip_list: "{ip_list_base64}"' + ) + + # Write modified manifest to temp file + temp_manifest_path = BASE_DIR / "cloudflare_test.yml" + with open(temp_manifest_path, 'w') as f: + f.write(manifest_modified) + + try: + # Apply manifest + subprocess.run( + [kubectl_cmd, "apply", "-f", str(temp_manifest_path), "-n", "default"], + check=True, + capture_output=True + ) + + # Wait for pods to be ready + time.sleep(5) + + # Wait for all pods to be running + max_wait = 60 + start_time = time.time() + while time.time() - start_time < max_wait: + result = subprocess.run( + [kubectl_cmd, "get", "pods", "-n", "default", "-l", "app=webapp", "-o", "json"], + check=True, + capture_output=True, + text=True + ) + pods = json.loads(result.stdout) + + if not pods['items']: + time.sleep(2) + continue + + all_running = all( + pod['status']['phase'] == 'Running' + for pod in pods['items'] + ) + + if all_running: + print("✓ All cloudflare webapp pods running") + break + + time.sleep(2) + + yield kubectl_cmd + + # Cleanup + subprocess.run( + [kubectl_cmd, "delete", "-f", str(temp_manifest_path), "-n", "default", + "--ignore-not-found=true"], + check=True, + capture_output=True + ) + finally: + # Clean up temp manifest + if temp_manifest_path.exists(): + os.unlink(temp_manifest_path) + + # ============================================================================= # Helper Functions for Tests # ============================================================================= @@ -1642,6 +1730,178 @@ class TestJWTValidatorSecret: f"Expected HTTP 200 for valid JWT token on explicit key ingress, got: {http_code}\nResponse: {result.stdout}" +# ============================================================================= +# Test: cloudflare.yml - Cloudflare IP Restoration Plugin +# ============================================================================= + +@pytest.mark.kubernetes +class TestCloudflare: + """Tests for cloudflare.yml - Cloudflare IP restoration from CDN""" + + def test_resources_created(self, k8s_cloudflare): + """Test that deployment, service, and ingress are created""" + kubectl = k8s_cloudflare + + # Check deployment exists + result = subprocess.run( + [kubectl, "get", "deployment", "webapp", "-n", "default"], + check=True, + capture_output=True, + text=True + ) + assert "webapp" in result.stdout + + # Check service exists + result = subprocess.run( + [kubectl, "get", "service", "webapp-service", "-n", "default"], + check=True, + capture_output=True, + text=True + ) + assert "webapp-service" in result.stdout + + # Check ingress exists + result = subprocess.run( + [kubectl, "get", "ingress", "webapp-ingress-cloudflare", "-n", "default"], + check=True, + capture_output=True, + text=True + ) + assert "webapp-ingress-cloudflare" in result.stdout + + def test_pods_running(self, k8s_cloudflare): + """Test that all webapp pods are running""" + kubectl = k8s_cloudflare + + # Wait for deployment to be ready + subprocess.run( + [kubectl, "wait", "--for=condition=Available", "deployment/webapp", + "-n", "default", "--timeout=30s"], + check=True + ) + + result = subprocess.run( + [kubectl, "get", "pods", "-n", "default", "-l", "app=webapp", "-o", "json"], + check=True, + capture_output=True, + text=True + ) + pods = json.loads(result.stdout) + + assert len(pods['items']) > 0, "No webapp pods found" + + for pod in pods['items']: + assert pod['status']['phase'] == 'Running', \ + f"Pod {pod['metadata']['name']} is not running: {pod['status']['phase']}" + + def test_haproxy_config_has_cloudflare_plugin(self, k8s_cloudflare): + """Test that HAProxy configuration contains Cloudflare plugin rules""" + kubectl = k8s_cloudflare + + # Wait for EasyHAProxy to discover the ingress + assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ + "EasyHAProxy did not discover myapp.example.local within 30 seconds" + + # Get the EasyHAProxy pod name + result = subprocess.run( + [kubectl, "get", "pods", "-n", "easyhaproxy", + "-l", "app.kubernetes.io/name=easyhaproxy", + "-o", "jsonpath={.items[0].metadata.name}"], + check=True, + capture_output=True, + text=True + ) + pod_name = result.stdout.strip() + assert pod_name, "EasyHAProxy pod not found" + + # Get HAProxy configuration + result = subprocess.run( + [kubectl, "exec", "-n", "easyhaproxy", pod_name, + "--", "cat", "/etc/haproxy/haproxy.cfg"], + check=True, + capture_output=True, + text=True + ) + config = result.stdout + + # Verify Cloudflare plugin comment + assert "# Cloudflare - Restore original visitor IP" in config, \ + "Cloudflare plugin comment not found in HAProxy config" + + # Verify ACL for Cloudflare IPs + assert "acl from_cloudflare src -f /etc/haproxy/cloudflare_ips.lst" in config, \ + "Cloudflare IP ACL not found in HAProxy config" + + # Verify real IP extraction from CF-Connecting-IP header + assert "http-request set-var(txn.real_ip) req.hdr(CF-Connecting-IP) if from_cloudflare" in config, \ + "CF-Connecting-IP header extraction not found" + + # Verify X-Forwarded-For header update + assert "http-request set-header X-Forwarded-For %[var(txn.real_ip)] if from_cloudflare" in config, \ + "X-Forwarded-For header update not found" + + def test_cloudflare_ip_file_contains_custom_ips(self, k8s_cloudflare): + """Test that custom base64-encoded IP list was written to the IP file""" + kubectl = k8s_cloudflare + + # Wait for EasyHAProxy to discover the ingress + assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ + "EasyHAProxy did not discover myapp.example.local within 30 seconds" + + # Get the EasyHAProxy pod name + result = subprocess.run( + [kubectl, "get", "pods", "-n", "easyhaproxy", + "-l", "app.kubernetes.io/name=easyhaproxy", + "-o", "jsonpath={.items[0].metadata.name}"], + check=True, + capture_output=True, + text=True + ) + pod_name = result.stdout.strip() + + # Read the Cloudflare IP list file + result = subprocess.run( + [kubectl, "exec", "-n", "easyhaproxy", pod_name, + "--", "cat", "/etc/haproxy/cloudflare_ips.lst"], + check=True, + capture_output=True, + text=True + ) + ip_file_content = result.stdout + + # Verify our custom IPs are in the file (from fixture) + assert "127.0.0.1" in ip_file_content, "127.0.0.1 not found in IP list" + assert "10.0.0.0/8" in ip_file_content, "10.0.0.0/8 not found in IP list" + assert "172.16.0.0/12" in ip_file_content, "172.16.0.0/12 not found in IP list" + assert "192.168.0.0/16" in ip_file_content, "192.168.0.0/16 not found in IP list" + + # Verify it DOESN'T contain built-in Cloudflare IPs + # (proves that ip_list took precedence over use_builtin_ips) + assert "173.245.48.0/20" not in ip_file_content, \ + "Built-in Cloudflare IP found (ip_list should take precedence)" + + def test_access_to_webapp(self, k8s_cloudflare): + """Test that the webapp is accessible via the Cloudflare ingress""" + kubectl = k8s_cloudflare + + # Wait for EasyHAProxy to discover and configure the ingress + assert wait_for_easyhaproxy_discovery(kubectl, "myapp.example.local", timeout=30), \ + "EasyHAProxy did not become ready for myapp.example.local within 30 seconds" + + # Test HTTP request + result = subprocess.run( + ["curl", "-s", "-H", "Host: myapp.example.local", + f"http://localhost:{HTTP_PORT}"], + capture_output=True, + text=True, + timeout=10 + ) + + assert result.returncode == 0, f"Curl failed with return code {result.returncode}" + assert "App Behind Cloudflare" in result.stdout, \ + f"Expected 'App Behind Cloudflare' in response, got: {result.stdout}" + + # ============================================================================= # Helper functions for manual testing # ============================================================================= diff --git a/src/plugins/builtin/cloudflare.py b/src/plugins/builtin/cloudflare.py index b7722b8..2dd954f 100644 --- a/src/plugins/builtin/cloudflare.py +++ b/src/plugins/builtin/cloudflare.py @@ -9,6 +9,7 @@ updated and written to the IP list file. Configuration: - ip_list_path: Path to file containing Cloudflare IP ranges (default: /etc/haproxy/cloudflare_ips.lst) + - ip_list: Base64-encoded list of IP ranges (one per line), takes precedence over ip_list_path - use_builtin_ips: Use built-in Cloudflare IP ranges (default: true) - update_log_format: Update HAProxy log format to show real visitor IP (default: true) @@ -20,6 +21,11 @@ Example YAML config: use_builtin_ips: true update_log_format: true +Example Kubernetes Ingress Annotation: + easyhaproxy.plugins: "cloudflare" + easyhaproxy.plugin.cloudflare.ip_list: "MTAuMC4wLjAvOAoxNzIuMTYuMC4wLzEyCjE5Mi4xNjguMC4wLzE2Cg==" + easyhaproxy.plugin.cloudflare.update_log_format: "true" + Example Container Label: easyhaproxy.http.plugins: "cloudflare" easyhaproxy.http.plugin.cloudflare.update_log_format: "true" @@ -35,6 +41,7 @@ Log Format (when update_log_format=true): Format: real_ip/connection_ip [timestamp] request status bytes ... """ +import base64 import os import sys @@ -82,6 +89,7 @@ class CloudflarePlugin(PluginInterface): self.enabled = True self.use_builtin_ips = True self.update_log_format = True + self.ip_list = None @property def name(self) -> str: @@ -98,6 +106,7 @@ class CloudflarePlugin(PluginInterface): Args: config: Dictionary with configuration options - ip_list_path: Path to Cloudflare IP list file + - ip_list: Base64-encoded list of IP ranges (one per line) - enabled: Whether plugin is enabled - use_builtin_ips: Use built-in Cloudflare IP ranges (default: true) - update_log_format: Update HAProxy log format to show real IP (default: true) @@ -105,6 +114,14 @@ class CloudflarePlugin(PluginInterface): if "ip_list_path" in config: self.ip_list_path = config["ip_list_path"] + if "ip_list" in config: + # Decode from base64 (consistent with JWT validator pubkey parameter) + try: + self.ip_list = base64.b64decode(config["ip_list"]).decode('utf-8') + except Exception as e: + logger_easyhaproxy.warning(f"Cloudflare plugin: Failed to decode ip_list: {e}") + self.ip_list = None + if "enabled" in config: self.enabled = str(config["enabled"]).lower() in ["true", "1", "yes"] @@ -127,22 +144,41 @@ class CloudflarePlugin(PluginInterface): if not self.enabled: return PluginResult() - # Write built-in Cloudflare IPs to file if using built-in IPs - if self.use_builtin_ips: + # Determine which IPs to write to file + ips_to_write = None + ip_source = None + + if self.ip_list: + # Priority 1: Base64-encoded ip_list from annotation + ip_lines = [line.strip() for line in self.ip_list.split('\n') if line.strip()] + ips_to_write = ip_lines + ip_source = "base64 ip_list" + elif self.use_builtin_ips: + # Priority 2: Built-in Cloudflare IPs + ips_to_write = self.CLOUDFLARE_IPS + ip_source = "built-in IPs" + + # Write IPs to file if we have any + if ips_to_write: try: - # Create directory if it doesn't exist + # Create directory if needed ip_list_dir = os.path.dirname(self.ip_list_path) if ip_list_dir and not os.path.exists(ip_list_dir): os.makedirs(ip_list_dir, exist_ok=True) - # Write Cloudflare IPs to file + # Write IPs to file with open(self.ip_list_path, 'w') as f: - for ip_range in self.CLOUDFLARE_IPS: + for ip_range in ips_to_write: f.write(f"{ip_range}\n") - logger_easyhaproxy.info(f"Cloudflare plugin: Written {len(self.CLOUDFLARE_IPS)} IP ranges to {self.ip_list_path}") + logger_easyhaproxy.info( + f"Cloudflare plugin: Written {len(ips_to_write)} IP ranges " + f"from {ip_source} to {self.ip_list_path}" + ) except Exception as e: - logger_easyhaproxy.warning(f"Cloudflare plugin: Failed to write IP list to {self.ip_list_path}: {e}") + logger_easyhaproxy.warning( + f"Cloudflare plugin: Failed to write IP list to {self.ip_list_path}: {e}" + ) # Generate HAProxy config snippet for backend haproxy_config = f"""# Cloudflare - Restore original visitor IP @@ -164,9 +200,11 @@ log-format "%{+Q}[var(txn.real_ip)]:-/%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%T metadata={ "domain": context.domain, "ip_list_path": self.ip_list_path, + "ip_list_provided": self.ip_list is not None, "use_builtin_ips": self.use_builtin_ips, "update_log_format": self.update_log_format, "defaults_config": log_format_config, - "ip_count": len(self.CLOUDFLARE_IPS) if self.use_builtin_ips else None + "ip_count": len(ips_to_write) if ips_to_write else None, + "ip_source": ip_source if ips_to_write else "existing file" } ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index fd4c7cc..397b860 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -199,6 +199,131 @@ class TestCloudflarePlugin: assert result.metadata["use_builtin_ips"] is False assert result.metadata["ip_count"] is None + def test_cloudflare_plugin_with_base64_ip_list(self): + """Test Cloudflare plugin with base64-encoded IP list""" + import base64 + import os + + plugin = CloudflarePlugin() + + # Create test IP list + test_ips = "10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16" + ip_list_base64 = base64.b64encode(test_ips.encode('utf-8')).decode('ascii') + + # Configure with base64 IP list + plugin.configure({ + "ip_list": ip_list_base64, + "ip_list_path": "/tmp/test_cloudflare_ips.lst" + }) + + # Verify it was decoded + assert plugin.ip_list == test_ips + + # Process and verify file creation + context = PluginContext( + parsed_object={}, + easymapping=[], + container_env={}, + domain="test.example.com", + port="80", + host_config={} + ) + result = plugin.process(context) + + # Verify file was written with our IPs + assert os.path.exists("/tmp/test_cloudflare_ips.lst") + with open("/tmp/test_cloudflare_ips.lst", 'r') as f: + content = f.read() + + assert "10.0.0.0/8" in content + assert "172.16.0.0/12" in content + assert "192.168.0.0/16" in content + + # Verify built-in IPs were NOT written + assert "173.245.48.0/20" not in content + + # Cleanup + os.unlink("/tmp/test_cloudflare_ips.lst") + + def test_cloudflare_plugin_ip_list_precedence(self): + """Test that ip_list takes precedence over use_builtin_ips""" + import base64 + import os + + plugin = CloudflarePlugin() + + test_ips = "127.0.0.1" + ip_list_base64 = base64.b64encode(test_ips.encode('utf-8')).decode('ascii') + + # Configure with BOTH ip_list and use_builtin_ips + plugin.configure({ + "ip_list": ip_list_base64, + "use_builtin_ips": "true", + "ip_list_path": "/tmp/test_precedence.lst" + }) + + # Process + context = PluginContext( + parsed_object={}, + easymapping=[], + container_env={}, + domain="test.example.com", + port="80", + host_config={} + ) + result = plugin.process(context) + + # Verify file contains ONLY our IP, not built-in IPs + with open("/tmp/test_precedence.lst", 'r') as f: + content = f.read() + + assert "127.0.0.1" in content + assert "173.245.48.0/20" not in content # Built-in IP should NOT be there + + # Verify metadata shows ip_list was provided + assert result.metadata["ip_list_provided"] is True + assert result.metadata["ip_source"] == "base64 ip_list" + + # Cleanup + os.unlink("/tmp/test_precedence.lst") + + def test_cloudflare_plugin_invalid_base64(self): + """Test Cloudflare plugin handles invalid base64 gracefully""" + import os + + plugin = CloudflarePlugin() + + # Configure with invalid base64 + plugin.configure({ + "ip_list": "not-valid-base64!!!", + "use_builtin_ips": "true", + "ip_list_path": "/tmp/test_invalid.lst" + }) + + # Should fall back to use_builtin_ips + assert plugin.ip_list is None + + # Process should still work with built-in IPs + context = PluginContext( + parsed_object={}, + easymapping=[], + container_env={}, + domain="test.example.com", + port="80", + host_config={} + ) + result = plugin.process(context) + + # Verify fallback to built-in IPs + assert os.path.exists("/tmp/test_invalid.lst") + with open("/tmp/test_invalid.lst", 'r') as f: + content = f.read() + + assert "173.245.48.0/20" in content # Built-in IP + + # Cleanup + os.unlink("/tmp/test_invalid.lst") + class TestCleanupPlugin: """Test cases for CleanupPlugin (GLOBAL plugin)"""