Refactor E2E tests and configuration format
- Replaced `easymapping` configuration with `containers` for better maintainability and clarity. - Introduced `DockerComposeFixture` class in `utils.py` to manage Docker Compose lifecycle and smart build strategy. - Enhanced YAML-to-environment variable conversion in `ContainerEnv` for dynamic configuration support. - Updated HAProxy configurations and test fixtures to reflect the new format. - Improved test coverage for YAML parsing, environment variable handling, and HAProxy config generation.
This commit is contained in:
Joao Gilberto Magalhaes
parent
c125985150
commit
97845b8a52
22 changed files with 1042 additions and 442 deletions
|
|
@ -32,7 +32,9 @@ class DockerLabelHandler:
|
|||
return self.__data[label].lower() in ["true", "1", "yes"]
|
||||
return default_value
|
||||
|
||||
def get_json(self, label, default_value={}):
|
||||
def get_json(self, label, default_value=None):
|
||||
if default_value is None:
|
||||
default_value = {}
|
||||
if self.has_label(label):
|
||||
value = self.__data[label]
|
||||
if not value: # Handle empty strings
|
||||
|
|
@ -178,6 +180,12 @@ class HaproxyConfigGenerator:
|
|||
self.label.create([definition, "clone_to_ssl"])
|
||||
)
|
||||
|
||||
# Check if this is a redirect-only entry (no backend)
|
||||
redirect_only = self.label.get_bool(
|
||||
self.label.create([definition, "redirect_only"]),
|
||||
False
|
||||
)
|
||||
|
||||
if port not in easymapping:
|
||||
easymapping[port] = {
|
||||
"mode": mode,
|
||||
|
|
@ -187,6 +195,12 @@ class HaproxyConfigGenerator:
|
|||
"redirect": dict(),
|
||||
}
|
||||
|
||||
if redirect_only:
|
||||
easymapping[port]["redirect"].update(self.label.get_json(
|
||||
self.label.create([definition, "redirect"])
|
||||
))
|
||||
continue
|
||||
|
||||
# TODO: this could use `EXPOSE` from `Dockerfile`?
|
||||
ct_port = self.label.get(
|
||||
self.label.create([definition, "localport"]),
|
||||
|
|
|
|||
|
|
@ -15,7 +15,20 @@ from OpenSSL import crypto
|
|||
|
||||
class ContainerEnv:
|
||||
@staticmethod
|
||||
def read():
|
||||
def read(yaml_config=None):
|
||||
"""
|
||||
Read configuration from environment variables, optionally merged with YAML config.
|
||||
|
||||
Args:
|
||||
yaml_config: Optional dict from YAML file (for static mode). YAML values take precedence.
|
||||
|
||||
Returns:
|
||||
Dict with configuration settings
|
||||
"""
|
||||
# Convert YAML config to environment variables first (if provided)
|
||||
if yaml_config:
|
||||
ContainerEnv._yaml_to_env(yaml_config)
|
||||
|
||||
env_vars = {
|
||||
"customerrors": True if os.getenv("HAPROXY_CUSTOMERRORS") == "true" else False,
|
||||
"ssl_mode": os.getenv("EASYHAPROXY_SSL_MODE").lower() if os.getenv("EASYHAPROXY_SSL_MODE") else 'default'
|
||||
|
|
@ -115,6 +128,68 @@ class ContainerEnv:
|
|||
|
||||
return env_vars
|
||||
|
||||
@staticmethod
|
||||
def _yaml_to_env(yaml_config):
|
||||
"""Convert YAML configuration to environment variables"""
|
||||
|
||||
# Convert customerrors
|
||||
if 'customerrors' in yaml_config:
|
||||
os.environ['HAPROXY_CUSTOMERRORS'] = 'true' if yaml_config['customerrors'] else 'false'
|
||||
|
||||
# Convert ssl_mode
|
||||
if 'ssl_mode' in yaml_config:
|
||||
os.environ['EASYHAPROXY_SSL_MODE'] = str(yaml_config['ssl_mode'])
|
||||
|
||||
# Convert stats
|
||||
if 'stats' in yaml_config:
|
||||
stats = yaml_config['stats']
|
||||
if 'username' in stats:
|
||||
os.environ['HAPROXY_USERNAME'] = str(stats['username'])
|
||||
if 'password' in stats:
|
||||
os.environ['HAPROXY_PASSWORD'] = str(stats['password'])
|
||||
if 'port' in stats:
|
||||
os.environ['HAPROXY_STATS_PORT'] = str(stats['port'])
|
||||
|
||||
# Convert logLevel
|
||||
if 'logLevel' in yaml_config:
|
||||
log_level = yaml_config['logLevel']
|
||||
for source, level in log_level.items():
|
||||
os.environ[source.upper() + '_LOG_LEVEL'] = str(level)
|
||||
|
||||
# Convert certbot
|
||||
if 'certbot' in yaml_config:
|
||||
certbot = yaml_config['certbot']
|
||||
for config, value in certbot.items():
|
||||
os.environ['EASYHAPROXY_CERTBOT_' + config.upper()] = str(value)
|
||||
|
||||
# Convert plugins
|
||||
if 'plugins' in yaml_config:
|
||||
plugins = yaml_config['plugins']
|
||||
|
||||
# Convert enabled list
|
||||
if 'enabled' in plugins:
|
||||
enabled_list = plugins['enabled'] if isinstance(plugins['enabled'], list) else [plugins['enabled']]
|
||||
os.environ['EASYHAPROXY_PLUGINS_ENABLED'] = ','.join(enabled_list)
|
||||
|
||||
# Convert abort_on_error
|
||||
if 'abort_on_error' in plugins:
|
||||
os.environ['EASYHAPROXY_PLUGINS_ABORT_ON_ERROR'] = 'true' if plugins['abort_on_error'] else 'false'
|
||||
|
||||
# Convert plugin configs
|
||||
if 'config' in plugins:
|
||||
for plugin_name, plugin_config in plugins['config'].items():
|
||||
for config_key, config_value in plugin_config.items():
|
||||
# Convert to env var format: EASYHAPROXY_PLUGIN_<NAME>_<KEY>
|
||||
env_key = f"EASYHAPROXY_PLUGIN_{plugin_name.upper()}_{config_key.upper()}"
|
||||
|
||||
# Convert list values to comma-separated strings
|
||||
if isinstance(config_value, list):
|
||||
env_value = ','.join(str(v) for v in config_value)
|
||||
else:
|
||||
env_value = str(config_value)
|
||||
|
||||
os.environ[env_key] = env_value
|
||||
|
||||
|
||||
class Functions:
|
||||
HAPROXY_LOG: Final[str] = "HAPROXY"
|
||||
|
|
|
|||
|
|
@ -98,48 +98,132 @@ class Static(ProcessorInterface):
|
|||
super().__init__(filename)
|
||||
|
||||
def inspect_network(self):
|
||||
self.parsed_object = {}
|
||||
self.static_content = None
|
||||
|
||||
def get_parsed_object(self):
|
||||
return self.static_content["easymapping"] if "easymapping" in self.static_content else []
|
||||
|
||||
def get_hosts(self):
|
||||
hosts = []
|
||||
for obj in self.get_parsed_object():
|
||||
if "hosts" not in obj:
|
||||
continue
|
||||
for host in obj["hosts"].keys():
|
||||
hosts.append(f"{host}:{obj['port']}")
|
||||
return hosts
|
||||
|
||||
def parse(self):
|
||||
"""Load YAML and convert containers to Docker-style container metadata"""
|
||||
# Load YAML
|
||||
self.static_content = yaml.load(Functions.load(self.filename), Loader=yaml.FullLoader)
|
||||
|
||||
# Merge plugin config from YAML with env vars
|
||||
if "plugins" in self.static_content:
|
||||
# Get env var config
|
||||
container_env = ContainerEnv.read()
|
||||
# Convert containers to label format
|
||||
self.parsed_object = self._convert_yaml_to_labels()
|
||||
|
||||
# Merge YAML plugins config with env config
|
||||
# YAML config takes precedence over env vars
|
||||
if "plugins" not in self.static_content:
|
||||
self.static_content["plugins"] = container_env.get("plugins", {})
|
||||
def _convert_yaml_to_labels(self):
|
||||
"""
|
||||
Convert static YAML containers to Docker label format.
|
||||
Returns: {IP: {labels}} structure that parse() can process
|
||||
"""
|
||||
import json
|
||||
|
||||
container_metadata = {}
|
||||
|
||||
# Get global plugin configuration
|
||||
global_plugins = self.static_content.get("plugins", {})
|
||||
global_enabled = global_plugins.get("enabled", [])
|
||||
global_plugin_config = global_plugins.get("config", {})
|
||||
|
||||
for host_port, config in self.static_content.get("containers", {}).items():
|
||||
# Parse hostname:port from key
|
||||
if ":" in host_port:
|
||||
hostname, port = host_port.rsplit(":", 1)
|
||||
else:
|
||||
# Merge configs - YAML overrides env vars
|
||||
yaml_plugins = self.static_content["plugins"]
|
||||
env_plugins = container_env.get("plugins", {})
|
||||
hostname = host_port
|
||||
port = "80"
|
||||
|
||||
# Merge individual plugin configs
|
||||
for plugin_name, plugin_config in env_plugins.get("config", {}).items():
|
||||
if plugin_name not in yaml_plugins:
|
||||
yaml_plugins[plugin_name] = {}
|
||||
# Env vars fill in missing keys, YAML takes precedence
|
||||
for key, value in plugin_config.items():
|
||||
if key not in yaml_plugins[plugin_name]:
|
||||
yaml_plugins[plugin_name][key] = value
|
||||
# Create definition: hostname_port (e.g., host1_com_br_80)
|
||||
definition = hostname.replace(".", "_") + f"_{port}"
|
||||
|
||||
self.cfg = HaproxyConfigGenerator(self.static_content)
|
||||
# Handle redirect-only entries (no backend)
|
||||
if "redirect" in config and "ip" not in config:
|
||||
# Create metadata with redirect but mark as redirect-only to skip backend creation
|
||||
fake_ip = f"redirect-{hostname}-{port}"
|
||||
if fake_ip not in container_metadata:
|
||||
container_metadata[fake_ip] = {}
|
||||
|
||||
container_metadata[fake_ip].update({
|
||||
f"easyhaproxy.{definition}.host": hostname,
|
||||
f"easyhaproxy.{definition}.port": port,
|
||||
f"easyhaproxy.{definition}.redirect": json.dumps({hostname: config["redirect"]}),
|
||||
f"easyhaproxy.{definition}.redirect_only": "true", # Marker to skip backend
|
||||
})
|
||||
continue
|
||||
|
||||
# Get IPs/containers
|
||||
ip_list = config.get("ip", [hostname])
|
||||
|
||||
# Process each container/IP
|
||||
for container_spec in ip_list:
|
||||
# Parse container:localport
|
||||
if ":" in container_spec:
|
||||
container_addr, localport = container_spec.rsplit(":", 1)
|
||||
else:
|
||||
container_addr = container_spec
|
||||
localport = "80"
|
||||
|
||||
# Use container address as IP (could be IP, DNS, or container name)
|
||||
ip = container_addr
|
||||
|
||||
# Build labels dict
|
||||
labels = {
|
||||
f"easyhaproxy.{definition}.host": hostname,
|
||||
f"easyhaproxy.{definition}.port": port,
|
||||
f"easyhaproxy.{definition}.localport": localport,
|
||||
}
|
||||
|
||||
# Add optional settings
|
||||
for key in ["mode", "certbot", "redirect_ssl", "ssl", "balance", "proto", "ssl-check", "clone_to_ssl"]:
|
||||
if key in config:
|
||||
value = config[key]
|
||||
# Convert boolean to string
|
||||
if isinstance(value, bool):
|
||||
value = "true" if value else "false"
|
||||
labels[f"easyhaproxy.{definition}.{key}"] = str(value)
|
||||
|
||||
# Handle plugins
|
||||
host_plugins = config.get("plugins", global_enabled)
|
||||
if host_plugins:
|
||||
# Convert list to comma-separated string if needed
|
||||
if isinstance(host_plugins, list):
|
||||
plugins_str = ",".join(host_plugins)
|
||||
else:
|
||||
plugins_str = host_plugins
|
||||
labels[f"easyhaproxy.{definition}.plugins"] = plugins_str
|
||||
|
||||
# Process plugin configurations
|
||||
host_plugin_config = config.get("plugin", {})
|
||||
|
||||
# Parse plugins list
|
||||
plugins_list = host_plugins if isinstance(host_plugins, list) else [p.strip() for p in host_plugins.split(",")]
|
||||
|
||||
for plugin_name in plugins_list:
|
||||
# Merge global and host-specific config
|
||||
merged_config = {}
|
||||
if plugin_name in global_plugin_config:
|
||||
merged_config.update(global_plugin_config[plugin_name])
|
||||
if plugin_name in host_plugin_config:
|
||||
merged_config.update(host_plugin_config[plugin_name])
|
||||
|
||||
# Convert plugin config to labels
|
||||
for config_key, config_value in merged_config.items():
|
||||
label_key = f"easyhaproxy.{definition}.plugin.{plugin_name}.{config_key}"
|
||||
|
||||
# Convert list values to comma-separated strings
|
||||
if isinstance(config_value, list):
|
||||
label_value = ",".join(str(v) for v in config_value)
|
||||
else:
|
||||
label_value = str(config_value)
|
||||
|
||||
labels[label_key] = label_value
|
||||
|
||||
# Initialize container entry if it doesn't exist
|
||||
if ip not in container_metadata:
|
||||
container_metadata[ip] = {}
|
||||
|
||||
# Merge labels instead of overwriting
|
||||
container_metadata[ip].update(labels)
|
||||
|
||||
return container_metadata
|
||||
|
||||
def parse(self):
|
||||
"""Create HaproxyConfigGenerator with YAML config merged into env vars"""
|
||||
self.cfg = HaproxyConfigGenerator(ContainerEnv.read(self.static_content))
|
||||
|
||||
|
||||
class Docker(ProcessorInterface):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue