From 3b2e9a43fdbd12bcedcb11113325f3eae96fd91e Mon Sep 17 00:00:00 2001 From: Joao Gilberto Magalhaes Date: Sat, 1 Jul 2023 17:55:25 -0500 Subject: [PATCH] Fixing errors / reformating code --- src/easymapping/__init__.py | 27 +++--- src/functions/__init__.py | 59 +++++++++--- src/main.py | 24 +++-- src/processor/__init__.py | 148 ++++++++++++++-------------- src/tests/context.py | 4 +- src/tests/test_containerenv.py | 170 +++++++++++++++++---------------- src/tests/test_docker.py | 6 +- src/tests/test_functions.py | 25 +++-- src/tests/test_labels.py | 11 ++- src/tests/test_parser.py | 161 ++++++++++++++++--------------- src/tests/test_static.py | 49 +++++----- 11 files changed, 368 insertions(+), 316 deletions(-) diff --git a/src/easymapping/__init__.py b/src/easymapping/__init__.py index c0db5f7..34fa05c 100644 --- a/src/easymapping/__init__.py +++ b/src/easymapping/__init__.py @@ -1,12 +1,13 @@ import base64 -import hashlib -from jinja2 import Environment, FileSystemLoader import json -import os import re +from jinja2 import Environment, FileSystemLoader + + class DockerLabelHandler: def __init__(self, label): + self.__data = None self.__label_base = label def get_lookup_label(self): @@ -18,19 +19,17 @@ class DockerLabelHandler: return "{}.{}".format(self.__label_base, ".".join(key)) - - def get(self, label, default_value = ""): + def get(self, label, default_value=""): if self.has_label(label): return self.__data[label] return default_value - - def get_bool(self, label, default_value = False): + def get_bool(self, label, default_value=False): if self.has_label(label): return self.__data[label].lower() in ["true", "1", "yes"] return default_value - def get_json(self, label, default_value = {}): + def get_json(self, label, default_value={}): if self.has_label(label): return json.loads(self.__data[label]) return default_value @@ -38,7 +37,6 @@ class DockerLabelHandler: def set_data(self, data): self.__data = data - def has_label(self, label): if label in self.__data: return True @@ -55,10 +53,10 @@ class HaproxyConfigGenerator: self.certbot_hosts = [] self.serving_hosts = [] self.certs = {} - - def generate(self, container_metadata = {}): + + def generate(self, container_metadata={}): self.mapping.setdefault("easymapping", []) - + if container_metadata != {}: self.mapping["easymapping"] = self.parse(container_metadata) @@ -70,7 +68,6 @@ class HaproxyConfigGenerator: template = env.get_template('haproxy.cfg.j2') return template.render(data=self.mapping) - def parse(self, container_metadata): easymapping = dict() @@ -163,8 +160,8 @@ class HaproxyConfigGenerator: easymapping["443"]["hosts"][hostname]["certbot"] = False easymapping["443"]["hosts"][hostname]["redirect_ssl"] = False easymapping["443"]["ssl"] = True - self.certbot_hosts.append(hostname) if certbot and hostname not in self.certbot_hosts else self.certbot_hosts - + self.certbot_hosts.append( + hostname) if certbot and hostname not in self.certbot_hosts else self.certbot_hosts # handle SSL ssl_label = self.label.create([definition, "sslcert"]) diff --git a/src/functions/__init__.py b/src/functions/__init__.py index 9713843..e704cb7 100644 --- a/src/functions/__init__.py +++ b/src/functions/__init__.py @@ -1,14 +1,41 @@ -from datetime import datetime -from multiprocessing import Process, Lock -import subprocess -import shlex -import time import os -import re +import shlex +import subprocess import time +from datetime import datetime +from multiprocessing import Process + from OpenSSL import crypto +class ContainerEnv: + @staticmethod + def read(): + env_vars = { + "customerrors": True if os.getenv("HAPROXY_CUSTOMERRORS") == "true" else False, + "ssl_mode": os.getenv("EASYHAPROXY_SSL_MODE").lower() if os.getenv("EASYHAPROXY_SSL_MODE") else 'default' + } + + if os.getenv("HAPROXY_PASSWORD"): + env_vars["stats"] = { + "username": os.getenv("HAPROXY_USERNAME") if os.getenv("HAPROXY_USERNAME") else "admin", + "password": os.getenv("HAPROXY_PASSWORD"), + "port": os.getenv("HAPROXY_STATS_PORT") if os.getenv("HAPROXY_STATS_PORT") else "1936", + } + + env_vars["lookup_label"] = os.getenv("EASYHAPROXY_LABEL_PREFIX") if os.getenv( + "EASYHAPROXY_LABEL_PREFIX") else "easyhaproxy" + + env_vars["certbot"] = { + "email": os.getenv("EASYHAPROXY_CERTBOT_EMAIL", ""), + "server": os.getenv("EASYHAPROXY_CERTBOT_SERVER", False), + "eab_kid": os.getenv("EASYHAPROXY_CERTBOT_EAB_KID", ""), + "eab_hmac_key": os.getenv("EASYHAPROXY_CERTBOT_EAB_HMAC_KEY", ""), + } + + return env_vars + + class Functions: HAPROXY_LOG = "HAPROXY" EASYHAPROXY_LOG = "EASYHAPROXY" @@ -119,8 +146,8 @@ class DaemonizeHAProxy: else: pid = "".join(Functions().run_bash(Functions.HAPROXY_LOG, "cat /run/haproxy.pid", log_output=False)) self.__prepare( - "/usr/sbin/haproxy -W -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -x /var/run/haproxy.sock -sf %s" % ( - pid)) + "/usr/sbin/haproxy -W -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -x /var/run/haproxy.sock -sf %s" % + pid) if self.process is None: return @@ -151,8 +178,8 @@ class DaemonizeHAProxy: for line in iter(self.process.stdout.readline, b''): Functions.log(source, Functions.INFO, line) - returncode = self.process.wait() - Functions.log(source, Functions.DEBUG, "Return code %s" % (returncode)) + return_code = self.process.wait() + Functions.log(source, Functions.DEBUG, "Return code %s" % return_code) except Exception as e: Functions.log(source, Functions.ERROR, "%s" % e) @@ -188,7 +215,8 @@ class Certbot: self.eab_kid = self.set_eab_kid(env["certbot"]["eab_kid"]) self.eab_hmac_key = self.set_eab_hmac_key(env["certbot"]["eab_hmac_key"]) - def set_acme_server(self, acme_server): + @staticmethod + def set_acme_server(acme_server): if acme_server.lower() == "staging": return "--staging" elif acme_server.lower().startswith("http"): @@ -196,13 +224,15 @@ class Certbot: else: return "" - def set_eab_kid(self, eab_kid): + @staticmethod + def set_eab_kid(eab_kid): if eab_kid != "": return "--eab-kid \"%s\"" % eab_kid else: return "" - def set_eab_hmac_key(self, eab_hmac_key): + @staticmethod + def set_eab_hmac_key(eab_hmac_key): if eab_hmac_key != "": return "--eab-hmac-key \"%s\"" % eab_hmac_key else: @@ -274,7 +304,8 @@ class Certbot: Functions.log(Functions.CERTBOT_LOG, Functions.ERROR, "%s" % e) return False - def merge_certificate(self, cert, key, filename): + @staticmethod + def merge_certificate(cert, key, filename): Functions.save(filename, cert + key) def find_live_certificates(self): diff --git a/src/main.py b/src/main.py index c50e0d2..d55c148 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,10 @@ +import os + +from deepdiff import DeepDiff + from functions import Functions, DaemonizeHAProxy, Certbot, Consts from processor import ProcessorInterface -import os -from deepdiff import DeepDiff + def start(): processor_obj = ProcessorInterface.factory(os.getenv("EASYHAPROXY_DISCOVER")) @@ -14,7 +17,8 @@ def start(): processor_obj.save_config(Consts.haproxy_config) processor_obj.save_certs(Consts.certs_haproxy) certbot_certs_found = processor_obj.get_certbot_hosts() - Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, 'Found hosts: %s' % ", ".join(processor_obj.get_hosts())) # Needs to run after save_config + Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, + 'Found hosts: %s' % ", ".join(processor_obj.get_hosts())) # Needs to run after save_config Functions.log(Functions.EASYHAPROXY_LOG, Functions.TRACE, 'Object Found: %s' % (processor_obj.get_parsed_object())) old_haproxy = None @@ -31,27 +35,28 @@ def start(): try: old_parsed = processor_obj.get_parsed_object() processor_obj.refresh() - if certbot.check_certificates(certbot_certs_found) or DeepDiff(old_parsed, processor_obj.get_parsed_object()) != {} or not haproxy.is_alive(): + if certbot.check_certificates(certbot_certs_found) or DeepDiff(old_parsed, + processor_obj.get_parsed_object()) != {} or not haproxy.is_alive(): Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, 'New configuration found. Reloading...') - Functions.log(Functions.EASYHAPROXY_LOG, Functions.TRACE, 'Object Found: %s' % (processor_obj.get_parsed_object())) + Functions.log(Functions.EASYHAPROXY_LOG, Functions.TRACE, + 'Object Found: %s' % (processor_obj.get_parsed_object())) processor_obj.save_config(Consts.haproxy_config) processor_obj.save_certs(Consts.certs_haproxy) certbot_certs_found = processor_obj.get_certbot_hosts() - Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, 'Found hosts: %s' % ", ".join(processor_obj.get_hosts())) # Needs to after save_config + Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, + 'Found hosts: %s' % ", ".join(processor_obj.get_hosts())) # Needs to after save_config old_haproxy = haproxy haproxy = DaemonizeHAProxy() haproxy.haproxy("reload") old_haproxy.terminate() except Exception as e: - Functions.log(Functions.EASYHAPROXY_LOG, Functions.FATAL, "Err: %s" % (e)) + Functions.log(Functions.EASYHAPROXY_LOG, Functions.FATAL, "Err: %s" % e) Functions.log(Functions.EASYHAPROXY_LOG, Functions.DEBUG, 'Heartbeat') haproxy.sleep() - - def main(): Functions.run_bash(Functions.INIT_LOG, '/usr/sbin/haproxy -v') @@ -69,5 +74,6 @@ def main(): start() + if __name__ == '__main__': main() diff --git a/src/processor/__init__.py b/src/processor/__init__.py index 9583c32..68d55c6 100644 --- a/src/processor/__init__.py +++ b/src/processor/__init__.py @@ -1,46 +1,26 @@ -from easymapping import HaproxyConfigGenerator -from functions import Functions, Consts -import yaml -import sys -import os -import json import base64 -import docker import socket + +import docker +import yaml from kubernetes import client, config from kubernetes.client.rest import ApiException -class ContainerEnv: - @staticmethod - def read(): - env_vars = { - "customerrors": True if os.getenv("HAPROXY_CUSTOMERRORS") == "true" else False, - "ssl_mode": os.getenv("EASYHAPROXY_SSL_MODE").lower() if os.getenv("EASYHAPROXY_SSL_MODE") else 'default' - } - - if os.getenv("HAPROXY_PASSWORD"): - env_vars["stats"] = { - "username": os.getenv("HAPROXY_USERNAME") if os.getenv("HAPROXY_USERNAME") else "admin", - "password": os.getenv("HAPROXY_PASSWORD"), - "port": os.getenv("HAPROXY_STATS_PORT") if os.getenv("HAPROXY_STATS_PORT") else "1936", - } - - env_vars["lookup_label"] = os.getenv("EASYHAPROXY_LABEL_PREFIX") if os.getenv("EASYHAPROXY_LABEL_PREFIX") else "easyhaproxy" - - env_vars["certbot"] = { - "email": os.getenv("EASYHAPROXY_CERTBOT_EMAIL", ""), - "server": os.getenv("EASYHAPROXY_CERTBOT_SERVER", False), - "eab_kid": os.getenv("EASYHAPROXY_CERTBOT_EAB_KID", ""), - "eab_hmac_key": os.getenv("EASYHAPROXY_CERTBOT_EAB_HMAC_KEY", ""), - } - - return env_vars +from easymapping import HaproxyConfigGenerator +from functions import Functions, Consts, ContainerEnv class ProcessorInterface: static_file = Consts.easyhaproxy_config - def __init__(self, filename = None): + def __init__(self, filename=None): + self.certbot_hosts = None + self.parsed_object = None + self.cfg = None + self.hosts = None + self.cfg = None + self.certbot_hosts = None + self.hosts = None self.filename = filename self.refresh() @@ -55,7 +35,8 @@ class ProcessorInterface: elif mode == "kubernetes": return Kubernetes() else: - Functions.log("EASYHAPROXY", Functions.FATAL, "Expected mode to be 'static', 'docker', 'swarm' or 'kubernetes'. I got '%s'" % (mode)) + Functions.log("EASYHAPROXY", Functions.FATAL, + "Expected mode to be 'static', 'docker', 'swarm' or 'kubernetes'. I got '%s'" % mode) return None def refresh(self): @@ -67,7 +48,7 @@ class ProcessorInterface: self.parse() def inspect_network(self): - #Abstract + # Abstract pass def parse(self): @@ -82,7 +63,7 @@ class ProcessorInterface: def get_parsed_object(self): return self.parsed_object - def get_certs(self, key = None): + def get_certs(self, key=None): if key is None: return self.cfg.certs else: @@ -103,20 +84,27 @@ class ProcessorInterface: class Static(ProcessorInterface): + def __init__(self, filename=None): + self.parsed_object = None + self.static_content = None + self.static_content = None + self.cfg = None + super().__init__(filename) + def inspect_network(self): self.parsed_object = {} self.static_content = None - + def get_parsed_object(self): return self.static_content["easymapping"] if "easymapping" in self.static_content else [] def get_hosts(self): hosts = [] - for object in self.get_parsed_object(): - if "hosts" not in object: + for obj in self.get_parsed_object(): + if "hosts" not in obj: continue - for host in object["hosts"].keys(): - hosts.append("%s:%s" % (host, object["port"])) + for host in obj["hosts"].keys(): + hosts.append("%s:%s" % (host, obj["port"])) return hosts def parse(self): @@ -125,18 +113,21 @@ class Static(ProcessorInterface): class Docker(ProcessorInterface): - def __init__(self, filename = None): + def __init__(self, filename=None): + self.parsed_object = None self.client = docker.from_env() super().__init__() - + def inspect_network(self): try: - ha_proxy_network_name = next(iter(self.client.containers.get(socket.gethostname()).attrs["NetworkSettings"]["Networks"])) + ha_proxy_network_name = next( + iter(self.client.containers.get(socket.gethostname()).attrs["NetworkSettings"]["Networks"])) except: # HAProxy is not running in a container, get first container network if len(self.client.containers.list()) == 0: return - ha_proxy_network_name = next(iter(self.client.containers.get(self.client.containers.list()[0].name).attrs["NetworkSettings"]["Networks"])) + ha_proxy_network_name = next(iter( + self.client.containers.get(self.client.containers.list()[0].name).attrs["NetworkSettings"]["Networks"])) ha_proxy_network = self.client.networks.get(ha_proxy_network_name) @@ -152,17 +143,22 @@ class Docker(ProcessorInterface): class Swarm(ProcessorInterface): - def __init__(self, filename = None): + def __init__(self, filename=None): + self.parsed_object = None self.client = docker.from_env() super().__init__() def inspect_network(self): ha_proxy_service_name = self.client.containers.get(socket.gethostname()).name.split('.')[0] - for endpoint in self.client.services.get(ha_proxy_service_name).attrs['Endpoint']["VirtualIPs"]: + ha_proxy_network_id = None + for endpoint in self.client.services.get(ha_proxy_service_name).attrs['Endpoint']["VirtualIPs"]: ha_proxy_network_id = endpoint["NetworkID"] if self.client.networks.get(ha_proxy_network_id).name != 'ingress': break + if ha_proxy_network_id is None: + raise "Could not find ingress network" + self.parsed_object = {} for service in self.client.services.list(): ip_address = None @@ -172,17 +168,18 @@ class Swarm(ProcessorInterface): ip_address = endpoint["Addr"].split("/")[0] break network_list.append(endpoint["NetworkID"]) - + if ip_address is None: network_list.append(ha_proxy_network_id) - service.update(networks = network_list) - continue # skip to the next service to give time to update the network - + service.update(networks=network_list) + continue # skip to the next service to give time to update the network + self.parsed_object[ip_address] = service.attrs["Spec"]["Labels"] class Kubernetes(ProcessorInterface): - def __init__(self, filename = None): + def __init__(self, filename=None): + self.parsed_object = None config.load_incluster_config() config.verify_ssl = False self.api_instance = client.CoreV1Api() @@ -190,13 +187,14 @@ class Kubernetes(ProcessorInterface): self.cert_cache = {} super().__init__() - def _check_annotation(self, annotations, key): + @staticmethod + def _check_annotation(annotations, key): if key not in annotations: return None return annotations[key] def inspect_network(self): - + ret = self.v1.list_ingress_for_all_namespaces(watch=False) self.parsed_object = {} @@ -216,10 +214,8 @@ class Kubernetes(ProcessorInterface): if listen_port is None: listen_port = 80 - data = {} - data["creation_timestamp"] = ingress.metadata.creation_timestamp.strftime("%x %X") - data["resource_version"] = ingress.metadata.resource_version - data["namespace"] = ingress.metadata.namespace + data = {"creation_timestamp": ingress.metadata.creation_timestamp.strftime("%x %X"), + "resource_version": ingress.metadata.resource_version, "namespace": ingress.metadata.namespace} ingress_name = ingress.metadata.namespace @@ -233,33 +229,36 @@ class Kubernetes(ProcessorInterface): if tls.secret_name not in self.cert_cache or self.cert_cache[tls.secret_name] != secret.data: self.cert_cache[tls.secret_name] = secret.data Functions.save( - "{0}/{1}.pem".format(Consts.certs_haproxy, tls.secret_name), - base64.b64decode(secret.data["tls.crt"]).decode('ascii') + "\n" + base64.b64decode(secret.data["tls.key"]).decode('ascii') + "{0}/{1}.pem".format(Consts.certs_haproxy, tls.secret_name), + base64.b64decode(secret.data["tls.crt"]).decode('ascii') + "\n" + base64.b64decode( + secret.data["tls.key"]).decode('ascii') ) ssl_hosts.extend(tls.hosts) except Exception as e: - Functions.log("EASYHAPROXY", Functions.WARN, "Ingress %s - Get secret failed: '%s'" % (ingress_name, e)) + Functions.log("EASYHAPROXY", Functions.WARN, + "Ingress %s - Get secret failed: '%s'" % (ingress_name, e)) - Functions.log("EASYHAPROXY", Functions.TRACE, "Ingress %s - SSL Hosts found '%s'" % (ingress_name, ssl_hosts)) + Functions.log("EASYHAPROXY", Functions.TRACE, + "Ingress %s - SSL Hosts found '%s'" % (ingress_name, ssl_hosts)) for rule in ingress.spec.rules: rule_data = {} port_number = rule.http.paths[0].backend.service.port.number definition = "easyhaproxy.%s_%s" % (rule.host.replace(".", "-"), port_number) - rule_data["%s.host" % (definition)] = rule.host - rule_data["%s.port" % (definition)] = listen_port - rule_data["%s.localport" % (definition)] = port_number + rule_data["%s.host" % definition] = rule.host + rule_data["%s.port" % definition] = listen_port + rule_data["%s.localport" % definition] = port_number if rule.host in ssl_hosts: - rule_data["%s.clone_to_ssl" % (definition)] = 'true' + rule_data["%s.clone_to_ssl" % definition] = 'true' if redirect_ssl is not None: - rule_data["%s.redirect_ssl" % (definition)] = redirect_ssl + rule_data["%s.redirect_ssl" % definition] = redirect_ssl if certbot is not None: - rule_data["%s.certbot" % (definition)] = certbot + rule_data["%s.certbot" % definition] = certbot if redirect is not None: - rule_data["%s.redirect" % (definition)] = redirect + rule_data["%s.redirect" % definition] = redirect if mode is not None: - rule_data["%s.mode" % (definition)] = mode + rule_data["%s.mode" % definition] = mode service_name = rule.http.paths[0].backend.service.name try: @@ -267,13 +266,10 @@ class Kubernetes(ProcessorInterface): cluster_ip = api_response.spec.cluster_ip except ApiException as e: cluster_ip = None - Functions.log("EASYHAPROXY", Functions.WARN, "Ingress %s - Service %s - Failed: '%s'" % (ingress_name, service_name, e)) - + Functions.log("EASYHAPROXY", Functions.WARN, + "Ingress %s - Service %s - Failed: '%s'" % (ingress_name, service_name, e)) + if cluster_ip is not None: if cluster_ip not in self.parsed_object.keys(): self.parsed_object[cluster_ip] = data self.parsed_object[cluster_ip].update(rule_data) - - - - diff --git a/src/tests/context.py b/src/tests/context.py index 67dda65..66ad651 100644 --- a/src/tests/context.py +++ b/src/tests/context.py @@ -1,5 +1,5 @@ import os import sys -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -import easymapping + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/src/tests/test_containerenv.py b/src/tests/test_containerenv.py index 4b84810..0347788 100644 --- a/src/tests/test_containerenv.py +++ b/src/tests/test_containerenv.py @@ -1,108 +1,113 @@ -import pytest import os -from processor import ContainerEnv + +from functions import ContainerEnv + def test_container_env_empty(): assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() # os.environ['CERTBOT_LOG_LEVEL'] = 'warn' + def test_container_env_customerrors(): os.environ['HAPROXY_CUSTOMERRORS'] = 'true' try: assert { - "customerrors": True, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + "customerrors": True, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() finally: os.environ['HAPROXY_CUSTOMERRORS'] = '' + def test_container_env_sslmode(): os.environ['EASYHAPROXY_SSL_MODE'] = 'STRICT' try: assert { - "customerrors": False, - "ssl_mode": "strict", - "lookup_label": "easyhaproxy", - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "strict", + "lookup_label": "easyhaproxy", + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() finally: os.environ['EASYHAPROXY_SSL_MODE'] = '' + def test_container_env_stats(): os.environ['HAPROXY_USERNAME'] = 'abc' os.environ['HAPROXY_STATS_PORT'] = '2101' try: assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() finally: os.environ['HAPROXY_USERNAME'] = '' os.environ['HAPROXY_STATS_PORT'] = '' + def test_container_env_stats_password(): os.environ['HAPROXY_PASSWORD'] = 'xyz' try: assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "stats": { - "username": "admin", - "password": "xyz", - "port": "1936" + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "stats": { + "username": "admin", + "password": "xyz", + "port": "1936" - }, - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + }, + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() finally: os.environ['HAPROXY_PASSWORD'] = '' -def test_container_env_stats_password(): +def test_container_env_stats_password_2(): os.environ['HAPROXY_USERNAME'] = 'abc' os.environ['HAPROXY_STATS_PORT'] = '2101' os.environ['HAPROXY_PASSWORD'] = 'xyz' try: assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "stats": { - "username": "abc", - "password": "xyz", - "port": "2101" - }, - "certbot": {"eab_hmac_key": "", - "eab_kid": "", - "email": "", - "server": False} - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "stats": { + "username": "abc", + "password": "xyz", + "port": "2101" + }, + "certbot": {"eab_hmac_key": "", + "eab_kid": "", + "email": "", + "server": False} + } == ContainerEnv.read() finally: os.environ['HAPROXY_USERNAME'] = '' os.environ['HAPROXY_STATS_PORT'] = '' @@ -113,19 +118,20 @@ def test_container_env_certbot_email(): os.environ['EASYHAPROXY_CERTBOT_EMAIL'] = 'acme@example.org' try: assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "certbot": { - 'eab_hmac_key': "", - 'eab_kid': "", - "email": "acme@example.org", - "server": False - } - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "certbot": { + 'eab_hmac_key': "", + 'eab_kid': "", + "email": "acme@example.org", + "server": False + } + } == ContainerEnv.read() finally: os.environ['EASYHAPROXY_CERTBOT_EMAIL'] = '' + def test_container_env_certbot_full(): os.environ['EASYHAPROXY_CERTBOT_EMAIL'] = 'acme@example.org' os.environ['EASYHAPROXY_CERTBOT_SERVER'] = 'schema://url/a' @@ -133,15 +139,15 @@ def test_container_env_certbot_full(): os.environ['EASYHAPROXY_CERTBOT_EAB_HMAC_KEY'] = 'eab_hmac_key' try: assert { - "customerrors": False, - "ssl_mode": "default", - "lookup_label": "easyhaproxy", - "certbot": { - "email": "acme@example.org", - "server": "schema://url/a", - 'eab_hmac_key': 'eab_hmac_key', - 'eab_kid': 'eab_kid', - } - } == ContainerEnv.read() + "customerrors": False, + "ssl_mode": "default", + "lookup_label": "easyhaproxy", + "certbot": { + "email": "acme@example.org", + "server": "schema://url/a", + 'eab_hmac_key': 'eab_hmac_key', + 'eab_kid': 'eab_kid', + } + } == ContainerEnv.read() finally: - os.environ['EASYHAPROXY_CERTBOT_EMAIL'] = '' \ No newline at end of file + os.environ['EASYHAPROXY_CERTBOT_EMAIL'] = '' diff --git a/src/tests/test_docker.py b/src/tests/test_docker.py index 6aaaf61..b6b95e1 100644 --- a/src/tests/test_docker.py +++ b/src/tests/test_docker.py @@ -1,10 +1,11 @@ -import pytest import os import time + import docker +import pytest + from functions import Functions from processor import ProcessorInterface -from processor import Docker def _get_hydrated_object(parsed_objects, lookup_key): @@ -17,7 +18,6 @@ def _get_hydrated_object(parsed_objects, lookup_key): def _get_ip_host(parsed_objects, lookup_key): - hydrated_object = {} for key in parsed_objects: for keys in parsed_objects[key]: if lookup_key in keys: diff --git a/src/tests/test_functions.py b/src/tests/test_functions.py index 5f40267..861e8d1 100644 --- a/src/tests/test_functions.py +++ b/src/tests/test_functions.py @@ -1,11 +1,11 @@ -import json -import pytest import os -import re import random +import re import string + from functions import Functions + def test_functions_check_local_level(): assert Functions.skip_log('CERTBOT', Functions.INFO) == False assert Functions.skip_log('HAPOROXY', Functions.INFO) == False @@ -23,6 +23,7 @@ def test_functions_check_local_level(): assert Functions.skip_log('EASYHAPROXY', Functions.INFO) == True os.environ['EASYHAPROXY_LOG_LEVEL'] = '' + def test_function_load_and_save(): filename = '/tmp/x.txt' try: @@ -34,6 +35,7 @@ def test_function_load_and_save(): finally: os.unlink(filename) + def test_functions_check_log_sanity(): print() Functions.log(Functions.EASYHAPROXY_LOG, Functions.INFO, "Test 1") @@ -51,44 +53,51 @@ def test_functions_check_log_sanity(): assert re.match("\[EASYHAPROXY\] .* \[INFO\]: Test 3", Functions.debug_log[1]) os.environ['EASYHAPROXY_LOG_LEVEL'] = 'warn' - Functions.log(Functions.EASYHAPROXY_LOG, Functions.INFO, "Test 4") # Should not log to debug + Functions.log(Functions.EASYHAPROXY_LOG, Functions.INFO, "Test 4") # Should not log to debug assert len(Functions.debug_log) == 2 finally: os.environ['EASYHAPROXY_LOG_LEVEL'] = '' Functions.debug_log = None + def test_functions_run_bash_log_output(): print() Functions.debug_log = [] try: - result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 1'", log_output=True, return_result=False) + result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 1'", log_output=True, + return_result=False) assert result == [] assert len(Functions.debug_log) == 1 assert re.match("\[EASYHAPROXY\] .* \[INFO\]: test run 1", Functions.debug_log[0]) finally: Functions.debug_log = None + def test_functions_run_bash_no_log_output(): print() Functions.debug_log = [] try: - result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 2'", log_output=False, return_result=False) + result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 2'", log_output=False, + return_result=False) assert result == [] assert len(Functions.debug_log) == 0 finally: Functions.debug_log = None + def test_functions_run_bash_return(): print() Functions.debug_log = [] try: - result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 3'", log_output=False, return_result=True) + result = Functions.run_bash(Functions.EASYHAPROXY_LOG, "echo 'test run 3'", log_output=False, + return_result=True) assert len(Functions.debug_log) == 0 assert "".join(result) == 'test run 3' finally: Functions.debug_log = None + def test_functions_run_bash_log_and_return_output(): print() Functions.debug_log = [] @@ -98,4 +107,4 @@ def test_functions_run_bash_log_and_return_output(): assert len(Functions.debug_log) == 1 assert re.match("\[EASYHAPROXY\] .* \[INFO\]: test run 4", Functions.debug_log[0]) finally: - Functions.debug_log = None \ No newline at end of file + Functions.debug_log = None diff --git a/src/tests/test_labels.py b/src/tests/test_labels.py index f56c212..14ce279 100644 --- a/src/tests/test_labels.py +++ b/src/tests/test_labels.py @@ -1,16 +1,17 @@ -from .context import easymapping import json -import pytest + +from easymapping import DockerLabelHandler + def test_label_generation(): - label = easymapping.DockerLabelHandler("foo") + label = DockerLabelHandler("foo") assert label.create("bar") == "foo.bar" assert label.create(["bar", "foobar"]) == "foo.bar.foobar" def test_label_data(): - label = easymapping.DockerLabelHandler("base") + label = DockerLabelHandler("base") label.set_data(json.loads('{"base.definitions":"h2"}')) label_name = label.create("definitions") @@ -20,7 +21,7 @@ def test_label_data(): def test_label_complex_key(): - label = easymapping.DockerLabelHandler("till") + label = DockerLabelHandler("till") data = dict() data["till.definitions"] = "h2" diff --git a/src/tests/test_parser.py b/src/tests/test_parser.py index c5ef3df..6ab72cb 100644 --- a/src/tests/test_parser.py +++ b/src/tests/test_parser.py @@ -1,12 +1,14 @@ -import easymapping -import pytest -import os -import yaml import json +import os + +import yaml + +import easymapping + +CERTS_FOLDER = "/tmp/certs" +CERT_FILE = "/tmp/certs/haproxy/www.somehost.com.br.pem" +CERTBOT_EMAIL = "some@email.com" -CERTS_FOLDER="/tmp/certs" -CERT_FILE="/tmp/certs/haproxy/www.somehost.com.br.pem" -CERTBOT_EMAIL="some@email.com" def load_fixture(file): path = os.path.dirname(os.path.realpath(__file__)) @@ -35,6 +37,7 @@ def test_parser_doesnt_crash(): assert expected_file.read() == haproxy_config assert [] == cfg.certbot_hosts + def test_parser_finds_services(): line_list = load_fixture("services") @@ -56,10 +59,11 @@ def test_parser_finds_services(): with open(path + "/expected/services.txt", 'r') as expected_file: assert expected_file.read() == haproxy_config - assert {"www.somehost.com.br.pem":"Some PEM Certificate"} == cfg.certs + assert {"www.somehost.com.br.pem": "Some PEM Certificate"} == cfg.certs assert ['node-exporter.quantum.example.org'] == cfg.certbot_hosts + def test_parser_finds_services_changed_label(): line_list = load_fixture("services-changed-label") @@ -85,10 +89,11 @@ def test_parser_finds_services_changed_label(): with open(path + "/expected/services.txt", 'r') as expected_file: assert expected_file.read() == haproxy_config - assert {"www.somehost.com.br.pem":"Some PEM Certificate"} == cfg.certs + assert {"www.somehost.com.br.pem": "Some PEM Certificate"} == cfg.certs assert ['node-exporter.quantum.example.org'] == cfg.certbot_hosts + def test_parser_finds_services_raw(): line_list = load_fixture("services") @@ -109,10 +114,10 @@ def test_parser_finds_services_raw(): parsed_object = [ { - "mode":"tcp", - "health-check":"", - "port":"31339", - "hosts":{ + "mode": "tcp", + "health-check": "", + "port": "31339", + "hosts": { "agent.quantum.example.org": { "containers": [ "my-stack_agent:9001" @@ -121,23 +126,23 @@ def test_parser_finds_services_raw(): "redirect_ssl": False } }, - "redirect":{ - + "redirect": { + } }, { - "mode":"http", - "health-check":"", - "port":"31337", - "hosts":{ - "cadvisor.quantum.example.org":{ + "mode": "http", + "health-check": "", + "port": "31337", + "hosts": { + "cadvisor.quantum.example.org": { "containers": [ "my-stack_cadvisor:8080" ], "certbot": False, "redirect_ssl": False }, - "node-exporter.quantum.example.org":{ + "node-exporter.quantum.example.org": { "containers": [ "my-stack_node-exporter:9100" ], @@ -145,15 +150,15 @@ def test_parser_finds_services_raw(): "redirect_ssl": False } }, - "redirect":{ - + "redirect": { + }, }, { - "mode":"http", - "health-check":"", - "port":"443", - "hosts":{ + "mode": "http", + "health-check": "", + "port": "443", + "hosts": { "node-exporter.quantum.example.org": { "containers": [ "my-stack_node-exporter:9100" @@ -161,7 +166,7 @@ def test_parser_finds_services_raw(): "certbot": False, "redirect_ssl": False }, - "www.somehost.com.br":{ + "www.somehost.com.br": { "containers": [ "some-service:80" ], @@ -169,21 +174,21 @@ def test_parser_finds_services_raw(): "redirect_ssl": False } }, - "redirect":{ - "somehost.com.br":"https://www.somehost.com.br", - "somehost.com":"https://www.somehost.com.br", - "www.somehost.com":"https://www.somehost.com.br", - "byjg.ca":"https://www.somehost.com.br", - "www.byjg.ca":"https://www.somehost.com.br" + "redirect": { + "somehost.com.br": "https://www.somehost.com.br", + "somehost.com": "https://www.somehost.com.br", + "www.somehost.com": "https://www.somehost.com.br", + "byjg.ca": "https://www.somehost.com.br", + "www.byjg.ca": "https://www.somehost.com.br" }, "ssl": True }, { - "mode":"http", - "health-check":"", - "port":"80", - "hosts":{ - "www.somehost.com.br":{ + "mode": "http", + "health-check": "", + "port": "80", + "hosts": { + "www.somehost.com.br": { "containers": [ "some-service:80" ], @@ -191,12 +196,12 @@ def test_parser_finds_services_raw(): "redirect_ssl": False } }, - "redirect":{ - "somehost.com.br":"https://www.somehost.com.br", - "somehost.com":"https://www.somehost.com.br", - "www.somehost.com":"https://www.somehost.com.br", - "byjg.ca":"https://www.somehost.com.br", - "www.byjg.ca":"https://www.somehost.com.br" + "redirect": { + "somehost.com.br": "https://www.somehost.com.br", + "somehost.com": "https://www.somehost.com.br", + "www.somehost.com": "https://www.somehost.com.br", + "byjg.ca": "https://www.somehost.com.br", + "www.byjg.ca": "https://www.somehost.com.br" }, } ] @@ -207,7 +212,6 @@ def test_parser_finds_services_raw(): assert ['node-exporter.quantum.example.org'] == cfg.certbot_hosts - def test_parser_static(): path = os.path.dirname(os.path.realpath(__file__)) with open(path + "/fixtures/static.yml", 'r') as content_file: @@ -221,6 +225,7 @@ def test_parser_static(): assert expected_file.read() == haproxy_config assert [] == cfg.certbot_hosts + def test_parser_static_raw(): path = os.path.dirname(os.path.realpath(__file__)) with open(path + "/fixtures/static.yml", 'r') as content_file: @@ -280,7 +285,6 @@ def test_parser_static_raw(): assert expected == parsed - def test_parser_tcp(): line_list = load_fixture("services-tcp") @@ -301,6 +305,7 @@ def test_parser_tcp(): assert expected_file.read() == haproxy_config assert [] == cfg.certbot_hosts + def test_parser_multi_containers(): line_list = load_fixture("services-multi-containers") @@ -384,6 +389,7 @@ def test_parser_ssl_strict(): assert expected_file.read() == haproxy_config assert [] == cfg.certbot_hosts + def test_parser_ssl_loose(): line_list = load_fixture("no-services") @@ -401,6 +407,7 @@ def test_parser_ssl_loose(): assert expected_file.read() == haproxy_config assert [] == cfg.certbot_hosts + def test_parser_ssl_letsencrypt(): line_list = load_fixture("services-letsencrypt") @@ -444,51 +451,51 @@ def test_parser_finds_services_clone_to_ssl_raw(): parsed_object = [ { - "health-check":"", - "hosts":{ - "host2.local":{ - "containers":[ - "10.152.183.215:8080" + "health-check": "", + "hosts": { + "host2.local": { + "containers": [ + "10.152.183.215:8080" ], "certbot": False, "redirect_ssl": False }, - "valida.me":{ - "containers":[ - "10.152.183.62:8080" + "valida.me": { + "containers": [ + "10.152.183.62:8080" ], "certbot": False, "redirect_ssl": False }, - "www.valida.me":{ - "containers":[ - "10.152.183.62:8080" + "www.valida.me": { + "containers": [ + "10.152.183.62:8080" ], "certbot": False, "redirect_ssl": False } }, - "mode":"http", - "port":"80", - "redirect":{ - + "mode": "http", + "port": "80", + "redirect": { + } }, { - "health-check":"ssl", - "hosts":{ - "host2.local":{ - "containers":[ - "10.152.183.215:8080" + "health-check": "ssl", + "hosts": { + "host2.local": { + "containers": [ + "10.152.183.215:8080" ], "certbot": False, "redirect_ssl": False } }, - "mode":"http", - "port":"443", - "redirect":{ - + "mode": "http", + "port": "443", + "redirect": { + }, "ssl": True } @@ -498,10 +505,8 @@ def test_parser_finds_services_clone_to_ssl_raw(): assert parsed_object == processed assert [] == cfg.certbot_hosts - - -#test_parser_finds_services_raw() -#test_parser_tcp() -#test_parser_multiple_hosts() -#test_parser_ssl_certbot() -#test_parser_finds_services() \ No newline at end of file +# test_parser_finds_services_raw() +# test_parser_tcp() +# test_parser_multiple_hosts() +# test_parser_ssl_certbot() +# test_parser_finds_services() diff --git a/src/tests/test_static.py b/src/tests/test_static.py index f8800ac..7e6998c 100644 --- a/src/tests/test_static.py +++ b/src/tests/test_static.py @@ -1,8 +1,8 @@ -import pytest import os + from functions import Functions from processor import ProcessorInterface -from processor import Static + def test_processor_static(): ProcessorInterface.static_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./fixtures/static.yml") @@ -10,44 +10,44 @@ def test_processor_static(): parsed_object = [ { - "hosts":{ - "host1.com.br":{ - "containers":[ - "container:5000" + "hosts": { + "host1.com.br": { + "containers": [ + "container:5000" ], "certbot": True }, - "host2.com.br":{ - "containers":[ - "other:3000" + "host2.com.br": { + "containers": [ + "other:3000" ] } }, - "port":80, - "redirect":{ - "www.host1.com.br":"http://host1.com.br" + "port": 80, + "redirect": { + "www.host1.com.br": "http://host1.com.br" } }, { - "hosts":{ - "host1.com.br":{ - "containers":[ - "container:80" + "hosts": { + "host1.com.br": { + "containers": [ + "container:80" ] } }, - "port":443, + "port": 443, "ssl": True }, { - "hosts":{ - "host3.com.br":{ - "containers":[ - "domain:8181" + "hosts": { + "host3.com.br": { + "containers": [ + "domain:8181" ] } }, - "port":8080 + "port": 8080 } ] hosts = [ @@ -63,11 +63,12 @@ def test_processor_static(): haproxy_cfg = static.get_haproxy_conf() - assert haproxy_cfg == Functions.load(os.path.join(os.path.dirname(os.path.realpath(__file__)), "./expected/static.txt")) + assert haproxy_cfg == Functions.load( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "./expected/static.txt")) # @todo: Static doesnt populate this fields assert static.get_certbot_hosts() == [] assert static.get_parsed_object() == parsed_object assert static.get_hosts() == hosts -# test_processor_static() \ No newline at end of file +# test_processor_static()