Refactor: tried to refactor code

- move scripts to assets/scripts (and updated build)
 - put most logic into HaproxyConfigGenerator
 - created DockerLabelHandler for all label handling from previous code
 - added unit tests and fixtures to cover HaproxyConfigGenerator and DockerLabelHandler
 - added a Makefile with build (for docker build) and test targets

Dockerfile

@@ -8,8 +8,6 @@ COPY requirements.txt /scripts
 RUN pip3 install --upgrade pip \
  && pip install -r requirements.txt
 
-COPY swarm.* /scripts/
-COPY static.* /scripts/
 COPY templates /scripts/templates/
 COPY easymapping /scripts/easymapping/
 

Makefile

@@ -0,0 +1,7 @@
+.PHONY: build
+build:
+	docker build -t byjg/easy-haproxy -t byjg/easy-haproxy:local .
+
+.PHONY: test
+test:
+	pytest tests/

assets/scripts/swarm.py

@@ -0,0 +1,24 @@
+import os
+from easymapping import HaproxyConfigGenerator
+
+# path = os.path.dirname(os.path.realpath(__file__))
+with open("/tmp/.docker_data", 'r') as content_file:
+    lineList = content_file.readlines()
+
+result = {
+    "customerrors": True if os.getenv("HAPROXY_CUSTOMERRORS") == "true" else False
+}
+
+if os.getenv("HAPROXY_PASSWORD"):
+    result["stats"] = {
+        "username": os.getenv("HAPROXY_USERNAME") if os.getenv("HAPROXY_USERNAME") else "admin",
+        "password": os.getenv("HAPROXY_PASSWORD"),
+        "port": os.getenv("HAPROXY_STATS_PORT") if os.getenv("HAPROXY_STATS_PORT") else "1936",
+    }
+
+cfg = HaproxyConfigGenerator(result)
+print(cfg.generate(lineList))
+
+# print(jsonStr)
+
+

easymapping/__init__.py

@@ -1,12 +1,138 @@
+import base64
+import hashlib
 from jinja2 import Environment, FileSystemLoader
+import json
+import time
+
+
+class DockerLabelHandler:
+    def __init__(self, label):
+        self.__label_base = label
+
+
+    def create(self, key):
+        if isinstance(key, str):
+            return "{}.{}".format(self.__label_base, key)
+
+        return "{}.{}".format(self.__label_base, ".".join(key))
+
+
+    def get(self, label, default_value = ""):
+        if self.has_label(label):
+            return self.__data[label]
+        return default_value
+
+
+    def set_data(self, data):
+        self.__data = data
+
+
+    def has_label(self, label):
+        if label in self.__data:
+            return True
+        return False
 
 
 class HaproxyConfigGenerator:
     def __init__(self, mapping):
         self.mapping = mapping
+        self.label = DockerLabelHandler("com.byjg.easyhaproxy")
+
+
+    def generate(self, lineList = []):
+        # static?
+        if len(lineList) > 0:
+            self.mapping["easymapping"] = self.__parse(lineList)
+
+        # still 'None' -> default to [] for jinja2
+        if self.mapping["easymapping"] is None:
+            self.mapping["easymapping"] = []
 
-    def generate(self):
         file_loader = FileSystemLoader('templates')
         env = Environment(loader=file_loader)
+        env.trim_blocks = True
+        env.lstrip_blocks = True
+        env.rstrip_blocks = True
         template = env.get_template('haproxy.cfg.j2')
         return template.render(data=self.mapping)
+
+
+    def __parse(self, lineList):
+        easymapping = dict()
+
+        for line in lineList:
+            line = line.strip()
+            i = line.find("=")
+            container = line[:i]
+            jsonStr = line[i+1:]
+            d = json.loads(jsonStr)
+
+            if self.label.create("definitions") not in d.keys():
+                continue
+
+            self.label.set_data(d)
+
+            definitions = d[self.label.create("definitions")].split(",")
+            for definition in definitions:
+                mode = self.label.get(
+                    self.label.create(["mode", definition]),
+                    "http"
+                )
+
+                # TODO: we can ignore "host" in TCP, but it would break the template
+                host_label = self.label.create(["host", definition])
+                if not self.label.has_label(host_label):
+                    continue
+
+                port = self.label.get(
+                    self.label.create(["port", definition]),
+                    "80"
+                )
+
+                if self.label.create(["sslcert", definition]) in d:
+                    hash = hashlib.md5(
+                        d[self.label.create(["sslcert", definition])].encode('utf-8')
+                    ).hexdigest()
+                else:
+                    hash = ""
+
+                key = port+hash
+
+                if key not in easymapping:
+                    easymapping[key] = {
+                        "mode": mode,
+                        "port": port,
+                        "hosts": dict(),
+                        "redirect": dict(),
+                    }
+
+                # TODO: this could use `EXPOSE` from `Dockerfile`?
+                ct_port = self.label.get(
+                    self.label.create(["localport", definition]),
+                    "80"
+                )
+
+                easymapping[key]["hosts"][d[host_label]] = "{}:{}".format(container, ct_port)
+
+                # handle SSL
+                ssl_label = self.label.create(["sslcert", definition])
+                if self.label.has_label(ssl_label):
+                    filename = "/etc/haproxy/certs/{}.{}.pem".format(
+                        d[ssl_label], str(time.time())
+                    )
+                    easymapping[key]["ssl_cert"] = filename
+                    with open(filename, 'wb') as file:
+                        file.write(
+                            base64.b64decode(d[ssl_label])
+                        )
+
+                # handle redirects
+                redirect = self.label.get(
+                    self.label.create(["redirect", definition])
+                )
+                if len(redirect) > 0:
+                    for r in redirect.split(","):
+                        r_parts = r.split("--")
+                        easymapping[key]["redirect"][r_parts[0]] = r_parts[1]
+
+        return easymapping.values()

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -v -p no:warnings

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup, find_packages
+
+
+with open('README.md') as f:
+    readme = f.read()
+
+with open('LICENSE') as f:
+    license = f.read()
+
+setup(
+    name='easymapping',
+    version='0.1.0',
+    description='HAProxy label based routing',
+    long_description=readme,
+    author='',
+    author_email='',
+    url='',
+    license=license,
+    packages=find_packages(exclude=('tests', 'docs'))
+)

swarm.py

@@ -1,76 +0,0 @@
-import os
-import json
-import time
-import base64
-import hashlib
-from easymapping import HaproxyConfigGenerator
-
-# path = os.path.dirname(os.path.realpath(__file__))
-with open("/tmp/.docker_data", 'r') as content_file:
-    lineList = content_file.readlines()
-
-result = {
-    "easymapping": [],
-    "customerrors": True if os.getenv("HAPROXY_CUSTOMERRORS") == "true" else False
-}
-easymapping = dict()
-
-if os.getenv("HAPROXY_PASSWORD"):
-    result["stats"] = {
-        "username": os.getenv("HAPROXY_USERNAME") if os.getenv("HAPROXY_USERNAME") else "admin",
-        "password": os.getenv("HAPROXY_PASSWORD"),
-        "port": os.getenv("HAPROXY_STATS_PORT") if os.getenv("HAPROXY_STATS_PORT") else "1936",
-    }
-
-for line in lineList:
-    line = line.strip()
-    i = line.find("=")
-    container = line[:i]
-    jsonStr = line[i+1:]
-    d = json.loads(jsonStr)
-
-    if "com.byjg.easyhaproxy.definitions" in d.keys():
-        definitions = d["com.byjg.easyhaproxy.definitions"].split(",")
-
-        for definition in definitions:
-            if "com.byjg.easyhaproxy.host." + definition not in d:
-                continue
-
-            mode = d["com.byjg.easyhaproxy.mode." + definition] if "com.byjg.easyhaproxy.mode." + definition in d else "http"
-            port = d["com.byjg.easyhaproxy.port." + definition] if "com.byjg.easyhaproxy.port." + definition in d else "80"
-            hash = hashlib.md5(d["com.byjg.easyhaproxy.sslcert." + definition].encode('utf-8')).hexdigest() if "com.byjg.easyhaproxy.sslcert." + definition in d else ""
-
-            key = port+hash
-
-            if key not in easymapping:
-                easymapping[key] = {
-                    "mode": mode,
-                    "port": port,
-                    "hosts": dict(),
-                    "redirect": dict(),
-                    # "ssl_cert": ""
-                }
-
-            easymapping[key]["hosts"][d["com.byjg.easyhaproxy.host." + definition]] = container + ":" + (d["com.byjg.easyhaproxy.localport." + definition] if "com.byjg.easyhaproxy.localport." + definition in d else "80")
-
-            if "com.byjg.easyhaproxy.sslcert." + definition in d:
-                filename = '/etc/haproxy/certs/' + d["com.byjg.easyhaproxy.host." + definition] + "." + str(time.time()) + ".pem"
-                easymapping[key]["ssl_cert"] = filename
-                with open(filename, 'wb') as file:
-                    file.write(base64.b64decode(d["com.byjg.easyhaproxy.sslcert." + definition]))
-
-            if "com.byjg.easyhaproxy.redirect." + definition in d:
-                redirect = d["com.byjg.easyhaproxy.redirect." + definition] if "com.byjg.easyhaproxy.redirect." + definition in d else ""
-                for r in redirect.split(","):
-                    r_parts = r.split("--")
-                    easymapping[key]["redirect"][r_parts[0]] = r_parts[1]
-
-        result["easymapping"] = easymapping.values()
-
-
-cfg = HaproxyConfigGenerator(result)
-print(cfg.generate())
-
-# print(jsonStr)
-
-

templates/bind.j2

@@ -0,0 +1,8 @@
+    {% if "ssl_cert" in o %}
+    bind *:{{ o["port"] }}  ssl crt {{ o["ssl_cert"] }}
+    {% elif "h2" in o and o["h2"] %}
+    bind *:{{ o["port"] }} proto h2
+    option http-use-htx
+    {% else %}
+    bind *:{{ o["port"] }}
+    {% endif %}

templates/frontend-mode-http.j2

@@ -0,0 +1,11 @@
+    mode http
+    {% for k in o["redirect"] %}
+    redirect prefix {{ o["redirect"][k] }} code 301 if { hdr(host) -i {{ k }} }
+    {% endfor %}
+    {% for k in o["hosts"] %}
+        {% set host = k.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) %}
+
+    acl is_rule_{{ host }}_1 hdr(host) -i {{ k }}
+    acl is_rule_{{ host }}_2 hdr(host) -i {{ k }}:{{ o["port"] }}
+    use_backend srv_{{ host }} if is_rule_{{ host }}_1 OR is_rule_{{ host }}_2
+    {% endfor %}

templates/frontend-mode-tcp.j2

@@ -0,0 +1,6 @@
+    mode tcp
+    option tcplog
+    log global
+{% set backend = (o["hosts"]|first) %}
+    default_backend srv_{{ backend.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) }}
+

templates/haproxy.cfg.j2

@@ -1,3 +1,8 @@
+global
+    log stdout  format raw  local0  info
+    maxconn 2000
+    tune.ssl.default-dh-param 2048
+
 defaults
     log global
 
@@ -14,11 +19,6 @@ defaults
     errorfile 504 /etc/haproxy/errors-custom/504.http
 {% endif %}
 
-global
-    log stdout  format raw  local0  info
-    maxconn 2000
-    tune.ssl.default-dh-param 2048
-
 {% if "stats" in data %}
 frontend stats
     bind *:{{ data["stats"]["port"] | default(1936) }}
@@ -37,43 +37,31 @@ backend srv_stats
     mode http
     server Local 127.0.0.1:{{ data["stats"]["port"] | default(1936) }}
 {% endif %}
-
+{% for o in data["easymapping"] -%}
-{% for o in data["easymapping"] %}
     {% set mode = o["mode"] or "http" %}
     {% set salt = loop.index %}
+
 frontend {{ mode }}_in_{{ o["port"] }}_{{ salt }}
-    bind *:{{ o["port"] }} {{ " ssl crt " + o["ssl_cert"] if "ssl_cert" in o else "" }}
+    {% include "bind.j2" %}
-    mode {{ mode }}
+    {% if mode == "http" %}
+        {% include "frontend-mode-http.j2" %}
+    {% else %}
+        {% include "frontend-mode-tcp.j2" %}
+    {% endif %}
 
-    {% if mode == "tcp" -%}
+    {% for k in o["hosts"] -%}
-    option tcplog
-    tcp-request inspect-delay 5s
-    tcp-request content accept if { req.ssl_hello_type 1 }
-    {% endif -%}
-
-    {% for k in o["redirect"] -%}
-    redirect prefix {{ o["redirect"][k] }} code 301 if { hdr(host) -i {{ k }} }
-    {% endfor -%}
-
-    {% for k in o["hosts"] %}
-        {% set host = k.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) %}
-    acl is_rule_{{ host }}_1 hdr(host) -i {{ k }}
-    acl is_rule_{{ host }}_2 hdr(host) -i {{ k }}:{{ o["port"] }}
-    use_backend srv_{{ host }} if is_rule_{{ host }}_1 OR is_rule_{{ host }}_2
-    {% endfor %}
-
-    {% for k in o["hosts"] %}
         {% set host = k.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) %}
 backend srv_{{ host }}
     balance roundrobin
     mode {{ mode }}
-
+        {% if mode == "http" %}
-{% if mode == "http" %}
     option forwardfor
     http-request set-header X-Forwarded-Port %[dst_port]
     http-request add-header X-Forwarded-Proto https if { ssl_fc }
-{% endif %}
+        {% elif mode == "tcp" %}
-
+    option tcp-check
+    tcp-check connect
+        {% endif %}
     server srv {{ o["hosts"][k] }} check weight 1
     {% endfor %}
 {% endfor %}

tests/context.py

@@ -0,0 +1,5 @@
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import easymapping

tests/fixtures/no-services

@@ -0,0 +1,5 @@
+swarm-prom_caddy={"com.docker.stack.image":"stefanprodan/caddy","com.docker.stack.namespace":"swarm-prom"}
+swarm-prom_cadvisor={"com.docker.stack.image":"google/cadvisor","com.docker.stack.namespace":"swarm-prom"}
+swarm-prom_dockerd-exporter={"com.docker.stack.image":"stefanprodan/caddy","com.docker.stack.namespace":"swarm-prom"}
+swarm-prom_unsee={"com.docker.stack.image":"cloudflare/unsee:v0.8.0","com.docker.stack.namespace":"swarm-prom"}
+test_proxy={"com.docker.stack.image":"byjg/easy-haproxy","com.docker.stack.namespace":"test"}

tests/fixtures/services

@@ -0,0 +1,5 @@
+portainer-agent_agent={"com.docker.stack.image":"portainer/agent:1.5.1","com.docker.stack.namespace":"portainer-agent"}
+my-stack_agent={"com.byjg.easyhaproxy.definitions":"agent","com.byjg.easyhaproxy.host.agent":"agent.quantum.example.org","com.byjg.easyhaproxy.localport.agent":"9001","com.byjg.easyhaproxy.mode.agent":"tcp","com.byjg.easyhaproxy.port.agent":"31339","com.docker.stack.image":"portainer/agent:1.5.1","com.docker.stack.namespace":"my-stack","com.planetary-quantum":"monitoring"}
+my-stack_cadvisor={"com.byjg.easyhaproxy.definitions":"cadvisor","com.byjg.easyhaproxy.host.cadvisor":"cadvisor.quantum.example.org","com.byjg.easyhaproxy.localport.cadvisor":"8080","com.byjg.easyhaproxy.port.cadvisor":"31337","com.docker.stack.image":"gcr.io/google-containers/cadvisor:v0.34.0","com.docker.stack.namespace":"my-stack","com.planetary-quantum":"monitoring"}
+my-stack_node-exporter={"com.byjg.easyhaproxy.definitions":"exp","com.byjg.easyhaproxy.host.exp":"node-exporter.quantum.example.org","com.byjg.easyhaproxy.localport.exp":"9100","com.byjg.easyhaproxy.port.exp":"31337","com.docker.stack.image":"stefanprodan/swarmprom-node-exporter:v0.16.0","com.docker.stack.namespace":"my-stack","com.planetary-quantum":"monitoring"}
+my-stack_reverse-proxy={"com.docker.stack.image":"quay.io/pngmbh/easy-haproxy:tcp-mode","com.docker.stack.namespace":"my-stack","com.planetary-quantum":"monitoring"}

tests/fixtures/services-tcp

@@ -0,0 +1,2 @@
+test_agent={"com.byjg.easyhaproxy.definitions":"agent","com.byjg.easyhaproxy.host.agent":"agent.quantum.local","com.byjg.easyhaproxy.localport.agent":"9001","com.byjg.easyhaproxy.mode.agent":"tcp","com.byjg.easyhaproxy.port.agent":"31339","com.docker.stack.image":"portainer/agent:1.5.1","com.docker.stack.namespace":"test","com.planetary-quantum":"monitoring"}
+test_proxy={"com.docker.stack.image":"byjg/easy-haproxy:local","com.docker.stack.namespace":"test"}

tests/fixtures/static.yml

@@ -0,0 +1,23 @@
+stats:
+  username: admin
+  password: test123
+  port: 1936         # Optional (default 1936)
+
+customerrors: true   # Optional (default false)
+
+easymapping:
+  - port: 80
+    hosts:                                     
+      host1.com.br: container:5000
+      host2.com.br: other:3000
+    redirect:
+      www.host1.com.br: http://host1.com.br
+      
+  - port: 443
+    ssl_cert: BASE64_PEM_CERTIFICATE
+    hosts:
+      host1.com.br: container:80
+
+  - port: 8080
+    hosts:
+      host3.com.br: domain:8181

tests/test_labels.py

@@ -0,0 +1,32 @@
+from .context import easymapping
+import json
+import pytest
+
+def test_label_generation():
+    label = easymapping.DockerLabelHandler("foo")
+
+    assert label.create("bar") == "foo.bar"
+    assert label.create(["bar", "foobar"]) == "foo.bar.foobar"
+
+
+def test_label_data():
+    label = easymapping.DockerLabelHandler("base")
+    label.set_data(json.loads('{"base.definitions":"h2"}'))
+
+    label_name = label.create("definitions")
+    assert label_name == "base.definitions"
+    assert label.has_label(label_name)
+    assert label.get(label_name) == "h2"
+
+
+def test_label_complex_key():
+    label = easymapping.DockerLabelHandler("till")
+    
+    data = dict()
+    data["till.definitions"] = "h2"
+    data["till.host.h2"] = "fqdn.example.org"
+    data["till.mode.h2"] = "tcp"
+    label.set_data(json.loads(json.dumps(data)))
+
+    assert label.get(label.create(["host", "h2"])) == "fqdn.example.org"
+    assert label.get(label.create(["mode", "h2"])) == "tcp"

tests/test_parser.py

@@ -0,0 +1,70 @@
+from .context import easymapping
+import pytest
+import os
+import yaml
+
+
+def load_fixture(file):
+    path = os.path.dirname(os.path.realpath(__file__))
+    with open(path + "/fixtures/" + file, 'r') as content_file:
+        lineList = content_file.readlines()
+
+    return lineList
+
+
+def test_parser_doesnt_crash():
+    lineList = load_fixture("no-services")
+
+    result = {
+        "customerrors": False
+    }
+
+    cfg = easymapping.HaproxyConfigGenerator(result)
+    haproxy_config = cfg.generate(lineList)
+    assert len(haproxy_config) > 0
+    assert "frontend" not in haproxy_config
+    assert "backend" not in haproxy_config
+
+
+def test_parser_finds_services():
+    lineList = load_fixture("services")
+
+    result = {
+        "customerrors": False
+    }
+
+    cfg = easymapping.HaproxyConfigGenerator(result)
+    haproxy_config = cfg.generate(lineList)
+    assert len(haproxy_config) > 0
+    assert "mode tcp" in haproxy_config
+    assert "mode http" in haproxy_config
+
+    assert "frontend tcp_in_31339_1" in haproxy_config
+    assert "frontend http_in_31337_2" in haproxy_config
+
+
+def test_parser_static():
+    path = os.path.dirname(os.path.realpath(__file__))
+    with open(path + "/fixtures/static.yml", 'r') as content_file:
+        parsed = yaml.load(content_file.read(), Loader=yaml.FullLoader)
+
+    cfg = easymapping.HaproxyConfigGenerator(parsed)
+    haproxy_config = cfg.generate()
+    assert len(haproxy_config) > 0
+
+    # assert on auth on stats
+    assert "stats auth admin:test123" in haproxy_config
+
+    # assert that we found redirect
+    assert "redirect prefix http://host1.com.br code 301 if { hdr(host) -i www.host1.com.br }" in haproxy_config
+
+    # assert that we found the services
+    assert "frontend http_in_80_1" in haproxy_config
+    assert "bind *:80"
+    assert "frontend http_in_443_2" in haproxy_config
+    assert "bind *:443"
+    assert "frontend http_in_8080_3" in haproxy_config
+    assert "bind :*8080"
+
+    # verify ssl config
+    assert "frontend http_in_443_2\n    bind *:443  ssl crt BASE64_PEM_CERTIFICATE" in haproxy_config