Merge pull request #35 from byjg/issues

Issues

.vscode/launch.json

@@ -4,6 +4,22 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "PyTest Current File",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "justMyCode": true,
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src",
+            "args": [
+                "-vv",
+                "${file}"
+            ],
+            "env": {
+                "PYTHONPATH": "${cwd}/src"
+            }
+        },
         {
             "name": "Python: Current File",
             "type": "python",

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cSpell.words": [
+        "certonly",
+        "letsencrypt"
+    ]
+}

docs/docker-environment.md

@@ -1,10 +1,11 @@
 # Docker environment variables
 
 | Environment Variable            | Description                                                                                     | Default          |
-|-------------------------------|-------------------------------------------------------------------------------------------------|------------------|
+|---------------------------------|-------------------------------------------------------------------------------------------------|------------------|
 | EASYHAPROXY_DISCOVER            | How the services will be discovered to create `haproxy.cfg`:  `static`, `docker`, `swarm` or `kubernetes`     | **required**     |
 | EASYHAPROXY_LABEL_PREFIX        | (Optional) The key will search for matching resources.                                          | `easyhaproxy`    |
 | EASYHAPROXY_LETSENCRYPT_EMAIL   | (Optional) The email will be used to request the certificate to Letsencrypt                     | *empty*          |
+| EASYHAPROXY_LETSENCRYPT_SERVER  | (Optional) Can be `staging` or 'schema://domain.tld'. If set, will try to connect to the Letsencrypt test server  | *empty*            |
 | EASYHAPROXY_SSL_MODE            | (Optional) `strict` supports only the most recent TLS version; `default` good SSL integration with recent browsers; `loose` supports all old SSL protocols for old browsers (not recommended).  | `default`|
 | EASYHAPROXY_REFRESH_CONF        | (Optional) Check configuration every N seconds.                                                 | 10               |
 | EASYHAPROXY_LOG_LEVEL           | (Optional) The log level for EasyHAproxy messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL | DEBUG            |

docs/docker.md

@@ -5,6 +5,7 @@
 This method will use a docker standalone installation to discover the containers and configure the HAProxy.
 
 The only requirement is that containers and EasyHAProxy must be in the same docker network.
+If not, EasyHAProxy will connect the container with the EasyHAProxy network.
 
 e.g.:
 

docs/swarm.md

@@ -6,6 +6,7 @@ This method will use a docker swarm installation to discover the containers and
 The advantage of this method is that you can discover containers in other nodes from the cluster.
 
 The only requirement is that containers and EasyHAProxy must be in the same docker swarm network.
+If not, EasyHAProxy will connect the service with the EasyHAProxy service network.
 
 e.g.:
 

examples/docker/docker-compose.yml

@@ -13,7 +13,7 @@
 # location: https://host1.local/
 #
 # Test SSL:
-# openssl s_client -showcerts -connect 127.0.0.1:443 --servername host1.local
+# openssl s_client -showcerts -connect 127.0.0.1:443 -servername host1.local
 
 version: "3"
 

examples/swarm/docker-compose.yml

@@ -13,7 +13,7 @@
 # location: https://host1.local/
 #
 # Test SSL:
-# openssl s_client -showcerts -connect 127.0.0.1:443 --servername host1.local
+# openssl s_client -showcerts -connect 127.0.0.1:443 -servername host1.local
 
 version: "3"
 

src/easymapping/__init__.py

@@ -49,7 +49,7 @@ class HaproxyConfigGenerator:
     def __init__(self, mapping):
         self.mapping = mapping
         self.mapping.setdefault("ssl_mode", 'default')
-        self.mapping.setdefault("letsencrypt", {"email": ""})
+        self.mapping.setdefault("letsencrypt", {"email": "", "staging": False})
         self.mapping["ssl_mode"] = self.mapping["ssl_mode"].lower()
         self.label = DockerLabelHandler(mapping['lookup_label'] if 'lookup_label' in mapping else "easyhaproxy")
         self.letsencrypt_hosts = []

src/functions/__init__.py

@@ -174,9 +174,18 @@ class DaemonizeHAProxy:
 
 
 class Certbot:
-    def __init__(self, certs, email):
+    def __init__(self, certs, email, test_server):
         self.certs = certs
         self.email = email
+        self.test_server = self.set_test_server(test_server)
+
+    def set_test_server(self, test_server):
+        if test_server.lower() == "staging":
+            return "--staging"
+        elif test_server.lower().startswith("http"):
+            return "--server " + test_server
+        else:
+            return ""
 
     def check_certificates(self, hosts):
         if self.email == "" or len(hosts) == 0:
@@ -201,7 +210,7 @@ class Certbot:
                         Functions.log(Functions.CERTBOT_LOG, Functions.DEBUG, "Renew certificate for %s" % (host))
                         renew_certs.append(host_arg)
 
-            certbot_certonly = ('/usr/bin/certbot certonly '
+            certbot_certonly = ('/usr/bin/certbot certonly {test_server}'
                                 '    --standalone'
                                 '    --preferred-challenges http'
                                 '    --http-01-port 2080'
@@ -210,7 +219,9 @@ class Certbot:
                                 '    --no-eff-email'
                                 '    --non-interactive'
                                 '    --max-log-backups=0'
-                                '    %s --email %s' % (' '.join(request_certs), self.email)
+                                '    {certs} --email {email}'.format(certs = ' '.join(request_certs),
+                                                           email = self.email,
+                                                           test_server = self.test_server)
                             )
 
             ret_reload = False
@@ -235,6 +246,8 @@ class Certbot:
     
     def find_live_certificates(self):
         letsencrypt_certs = "/etc/letsencrypt/live/"
+        if not os.path.exists(letsencrypt_certs):
+            return
         for item in os.listdir(letsencrypt_certs):
             path = os.path.join(letsencrypt_certs, item)
             if os.path.isdir(path):

src/main.py

@@ -22,7 +22,7 @@ def start():
     haproxy.haproxy("start")
     haproxy.sleep()
 
-    certbot = Certbot(Consts.certs_letsencrypt, os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"))
+    certbot = Certbot(Consts.certs_letsencrypt, os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"), os.getenv("EASYHAPROXY_LETSENCRYPT_SERVER", "").lower())
 
     while True:
         if old_haproxy is not None:

src/processor/__init__.py

@@ -6,6 +6,7 @@ import os
 import json
 import base64
 import docker
+import socket
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 
@@ -27,7 +28,8 @@ class ContainerEnv:
         env_vars["lookup_label"] = os.getenv("EASYHAPROXY_LABEL_PREFIX") if os.getenv("EASYHAPROXY_LABEL_PREFIX") else "easyhaproxy"
         if (os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL")):
             env_vars["letsencrypt"] = {
-                "email": os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL")
+                "email": os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"),
+                "server": os.getenv("EASYHAPROXY_LETSENCRYPT_SERVER", "false").lower() in ["true", "1", "yes"]
             }
         
         return env_vars
@@ -126,9 +128,25 @@ class Docker(ProcessorInterface):
         super().__init__()
     
     def inspect_network(self):
+        try:
+            ha_proxy_network_name = next(iter(self.client.containers.get(socket.gethostname()).attrs["NetworkSettings"]["Networks"]))
+        except:
+            # HAProxy is not running in a container, get first container network
+            if len(self.client.containers.list()) == 0:
+                return
+            ha_proxy_network_name = next(iter(self.client.containers.get(self.client.containers.list()[0].name).attrs["NetworkSettings"]["Networks"]))
+
+        ha_proxy_network = self.client.networks.get(ha_proxy_network_name)
+
         self.parsed_object = {}
         for container in self.client.containers.list():
-            self.parsed_object[container.name] = container.labels
+            # Issue 32 - Docker container cannot connect to containers in different network.
+            if ha_proxy_network_name not in container.attrs["NetworkSettings"]["Networks"].keys():
+                ha_proxy_network.connect(container.name)
+                container = self.client.containers.get(container.name)  # refresh object
+
+            ip_address = container.attrs["NetworkSettings"]["Networks"][ha_proxy_network_name]["IPAddress"]
+            self.parsed_object[ip_address] = container.labels
 
 
 class Swarm(ProcessorInterface):
@@ -137,9 +155,28 @@ class Swarm(ProcessorInterface):
         super().__init__()
 
     def inspect_network(self):
+        ha_proxy_service_name = self.client.containers.get(socket.gethostname()).name.split('.')[0]
+        for endpoint in self.client.services.get(ha_proxy_service_name).attrs['Endpoint']["VirtualIPs"]:                
+            ha_proxy_network_id = endpoint["NetworkID"]
+            if self.client.networks.get(ha_proxy_network_id).name != 'ingress':
+                break
+
         self.parsed_object = {}
-        for container in self.client.services.list():
+        for service in self.client.services.list():
-            self.parsed_object[container.attrs["Spec"]["Name"]] = container.attrs["Spec"]["Labels"]
+            ip_address = None
+            network_list = []
+            for endpoint in service.attrs["Endpoint"]["VirtualIPs"]:
+                if ha_proxy_network_id == endpoint["NetworkID"]:
+                    ip_address = endpoint["Addr"].split("/")[0]
+                    break
+                network_list.append(endpoint["NetworkID"])
+            
+            if ip_address is None:
+                network_list.append(ha_proxy_network_id)
+                service.update(networks = network_list)
+                continue # skip to the next service to give time to update the network
+                
+            self.parsed_object[ip_address] = service.attrs["Spec"]["Labels"]
 
 
 class Kubernetes(ProcessorInterface):
@@ -162,6 +199,8 @@ class Kubernetes(ProcessorInterface):
 
         self.parsed_object = {}
         for ingress in ret.items:
+            if 'kubernetes.io/ingress.class' not in ingress.metadata.annotations:
+                continue
             if ingress.metadata.annotations['kubernetes.io/ingress.class'] != "easyhaproxy-ingress":
                 continue
 

src/tests/test_containerenv.py

@@ -95,8 +95,24 @@ def test_container_env_stats_password():
             "lookup_label": "easyhaproxy",
             "letsencrypt": {
                 "email": "acme@example.org",
+                "server": False
             }
         } == ContainerEnv.read()
     finally:
         os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = ''
 
+def test_container_env_letsencrypt():
+    os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = 'acme@example.org'
+    os.environ['EASYHAPROXY_LETSENCRYPT_SERVER'] = 'true'
+    try:
+        assert {
+            "customerrors": False,
+            "ssl_mode": "default",
+            "lookup_label": "easyhaproxy",
+            "letsencrypt": {
+                "email": "acme@example.org",
+                "server": True
+            }
+        } == ContainerEnv.read()
+    finally:
+        os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = ''

src/tests/test_docker.py

@@ -6,15 +6,24 @@ from functions import Functions
 from processor import ProcessorInterface
 from processor import Docker
 
-def _get_hydrated_object(parsed_objects, key):
+
-    assert key in parsed_objects.keys()
+def _get_hydrated_object(parsed_objects, lookup_key):
     hydrated_object = {}
+    for key in parsed_objects:
         for keys in parsed_objects[key]:
-        if "easyhaproxy" in keys:
+            if lookup_key in keys:
                 hydrated_object[keys] = parsed_objects[key][keys]
     return hydrated_object
 
 
+def _get_ip_host(parsed_objects, lookup_key):
+    hydrated_object = {}
+    for key in parsed_objects:
+        for keys in parsed_objects[key]:
+            if lookup_key in keys:
+                return key
+
+
 def test_processor_docker():
     try:
         client = docker.from_env()
@@ -66,27 +75,34 @@ def test_processor_docker():
             'easyhaproxy.http2.localport': '9000',
             'easyhaproxy.http2.port': '90',
             'easyhaproxy.http2.letsencrypt': 'true',
-        } == _get_hydrated_object(static.get_parsed_object(), "test_processor_docker")
+        } == _get_hydrated_object(static.get_parsed_object(), "easyhaproxy.http")
         assert {
             'easyhaproxy.ssl.host': 'hostssl.local',
             'easyhaproxy.ssl.localport': '8080',
             'easyhaproxy.ssl.port': '443',
             'easyhaproxy.ssl.sslcert': 'U29tZSBQRU0gQ2VydGlmaWNhdGU='
-        } == _get_hydrated_object(static.get_parsed_object(), "test2_processor_docker")
+        } == _get_hydrated_object(static.get_parsed_object(), "easyhaproxy.ssl.")
 
         assert static.get_hosts() is None
         assert static.get_certs() == {}
 
         haproxy_cfg = static.get_haproxy_conf()
-        assert haproxy_cfg == Functions.load(os.path.join(os.path.dirname(os.path.realpath(__file__)), "./expected/docker.txt"))
+        assert haproxy_cfg == Functions.load(os.path.join(os.path.dirname(os.path.realpath(__file__)), "./expected/docker.txt")).replace("test_processor_docker", _get_ip_host(
+            static.get_parsed_object(), "easyhaproxy.http")).replace("test2_processor_docker", _get_ip_host(static.get_parsed_object(), "easyhaproxy.ssl"))
 
         assert static.get_letsencrypt_hosts() == ['host2.local']
-        assert static.get_hosts() == ['hostssl.local:443', 'host1.local:80', 'host2.local:90']
+        assert static.get_hosts() == [
-        assert static.get_certs() == {'hostssl.local.pem': 'Some PEM Certificate'}
+            'hostssl.local:443',
+            'host1.local:80',
+            'host2.local:90'
+        ]
+        assert static.get_certs() == {
+            'hostssl.local.pem': 'Some PEM Certificate'
+        }
     finally:
         os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = ''
         container.stop()
         container2.stop()
 
 
-#test_processor_docker()
+# test_processor_docker()