From b1969053209bc8bba85709243e5fde5386bffacb Mon Sep 17 00:00:00 2001 From: Joao Gilberto Magalhaes Date: Wed, 17 Jul 2019 00:14:58 -0500 Subject: [PATCH] Directory restructuration --- Dockerfile | 18 ++- assets/{crontab => etc/crontabs/root} | 2 +- .../etc/haproxy/conf.d}/00-simple-comment.cfg | 0 .../etc/haproxy/conf.d}/README.md | 0 .../{ => etc/haproxy}/errors-custom/400.http | 0 .../{ => etc/haproxy}/errors-custom/403.http | 0 .../{ => etc/haproxy}/errors-custom/408.http | 0 .../{ => etc/haproxy}/errors-custom/500.http | 0 .../{ => etc/haproxy}/errors-custom/502.http | 0 .../{ => etc/haproxy}/errors-custom/503.http | 0 .../{ => etc/haproxy}/errors-custom/504.http | 0 assets/{ => etc}/supervisord.conf | 4 +- .../scripts}/exit-event-listener.py | 0 {scripts => assets/scripts}/haproxy-reload.sh | 0 {scripts => assets/scripts}/haproxy.sh | 0 easymapping/__init__.py | 12 ++ entrypoint.py | 109 ------------------ entrypoint.sh | 30 ----- requirements.txt | 3 +- static.py | 16 +++ discover.py => swarm.py | 25 ++-- discover.sh => swarm.sh | 6 +- templates/haproxy.cfg.j2 | 68 +++++++++++ 23 files changed, 127 insertions(+), 166 deletions(-) rename assets/{crontab => etc/crontabs/root} (86%) rename {conf.d => assets/etc/haproxy/conf.d}/00-simple-comment.cfg (100%) rename {conf.d => assets/etc/haproxy/conf.d}/README.md (100%) rename assets/{ => etc/haproxy}/errors-custom/400.http (100%) rename assets/{ => etc/haproxy}/errors-custom/403.http (100%) rename assets/{ => etc/haproxy}/errors-custom/408.http (100%) rename assets/{ => etc/haproxy}/errors-custom/500.http (100%) rename assets/{ => etc/haproxy}/errors-custom/502.http (100%) rename assets/{ => etc/haproxy}/errors-custom/503.http (100%) rename assets/{ => etc/haproxy}/errors-custom/504.http (100%) rename assets/{ => etc}/supervisord.conf (95%) rename {scripts => assets/scripts}/exit-event-listener.py (100%) rename {scripts => assets/scripts}/haproxy-reload.sh (100%) rename {scripts => assets/scripts}/haproxy.sh (100%) create mode 100644 easymapping/__init__.py delete mode 100644 entrypoint.py delete mode 100755 entrypoint.sh create mode 100644 static.py rename discover.py => swarm.py (55%) rename discover.sh => swarm.sh (94%) create mode 100644 templates/haproxy.cfg.j2 diff --git a/Dockerfile b/Dockerfile index facbae7..97c7a41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,19 @@ FROM haproxy:1.9-alpine -WORKDIR /root +WORKDIR /scripts RUN apk add --no-cache bash python3 py-yaml supervisor docker -COPY requirements.txt /root +COPY requirements.txt /scripts RUN pip3 install --upgrade pip \ && pip install -r requirements.txt +COPY swarm.* /scripts/ +COPY static.* /scripts/ +COPY templates /scripts/templates/ +COPY easymapping /scripts/easymapping/ -COPY haproxy.cfg /etc/haproxy/haproxy.cfg -COPY conf.d /etc/haproxy/conf.d -COPY assets/errors-custom/* /etc/haproxy/errors-custom/ -COPY assets/supervisord.conf /etc/supervisord.conf -COPY assets/crontab /etc/crontabs/root - -COPY scripts/exit-event-listener.py /exit-event-listener.py -COPY scripts/haproxy.sh /haproxy.sh -COPY scripts/haproxy-reload.sh /haproxy-reload.sh +COPY assets / #CMD ["haproxy", "-f", "/etc/haproxy/haproxy.cfg"] diff --git a/assets/crontab b/assets/etc/crontabs/root similarity index 86% rename from assets/crontab rename to assets/etc/crontabs/root index 7b75766..cabb0aa 100644 --- a/assets/crontab +++ b/assets/etc/crontabs/root @@ -5,5 +5,5 @@ 0 2 * * * run-parts /etc/periodic/daily 0 3 * * 6 run-parts /etc/periodic/weekly 0 5 1 * * run-parts /etc/periodic/monthly -* * * * * /haproxy-reload.sh +* * * * * /scripts/haproxy-reload.sh diff --git a/conf.d/00-simple-comment.cfg b/assets/etc/haproxy/conf.d/00-simple-comment.cfg similarity index 100% rename from conf.d/00-simple-comment.cfg rename to assets/etc/haproxy/conf.d/00-simple-comment.cfg diff --git a/conf.d/README.md b/assets/etc/haproxy/conf.d/README.md similarity index 100% rename from conf.d/README.md rename to assets/etc/haproxy/conf.d/README.md diff --git a/assets/errors-custom/400.http b/assets/etc/haproxy/errors-custom/400.http similarity index 100% rename from assets/errors-custom/400.http rename to assets/etc/haproxy/errors-custom/400.http diff --git a/assets/errors-custom/403.http b/assets/etc/haproxy/errors-custom/403.http similarity index 100% rename from assets/errors-custom/403.http rename to assets/etc/haproxy/errors-custom/403.http diff --git a/assets/errors-custom/408.http b/assets/etc/haproxy/errors-custom/408.http similarity index 100% rename from assets/errors-custom/408.http rename to assets/etc/haproxy/errors-custom/408.http diff --git a/assets/errors-custom/500.http b/assets/etc/haproxy/errors-custom/500.http similarity index 100% rename from assets/errors-custom/500.http rename to assets/etc/haproxy/errors-custom/500.http diff --git a/assets/errors-custom/502.http b/assets/etc/haproxy/errors-custom/502.http similarity index 100% rename from assets/errors-custom/502.http rename to assets/etc/haproxy/errors-custom/502.http diff --git a/assets/errors-custom/503.http b/assets/etc/haproxy/errors-custom/503.http similarity index 100% rename from assets/errors-custom/503.http rename to assets/etc/haproxy/errors-custom/503.http diff --git a/assets/errors-custom/504.http b/assets/etc/haproxy/errors-custom/504.http similarity index 100% rename from assets/errors-custom/504.http rename to assets/etc/haproxy/errors-custom/504.http diff --git a/assets/supervisord.conf b/assets/etc/supervisord.conf similarity index 95% rename from assets/supervisord.conf rename to assets/etc/supervisord.conf index 5ad57e6..29e789b 100644 --- a/assets/supervisord.conf +++ b/assets/etc/supervisord.conf @@ -22,7 +22,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface serverurl=unix:///dev/shm/supervisor.sock ; use a unix:// URL for a unix socket [program:haproxy] -command = /haproxy.sh +command = /scripts/haproxy.sh autostart=true autorestart=false priority=5 @@ -41,5 +41,5 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [eventlistener:exit_on_any_fatal] -command=/exit-event-listener.py +command=/scripts/exit-event-listener.py events=PROCESS_STATE_FATAL,PROCESS_STATE_EXITED,PROCESS_STATE_STOPPED diff --git a/scripts/exit-event-listener.py b/assets/scripts/exit-event-listener.py similarity index 100% rename from scripts/exit-event-listener.py rename to assets/scripts/exit-event-listener.py diff --git a/scripts/haproxy-reload.sh b/assets/scripts/haproxy-reload.sh similarity index 100% rename from scripts/haproxy-reload.sh rename to assets/scripts/haproxy-reload.sh diff --git a/scripts/haproxy.sh b/assets/scripts/haproxy.sh similarity index 100% rename from scripts/haproxy.sh rename to assets/scripts/haproxy.sh diff --git a/easymapping/__init__.py b/easymapping/__init__.py new file mode 100644 index 0000000..1b54af2 --- /dev/null +++ b/easymapping/__init__.py @@ -0,0 +1,12 @@ +from jinja2 import Environment, FileSystemLoader + + +class HaproxyConfigGenerator: + def __init__(self, mapping): + self.mapping = mapping + + def generate(self): + file_loader = FileSystemLoader('templates') + env = Environment(loader=file_loader) + template = env.get_template('haproxy.cfg.j2') + return template.render(data=self.mapping) diff --git a/entrypoint.py b/entrypoint.py deleted file mode 100644 index 7962ed2..0000000 --- a/entrypoint.py +++ /dev/null @@ -1,109 +0,0 @@ -import yaml -import sys - -if len(sys.argv) != 2: - print("You need to pass the easyconfig.cfg path") - exit(1) - - -def defaults(custom): - result = """ -defaults - log global - - timeout connect 3s - timeout client 10s - timeout server 10m -""" - if custom: - result += """ - errorfile 400 /etc/haproxy/errors-custom/400.http - errorfile 403 /etc/haproxy/errors-custom/403.http - errorfile 408 /etc/haproxy/errors-custom/408.http - errorfile 500 /etc/haproxy/errors-custom/500.http - errorfile 502 /etc/haproxy/errors-custom/502.http - errorfile 503 /etc/haproxy/errors-custom/503.http - errorfile 504 /etc/haproxy/errors-custom/504.http -""" - result += """ -global - log /dev/log local0 - maxconn 2000 - tune.ssl.default-dh-param 2048 -""" - return result - - -def stats(map): - return """ -frontend stats - bind *:{2} - mode http - stats enable - stats hide-version - stats realm Haproxy\ Statistics - stats uri / - stats auth {0}:{1} -# acl is_proxystats hdr(host) -i some.host.com -# default_backend srv_stats -# use_backend srv_stats if is_proxystats - default_backend srv_stats - -backend srv_stats - mode http - server Local 127.0.0.1:{2} -""".format(map["username"], map["password"], map["port"] if "port" in map else 1936) - - -def easymapping(o, salt): - port = o["port"] - ssl = " ssl crt " + o["ssl_cert"] if "ssl_cert" in o else "" - hosts = o["hosts"] if "hosts" in o else dict() - redir = o["redirect"] if "redirect" in o else dict() - - result = """ -frontend http_in_{0}_{1} - bind *:{0} {2} - mode http - -""".format(port, salt, ssl) - - for k in redir: - result += " redirect prefix " + redir[k] + " code 301 if { hdr(host) -i " + k + " }\n" - - result += "\n" - for k in hosts: - host = k.replace(".", "_") + "_{0}_{1}".format(port, salt) - result += " acl is_rule_{0}_1 hdr(host) -i {1}\n".format(host, k) - result += " acl is_rule_{0}_2 hdr(host) -i {1}:{2}\n".format(host, k, port) - result += " use_backend srv_{0} if is_rule_{0}_1 OR is_rule_{0}_2\n\n".format(host) - - for k in hosts: - host = k.replace(".", "_") + "_{0}_{1}".format(port, salt) - result += """ -backend srv_{0} - balance roundrobin - mode http - option forwardfor - http-request set-header X-Forwarded-Port %[dst_port]""".format(host) + """ - http-request add-header X-Forwarded-Proto https if { ssl_fc }""" + """ - server srv {0} check weight 1 -""".format(hosts[k]) - - return result - - -with open(sys.argv[1], 'r') as content_file: - parsed = yaml.load(content_file.read()) - -n = 0 - -print(defaults(parsed["customerrors"] if "customerrors" in parsed else False)) -if "stats" in parsed: - print(stats(parsed["stats"])) -if "easymapping" in parsed: - for k in parsed["easymapping"]: - n = n + 1 - print(easymapping(k, n)) - - diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 9eb6d31..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo " ______ _ _ _____ " -echo "| ____| | | | | /\ | __ \ " -echo "| |__ __ _ ___ _ _ | |__| | / \ | |__) | __ _____ ___ _ " -echo "| __| / _\` / __| | | | | __ | / /\ \ | ___/ '__/ _ \ \/ / | | |" -echo "| |___| (_| \__ \ |_| | | | | |/ ____ \| | | | | (_) > <| |_| |" -echo "|______\__,_|___/\__, | |_| |_/_/ \_\_| |_| \___/_/\_\\__, |" -echo " __/ | __/ |" -echo " |___/ |___/ " -echo "" - -HAPROXY_CFG="/etc/haproxy/haproxy.cfg" -EASYCONFIG_CFG="/etc/easyconfig/easyconfig.cfg" - -if [ ! -f "$EASYCONFIG_CFG" ] -then - echo "File '$EASYCONFIG_CFG' does not exist" - exit 1 -fi - - - -python3 /entrypoint.py "$EASYCONFIG_CFG" > $HAPROXY_CFG - - -/sbin/syslogd -O /proc/1/fd/1 -/docker-entrypoint.sh "$@" diff --git a/requirements.txt b/requirements.txt index 949d85b..fa90f6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pyyaml -docker \ No newline at end of file +docker +jinja2 \ No newline at end of file diff --git a/static.py b/static.py new file mode 100644 index 0000000..87efb42 --- /dev/null +++ b/static.py @@ -0,0 +1,16 @@ +import yaml +import sys +from easymapping import HaproxyConfigGenerator + +if len(sys.argv) != 2: + print("You need to pass the easyconfig.cfg path") + exit(1) + + +with open(sys.argv[1], 'r') as content_file: + parsed = yaml.load(content_file.read()) + +cfg = HaproxyConfigGenerator(parsed) +print(cfg.generate()) + +exit(0) diff --git a/discover.py b/swarm.py similarity index 55% rename from discover.py rename to swarm.py index a077fad..dd2921a 100644 --- a/discover.py +++ b/swarm.py @@ -1,11 +1,14 @@ import os import json +from easymapping import HaproxyConfigGenerator path = os.path.dirname(os.path.realpath(__file__)) with open(path + "/.docker_data", 'r') as content_file: lineList = content_file.readlines() -result = dict() +result = { + "easymapping": [] +} for line in lineList: line = line.strip() @@ -19,19 +22,25 @@ for line in lineList: for definition in definitions: port = d["com.byjg.easyhaproxy.port." + definition] if "com.byjg.easyhaproxy.port." + definition in d else "80" - if port not in result: - result[port] = dict() - result[port]["hosts"] = dict() - result[port]["redirect"] = dict() - result[port]["ssl_cert"] = None + data = { + "port": port, + "hosts": dict(), + "redirect": dict(), + # "ssl_cert": "" + } - result[port]["hosts"][d["com.byjg.easyhaproxy.host." + definition] if "com.byjg.easyhaproxy.host." + definition in d else ""] = container + ":" + (d["com.byjg.easyhaproxy.localport." + definition] if "com.byjg.easyhaproxy.localport." + definition in d else "80") + data["hosts"][d["com.byjg.easyhaproxy.host." + definition] if "com.byjg.easyhaproxy.host." + definition in d else ""] = container + ":" + (d["com.byjg.easyhaproxy.localport." + definition] if "com.byjg.easyhaproxy.localport." + definition in d else "80") redirect = d["com.byjg.easyhaproxy.redirect." + definition] if "com.byjg.easyhaproxy.redirect." + definition in d else "" for r in redirect.split(","): r_parts = r.split("=>") - result[port]["redirect"][r_parts[0]] = r_parts[1] + data["redirect"][r_parts[0]] = r_parts[1] + result["easymapping"].append(data) + + +cfg = HaproxyConfigGenerator(result) +print(cfg.generate()) print(result) # print(jsonStr) diff --git a/discover.sh b/swarm.sh similarity index 94% rename from discover.sh rename to swarm.sh index d860b7f..1948b52 100644 --- a/discover.sh +++ b/swarm.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash - #docker run \ # -l com.byjg.easyhaproxy.definitions=http \ # -l com.byjg.easyhaproxy.port.http=80 \ @@ -22,9 +21,8 @@ done if cmp -s ./.docker_data ./.docker_data_old ; then - echo "Nothing changed" -else - echo "Something changed" + exit 0 fi +python3 swarm.py # byjg_site={"com.byjg.easyhaproxy.definition":"http","com.byjg.easyhaproxy.host.http":"byjg.com.br","com.byjg.easyhaproxy.port.http":"80","com.byjg.easyhaproxy.redirect.http":"byjg.com.br%http://www.byjg.com.br,byjg.com%http://www.byjg.com.br","maintainer":"Joao Gilberto Magalhaes"} diff --git a/templates/haproxy.cfg.j2 b/templates/haproxy.cfg.j2 new file mode 100644 index 0000000..3ac2805 --- /dev/null +++ b/templates/haproxy.cfg.j2 @@ -0,0 +1,68 @@ +defaults + log global + + timeout connect 3s + timeout client 10s + timeout server 10m +{% if data["customerrors"] %} + errorfile 400 /etc/haproxy/errors-custom/400.http + errorfile 403 /etc/haproxy/errors-custom/403.http + errorfile 408 /etc/haproxy/errors-custom/408.http + errorfile 500 /etc/haproxy/errors-custom/500.http + errorfile 502 /etc/haproxy/errors-custom/502.http + errorfile 503 /etc/haproxy/errors-custom/503.http + errorfile 504 /etc/haproxy/errors-custom/504.http +{% endif %} + +global + log /dev/log local0 + maxconn 2000 + tune.ssl.default-dh-param 2048 + +{% if "stats" in data %} +frontend stats + bind *:{{ data["stats"]["port"] | default(1936) }} + mode http + stats enable + stats hide-version + stats realm Haproxy\ Statistics + stats uri / + stats auth {{ data["stats"]["username"] }}:{{ data["stats"]["password"] }} +# acl is_proxystats hdr(host) -i some.host.com +# default_backend srv_stats +# use_backend srv_stats if is_proxystats + default_backend srv_stats + +backend srv_stats + mode http + server Local 127.0.0.1:{{ data["stats"]["port"] | default(1936) }} +{% endif %} + +{% for o in data["easymapping"] %} + {% set salt = loop.index %} +frontend http_in_{{ o["port"] }}_{{ salt }} + bind *:{{ o["port"] }} {{ " ssl crt " + o["ssl_cert"] if "ssl_cert" in o else "" }} + mode http + + {% for k in o["redirect"] -%} + redirect prefix {{ o["redirect"][k] }} code 301 if { hdr(host) -i {{ k }} } + {% endfor -%} + + {% for k in o["hosts"] %} + {% set host = k.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) %} + acl is_rule_{{ host }}_1 hdr(host) -i {{ k }} + acl is_rule_{{ host }}_2 hdr(host) -i {{ k }}:{{ o["port"] }} + use_backend srv_{{ host }} if is_rule_{{ host }}_1 OR is_rule_{{ host }}_2 + {% endfor %} + + {% for k in o["hosts"] %} + {% set host = k.replace(".", "_") + "_{0}_{1}".format(o["port"], salt) %} +backend srv_{{ host }} + balance roundrobin + mode http + option forwardfor + http-request set-header X-Forwarded-Port %[dst_port] + http-request add-header X-Forwarded-Proto https if { ssl_fc } + server srv {{ k }} check weight 1 + {% endfor %} +{% endfor %}