diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index 3399800..bf61f71 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -241,36 +241,59 @@ jobs: echo "๐Ÿงน Remove the docker image artifact" rm "${{ runner.temp }}/${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar" - echo "๐Ÿ›‘ Stop and remove containers before updating compose files" - #ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_core.yml down" - if [ "${PROD}" = true ]; then - ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_web.yml down" - else - ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_staging.yml down" - fi + echo "๐Ÿ’พ Copy new files to server" + ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new" + scp -pr ./server/* deploy:${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/ - echo "๐Ÿ’พ Copy files to server" - ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}" - scp -pr ./server/* deploy:${{ steps.meta.outputs.REPO_PROJECT_PATH}}/ + echo "๐Ÿ“ Create new .env file" + printf "%s" "${ENV_FILE_BASE64}" | base64 -d | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/.env && chmod 600 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/.env" - echo "๐Ÿ“ Create .env file" - printf "%s" "${ENV_FILE_BASE64}" | base64 -d | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/.env && chmod 600 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/.env" + echo "๐Ÿ”‘ Set up certs" + ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs && chmod 550 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs" + printf "%s" "$CF_PEM_CERT" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/crt.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/crt.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/crt.pem" + printf "%s" "$CF_PEM_CA" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/ca.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/ca.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/new/certs/ca.pem" - echo "๐Ÿ”‘ Set up certificates" - ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs && chmod 550 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs" - printf "%s" "$CF_PEM_CERT" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem" - printf "%s" "$CF_PEM_CA" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem" + ssh -T deploy <=3.18.3", "django-linear-migrations>=2.17.0", "django-meta==2.4.2", "django-tailwind-4[reload]==0.1.4", @@ -55,6 +56,7 @@ dependencies = [ "packaging==23.1", "pillow>=11.2.1", "playwright==1.52.0", + "psutil>=7.0.0", "psycopg==3.2.3", "psycopg-binary==3.2.3", "pycparser==2.21", diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index c63b65e..972a189 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -13,8 +13,5 @@ else export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE fi -echo "Running deploy.sh... (if you get a APP_REGISTRY_NOT_READY error, there's probably an error in settings.py)" -/deploy.sh - -echo "Environment is correct and deploy.sh has been run - executing command: '$@'" +echo "Environment is correct - executing command: '$@'" exec "$@" && exit 0 \ No newline at end of file diff --git a/server/docker-compose_core.yml b/server/docker-compose_core.yml index 06a5887..9925cdf 100644 --- a/server/docker-compose_core.yml +++ b/server/docker-compose_core.yml @@ -1,5 +1,5 @@ services: - db-healthcheck: + db-redis-healthcheck: image: stephenc/postgresql-cli:latest command: - "sh" @@ -9,26 +9,47 @@ services: sleep 30; while true; do pg_output=$$(pg_isready -d ${DJANGO_DATABASE_URL} 2>&1); - exit_code=$$?; - if [ $$exit_code -eq 0 ]; then - success="true"; - error=""; + pg_exit_code=$$?; + if [ $$pg_exit_code -eq 0 ]; then + pg_success="true"; + pg_error=""; else - success="false"; - error="$$pg_output"; + pg_success="false"; + pg_error="$$pg_output"; fi; curl -s -f -X POST \ --connect-timeout 10 \ --max-time 15 \ --header "Authorization: Bearer ${GATUS_TOKEN}" \ - http://health:8080/api/v1/endpoints/db_pg-isready/external?success=$$success&error=$$error; - if [ "$$success" = "true" ]; then + http://health:8080/api/v1/endpoints/services_database/external?success=$$pg_success&error=$$pg_error; + if [ "$$pg_success" = "true" ]; then echo " Database is OK"; - sleep 60; else echo "Database is not OK: $$pg_output"; exit 1; fi; + + redis_output=$$(echo -e "ping\nquit" | curl -v --max-time 10 --connect-timeout 10 telnet://redis:6379 2>&1 | grep -q "+PONG"); + redis_exit_code=$$?; + if [ $$redis_exit_code -eq 0 ]; then + redis_success="true"; + redis_error=""; + else + redis_success="false"; + redis_error="$$redis_output"; + fi; + curl -s -f -X POST \ + --connect-timeout 10 \ + --max-time 15 \ + --header "Authorization: Bearer ${GATUS_TOKEN}" \ + http://health:8080/api/v1/endpoints/services_redis/external?success=$$redis_success&error=$$redis_error; + if [ "$$redis_success" = "true" ]; then + echo " Redis is OK"; + else + echo "Redis is not OK: $$redis_output"; + exit 1; + fi; + sleep 60; done env_file: - .env @@ -46,41 +67,114 @@ services: feedback: restart: always image: getfider/fider:stable + labels: + - "enable_gatekeeper=true" env_file: - .env - cadvisor: - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - /dev/disk/:/dev/disk:ro - privileged: true - devices: - - /dev/kmsg - image: gcr.io/cadvisor/cadvisor:v0.52.1 + # cadvisor: + # volumes: + # - /:/rootfs:ro + # - /var/run:/var/run:ro + # - /sys:/sys:ro + # - /var/lib/docker/:/var/lib/docker:ro + # - /dev/disk/:/dev/disk:ro + # privileged: true + # devices: + # - /dev/kmsg + # image: gcr.io/cadvisor/cadvisor:v0.52.1 redis: image: redis:latest restart: always - ports: - - 6379:6379 - # anubis: - # image: ghcr.io/techarohq/anubis:latest - # env_file: - # - .env - # dockergen: - # image: jwilder/docker-gen:latest - # container_name: dockergen_gatus_config - # command: -watch -notify-sighup gatus_service -only-exposed /app/config.template.yml /app/config.yaml - # restart: unless-stopped - # volumes: - # - /var/run/docker.sock:/tmp/docker.sock:ro - # - ./gatus:/app - # depends_on: - # - health + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + dockergen-health: + image: nginxproxy/docker-gen:latest + command: -wait 15s -watch /gatus/config.template.yaml /gatus/config.yaml + restart: unless-stopped + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + - ./gatus:/gatus + dockergen-gatekeeper: + image: nginxproxy/docker-gen:latest + command: -wait 15s -watch /gatekeeper/docker-compose_gatekeeper.template.yml /gatekeeper/docker-compose_gatekeeper.yml -notify-sighup gatekeeper-manager + restart: unless-stopped + volumes: + - /var/run/docker.sock:/tmp/docker.sock:ro + - ./:/gatekeeper + gatekeeper-manager: + image: docker:latest + restart: always + stop_signal: SIGTERM + volumes: + - /srv:/srv:ro + - /var/run/docker.sock:/var/run/docker.sock + environment: + - REFRESH_INTERVAL=60 + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu -o pipefail + + COMPOSE_FILE_PATH="/srv/pkmntrade-club/docker-compose_gatekeeper.yml" + PROJECT_DIR_PATH="/srv/pkmntrade-club" + PROJECT_NAME_TAG="gatekeepers" + + gatekeeper_down() { + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Taking gatekeepers down (Project: $$PROJECT_NAME_TAG)..." + cd "$$PROJECT_DIR_PATH" + if ! docker compose -p "$$PROJECT_NAME_TAG" -f "$$COMPOSE_FILE_PATH" down; then + echo "$(date +'%Y-%m-%d %H:%M:%S') [WARN]: 'docker compose down' for $$PROJECT_NAME_TAG encountered an issue, but proceeding." + fi + } + + gatekeeper_up() { + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Bringing gatekeepers up/updating (Project: $$PROJECT_NAME_TAG, File: $$COMPOSE_FILE_PATH)..." + cd "$$PROJECT_DIR_PATH" + if ! docker compose -p "$$PROJECT_NAME_TAG" -f "$$COMPOSE_FILE_PATH" up -d --remove-orphans; then + echo "$(date +'%Y-%m-%d %H:%M:%S') [ERROR]: 'docker compose up' for $$PROJECT_NAME_TAG failed. Will retry." + fi + } + + handle_sigterm() { + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: SIGTERM received. Initiating graceful shutdown for gatekeepers." + gatekeeper_down + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Gatekeepers shut down. Gatekeeper Manager exiting." + exit 0 + } + + handle_sighup() { + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: SIGHUP received. Restarting gatekeepers." + gatekeeper_down + gatekeeper_up + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Gatekeepers restarted following SIGHUP." + } + + trap 'handle_sigterm' SIGTERM + trap 'handle_sighup' SIGHUP + + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Gatekeeper Manager started." + + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Periodic refresh enabled: $$REFRESH_INTERVAL seconds." + while true; do + gatekeeper_up + + # 'sleep 60 &' and 'wait $!' allows signals to interrupt the sleep. + sleep $$REFRESH_INTERVAL & + # '|| true' ensures the loop continues if 'wait' is killed by a handled signal (SIGHUP/SIGTERM) + # SIGTERM handler exits completely, so loop won't continue. SIGHUP handler doesn't exit. + wait $! || true + + echo "$(date +'%Y-%m-%d %H:%M:%S') [INFO]: Periodic refresh triggered." + done health: image: twinproduction/gatus:latest restart: always + labels: + - "enable_gatekeeper=true" env_file: - .env environment: diff --git a/server/docker-compose_gatekeeper.template.yml b/server/docker-compose_gatekeeper.template.yml new file mode 100644 index 0000000..29a2420 --- /dev/null +++ b/server/docker-compose_gatekeeper.template.yml @@ -0,0 +1,37 @@ +services: + {{ $all_containers := whereLabelValueMatches . "enable_gatekeeper" "true" }} + {{ $all_containers = sortObjectsByKeysAsc $all_containers "Name" }} + + {{ range $container := $all_containers }} + {{ $serviceLabel := index $container.Labels "com.docker.compose.service" }} + {{ $containerNumber := index $container.Labels "com.docker.compose.container-number" }} + {{ $port := "" }} + {{ if eq $serviceLabel "web" }} + {{ $port = ":8000" }} + {{ end }} + {{ if eq $serviceLabel "web-staging" }} + {{ $port = ":8000" }} + {{ end }} + {{ if eq $serviceLabel "feedback" }} + {{ $port = ":3000" }} + {{ end }} + {{ if eq $serviceLabel "health" }} + {{ $port = ":8080" }} + {{ end }} + gatekeeper-{{ $serviceLabel }}-{{ $containerNumber }}: + image: ghcr.io/techarohq/anubis:latest + container_name: pkmntrade-club-gatekeeper-{{ $serviceLabel }}-{{ $containerNumber }} + env_file: + - .env + environment: + - TARGET=http://{{ $container.Name }}{{ $port }} + networks: + default: + aliases: + - pkmntrade-club-gatekeeper-{{ $serviceLabel }} + - gatekeeper-{{ $serviceLabel }} + {{ end }} +networks: + default: + name: pkmntrade-club_default + external: true diff --git a/server/docker-compose_staging.yml b/server/docker-compose_staging.yml index 85b82a1..1b487bb 100644 --- a/server/docker-compose_staging.yml +++ b/server/docker-compose_staging.yml @@ -3,15 +3,17 @@ x-common: &common restart: always env_file: - .env - environment: - - DEBUG=True - - DISABLE_SIGNUPS=True - - PUBLIC_HOST=staging.pkmntrade.club - - ALLOWED_HOSTS=staging.pkmntrade.club,127.0.0.1 services: web-staging: <<: *common + environment: + - DEBUG=False + - DISABLE_SIGNUPS=True + - PUBLIC_HOST=staging.pkmntrade.club + - ALLOWED_HOSTS=staging.pkmntrade.club,127.0.0.1,pkmntrade-club-web-staging-1,pkmntrade-club-web-staging-2 + labels: + - "enable_gatekeeper=true" deploy: mode: replicated replicas: 2 @@ -23,4 +25,9 @@ services: # start_period: 30s celery-staging: <<: *common + environment: + - DEBUG=False + - DISABLE_SIGNUPS=True + - PUBLIC_HOST=staging.pkmntrade.club + - ALLOWED_HOSTS=staging.pkmntrade.club,127.0.0.1,pkmntrade-club-celery-staging-1 command: ["celery", "-A", "pkmntrade_club.django_project", "worker", "-l", "INFO", "-B", "-E"] \ No newline at end of file diff --git a/server/docker-compose_web.yml b/server/docker-compose_web.yml index bd9b342..51605e8 100644 --- a/server/docker-compose_web.yml +++ b/server/docker-compose_web.yml @@ -2,11 +2,6 @@ x-common: &common restart: always env_file: - .env - environment: - - DEBUG=False - - DISABLE_SIGNUPS=True - - PUBLIC_HOST=pkmntrade.club - - ALLOWED_HOSTS=pkmntrade.club,127.0.0.1 services: web: @@ -15,6 +10,13 @@ services: entrypoint: ["/ko-app/httpdebug", "--bind", ":8000"] #image: badbl0cks/pkmntrade-club:stable #command: ["granian", "--interface", "wsgi", "pkmntrade_club.django_project.wsgi:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--workers-kill-timeout", "180", "--access-log"] + environment: + - DEBUG=False + - DISABLE_SIGNUPS=True + - PUBLIC_HOST=pkmntrade.club + - ALLOWED_HOSTS=pkmntrade.club,127.0.0.1,pkmntrade-club-web-1,pkmntrade-club-web-2,pkmntrade-club-web-3,pkmntrade-club-web-4 + labels: + - "enable_gatekeeper=true" deploy: mode: replicated replicas: 4 @@ -24,7 +26,12 @@ services: # timeout: 10s # retries: 3 # start_period: 30s - celery: - <<: *common - image: badbl0cks/pkmntrade-club:stable - command: ["celery", "-A", "pkmntrade_club.django_project", "worker", "-l", "INFO", "-B", "-E"] \ No newline at end of file + # celery: + # <<: *common + # image: badbl0cks/pkmntrade-club:stable + # environment: + # - DEBUG=False + # - DISABLE_SIGNUPS=True + # - PUBLIC_HOST=pkmntrade.club + # - ALLOWED_HOSTS=pkmntrade.club,127.0.0.1,pkmntrade-club-celery-1,pkmntrade-club-celery-2 + # command: ["celery", "-A", "pkmntrade_club.django_project", "worker", "-l", "INFO", "-B", "-E"] \ No newline at end of file diff --git a/server/gatus/config.template.yaml b/server/gatus/config.template.yaml new file mode 100644 index 0000000..0e3c7e9 --- /dev/null +++ b/server/gatus/config.template.yaml @@ -0,0 +1,145 @@ +storage: + type: postgres + path: "${GATUS_DATABASE_URL}" +web: + read-buffer-size: 32768 +connectivity: + checker: + target: 1.1.1.1:53 + interval: 60s +external-endpoints: + - name: Database + group: Services + token: "${GATUS_TOKEN}" + alerts: + - type: email + - name: Redis + group: Services + token: "${GATUS_TOKEN}" + alerts: + - type: email +endpoints: + - name: Domain + group: Expirations + url: "https://pkmntrade.club" + interval: 1h + conditions: + - "[DOMAIN_EXPIRATION] > 720h" + alerts: + - type: email + - name: Certificate + group: Expirations + url: "https://pkmntrade.club" + interval: 1h + conditions: + - "[CERTIFICATE_EXPIRATION] > 240h" + alerts: + - type: email + - name: Cloudflare + group: DNS + url: "1.1.1.1" + interval: 60s + dns: + query-name: "pkmntrade.club" + query-type: "A" + conditions: + - "[DNS_RCODE] == NOERROR" + alerts: + - type: email + - name: Google + group: DNS + url: "8.8.8.8" + interval: 60s + dns: + query-name: "pkmntrade.club" + query-type: "A" + conditions: + - "[DNS_RCODE] == NOERROR" + alerts: + - type: email + - name: Quad9 + group: DNS + url: "9.9.9.9" + interval: 60s + dns: + query-name: "pkmntrade.club" + query-type: "A" + conditions: + - "[DNS_RCODE] == NOERROR" + alerts: + - type: email + - name: HAProxy + group: Load Balancer + url: "http://loba/" + interval: 60s + conditions: + - "[STATUS] == 200" + - "[BODY] == OK/HEALTHY" + alerts: + - type: email + - name: Feedback + group: Services + url: "http://feedback:3000/" + interval: 60s + conditions: + - "[STATUS] == 200" + alerts: + - type: email + {{ $all_containers := . }} + {{ $web_containers := list }} + {{ $web_staging_containers := list }} + + {{ range $container := $all_containers }} + {{ $serviceLabel := index $container.Labels "com.docker.compose.service" }} + {{ if eq $serviceLabel "web" }} + {{ $web_containers = append $web_containers $container }} + {{ end }} + {{ if eq $serviceLabel "web-staging" }} + {{ $web_staging_containers = append $web_staging_containers $container }} + {{ end }} + {{ end }} + + {{ $web_containers = sortObjectsByKeysAsc $web_containers "Name" }} + {{ $web_staging_containers = sortObjectsByKeysAsc $web_staging_containers "Name" }} + + {{ range $container := $web_containers }} + {{ $containerNumber := index $container.Labels "com.docker.compose.container-number" }} + - name: "Web Worker {{ $containerNumber }}" + group: Main + url: "http://{{ $container.Name }}:8000/health/" + interval: 60s + conditions: + - "[STATUS] == 200" + # - "[BODY] == OK/HEALTHY" + alerts: + - type: email + {{ end }} + + {{ range $container := $web_staging_containers }} + {{ $containerNumber := index $container.Labels "com.docker.compose.container-number" }} + - name: "Web Worker {{ $containerNumber }}" + group: Staging + url: "http://{{ $container.Name }}:8000/health/" + interval: 60s + conditions: + - "[STATUS] == 200" + # - "[BODY] == OK/HEALTHY" + alerts: + - type: email + {{ end }} + +alerting: + email: + from: "${GATUS_SMTP_FROM}" + username: "${GATUS_SMTP_USER}" + password: "${GATUS_SMTP_PASS}" + host: "${GATUS_SMTP_HOST}" + port: ${GATUS_SMTP_PORT} + to: "${GATUS_SMTP_TO}" + client: + insecure: false + default-alert: + enabled: true + failure-threshold: 3 + success-threshold: 2 + send-on-resolved: true diff --git a/server/gatus/config.yaml b/server/gatus/config.yaml index 98ae997..8261132 100644 --- a/server/gatus/config.yaml +++ b/server/gatus/config.yaml @@ -8,14 +8,19 @@ connectivity: target: 1.1.1.1:53 interval: 60s external-endpoints: - - name: pg_isready - group: db + - name: Database + group: Services + token: "${GATUS_TOKEN}" + alerts: + - type: email + - name: Redis + group: Services token: "${GATUS_TOKEN}" alerts: - type: email endpoints: - name: Domain - group: expirations + group: Expirations url: "https://pkmntrade.club" interval: 1h conditions: @@ -23,7 +28,7 @@ endpoints: alerts: - type: email - name: Certificate - group: expirations + group: Expirations url: "https://pkmntrade.club" interval: 1h conditions: @@ -31,7 +36,7 @@ endpoints: alerts: - type: email - name: Cloudflare - group: dns + group: DNS url: "1.1.1.1" interval: 60s dns: @@ -42,7 +47,7 @@ endpoints: alerts: - type: email - name: Google - group: dns + group: DNS url: "8.8.8.8" interval: 60s dns: @@ -53,7 +58,7 @@ endpoints: alerts: - type: email - name: Quad9 - group: dns + group: DNS url: "9.9.9.9" interval: 60s dns: @@ -64,7 +69,7 @@ endpoints: alerts: - type: email - name: HAProxy - group: loadbalancer + group: Load Balancer url: "http://loba/" interval: 60s conditions: @@ -73,60 +78,22 @@ endpoints: alerts: - type: email - name: Feedback - group: backends + group: Services url: "http://feedback:3000/" interval: 60s conditions: - "[STATUS] == 200" alerts: - type: email - - name: Web Worker 1 - group: backends - url: "http://pkmntrade-club-web-1:8000/health/" - interval: 60s - conditions: - - "[STATUS] == 200" - #- "[BODY] == OK/HEALTHY" - #- [BODY].database == UP - # must return json like {"database": "UP"} first - alerts: - - type: email - - name: Web Worker 2 - group: backends - url: "http://pkmntrade-club-web-2:8000/health/" - interval: 60s - conditions: - - "[STATUS] == 200" - #- "[BODY] == OK/HEALTHY" - alerts: - - type: email - - name: Web Worker 3 - group: backends - url: "http://pkmntrade-club-web-3:8000/health/" - interval: 60s - conditions: - - "[STATUS] == 200" - #- "[BODY] == OK/HEALTHY" - alerts: - - type: email - - name: Web Worker 4 - group: backends - url: "http://pkmntrade-club-web-4:8000/health/" - interval: 60s - conditions: - - "[STATUS] == 200" - #- "[BODY] == OK/HEALTHY" - alerts: - - type: email - # todo: add cadvisor checks via api https://github.com/google/cadvisor/blob/master/docs/api.md + alerting: email: - from: noreply@pkmntrade.club - username: dd2cd354-de6d-4fa4-bfe8-31c961cb4e90 - password: 1622e8a1-9a45-4a7f-8071-cccca29d8675 - host: smtp.tem.scaleway.com - port: 465 - to: rob@badblocks.email + from: "${GATUS_SMTP_FROM}" + username: "${GATUS_SMTP_USER}" + password: "${GATUS_SMTP_PASS}" + host: "${GATUS_SMTP_HOST}" + port: ${GATUS_SMTP_PORT} + to: "${GATUS_SMTP_TO}" client: insecure: false default-alert: diff --git a/server/haproxy.cfg b/server/haproxy.cfg index 6c9bde8..a06f079 100644 --- a/server/haproxy.cfg +++ b/server/haproxy.cfg @@ -21,7 +21,7 @@ defaults timeout http-request 120s option httplog -frontend web_frontend +frontend haproxy_entrypoint bind :443 ssl crt /certs/crt.pem verify required ca-file /certs/ca.pem use_backend %[req.hdr(host),lower,word(1,:)] # strip out port from host @@ -34,17 +34,17 @@ backend basic_check backend pkmntrade.club balance leastconn - server-template web- 10 web:8000 check resolvers docker_resolver init-addr libc,none + server-template gatekeeper-web- 4 gatekeeper-web:8000 check resolvers docker_resolver init-addr libc,none backend staging.pkmntrade.club balance leastconn - server-template web-staging- 10 web-staging:8000 check resolvers docker_resolver init-addr libc,none + server-template gatekeeper-web-staging- 4 gatekeeper-web-staging:8000 check resolvers docker_resolver init-addr libc,none backend feedback.pkmntrade.club - server feedback-1 feedback:3000 + server-template gatekeeper-feedback- 4 gatekeeper-feedback:8000 check resolvers docker_resolver init-addr libc,none backend health.pkmntrade.club - server health-1 health:8080 + server-template gatekeeper-health- 4 gatekeeper-health:8000 check resolvers docker_resolver init-addr libc,none #EOF - trailing newline required diff --git a/src/pkmntrade_club/django_project/settings.py b/src/pkmntrade_club/django_project/settings.py index 819d2d0..75c5f9e 100644 --- a/src/pkmntrade_club/django_project/settings.py +++ b/src/pkmntrade_club/django_project/settings.py @@ -118,6 +118,15 @@ INSTALLED_APPS = [ "crispy_tailwind", "tailwind", "django_linear_migrations", + 'health_check', + 'health_check.db', + 'health_check.cache', + 'health_check.storage', + 'health_check.contrib.migrations', + 'health_check.contrib.celery', + 'health_check.contrib.celery_ping', + 'health_check.contrib.psutil', + 'health_check.contrib.redis', "meta", ] + FIRST_PARTY_APPS @@ -155,6 +164,11 @@ if DEBUG: "django_browser_reload.middleware.BrowserReloadMiddleware", ] +HEALTH_CHECK = { + 'DISK_USAGE_MAX': 90, # percent + 'MEMORY_MIN': 100, # in MB +} + DAISY_SETTINGS = { 'SITE_TITLE': 'PKMN Trade Club Admin', 'DONT_SUPPORT_ME': True, @@ -208,7 +222,6 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] - # Internationalization # https://docs.djangoproject.com/en/dev/topics/i18n/ # https://docs.djangoproject.com/en/dev/ref/settings/#language-code @@ -344,6 +357,8 @@ CACHE_TIMEOUT = 604800 # 1 week DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda request: DEBUG} +REDIS_URL = "redis://redis:6379" + DISABLE_CACHE = env('DISABLE_CACHE', default=DEBUG) if DISABLE_CACHE: @@ -356,12 +371,12 @@ else: CACHES = { "default": { "BACKEND": "django.core.cache.backends.redis.RedisCache", - "LOCATION": "redis://redis:6379", + "LOCATION": REDIS_URL, } } -CELERY_BROKER_URL = "redis://redis:6379" -CELERY_RESULT_BACKEND = "redis://redis:6379" +CELERY_BROKER_URL = REDIS_URL +CELERY_RESULT_BACKEND = REDIS_URL CELERY_TIMEZONE = "America/Los_Angeles" CELERY_ENABLE_UTC = True CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/src/pkmntrade_club/django_project/urls.py b/src/pkmntrade_club/django_project/urls.py index e01ec1f..61cac00 100644 --- a/src/pkmntrade_club/django_project/urls.py +++ b/src/pkmntrade_club/django_project/urls.py @@ -4,10 +4,11 @@ from debug_toolbar.toolbar import debug_toolbar_urls urlpatterns = [ path("admin/", admin.site.urls), + path('account/', include('pkmntrade_club.accounts.urls')), path("accounts/", include("allauth.urls")), path("", include("pkmntrade_club.home.urls")), path("cards/", include("pkmntrade_club.cards.urls")), - path('account/', include('pkmntrade_club.accounts.urls')), + path("health/", include('health_check.urls')), path("trades/", include("pkmntrade_club.trades.urls")), path("__reload__/", include("django_browser_reload.urls")), ] + debug_toolbar_urls() diff --git a/src/pkmntrade_club/home/urls.py b/src/pkmntrade_club/home/urls.py index 0eae707..c135f7a 100644 --- a/src/pkmntrade_club/home/urls.py +++ b/src/pkmntrade_club/home/urls.py @@ -4,6 +4,4 @@ from .views import HomePageView, HealthCheckView urlpatterns = [ path("", HomePageView.as_view(), name="home"), - path("health", HealthCheckView.as_view(), name="health"), - path("health/", HealthCheckView.as_view(), name="health"), ] diff --git a/uv.lock b/uv.lock index c4187fd..f7c0324 100644 --- a/uv.lock +++ b/uv.lock @@ -313,6 +313,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/b3/0a3bec4ecbfee960f39b1842c2f91e4754251e0a6ed443db9fe3f666ba8f/django_environ-0.12.0-py2.py3-none-any.whl", hash = "sha256:92fb346a158abda07ffe6eb23135ce92843af06ecf8753f43adf9d2366dcc0ca", size = 19957 }, ] +[[package]] +name = "django-health-check" +version = "3.18.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/e9/0699ea3debfda75e5960ff99f56974136380e6f8202d453de7357e1f67fc/django_health_check-3.18.3.tar.gz", hash = "sha256:18b75daca4551c69a43f804f9e41e23f5f5fb9efd06cf6a313b3d5031bb87bd0", size = 20919 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/1e/3b23b580762cca7456427731de9b90718d15eec02ebe096437469d767dfe/django_health_check-3.18.3-py2.py3-none-any.whl", hash = "sha256:f5f58762b80bdf7b12fad724761993d6e83540f97e2c95c42978f187e452fa07", size = 30331 }, +] + [[package]] name = "django-linear-migrations" version = "2.17.0" @@ -591,6 +603,7 @@ dependencies = [ { name = "django-daisy" }, { name = "django-debug-toolbar" }, { name = "django-environ" }, + { name = "django-health-check" }, { name = "django-linear-migrations" }, { name = "django-meta" }, { name = "django-tailwind-4", extra = ["reload"] }, @@ -603,6 +616,7 @@ dependencies = [ { name = "packaging" }, { name = "pillow" }, { name = "playwright" }, + { name = "psutil" }, { name = "psycopg" }, { name = "psycopg-binary" }, { name = "pycparser" }, @@ -635,6 +649,7 @@ requires-dist = [ { name = "django-daisy", specifier = "==1.0.13" }, { name = "django-debug-toolbar", specifier = "==4.4.6" }, { name = "django-environ", specifier = "==0.12.0" }, + { name = "django-health-check", specifier = ">=3.18.3" }, { name = "django-linear-migrations", specifier = ">=2.17.0" }, { name = "django-meta", specifier = "==2.4.2" }, { name = "django-tailwind-4", extras = ["reload"], specifier = "==0.1.4" }, @@ -647,6 +662,7 @@ requires-dist = [ { name = "packaging", specifier = "==23.1" }, { name = "pillow", specifier = ">=11.2.1" }, { name = "playwright", specifier = "==1.52.0" }, + { name = "psutil", specifier = ">=7.0.0" }, { name = "psycopg", specifier = "==3.2.3" }, { name = "psycopg-binary", specifier = "==3.2.3" }, { name = "pycparser", specifier = "==2.21" }, @@ -692,6 +708,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/4f/5249960887b1fbe561d9ff265496d170b55a735b76724f10ef19f9e40716/prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07", size = 387810 }, ] +[[package]] +name = "psutil" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051 }, + { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535 }, + { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004 }, + { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986 }, + { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544 }, + { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053 }, + { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885 }, +] + [[package]] name = "psycopg" version = "3.2.3"