Finish packaging and change to src-based packaging layout, replace caddy with haproxy for performance, and update docker-compose and Dockerfiles for new packaging.

This commit is contained in:
badblocks 2025-05-09 18:39:04 -07:00
parent 959b06c425
commit 762361a21b
210 changed files with 235 additions and 168 deletions

2
.vscode/launch.json vendored
View file

@ -9,7 +9,7 @@
"args": ["runserver"],
"django": true,
"justMyCode": true,
"preLaunchTask": "Run db"
"preLaunchTask": "Run db standalone"
}
]
}

20
.vscode/tasks.json vendored
View file

@ -8,15 +8,21 @@
"problemMatcher": []
},
{
"label": "Run app & db (db in docker)",
"label": "Build & run app",
"type": "shell",
"command": "./scripts/entrypoint.sh",
"command": "docker compose up --build",
"problemMatcher": []
},
{
"label": "Run app & db (both in Docker)",
"label": "Build & run app (no build cache)",
"type": "shell",
"command": "docker compose -f docker-compose_entire_app.yml up",
"command": "docker compose up --build --no-cache",
"problemMatcher": []
},
{
"label": "Run db standalone",
"type": "shell",
"command": "docker compose up -d db",
"problemMatcher": []
},
{
@ -24,12 +30,6 @@
"type": "shell",
"command": "cd theme/static_src && npm run dev",
"problemMatcher": [],
},
{
"label": "Run db",
"type": "shell",
"command": "docker compose -f docker-compose_db_only.yml up",
"problemMatcher": []
}
]
}

View file

@ -94,4 +94,4 @@ EXPOSE 8000
HEALTHCHECK CMD curl --fail http://localhost:8000/health/ || exit 1
CMD ["granian", "--interface", "wsgi", "django_project.wsgi", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--respawn-failed-workers", "--workers-kill-timeout", "60"]
CMD ["granian", "--interface", "wsgi", "pkmntrade_club.django_project.wsgi", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--respawn-failed-workers", "--workers-kill-timeout", "60"]

View file

@ -1,29 +1,30 @@
include README.md
include LICENSE
graft accounts/templates
graft accounts/migrations
graft src/pkmntrade_club/accounts/templates
graft src/pkmntrade_club/accounts/migrations
graft cards/templates
graft cards/migrations
graft src/pkmntrade_club/cards/templates
graft src/pkmntrade_club/cards/migrations
graft home/migrations
graft src/pkmntrade_club/home/migrations
graft theme/templates
graft theme/templatetags
graft theme/static
graft src/pkmntrade_club/theme/templates
graft src/pkmntrade_club/theme/templatetags
graft src/pkmntrade_club/theme/static
graft trades/templates
graft trades/migrations
graft src/pkmntrade_club/trades/templates
graft src/pkmntrade_club/trades/migrations
graft static
graft src/pkmntrade_club/static
recursive-include accounts *.py
recursive-include cards *.py
recursive-include common *.py
recursive-include django_project *.py
recursive-include home *.py
recursive-include theme *.py
recursive-include trades *.py
recursive-include src/pkmntrade_club/accounts *.html *.py
recursive-include src/pkmntrade_club/cards *.html *.py
recursive-include src/pkmntrade_club/common *.html *.py
recursive-include src/pkmntrade_club/django_project *.html *.py
recursive-include src/pkmntrade_club/home *.html *.py
recursive-include src/pkmntrade_club/static *
recursive-include src/pkmntrade_club/theme *.html *.py
recursive-include src/pkmntrade_club/trades *.html *.py
global-exclude *.py[cod] __pycache__ .DS_Store .*

View file

@ -1,9 +0,0 @@
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_project.settings")
# use app for granian, application for gunicorn
app = get_wsgi_application()
application = app

View file

@ -1,26 +1,27 @@
services:
loba:
image: lucaslorentz/caddy-docker-proxy:2.9
image: haproxy:3.1
stop_signal: SIGTERM
ports:
- 8000:8000
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- caddy_data:/data
- ./haproxy/haproxy.dev.cfg:/usr/local/etc/haproxy/haproxy.cfg
depends_on:
- web
web:
build: .
volumes:
#- ./seed:/seed:ro
- ./.env.dev:/.env:ro
#- ./src:/src
- ./seed:/seed:ro
- ./src/pkmntrade_club/accounts/migrations:/app/lib/python3.12/site-packages/pkmntrade_club/accounts/migrations # for makemigrations
- ./src/pkmntrade_club/cards/migrations:/app/lib/python3.12/site-packages/pkmntrade_club/cards/migrations # for makemigrations
- ./src/pkmntrade_club/trades/migrations:/app/lib/python3.12/site-packages/pkmntrade_club/trades/migrations # for makemigrations
- ./src/pkmntrade_club/home/migrations:/app/lib/python3.12/site-packages/pkmntrade_club/home/migrations # for makemigrations
env_file:
- .env.dev
labels:
caddy: ":8000"
caddy.reverse_proxy: "{{upstreams 8000}}"
depends_on:
- db
db:
condition: service_healthy
db:
image: postgres:16
ports:
@ -29,9 +30,11 @@ services:
- postgres_data:/var/lib/postgresql/data/
environment:
- "POSTGRES_HOST_AUTH_METHOD=trust"
healthcheck:
test: ["CMD", "pg_isready", "-U", "postgres", "-d", "postgres"]
interval: 10s
timeout: 5s
retries: 5
volumes:
postgres_data:
labels:
- "db_is_resettable_via_script"
caddy_data:

View file

@ -1,26 +1,21 @@
services:
loba:
image: lucaslorentz/caddy-docker-proxy:2.9
image: haproxy:3.1
stop_signal: SIGTERM
ports:
- 8000:8000
- 443:443
entrypoint: ["/usr/local/bin/haproxy.entrypoint.sh"]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- caddy_data:/data
- ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
- ./scripts/haproxy.entrypoint.sh:/usr/local/bin/haproxy.entrypoint.sh
depends_on:
- web
web:
build: .
volumes:
- ./seed:/seed:ro
- ./.env.production:/.env:ro
env_file:
- .env.production
deploy:
mode: replicated
replicas: 4
labels:
caddy: ":8000"
caddy.reverse_proxy: "{{upstreams 8000}}"
volumes:
caddy_data:

21
haproxy/haproxy.cfg Normal file
View file

@ -0,0 +1,21 @@
defaults
mode http
timeout client 10s
timeout connect 5s
timeout server 10s
timeout http-request 10s
frontend cf
bind :443 ssl crt /certs/crt.pem verify required #ca-file /certs/ca.pem
default_backend django
backend django
balance leastconn
option httpchk
http-check send meth GET uri /health/
http-check expect string OK/HEALTHY
default-server check maxconn 10000 observe layer7
server django1 pkmntradeclub-web-1:8000
server django2 pkmntradeclub-web-2:8000
server django3 pkmntradeclub-web-3:8000
server django4 pkmntradeclub-web-4:8000

18
haproxy/haproxy.dev.cfg Normal file
View file

@ -0,0 +1,18 @@
defaults
mode http
timeout client 10s
timeout connect 5s
timeout server 10s
timeout http-request 10s
frontend cf
bind :8000
default_backend django
backend django
option httpchk
http-check send meth GET uri /health/
http-check expect string OK/HEALTHY
default-server check maxconn 10000 observe layer7
server django web:8000

View file

@ -3,10 +3,9 @@
import os
import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_project.settings")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pkmntrade_club.django_project.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:

View file

@ -72,14 +72,5 @@ pkmntrade-manage = "manage:main"
[project.urls]
Homepage = "https://pkmntrade.club"
[tool.setuptools]
packages = [
"accounts",
"cards",
"common",
"django_project",
"home",
"static",
"theme",
"trades"
]
[tool.setuptools.packages.find]
where = ["src"]

View file

@ -1,15 +1,15 @@
#!/bin/bash
echo "Running makemigrations --check to make sure migrations are up to date..."
django-admin makemigrations --noinput --check 2>&1
echo "*** Running makemigrations --check to make sure migrations are up to date..."
django-admin makemigrations --check --noinput 2>&1 || exit 1
echo "Running migrate to apply migrations..."
echo "*** Running migrate to apply migrations..."
django-admin migrate --noinput 2>&1
echo "Clearing django cache..."
echo "*** Clearing django cache..."
django-admin clear_cache 2>&1
echo "Running collectstatic..."
echo "*** Running collectstatic..."
django-admin collectstatic -c --no-input 2>&1
echo "Deployed successfully!"
echo "*** Deployed successfully!"

View file

@ -0,0 +1,25 @@
#!/bin/sh
CERT_PATH="/certs/crt.pem"
CA_PATH="/certs/ca.pem"
# Create the directory if it doesn't exist
mkdir -p "$(dirname "$CERT_PATH")" "$(dirname "$CA_PATH")"
if [ -n "$HAPROXY_PEM_CERT" ]; then
printf "%s" "$HAPROXY_PEM_CERT" > "$CERT_PATH"
chmod 600 "$CERT_PATH"
echo "HAProxy SSL certificate written to $CERT_PATH"
else
echo "Warning: HAPROXY_PEM_CERT environment variable is not set. SSL may not be configured."
fi
if [ -n "$HAPROXY_PEM_CA" ]; then
printf "%s" "$HAPROXY_PEM_CA" > "$CA_PATH"
chmod 600 "$CA_PATH" # Set restrictive permissions
echo "HAProxy SSL CA written to $CA_PATH"
else
echo "Warning: HAPROXY_PEM_CA environment variable is not set. SSL may not be configured."
fi
exec /usr/local/bin/docker-entrypoint.sh "$@"

View file

@ -7,6 +7,6 @@ if [ -d "staticfiles" ]; then
fi
# Build the tailwind theme css
cd theme/static_src
cd src/pkmntrade_club/theme/static_src
npm install . && npm run build
cd ../../

View file

@ -2,18 +2,23 @@
# Exit immediately if any command exits with a non-zero status.
set -e
# Reset the database and migrations.
echo "Resetting database and migrations... "
echo "Remaking migrations..."
docker compose up -d
find . -path "*/migrations/0*.py" -delete
docker compose exec -it web bash -c "django-admin makemigrations --noinput"
echo "Resetting database... "
docker compose down \
&& docker volume prune -a --filter label=db_is_resettable_via_script \
&& find . -path "*/migrations/00*.py" -delete \
&& docker volume rm -f pkmntradeclub_postgres_data \
&& docker compose up -d
# Wait for the database to be ready.
echo "Waiting for the database to be ready..."
echo "Waiting for the database to be ready, and migrations to be autorun..."
sleep 10
echo "Loading seed data..."
docker compose exec -it web bash -c "django-admin loaddata /seed/0*"
echo "Done & Started!"
docker compose down
echo "Done!"

View file

@ -3,4 +3,4 @@ from django.apps import AppConfig
class AccountsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'accounts'
name = 'pkmntrade_club.accounts'

View file

@ -1,10 +1,10 @@
# Generated by Django 5.1 on 2025-05-09 01:49
# Generated by Django 5.1 on 2025-05-10 01:22
import accounts.models
import django.contrib.auth.models
import django.contrib.auth.validators
import django.db.models.deletion
import django.utils.timezone
import pkmntrade_club.accounts.models
from django.conf import settings
from django.db import migrations, models
@ -51,7 +51,7 @@ class Migration(migrations.Migration):
name='FriendCode',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('friend_code', models.CharField(max_length=19, validators=[accounts.models.validate_friend_code])),
('friend_code', models.CharField(max_length=19, validators=[pkmntrade_club.accounts.models.validate_friend_code])),
('in_game_name', models.CharField(max_length=14)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),

View file

@ -8,10 +8,10 @@ from django.urls import reverse
from django.core.exceptions import ValidationError
from django.contrib.sessions.middleware import SessionMiddleware
from accounts.models import FriendCode
from accounts.forms import FriendCodeForm, CustomUserCreationForm, UserSettingsForm
from accounts.templatetags import gravatar
from trades.models import TradeOffer
from pkmntrade_club.accounts.models import FriendCode
from pkmntrade_club.accounts.forms import FriendCodeForm, CustomUserCreationForm, UserSettingsForm
from pkmntrade_club.accounts.templatetags import gravatar
from pkmntrade_club.trades.models import TradeOffer
from tests.utils.rarity import RARITY_MAPPING
# Create your tests here.
@ -605,7 +605,7 @@ class TemplateTagTests(TestCase):
self.assertIn('img src="', result)
self.assertIn(f'width="{size}"', result)
@patch("accounts.templatetags.gravatar.requests.get")
@patch("pkmntrade_club.accounts.templatetags.gravatar.requests.get")
def test_gravatar_profile_data_success(self, mock_get):
"""Test that gravatar_profile_data returns the first entry when JSON response is valid."""
dummy_entry = {"name": "Test User"}
@ -616,7 +616,7 @@ class TemplateTagTests(TestCase):
data = gravatar.gravatar_profile_data("user@example.com")
self.assertEqual(data, dummy_entry)
@patch("accounts.templatetags.gravatar.requests.get")
@patch("pkmntrade_club.accounts.templatetags.gravatar.requests.get")
def test_gravatar_profile_data_failure(self, mock_get):
"""
If requests.get fails or the JSON is not valid,

View file

@ -3,13 +3,13 @@ from django.contrib.auth.mixins import LoginRequiredMixin
from django.urls import reverse_lazy
from django.shortcuts import redirect, get_object_or_404, render
from django.views.generic import ListView, CreateView, DeleteView, View, TemplateView, UpdateView
from accounts.models import FriendCode, CustomUser
from accounts.forms import FriendCodeForm, UserSettingsForm
from pkmntrade_club.accounts.models import FriendCode, CustomUser
from pkmntrade_club.accounts.forms import FriendCodeForm, UserSettingsForm
from django.db.models import Case, When, Value, BooleanField
from trades.models import TradeOffer, TradeAcceptance
from pkmntrade_club.trades.models import TradeOffer, TradeAcceptance
from django.core.exceptions import PermissionDenied
from trades.mixins import FriendCodeRequiredMixin
from common.mixins import ReusablePaginationMixin
from pkmntrade_club.trades.mixins import FriendCodeRequiredMixin
from pkmntrade_club.common.mixins import ReusablePaginationMixin
from django.urls import reverse
from django.utils.http import urlencode
@ -140,7 +140,7 @@ class DashboardView(LoginRequiredMixin, FriendCodeRequiredMixin, ReusablePaginat
def post(self, request, *args, **kwargs):
if 'update_settings' in request.POST:
from accounts.forms import UserSettingsForm
from pkmntrade_club.accounts.forms import UserSettingsForm
form = UserSettingsForm(request.POST, instance=request.user)
if form.is_valid():
form.save()
@ -314,7 +314,7 @@ class DashboardView(LoginRequiredMixin, FriendCodeRequiredMixin, ReusablePaginat
context["closed_acceptances_paginated"] = self.get_closed_acceptances_paginated(closed_acceptances_page)
context["rejected_by_me_paginated"] = self.get_rejected_by_me_paginated(rejected_by_me_page)
context["rejected_by_them_paginated"] = self.get_rejected_by_them_paginated(rejected_by_them_page)
from accounts.forms import UserSettingsForm
from pkmntrade_club.accounts.forms import UserSettingsForm
context["settings_form"] = UserSettingsForm(instance=request.user)
context["active_tab"] = request.GET.get("tab", "dash")
return context

View file

@ -2,7 +2,7 @@ from django.apps import AppConfig
class CardsConfig(AppConfig):
name = "cards"
name = "pkmntrade_club.cards"
def ready(self):
import cards.signals
import pkmntrade_club.cards.signals

View file

@ -1,4 +1,4 @@
# Generated by Django 5.1 on 2025-05-09 01:49
# Generated by Django 5.1 on 2025-05-10 01:22
import django.db.models.deletion
from django.db import migrations, models

View file

@ -1,6 +1,6 @@
import uuid
from django import template
from cards.models import Card
from pkmntrade_club.cards.models import Card
from django.db.models.query import QuerySet
import json
import hashlib

View file

@ -5,10 +5,10 @@ from datetime import timedelta
from django.urls import reverse
from django.utils import timezone
from accounts.models import CustomUser, FriendCode
from cards.models import Card, Deck, DeckNameTranslation, CardNameTranslation
from trades.models import TradeOffer, TradeOfferHaveCard, TradeOfferWantCard
from cards.templatetags import card_badge, card_multiselect
from pkmntrade_club.accounts.models import CustomUser, FriendCode
from pkmntrade_club.cards.models import Card, Deck, DeckNameTranslation, CardNameTranslation
from pkmntrade_club.trades.models import TradeOffer, TradeOfferHaveCard, TradeOfferWantCard
from pkmntrade_club.cards.templatetags import card_badge, card_multiselect
from tests.utils.rarity import RARITY_MAPPING
class CardsModelsTests(TestCase):

View file

@ -1,9 +1,9 @@
from django.views.generic import TemplateView
from django.urls import reverse_lazy
from django.views.generic import UpdateView, DeleteView, CreateView, ListView, DetailView
from cards.models import Card
from trades.models import TradeOffer
from common.mixins import ReusablePaginationMixin
from pkmntrade_club.cards.models import Card
from pkmntrade_club.trades.models import TradeOffer
from pkmntrade_club.common.mixins import ReusablePaginationMixin
from django.views import View
from django.shortcuts import get_object_or_404, render

View file

@ -0,0 +1,8 @@
from django.apps import AppConfig
class CommonConfig(AppConfig):
name = "pkmntrade_club.common"
def ready(self):
pass

View file

@ -2,6 +2,6 @@ import os
from django.core.asgi import get_asgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_project.settings")
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pkmntrade_club.django_project.settings')
application = get_asgi_application()

View file

@ -7,6 +7,8 @@ class LogRequestsMiddleware:
self.get_response = get_response
def __call__(self, request):
if request.path == "/health/":
return self.get_response(request)
start = time.perf_counter()
response = self.get_response(request)
end = time.perf_counter()

View file

@ -70,12 +70,12 @@ ALLOWED_HOSTS = env('ALLOWED_HOSTS').split(',')
CSRF_TRUSTED_ORIGINS = env('CSRF_TRUSTED_ORIGINS').split(',')
FIRST_PARTY_APPS = [
"theme",
"common",
"accounts",
"cards",
"home",
"trades"
'pkmntrade_club.accounts',
'pkmntrade_club.cards',
'pkmntrade_club.common',
'pkmntrade_club.home',
'pkmntrade_club.theme',
'pkmntrade_club.trades',
]
# Application definition
@ -122,7 +122,7 @@ MIDDLEWARE = [
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"allauth.account.middleware.AccountMiddleware", # django-allauth
"django_project.middleware.LogRequestsMiddleware",
"pkmntrade_club.django_project.middleware.LogRequestsMiddleware",
]
if DEBUG:
@ -136,10 +136,12 @@ DAISY_SETTINGS = {
}
# https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = "django_project.urls"
ROOT_URLCONF = 'pkmntrade_club.django_project.urls'
# https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = "django_project.wsgi.application"
WSGI_APPLICATION = 'pkmntrade_club.django_project.wsgi.app'
ASGI_APPLICATION = 'pkmntrade_club.django_project.asgi.application'
# https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES = [
@ -153,7 +155,7 @@ TEMPLATES = [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"common.context_processors.cache_settings",
"pkmntrade_club.common.context_processors.cache_settings",
],
},
},

View file

@ -5,10 +5,10 @@ from django.urls import path, include
urlpatterns = [
path("admin/", admin.site.urls),
path("accounts/", include("allauth.urls")),
path("", include("home.urls")),
path("cards/", include("cards.urls")),
path('account/', include('accounts.urls')),
path("trades/", include("trades.urls")),
path("", include("pkmntrade_club.home.urls")),
path("cards/", include("pkmntrade_club.cards.urls")),
path('account/', include('pkmntrade_club.accounts.urls')),
path("trades/", include("pkmntrade_club.trades.urls")),
path("__reload__/", include("django_browser_reload.urls")),
]

View file

@ -0,0 +1,7 @@
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pkmntrade_club.django_project.settings")
app = get_wsgi_application()

View file

@ -2,4 +2,4 @@ from django.apps import AppConfig
class HomeConfig(AppConfig):
name = "home"
name = "pkmntrade_club.home"

View file

@ -1,10 +1,10 @@
from django.test import TestCase, Client, RequestFactory
from django.urls import reverse
from django.contrib.auth import get_user_model
from cards.models import Card, Deck
from trades.models import TradeOffer, TradeOfferHaveCard, TradeOfferWantCard
from accounts.models import FriendCode
from home.views import HomePageView
from pkmntrade_club.cards.models import Card, Deck
from pkmntrade_club.trades.models import TradeOffer, TradeOfferHaveCard, TradeOfferWantCard
from pkmntrade_club.accounts.models import FriendCode
from pkmntrade_club.home.views import HomePageView
import json
from collections import OrderedDict
from unittest.mock import patch, MagicMock

View file

@ -4,8 +4,8 @@ from django.urls import reverse_lazy
from django.db.models import Count, Q, Prefetch, Sum, F, IntegerField, Value, BooleanField, Case, When
from django.db.models.functions import Coalesce
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from trades.models import TradeOffer, TradeAcceptance, TradeOfferHaveCard, TradeOfferWantCard
from cards.models import Card
from pkmntrade_club.trades.models import TradeOffer, TradeAcceptance, TradeOfferHaveCard, TradeOfferWantCard
from pkmntrade_club.cards.models import Card
from django.utils.decorators import method_decorator
from django.template.response import TemplateResponse
from django.http import HttpResponseRedirect
@ -147,7 +147,7 @@ class HealthCheckView(View):
return HttpResponse("Database connection failed", status=500)
try:
from trades.models import TradeOffer
from pkmntrade_club.trades.models import TradeOffer
with contextlib.redirect_stdout(None):
print(TradeOffer.objects.count())
except Exception as e:
@ -161,5 +161,4 @@ class HealthCheckView(View):
except Exception as e:
return HttpResponse("Cache not reachable", status=500)
logger.info("OK/HEALTHY")
return HttpResponse("OK/HEALTHY")

View file

Before

Width:  |  Height:  |  Size: 549 B

After

Width:  |  Height:  |  Size: 549 B

Before After
Before After

View file

@ -2,4 +2,4 @@ from django.apps import AppConfig
class ThemeConfig(AppConfig):
name = 'theme'
name = 'pkmntrade_club.theme'

Some files were not shown because too many files have changed in this diff Show more