Major refactoring of build_deploy action, along with docker building and packaging improvements. Added no_signups and other .env improvements. There is no longer a separate .env.dev, both use .env now.

.github/workflows/build.yml

@@ -1,62 +0,0 @@
-name: build-image-from-tag
-on:
-  workflow_dispatch:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-jobs:
-  build:
-    # Don't build the image if the registry credentials are not set, the ref is not a tag or it doesn't contain '-v'
-    if: ${{ vars.REGISTRY_USER != '' && secrets.REGISTRY_PASS != '' && startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-v') }}
-    runs-on: docker
-    container:
-      image: git.badblocks.dev/oci/pkmntrade-club_web:latest
-      # Mount the dind socket on the container at the default location
-      options: -v /dind/docker.sock:/var/run/docker.sock
-    steps:
-      - name: Extract image name and tag from git and get registry name from env
-        id: job_data
-        run: |
-          echo "::set-output name=img_name::${GITHUB_REF_NAME%%-v*}"
-          echo "::set-output name=img_tag::${GITHUB_REF_NAME##*-v}"
-          echo "::set-output name=registry::$(
-            echo "${{ github.server_url }}" | sed -e 's%https://%%'
-          )"
-          echo "::set-output name=oci_registry_prefix::$(
-            echo "${{ github.server_url }}/oci" | sed -e 's%https://%%'
-          )"
-      - name: Checkout the repo
-        uses: actions/checkout@v4
-      - name: Export build dir and Dockerfile
-        id: build_data
-        run: |
-          img="${{ steps.job_data.outputs.img_name }}"
-          build_dir="$(pwd)/${img}"
-          dockerfile="${build_dir}/Dockerfile"
-          if [ -f "$dockerfile" ]; then
-            echo "::set-output name=build_dir::$build_dir"
-            echo "::set-output name=dockerfile::$dockerfile"
-          else
-            echo "Couldn't find the Dockerfile for the '$img' image"
-            exit 1
-          fi
-      - name: Login to the Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ steps.job_data.outputs.registry }}
-          username: ${{ vars.REGISTRY_USER }}
-          password: ${{ secrets.REGISTRY_PASS }}
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Build and Push
-        uses: docker/build-push-action@v6
-        with:
-          push: true
-          tags: |
-            ${{ steps.job_data.outputs.oci_registry_prefix }}/${{ steps.job_data.outputs.img_name }}:${{ steps.job_data.outputs.img_tag }}
-            ${{ steps.job_data.outputs.oci_registry_prefix }}/${{ steps.job_data.outputs.img_name }}:latest
-          context: ${{ steps.build_data.outputs.build_dir }}
-          file: ${{ steps.build_data.outputs.dockerfile }}
-          build-args: |
-            OCI_REGISTRY_PREFIX=${{ steps.job_data.outputs.oci_registry_prefix }}/

.github/workflows/build_deploy.yml

@@ -0,0 +1,308 @@
+name: Build & Deploy
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+      - name: Get full and partial repository name
+        id: meta
+        run: |
+          echo "GITHUB_REPOSITORY: ${{ github.repository }}"
+          if [[ "${{ github.repository }}" == *".git" ]]; then
+            if [[ "${{ github.repository }}" == "https://"* ]]; then
+              echo "GITHUB_REPOSITORY ends in .git and is a URL"
+              REPO=$(echo "${{ github.repository }}" | sed 's/\.git$//' | cut -d'/' -f4-5 | sed 's/[^a-zA-Z0-9\/-]/-/g')
+            else
+              echo "GITHUB_REPOSITORY ends in .git and is not a URL"
+              REPO=$(echo "${{ github.repository }}" | sed 's/\.git$//' | sed 's/[^a-zA-Z0-9\/-]/-/g')
+            fi
+          else
+            echo "GITHUB_REPOSITORY is not a URL"
+            REPO=$(echo "${{ github.repository }}" | sed 's/[^a-zA-Z0-9\/-]/-/g')
+          fi
+          echo "REPO=$REPO" >> $GITHUB_OUTPUT
+
+          REPO_NAME_ONLY=$(echo "$REPO" | cut -d'/' -f2)
+          echo "REPO_NAME_ONLY=$REPO_NAME_ONLY" >> $GITHUB_OUTPUT
+
+          REPO_PROJECT_PATH=/srv/$(echo "$REPO_NAME_ONLY")
+          echo "REPO_PROJECT_PATH=$REPO_PROJECT_PATH" >> $GITHUB_OUTPUT
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set PROD environment variable
+        run: |
+          echo "✅ Exit script on any error"
+          set -eu -o pipefail
+
+          prod_value=""
+
+          echo "🔍 Check if PROD is set via vars; if not, determine from github.ref"
+          if [ -z "${{ vars.PROD }}" ]; then
+            prod_value="${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-prerelease') }}"
+            echo "📦 PROD mode unset, determined from github.ref (starts with v and does not end with -prerelease?): ${prod_value}"
+          else
+            prod_value="${{ vars.PROD }}"
+            echo "📦 PROD mode already set to: ${prod_value}"
+          fi
+
+          echo "🖊️ Writing determined values to GITHUB_ENV:"
+          echo "PROD=${prod_value}" >> $GITHUB_ENV
+          echo "PROD=${prod_value} -> GITHUB_ENV"
+      - name: Generate tags
+        id: generated_docker_tags
+        run: |
+          echo "✅ Exit script on any error"
+          set -eu -o pipefail
+
+          # echo current shell
+          echo "Current shell: $SHELL"
+
+          IMAGE_BASE_NAME="${{ steps.meta.outputs.REPO }}"
+          GIT_SHA="${{ github.sha }}"
+          GIT_REF="${{ github.ref }}"
+
+          echo "Inputs for tagging:"
+          echo "IMAGE_BASE_NAME: $IMAGE_BASE_NAME"
+          echo "GIT_SHA: $GIT_SHA"
+          echo "GIT_REF: $GIT_REF"
+          echo "PROD status for tagging: ${PROD}"
+
+          TAG_LIST=()
+          VERSION_TAG_LIST=()
+
+          if [[ -n "$GIT_SHA" ]]; then
+            SHORT_SHA=$(echo "$GIT_SHA" | cut -c1-7)
+            TAG_LIST+=("${IMAGE_BASE_NAME}:sha-${SHORT_SHA}")
+            TAG_LIST+=("${IMAGE_BASE_NAME}:sha-${GIT_SHA}")
+          else
+            echo "🔴 No Git SHA found, cannot generate tags. Aborting."
+            exit 1
+          fi
+
+          GIT_TAG_VERSION=""
+          # Extract version only if GIT_REF is a tag like refs/tags/vX.Y.Z or refs/tags/vX.Y.Z-prerelease
+          if [[ "$GIT_REF" == refs/tags/v* ]]; then
+            GIT_TAG_VERSION=$(echo "$GIT_REF" | sed 's%refs/tags/v%%' | sed 's%-prerelease$%%')
+
+            if [[ "$GIT_TAG_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
+              echo "Detected Git tag: v$GIT_TAG_VERSION"
+              MAJOR=${BASH_REMATCH[1]}
+              MINOR=${BASH_REMATCH[2]}
+              PATCH=${BASH_REMATCH[3]}
+              echo "Parsed version: Major=$MAJOR, Minor=$MINOR, Patch=$PATCH from v$GIT_TAG_VERSION"
+
+              if [ "$MAJOR" -gt 0 ]; then
+                VERSION_TAG_LIST+=("${IMAGE_BASE_NAME}:v${MAJOR}")
+              else
+                echo "ℹ️ Major version is 0 (v$GIT_TAG_VERSION). Skipping MAJOR-only tag v0. Please reference by MAJOR.MINOR or MAJOR.MINOR.PATCH."
+              fi
+              
+              VERSION_TAG_LIST+=("${IMAGE_BASE_NAME}:v${MAJOR}.${MINOR}")
+              VERSION_TAG_LIST+=("${IMAGE_BASE_NAME}:v${MAJOR}.${MINOR}.${PATCH}")
+            else
+              echo "⚠️ Git tag 'v$GIT_TAG_VERSION' is not a valid semantic version (x.y.z) but should be. Aborting."
+              exit 1
+            fi
+          fi
+
+          if [ "$PROD" = "true" ]; then
+            echo "📦 Generating PROD tags."
+            TAG_LIST+=("${IMAGE_BASE_NAME}:stable")
+            TAG_LIST+=("${IMAGE_BASE_NAME}:latest")
+
+            if [ ${#VERSION_TAG_LIST[@]} -gt 0 ]; then
+              TAG_LIST+=("${VERSION_TAG_LIST[@]}")
+            else
+              echo "🔴 PROD mode is true, but Git ref ($GIT_REF) is not a valid version tag. This is unexpected, aborting."
+              exit 1
+            fi
+          else # Non-PROD
+            echo "🔨 Generating STAGING tags."
+            TAG_LIST+=("${IMAGE_BASE_NAME}:staging")
+            TAG_LIST+=("${IMAGE_BASE_NAME}:latest-staging")
+
+            if [ ${#VERSION_TAG_LIST[@]} -gt 0 ]; then
+              echo "🔨 This is also a prerelease version, generating version tags with '-prerelease' suffix."
+              VERSION_TAG_LIST=("${VERSION_TAG_LIST[@]/%/-prerelease}")
+
+              TAG_LIST+=("${VERSION_TAG_LIST[@]}")
+            else
+              echo "ℹ️ Git ref ($GIT_REF) is not a valid version tag. Skipping versioned -prerelease tag generation."
+            fi
+          fi
+
+          echo "Final list of generated tags:"
+          printf "%s\n" "${TAG_LIST[@]}"
+
+          if [[ -z "$GIT_SHA" || ${#TAG_LIST[@]} -lt 4 ]]; then
+            echo "⚠️ No tags (or too few) were generated based on the logic. Need at least 4 tags: Git commit short and full length SHA tags, a latest/latest-staging tag, and a stable/staging tag. This is unexpected, aborting."
+            exit 1
+          fi
+
+          # Output the tags for the docker build action (output name is 'tag')
+          {
+            echo "tag<<EOF"
+            printf "%s\n" "${TAG_LIST[@]}"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+      - name: Run prebuild tasks
+        run: |
+          echo "🔄 Chdir to src/pkmntrade_club/"
+          cd src/pkmntrade_club/
+
+          echo "🔄 Chdir to theme/static_src"
+          cd theme/static_src
+
+          echo "📦 Install npm dependencies"
+          npm install .
+
+          echo "🔨 Build the tailwind theme css"
+          npm run build
+      # - name: Cache Docker layers
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: ${{ runner.temp }}/.cache
+      #     key: ${{ runner.os }}-${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}-${{ github.sha }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}-
+      # - name: Inject buildx-cache
+      #   uses: reproducible-containers/buildkit-cache-dance@4b2444fec0c0fb9dbf175a96c094720a692ef810 # v2.1.4
+      #   with:
+      #     cache-source: ${{ runner.temp }}/.cache/buildx-cache
+      - name: Build container
+        uses: docker/build-push-action@v6
+        with:
+          outputs: type=docker,dest=${{ runner.temp }}/${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar
+          tags: ${{ steps.generated_docker_tags.outputs.tag }}
+          build-args: CACHE_DIR=${{ runner.temp }}/.cache/dockerfile-cache
+          context: .
+          #cache-from: type=local,src=${{ runner.temp }}/.cache/buildx-cache
+          #cache-to: type=local,src=${{ runner.temp }}/.cache/buildx-cache-new,mode=max
+      # - name: Rotate cache # along with cache-from & cache-to: prevents cache from growing indefinitely
+      #   run: |
+      #     rm -rf ${{ runner.temp }}/.cache/buildx-cache
+      #     mv ${{ runner.temp }}/.cache/buildx-cache-new ${{ runner.temp }}/.cache/buildx-cache
+      # - name: Upload container as artifact
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: ${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar
+      #     path: ${{ runner.temp }}/${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar
+      #     if-no-files-found: error
+      #     compression-level: 0
+      - name: Get Deploy Secrets
+        uses: bitwarden/sm-action@v2
+        with:
+          access_token: ${{ secrets.BW_ACCESS_TOKEN }}
+          secrets: |
+            27a8da8d-5fb4-4f58-baaf-b2dd0032eca8 > DEPLOY_HOST
+            4cf9ab8d-1772-4ab0-9219-b2dd003315d4 > DEPLOY_KEY
+            9aefe34e-c2cf-442e-973c-b2dd0032b6cf > ENV_FILE_BASE64
+            d3bb47f8-bfc0-4a61-9cee-b2df0147a02a > CF_PEM_CERT
+            5f658ddf-aadd-4464-b501-b2df0147c338 > CF_PEM_CA
+      - name: Set up SSH
+        run: |
+          mkdir -p $HOME/.ssh
+          echo -e "${DEPLOY_KEY}" > $HOME/.ssh/deploy.key
+          chmod 700 $HOME/.ssh
+          chmod 600 $HOME/.ssh/deploy.key
+
+          cat >>$HOME/.ssh/config <<END
+          Host deploy
+              HostName ${DEPLOY_HOST}
+              Port ${{ vars.DEPLOY_PORT }}
+              User ${{ vars.DEPLOY_USER }}
+              IdentityFile $HOME/.ssh/deploy.key
+              UserKnownHostsFile /dev/null
+              StrictHostKeyChecking no
+              ControlMaster auto
+              ControlPath $HOME/.ssh/control-%C
+              ControlPersist yes
+          END
+      - name: Run Deploy Script
+        run: |
+          echo "✅ Exit script on any error"
+          set -eu -o pipefail
+
+          echo "⚙️ Set docker host to ssh://deploy so that all docker commands are run on the remote server"
+          export DOCKER_HOST=ssh://deploy          
+
+          echo "🚀 Enable and start docker service"
+          ssh deploy "sudo systemctl enable --now docker.service"
+
+          echo "💾 Load the new docker image (${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar)"
+          docker load -i "${{ runner.temp }}/${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar"
+
+          echo "🧹 Remove the docker image artifact"
+          rm "${{ runner.temp }}/${{ steps.meta.outputs.REPO_NAME_ONLY }}-${{ github.ref_name }}_${{ github.sha }}.tar"
+
+          echo "🛑 Stop and remove containers before updating compose files"
+          #ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_core.yml down"
+          if [ "${PROD}" = true ]; then
+            ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_web.yml down"
+          else
+            ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}} && docker compose -f docker-compose_staging.yml down"
+          fi
+
+          echo "💾 Copy files to server"
+          ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}"
+          scp -pr ./server/* deploy:${{ steps.meta.outputs.REPO_PROJECT_PATH}}/
+
+          echo "📝 Create .env file"
+          printf "%s" "${ENV_FILE_BASE64}" | base64 -d | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/.env && chmod 600 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/.env"
+
+          echo "🔑 Set up certificates"
+          ssh deploy "mkdir -p ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs && chmod 550 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs"
+          printf "%s" "$CF_PEM_CERT" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/crt.pem"
+          printf "%s" "$CF_PEM_CA" | ssh deploy "cat > ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem && chmod 440 ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem && chown 99:root ${{ steps.meta.outputs.REPO_PROJECT_PATH}}/certs/ca.pem"         
+
+          echo "🚀 Start the new containers"
+          if [ "${PROD}" = true ]; then
+            ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH }} && docker compose -f docker-compose_core.yml -f docker-compose_web.yml up -d --no-build"
+          else
+            ssh deploy "cd ${{ steps.meta.outputs.REPO_PROJECT_PATH }} && docker compose -f docker-compose_core.yml -f docker-compose_staging.yml up -d --no-build"
+          fi
+
+          # echo "🚀 Start the new containers"
+          # if [ "${PROD}" = true ]; then
+          #   ssh deploy <<<END
+          #     cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}}
+          #     old_container_id=$(docker compose -f docker-compose_web.yml ps -f name=web -q | tail -n1)
+          #     docker compose -f docker-compose_web.yml up -d --no-build --no-recreate
+          #     new_container_id=$(docker compose -f docker-compose_web.yml ps -f name=web -q | head -n1)
+          #     # not needed, but might be useful at some point
+          #     #new_container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $new_container_id)
+          #     #new_container_name=$(docker inspect -f '{{.Name}}' $new_container_id | cut -c2-) 
+          #     sleep 100 # change to wait for healthcheck in the future
+          #     #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
+          #     docker stop $old_container_id
+          #     docker rm $old_container_id
+          #     #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
+          #   END
+          # else
+          #   ssh deploy <<<END
+          #     cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}}
+          #     old_container_id=$(docker compose -f docker-compose_staging.yml ps -f name=web-staging -q | tail -n1)
+          #     docker compose -f docker-compose_staging.yml up -d --no-build --no-recreate
+          #     new_container_id=$(docker compose -f docker-compose_staging.yml ps -f name=web-staging -q | head -n1)
+          #     # not needed, but might be useful at some point
+          #     #new_container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $new_container_id)
+          #     #new_container_name=$(docker inspect -f '{{.Name}}' $new_container_id | cut -c2-) 
+          #     sleep 100 # change to wait for healthcheck in the future
+          #     #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
+          #     docker stop $old_container_id
+          #     docker rm $old_container_id
+          #     #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
+          #   END
+          # fi
+
+          echo "🧹 Prune all unused images"
+          docker system prune -f

.github/workflows/test.yml

@@ -1,20 +1,19 @@
-name: Build, Test, and Deploy
+name: Test - Step 2
 
 on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
+  workflow_dispatch:
+  # workflow_run:
+  #   workflows: ["Build - Step 1"]
+  #   types:
+  #     - completed
 
 jobs:
-
-  build:
-
+  test-step-2:
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [ 3.13 ]
+        python-version: [ 3.12 ]
         database-name:
           - test_db
         database-password:
@@ -41,8 +40,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
-
     steps:
     - uses: actions/checkout@v2.4.0
     - name: Set up Python ${{ matrix.python-version }}
@@ -100,20 +97,7 @@ jobs:
           python manage.py test
       env:
         DATABASE_URL: postgres://${{ matrix.database-user }}:${{ matrix.database-password }}@${{ matrix.database-host }}:${{ matrix.database-port }}/${{ matrix.database-name }}
-        SECRET_KEY: test-secret-key
         DEBUG: 1
         ALLOWED_HOSTS: localhost
         GITHUB_WORKFLOW: True
         MODE: workflow
-    - uses: actions/checkout@v2.4.0
-    - name: Build the images and start the containers
-      run: |
-        export GITHUB_WORKFLOW=True
-        export MODE="Test"
-        docker-compose -f docker-compose.yml build
-        docker-compose -f docker-compose.yml up -d
-        # run: docker-compose up -d --build
-    - name: Stop containers
-      if: always()
-      run: docker-compose -f "docker-compose.yml" down
-