feat: add dynamic versioning and automated deployment with rollback capability

- Implement setuptools-scm for dynamic version management from git tags
- Refactor CI/CD into separate build and deploy jobs with artifact sharing
- Add versioned releases with timestamp-based deployment directories
- Implement health checks and automatic rollback on deployment failure
- Extract deployment logic into reusable shell scripts
- Add Docker layer caching to speed up builds
- Include version info in Django context and build args
This commit is contained in:
badblocks 2025-06-06 14:38:23 -07:00
parent 46619bd5e1
commit f20c4f9474
No known key found for this signature in database
14 changed files with 719 additions and 233 deletions

124
scripts/deploy-to-server.sh Normal file
View file

@ -0,0 +1,124 @@
#!/bin/bash
set -euo pipefail
# Main deployment script with versioned releases
# Usage: ./deploy-to-server.sh
# Source retry function
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/retry.sh"
# Required environment variables (should be set by GitHub Actions)
: "${DOCKER_HOST:?Error: DOCKER_HOST not set}"
: "${REPO_PROJECT_PATH:?Error: REPO_PROJECT_PATH not set}"
: "${REPO_NAME_ONLY:?Error: REPO_NAME_ONLY not set}"
: "${IMAGE_TAR:?Error: IMAGE_TAR not set}"
: "${ENV_FILE_BASE64:?Error: ENV_FILE_BASE64 not set}"
: "${CF_PEM_CERT:?Error: CF_PEM_CERT not set}"
: "${CF_PEM_CA:?Error: CF_PEM_CA not set}"
: "${IS_PROD:?Error: IS_PROD not set}"
echo "⚙️ Docker host: $DOCKER_HOST"
# Generate deployment timestamp
DEPLOYMENT_TIMESTAMP=$(date +%Y%m%d_%H%M%S)
RELEASES_PATH="${REPO_PROJECT_PATH}/releases"
NEW_RELEASE_PATH="${RELEASES_PATH}/${DEPLOYMENT_TIMESTAMP}"
CURRENT_LINK_PATH="${REPO_PROJECT_PATH}/current"
echo "📅 Deployment version: ${DEPLOYMENT_TIMESTAMP}"
echo "🚀 Enable and start docker service"
retry ssh deploy "sudo systemctl enable --now docker.service"
echo "💾 Load the new docker image ($IMAGE_TAR)"
if [ ! -f "$IMAGE_TAR" ]; then
echo "Error: Docker image tar file not found: $IMAGE_TAR"
exit 1
fi
retry docker load -i "$IMAGE_TAR"
echo "📁 Create versioned release directory"
ssh deploy "mkdir -p '${NEW_RELEASE_PATH}'"
echo "💾 Copy new files to server"
# Check if server directory exists before copying
if [ -d "./server" ]; then
retry scp -pr ./server/* "deploy:${NEW_RELEASE_PATH}/"
else
echo "⚠️ No server directory found, erroring out"
exit 1
fi
echo "📝 Create new .env file"
printf "%s" "${ENV_FILE_BASE64}" | base64 -d | ssh deploy "cat > '${NEW_RELEASE_PATH}/.env' && chmod 600 '${NEW_RELEASE_PATH}/.env'"
echo "🔑 Set up certs"
ssh deploy "mkdir -p '${NEW_RELEASE_PATH}/certs' && chmod 550 '${NEW_RELEASE_PATH}/certs' && chown 99:root '${NEW_RELEASE_PATH}/certs'"
printf "%s" "$CF_PEM_CERT" | ssh deploy "cat > '${NEW_RELEASE_PATH}/certs/crt.pem' && chmod 440 '${NEW_RELEASE_PATH}/certs/crt.pem' && chown 99:root '${NEW_RELEASE_PATH}/certs/crt.pem'"
printf "%s" "$CF_PEM_CA" | ssh deploy "cat > '${NEW_RELEASE_PATH}/certs/ca.pem' && chmod 440 '${NEW_RELEASE_PATH}/certs/ca.pem' && chown 99:root '${NEW_RELEASE_PATH}/certs/ca.pem'"
echo "🔄 Prepare deployment (stop current containers)"
# Copy script to remote and execute with parameters
scp "${SCRIPT_DIR}/prepare-deployment.sh" deploy:/tmp/
ssh deploy "chmod +x /tmp/prepare-deployment.sh && /tmp/prepare-deployment.sh '${REPO_PROJECT_PATH}' '${IS_PROD}' '${CURRENT_LINK_PATH}'"
ssh deploy "rm -f /tmp/prepare-deployment.sh"
echo "📝 Save deployment metadata"
ssh deploy "echo '${DEPLOYMENT_TIMESTAMP}' > '${NEW_RELEASE_PATH}/.deployment_version'"
ssh deploy "echo '${IS_PROD}' > '${NEW_RELEASE_PATH}/.deployment_env'"
# Save previous version info for potential rollback
ssh deploy "if [ -L '${CURRENT_LINK_PATH}' ]; then readlink -f '${CURRENT_LINK_PATH}' > '${NEW_RELEASE_PATH}/.previous_version'; fi"
echo "🔗 Update current symlink to new release"
ssh deploy "ln -sfn '${NEW_RELEASE_PATH}' '${CURRENT_LINK_PATH}'"
# TODO: implement zero-downtime deployment
# echo "🚀 Start the new containers, zero-downtime"
# if [ "${PROD}" = true ]; then
# ssh deploy <<EOF
# cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}}
# old_container_id=$(docker compose -f docker-compose_web.yml ps -f name=web -q | tail -n1)
# docker compose -f docker-compose_web.yml up -d --no-build --no-recreate
# new_container_id=$(docker compose -f docker-compose_web.yml ps -f name=web -q | head -n1)
# # not needed, but might be useful at some point
# #new_container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $new_container_id)
# #new_container_name=$(docker inspect -f '{{.Name}}' $new_container_id | cut -c2-)
# sleep 100 # change to wait for healthcheck in the future
# #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
# docker stop $old_container_id
# docker rm $old_container_id
# #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
# EOF
# else
# ssh deploy <<EOF
# cd ${{ steps.meta.outputs.REPO_PROJECT_PATH}}
# old_container_id=$(docker compose -f docker-compose_staging.yml ps -f name=web-staging -q | tail -n1)
# docker compose -f docker-compose_staging.yml up -d --no-build --no-recreate
# new_container_id=$(docker compose -f docker-compose_staging.yml ps -f name=web-staging -q | head -n1)
# # not needed, but might be useful at some point
# #new_container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $new_container_id)
# #new_container_name=$(docker inspect -f '{{.Name}}' $new_container_id | cut -c2-)
# sleep 100 # change to wait for healthcheck in the future
# #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
# docker stop $old_container_id
# docker rm $old_container_id
# #docker compose -f docker-compose_core.yml kill -s SIGUSR2 loba
# EOF
# fi
echo "🚀 Start the new containers"
if [ "$IS_PROD" = "true" ]; then
retry ssh deploy "cd '${CURRENT_LINK_PATH}' && docker compose -f docker-compose_core.yml -f docker-compose_web.yml up -d --no-build"
else
retry ssh deploy "cd '${CURRENT_LINK_PATH}' && docker compose -f docker-compose_core.yml -f docker-compose_web.yml -f docker-compose_staging.yml up -d --no-build"
fi
echo "🧹 Prune unused Docker resources"
ssh deploy "docker system prune -f"
echo "🗑️ Clean up old releases (keep last 5)"
ssh deploy "cd '${RELEASES_PATH}' && ls -dt */ 2>/dev/null | tail -n +6 | xargs -r rm -rf || true"
echo "✅ Deployment completed. Version: ${DEPLOYMENT_TIMESTAMP}"

View file

@ -0,0 +1,49 @@
#!/bin/bash
set -euo pipefail
# Generate Docker tags based on git ref and environment
# Usage: ./generate-docker-tags.sh IMAGE_BASE GIT_SHA GIT_REF IS_PROD
if [ $# -ne 4 ]; then
echo "Error: Invalid number of arguments"
echo "Usage: $0 IMAGE_BASE GIT_SHA GIT_REF IS_PROD"
exit 1
fi
IMAGE_BASE="$1"
GIT_SHA="$2"
GIT_REF="$3"
IS_PROD="$4"
# Validate inputs
if [ -z "$IMAGE_BASE" ] || [ -z "$GIT_SHA" ]; then
echo "Error: IMAGE_BASE and GIT_SHA cannot be empty"
exit 1
fi
# Always include SHA tags
echo "${IMAGE_BASE}:sha-${GIT_SHA:0:7}"
echo "${IMAGE_BASE}:sha-${GIT_SHA}"
# Handle version tags
if [[ "$GIT_REF" =~ ^refs/tags/v([0-9]+)\.([0-9]+)\.([0-9]+)(-.*)?$ ]]; then
MAJOR="${BASH_REMATCH[1]}"
MINOR="${BASH_REMATCH[2]}"
PATCH="${BASH_REMATCH[3]}"
PRERELEASE="${BASH_REMATCH[4]}"
if [[ -z "$PRERELEASE" ]] && [[ "$IS_PROD" == "true" ]]; then
echo "${IMAGE_BASE}:latest"
echo "${IMAGE_BASE}:stable"
[[ "$MAJOR" -gt 0 ]] && echo "${IMAGE_BASE}:v${MAJOR}"
echo "${IMAGE_BASE}:v${MAJOR}.${MINOR}"
echo "${IMAGE_BASE}:v${MAJOR}.${MINOR}.${PATCH}"
else
echo "${IMAGE_BASE}:latest-staging"
echo "${IMAGE_BASE}:staging"
echo "${IMAGE_BASE}:v${MAJOR}.${MINOR}.${PATCH}-prerelease"
fi
elif [[ "$IS_PROD" == "false" ]]; then
echo "${IMAGE_BASE}:latest-staging"
echo "${IMAGE_BASE}:staging"
fi

View file

@ -0,0 +1,102 @@
#!/bin/bash
set -euo pipefail
# Perform health check and rollback if necessary
# Usage: ./health-check-and-rollback.sh REPO_PROJECT_PATH IS_PROD HEALTH_CHECK_URL [MAX_ATTEMPTS]
if [ $# -lt 3 ]; then
echo "Error: Invalid number of arguments"
echo "Usage: $0 REPO_PROJECT_PATH IS_PROD HEALTH_CHECK_URL [MAX_ATTEMPTS]"
exit 1
fi
REPO_PROJECT_PATH="$1"
IS_PROD="$2"
HEALTH_CHECK_URL="$3"
MAX_ATTEMPTS="${4:-30}"
CURRENT_LINK_PATH="${REPO_PROJECT_PATH}/current"
RELEASES_PATH="${REPO_PROJECT_PATH}/releases"
echo "🏥 Performing health check..."
echo "Health check URL: $HEALTH_CHECK_URL"
get_current_version() {
if [ -L "$CURRENT_LINK_PATH" ]; then
basename "$(readlink -f "$CURRENT_LINK_PATH")"
else
echo "unknown"
fi
}
ATTEMPT=0
while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do
# Check if the service is responding with 200 OK
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 10 "$HEALTH_CHECK_URL" || echo '000')
if [ "$HTTP_CODE" = "200" ]; then
echo "✅ Health check passed! (HTTP $HTTP_CODE)"
CURRENT_VERSION=$(get_current_version)
echo "📌 Current version: ${CURRENT_VERSION}"
exit 0
fi
ATTEMPT=$((ATTEMPT + 1))
if [ "$ATTEMPT" -eq "$MAX_ATTEMPTS" ]; then
echo "❌ Health check failed after $MAX_ATTEMPTS attempts (Last HTTP code: $HTTP_CODE)"
echo "🔄 Rolling back deployment..."
FAILED_VERSION=$(get_current_version)
echo "❌ Failed version: ${FAILED_VERSION}"
# Check if we have a previous version to roll back to
if [ -f "${CURRENT_LINK_PATH}/.previous_version" ]; then
PREVIOUS_VERSION_PATH=$(cat "${CURRENT_LINK_PATH}/.previous_version")
PREVIOUS_VERSION=$(basename "$PREVIOUS_VERSION_PATH")
if [ -d "$PREVIOUS_VERSION_PATH" ]; then
echo "🔄 Rolling back to version: ${PREVIOUS_VERSION}"
# Stop failed deployment containers
cd "$CURRENT_LINK_PATH"
echo "Stopping failed deployment containers..."
docker compose -f docker-compose_web.yml down || true
if [ "$IS_PROD" = "false" ]; then
docker compose -f docker-compose_staging.yml down || true
fi
docker compose -f docker-compose_core.yml down || true
# Switch symlink back to previous version
ln -sfn "$PREVIOUS_VERSION_PATH" "$CURRENT_LINK_PATH"
# Start previous version containers
cd "$CURRENT_LINK_PATH"
docker compose -f docker-compose_core.yml up -d --no-build
if [ "$IS_PROD" = "true" ]; then
docker compose -f docker-compose_web.yml up -d --no-build
else
docker compose -f docker-compose_web.yml -f docker-compose_staging.yml up -d --no-build
fi
echo "✅ Rollback completed to version: ${PREVIOUS_VERSION}"
# Mark failed version
if [ -d "${RELEASES_PATH}/${FAILED_VERSION}" ]; then
touch "${RELEASES_PATH}/${FAILED_VERSION}/.failed"
echo "$(date): Health check failed, rolled back to ${PREVIOUS_VERSION}" > "${RELEASES_PATH}/${FAILED_VERSION}/.failure_reason"
fi
else
echo "❌ Previous version directory not found: $PREVIOUS_VERSION_PATH"
exit 1
fi
else
echo "❌ No previous version information found. Cannot rollback!"
echo "💡 This might be the first deployment or the previous version info is missing."
exit 1
fi
exit 1
fi
echo "⏳ Waiting for service to be healthy... (attempt $ATTEMPT/$MAX_ATTEMPTS, HTTP code: $HTTP_CODE)"
sleep 10
done

120
scripts/manage-releases.sh Normal file
View file

@ -0,0 +1,120 @@
#!/bin/bash
set -euo pipefail
# Manage deployment releases
# Usage: ./manage-releases.sh REPO_PROJECT_PATH COMMAND [ARGS]
if [ $# -lt 2 ]; then
echo "Error: Invalid number of arguments"
echo "Usage: $0 REPO_PROJECT_PATH COMMAND [ARGS]"
echo "Commands:"
echo " list - List all releases"
echo " current - Show current release"
echo " rollback VERSION - Rollback to specific version"
echo " cleanup [KEEP] - Clean up old releases (default: keep 5)"
exit 1
fi
REPO_PROJECT_PATH="$1"
COMMAND="$2"
CURRENT_LINK_PATH="${REPO_PROJECT_PATH}/current"
RELEASES_PATH="${REPO_PROJECT_PATH}/releases"
case "$COMMAND" in
list)
echo "📋 Available releases:"
if [ -d "$RELEASES_PATH" ]; then
for release in $(ls -dt "${RELEASES_PATH}"/*/); do
version=$(basename "$release")
status=""
# Check if it's current
if [ -L "$CURRENT_LINK_PATH" ] && [ "$(readlink -f "$CURRENT_LINK_PATH")" = "$(realpath "$release")" ]; then
status=" [CURRENT]"
fi
# Check if it failed
if [ -f "${release}/.failed" ]; then
status="${status} [FAILED]"
fi
echo " - ${version}${status}"
done
else
echo "No releases found"
fi
;;
current)
if [ -L "$CURRENT_LINK_PATH" ]; then
current_version=$(basename "$(readlink -f "$CURRENT_LINK_PATH")")
echo "📌 Current version: ${current_version}"
else
echo "❌ No current deployment found"
fi
;;
rollback)
if [ $# -lt 3 ]; then
echo "Error: VERSION required for rollback"
exit 1
fi
TARGET_VERSION="$3"
TARGET_PATH="${RELEASES_PATH}/${TARGET_VERSION}"
if [ ! -d "$TARGET_PATH" ]; then
echo "Error: Version ${TARGET_VERSION} not found"
exit 1
fi
echo "🔄 Rolling back to version: ${TARGET_VERSION}"
# Read environment from target version
if [ -f "${TARGET_PATH}/.deployment_env" ]; then
IS_PROD=$(cat "${TARGET_PATH}/.deployment_env")
else
echo "Warning: Could not determine environment, assuming staging"
IS_PROD="false"
fi
# Stop current containers
if [ -L "$CURRENT_LINK_PATH" ] && [ -d "$CURRENT_LINK_PATH" ]; then
cd "$CURRENT_LINK_PATH"
docker compose -f docker-compose_web.yml down || true
[ "$IS_PROD" = "false" ] && docker compose -f docker-compose_staging.yml down || true
docker compose -f docker-compose_core.yml down || true
fi
# Update symlink
ln -sfn "$TARGET_PATH" "$CURRENT_LINK_PATH"
# Start containers
cd "$CURRENT_LINK_PATH"
docker compose -f docker-compose_core.yml up -d --no-build
if [ "$IS_PROD" = "true" ]; then
docker compose -f docker-compose_web.yml up -d --no-build
else
docker compose -f docker-compose_web.yml -f docker-compose_staging.yml up -d --no-build
fi
echo "✅ Rollback completed"
;;
cleanup)
KEEP_COUNT="${3:-5}"
echo "🗑️ Cleaning up old releases (keeping last ${KEEP_COUNT})"
if [ -d "$RELEASES_PATH" ]; then
cd "$RELEASES_PATH"
ls -dt */ 2>/dev/null | tail -n +$((KEEP_COUNT + 1)) | xargs -r rm -rf || true
echo "✅ Cleanup completed"
else
echo "No releases directory found"
fi
;;
*)
echo "Error: Unknown command: $COMMAND"
exit 1
;;
esac

View file

@ -0,0 +1,36 @@
#!/bin/bash
set -euo pipefail
# Parse repository name and generate project paths
# Usage: ./parse-repository-name.sh GITHUB_REPOSITORY
if [ $# -eq 0 ]; then
echo "Error: No repository name provided"
echo "Usage: $0 GITHUB_REPOSITORY"
exit 1
fi
GITHUB_REPOSITORY="$1"
echo "GITHUB_REPOSITORY: $GITHUB_REPOSITORY"
if [[ "$GITHUB_REPOSITORY" == *".git" ]]; then
if [[ "$GITHUB_REPOSITORY" == "https://"* ]]; then
echo "GITHUB_REPOSITORY ends in .git and is a URL"
REPO=$(echo "$GITHUB_REPOSITORY" | sed 's/\.git$//' | cut -d'/' -f4-5 | sed 's/[^a-zA-Z0-9\/-]/-/g')
else
echo "GITHUB_REPOSITORY ends in .git and is not a URL"
REPO=$(echo "$GITHUB_REPOSITORY" | sed 's/\.git$//' | sed 's/[^a-zA-Z0-9\/-]/-/g')
fi
else
echo "GITHUB_REPOSITORY is not a URL"
REPO=$(echo "$GITHUB_REPOSITORY" | sed 's/[^a-zA-Z0-9\/-]/-/g')
fi
REPO_NAME_ONLY=$(echo "$REPO" | cut -d'/' -f2)
REPO_PROJECT_PATH="/srv/${REPO_NAME_ONLY}"
# Output in format that can be sourced - using printf %q for proper escaping
printf "export REPO=%q\n" "$REPO"
printf "export REPO_NAME_ONLY=%q\n" "$REPO_NAME_ONLY"
printf "export REPO_PROJECT_PATH=%q\n" "$REPO_PROJECT_PATH"

View file

@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
# Prepare deployment by stopping containers
# Usage: ./prepare-deployment.sh REPO_PROJECT_PATH IS_PROD CURRENT_LINK_PATH
if [ $# -ne 3 ]; then
echo "Error: Invalid number of arguments"
echo "Usage: $0 REPO_PROJECT_PATH IS_PROD CURRENT_LINK_PATH"
exit 1
fi
REPO_PROJECT_PATH="$1"
IS_PROD="$2"
CURRENT_LINK_PATH="$3"
# Ensure base directory exists
if [ ! -d "$REPO_PROJECT_PATH" ]; then
echo "⚠️ Directory $REPO_PROJECT_PATH does not exist, creating it..."
mkdir -p "$REPO_PROJECT_PATH"
fi
# If current symlink exists, stop containers in that directory
if [ -L "$CURRENT_LINK_PATH" ] && [ -d "$CURRENT_LINK_PATH" ]; then
echo "🛑 Stopping containers in current deployment..."
cd "$CURRENT_LINK_PATH"
# Stop containers
if [ -f "docker-compose_web.yml" ]; then
docker compose -f docker-compose_web.yml down || true
fi
if [ "$IS_PROD" = "false" ] && [ -f "docker-compose_staging.yml" ]; then
docker compose -f docker-compose_staging.yml down || true
fi
if [ -f "docker-compose_core.yml" ]; then
docker compose -f docker-compose_core.yml down || true
fi
echo "✅ Containers stopped"
else
echo " No current deployment found (symlink doesn't exist or point to valid directory)"
fi

23
scripts/retry.sh Normal file
View file

@ -0,0 +1,23 @@
#!/bin/bash
# Retry function with exponential backoff
# Usage: source retry.sh && retry <command>
retry() {
local max_attempts=3
local delay=5
local attempt=1
until "$@"; do
if [ "$attempt" -ge "$max_attempts" ]; then
echo "Command failed after $max_attempts attempts: $*"
return 1
fi
echo "Command failed (attempt $attempt/$max_attempts): $*"
echo "Retrying in $delay seconds..."
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2)) # Exponential backoff
done
}