badbl0cks
f20c4f9474
- Implement setuptools-scm for dynamic version management from git tags - Refactor CI/CD into separate build and deploy jobs with artifact sharing - Add versioned releases with timestamp-based deployment directories - Implement health checks and automatic rollback on deployment failure - Extract deployment logic into reusable shell scripts - Add Docker layer caching to speed up builds - Include version info in Django context and build args
102 lines
No EOL
3.6 KiB
Bash
102 lines
No EOL
3.6 KiB
Bash
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# Perform health check and rollback if necessary
|
|
# Usage: ./health-check-and-rollback.sh REPO_PROJECT_PATH IS_PROD HEALTH_CHECK_URL [MAX_ATTEMPTS]
|
|
|
|
if [ $# -lt 3 ]; then
|
|
echo "Error: Invalid number of arguments"
|
|
echo "Usage: $0 REPO_PROJECT_PATH IS_PROD HEALTH_CHECK_URL [MAX_ATTEMPTS]"
|
|
exit 1
|
|
fi
|
|
|
|
REPO_PROJECT_PATH="$1"
|
|
IS_PROD="$2"
|
|
HEALTH_CHECK_URL="$3"
|
|
MAX_ATTEMPTS="${4:-30}"
|
|
|
|
CURRENT_LINK_PATH="${REPO_PROJECT_PATH}/current"
|
|
RELEASES_PATH="${REPO_PROJECT_PATH}/releases"
|
|
|
|
echo "🏥 Performing health check..."
|
|
echo "Health check URL: $HEALTH_CHECK_URL"
|
|
|
|
get_current_version() {
|
|
if [ -L "$CURRENT_LINK_PATH" ]; then
|
|
basename "$(readlink -f "$CURRENT_LINK_PATH")"
|
|
else
|
|
echo "unknown"
|
|
fi
|
|
}
|
|
|
|
ATTEMPT=0
|
|
while [ "$ATTEMPT" -lt "$MAX_ATTEMPTS" ]; do
|
|
# Check if the service is responding with 200 OK
|
|
HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' -m 10 "$HEALTH_CHECK_URL" || echo '000')
|
|
|
|
if [ "$HTTP_CODE" = "200" ]; then
|
|
echo "✅ Health check passed! (HTTP $HTTP_CODE)"
|
|
CURRENT_VERSION=$(get_current_version)
|
|
echo "📌 Current version: ${CURRENT_VERSION}"
|
|
exit 0
|
|
fi
|
|
|
|
ATTEMPT=$((ATTEMPT + 1))
|
|
if [ "$ATTEMPT" -eq "$MAX_ATTEMPTS" ]; then
|
|
echo "❌ Health check failed after $MAX_ATTEMPTS attempts (Last HTTP code: $HTTP_CODE)"
|
|
echo "🔄 Rolling back deployment..."
|
|
|
|
FAILED_VERSION=$(get_current_version)
|
|
echo "❌ Failed version: ${FAILED_VERSION}"
|
|
|
|
# Check if we have a previous version to roll back to
|
|
if [ -f "${CURRENT_LINK_PATH}/.previous_version" ]; then
|
|
PREVIOUS_VERSION_PATH=$(cat "${CURRENT_LINK_PATH}/.previous_version")
|
|
PREVIOUS_VERSION=$(basename "$PREVIOUS_VERSION_PATH")
|
|
|
|
if [ -d "$PREVIOUS_VERSION_PATH" ]; then
|
|
echo "🔄 Rolling back to version: ${PREVIOUS_VERSION}"
|
|
|
|
# Stop failed deployment containers
|
|
cd "$CURRENT_LINK_PATH"
|
|
echo "Stopping failed deployment containers..."
|
|
docker compose -f docker-compose_web.yml down || true
|
|
if [ "$IS_PROD" = "false" ]; then
|
|
docker compose -f docker-compose_staging.yml down || true
|
|
fi
|
|
docker compose -f docker-compose_core.yml down || true
|
|
|
|
# Switch symlink back to previous version
|
|
ln -sfn "$PREVIOUS_VERSION_PATH" "$CURRENT_LINK_PATH"
|
|
|
|
# Start previous version containers
|
|
cd "$CURRENT_LINK_PATH"
|
|
docker compose -f docker-compose_core.yml up -d --no-build
|
|
if [ "$IS_PROD" = "true" ]; then
|
|
docker compose -f docker-compose_web.yml up -d --no-build
|
|
else
|
|
docker compose -f docker-compose_web.yml -f docker-compose_staging.yml up -d --no-build
|
|
fi
|
|
|
|
echo "✅ Rollback completed to version: ${PREVIOUS_VERSION}"
|
|
|
|
# Mark failed version
|
|
if [ -d "${RELEASES_PATH}/${FAILED_VERSION}" ]; then
|
|
touch "${RELEASES_PATH}/${FAILED_VERSION}/.failed"
|
|
echo "$(date): Health check failed, rolled back to ${PREVIOUS_VERSION}" > "${RELEASES_PATH}/${FAILED_VERSION}/.failure_reason"
|
|
fi
|
|
else
|
|
echo "❌ Previous version directory not found: $PREVIOUS_VERSION_PATH"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "❌ No previous version information found. Cannot rollback!"
|
|
echo "💡 This might be the first deployment or the previous version info is missing."
|
|
exit 1
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
echo "⏳ Waiting for service to be healthy... (attempt $ATTEMPT/$MAX_ATTEMPTS, HTTP code: $HTTP_CODE)"
|
|
sleep 10
|
|
done |