ImanThiyanga
8e472ac996
CI/CD - ERP Core / scripts (pull_request) Failing after 12m21s
CI/CD - ERP Core / e2e-typecheck (pull_request) Failing after 13m13s
CI/CD - ERP Core / frontend (pull_request) Failing after 14m5s
CI/CD - ERP Core / backend (pull_request) Failing after 14m57s
CI/CD - ERP Core / detect (pull_request) Failing after 13m42s
CI/CD - ERP Core / Build & Deploy (pull_request) Failing after 12m47s
CI/CD - ERP Core / ci-ok (pull_request) Failing after 14m58s
92 lines
3.7 KiB
Bash
92 lines
3.7 KiB
Bash
#!/usr/bin/env bash
|
||
# deploy.yml › backend — switch an environment to BACKEND_HASH:
|
||
# pg_dump (policy) → stop → repoint `current` → start → wait for /health
|
||
# On failure the previous release is restored and the job fails (red).
|
||
# The DATABASE is never rolled back — hence add-only migrations (docs/40-DEPLOYMENT.md §8).
|
||
# shellcheck source=common.sh
|
||
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
|
||
|
||
: "${ERP_ENV:?ERP_ENV is required}" "${BACKEND_HASH:?BACKEND_HASH is required}"
|
||
load_env "$ERP_ENV"
|
||
is_rollback="${IS_ROLLBACK:-false}"
|
||
migrations_changed="${MIGRATIONS_CHANGED:-false}"
|
||
|
||
target="$RELEASES_DIR/$BACKEND_HASH"
|
||
[[ -x "$target/ERPCore" ]] || fail "backend release not found: $target"
|
||
|
||
backup_database() {
|
||
local pg_dump file
|
||
pg_dump="$(command -v pg_dump || echo /usr/pgsql-16/bin/pg_dump)"
|
||
[[ -x "$pg_dump" ]] || fail "pg_dump not found (install postgresql16 client tools or fix PATH)"
|
||
mkdir -p "$BACKUP_DIR"
|
||
file="$BACKUP_DIR/$ERP_ENV-$(date -u +%Y%m%dT%H%M%SZ)-${BACKEND_HASH:0:12}.dump"
|
||
log "backing up database $DB_NAME → $file"
|
||
runuser -u postgres -- "$pg_dump" -Fc "$DB_NAME" >"$file.partial"
|
||
mv "$file.partial" "$file"
|
||
}
|
||
|
||
# wait_healthy — succeeds once /health answers 200. EF migrations + seeding run BEFORE the app
|
||
# starts listening, so a slow start is normal. Only a crash (unit failed / auto-restarted) or the
|
||
# generous per-environment ceiling counts as failure: never cut a live migration on a tight timer.
|
||
wait_healthy() {
|
||
local deadline=$((SECONDS + HEALTH_TIMEOUT)) restarts_at_start state restarts
|
||
restarts_at_start="$(systemctl show -p NRestarts --value "$SERVICE")"
|
||
while ((SECONDS < deadline)); do
|
||
if curl -fsS --max-time 5 -o /dev/null "http://127.0.0.1:$BACKEND_PORT/health" 2>/dev/null; then
|
||
return 0
|
||
fi
|
||
state="$(systemctl show -p ActiveState --value "$SERVICE")"
|
||
restarts="$(systemctl show -p NRestarts --value "$SERVICE")"
|
||
if [[ "$state" == failed || "$restarts" != "$restarts_at_start" ]]; then
|
||
warn "$SERVICE crashed during startup (state=$state, restarts $restarts_at_start → $restarts)"
|
||
return 1
|
||
fi
|
||
sleep 3
|
||
done
|
||
warn "$SERVICE was not healthy within ${HEALTH_TIMEOUT}s"
|
||
return 1
|
||
}
|
||
|
||
switch_to() {
|
||
systemctl stop "$SERVICE" || true
|
||
mkdir -p "$(dirname "$CURRENT_LINK")"
|
||
ln -sfn "$1" "$CURRENT_LINK"
|
||
systemctl reset-failed "$SERVICE" 2>/dev/null || true
|
||
systemctl start "$SERVICE" || warn "systemctl start $SERVICE returned an error"
|
||
}
|
||
|
||
if [[ "$is_rollback" != true ]] && [[ "$BACKUP_POLICY" == always || "$migrations_changed" == true ]]; then
|
||
backup_database
|
||
else
|
||
log "no database backup (policy=$BACKUP_POLICY, migrations changed=$migrations_changed, rollback=$is_rollback)"
|
||
fi
|
||
|
||
previous="$(readlink "$CURRENT_LINK" 2>/dev/null || true)"
|
||
log "deploying backend $BACKEND_HASH to $ERP_ENV (previous: ${previous:-none})"
|
||
switch_to "$target"
|
||
|
||
if wait_healthy; then
|
||
write_state backend "$BACKEND_HASH"
|
||
if [[ "$is_rollback" != true && -n "${MIGRATIONS_HASH:-}" ]]; then write_state migrations "$MIGRATIONS_HASH"; fi
|
||
if [[ "$is_rollback" == true ]]; then result=rollback; else result=deployed; fi
|
||
record_history backend "$BACKEND_HASH" "$result"
|
||
log "✅ backend $BACKEND_HASH is healthy on $ERP_ENV"
|
||
exit 0
|
||
fi
|
||
|
||
journalctl -u "$SERVICE" -n 80 --no-pager || true
|
||
record_history backend "$BACKEND_HASH" failed
|
||
if [[ -n "$previous" && "$previous" != "$target" && -x "$previous/ERPCore" ]]; then
|
||
warn "restoring previous release $previous"
|
||
switch_to "$previous"
|
||
if wait_healthy; then
|
||
log "previous release is serving again"
|
||
else
|
||
warn "the previous release is not healthy either — investigate now"
|
||
fi
|
||
else
|
||
warn "no previous release to restore; stopping $SERVICE"
|
||
systemctl stop "$SERVICE" || true
|
||
fi
|
||
fail "backend deploy of $BACKEND_HASH to $ERP_ENV failed"
|