Harden shared lifecycle rollback
This commit is contained in:
206
bin/context-kit
206
bin/context-kit
@@ -64,6 +64,13 @@ SHARED_SERVICES=(searxng "${WEB_SEARCH_SERVICE_NAME}" "${DOCS_SERVICE_NAME}")
|
||||
SNAPSHOT_IDS=()
|
||||
SNAPSHOT_RUNNING=()
|
||||
DOCS_SOURCES_FILE="${DATA_DIR}/docs-sources.txt"
|
||||
DOCS_SOURCES_TRANSACTION_ACTIVE=0
|
||||
DOCS_SOURCES_PRIOR_PRESENT=0
|
||||
DOCS_SOURCES_BACKUP_DIR=''
|
||||
DOCS_SOURCES_BACKUP_FILE=''
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
|
||||
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=false
|
||||
DOCS_DATA_DIR="${DATA_DIR}/docs"
|
||||
MODELS_DATA_DIR="${DATA_DIR}/models"
|
||||
DOCS_LOCAL_SOURCES_DIR="${CONTEXT_KIT_DOCS_LOCAL_SOURCES_DIR:-${DATA_DIR}/local-sources}"
|
||||
@@ -130,12 +137,104 @@ require_no_args() {
|
||||
|
||||
write_docs_sources_file() {
|
||||
mkdir -p "$(dirname "${DOCS_SOURCES_FILE}")"
|
||||
local tmp="${DOCS_SOURCES_FILE}.tmp.$$"
|
||||
{
|
||||
local tmp="${DOCS_SOURCES_FILE}.tmp.${BASHPID}"
|
||||
DOCS_SOURCES_RENDER_TMP="${tmp}"
|
||||
if ! {
|
||||
printf '# generated by context-kit lifecycle commands; edit your CONTEXT_KIT_DOCS_SOURCES file(s) instead\n'
|
||||
resolved_sources
|
||||
} > "${tmp}"
|
||||
mv "${tmp}" "${DOCS_SOURCES_FILE}"
|
||||
} > "${tmp}"; then
|
||||
rm -f -- "${tmp}"
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
return 1
|
||||
fi
|
||||
if ! mv -fT -- "${tmp}" "${DOCS_SOURCES_FILE}"; then
|
||||
rm -f -- "${tmp}"
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
return 1
|
||||
fi
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
}
|
||||
|
||||
reset_docs_sources_transaction() {
|
||||
DOCS_SOURCES_TRANSACTION_ACTIVE=0
|
||||
DOCS_SOURCES_PRIOR_PRESENT=0
|
||||
DOCS_SOURCES_BACKUP_DIR=''
|
||||
DOCS_SOURCES_BACKUP_FILE=''
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
}
|
||||
|
||||
begin_docs_sources_transaction() {
|
||||
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 0 ]] || return 1
|
||||
mkdir -p "$(dirname "${DOCS_SOURCES_FILE}")"
|
||||
|
||||
local backup_dir
|
||||
backup_dir="$(mktemp -d "${DOCS_SOURCES_FILE}.lifecycle-backup.XXXXXX")" || return 1
|
||||
DOCS_SOURCES_BACKUP_DIR="${backup_dir}"
|
||||
DOCS_SOURCES_BACKUP_FILE="${backup_dir}/docs-sources.txt"
|
||||
DOCS_SOURCES_PRIOR_PRESENT=0
|
||||
DOCS_SOURCES_TRANSACTION_ACTIVE=1
|
||||
|
||||
if [[ -e "${DOCS_SOURCES_FILE}" || -L "${DOCS_SOURCES_FILE}" ]]; then
|
||||
DOCS_SOURCES_PRIOR_PRESENT=1
|
||||
if ! mv -T -- "${DOCS_SOURCES_FILE}" "${DOCS_SOURCES_BACKUP_FILE}"; then
|
||||
rmdir -- "${DOCS_SOURCES_BACKUP_DIR}" 2>/dev/null || true
|
||||
reset_docs_sources_transaction
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
restore_docs_sources_transaction() {
|
||||
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 1 ]] || return 0
|
||||
local failed=0
|
||||
|
||||
if [[ -n "${DOCS_SOURCES_RENDER_TMP}" ]]; then
|
||||
rm -f -- "${DOCS_SOURCES_RENDER_TMP}" || failed=1
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
fi
|
||||
if [[ "${DOCS_SOURCES_PRIOR_PRESENT}" -eq 1 ]]; then
|
||||
if [[ -e "${DOCS_SOURCES_BACKUP_FILE}" || -L "${DOCS_SOURCES_BACKUP_FILE}" ]]; then
|
||||
if ! mv -fT -- "${DOCS_SOURCES_BACKUP_FILE}" "${DOCS_SOURCES_FILE}"; then
|
||||
warn "failed to restore prior docs sources file: ${DOCS_SOURCES_FILE}"
|
||||
failed=1
|
||||
fi
|
||||
else
|
||||
warn "prior docs sources backup is missing: ${DOCS_SOURCES_BACKUP_FILE}"
|
||||
failed=1
|
||||
fi
|
||||
elif ! rm -f -- "${DOCS_SOURCES_FILE}"; then
|
||||
warn "failed to restore prior absence of docs sources file: ${DOCS_SOURCES_FILE}"
|
||||
failed=1
|
||||
fi
|
||||
|
||||
if [[ "${failed}" -eq 0 ]] && ! rmdir -- "${DOCS_SOURCES_BACKUP_DIR}"; then
|
||||
warn "failed to remove docs sources backup directory: ${DOCS_SOURCES_BACKUP_DIR}"
|
||||
failed=1
|
||||
fi
|
||||
if [[ "${failed}" -eq 0 ]]; then
|
||||
reset_docs_sources_transaction
|
||||
fi
|
||||
return "${failed}"
|
||||
}
|
||||
|
||||
discard_docs_sources_transaction() {
|
||||
[[ "${DOCS_SOURCES_TRANSACTION_ACTIVE}" -eq 1 ]] || return 0
|
||||
local failed=0
|
||||
|
||||
if [[ -n "${DOCS_SOURCES_RENDER_TMP}" ]]; then
|
||||
rm -f -- "${DOCS_SOURCES_RENDER_TMP}" || failed=1
|
||||
DOCS_SOURCES_RENDER_TMP=''
|
||||
fi
|
||||
if [[ "${DOCS_SOURCES_PRIOR_PRESENT}" -eq 1 ]]; then
|
||||
rm -f -- "${DOCS_SOURCES_BACKUP_FILE}" || failed=1
|
||||
fi
|
||||
if [[ "${failed}" -eq 0 ]] && ! rmdir -- "${DOCS_SOURCES_BACKUP_DIR}"; then
|
||||
failed=1
|
||||
fi
|
||||
if [[ "${failed}" -eq 0 ]]; then
|
||||
reset_docs_sources_transaction
|
||||
fi
|
||||
return "${failed}"
|
||||
}
|
||||
|
||||
ensure_writable_dir() {
|
||||
@@ -491,6 +590,34 @@ restore_shared_service_states() {
|
||||
return "${failed}"
|
||||
}
|
||||
|
||||
rollback_lifecycle_transaction() {
|
||||
local failed=0
|
||||
restore_docs_sources_transaction || failed=1
|
||||
if [[ "${LIFECYCLE_STATE_ROLLBACK_ACTIVE}" -eq 1 ]]; then
|
||||
restore_shared_service_states "${LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW}" || failed=1
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
|
||||
fi
|
||||
return "${failed}"
|
||||
}
|
||||
|
||||
lifecycle_transaction_exit() {
|
||||
local status="$1"
|
||||
trap - EXIT HUP INT TERM
|
||||
rollback_lifecycle_transaction || true
|
||||
exit "${status}"
|
||||
}
|
||||
|
||||
arm_lifecycle_transaction_traps() {
|
||||
trap 'lifecycle_transaction_exit "$?"' EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
}
|
||||
|
||||
disarm_lifecycle_transaction_traps() {
|
||||
trap - EXIT HUP INT TERM
|
||||
}
|
||||
|
||||
shared_services_ready() {
|
||||
wait_for_searxng && wait_for_web_search_mcp && wait_for_docs_mcp
|
||||
}
|
||||
@@ -500,18 +627,42 @@ start_locked() {
|
||||
if ! docker image inspect "${WEB_SEARCH_IMAGE}" >/dev/null 2>&1 || ! docker image inspect "${DOCS_IMAGE}" >/dev/null 2>&1; then
|
||||
cmd_build
|
||||
fi
|
||||
write_docs_sources_file
|
||||
|
||||
local result=0
|
||||
local result=0 rollback_failed=0
|
||||
arm_lifecycle_transaction_traps
|
||||
if begin_docs_sources_transaction; then
|
||||
:
|
||||
else
|
||||
result=$?
|
||||
disarm_lifecycle_transaction_traps
|
||||
return "${result}"
|
||||
fi
|
||||
if write_docs_sources_file; then
|
||||
:
|
||||
else
|
||||
result=$?
|
||||
rollback_lifecycle_transaction || rollback_failed=1
|
||||
disarm_lifecycle_transaction_traps
|
||||
[[ "${rollback_failed}" -eq 0 ]] || return 1
|
||||
return "${result}"
|
||||
fi
|
||||
snapshot_shared_services
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=1
|
||||
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=true
|
||||
compose up -d --no-recreate "${SHARED_SERVICES[@]}" || result=$?
|
||||
if [[ "${result}" -eq 0 ]] && ! shared_services_ready; then
|
||||
result=1
|
||||
fi
|
||||
[[ "${result}" -ne 0 ]] || return 0
|
||||
if [[ "${result}" -eq 0 ]] && discard_docs_sources_transaction; then
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
|
||||
disarm_lifecycle_transaction_traps
|
||||
return 0
|
||||
fi
|
||||
[[ "${result}" -ne 0 ]] || result=1
|
||||
|
||||
warn "shared service startup failed; restoring prior container states"
|
||||
restore_shared_service_states true || true
|
||||
warn "shared service startup failed; restoring prior docs sources and container states"
|
||||
rollback_lifecycle_transaction || rollback_failed=1
|
||||
disarm_lifecycle_transaction_traps
|
||||
[[ "${rollback_failed}" -eq 0 ]] || return 1
|
||||
return "${result}"
|
||||
}
|
||||
|
||||
@@ -538,17 +689,42 @@ cmd_restart() {
|
||||
}
|
||||
|
||||
restart_locked() {
|
||||
local result=0
|
||||
write_docs_sources_file
|
||||
local result=0 rollback_failed=0
|
||||
arm_lifecycle_transaction_traps
|
||||
if begin_docs_sources_transaction; then
|
||||
:
|
||||
else
|
||||
result=$?
|
||||
disarm_lifecycle_transaction_traps
|
||||
return "${result}"
|
||||
fi
|
||||
if write_docs_sources_file; then
|
||||
:
|
||||
else
|
||||
result=$?
|
||||
rollback_lifecycle_transaction || rollback_failed=1
|
||||
disarm_lifecycle_transaction_traps
|
||||
[[ "${rollback_failed}" -eq 0 ]] || return 1
|
||||
return "${result}"
|
||||
fi
|
||||
snapshot_shared_services
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=1
|
||||
LIFECYCLE_STATE_ROLLBACK_REMOVE_NEW=false
|
||||
compose restart "${SHARED_SERVICES[@]}" || result=$?
|
||||
if [[ "${result}" -eq 0 ]] && ! shared_services_ready; then
|
||||
result=1
|
||||
fi
|
||||
[[ "${result}" -ne 0 ]] || return 0
|
||||
if [[ "${result}" -eq 0 ]] && discard_docs_sources_transaction; then
|
||||
LIFECYCLE_STATE_ROLLBACK_ACTIVE=0
|
||||
disarm_lifecycle_transaction_traps
|
||||
return 0
|
||||
fi
|
||||
[[ "${result}" -ne 0 ]] || result=1
|
||||
|
||||
warn "shared service restart failed; restoring prior container states"
|
||||
restore_shared_service_states false || true
|
||||
warn "shared service restart failed; restoring prior docs sources and container states"
|
||||
rollback_lifecycle_transaction || rollback_failed=1
|
||||
disarm_lifecycle_transaction_traps
|
||||
[[ "${rollback_failed}" -eq 0 ]] || return 1
|
||||
return "${result}"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user