Initial commit: Backup Restore Orchestrator
Windmill-Flow + restore.sh für das automatische tägliche Backup-Verifikationssystem. Direkter Windmill-Sync via `wmill sync push` möglich. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Sebastian Serfling
@@ -0,0 +1,734 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# /opt/windmill-restore/restore.sh
|
||||
# Windmill Backup Restore Worker
|
||||
# Version: 1.0.26
|
||||
#
|
||||
# Unterstützt sowohl VM (qm) als auch CT (pct) Backups.
|
||||
# Backup-Typ wird automatisch aus dem Backup-Pfad erkannt (vm/ oder ct/).
|
||||
#
|
||||
# ABLAUF:
|
||||
# [0] 7z-Passwort holen – password_7z.txt per Rsync vom PBS-Server
|
||||
# [1] Space-Check – Freier Platz auf restore-mount prüfen
|
||||
# [2] ID ermitteln – Original aus Backup-Pfad, Restore-ID ab 1000
|
||||
# [3] Restore – qmrestore (VM) oder pct restore (CT)
|
||||
# [4] IMAGE_DIR – Dynamisch aus PVE-Storage-Pfad ermitteln
|
||||
# [5] Images prüfen – Abbruch wenn leer/nicht vorhanden
|
||||
# [6] Vorbereiten – VM: unlock/cdrom/net entfernen/Agent
|
||||
# CT: unlock/net entfernen
|
||||
# [7] Starten & prüfen – VM: qm-Agent 120s | CT: pct exec ping
|
||||
# [8] Stoppen – VM: qm shutdown | CT: pct stop
|
||||
# [9] Config sichern – Originale Config ins ZIP-Verzeichnis
|
||||
# [10] 7z-Archiv – Images verschlüsselt zippen
|
||||
# [11] Rsync – ZIP zum Backup-Server
|
||||
# [12] Aufräumen – destroy, ZIP löschen
|
||||
# [13] Webhook – JSON → Windmill
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ── Konfigdatei laden ─────────────────────────────────────────────────────────
|
||||
CONF_FILE="/opt/windmill-restore/pbs.conf"
|
||||
[[ ! -f "$CONF_FILE" ]] && { echo "FEHLER: $CONF_FILE fehlt!" >&2; exit 1; }
|
||||
source "$CONF_FILE"
|
||||
|
||||
# ── Argument-Parser ───────────────────────────────────────────────────────────
|
||||
JOB_UUID=""
|
||||
BACKUP_PATH=""
|
||||
CLIENT_NAME=""
|
||||
RESTORE_MOUNT=""
|
||||
RESTORE_PATH=""
|
||||
RSYNC_TARGET=""
|
||||
PBS_STORAGE=""
|
||||
WEBHOOK_URL=""
|
||||
WEBHOOK_TOKEN=""
|
||||
SERVER_HOSTNAME=""
|
||||
BACKUP_SIZE_BYTES=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--job-uuid) JOB_UUID="$2"; shift 2 ;;
|
||||
--backup-path) BACKUP_PATH="$2"; shift 2 ;;
|
||||
--client) CLIENT_NAME="$2"; shift 2 ;;
|
||||
--restore-mount) RESTORE_MOUNT="$2"; shift 2 ;;
|
||||
--restore-path) RESTORE_PATH="$2"; shift 2 ;;
|
||||
--rsync-target) RSYNC_TARGET="$2"; shift 2 ;;
|
||||
--pbs-storage) PBS_STORAGE="$2"; shift 2 ;;
|
||||
--webhook-url) WEBHOOK_URL="$2"; shift 2 ;;
|
||||
--webhook-token) WEBHOOK_TOKEN="$2"; shift 2 ;;
|
||||
--server-hostname) SERVER_HOSTNAME="$2"; shift 2 ;;
|
||||
--backup-size) BACKUP_SIZE_BYTES="$2"; shift 2 ;;
|
||||
*) echo "Unbekannter Parameter: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
for var in JOB_UUID BACKUP_PATH CLIENT_NAME \
|
||||
RESTORE_MOUNT RESTORE_PATH RSYNC_TARGET PBS_STORAGE WEBHOOK_URL; do
|
||||
[[ -z "${!var}" ]] && { echo "FEHLER: --${var//_/-} fehlt" >&2; exit 1; }
|
||||
done
|
||||
|
||||
[[ ! -d "$RESTORE_MOUNT" ]] && {
|
||||
echo "FEHLER: Restore-Mount '$RESTORE_MOUNT' existiert nicht!" >&2; exit 1
|
||||
}
|
||||
|
||||
# Fallback SERVER_HOSTNAME
|
||||
SERVER_HOSTNAME="${SERVER_HOSTNAME:-$(hostname -f 2>/dev/null || hostname)}"
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||||
LOG_DIR="/opt/windmill-restore/logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
SAFE_CLIENT="${CLIENT_NAME//\//_}"
|
||||
SAFE_CLIENT="${SAFE_CLIENT//:/_}"
|
||||
LOG_FILE="$LOG_DIR/${SAFE_CLIENT}.log"
|
||||
exec >> "$LOG_FILE" 2>&1
|
||||
|
||||
# ── Backup-Pfad zerlegen ──────────────────────────────────────────────────────
|
||||
DATASTORE=$(echo "$BACKUP_PATH" | cut -d: -f1)
|
||||
SNAPSHOT_PATH=$(echo "$BACKUP_PATH" | cut -d: -f2-)
|
||||
BACKUP_TYPE=$(echo "$SNAPSHOT_PATH" | cut -d/ -f1)
|
||||
PVE_BACKUP_REF="${PBS_STORAGE}:backup/${SNAPSHOT_PATH}"
|
||||
|
||||
# ── Komprimierungsstufe festlegen ─────────────────────────────────────────────
|
||||
# Standard: mx=1 (schnellste Komprimierung)
|
||||
# Ausnahme: tnp-Invest-GmbH vm/108 → mx=0 (Store-Modus, kein Komprimieren)
|
||||
# Hintergrund: Diese VM ist sehr groß und würde mit mx=1 ~10h brauchen.
|
||||
COMPRESS_LEVEL=0
|
||||
|
||||
# ── 7z Thread-Anzahl je Host festlegen ────────────────────────────────────────
|
||||
# STI-BAC01 → Ryzen 9 5950X (16 Kerne / 32 Threads) → mmt=16
|
||||
# ITD-PROX01 → Ryzen 7 3700X ( 8 Kerne / 16 Threads) → mmt=8
|
||||
# STI-PROX01 → Xeon E5-1650v3 ( 6 Kerne / 12 Threads) → mmt=6
|
||||
# Fallback → mmt=4
|
||||
case "$SERVER_HOSTNAME" in
|
||||
STI-BAC01) MMT_THREADS=30 ;;
|
||||
ITD-PROX01) MMT_THREADS=8 ;;
|
||||
STI-PROX01) MMT_THREADS=16 ;;
|
||||
*) MMT_THREADS=4 ;;
|
||||
esac
|
||||
echo "INFO: Server '$SERVER_HOSTNAME' → 7z mmt=${MMT_THREADS}"
|
||||
|
||||
# ── Messvariablen ─────────────────────────────────────────────────────────────
|
||||
LAST_DATE=$(TZ="Europe/Berlin" date +"%Y-%m-%d" -d "1 day ago")
|
||||
|
||||
# STI-BAC01: rsync_target lokal gemountet → ZIP direkt dorthin, kein Rsync
|
||||
if [[ "$SERVER_HOSTNAME" == "STI-BAC01" ]]; then
|
||||
ZIP_DIR="${RSYNC_TARGET}/${LAST_DATE}"
|
||||
SKIP_RSYNC=1
|
||||
else
|
||||
ZIP_DIR="${RESTORE_MOUNT}/zips/${LAST_DATE}"
|
||||
SKIP_RSYNC=0
|
||||
fi
|
||||
|
||||
BACKUP_SERVER_HOST=$(cat /opt/windmill-restore/backup_server_host 2>/dev/null \
|
||||
|| echo "backup-server")
|
||||
KEY_DIR="/opt/windmill-restore/keys"
|
||||
|
||||
RESTORE_START=$(date +%s)
|
||||
STATUS="success"
|
||||
ERROR_MSG=""
|
||||
VM_ID_ORIGINAL=0
|
||||
VM_ID_RESTORED=0
|
||||
VM_NAME=""
|
||||
IMAGE_DIR=""
|
||||
ACTUAL_DISK_BYTES=0
|
||||
ZIP_SIZE_BYTES=0
|
||||
ZIP_DURATION=0
|
||||
RSYNC_SIZE_BYTES=0
|
||||
RSYNC_OK="true"
|
||||
RSYNC_RETRIES=0
|
||||
QM_AGENT_OK="false"
|
||||
ZIP_FILE=""
|
||||
ZIP_PASSWORD=""
|
||||
FREE_GB=0
|
||||
|
||||
echo "============================================================"
|
||||
echo " Windmill Restore Worker"
|
||||
echo " Client: $CLIENT_NAME"
|
||||
echo " Typ: $BACKUP_TYPE"
|
||||
echo " Datastore: $DATASTORE"
|
||||
echo " Backup: $BACKUP_PATH"
|
||||
echo " PBS-Storage: $PBS_STORAGE"
|
||||
echo " Restore-Mount: $RESTORE_MOUNT"
|
||||
echo " Restore-Path: $RESTORE_PATH"
|
||||
echo " Rsync-Target: $RSYNC_TARGET"
|
||||
echo " Server: $SERVER_HOSTNAME"
|
||||
echo " Skip-Rsync: $SKIP_RSYNC"
|
||||
echo " Job-UUID: $JOB_UUID"
|
||||
echo " 7z-Level: mx=${COMPRESS_LEVEL} mmt=${MMT_THREADS}"
|
||||
echo " Start: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "============================================================"
|
||||
|
||||
# ── JSON Escape Funktion ──────────────────────────────────────────────────────
|
||||
escape_json() {
|
||||
local input="$1"
|
||||
input="${input//\\/\\\\}"
|
||||
input="${input//\"/\\\"}"
|
||||
input="${input//$'\n'/\\n}"
|
||||
input="${input//$'\r'/\\r}"
|
||||
input="${input//$'\t'/\\t}"
|
||||
echo "$input"
|
||||
}
|
||||
|
||||
# ── Webhook-Funktion ──────────────────────────────────────────────────────────
|
||||
send_webhook() {
|
||||
local wh_status="$1"
|
||||
local wh_error
|
||||
wh_error=$(escape_json "${2:-}")
|
||||
local wh_vm_name
|
||||
wh_vm_name=$(escape_json "${VM_NAME:-$SAFE_CLIENT}")
|
||||
local duration=$(( $(date +%s) - RESTORE_START ))
|
||||
local payload
|
||||
payload=$(printf '{
|
||||
"job_uuid": "%s",
|
||||
"client_name": "%s",
|
||||
"status": "%s",
|
||||
"error_message": "%s",
|
||||
"server_hostname": "%s",
|
||||
"free_space_gb": %d,
|
||||
"vm_name": "%s",
|
||||
"vm_id_original": %d,
|
||||
"vm_id_restored": %d,
|
||||
"restore_duration_sec": %d,
|
||||
"actual_disk_used_bytes": %d,
|
||||
"zip_size_bytes": %d,
|
||||
"zip_duration_sec": %d,
|
||||
"rsync_size_bytes": %d,
|
||||
"rsync_ok": %s,
|
||||
"rsync_retries": %d,
|
||||
"qm_agent_ok": "%s",
|
||||
"log_file": "%s"
|
||||
}' \
|
||||
"$JOB_UUID" "$CLIENT_NAME" "$wh_status" "$wh_error" \
|
||||
"$SERVER_HOSTNAME" "$FREE_GB" "$wh_vm_name" \
|
||||
"$VM_ID_ORIGINAL" "$VM_ID_RESTORED" \
|
||||
"$duration" "$ACTUAL_DISK_BYTES" \
|
||||
"$ZIP_SIZE_BYTES" "$ZIP_DURATION" \
|
||||
"$RSYNC_SIZE_BYTES" "$RSYNC_OK" "$RSYNC_RETRIES" \
|
||||
"$QM_AGENT_OK" "$LOG_FILE")
|
||||
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> Sende Webhook..."
|
||||
echo " Payload: $payload"
|
||||
local http_response
|
||||
http_response=$(curl -s -w "\n%{http_code}" \
|
||||
-X POST "$WEBHOOK_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
${WEBHOOK_TOKEN:+-H "Authorization: Bearer ${WEBHOOK_TOKEN}"} \
|
||||
-d "$payload")
|
||||
local http_code
|
||||
http_code=$(echo "$http_response" | tail -1)
|
||||
local http_body
|
||||
http_body=$(echo "$http_response" | head -n -1)
|
||||
echo " HTTP: $http_code"
|
||||
echo " Response: $http_body"
|
||||
[[ "$http_code" =~ ^2 ]] && echo " Webhook OK." \
|
||||
|| echo " WARNUNG: HTTP $http_code"
|
||||
}
|
||||
|
||||
# ── ERR-Trap ──────────────────────────────────────────────────────────────────
|
||||
trap 'STATUS="failed"
|
||||
ERROR_LINE=$LINENO
|
||||
echo ""
|
||||
echo "FEHLER in Zeile ${ERROR_LINE} – räume auf..."
|
||||
if [[ ${VM_ID_RESTORED:-0} -gt 0 ]]; then
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct stop "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
sleep 3
|
||||
pct destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
|
||||
else
|
||||
qm stop "$VM_ID_RESTORED" --skiplock 1 2>/dev/null || true
|
||||
sleep 5
|
||||
qm destroy "$VM_ID_RESTORED" \
|
||||
--destroy-unreferenced-disks 1 --purge 1 2>/dev/null || true
|
||||
fi
|
||||
echo " ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt."
|
||||
fi
|
||||
[[ -n "${ZIP_FILE:-}" && -f "$ZIP_FILE" ]] && rm -f "$ZIP_FILE"
|
||||
[[ -n "${IMAGE_DIR:-}" && -d "$IMAGE_DIR" ]] && rm -rf "$IMAGE_DIR"
|
||||
send_webhook "failed" "Abgebrochen in Zeile ${ERROR_LINE} – $LOG_FILE"' ERR
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [0/13] 7Z-PASSWORT VOM PBS-SERVER HOLEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [0/13] 7z-Passwort vom PBS-Server holen ($PBS_HOST)..."
|
||||
mkdir -p "$KEY_DIR"
|
||||
chmod 700 "$KEY_DIR"
|
||||
|
||||
PW_FILE_LOCAL="${KEY_DIR}/password_7z.txt"
|
||||
if [[ ! -f "$PW_FILE_LOCAL" || ! -s "$PW_FILE_LOCAL" ]]; then
|
||||
echo " Hole password_7z.txt..."
|
||||
rsync -az \
|
||||
-e "ssh -o StrictHostKeyChecking=no" \
|
||||
"root@${PBS_HOST}:/root/Scripte/password_7z.txt" \
|
||||
"$PW_FILE_LOCAL" \
|
||||
2>&1
|
||||
chmod 600 "$PW_FILE_LOCAL"
|
||||
echo " password_7z.txt gespeichert ✓"
|
||||
else
|
||||
echo " password_7z.txt bereits vorhanden."
|
||||
fi
|
||||
|
||||
ZIP_PASSWORD=$(grep -m1 "^${DATASTORE}:" "$PW_FILE_LOCAL" \
|
||||
| awk -F': ' '{print $2}' | tr -d '[:space:]')
|
||||
|
||||
[[ -z "$ZIP_PASSWORD" ]] && {
|
||||
echo "FEHLER: Kein 7z-Passwort für '$DATASTORE' in password_7z.txt" >&2; exit 1
|
||||
}
|
||||
echo " 7z-Passwort geladen ✓"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [1/13] SPACE-CHECK
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [1/13] Prüfe freien Speicherplatz auf $RESTORE_MOUNT..."
|
||||
mkdir -p "$ZIP_DIR"
|
||||
FREE_KB=$(df "$RESTORE_MOUNT" 2>/dev/null | awk 'NR==2{print $4}' || echo "0")
|
||||
FREE_GB=$(( FREE_KB / 1024 / 1024 ))
|
||||
FREE_BYTES=$(( FREE_KB * 1024 ))
|
||||
echo " Frei: ${FREE_GB} GB"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [2/13] ID ERMITTELN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [2/13] Ermittle IDs..."
|
||||
VM_ID_ORIGINAL=$(echo "$SNAPSHOT_PATH" | grep -oP '\d+' | head -1 || echo "0")
|
||||
echo " Original-ID: $VM_ID_ORIGINAL (Typ: $BACKUP_TYPE)"
|
||||
|
||||
VM_ID_RESTORED=$(
|
||||
{
|
||||
pvesh get /nodes/localhost/qemu --output-format json 2>/dev/null || echo "[]"
|
||||
pvesh get /nodes/localhost/lxc --output-format json 2>/dev/null || echo "[]"
|
||||
} | python3 -c "
|
||||
import json, sys
|
||||
data = []
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try: data.extend(json.loads(line))
|
||||
except: pass
|
||||
existing = {int(v.get('vmid', 0)) for v in data}
|
||||
for i in range(1000, 2000):
|
||||
if i not in existing:
|
||||
print(i); break
|
||||
" 2>/dev/null || echo "1000"
|
||||
)
|
||||
echo " Restore-ID: $VM_ID_RESTORED"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [2.5/13] CONFIG-CHECK
|
||||
# Config direkt aus PBS-Backup lesen um VM-Name zu ermitteln und zu prüfen
|
||||
# ob ZIP bereits auf dem Backup-Server existiert → Restore überspringen
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [2.5/13] Config aus PBS-Backup lesen..."
|
||||
|
||||
CONFIG_VM_NAME=""
|
||||
CONFIG_TMP="/tmp/pbs_config_${VM_ID_ORIGINAL}_$$.conf"
|
||||
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
CONF_FILE_IN_BACKUP="pct.conf"
|
||||
NAME_KEY="^hostname:"
|
||||
else
|
||||
CONF_FILE_IN_BACKUP="qemu-server.conf"
|
||||
NAME_KEY="^name:"
|
||||
fi
|
||||
|
||||
export PBS_PASSWORD
|
||||
export PBS_REPOSITORY="${PBS_USER}@${PBS_HOST}:${DATASTORE}"
|
||||
|
||||
SNAP_ID=$(echo "$SNAPSHOT_PATH" | cut -d/ -f3)
|
||||
echo " Repository: $PBS_REPOSITORY"
|
||||
echo " Snapshot: ${BACKUP_TYPE}/${VM_ID_ORIGINAL}/${SNAP_ID}"
|
||||
echo " Config: $CONF_FILE_IN_BACKUP"
|
||||
echo " Keyfile: ${KEY_DIR}/${DATASTORE}.keyfile"
|
||||
|
||||
proxmox-backup-client restore \
|
||||
--keyfile "${KEY_DIR}/${DATASTORE}.keyfile" \
|
||||
"${BACKUP_TYPE}/${VM_ID_ORIGINAL}/${SNAP_ID}" \
|
||||
"$CONF_FILE_IN_BACKUP" \
|
||||
"$CONFIG_TMP" \
|
||||
2>&1 || echo " WARNUNG: proxmox-backup-client restore fehlgeschlagen (exit $?)"
|
||||
|
||||
if [[ -f "$CONFIG_TMP" ]]; then
|
||||
CONFIG_VM_NAME=$(grep -m1 "$NAME_KEY" "$CONFIG_TMP" 2>/dev/null \
|
||||
| awk -F': ' '{print $2}' | tr -d '[:space:]' || echo "")
|
||||
rm -f "$CONFIG_TMP"
|
||||
echo " VM-Name: ${CONFIG_VM_NAME:-unbekannt}"
|
||||
else
|
||||
echo " Config nicht lesbar – überspringe ZIP-Check."
|
||||
fi
|
||||
|
||||
# Prüfen ob ZIP bereits vorhanden
|
||||
if [[ -n "$CONFIG_VM_NAME" ]]; then
|
||||
ZIP_CHECK="${RSYNC_TARGET}/${LAST_DATE}/${CONFIG_VM_NAME}-${VM_ID_ORIGINAL}.7z"
|
||||
if [[ "$SKIP_RSYNC" == "1" ]]; then
|
||||
if [[ -f "$ZIP_CHECK" ]]; then
|
||||
echo " ZIP bereits vorhanden (lokal): $ZIP_CHECK"
|
||||
VM_NAME="$CONFIG_VM_NAME"
|
||||
ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_CHECK" 2>/dev/null || echo "0")
|
||||
RSYNC_OK="true"
|
||||
RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
|
||||
QM_AGENT_OK="skipped"
|
||||
trap - ERR
|
||||
send_webhook "success" ""
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
if ssh "$BACKUP_SERVER_HOST" "test -f '$ZIP_CHECK'" 2>/dev/null; then
|
||||
echo " ZIP bereits vorhanden (remote): $ZIP_CHECK"
|
||||
VM_NAME="$CONFIG_VM_NAME"
|
||||
ZIP_SIZE_BYTES=$(ssh "$BACKUP_SERVER_HOST" \
|
||||
"stat -c%s '$ZIP_CHECK'" 2>/dev/null || echo "0")
|
||||
RSYNC_OK="true"
|
||||
RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
|
||||
QM_AGENT_OK="skipped"
|
||||
trap - ERR
|
||||
send_webhook "success" ""
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo " Kein vorhandenes ZIP – starte vollständigen Restore."
|
||||
fi
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [3/13] RESTORE
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [3/13] Restore vom PBS-Storage ($BACKUP_TYPE)..."
|
||||
echo " Backup-Ref: $PVE_BACKUP_REF"
|
||||
echo " Storage: $RESTORE_PATH"
|
||||
echo " ID: $VM_ID_RESTORED"
|
||||
|
||||
RESTORE_START_INNER=$(date +%s)
|
||||
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct restore "$VM_ID_RESTORED" "$PVE_BACKUP_REF" \
|
||||
--storage "$RESTORE_PATH" \
|
||||
--unique 1 \
|
||||
2>&1
|
||||
else
|
||||
qmrestore "$PVE_BACKUP_REF" "$VM_ID_RESTORED" \
|
||||
--storage "$RESTORE_PATH" \
|
||||
--unique 1 \
|
||||
2>&1
|
||||
fi
|
||||
|
||||
RESTORE_DURATION=$(( $(date +%s) - RESTORE_START_INNER ))
|
||||
echo " Restore abgeschlossen in ${RESTORE_DURATION}s"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [4/13] IMAGE_DIR DYNAMISCH ERMITTELN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [4/13] Ermittle Image-Verzeichnis..."
|
||||
STORAGE_BASE=$(pvesh get "/storage/${RESTORE_PATH}" --output-format json \
|
||||
2>/dev/null | python3 -c "
|
||||
import json, sys
|
||||
cfg = json.load(sys.stdin)
|
||||
print(cfg.get('path', ''))
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$STORAGE_BASE" ]]; then
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
IMAGE_DIR=""
|
||||
for candidate in \
|
||||
"${STORAGE_BASE}/images/${VM_ID_RESTORED}" \
|
||||
"${STORAGE_BASE}/private/${VM_ID_RESTORED}" \
|
||||
"${STORAGE_BASE}/rootdir/${VM_ID_RESTORED}"; do
|
||||
if [[ -d "$candidate" ]] && [[ -n "$(ls -A "$candidate" 2>/dev/null)" ]]; then
|
||||
IMAGE_DIR="$candidate"
|
||||
echo " CT-Image gefunden: $IMAGE_DIR"
|
||||
break
|
||||
else
|
||||
echo " Nicht gefunden: $candidate"
|
||||
fi
|
||||
done
|
||||
if [[ -z "$IMAGE_DIR" ]]; then
|
||||
IMAGE_DIR=$(find "$STORAGE_BASE" -maxdepth 2 -type d \
|
||||
-name "$VM_ID_RESTORED" 2>/dev/null | head -1 || echo "")
|
||||
[[ -n "$IMAGE_DIR" ]] && echo " CT-Image via find: $IMAGE_DIR"
|
||||
fi
|
||||
else
|
||||
IMAGE_DIR="${STORAGE_BASE}/images/${VM_ID_RESTORED}"
|
||||
fi
|
||||
else
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}"
|
||||
else
|
||||
IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}"
|
||||
fi
|
||||
echo " WARNUNG: Storage-Pfad nicht ermittelt, Fallback: $IMAGE_DIR"
|
||||
fi
|
||||
|
||||
if [[ -z "$IMAGE_DIR" ]]; then
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
IMAGE_DIR="/var/lib/vz/private/${VM_ID_RESTORED}"
|
||||
else
|
||||
IMAGE_DIR="/var/lib/vz/images/${VM_ID_RESTORED}"
|
||||
fi
|
||||
echo " WARNUNG: Fallback: $IMAGE_DIR"
|
||||
fi
|
||||
echo " Image-Dir: $IMAGE_DIR"
|
||||
|
||||
ACTUAL_DISK_BYTES=$(du -sb "$IMAGE_DIR" 2>/dev/null | awk '{print $1}' || echo "0")
|
||||
echo " Image-Größe: $(( ACTUAL_DISK_BYTES / 1024 / 1024 / 1024 )) GB"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [5/13] IMAGES PRÜFEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [5/13] Prüfe Images..."
|
||||
if [[ ! -d "$IMAGE_DIR" ]] || [[ -z "$(ls -A "$IMAGE_DIR" 2>/dev/null)" ]]; then
|
||||
ERROR_MSG="IMAGE_DIR leer oder nicht vorhanden: $IMAGE_DIR"
|
||||
echo " FEHLER: $ERROR_MSG"
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct destroy "$VM_ID_RESTORED" --purge 1 2>/dev/null || true
|
||||
else
|
||||
qm destroy "$VM_ID_RESTORED" \
|
||||
--destroy-unreferenced-disks 1 --purge 1 2>/dev/null || true
|
||||
fi
|
||||
trap - ERR
|
||||
send_webhook "failed" "$ERROR_MSG"
|
||||
exit 0
|
||||
fi
|
||||
echo " Images vorhanden ✓"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [6/13] VORBEREITEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [6/13] Vorbereiten ($BACKUP_TYPE)..."
|
||||
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct unlock "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
pct stop "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
sleep 3
|
||||
for ((net=0; net<=10; net++)); do
|
||||
pct set "$VM_ID_RESTORED" --delete "net${net}" 2>/dev/null || true
|
||||
done
|
||||
echo " CT vorbereitet (Netzwerkkarten entfernt)."
|
||||
else
|
||||
qm unlock "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
qm stop "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
sleep 3
|
||||
qm set "$VM_ID_RESTORED" -delete cdrom 2>/dev/null || true
|
||||
qm set "$VM_ID_RESTORED" -delete ide0 2>/dev/null || true
|
||||
for ((net=0; net<=10; net++)); do
|
||||
qm set "$VM_ID_RESTORED" -delete "net${net}" 2>/dev/null || true
|
||||
done
|
||||
qm set "$VM_ID_RESTORED" --agent "enabled=1,type=virtio" 2>/dev/null || true
|
||||
echo " VM vorbereitet (Netzwerkkarten entfernt, Agent aktiviert)."
|
||||
fi
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [7/13] STARTEN & PRÜFEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [7/13] Starte & prüfe ($BACKUP_TYPE)..."
|
||||
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct start "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
sleep 10
|
||||
if pct status "$VM_ID_RESTORED" 2>/dev/null | grep -q "running"; then
|
||||
QM_AGENT_OK="true"
|
||||
echo " CT läuft ✓"
|
||||
CT_HOSTNAME=$(pct exec "$VM_ID_RESTORED" -- hostname 2>/dev/null || echo "unbekannt")
|
||||
echo " Hostname: $CT_HOSTNAME"
|
||||
else
|
||||
QM_AGENT_OK="false"
|
||||
echo " CT nicht gestartet."
|
||||
fi
|
||||
else
|
||||
qm start "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
AGENT_WAIT=0
|
||||
AGENT_MAX=120
|
||||
AGENT_INTERVAL=10
|
||||
while [[ $AGENT_WAIT -lt $AGENT_MAX ]]; do
|
||||
sleep $AGENT_INTERVAL
|
||||
AGENT_WAIT=$(( AGENT_WAIT + AGENT_INTERVAL ))
|
||||
echo -n " [${AGENT_WAIT}s/${AGENT_MAX}s] qm-Agent... "
|
||||
if qm agent "$VM_ID_RESTORED" ping 2>/dev/null | grep -qi "pong\|ping"; then
|
||||
QM_AGENT_OK="true"
|
||||
echo "ONLINE ✓"
|
||||
hostname_info=$(qm agent "$VM_ID_RESTORED" get-host-name 2>/dev/null \
|
||||
| grep host-name | tr -d '"' || true)
|
||||
echo " Hostname: ${hostname_info:-unbekannt}"
|
||||
break
|
||||
else
|
||||
echo "nicht erreichbar."
|
||||
fi
|
||||
done
|
||||
[[ "$QM_AGENT_OK" == "false" ]] && \
|
||||
echo " qm-Agent nicht erreichbar – qm_agent_ok=false in DB."
|
||||
fi
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [8/13] STOPPEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [8/13] Stoppe $BACKUP_TYPE..."
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct stop "$VM_ID_RESTORED" 2>/dev/null || true
|
||||
sleep 10
|
||||
else
|
||||
# Graceful shutdown mit 2 Minuten Timeout, danach force-stop
|
||||
qm shutdown "$VM_ID_RESTORED" --timeout 120 2>/dev/null || true
|
||||
# Prüfen ob VM noch läuft → force-stop
|
||||
if qm status "$VM_ID_RESTORED" 2>/dev/null | grep -q "running"; then
|
||||
echo " VM läuft noch nach 120s – force stop..."
|
||||
qm stop "$VM_ID_RESTORED" --skiplock 1 2>/dev/null || true
|
||||
sleep 5
|
||||
fi
|
||||
fi
|
||||
echo " Gestoppt."
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [9/13] CONFIG SICHERN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [9/13] Config sichern..."
|
||||
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
PVE_CONF="/etc/pve/lxc/${VM_ID_RESTORED}.conf"
|
||||
CONF_FILENAME="lxc.conf"
|
||||
VM_NAME=$(grep -m1 "^hostname:" "$PVE_CONF" 2>/dev/null \
|
||||
| awk -F': ' '{print $2}' | tr -d '[:space:]' \
|
||||
|| echo "${CONFIG_VM_NAME:-$SAFE_CLIENT}")
|
||||
else
|
||||
PVE_CONF="/etc/pve/qemu-server/${VM_ID_RESTORED}.conf"
|
||||
CONF_FILENAME="qemu-server.conf"
|
||||
VM_NAME=$(grep -m1 "^name:" "$PVE_CONF" 2>/dev/null \
|
||||
| awk -F': ' '{print $2}' | tr -d '[:space:]' \
|
||||
|| echo "${CONFIG_VM_NAME:-$SAFE_CLIENT}")
|
||||
fi
|
||||
|
||||
if [[ -f "$PVE_CONF" ]]; then
|
||||
cp "$PVE_CONF" "${IMAGE_DIR}/${CONF_FILENAME}"
|
||||
echo " Config gesichert: ${IMAGE_DIR}/${CONF_FILENAME}"
|
||||
else
|
||||
echo " WARNUNG: Config nicht gefunden: $PVE_CONF"
|
||||
fi
|
||||
echo " Name: $VM_NAME"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [10/13] 7Z-ARCHIV
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [10/13] Erstelle verschlüsseltes 7z-Archiv (mx=${COMPRESS_LEVEL})..."
|
||||
|
||||
ZIP_FILE="${ZIP_DIR}/${VM_NAME}-${VM_ID_ORIGINAL}.7z"
|
||||
ZIP_START=$(date +%s)
|
||||
|
||||
7z a -t7z \
|
||||
-mmt=${MMT_THREADS} \
|
||||
-mx=${COMPRESS_LEVEL} \
|
||||
-md=16M \
|
||||
-p"${ZIP_PASSWORD}" \
|
||||
-mhe=on \
|
||||
"$ZIP_FILE" \
|
||||
"${IMAGE_DIR}/"* \
|
||||
2>&1 | tail -5
|
||||
|
||||
ZIP_DURATION=$(( $(date +%s) - ZIP_START ))
|
||||
ZIP_SIZE_BYTES=$(stat -c%s "$ZIP_FILE" 2>/dev/null || echo "0")
|
||||
echo " ZIP: $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB in ${ZIP_DURATION}s"
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [11/13] RSYNC ZUM BACKUP-SERVER
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
RSYNC_TARGET_DATE="${RSYNC_TARGET}/${LAST_DATE}"
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [11/13] Rsync / Datei-Transfer..."
|
||||
|
||||
if [[ "$SKIP_RSYNC" == "1" ]]; then
|
||||
echo " Lokaler Modus: ZIP bereits in ${RSYNC_TARGET_DATE} – kein Rsync."
|
||||
RSYNC_OK="true"
|
||||
RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
|
||||
echo " Groesse: $(( RSYNC_SIZE_BYTES / 1024 / 1024 )) MB"
|
||||
else
|
||||
MAX_RETRIES=3
|
||||
|
||||
rsync_transfer() {
|
||||
rsync -avz --progress --timeout=300 \
|
||||
"$ZIP_FILE" \
|
||||
"${BACKUP_SERVER_HOST}:${RSYNC_TARGET_DATE}/" \
|
||||
2>&1
|
||||
}
|
||||
|
||||
ssh "$BACKUP_SERVER_HOST" "mkdir -p '${RSYNC_TARGET_DATE}'" 2>/dev/null || true
|
||||
|
||||
while [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; do
|
||||
if rsync_transfer; then
|
||||
RSYNC_OK="true"
|
||||
RSYNC_SIZE_BYTES=$ZIP_SIZE_BYTES
|
||||
echo " Rsync OK: $(( RSYNC_SIZE_BYTES / 1024 / 1024 )) MB"
|
||||
break
|
||||
else
|
||||
RSYNC_RETRIES=$(( RSYNC_RETRIES + 1 ))
|
||||
if [[ $RSYNC_RETRIES -lt $MAX_RETRIES ]]; then
|
||||
echo " Fehlgeschlagen ($RSYNC_RETRIES/$MAX_RETRIES). Warte 60s..."
|
||||
sleep 60
|
||||
else
|
||||
RSYNC_OK="false"
|
||||
STATUS="failed"
|
||||
ERROR_MSG="Rsync fehlgeschlagen nach ${RSYNC_RETRIES} Versuchen"
|
||||
echo " FEHLER: $ERROR_MSG"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$RSYNC_OK" == "true" ]]; then
|
||||
REMOTE_SIZE=$(ssh "$BACKUP_SERVER_HOST" \
|
||||
"stat -c%s '${RSYNC_TARGET_DATE}/$(basename "$ZIP_FILE")'" \
|
||||
2>/dev/null || echo "0")
|
||||
if [[ "$REMOTE_SIZE" != "$ZIP_SIZE_BYTES" ]]; then
|
||||
echo " WARNUNG: Remote ${REMOTE_SIZE}B != lokal ${ZIP_SIZE_BYTES}B"
|
||||
RSYNC_OK="false"
|
||||
STATUS="failed"
|
||||
ERROR_MSG="Groessenabweichung: lokal=${ZIP_SIZE_BYTES} remote=${REMOTE_SIZE}"
|
||||
else
|
||||
echo " Groessenprüfung OK: ${REMOTE_SIZE} Bytes."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# [12/13] AUFRÄUMEN
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
echo ""
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') ==> [12/13] Aufräumen..."
|
||||
if [[ "$BACKUP_TYPE" == "ct" ]]; then
|
||||
pct destroy "$VM_ID_RESTORED" --purge 1 \
|
||||
2>/dev/null || echo " CT $VM_ID_RESTORED nicht mehr vorhanden."
|
||||
else
|
||||
qm destroy "$VM_ID_RESTORED" \
|
||||
--destroy-unreferenced-disks 1 \
|
||||
--purge 1 \
|
||||
2>/dev/null || echo " VM $VM_ID_RESTORED nicht mehr vorhanden."
|
||||
fi
|
||||
if [[ "$SKIP_RSYNC" == "1" ]]; then
|
||||
echo " ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt. ZIP bleibt am Zielort."
|
||||
else
|
||||
rm -f "$ZIP_FILE"
|
||||
echo " ${BACKUP_TYPE^^} ${VM_ID_RESTORED} entfernt, ZIP gelöscht."
|
||||
fi
|
||||
|
||||
# ── Zusammenfassung & Webhook ─────────────────────────────────────────────────
|
||||
TOTAL=$(( $(date +%s) - RESTORE_START ))
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo " Status: $STATUS"
|
||||
echo " Typ: $BACKUP_TYPE"
|
||||
echo " Gesamtdauer: ${TOTAL}s"
|
||||
echo " Name: ${VM_NAME:-$SAFE_CLIENT}"
|
||||
echo " Image-Dir: $IMAGE_DIR"
|
||||
echo " qm-Agent/CT: $QM_AGENT_OK"
|
||||
echo " Rsync: $RSYNC_OK (Versuche: $RSYNC_RETRIES)"
|
||||
echo " ZIP: $(( ZIP_SIZE_BYTES / 1024 / 1024 )) MB"
|
||||
echo " 7z-Level: mx=${COMPRESS_LEVEL} mmt=${MMT_THREADS}"
|
||||
[[ -n "$ERROR_MSG" ]] && echo " Fehler: $ERROR_MSG"
|
||||
echo "============================================================"
|
||||
|
||||
trap - ERR
|
||||
send_webhook "$STATUS" "$ERROR_MSG"
|
||||
Reference in New Issue
Block a user