f2a979f047
The repo is public and files are fetched by raw URL, so a reader who lands on one runbook never sees the README -- the repo's context does not travel with the file. Each .md now carries two lines under the title, each .ps1 the equivalent at the end of its .NOTES block. Deliberately two lines, not a paragraph. These files are read through `| more` on a client console mid-incident, and the top of the file is where the procedure-specific warnings live -- never a live chart, stop the service before copying, confirm authorization before acting. A legal preamble above those competes with them and trains people to skip past. Wording aims at a stranger who found the repo, not at the quality of the procedure: these double as documented-procedure evidence for E&O, and language implying the content is unreliable works against that. MIT rather than no license: the warranty and liability disclaimer is the part that does the work, and leaving it unlicensed makes reuse ambiguous rather than disclaimed. Also fixes 5 stale ops/rb URLs in scripts/*.ps1 that the previous commit missed -- it only swept the .md files. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HwcG1jLs1T425QRMxtjxP7
152 lines
6.8 KiB
PowerShell
152 lines
6.8 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Read-only health check of the latest Open Dental cold backup: freshness,
|
|
completeness, and size sanity. Optionally pings a dead-man's-switch heartbeat
|
|
URL on success so an external monitor alerts when a backup silently stops.
|
|
|
|
.DESCRIPTION
|
|
Points at the backup destination root (<DEST>) produced by od-db-backup.ps1,
|
|
finds the newest od-backup-<timestamp> folder, and verifies it is:
|
|
- FRESH : written within the max-age window (default 24h)
|
|
- COMPLETE : has data\, (optionally) OpenDentImages\, MANIFEST.txt, and a
|
|
backup.log that ends in success with no ERROR/CRITICAL lines
|
|
- SANE SIZE : data\ isn't near-empty, and isn't a fraction of the prior run
|
|
Prints a clear PASS/FAIL summary, appends a line to <DEST>\backup-check.log,
|
|
and — if healthy and a heartbeat URL was given — pings it. Pinging ONLY on
|
|
success is deliberate: if this check fails, crashes, or the whole server is
|
|
off, no ping arrives and the external monitor raises the alarm. "No news" is
|
|
never mistaken for good news.
|
|
|
|
Fully READ-ONLY: it does not touch the database, the service, or the backup
|
|
files. Safe to schedule and safe to run ad hoc.
|
|
|
|
.NOTES
|
|
Manual spot-check:
|
|
irm rb.godwinsystems.com/scripts/od-backup-check.ps1 | iex
|
|
|
|
For unattended monitoring, schedule a filled-in LOCAL copy (with <DEST> and the
|
|
heartbeat URL baked in) — keep that copy in the private tier, not here. See
|
|
od-backup-schedule.md.
|
|
|
|
iex-safe: uses `return`, never `exit` (which would close an interactive
|
|
session). PUBLIC REPO: no client specifics hard-coded; prompt at run time.
|
|
|
|
Provided as-is, without warranty. This runs in your session via `iex` —
|
|
read it before you run it. You are responsible for the systems you run
|
|
it on. See LICENSE.
|
|
#>
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
Write-Host '== Open Dental: backup health check (read-only) ==' -ForegroundColor Cyan
|
|
|
|
# --- Prompts (no param block; iex-safe) ---
|
|
$destRoot = Read-Host 'Backup destination root (<DEST>, e.g. E:\OD-Backups)'
|
|
if (-not (Test-Path $destRoot)) {
|
|
Write-Warning "Destination not found: $destRoot"
|
|
return
|
|
}
|
|
$maxAgeInput = Read-Host 'Max age (hours) for a healthy backup [24]'
|
|
$maxAgeHrs = if ($maxAgeInput -match '^\d+$') { [int]$maxAgeInput } else { 24 }
|
|
$expectImages = (Read-Host 'Should each backup include an OpenDentImages folder? (Y/n)') -ne 'n'
|
|
$heartbeat = Read-Host 'Heartbeat/ping URL to hit on SUCCESS (<HEARTBEAT_URL>) [blank to skip]'
|
|
|
|
$problems = New-Object System.Collections.Generic.List[string]
|
|
function Fail($msg) { $problems.Add($msg) }
|
|
|
|
function Get-DirSizeBytes($path) {
|
|
if (-not (Test-Path $path)) { return $null }
|
|
(Get-ChildItem -LiteralPath $path -Recurse -Force -File -ErrorAction SilentlyContinue |
|
|
Measure-Object -Property Length -Sum).Sum
|
|
}
|
|
|
|
# --- Find the newest backup folder ---
|
|
$folders = @(Get-ChildItem -LiteralPath $destRoot -Directory -Filter 'od-backup-*' -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime -Descending)
|
|
if ($folders.Count -eq 0) {
|
|
Fail "No od-backup-* folders found under $destRoot."
|
|
}
|
|
else {
|
|
$latest = $folders[0]
|
|
Write-Host "Latest backup: $($latest.Name) ($($latest.LastWriteTime))" -ForegroundColor Green
|
|
|
|
# 1) Freshness
|
|
$ageHrs = [math]::Round(((Get-Date) - $latest.LastWriteTime).TotalHours, 1)
|
|
if ($ageHrs -gt $maxAgeHrs) {
|
|
Fail "Stale: newest backup is $ageHrs h old (limit $maxAgeHrs h). Backup may have stopped running."
|
|
}
|
|
if (((Get-Date) - $latest.LastWriteTime).TotalMinutes -lt 5) {
|
|
Write-Warning 'Newest folder was written <5 min ago — a backup may be in progress. Re-check after it finishes.'
|
|
}
|
|
|
|
# 2) Completeness
|
|
$dataDir = Join-Path $latest.FullName 'data'
|
|
$imgDir = Join-Path $latest.FullName 'OpenDentImages'
|
|
$manifest = Join-Path $latest.FullName 'MANIFEST.txt'
|
|
$logFile = Join-Path $latest.FullName 'backup.log'
|
|
|
|
if (-not (Test-Path $dataDir)) { Fail "Missing data\ directory in $($latest.Name)." }
|
|
if (-not (Test-Path $manifest)) { Fail "Missing MANIFEST.txt (backup may not have completed) in $($latest.Name)." }
|
|
if ($expectImages -and -not (Test-Path $imgDir)) {
|
|
Fail "Missing OpenDentImages\ in $($latest.Name) — a DB-only backup loses all documents."
|
|
}
|
|
if (Test-Path $logFile) {
|
|
$tail = Get-Content -LiteralPath $logFile -Tail 40 -ErrorAction SilentlyContinue
|
|
if ($tail -match 'ERROR|CRITICAL') { Fail "backup.log contains ERROR/CRITICAL lines in $($latest.Name)." }
|
|
if (-not ($tail -match 'completed successfully')) {
|
|
Fail "backup.log does not show a successful completion in $($latest.Name)."
|
|
}
|
|
} else {
|
|
Fail "Missing backup.log in $($latest.Name)."
|
|
}
|
|
|
|
# 3) Size sanity
|
|
$dataBytes = Get-DirSizeBytes $dataDir
|
|
if ($null -ne $dataBytes) {
|
|
$dataMB = [math]::Round($dataBytes / 1MB, 1)
|
|
Write-Host " data\ size: $dataMB MB"
|
|
if ($dataBytes -lt 1MB) { Fail "data\ is only $dataMB MB — implausibly small; likely a broken/partial backup." }
|
|
if ($folders.Count -gt 1) {
|
|
$prevData = Get-DirSizeBytes (Join-Path $folders[1].FullName 'data')
|
|
if ($prevData -and $dataBytes -lt ($prevData * 0.5)) {
|
|
Fail ("data\ shrank to {0}% of the previous run ({1} MB -> {2} MB) — possible truncation." -f `
|
|
[math]::Round($dataBytes / $prevData * 100), [math]::Round($prevData/1MB,1), $dataMB)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# --- Report ---
|
|
$healthy = ($problems.Count -eq 0)
|
|
$stamp = Get-Date -Format 'yyyy-MM-dd HH:mm:ss'
|
|
Write-Host ''
|
|
if ($healthy) {
|
|
Write-Host "PASS Backup is healthy as of $stamp." -ForegroundColor Green
|
|
} else {
|
|
Write-Host "FAIL Backup health problems:" -ForegroundColor Red
|
|
$problems | ForEach-Object { Write-Host " - $_" -ForegroundColor Red }
|
|
}
|
|
|
|
# On-disk trail at the destination
|
|
try {
|
|
$line = "$stamp $(if($healthy){'PASS'}else{'FAIL'}) $(if($healthy){'healthy'}else{$problems -join ' | '})"
|
|
Add-Content -LiteralPath (Join-Path $destRoot 'backup-check.log') -Value $line
|
|
} catch { Write-Warning "Could not write backup-check.log: $($_.Exception.Message)" }
|
|
|
|
# --- Dead-man's-switch heartbeat: ping ONLY on success ---
|
|
if ($heartbeat) {
|
|
if ($healthy) {
|
|
try {
|
|
Invoke-WebRequest -Uri $heartbeat -UseBasicParsing -TimeoutSec 20 -Method Get | Out-Null
|
|
Write-Host 'Heartbeat ping sent.' -ForegroundColor Green
|
|
} catch {
|
|
Write-Warning "Heartbeat ping failed: $($_.Exception.Message) (monitor will alarm on the missing check-in)."
|
|
}
|
|
} else {
|
|
Write-Host 'Unhealthy — NOT pinging heartbeat. The external monitor should alert on the missing ping.' -ForegroundColor Yellow
|
|
Write-Host '(healthchecks.io users: append /fail to the URL for an immediate alert.)'
|
|
}
|
|
}
|
|
|
|
if (-not $healthy) { Write-Warning 'Investigate now — see od-backup-schedule.md triage.' }
|