#!/bin/bash
###############################################################################
# WordPress / WooCommerce Daily Health-Check Script  (v2.9.1)
# Changes vs v2.9:
#   - FIX: Force CLI PHP SAPI for WP-CLI. On cPanel, /usr/local/bin/wp
#     silently ran under 'cgi-fcgi', which printed HTTP headers to stdout
#     ("X-Powered-By: PHP/8.2.31 ..."). Those got captured into $AS_FAILED,
#     $AS_PENDING, $AS_INPROG etc., producing the errors:
#        line NN: [: X-Powered-By...: integer expression expected
#     Now every WP-CLI call goes through $PHP_CLI $WP_PHAR explicitly, and
#     wpq() has a numeric-only guard that returns "NA" on garbage output.
#   - FIX: Removed illegal 'local' declaration outside a function
#     ("local: can only be used in a function") in the CRIT dispatch block.
#   - FIX: Numeric guards on CRON_OVERDUE capture as well.
#   - FIX: SAPI sanity check at startup; abort with clear message if the
#     configured PHP binary isn't CLI.
# Changes vs v2.8:
#   - WEEKLY_REPORT_SITES allowlist: weekly digest can now filter to a subset
#     of sites (default: "prod"). Prevents legacy dev rows from lingering in
#     the weekly HTML while the CSV file still contains them.
#   - "As of" metadata block in the weekly HTML: shows which sites are
#     included and how many legacy rows from excluded sites were skipped.
#   - Auto-purge of stale state entries for sites no longer in SITES:
#     any dev alerts still in .alert_state now fire one RESOLVED email
#     and are cleaned up, so removed sites don't linger indefinitely.
# Changes vs v2.7:
#   - Dropped 'dev' from routine monitoring
#   - Rich alert emails with per-rule playbooks
# Changes vs v2.6:
#   - WARN safety nets: GROWTH re-alert (>=100%) + STALE reminder (>=7d)
###############################################################################
# WordPress / WooCommerce Daily Health-Check Script  (v2.6)
# Changes vs v2.5:
#   - CRITICAL now sends periodic REMINDERS (default: every 24h) as requested.
#   - WARN is now FIRE-ONCE + auto-resolved (no more daily rollup spam
#     while the same warnings persist).
#   - Replaced the arbitrary "50% growth" escalation with SEVERITY BANDS,
#     which is far more intuitive:
#       * Each critical rule has a "severe" threshold.
#       * Crossing UP into the severe band forces an immediate re-alert
#         regardless of cooldown.
#       * Small drifts (e.g. RAM 96 -> 97) no longer spam email.
#   - State file now records the band the alert was sent at
#       Format: site|rule|last_epoch|last_value|last_band
#       Bands: warn | critical | severe
# Schedule: 55 5 * * * /home/digitechstores/scripts/wp_health_check.sh
###############################################################################
set -u

# --------------------------- CONFIG ------------------------------------------
SITES=(
  "prod|/home/digitechstores/public_html|/home/digitechstores/public_html/error_log"
  # Uncomment during release weeks / debugging to also monitor dev:
  # "dev|/home/digitechstores/devnew.digitechstores.com|/home/digitechstores/devnew.digitechstores.com/error_log"
)
REPORT_DIR="/home/digitechstores/health_reports"

# ---- WP-CLI invocation (v2.9.1) ---------------------------------------------
# On cPanel, /usr/local/bin/wp is often a wrapper that resolves to 'php-cgi',
# not the 'cli' SAPI. WP-CLI refuses to run under CGI and prints HTTP headers
# to stdout. Those get captured into variables and blow up numeric comparisons.
# Fix: always invoke the CLI PHP binary explicitly with the wp.phar.
PHP_CLI="/opt/cpanel/ea-php82/root/usr/bin/php"
WP_PHAR="/usr/local/bin/wp"
WP_BIN="${PHP_CLI} ${WP_PHAR}"     # kept for backward-compat with any old refs

TABLE_PREFIX="wp7e_"
LOOKBACK_MIN=1440

# ---- Weekly report ----
WEEKLY_REPORT_DOW=7
WEEKLY_RECIPIENTS="munzir.mukhtar@ctcgroupltd.com"
# WEEKLY_RECIPIENTS="munzir.mukhtar@ctcgroupltd.com,mohamed.gsmallah@ctcgroupltd.com,osman.eltayeb@ctcgroupltd.com"
# WEEKLY_CC="yazeed.awad@ctcgroupltd.com,mohamed.abnaof@ctcgroupltd.com"
WEEKLY_FROM="digitechstores@server.digitechstores.com"
WEEKLY_SUBJECT_PREFIX="[Weekly WP Health Report] Digitech Stores"
# Which sites to include in the weekly summary (comma-separated allowlist).
# Legacy rows from other sites still in the CSV will be skipped and reported
# as "legacy_skipped" in the HTML header.
WEEKLY_REPORT_SITES="prod"

# ---- Alerts (v2.3+) ----
ALERT_RECIPIENTS="munzir.mukhtar@ctcgroupltd.com"
ALERT_CC=""
ALERT_FROM="digitechstores@server.digitechstores.com"
ALERT_CRIT_SUBJECT_PREFIX="[CRITICAL - WP HEALTH] Digitech Stores"
ALERT_SEVERE_SUBJECT_PREFIX="[SEVERE - WP HEALTH] Digitech Stores"
ALERT_WARN_SUBJECT_PREFIX="[WARN - WP HEALTH] Digitech Stores"
ALERT_RESOLVED_SUBJECT_PREFIX="[RESOLVED - WP HEALTH] Digitech Stores"

# ---- REMINDER POLICY (v2.6) ----
#   CRITICAL alerts:  reminder every COOLDOWN_HOURS while still active
#                     PLUS immediate re-alert if it crosses into SEVERE band
#   WARN alerts:      FIRE-ONCE + auto RESOLVED email when it clears
#                     (no reminders, since warnings are non-urgent)
CRITICAL_REMINDERS_ENABLED=true
COOLDOWN_HOURS=24

# WARN safety nets (v2.7)
WARN_GROWTH_PCT=100     # re-alert if metric grows >= this % vs last WARN email
WARN_STALE_DAYS=7       # send a "still active" reminder after this many days

ALERT_STATE_FILE="${REPORT_DIR}/.alert_state"

# ---- Thresholds ----
# WARN = "should investigate" (fire once, no reminders)
# CRIT = "incident, act now" (reminders on)
# SEVERE = "getting worse - force re-alert" (bypasses cooldown)

# Commands out of sync (baseline ~2/day normal WPML noise)
WARN_COOS=10
CRIT_COOS=50
SEVERE_COOS=200

# PHP Fatal errors (any is bad; severe = clearly bleeding)
CRIT_FATAL=0
SEVERE_FATAL=20

# WP DB errors
WARN_DBERR=0

# Action Scheduler failed jobs
WARN_AS_FAILED=5

# Expired transients (housekeeping)
WARN_TRANS_EXPIRED=5000

# WP-Cron overdue
WARN_CRON_OVERDUE=5

# Memory %
WARN_RAM_PCT=85
CRIT_RAM_PCT=95
SEVERE_RAM_PCT=98

# Disk %
WARN_DISK_PCT=85
CRIT_DISK_PCT=90
SEVERE_DISK_PCT=95

# Load average (5m avg) — warn if > vCPU * multiplier
WARN_LOAD5_MULT=1.5
# ------------------------ END CONFIG -----------------------------------------

# ---- SAPI SANITY CHECK (v2.9.1) ---------------------------------------------
# Abort loudly if PHP_CLI isn't actually the CLI SAPI. This is the single most
# common cause of the "X-Powered-By: ..." garbage-value bug on cPanel hosts.
if [ ! -x "$PHP_CLI" ]; then
  echo "[FATAL] PHP_CLI not executable: $PHP_CLI" >&2
  echo "        Edit wp_health_check.sh and set PHP_CLI to the CLI PHP binary." >&2
  echo "        Try: ls /opt/cpanel/ea-php*/root/usr/bin/php" >&2
  exit 2
fi
if ! "$PHP_CLI" -v 2>/dev/null | head -1 | grep -qi 'cli'; then
  echo "[FATAL] $PHP_CLI is not the CLI SAPI. Refusing to run to avoid" >&2
  echo "        capturing HTTP headers as data values." >&2
  echo "        Actual PHP -v output:" >&2
  "$PHP_CLI" -v 2>&1 | sed 's/^/          /' >&2
  exit 2
fi
if [ ! -f "$WP_PHAR" ]; then
  echo "[FATAL] WP_PHAR not found: $WP_PHAR" >&2
  exit 2
fi
# -----------------------------------------------------------------------------

mkdir -p "$REPORT_DIR"
MONTH_TAG=$(date -u +'%Y%m')
CSV_FILE="${REPORT_DIR}/wp_health_${MONTH_TAG}.csv"
TS_UTC=$(date -u +'%Y-%m-%d %H:%M:%S')
TS_LOCAL=$(date +'%Y-%m-%d %H:%M:%S %Z')
TODAY_DOW=$(date +%u)
HOSTNAME_STR=$(hostname 2>/dev/null || echo "unknown-host")
VCPU_COUNT=$(nproc 2>/dev/null || echo 1)

UPTIME_STR=$(uptime -p 2>/dev/null | sed 's/^up //' | tr ',' ' ')
LOADAVG=$(awk '{print $1"|"$2"|"$3}' /proc/loadavg 2>/dev/null || echo "NA|NA|NA")
LOAD_1=$(echo "$LOADAVG"  | cut -d'|' -f1)
LOAD_5=$(echo "$LOADAVG"  | cut -d'|' -f2)
LOAD_15=$(echo "$LOADAVG" | cut -d'|' -f3)

if command -v free >/dev/null 2>&1; then
  RAM_TOTAL_MB=$(free -m | awk '/^Mem:/ {print $2}')
  RAM_USED_MB=$(free -m  | awk '/^Mem:/ {print $3}')
  RAM_PCT=$(awk -v u="$RAM_USED_MB" -v t="$RAM_TOTAL_MB" 'BEGIN{ if (t>0) printf "%.1f", (u/t)*100; else print "NA" }')
else
  RAM_TOTAL_MB="NA"; RAM_USED_MB="NA"; RAM_PCT="NA"
fi

if [ -r /proc/stat ]; then
  read cpu a b c idle rest < /proc/stat
  prev_total=$((a+b+c+idle)); prev_idle=$idle
  sleep 1
  read cpu a b c idle rest < /proc/stat
  total=$((a+b+c+idle))
  diff_total=$((total-prev_total)); diff_idle=$((idle-prev_idle))
  if [ "$diff_total" -gt 0 ]; then
    CPU_PCT=$(awk -v t="$diff_total" -v i="$diff_idle" 'BEGIN{ printf "%.1f", (1 - i/t)*100 }')
  else
    CPU_PCT="NA"
  fi
else
  CPU_PCT="NA"
fi

DISK_PCT=$(df -P /home 2>/dev/null | awk 'NR==2 {gsub("%",""); print $5}')
[ -z "${DISK_PCT:-}" ] && DISK_PCT="NA"

if [ ! -f "$CSV_FILE" ]; then
  echo "timestamp_utc,timestamp_local,site,uptime,load_1m,load_5m,load_15m,ram_used_mb,ram_total_mb,ram_pct,cpu_pct,disk_pct,commands_out_of_sync_24h,php_fatal_24h,db_error_24h,transients_total,transients_expired,options_rows,as_pending,as_in_progress,as_failed,as_complete_24h,cron_overdue" > "$CSV_FILE"
fi

###############################################################################
# HELPERS
###############################################################################
count_recent_log_matches() {
  local logfile="$1"; local pattern="$2"
  [ -r "$logfile" ] || { echo 0; return; }
  local cutoff_epoch
  cutoff_epoch=$(date -u -d "-${LOOKBACK_MIN} minutes" +%s 2>/dev/null)
  if [ -z "$cutoff_epoch" ]; then
    grep -c "$pattern" "$logfile" 2>/dev/null || echo 0
    return
  fi
  awk -v cutoff="$cutoff_epoch" -v pat="$pattern" '
    BEGIN{ count=0 }
    {
      if (match($0, /^\[([0-9]{2})-([A-Za-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2})/, m)) {
        mo["Jan"]=1;mo["Feb"]=2;mo["Mar"]=3;mo["Apr"]=4;mo["May"]=5;mo["Jun"]=6;
        mo["Jul"]=7;mo["Aug"]=8;mo["Sep"]=9;mo["Oct"]=10;mo["Nov"]=11;mo["Dec"]=12;
        ts_str = sprintf("%04d %02d %02d %02d %02d %02d", m[3], mo[m[2]], m[1], m[4], m[5], m[6]);
        ts = mktime(ts_str);
        if (ts >= cutoff && index($0, pat) > 0) count++;
      }
    }
    END{ print count+0 }
  ' "$logfile" 2>/dev/null || echo 0
}

# wpq: run a WP-CLI query, return a single integer or "NA" on garbage.
# v2.9.1: uses $PHP_CLI $WP_PHAR explicitly, plus numeric-only output guard.
wpq() {
  local wp_path="$1"; shift
  cd "$wp_path" 2>/dev/null || { echo "NA"; return; }
  local out
  out=$("$PHP_CLI" "$WP_PHAR" --skip-plugins --skip-themes "$@" 2>/dev/null | tr -d '\r\n' | tr -d ' ')
  # Numeric guard: defends against wp-cli somehow running under a non-CLI
  # SAPI (which prints HTTP headers to stdout). Only accept plain integers.
  if [[ -z "$out" || ! "$out" =~ ^-?[0-9]+$ ]]; then
    echo "NA"
  else
    echo "$out"
  fi
}

###############################################################################
# PLAYBOOKS (v2.8) - context + remediation guidance per rule
###############################################################################
# Each playbook prints two blocks: "meaning" and "actions".
# Usage: get_playbook <rule_name>  ->  echoes the block.
get_playbook() {
  case "$1" in
    php_fatal)
      cat <<'PB'
What this means:
  PHP threw one or more uncaught fatal errors. These crash the request
  and return a 500 error to the customer. Common causes: bad plugin
  update, missing class/function after a version bump, memory exhaustion,
  or a broken hook.

Recommended actions:
  1. Grab the last stack traces:
       tail -300 /home/digitechstores/public_html/error_log | grep -A5 "PHP Fatal"
  2. Check what changed recently:
       cd /home/digitechstores/public_html && wp plugin list --update=available
  3. If a specific plugin is at fault, deactivate it:
       wp plugin deactivate <slug>
  4. Correlate with WooCommerce checkout / order flow. Any drop in orders today?
PB
      ;;
    cmd_out_of_sync|cmd_out_of_sync_warn)
      cat <<'PB'
What this means:
  MySQL is reporting "Commands out of sync; you can't run this command now".
  Usually means a query result set was left unread, a connection was reused
  mid-transaction, or WPML/WooCommerce Action Scheduler is holding a
  connection incorrectly. A baseline of ~2/day is normal noise.

Recommended actions:
  1. Recent occurrences:
       grep "Commands out of sync" /home/digitechstores/public_html/error_log | tail -20
  2. Check Action Scheduler queue health:
       cd /home/digitechstores/public_html && wp action-scheduler status
  3. If bursts correlate with cron:
       wp cron event list
  4. Confirm mysql wait_timeout / max_allowed_packet are sane.
PB
      ;;
    db_error)
      cat <<'PB'
What this means:
  WordPress logged one or more "WordPress database error" entries in
  the last 24h. Common causes: connection drops, deadlocks, missing
  tables, or corruption after a schema change.

Recommended actions:
  1. See the actual queries:
       grep -A2 "WordPress database error" /home/digitechstores/public_html/error_log | tail -50
  2. Check DB connection health:
       mysqladmin -u root -p status
  3. Run integrity check:
       cd /home/digitechstores/public_html && wp db check --allow-root
  4. If deadlocks, check InnoDB status:
       mysql -e "SHOW ENGINE INNODB STATUS\G" | less
PB
      ;;
    ram_pct|ram_pct_warn)
      cat <<'PB'
What this means:
  Server memory utilisation is elevated. On 15 GB RAM with 12 vCPUs,
  a sustained spike usually points to PHP worker leaks (LiteSpeed LSAPI),
  a runaway plugin, or an unusually large query result set.

Recommended actions:
  1. Top memory consumers right now:
       ps aux --sort=-%mem | head -15
  2. LiteSpeed workers:
       systemctl status lsws  ;  ps aux | grep lsphp | head
  3. Consider bouncing PHP workers:
       systemctl restart lsws
  4. If it's a leak: reduce PHP OPcache max size or worker count temporarily.
PB
      ;;
    disk_pct|disk_pct_warn)
      cat <<'PB'
What this means:
  /home partition usage is high. When it fills up, WordPress cannot write
  uploads, cache, or logs -- users see broken images and 500 errors.

Recommended actions:
  1. Find the biggest offenders:
       du -h --max-depth=2 /home/digitechstores | sort -rh | head -20
  2. Rotate/prune logs:
       ls -lhS /home/digitechstores/public_html/error_log \
               /home/digitechstores/public_html/wp-content/debug.log
  3. Clear old backups / archives.
  4. Empty WooCommerce log directory:
       find /home/digitechstores/public_html/wp-content/uploads/wc-logs -type f -mtime +14 -delete
PB
      ;;
    as_failed)
      cat <<'PB'
What this means:
  Action Scheduler is accumulating FAILED jobs. These are background tasks
  (emails, stock sync, subscription renewals, WooCommerce webhooks) that
  did not complete. Every failure is potentially a customer-facing effect.

Recommended actions:
  1. List failed jobs:
       cd /home/digitechstores/public_html && wp action-scheduler status
       wp action-scheduler run --group=woocommerce --batches=1
  2. Inspect via UI:  WP Admin -> Tools -> Scheduled Actions -> filter Failed
  3. Common cause: PHP fatal inside a scheduled hook.
     Correlate with the "PHP fatals" section if you got one.
  4. After fixing root cause, retry or clear old failures.
PB
      ;;
    trans_expired)
      cat <<'PB'
What this means:
  Thousands of expired transient rows are still sitting in wp_options.
  This bloats the DB and slows autoload. Normally WP or a plugin cleans
  these up; a growing pile means the cleanup cron is broken.

Recommended actions:
  1. Quick manual cleanup:
       cd /home/digitechstores/public_html && wp transient delete --expired
  2. Verify WP-Cron is running (see "WP-Cron overdue" if flagged).
  3. Check wp_options autoload weight:
       wp db query "SELECT COUNT(*), SUM(LENGTH(option_value))
         FROM wp7e_options WHERE autoload='yes';"
PB
      ;;
    cron_overdue)
      cat <<'PB'
What this means:
  Multiple WP-Cron events are past their scheduled time. WP-Cron is what
  fires WooCommerce emails, subscription renewals, transient cleanup,
  scheduled posts, and Action Scheduler batches. If it's stuck, a lot
  of business logic silently stops working.

Recommended actions:
  1. List overdue events:
       cd /home/digitechstores/public_html && wp cron event list | head -30
  2. Force-run one to confirm cron works:
       wp cron event run --due-now
  3. Check that the OS cron entry hitting wp-cron.php is present:
       crontab -l | grep wp-cron
  4. Make sure DISABLE_WP_CRON is set correctly in wp-config.php.
PB
      ;;
    load_5m)
      cat <<'PB'
What this means:
  The 5-minute load average is above healthy levels for the number of
  vCPUs available. Sustained high load = requests queuing up = slower
  response times for customers.

Recommended actions:
  1. Who's using CPU right now?
       top -bn1 | head -20
  2. Any single process spiking?
       ps -eo pid,pcpu,pmem,cmd --sort=-pcpu | head -15
  3. Is it a bot storm? Check web access log for top IPs.
  4. If it's PHP: bounce workers (systemctl restart lsws).
PB
      ;;
    *)
      echo "What this means:"
      echo "  (No playbook defined for rule '$1' yet.)"
      echo ""
      echo "Recommended actions:"
      echo "  1. Check /home/digitechstores/public_html/error_log"
      echo "  2. Review the daily CSV for context."
      ;;
  esac
}

###############################################################################
# EMAIL + STATE HELPERS
###############################################################################
send_email() {
  # Args: subject, body_text, priority (normal|high)
  local subject="$1" body="$2" priority="${3:-normal}"
  if [ ! -x /usr/sbin/sendmail ]; then
    echo "[WARN] /usr/sbin/sendmail not available - alert not emailed:"
    echo -e "$body"
    return 1
  fi
  {
    echo "From: ${ALERT_FROM}"
    echo "To: ${ALERT_RECIPIENTS}"
    [ -n "$ALERT_CC" ] && echo "Cc: ${ALERT_CC}"
    echo "Subject: ${subject}"
    echo "MIME-Version: 1.0"
    echo "Content-Type: text/plain; charset=UTF-8"
    echo "Content-Transfer-Encoding: 8bit"
    if [ "$priority" = "high" ]; then
      echo "X-Priority: 1"
      echo "Importance: High"
    fi
    echo ""
    echo -e "$body"
    echo ""
    echo "-- "
    echo "Server   : ${HOSTNAME_STR}"
    echo "Time UTC : ${TS_UTC}"
    echo "Time Loc : ${TS_LOCAL}"
    echo "Script   : wp_health_check.sh v2.9.1"
  } | /usr/sbin/sendmail -t -oi
}

# State file: site|rule|last_epoch|last_value|last_band
mkdir -p "$(dirname "$ALERT_STATE_FILE")" 2>/dev/null
touch "$ALERT_STATE_FILE" 2>/dev/null

get_alert_state() {
  # Args: site, rule. Prints: last_epoch|last_value|last_band  (empty if none)
  awk -F'|' -v s="$1" -v r="$2" '$1==s && $2==r {print $3"|"$4"|"$5; exit}' "$ALERT_STATE_FILE" 2>/dev/null
}

set_alert_state() {
  # Args: site, rule, value, band
  local site="$1" rule="$2" value="$3" band="$4"
  local now; now=$(date +%s)
  local tmp="${ALERT_STATE_FILE}.tmp.$$"
  awk -F'|' -v s="$site" -v r="$rule" '!($1==s && $2==r)' "$ALERT_STATE_FILE" > "$tmp" 2>/dev/null
  echo "${site}|${rule}|${now}|${value}|${band}" >> "$tmp"
  mv "$tmp" "$ALERT_STATE_FILE"
}

clear_alert_state() {
  local tmp="${ALERT_STATE_FILE}.tmp.$$"
  awk -F'|' -v s="$1" -v r="$2" '!($1==s && $2==r)' "$ALERT_STATE_FILE" > "$tmp" 2>/dev/null
  mv "$tmp" "$ALERT_STATE_FILE"
}

# band_rank: warn=1, critical=2, severe=3
band_rank() {
  case "$1" in
    warn) echo 1;;
    critical) echo 2;;
    severe) echo 3;;
    *) echo 0;;
  esac
}

# should_alert_critical: decides whether to send a CRIT/SEVERE email.
# Args: site, rule, current_value, current_band ("critical" or "severe")
# Echoes: "send" | "suppress"
should_alert_critical() {
  local site="$1" rule="$2" value="$3" band="$4"
  local state; state=$(get_alert_state "$site" "$rule")

  if [ -z "$state" ]; then
    echo "send"
    echo "  [dedup] ${site}/${rule}: first occurrence (${band}) -> SEND" >&2
    return
  fi

  local last_epoch last_value last_band cur_rank last_rank now age_h
  last_epoch=$(echo "$state" | cut -d'|' -f1)
  last_value=$(echo "$state" | cut -d'|' -f2)
  last_band=$(echo  "$state" | cut -d'|' -f3)
  cur_rank=$(band_rank "$band")
  last_rank=$(band_rank "$last_band")

  # BAND ESCALATION: crossed UP into a higher severity band -> always send
  if [ "$cur_rank" -gt "$last_rank" ]; then
    echo "send"
    echo "  [dedup] ${site}/${rule}: escalated ${last_band}(${last_value}) -> ${band}(${value}) -> SEND" >&2
    return
  fi

  # COOLDOWN REMINDER (only if enabled)
  if [ "$CRITICAL_REMINDERS_ENABLED" = "true" ]; then
    now=$(date +%s)
    age_h=$(( (now - last_epoch) / 3600 ))
    if [ "$age_h" -ge "$COOLDOWN_HOURS" ]; then
      echo "send"
      echo "  [dedup] ${site}/${rule}: cooldown expired (${age_h}h) -> SEND (reminder)" >&2
      return
    fi
  fi

  echo "suppress"
  echo "  [dedup] ${site}/${rule}: suppressed (age $(( ($(date +%s) - last_epoch)/3600 ))h, band=${band}, prev=${last_band})" >&2
}

# should_alert_warn: WARN is fire-once, but with two safety nets:
#   1) Growth re-alert (metric >= WARN_GROWTH_PCT worse than last email)
#   2) Stale reminder (open for >= WARN_STALE_DAYS)
# Args: site, rule, current_value
# Echoes: "send" (new|growth|stale) | "suppress"
should_alert_warn() {
  local site="$1" rule="$2" value="$3"
  local state; state=$(get_alert_state "$site" "$rule")

  if [ -z "$state" ]; then
    echo "send"
    echo "  [dedup] ${site}/${rule}: new warning -> SEND" >&2
    return
  fi

  local last_epoch last_value now age_h age_d grow_pct
  last_epoch=$(echo "$state" | cut -d'|' -f1)
  last_value=$(echo "$state" | cut -d'|' -f2)
  now=$(date +%s)
  age_h=$(( (now - last_epoch) / 3600 ))
  age_d=$(( age_h / 24 ))

  # Growth safety net
  if [ -n "$last_value" ] && [ "$last_value" -gt 0 ] 2>/dev/null; then
    grow_pct=$(awk -v c="$value" -v p="$last_value" 'BEGIN{ if (p>0) printf "%.0f", ((c-p)/p)*100; else print 0 }')
    if [ "$grow_pct" -ge "$WARN_GROWTH_PCT" ] 2>/dev/null; then
      echo "send"
      echo "  [dedup] ${site}/${rule}: warning grew ${last_value} -> ${value} (+${grow_pct}%) -> SEND (getting worse)" >&2
      return
    fi
  fi

  # Stale safety net
  if [ "$age_d" -ge "$WARN_STALE_DAYS" ]; then
    echo "send"
    echo "  [dedup] ${site}/${rule}: warning open for ${age_d}d -> SEND (stale reminder)" >&2
    return
  fi

  echo "suppress"
  echo "  [dedup] ${site}/${rule}: warning still active (${age_d}d, was ${last_value}, now ${value}) -> silent" >&2
}

# Track which rules were active this run so we can send RESOLVED for the rest.
ACTIVE_RULES_THIS_RUN=""
mark_active() {
  ACTIVE_RULES_THIS_RUN="${ACTIVE_RULES_THIS_RUN}${1}|${2}
"
}

send_resolved_alert() {
  local site="$1" rule="$2" last_value="$3" last_band="$4"
  local subject="[WP HEALTH - RESOLVED] Digitech Stores | ${site} | ${rule}"
  local body=""
  body="${body}The following condition has CLEARED and is no longer alerting.\n"
  body="${body}==========================================================================\n\n"
  body="${body}  Site           : ${site}\n"
  body="${body}  Rule           : ${rule}\n"
  body="${body}  Previous band  : ${last_band}\n"
  body="${body}  Last value     : ${last_value}\n"
  body="${body}  Cleared at     : ${TS_LOCAL}\n\n"
  body="${body}No further action needed. Monitoring continues.\n"
  body="${body}If this recurs, a fresh alert will be sent as a new incident.\n\n"
  body="${body}Server       : ${HOSTNAME_STR}\n"
  body="${body}Daily CSV    : ${CSV_FILE}\n"
  send_email "$subject" "$body" "normal"
  echo "[INFO] RESOLVED notice emailed for ${site}/${rule}."
}

check_resolved_rules() {
  [ ! -s "$ALERT_STATE_FILE" ] && return
  local tmp_active; tmp_active=$(mktemp)
  echo -n "$ACTIVE_RULES_THIS_RUN" > "$tmp_active"
  while IFS='|' read -r s r ep val band; do
    [ -z "$s" ] && continue
    if ! grep -qxF "${s}|${r}" "$tmp_active"; then
      send_resolved_alert "$s" "$r" "$val" "$band"
      clear_alert_state "$s" "$r"
    fi
  done < "$ALERT_STATE_FILE"
  rm -f "$tmp_active"
}

# Purge state entries for sites that are no longer in SITES (v2.9).
# For each stale entry we send one RESOLVED-style notice so the team knows
# monitoring for that site has stopped, then remove it from state.
purge_removed_site_state() {
  [ ! -s "$ALERT_STATE_FILE" ] && return
  local current_sites=""
  for entry in "${SITES[@]}"; do
    current_sites="${current_sites}$(echo "$entry" | cut -d'|' -f1)|"
  done

  local tmp; tmp=$(mktemp)
  local purged=0
  while IFS='|' read -r s r ep val band; do
    [ -z "$s" ] && continue
    if [[ "|${current_sites}" != *"|${s}|"* ]]; then
      # Site no longer monitored - send closure notice
      local subject="[WP HEALTH - RESOLVED] Digitech Stores | ${s} | ${r} (site removed)"
      local body=""
      body="${body}Alert state for a site that is NO LONGER MONITORED has been cleaned up.\n"
      body="${body}==========================================================================\n\n"
      body="${body}  Site           : ${s}   (removed from SITES config)\n"
      body="${body}  Rule           : ${r}\n"
      body="${body}  Previous band  : ${band}\n"
      body="${body}  Last value     : ${val}\n"
      body="${body}  Cleared at     : ${TS_LOCAL}\n\n"
      body="${body}This rule was open when the site was removed from routine monitoring.\n"
      body="${body}No further alerts will be sent for '${s}' unless it is re-added to\n"
      body="${body}the SITES array in wp_health_check.sh.\n\n"
      body="${body}Server       : ${HOSTNAME_STR}\n"
      send_email "$subject" "$body" "normal"
      echo "[INFO] Purged stale state for removed site '${s}' rule '${r}'."
      purged=$((purged + 1))
    else
      # Keep this entry
      echo "${s}|${r}|${ep}|${val}|${band}" >> "$tmp"
    fi
  done < "$ALERT_STATE_FILE"
  mv "$tmp" "$ALERT_STATE_FILE"
  [ "$purged" -gt 0 ] && echo "[INFO] Purged ${purged} stale state entrie(s) from removed sites."
}

# Buffer for the WARN rollup email — only NEWLY-FIRED warns land here.
# Format per entry: site|rule|value|human_message  (newline separated)
WARN_BUFFER=""
WARN_COUNT=0
buffer_warn_email_line() {
  # Args: site, rule, value, human_message
  WARN_BUFFER="${WARN_BUFFER}$1|$2|$3|$4"$'\n'
  WARN_COUNT=$((WARN_COUNT + 1))
}
flush_warn_email() {
  [ -z "$WARN_BUFFER" ] && return 0
  local site_list; site_list=$(echo -n "$WARN_BUFFER" | awk -F'|' '{print $1}' | sort -u | paste -sd, -)
  local subject="[WP HEALTH - WARN] Digitech Stores | ${site_list} | ${WARN_COUNT} warning(s)"

  local body=""
  body="${body}WARNING: ${WARN_COUNT} rule(s) tripped the WARN tier during today's health check.\n"
  body="${body}==========================================================================\n\n"

  local n=0
  while IFS='|' read -r site rule value msg; do
    [ -z "$site" ] && continue
    n=$((n + 1))
    body="${body}--- Warning ${n} / ${WARN_COUNT} ------------------------------------------------------\n"
    body="${body}  Site   : ${site}\n"
    body="${body}  Rule   : ${rule}\n"
    body="${body}  Detail : ${msg}\n\n"
    body="${body}$(get_playbook "$rule")\n\n"
  done <<< "$(echo -n "$WARN_BUFFER")"

  body="${body}==========================================================================\n"
  body="${body}Alert policy (v2.9.1 - WARN tier)\n"
  body="${body}  * Fires ONCE when a new warning appears.\n"
  body="${body}  * Re-alerts only if the value grows by >= ${WARN_GROWTH_PCT}% (getting worse),\n"
  body="${body}    or if the warning has been open for >= ${WARN_STALE_DAYS} days (stale reminder).\n"
  body="${body}  * A RESOLVED email will be sent automatically when it clears.\n\n"
  body="${body}Server       : ${HOSTNAME_STR}\n"
  body="${body}Report time  : ${TS_LOCAL}\n"
  body="${body}Daily CSV    : ${CSV_FILE}\n"

  send_email "$subject" "$body" "normal"
  echo "[INFO] WARN rollup emailed (${WARN_COUNT} warnings)."
}

###############################################################################
# PER-SITE LOOP
###############################################################################
for entry in "${SITES[@]}"; do
  SITE_LABEL=$(echo "$entry" | cut -d'|' -f1)
  SITE_PATH=$(echo  "$entry" | cut -d'|' -f2)
  SITE_LOG=$(echo   "$entry" | cut -d'|' -f3)

  COOS=$(count_recent_log_matches "$SITE_LOG" "Commands out of sync")
  FATAL=$(count_recent_log_matches "$SITE_LOG" "PHP Fatal error")
  DBERR=$(count_recent_log_matches "$SITE_LOG" "WordPress database error")

  DEBUG_LOG="${SITE_PATH}/wp-content/debug.log"
  if [ -r "$DEBUG_LOG" ]; then
    COOS=$((  COOS  + $(count_recent_log_matches "$DEBUG_LOG" "Commands out of sync") ))
    FATAL=$(( FATAL + $(count_recent_log_matches "$DEBUG_LOG" "PHP Fatal error") ))
    DBERR=$(( DBERR + $(count_recent_log_matches "$DEBUG_LOG" "WordPress database error") ))
  fi

  TRANS_TOTAL=$(wpq   "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}options WHERE option_name LIKE '\\_transient\\_%';" --skip-column-names)
  TRANS_EXPIRED=$(wpq "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}options WHERE option_name LIKE '\\_transient\\_timeout\\_%' AND option_value < UNIX_TIMESTAMP();" --skip-column-names)
  OPT_ROWS=$(wpq      "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}options;" --skip-column-names)
  AS_PENDING=$(wpq    "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}actionscheduler_actions WHERE status='pending';" --skip-column-names)
  AS_INPROG=$(wpq     "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}actionscheduler_actions WHERE status='in-progress';" --skip-column-names)
  AS_FAILED=$(wpq     "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}actionscheduler_actions WHERE status='failed';" --skip-column-names)
  AS_COMPLETE=$(wpq   "$SITE_PATH" db query "SELECT COUNT(*) FROM ${TABLE_PREFIX}actionscheduler_actions WHERE status='complete' AND last_attempt_gmt >= (NOW() - INTERVAL 1 DAY);" --skip-column-names)

  # v2.9.1: use $PHP_CLI $WP_PHAR explicitly, and validate result is numeric.
  CRON_OVERDUE=$(cd "$SITE_PATH" 2>/dev/null && "$PHP_CLI" "$WP_PHAR" cron event list --format=csv --fields=hook,next_run_relative 2>/dev/null | \
    awk -F',' 'NR>1 && ($2 ~ /ago/ || $2 == "now") {c++} END{print c+0}')
  if [[ ! "${CRON_OVERDUE:-}" =~ ^[0-9]+$ ]]; then CRON_OVERDUE="NA"; fi

  echo "${TS_UTC},${TS_LOCAL},${SITE_LABEL},\"${UPTIME_STR}\",${LOAD_1},${LOAD_5},${LOAD_15},${RAM_USED_MB},${RAM_TOTAL_MB},${RAM_PCT},${CPU_PCT},${DISK_PCT},${COOS},${FATAL},${DBERR},${TRANS_TOTAL},${TRANS_EXPIRED},${OPT_ROWS},${AS_PENDING},${AS_INPROG},${AS_FAILED},${AS_COMPLETE},${CRON_OVERDUE}" >> "$CSV_FILE"

  echo "=========================================="
  echo " Health check -- site: ${SITE_LABEL}"
  echo " Time (UTC)      : ${TS_UTC}"
  echo " Load (1/5/15)   : ${LOAD_1} / ${LOAD_5} / ${LOAD_15}"
  echo " RAM used        : ${RAM_USED_MB} / ${RAM_TOTAL_MB} MB (${RAM_PCT}%)"
  echo " CPU             : ${CPU_PCT}%"
  echo " Disk (/home)    : ${DISK_PCT}%"
  echo " Errors 24h      : COOS=${COOS}  FATAL=${FATAL}  DBERR=${DBERR}"
  echo " AS              : pending=${AS_PENDING} inprog=${AS_INPROG} failed=${AS_FAILED} done24h=${AS_COMPLETE}"
  echo " Cron overdue    : ${CRON_OVERDUE}"
  echo "=========================================="

  ###########################################################################
  # ALERT EVALUATION (v2.6 - severity bands)
  ###########################################################################

  # ---- CRITICAL / SEVERE checks (with band promotion) ----------------------
  eval_crit_rule() {
    # Args: rule_name, current_value, crit_thresh, severe_thresh, human_line_fmt
    # For numeric integer metrics. Uses -gt.
    local rule="$1" v="$2" crit_t="$3" sev_t="$4" line_fmt="$5"
    [ "$v" = "NA" ] && return
    local band="" line=""
    if [ "$sev_t" != "" ] && [ "$v" -gt "$sev_t" ] 2>/dev/null; then
      band="severe"
    elif [ "$v" -gt "$crit_t" ] 2>/dev/null; then
      band="critical"
    else
      return
    fi
    # shellcheck disable=SC2059
    line=$(printf "$line_fmt" "$v" "$crit_t" "$sev_t")
    mark_active "$SITE_LABEL" "$rule"
    local decision; decision=$(should_alert_critical "$SITE_LABEL" "$rule" "$v" "$band")
    if [ "$decision" = "send" ]; then
      CRIT_EMAIL_LINES="${CRIT_EMAIL_LINES}- [${band^^}] ${line}\n"
      [ "$band" = "severe" ] && ANY_SEVERE=1
      set_alert_state "$SITE_LABEL" "$rule" "$v" "$band"
    fi
    CRIT_ALL_LINES="${CRIT_ALL_LINES}- [${band^^}] ${line}\n"
  }

  eval_crit_rule_pct() {
    # Same as eval_crit_rule but for float % metrics (use awk compare).
    local rule="$1" v="$2" crit_t="$3" sev_t="$4" line_fmt="$5"
    [ "$v" = "NA" ] && return
    local band=""
    if awk -v x="$v" -v t="$sev_t" 'BEGIN{ exit !(x+0 > t+0) }'; then
      band="severe"
    elif awk -v x="$v" -v t="$crit_t" 'BEGIN{ exit !(x+0 > t+0) }'; then
      band="critical"
    else
      return
    fi
    local v_int; v_int=$(awk -v p="$v" 'BEGIN{ printf "%d", p+0 }')
    local line; line=$(printf "$line_fmt" "$v" "$crit_t" "$sev_t")
    mark_active "$SITE_LABEL" "$rule"
    local decision; decision=$(should_alert_critical "$SITE_LABEL" "$rule" "$v_int" "$band")
    if [ "$decision" = "send" ]; then
      CRIT_EMAIL_LINES="${CRIT_EMAIL_LINES}- [${band^^}] ${line}\n"
      [ "$band" = "severe" ] && ANY_SEVERE=1
      set_alert_state "$SITE_LABEL" "$rule" "$v_int" "$band"
    fi
    CRIT_ALL_LINES="${CRIT_ALL_LINES}- [${band^^}] ${line}\n"
  }

  CRIT_EMAIL_LINES=""
  CRIT_ALL_LINES=""
  ANY_SEVERE=0

  eval_crit_rule     "php_fatal"       "$FATAL"    "$CRIT_FATAL"    "$SEVERE_FATAL"    "PHP Fatal errors in 24h: %s (crit >%s, severe >%s)"
  eval_crit_rule     "cmd_out_of_sync" "$COOS"     "$CRIT_COOS"     "$SEVERE_COOS"     "'Commands out of sync' in 24h: %s (crit >%s, severe >%s)"
  eval_crit_rule_pct "ram_pct"         "$RAM_PCT"  "$CRIT_RAM_PCT"  "$SEVERE_RAM_PCT"  "RAM usage: %s%% (crit >%s%%, severe >%s%%)"
  eval_crit_rule     "disk_pct"        "$DISK_PCT" "$CRIT_DISK_PCT" "$SEVERE_DISK_PCT" "Disk (/home): %s%% (crit >%s%%, severe >%s%%)"

  # ---- WARN checks (fire-once + growth + stale safety nets) ----------------
  eval_warn_rule() {
    # Args: rule_name, current_value (int for growth compare), human_message
    local rule="$1" v="$2" msg="$3"
    mark_active "$SITE_LABEL" "$rule"
    local decision; decision=$(should_alert_warn "$SITE_LABEL" "$rule" "$v")
    if [ "$decision" = "send" ]; then
      buffer_warn_email_line "$SITE_LABEL" "$rule" "$v" "$msg"
      set_alert_state "$SITE_LABEL" "$rule" "$v" "warn"
    fi
  }

  # COOS in the warn band (between WARN and CRIT thresholds)
  if [ "$COOS" != "NA" ] && [ "$COOS" -gt "$WARN_COOS" ] && [ "$COOS" -le "$CRIT_COOS" ]; then
    eval_warn_rule "cmd_out_of_sync_warn" "$COOS" "'Commands out of sync' elevated: $COOS in 24h (warn >$WARN_COOS, crit >$CRIT_COOS)"
  fi
  # DB errors
  if [ "$DBERR" != "NA" ] && [ "$DBERR" -gt "$WARN_DBERR" ]; then
    eval_warn_rule "db_error" "$DBERR" "WP DB errors in 24h: $DBERR"
  fi
  # Action Scheduler failed
  if [ "$AS_FAILED" != "NA" ] && [ "$AS_FAILED" -gt "$WARN_AS_FAILED" ]; then
    eval_warn_rule "as_failed" "$AS_FAILED" "Action Scheduler failed jobs: $AS_FAILED (warn >$WARN_AS_FAILED)"
  fi
  # Expired transients
  if [ "$TRANS_EXPIRED" != "NA" ] && [ "$TRANS_EXPIRED" -gt "$WARN_TRANS_EXPIRED" ]; then
    eval_warn_rule "trans_expired" "$TRANS_EXPIRED" "Expired transients: $TRANS_EXPIRED - cleanup may not be running"
  fi
  # Cron overdue
  if [ "$CRON_OVERDUE" != "NA" ] && [ "$CRON_OVERDUE" -gt "$WARN_CRON_OVERDUE" ]; then
    eval_warn_rule "cron_overdue" "$CRON_OVERDUE" "WP-Cron overdue events: $CRON_OVERDUE - cron may be stuck"
  fi
  # RAM in warn band (below crit) - use integer % for growth comparison
  if [ "$RAM_PCT" != "NA" ] && awk -v p="$RAM_PCT" -v w="$WARN_RAM_PCT" -v c="$CRIT_RAM_PCT" 'BEGIN{ exit !(p+0>w && p+0<=c) }'; then
    RAM_INT_W=$(awk -v p="$RAM_PCT" 'BEGIN{ printf "%d", p+0 }')
    eval_warn_rule "ram_pct_warn" "$RAM_INT_W" "RAM usage elevated: ${RAM_PCT}% (warn >${WARN_RAM_PCT}%)"
  fi
  # Disk in warn band
  if [ "$DISK_PCT" != "NA" ] && [ "$DISK_PCT" -gt "$WARN_DISK_PCT" ] && [ "$DISK_PCT" -le "$CRIT_DISK_PCT" ]; then
    eval_warn_rule "disk_pct_warn" "$DISK_PCT" "Disk (/home) elevated: ${DISK_PCT}% (warn >${WARN_DISK_PCT}%)"
  fi
  # Load average - value scaled to int (x100) so growth compare works with decimals
  if [ "$LOAD_5" != "NA" ] && [ "$VCPU_COUNT" -gt 0 ]; then
    if awk -v l="$LOAD_5" -v v="$VCPU_COUNT" -v m="$WARN_LOAD5_MULT" 'BEGIN{ exit !(l+0 > v*m) }'; then
      LOAD_INT=$(awk -v l="$LOAD_5" 'BEGIN{ printf "%d", l*100 }')
      eval_warn_rule "load_5m" "$LOAD_INT" "Load average (5m): $LOAD_5 exceeds $VCPU_COUNT vCPU x $WARN_LOAD5_MULT"
    fi
  fi

  # ---- Dispatch CRITICAL / SEVERE ------------------------------------------
  if [ -n "$CRIT_ALL_LINES" ]; then
    echo -e "\n[CRIT/SEVERE active on ${SITE_LABEL}]\n${CRIT_ALL_LINES}"
  fi

  if [ -n "$CRIT_EMAIL_LINES" ]; then
    # v2.9.1: removed illegal 'local' declarations here — this block runs
    # at file scope inside a for-loop, not inside a function, so 'local'
    # produced: "line NNN: local: can only be used in a function".
    tier_word="CRITICAL"
    priority="high"
    local_prefix="$ALERT_CRIT_SUBJECT_PREFIX"
    if [ "$ANY_SEVERE" -eq 1 ]; then
      local_prefix="$ALERT_SEVERE_SUBJECT_PREFIX"
      tier_word="SEVERE"
    fi
    subject="[WP HEALTH - ${tier_word}] Digitech Stores | ${SITE_LABEL} | $(date +'%Y-%m-%d %H:%M %Z')"

    CRIT_BODY=""
    CRIT_BODY="${CRIT_BODY}${tier_word} alerts detected on Digitech Stores production\n"
    CRIT_BODY="${CRIT_BODY}==========================================================================\n\n"
    CRIT_BODY="${CRIT_BODY}Rules being notified in this email (new, escalated, or reminder):\n"
    CRIT_BODY="${CRIT_BODY}${CRIT_EMAIL_LINES}\n"
    if [ "$CRIT_ALL_LINES" != "$CRIT_EMAIL_LINES" ]; then
      CRIT_BODY="${CRIT_BODY}All currently-active critical rules (for context):\n${CRIT_ALL_LINES}\n"
    fi

    CRIT_BODY="${CRIT_BODY}--- Server snapshot -----------------------------------------------------\n"
    CRIT_BODY="${CRIT_BODY}  Load (1/5/15) : ${LOAD_1} / ${LOAD_5} / ${LOAD_15}\n"
    CRIT_BODY="${CRIT_BODY}  RAM           : ${RAM_USED_MB} / ${RAM_TOTAL_MB} MB (${RAM_PCT}%)\n"
    CRIT_BODY="${CRIT_BODY}  CPU           : ${CPU_PCT}%\n"
    CRIT_BODY="${CRIT_BODY}  Disk (/home)  : ${DISK_PCT}%\n"
    CRIT_BODY="${CRIT_BODY}  AS failed     : ${AS_FAILED}\n"
    CRIT_BODY="${CRIT_BODY}  Cron overdue  : ${CRON_OVERDUE}\n\n"

    # Attach playbooks for each rule being notified this run
    CRIT_BODY="${CRIT_BODY}--- Playbooks -----------------------------------------------------------\n\n"
    while IFS='|' read -r st_site st_rule st_ep st_val st_band; do
      [ "$st_site" != "$SITE_LABEL" ] && continue
      # Only include rules that are in ACTIVE_RULES_THIS_RUN
      if echo -e "$ACTIVE_RULES_THIS_RUN" | grep -qxF "${st_site}|${st_rule}"; then
        CRIT_BODY="${CRIT_BODY}### Rule: ${st_rule}  (band=${st_band}, value=${st_val})\n"
        CRIT_BODY="${CRIT_BODY}$(get_playbook "$st_rule")\n\n"
      fi
    done < "$ALERT_STATE_FILE"

    CRIT_BODY="${CRIT_BODY}==========================================================================\n"
    if [ "$CRITICAL_REMINDERS_ENABLED" = "true" ]; then
      CRIT_BODY="${CRIT_BODY}Alert policy (v2.9.1): CRITICAL reminders every ${COOLDOWN_HOURS}h until resolved.\n"
      CRIT_BODY="${CRIT_BODY}Crossing into the SEVERE band re-alerts immediately.\n"
    else
      CRIT_BODY="${CRIT_BODY}Alert policy (v2.9.1): FIRE-ONCE. Crossing into SEVERE band still re-alerts.\n"
    fi
    CRIT_BODY="${CRIT_BODY}\nServer       : ${HOSTNAME_STR}\n"
    CRIT_BODY="${CRIT_BODY}Report time  : ${TS_LOCAL}\n"
    CRIT_BODY="${CRIT_BODY}Daily CSV    : ${CSV_FILE}\n"

    send_email "$subject" "$CRIT_BODY" "$priority"
    echo "[INFO] Alert emailed for ${SITE_LABEL} (severe=${ANY_SEVERE})."
  elif [ -n "$CRIT_ALL_LINES" ]; then
    echo "[INFO] Critical rules still active on ${SITE_LABEL} but suppressed (within cooldown, no band escalation)."
  fi
done

# After all sites processed:
flush_warn_email
check_resolved_rules
purge_removed_site_state

###############################################################################
# WEEKLY REPORT
###############################################################################
send_weekly_report() {
  local html_file="${REPORT_DIR}/weekly_report_$(date +%Y%m%d).html"
  local attach_csv="${REPORT_DIR}/weekly_export_$(date +%Y%m%d).csv"
  local period_start period_end
  period_start=$(date -u -d "-6 days" +'%Y-%m-%d')
  period_end=$(date -u +'%Y-%m-%d')

  local prev_month_tag; prev_month_tag=$(date -u -d "-1 month" +'%Y%m')
  local csv_current="${REPORT_DIR}/wp_health_${MONTH_TAG}.csv"
  local csv_prev="${REPORT_DIR}/wp_health_${prev_month_tag}.csv"

  # Build a pipe-delimited allowlist for awk (e.g. "prod" or "prod|dev")
  local sites_allow; sites_allow=$(echo "$WEEKLY_REPORT_SITES" | tr ',' '|' | tr -d ' ')

  # legacy_skipped counter file - awk writes count here, we read it later
  local legacy_file; legacy_file=$(mktemp)
  echo 0 > "$legacy_file"

  {
    head -1 "$csv_current" 2>/dev/null
    for f in "$csv_prev" "$csv_current"; do
      [ -r "$f" ] || continue
      awk -F',' -v start="$period_start" -v endd="$period_end" \
              -v allow="$sites_allow" -v lf="$legacy_file" '
        BEGIN {
          # Build allow set
          n = split(allow, arr, "|")
          for (i=1;i<=n;i++) allowed[arr[i]] = 1
          legacy = 0
        }
        NR==1 { next }
        {
          d = substr($1,1,10)
          if (d >= start && d <= endd) {
            # Detect legacy uptime-with-commas rows and rejoin field 4
            expected = 23
            if (NF > expected) {
              extra = NF - expected
              joined = $4
              for (i=5; i<=4+extra; i++) joined = joined " " $i
              site = $3
            } else {
              site = $3
            }
            # Site allowlist filter (v2.9)
            if (!(site in allowed)) {
              legacy++
              next
            }
            if (NF > expected) {
              out = $1 "," $2 "," $3 "," joined
              for (i=5+extra; i<=NF; i++) out = out "," $i
              print out
            } else {
              print $0
            }
          }
        }
        END {
          # Accumulate into the counter file across the two source files
          getline prev < lf; close(lf)
          if (prev == "") prev = 0
          print (prev + legacy) > lf
        }
      ' "$f"
    done
  } > "$attach_csv"

  local row_count legacy_skipped
  row_count=$(( $(wc -l < "$attach_csv") - 1 ))
  [ "$row_count" -lt 0 ] && row_count=0
  legacy_skipped=$(cat "$legacy_file" 2>/dev/null || echo 0)
  rm -f "$legacy_file"

  local agg
  agg=$(awk -F',' '
    NR==1 { next }
    NF < 23 { next }
    {
      site=$3
      coos[site]  += $13
      fatal[site] += $14
      dberr[site] += $15
      as_done[site] += $22
      if ($21+0 > max_as_fail[site]) max_as_fail[site] = $21+0
      cnt[site]++
      if ($10+0 > max_ram[site])  max_ram[site]  = $10+0
      if ($11+0 > max_cpu[site])  max_cpu[site]  = $11+0
      if ($12+0 > max_disk[site]) max_disk[site] = $12+0
      sum_ram[site] += $10+0
      sum_cpu[site] += $11+0
    }
    END {
      for (s in cnt) {
        printf "%s|%d|%d|%d|%d|%d|%d|%.1f|%.1f|%.1f|%.1f|%d\n",
          s, cnt[s], coos[s], fatal[s], dberr[s],
          max_as_fail[s], as_done[s],
          (cnt[s]>0? sum_ram[s]/cnt[s] : 0), max_ram[s],
          (cnt[s]>0? sum_cpu[s]/cnt[s] : 0), max_cpu[s],
          max_disk[s]
      }
    }
  ' "$attach_csv")

  {
    echo '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>Weekly WP Health Report</title>'
    echo '<style>body{font-family:Segoe UI,Arial,sans-serif;color:#222;max-width:900px;margin:0 auto;padding:20px}'
    echo 'h1{color:#0b5394;border-bottom:2px solid #0b5394;padding-bottom:6px}h2{color:#333;margin-top:26px}'
    echo 'table{border-collapse:collapse;width:100%;margin:12px 0;font-size:14px}'
    echo 'th,td{border:1px solid #ccc;padding:8px 10px;text-align:left}th{background:#0b5394;color:#fff}'
    echo 'tr:nth-child(even){background:#f6f8fa}.ok{color:#0a7d1f;font-weight:600}'
    echo '.warn{color:#b26a00;font-weight:600}.crit{color:#c00;font-weight:600}'
    echo '.meta{color:#666;font-size:12px}.footer{margin-top:30px;color:#666;font-size:12px;border-top:1px solid #ddd;padding-top:10px}'
    echo '</style></head><body>'
    echo '<h1>Weekly WordPress / WooCommerce Health Report</h1>'
    echo "<p class=\"meta\"><b>Period:</b> ${period_start} to ${period_end} (UTC) &nbsp;|&nbsp; <b>Data points:</b> ${row_count} rows</p>"
    # v2.9 "As of" metadata block
    legacy_note=""
    if [ "${legacy_skipped:-0}" -gt 0 ]; then
      legacy_note=" &nbsp;|&nbsp; <b>Legacy rows skipped:</b> ${legacy_skipped} (sites no longer in scope)"
    fi
    echo "<p class=\"meta\"><b>Reporting sites:</b> ${WEEKLY_REPORT_SITES}${legacy_note}</p>"
    echo '<h2>Per-site summary (last 7 days)</h2><table>'
    echo '<tr><th>Site</th><th>Rows</th><th>Cmds out-of-sync</th><th>PHP fatals</th><th>WP DB errors</th><th>AS failed (peak)</th><th>AS completed</th><th>Avg RAM %</th><th>Peak RAM %</th><th>Avg CPU %</th><th>Peak CPU %</th><th>Peak Disk %</th></tr>'
    if [ -n "$agg" ]; then
      echo "$agg" | while IFS='|' read -r s cnt coos fatal dberr as_fail as_done avg_ram max_ram avg_cpu max_cpu max_disk; do
        coos_c="ok"; [ "$coos"  -gt 50 ] 2>/dev/null && coos_c="crit"
        [ "$coos_c" = "ok" ] && [ "$coos" -gt 10 ] 2>/dev/null && coos_c="warn"
        fat_c="ok";  [ "$fatal" -gt 0 ] 2>/dev/null && fat_c="crit"
        db_c="ok";   [ "$dberr" -gt 0 ] 2>/dev/null && db_c="warn"
        asf_c="ok";  [ "$as_fail" -gt 5 ] 2>/dev/null && asf_c="warn"
        ram_c="ok";  awk -v v="$max_ram" 'BEGIN{ exit !(v+0>95) }' && ram_c="crit"
        [ "$ram_c" = "ok" ] && awk -v v="$max_ram" 'BEGIN{ exit !(v+0>85) }' && ram_c="warn"
        disk_c="ok"; [ "$max_disk" -gt 90 ] 2>/dev/null && disk_c="crit"
        [ "$disk_c" = "ok" ] && [ "$max_disk" -gt 85 ] 2>/dev/null && disk_c="warn"
        echo "<tr><td><b>${s}</b></td><td>${cnt}</td><td class=\"${coos_c}\">${coos}</td><td class=\"${fat_c}\">${fatal}</td><td class=\"${db_c}\">${dberr}</td><td class=\"${asf_c}\">${as_fail}</td><td>${as_done}</td><td>${avg_ram}%</td><td class=\"${ram_c}\">${max_ram}%</td><td>${avg_cpu}%</td><td>${max_cpu}%</td><td class=\"${disk_c}\">${max_disk}%</td></tr>"
      done
    else
      echo "<tr><td colspan='12'><i>No data collected in this window.</i></td></tr>"
    fi
    echo '</table><h2>How to read this report</h2>'
    echo '<ul><li><b>Cmds out-of-sync</b> baseline ~2/day. Warn &gt;10, critical &gt;50, severe &gt;200.</li>'
    echo '<li><b>PHP fatals</b> should stay at 0. Any &gt; 0 is critical, &gt; 20 is severe.</li>'
    echo '<li><b>WP DB errors</b> warn &gt; 0.</li>'
    echo '<li><b>AS failed (peak)</b> warn &gt; 5.</li>'
    echo '<li><b>RAM %</b>: warn &gt;85, critical &gt;95, severe &gt;98. <b>Disk %</b>: warn &gt;85, critical &gt;90, severe &gt;95.</li></ul>'
    echo "<p class=\"meta\"><b>Server time:</b> ${TS_LOCAL}</p>"
    echo '<div class="footer">Automated by <code>wp_health_check.sh</code> v2.9.1 - Digitech Stores IT - Contact: Munzir Mukhtar Almahdi</div>'
    echo '</body></html>'
  } > "$html_file"

  local subject="${WEEKLY_SUBJECT_PREFIX} -- ${period_start} to ${period_end}"

  if [ -x /usr/sbin/sendmail ]; then
    local boundary="====WPHEALTH_$(date +%s)_$$===="
    {
      echo "From: ${WEEKLY_FROM}"
      echo "To: ${WEEKLY_RECIPIENTS}"
      [ -n "$WEEKLY_CC" ] && echo "Cc: ${WEEKLY_CC}"
      echo "Subject: ${subject}"
      echo "MIME-Version: 1.0"
      echo "Content-Type: multipart/mixed; boundary=\"${boundary}\""
      echo ""
      echo "--${boundary}"
      echo "Content-Type: text/html; charset=UTF-8"
      echo "Content-Transfer-Encoding: 8bit"
      echo ""
      cat "$html_file"
      echo ""
      echo "--${boundary}"
      echo "Content-Type: text/csv; name=\"$(basename "$attach_csv")\""
      echo "Content-Disposition: attachment; filename=\"$(basename "$attach_csv")\""
      echo "Content-Transfer-Encoding: base64"
      echo ""
      base64 "$attach_csv"
      echo ""
      echo "--${boundary}--"
    } | /usr/sbin/sendmail -t -oi
    echo "[INFO] Weekly report emailed to: ${WEEKLY_RECIPIENTS}"
  else
    mail -s "$subject" "$WEEKLY_RECIPIENTS" < "$html_file"
  fi

  echo "[INFO] Weekly HTML : ${html_file}"
  echo "[INFO] Weekly CSV  : ${attach_csv}"
}

if [ "${1:-}" = "--weekly" ] || [ "$TODAY_DOW" = "$WEEKLY_REPORT_DOW" ]; then
  echo ""
  echo "=========================================="
  echo " Weekly report -- generating digest..."
  echo "=========================================="
  send_weekly_report || echo "[WARN] Weekly report generation reported an issue."
fi

exit 0