#!/usr/bin/env bash
# Collects a full diagnostic dump on the node and PUTs it to https://dist.gcp.emteria.com/inbox/ (Basic auth).
#   curl -fsS $PROXY_ARG http://dist.gcp.emteria.com/report.sh | bash -s -- enroll     # once: asks for the inbox password
#   curl -fsS $PROXY_ARG http://dist.gcp.emteria.com/report.sh | bash -s -- [label] [minutes]   # full dump; logs of the last N minutes (default 60)
set -uo pipefail
K="/usr/local/bin/k3s kubectl"; NETRC=/etc/emteria/inbox.netrc; URL=https://dist.gcp.emteria.com/inbox/
# The node's resolver does not know dist.gcp.emteria.com: connect to the IP but keep TLS verification for the name.
CT=(--connect-to "dist.gcp.emteria.com:443:${DIST_IP:-23.88.61.31}:443")
[ "$(id -u)" = 0 ] || { echo "run as root"; exit 1; }
if [ "${1:-}" = enroll ]; then
  mkdir -p /etc/emteria; read -r -s -p "inbox password for user 'node' (see /root/inbox-credential on the box): " PW </dev/tty; echo
  umask 077; printf 'machine dist.gcp.emteria.com login node password %s\n' "$PW" > "$NETRC"; unset PW
  if curl -fsS ${PROXY_ARG:-} "${CT[@]}" --netrc-file "$NETRC" -T /etc/hostname "${URL}enroll-$(hostname)-$(date -u +%Y%m%dT%H%M%SZ).txt" -o /dev/null; then echo "enrolled: PUT ok"; else echo "PUT failed - wrong password or no route; netrc kept at $NETRC"; exit 1; fi; exit 0
fi
[ -r "$NETRC" ] || { echo "not enrolled: run ... | bash -s -- enroll"; exit 1; }
LABEL="${1:-dump}"; MIN="${2:-60}"; TS=$(date -u +%Y%m%dT%H%M%SZ); D="/tmp/emteria-dump-$TS-$LABEL"; mkdir -p "$D"
sec() { printf '\n== %s\n' "$*"; }
echo "collecting into $D ..."
{ sec "host"; hostname; date -u +%FT%TZ; uptime; cat /etc/os-release | head -2; uname -r
  sec "clock"; timedatectl 2>/dev/null | head -8; chronyc tracking 2>/dev/null | head -6
  sec "memory"; free -m
  sec "df (real filesystems)"; df -h -x overlay -x tmpfs -x devtmpfs -x shm -x nsfs
  sec "mounts (bind mounts and data disk)"; findmnt -no TARGET,SOURCE,FSTYPE,OPTIONS | grep -E ' /data| /var/lib/emteria| /opt/local-path|^/ ' ; findmnt -no TARGET,SOURCE,FSTYPE /var/lib/emteria/registry /opt/local-path-provisioner /data /var/lib/rancher 2>/dev/null
  sec "what is on the ROOT filesystem (du -x, MiB, top level)"; du -xsm /* 2>/dev/null | sort -rn | head -15
  sec "root: /var/*"; du -xsm /var/* 2>/dev/null | sort -rn | head -12
  sec "root: /var/lib/*"; du -xsm /var/lib/* 2>/dev/null | sort -rn | head -15
  sec "root: /var/lib/rancher/k3s/*/*"; du -xsm /var/lib/rancher/k3s/agent/* /var/lib/rancher/k3s/server/* 2>/dev/null | sort -rn | head -12
  sec "root: /var/lib/rancher/k3s/agent/containerd/*"; du -xsm /var/lib/rancher/k3s/agent/containerd/* 2>/dev/null | sort -rn | head -10
  sec "root: /opt/*, /home/*, /root, /tmp, /var/tmp, /var/log"; du -xsm /opt/* /home/* /root /tmp /var/tmp /var/log 2>/dev/null | sort -rn | head -15
  sec "root: biggest files > 200 MiB"; find / -xdev -type f -size +200M -printf '%s\t%p\n' 2>/dev/null | sort -rn | head -20 | awk '{printf "%8d MiB  %s\n",$1/1048576,$2}'
  sec "/data (MiB)"; du -sm /data/* 2>/dev/null | sort -rn | head -15
  sec "/data/local-path-provisioner or /opt/local-path-provisioner (PVs, MiB)"; du -sm /data/local-path-provisioner/* /opt/local-path-provisioner/* 2>/dev/null | sort -rn | head -12
  sec "registry blob store"; du -sm /var/lib/emteria/registry 2>/dev/null; ls /var/lib/emteria/registry 2>/dev/null | head
  sec "containerd images"; /usr/local/bin/k3s crictl images 2>/dev/null | awk 'NR>1{n++} END{print n" images"}'; /usr/local/bin/k3s crictl imagefsinfo 2>/dev/null
  sec "journal"; journalctl --disk-usage 2>/dev/null; du -sm /var/log/journal 2>/dev/null
  sec "k3s service"; systemctl status k3s --no-pager 2>/dev/null | head -12
  sec "node"; $K get nodes -o wide 2>/dev/null; $K describe node 2>/dev/null | sed -n '/Allocated resources/,/Events/p' | head -14
  sec "cilium"; $K -n kube-system exec ds/cilium -c cilium-agent -- cilium-dbg status 2>/dev/null | head -30
  sec "argo applications"; $K -n argocd get applications 2>/dev/null
  sec "pods (all namespaces)"; $K get pods -A -o wide 2>/dev/null
  sec "pods not running"; $K get pods -A --no-headers 2>/dev/null | grep -vE 'Running|Completed' || echo none
  sec "externalsecrets"; $K get externalsecrets -A 2>/dev/null
  sec "pvc / pv"; $K get pvc -A 2>/dev/null; $K get pv 2>/dev/null
  sec "ingresses"; $K get ingress -A 2>/dev/null
  sec "cnpg cluster"; $K get clusters.postgresql.cnpg.io -A 2>/dev/null; $K -n default get pods -l cnpg.io/cluster -o wide 2>/dev/null
  sec "warning events (last hour)"; $K get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null | tail -40
  sec "emteria-status"; /usr/local/bin/emteria-status 2>&1 || true
  sec "hub config"; for cm in $($K -n default get cm -o name 2>/dev/null | grep -i hub); do echo "-- $cm"; $K -n default get "$cm" -o jsonpath='{.data}' 2>/dev/null | head -c 4000; echo; done
} > "$D/00-overview.txt" 2>&1
# logs
$K -n ingress-nginx logs deploy/ingress-nginx-controller --since="${MIN}m" > "$D/ingress-nginx.log" 2>&1
for d in mainservice website productmanager storagemanager streamservice eventmanager; do $K -n default logs deploy/$d --all-containers --since="${MIN}m" --timestamps > "$D/$d.log" 2>&1; done
$K -n default logs ds/mdm --since="${MIN}m" --timestamps > "$D/mdm.log" 2>&1
$K -n default logs -l cnpg.io/cluster --since="${MIN}m" --timestamps --tail=2000 > "$D/postgres.log" 2>&1
$K -n argocd logs deploy/argocd-repo-server --since="${MIN}m" --tail=500 > "$D/argocd-repo-server.log" 2>&1
journalctl -u k3s --since "-${MIN}min" --no-pager > "$D/k3s-journal.log" 2>&1
journalctl -p warning --since "-${MIN}min" --no-pager > "$D/journal-warnings.log" 2>&1
{ echo "== ingress lines with 4xx/5xx (last ${MIN}m)"; grep -E '" (4[0-9]{2}|5[0-9]{2}) ' "$D/ingress-nginx.log" | tail -100
  echo; echo "== app log lines with 422|error|exception (last ${MIN}m)"; grep -ihE '422|error|exception' "$D"/mainservice.log "$D"/website.log "$D"/productmanager.log "$D"/storagemanager.log "$D"/streamservice.log "$D"/eventmanager.log 2>/dev/null | tail -150
} > "$D/01-errors-summary.txt" 2>&1
TAR="/tmp/emteria-dump-$TS-$LABEL.tar.gz"; tar -C /tmp -czf "$TAR" "$(basename "$D")"; rm -rf "$D"
echo "dump: $TAR ($(du -h "$TAR" | cut -f1))"
if curl -fsS ${PROXY_ARG:-} "${CT[@]}" --netrc-file "$NETRC" -T "$TAR" "${URL}$(basename "$TAR")" -o /dev/null; then echo "sent to the box as $(basename "$TAR")"; else echo "PUT failed - dump kept at $TAR"; exit 1; fi