diff --git a/security-monitor.sh b/security-monitor.sh new file mode 100755 index 0000000..b441803 --- /dev/null +++ b/security-monitor.sh @@ -0,0 +1,398 @@ +#!/bin/bash +# Periodic compromise/change monitor for this host. +# +# Snapshots a set of security-relevant system facts (kernel/boot id, accounts and groups, +# UID-0 accounts, SSH authorized keys, sudoers, PAM, sshd config + effective policy, cron/at, +# enabled systemd units, shell startup files, dynamic-linker preload/config, apt sources, +# name resolution, listening TCP/UDP sockets, docker containers and their security +# posture, mounted block devices, SUID/SGID files with hashes, file capabilities, loaded +# kernel modules, host-level firewall policy, nftables table list, critical binary hashes, +# and the cumulative set of accepted SSH logins), diffs against the previous run, and pushes +# a Telegram alert (with a "[section]" label per changed block) when anything changed. Silent +# (no alert, no output) when nothing changed. First run just seeds the baseline. +# +# Deliberately NOT snapshotted, because they'd churn constantly on this host and just be noise: +# full process list/cmdlines (PIDs and cron-spawned procs change every tick), established +# network connections (ephemeral client churn), the full Docker-managed firewall ruleset and +# nftables rule bodies (rewritten on every container start/stop and on every fail2ban ban — +# see the scoped host-level-only checks instead), SUID files under the container image stores +# (/var/lib/{docker,containerd,haloy} — image layers carry their own su/mount/passwd and the +# layer directory names change on every image pull), swap (active devices/usage flap with +# ordinary memory pressure, not a security signal), a generic "recent /etc mtimes" catch-all +# (certbot renewals, resolv.conf, apt all touch /etc routinely), and /tmp, /dev/shm, /var/tmp +# contents (this host runs ad-hoc dev/test work there routinely — dropped after it turned out +# to be pure noise, see git history). +# +# Separately (not part of the diff above — disk % changes constantly and would spam it), +# checks each real filesystem's usage against a threshold and alerts once on crossing it, with +# hysteresis so it doesn't flap an alert every tick while hovering right at the line. +# +# Install: run once manually to seed the baseline, then drop into cron (e.g. /etc/cron.d, +# every 15 min). All state lives under $STATE_DIR; nothing else on the host is touched. +set -uo pipefail + +# Cron gives system crontabs a bare PATH (/usr/bin:/bin), so tools in /usr/sbin (iptables, +# nft, getcap...) silently vanish there while working fine in an interactive root shell — +# which made cron and manual runs produce *different* snapshots and fire false alerts (an +# empty section that should have had content, etc). Pin PATH so every run sees the same tools. +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +# Likewise pin the locale: sort(1) collation and date(1) names are locale-dependent, so an +# interactive UTF-8 root shell and a locale-less cron run would otherwise order/format the +# snapshot differently and diff against each other. +export LC_ALL=C +umask 077 # state files hold key fingerprints and source IPs + +TELEGRAM_BOT="8602530664:AAFE7tsRY79zhhEjU7sRR7MrS8p3FB0jvXg" +TELEGRAM_CHAT_ID="108712105" + +DISK_ALERT_PCT="${DISK_ALERT_PCT:-90}" # alert once usage reaches this +DISK_RECOVER_PCT="${DISK_RECOVER_PCT:-85}" # ...and don't re-arm until it drops back below this (hysteresis) + +STATE_DIR=/root/.security-monitor +CURRENT="$STATE_DIR/current.snapshot" +PREVIOUS="$STATE_DIR/previous.snapshot" +DISK_STATE="$STATE_DIR/disk_alert_state" +SSH_LOGINS="$STATE_DIR/ssh_logins" # cumulative user@ip set, append-only +SSH_CURSOR="$STATE_DIR/ssh_cursor" # journal cursor so we only read new entries +LOCK="$STATE_DIR/lock" +mkdir -p "$STATE_DIR" +chmod 700 "$STATE_DIR" +touch "$DISK_STATE" "$SSH_LOGINS" +# umask only governs newly created files; chmod explicitly so state written by an older +# version of this script (or by a cp onto an existing file) is tightened too. +chmod 600 "$DISK_STATE" "$SSH_LOGINS" "$CURRENT" "$PREVIOUS" 2>/dev/null + +# A slow run (big journal, hung docker) must not overlap the next cron tick and race on state. +exec 9>"$LOCK" +if ! flock -n 9; then + echo "another run is in progress, skipping this tick" + exit 0 +fi + +TMPSNAP="$CURRENT.tmp.$$" +trap 'rm -f "$TMPSNAP"' EXIT + +# Print " " for each argument that is a regular file. Unmatched globs arrive +# literally and are filtered out by the -f test; IFS-safe and quoting-safe throughout. +hash_files() { + local f + for f in "$@"; do + [ -f "$f" ] || continue + echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)" + done | sort +} + +# Listening sockets, normalised: pid/fd stripped (they change on every restart) and duplicate +# process entries collapsed (nginx reports one entry per worker, so a worker-count change from +# a routine reload would otherwise look like a socket change). +listening_sockets() { + local flag="$1" + ss -"$flag"lnp 2>/dev/null | tail -n +2 | awk ' + { + proc = $6 + gsub(/,?(pid|fd)=[0-9]+/, "", proc) + gsub(/^users:\(|\)$/, "", proc) + n = split(proc, parts, /\),\(/) + out = "" + delete seen + for (i = 1; i <= n; i++) { + gsub(/[()"]/, "", parts[i]) + if (parts[i] != "" && !(parts[i] in seen)) { + seen[parts[i]] = 1 + out = (out == "" ? parts[i] : out "," parts[i]) + } + } + print $1, $4, out + }' | sort -u +} + +# Accepted SSH logins, accumulated into a persistent set rather than re-derived from a rolling +# journal window. Two reasons: (1) a rolling window makes old-but-legitimate source IPs *fall +# out* of the snapshot as they age, which shows up as an alarming "-" line for something that +# never actually changed; (2) scanning 90 days of a multi-GB journal every 15 minutes cost +# ~10s per run — with a saved cursor we only read entries added since the last run. +collect_ssh_logins() { + local raw="" rc newcur + if [ -s "$SSH_CURSOR" ]; then + raw=$(journalctl -u ssh --after-cursor "$(cat "$SSH_CURSOR")" --no-pager --show-cursor 2>/dev/null) + rc=$? + if [ "$rc" -ne 0 ]; then # cursor no longer in the journal (rotated) — reseed + raw="" + rm -f "$SSH_CURSOR" + fi + fi + if [ ! -s "$SSH_CURSOR" ]; then + raw=$(journalctl -u ssh --since "-90 days" --no-pager --show-cursor 2>/dev/null) + fi + if [ -n "$raw" ]; then + printf '%s\n' "$raw" \ + | grep -oE 'Accepted [a-z-]+ for [^ ]+ from [0-9a-fA-F.:]+' \ + | awk '{print $4"@"$6}' >> "$SSH_LOGINS" + newcur=$(printf '%s\n' "$raw" | sed -n 's/^-- cursor: //p' | tail -1) + [ -n "$newcur" ] && printf '%s\n' "$newcur" > "$SSH_CURSOR" + fi + [ -s "$SSH_LOGINS" ] && sort -u "$SSH_LOGINS" -o "$SSH_LOGINS" + cat "$SSH_LOGINS" 2>/dev/null +} + +snapshot() { + echo "## kernel and boot" + uname -srvm + echo "boot_id $(cat /proc/sys/kernel/random/boot_id 2>/dev/null)" + + echo "## users (login shells)" + awk -F: '$7 !~ /nologin|false/ {print $1":"$3":"$6":"$7}' /etc/passwd | sort + + echo "## uid 0 accounts" + awk -F: '$3 == 0 {print $1":"$3":"$6":"$7}' /etc/passwd | sort + + echo "## all accounts (name:uid:gid:shell)" + awk -F: '{print $1":"$3":"$4":"$7}' /etc/passwd | sort + + echo "## groups (name:gid:members)" + getent group 2>/dev/null | awk -F: '{print $1":"$3":"$4}' | sort + + echo "## credential db hashes" + hash_files /etc/shadow /etc/gshadow /etc/securetty + + echo "## ssh authorized_keys" + for f in /root/.ssh/authorized_keys /root/.ssh/authorized_keys2 \ + /home/*/.ssh/authorized_keys /home/*/.ssh/authorized_keys2; do + [ -f "$f" ] || continue + echo "-- $f --" + awk '{print $1, $NF}' "$f" + done + + echo "## sudoers (path sha256)" + hash_files /etc/sudoers /etc/sudoers.d/* + + echo "## pam.d configs (path sha256)" + hash_files /etc/pam.d/* + + echo "## ssh daemon config (path sha256)" + hash_files /etc/ssh/sshd_config /etc/ssh/sshd_config.d/* + + echo "## sshd effective policy (resolved by sshd -T, catches Include changes)" + sshd -T 2>/dev/null | grep -E '^(permitrootlogin|passwordauthentication|permitemptypasswords|pubkeyauthentication|kbdinteractiveauthentication|usepam|x11forwarding|allowtcpforwarding|gatewayports|permittunnel|port|listenaddress|authorizedkeysfile|authorizedkeyscommand|allowusers|allowgroups|denyusers|denygroups|forcecommand|permituserenvironment|maxauthtries|logingracetime)\b' | sort + + echo "## cron files (path sha256)" + hash_files /etc/crontab /etc/cron.d/* /etc/cron.daily/* /etc/cron.hourly/* \ + /etc/cron.weekly/* /etc/cron.monthly/* /var/spool/cron/crontabs/* + echo "-- root crontab --" + crontab -l -u root 2>/dev/null | sha256sum + echo "-- at jobs --" + find /var/spool/cron/atjobs -type f 2>/dev/null | sort + + echo "## systemd enabled services/timers (unit path sha256)" + systemctl list-unit-files --type=service,timer --state=enabled --no-legend 2>/dev/null \ + | awk '{print $1}' | sort -u | while IFS= read -r u; do + f=$(systemctl show -p FragmentPath --value "$u" 2>/dev/null) + if [ -n "$f" ] && [ -f "$f" ]; then + echo "$u $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)" + else + echo "$u (no fragment)" + fi + done + + echo "## shell startup files (path sha256)" + hash_files /etc/profile /etc/profile.d/* /etc/bash.bashrc /etc/environment \ + /root/.bashrc /root/.profile /root/.bash_profile /root/.bash_login \ + /root/.bash_logout /home/*/.bashrc /home/*/.profile /home/*/.bash_profile + + echo "## dynamic linker preload/config (path sha256)" + if [ -e /etc/ld.so.preload ]; then + hash_files /etc/ld.so.preload + else + echo "/etc/ld.so.preload ABSENT" # its appearance is a classic rootkit signal + fi + hash_files /etc/ld.so.conf /etc/ld.so.conf.d/* + + echo "## apt sources and keyrings (path sha256)" + hash_files /etc/apt/sources.list /etc/apt/sources.list.d/* \ + /etc/apt/trusted.gpg.d/* /etc/apt/keyrings/* /etc/apt/apt.conf.d/* + + echo "## name resolution (path sha256)" + hash_files /etc/hosts /etc/resolv.conf /etc/nsswitch.conf /etc/host.conf + + echo "## listening tcp sockets" + listening_sockets t + + echo "## listening udp sockets" + listening_sockets u + + echo "## docker containers (name image)" + docker ps -a --format "{{.Names}} {{.Image}}" 2>/dev/null | sort + + echo "## docker security posture (haloy deploy id normalised, so redeploys are silent)" + local ids + ids=$(docker ps -q 2>/dev/null) + if [ -n "$ids" ]; then + # shellcheck disable=SC2086 + docker inspect --format '{{.Name}} privileged={{.HostConfig.Privileged}} net={{.HostConfig.NetworkMode}} pid={{.HostConfig.PidMode}} user={{.Config.User}} caps={{.HostConfig.CapAdd}} secopt={{.HostConfig.SecurityOpt}} binds={{.HostConfig.Binds}}' $ids 2>/dev/null \ + | sed -E 's#^/##; s/^([^ ]+)-[0-9a-z]{26} /\1- /' | sort + fi + + echo "## mounted filesystems (real block devices only — excludes overlay/tmpfs/container churn)" + mount | grep -E '^/dev/' | awk '{print $1, $3, $5, $6}' | sort + + echo "## suid/sgid regular files (path sha256)" + # -xdev keeps us off other filesystems; the -prune list keeps us out of the container image + # stores, which live on / and whose layer dirs (each carrying its own suid su/mount/passwd) + # are created and destroyed on every image pull — i.e. on every haloy deploy. + find / -xdev \( -path /var/lib/docker -o -path /var/lib/containerd -o -path /var/lib/haloy \) -prune \ + -o -type f \( -perm -4000 -o -perm -2000 \) -print 2>/dev/null \ + | sort | while IFS= read -r f; do + echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)" + done + + echo "## file capabilities (suid-equivalent privilege grants)" + getcap -r /usr /opt /srv /etc /root /home 2>/dev/null | sort + + echo "## kernel modules loaded" + lsmod | awk 'NR>1{print $1}' | sort + + echo "## firewall base-chain policies/rules (host-level only — DOCKER-managed nested chains excluded, they churn on every container start/stop)" + for c in INPUT OUTPUT FORWARD; do + iptables -S "$c" 2>/dev/null | sed 's/^/v4 /' + ip6tables -S "$c" 2>/dev/null | sed 's/^/v6 /' + done + + echo "## nftables tables (names only — f2b rule bodies churn on every ban)" + nft list tables 2>/dev/null | sort + + echo "## critical binary hashes" + sha256sum /usr/bin/sudo /usr/bin/su /usr/sbin/sshd /bin/bash /bin/sh /usr/bin/docker \ + /usr/bin/dockerd /usr/bin/containerd /usr/bin/runc /usr/bin/systemctl \ + /usr/lib/systemd/systemd /usr/bin/curl /usr/bin/wget /usr/sbin/nft /usr/sbin/iptables \ + /usr/local/bin/haloyd /root/.local/bin/haloy 2>/dev/null | awk '{print $2, $1}' | sort + + echo "## ssh new logins (user@source — appended once seen, never removed)" + collect_ssh_logins + + # Sentinel: proves the snapshot ran to completion. Without it, a run killed mid-snapshot + # (OOM, cron timeout, hung docker) would write a truncated file, diff it as a huge fake + # "everything disappeared" alert, and then poison the baseline with the truncated version. + echo "## end of snapshot" +} + +send_telegram() { + local resp attempt + for attempt in 1 2 3; do + resp=$(curl -s -m 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT}/sendMessage" \ + --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ + --data-urlencode "text=$1" \ + --data-urlencode "disable_web_page_preview=true" 2>/dev/null) + case "$resp" in *'"ok":true'*) return 0 ;; esac + [ "$attempt" -lt 3 ] && sleep 5 + done + echo "WARNING: telegram send failed after 3 attempts; last response: ${resp:0:200}" >&2 + return 1 +} + +check_disk_usage() { + local host; host=$(hostname) + # -P for POSIX single-line-per-fs output; skip pseudo filesystems that don't reflect real + # disk (tmpfs/devtmpfs/overlay churn with container lifecycle and mean nothing here). + df -P -x tmpfs -x devtmpfs -x overlay -x squashfs 2>/dev/null | tail -n +2 | while read -r _fs _blocks _used avail pct mount; do + local pct_num=${pct%\%} + case "$pct_num" in ''|*[!0-9]*) continue ;; esac # skip unparseable lines defensively + local prev + prev=$(grep "^${mount}:" "$DISK_STATE" 2>/dev/null | tail -1 | cut -d: -f2) + # Only record the new state once the alert actually went out, so a failed send is + # retried on the next tick instead of being silently swallowed. + if [ "$pct_num" -ge "$DISK_ALERT_PCT" ] && [ "$prev" != "ALERTED" ]; then + if send_telegram "🔴 disk-monitor: ${mount} on ${host} is ${pct_num}% full (${avail}KB free) — crossed ${DISK_ALERT_PCT}% threshold"; then + sed -i "\#^${mount}:#d" "$DISK_STATE" + echo "${mount}:ALERTED" >> "$DISK_STATE" + fi + elif [ "$pct_num" -le "$DISK_RECOVER_PCT" ] && [ "$prev" = "ALERTED" ]; then + if send_telegram "🟢 disk-monitor: ${mount} on ${host} recovered to ${pct_num}% full"; then + sed -i "\#^${mount}:#d" "$DISK_STATE" + echo "${mount}:OK" >> "$DISK_STATE" + fi + fi + done +} + +snapshot > "$TMPSNAP" 2>/dev/null +if [ "$(tail -n 1 "$TMPSNAP" 2>/dev/null)" != "## end of snapshot" ]; then + echo "ERROR: snapshot did not complete (no sentinel) — leaving state untouched" >&2 + exit 1 +fi +mv -f "$TMPSNAP" "$CURRENT" + +check_disk_usage + +if [ ! -f "$PREVIOUS" ]; then + cp "$CURRENT" "$PREVIOUS" + echo "baseline seeded, no previous snapshot to compare against" + exit 0 +fi + +# -U is deliberately larger than the snapshot can plausibly get: with one all-encompassing +# hunk, every "## section" header is present as a context line, which is what lets the awk +# below attribute each changed line to its section. +DIFF=$(diff -U 100000 "$PREVIOUS" "$CURRENT" 2>/dev/null || true) +if [ -n "$DIFF" ]; then + HOST=$(hostname) + # Label the first changed line under each section — e.g. "[docker containers (name image)]" + # — instead of dumping bare +/- lines with no indication of what actually changed. + # NB: the ---/+++ skip is anchored to the first two lines on purpose. The snapshot itself + # contains lines like "-- /root/.ssh/authorized_keys --", and when such a line is *removed* + # the diff renders it as "--- /root/.ssh/authorized_keys --" — an unanchored /^--- / rule + # silently swallowed exactly the kind of event this script exists to report. + CHANGED=$(echo "$DIFF" | awk ' + NR <= 2 && /^(--- |\+\+\+ )/ { next } + /^@@/ { next } + { + marker = substr($0, 1, 1) + text = substr($0, 2) + } + text ~ /^## / { + section = substr(text, 4) + if (marker == "+" || marker == "-") { + print "[" section "]" + prevsection = section + print $0 + } + next + } + marker == "+" || marker == "-" { + if (section != prevsection) { print "[" section "]"; prevsection = section } + print $0 + } + ') + # Summarise up front: the body gets truncated to fit Telegram's limit, and a burst of + # changes in one section must not hide that a second, quieter section also changed. + SECTIONS=$(printf '%s\n' "$CHANGED" | sed -n 's/^\[\(.*\)\]$/\1/p' \ + | awk '{ printf "%s%s", (NR > 1 ? "; " : ""), $0 } END { if (NR) print "" }') + N_ADD=$(printf '%s\n' "$CHANGED" | grep -c '^+' || true) + N_DEL=$(printf '%s\n' "$CHANGED" | grep -c '^-' || true) + MSG="⚠️ security-monitor: change detected on ${HOST} +$(date "+%a %Y-%m-%d %H:%M:%S %Z") +sections: ${SECTIONS} +(+${N_ADD} / -${N_DEL} lines) + +${CHANGED}" + if [ "${#MSG}" -gt 3900 ]; then + MSG="${MSG:0:3880} +… [truncated]" + fi + if send_telegram "$MSG"; then + # Commit the new baseline *before* writing the diff to stdout: if stdout is a pipe + # that has already closed (someone running this by hand through `head`), the echo + # takes a SIGPIPE and the script dies — and without this ordering it would die + # holding the stale baseline and re-alert the identical change on the next run. + cp "$CURRENT" "$PREVIOUS" + echo "CHANGE DETECTED — alert sent" + echo "$CHANGED" + else + # Keep the old baseline so the very next run re-detects and re-reports this change, + # rather than losing it forever to a transient network failure. + echo "CHANGE DETECTED — alert FAILED to send; keeping previous baseline to retry next run" >&2 + echo "$CHANGED" + fi +else + echo "no change" + cp "$CURRENT" "$PREVIOUS" +fi