#!/bin/bash
# Periodic compromise/change monitor for this host.
#
# Snapshots a set of security-relevant system facts (kernel/boot id, accounts and groups,
# UID-0 accounts, SSH authorized keys, sudoers, PAM, sshd config + effective policy, cron/at,
# enabled systemd units, shell startup files, dynamic-linker preload/config, apt sources,
# name resolution, listening TCP/UDP sockets, docker containers and their security
# posture, mounted block devices, SUID/SGID files with hashes, file capabilities, loaded
# kernel modules, host-level firewall policy, nftables table list, critical binary hashes,
# and the cumulative set of accepted SSH logins), diffs against the previous run, and pushes
# a Telegram alert (with a "[section]" label per changed block) when anything changed. Silent
# (no alert, no output) when nothing changed. First run just seeds the baseline.
#
# Deliberately NOT snapshotted, because they'd churn constantly on this host and just be noise:
# full process list/cmdlines (PIDs and cron-spawned procs change every tick), established
# network connections (ephemeral client churn), the full Docker-managed firewall ruleset and
# nftables rule bodies (rewritten on every container start/stop and on every fail2ban ban —
# see the scoped host-level-only checks instead), SUID files under the container image stores
# (/var/lib/{docker,containerd,haloy} — image layers carry their own su/mount/passwd and the
# layer directory names change on every image pull), swap (active devices/usage flap with
# ordinary memory pressure, not a security signal), a generic "recent /etc mtimes" catch-all
# (certbot renewals, resolv.conf, apt all touch /etc routinely), and /tmp, /dev/shm, /var/tmp
# contents (this host runs ad-hoc dev/test work there routinely — dropped after it turned out
# to be pure noise, see git history).
#
# Separately (not part of the diff above — disk % changes constantly and would spam it),
# checks each real filesystem's usage against a threshold and alerts once on crossing it, with
# hysteresis so it doesn't flap an alert every tick while hovering right at the line.
#
# Install: run once manually to seed the baseline, then drop into cron (e.g. /etc/cron.d,
# every 15 min). All state lives under $STATE_DIR; nothing else on the host is touched.
set -uo pipefail

# Cron gives system crontabs a bare PATH (/usr/bin:/bin), so tools in /usr/sbin (iptables,
# nft, getcap...) silently vanish there while working fine in an interactive root shell —
# which made cron and manual runs produce *different* snapshots and fire false alerts (an
# empty section that should have had content, etc). Pin PATH so every run sees the same tools.
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# Likewise pin the locale: sort(1) collation and date(1) names are locale-dependent, so an
# interactive UTF-8 root shell and a locale-less cron run would otherwise order/format the
# snapshot differently and diff against each other.
export LC_ALL=C
umask 077   # state files hold key fingerprints and source IPs

# Telegram credentials live outside the script (it's in a public repo): a root-only env file
# with TELEGRAM_BOT=... and TELEGRAM_CHAT_ID=... lines. Fail loudly if it's missing rather than
# running a monitor that silently can't alert.
TELEGRAM_ENV="${TELEGRAM_ENV:-/root/.security-monitor/telegram.env}"
if [ ! -r "$TELEGRAM_ENV" ]; then
    echo "ERROR: $TELEGRAM_ENV missing or unreadable — no way to send alerts" >&2
    exit 1
fi
# shellcheck source=/dev/null
. "$TELEGRAM_ENV"
: "${TELEGRAM_BOT:?not set in $TELEGRAM_ENV}" "${TELEGRAM_CHAT_ID:?not set in $TELEGRAM_ENV}"

DISK_ALERT_PCT="${DISK_ALERT_PCT:-90}"    # alert once usage reaches this
DISK_RECOVER_PCT="${DISK_RECOVER_PCT:-85}"  # ...and don't re-arm until it drops back below this (hysteresis)

STATE_DIR=/root/.security-monitor
CURRENT="$STATE_DIR/current.snapshot"
PREVIOUS="$STATE_DIR/previous.snapshot"
DISK_STATE="$STATE_DIR/disk_alert_state"
SSH_LOGINS="$STATE_DIR/ssh_logins"      # cumulative user@ip set, append-only
SSH_CURSOR="$STATE_DIR/ssh_cursor"      # journal cursor so we only read new entries
LOCK="$STATE_DIR/lock"
mkdir -p "$STATE_DIR"
chmod 700 "$STATE_DIR"
touch "$DISK_STATE" "$SSH_LOGINS"
# umask only governs newly created files; chmod explicitly so state written by an older
# version of this script (or by a cp onto an existing file) is tightened too.
chmod 600 "$DISK_STATE" "$SSH_LOGINS" "$CURRENT" "$PREVIOUS" 2>/dev/null

# A slow run (big journal, hung docker) must not overlap the next cron tick and race on state.
exec 9>"$LOCK"
if ! flock -n 9; then
    echo "another run is in progress, skipping this tick"
    exit 0
fi

TMPSNAP="$CURRENT.tmp.$$"
trap 'rm -f "$TMPSNAP"' EXIT

# Print "<path> <sha256>" for each argument that is a regular file. Unmatched globs arrive
# literally and are filtered out by the -f test; IFS-safe and quoting-safe throughout.
hash_files() {
    local f
    for f in "$@"; do
        [ -f "$f" ] || continue
        echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
    done | sort
}

# Listening sockets, normalised: pid/fd stripped (they change on every restart) and duplicate
# process entries collapsed (nginx reports one entry per worker, so a worker-count change from
# a routine reload would otherwise look like a socket change).
# UDP sockets of IGNORE_UDP_PROCS are dropped: helvetiscan (a known, trusted scanner) opens and
# closes dozens of random-port UDP sockets per second, so every tick alerted on lines like
# "UNCONN *:13786 helvetiscan". Its TCP listeners are still tracked. Ownerless UDP sockets on a
# port >= 1024 are dropped too: they're the same scanner's sockets caught mid-teardown, already
# detached from the process (sequential inodes, a new port every second). Ownerless sockets on
# privileged ports (e.g. the DHCP client on :68) stay in the snapshot.
IGNORE_UDP_PROCS="${IGNORE_UDP_PROCS:-helvetiscan}"
listening_sockets() {
    local flag="$1"
    ss -"$flag"lnp 2>/dev/null | tail -n +2 | awk -v flag="$flag" -v ignprocs="$IGNORE_UDP_PROCS" '
    BEGIN { n = split(ignprocs, ip, /[ ,]+/); for (i = 1; i <= n; i++) if (ip[i] != "") ign[ip[i]] = 1 }
    {
        proc = $6
        gsub(/,?(pid|fd)=[0-9]+/, "", proc)
        gsub(/^users:\(|\)$/, "", proc)
        n = split(proc, parts, /\),\(/)
        out = ""
        delete seen
        for (i = 1; i <= n; i++) {
            gsub(/[()"]/, "", parts[i])
            if (parts[i] != "" && !(parts[i] in seen)) {
                seen[parts[i]] = 1
                out = (out == "" ? parts[i] : out "," parts[i])
            }
        }
        if (flag == "u") {
            port = $4; sub(/.*:/, "", port)
            if (out == "" && port + 0 >= 1024) next
            n = split(out, names, ",")
            for (i = 1; i <= n; i++) if (names[i] in ign) next
        }
        print $1, $4, out
    }' | sort -u
}

# Accepted SSH logins, accumulated into a persistent set rather than re-derived from a rolling
# journal window. Two reasons: (1) a rolling window makes old-but-legitimate source IPs *fall
# out* of the snapshot as they age, which shows up as an alarming "-" line for something that
# never actually changed; (2) scanning 90 days of a multi-GB journal every 15 minutes cost
# ~10s per run — with a saved cursor we only read entries added since the last run.
collect_ssh_logins() {
    local raw="" rc newcur
    if [ -s "$SSH_CURSOR" ]; then
        raw=$(journalctl -u ssh --after-cursor "$(cat "$SSH_CURSOR")" --no-pager --show-cursor 2>/dev/null)
        rc=$?
        if [ "$rc" -ne 0 ]; then   # cursor no longer in the journal (rotated) — reseed
            raw=""
            rm -f "$SSH_CURSOR"
        fi
    fi
    if [ ! -s "$SSH_CURSOR" ]; then
        raw=$(journalctl -u ssh --since "-90 days" --no-pager --show-cursor 2>/dev/null)
    fi
    if [ -n "$raw" ]; then
        printf '%s\n' "$raw" \
            | grep -oE 'Accepted [a-z-]+ for [^ ]+ from [0-9a-fA-F.:]+' \
            | awk '{print $4"@"$6}' >> "$SSH_LOGINS"
        newcur=$(printf '%s\n' "$raw" | sed -n 's/^-- cursor: //p' | tail -1)
        [ -n "$newcur" ] && printf '%s\n' "$newcur" > "$SSH_CURSOR"
    fi
    [ -s "$SSH_LOGINS" ] && sort -u "$SSH_LOGINS" -o "$SSH_LOGINS"
    cat "$SSH_LOGINS" 2>/dev/null
}

snapshot() {
    echo "## kernel and boot"
    uname -srvm
    echo "boot_id $(cat /proc/sys/kernel/random/boot_id 2>/dev/null)"

    echo "## users (login shells)"
    awk -F: '$7 !~ /nologin|false/ {print $1":"$3":"$6":"$7}' /etc/passwd | sort

    echo "## uid 0 accounts"
    awk -F: '$3 == 0 {print $1":"$3":"$6":"$7}' /etc/passwd | sort

    echo "## all accounts (name:uid:gid:shell)"
    awk -F: '{print $1":"$3":"$4":"$7}' /etc/passwd | sort

    echo "## groups (name:gid:members)"
    getent group 2>/dev/null | awk -F: '{print $1":"$3":"$4}' | sort

    echo "## credential db hashes"
    hash_files /etc/shadow /etc/gshadow /etc/securetty

    echo "## ssh authorized_keys"
    for f in /root/.ssh/authorized_keys /root/.ssh/authorized_keys2 \
             /home/*/.ssh/authorized_keys /home/*/.ssh/authorized_keys2; do
        [ -f "$f" ] || continue
        echo "-- $f --"
        awk '{print $1, $NF}' "$f"
    done

    echo "## sudoers (path sha256)"
    hash_files /etc/sudoers /etc/sudoers.d/*

    echo "## pam.d configs (path sha256)"
    hash_files /etc/pam.d/*

    echo "## ssh daemon config (path sha256)"
    hash_files /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*

    echo "## sshd effective policy (resolved by sshd -T, catches Include changes)"
    sshd -T 2>/dev/null | grep -E '^(permitrootlogin|passwordauthentication|permitemptypasswords|pubkeyauthentication|kbdinteractiveauthentication|usepam|x11forwarding|allowtcpforwarding|gatewayports|permittunnel|port|listenaddress|authorizedkeysfile|authorizedkeyscommand|allowusers|allowgroups|denyusers|denygroups|forcecommand|permituserenvironment|maxauthtries|logingracetime)\b' | sort

    echo "## cron files (path sha256)"
    hash_files /etc/crontab /etc/cron.d/* /etc/cron.daily/* /etc/cron.hourly/* \
               /etc/cron.weekly/* /etc/cron.monthly/* /var/spool/cron/crontabs/*
    echo "-- root crontab --"
    crontab -l -u root 2>/dev/null | sha256sum
    echo "-- at jobs --"
    find /var/spool/cron/atjobs -type f 2>/dev/null | sort

    echo "## systemd enabled services/timers (unit path sha256)"
    systemctl list-unit-files --type=service,timer --state=enabled --no-legend 2>/dev/null \
        | awk '{print $1}' | sort -u | while IFS= read -r u; do
        f=$(systemctl show -p FragmentPath --value "$u" 2>/dev/null)
        if [ -n "$f" ] && [ -f "$f" ]; then
            echo "$u $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
        else
            echo "$u (no fragment)"
        fi
    done

    echo "## shell startup files (path sha256)"
    hash_files /etc/profile /etc/profile.d/* /etc/bash.bashrc /etc/environment \
               /root/.bashrc /root/.profile /root/.bash_profile /root/.bash_login \
               /root/.bash_logout /home/*/.bashrc /home/*/.profile /home/*/.bash_profile

    echo "## dynamic linker preload/config (path sha256)"
    if [ -e /etc/ld.so.preload ]; then
        hash_files /etc/ld.so.preload
    else
        echo "/etc/ld.so.preload ABSENT"   # its appearance is a classic rootkit signal
    fi
    hash_files /etc/ld.so.conf /etc/ld.so.conf.d/*

    echo "## apt sources and keyrings (path sha256)"
    hash_files /etc/apt/sources.list /etc/apt/sources.list.d/* \
               /etc/apt/trusted.gpg.d/* /etc/apt/keyrings/* /etc/apt/apt.conf.d/*

    echo "## name resolution (path sha256)"
    hash_files /etc/hosts /etc/resolv.conf /etc/nsswitch.conf /etc/host.conf

    echo "## listening tcp sockets"
    listening_sockets t

    echo "## listening udp sockets"
    listening_sockets u

    echo "## docker containers (name image)"
    docker ps -a --format "{{.Names}} {{.Image}}" 2>/dev/null | sort

    echo "## docker security posture (haloy deploy id normalised, so redeploys are silent)"
    local ids
    ids=$(docker ps -q 2>/dev/null)
    if [ -n "$ids" ]; then
        # shellcheck disable=SC2086
        docker inspect --format '{{.Name}} privileged={{.HostConfig.Privileged}} net={{.HostConfig.NetworkMode}} pid={{.HostConfig.PidMode}} user={{.Config.User}} caps={{.HostConfig.CapAdd}} secopt={{.HostConfig.SecurityOpt}} binds={{.HostConfig.Binds}}' $ids 2>/dev/null \
            | sed -E 's#^/##; s/^([^ ]+)-[0-9a-z]{26} /\1-<deploy> /' | sort
    fi

    echo "## mounted filesystems (real block devices only — excludes overlay/tmpfs/container churn)"
    mount | grep -E '^/dev/' | awk '{print $1, $3, $5, $6}' | sort

    echo "## suid/sgid regular files (path sha256)"
    # -xdev keeps us off other filesystems; the -prune list keeps us out of the container image
    # stores, which live on / and whose layer dirs (each carrying its own suid su/mount/passwd)
    # are created and destroyed on every image pull — i.e. on every haloy deploy.
    find / -xdev \( -path /var/lib/docker -o -path /var/lib/containerd -o -path /var/lib/haloy \) -prune \
         -o -type f \( -perm -4000 -o -perm -2000 \) -print 2>/dev/null \
        | sort | while IFS= read -r f; do
        echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
    done

    echo "## file capabilities (suid-equivalent privilege grants)"
    getcap -r /usr /opt /srv /etc /root /home 2>/dev/null | sort

    echo "## kernel modules loaded"
    lsmod | awk 'NR>1{print $1}' | sort

    echo "## firewall base-chain policies/rules (host-level only — DOCKER-managed nested chains excluded, they churn on every container start/stop)"
    for c in INPUT OUTPUT FORWARD; do
        iptables -S "$c" 2>/dev/null | sed 's/^/v4 /'
        ip6tables -S "$c" 2>/dev/null | sed 's/^/v6 /'
    done

    echo "## nftables tables (names only — f2b rule bodies churn on every ban)"
    nft list tables 2>/dev/null | sort

    echo "## critical binary hashes"
    sha256sum /usr/bin/sudo /usr/bin/su /usr/sbin/sshd /bin/bash /bin/sh /usr/bin/docker \
        /usr/bin/dockerd /usr/bin/containerd /usr/bin/runc /usr/bin/systemctl \
        /usr/lib/systemd/systemd /usr/bin/curl /usr/bin/wget /usr/sbin/nft /usr/sbin/iptables \
        /usr/local/bin/haloyd /root/.local/bin/haloy 2>/dev/null | awk '{print $2, $1}' | sort

    echo "## ssh new logins (user@source — appended once seen, never removed)"
    collect_ssh_logins

    # Sentinel: proves the snapshot ran to completion. Without it, a run killed mid-snapshot
    # (OOM, cron timeout, hung docker) would write a truncated file, diff it as a huge fake
    # "everything disappeared" alert, and then poison the baseline with the truncated version.
    echo "## end of snapshot"
}

send_telegram() {
    local resp attempt
    for attempt in 1 2 3; do
        resp=$(curl -s -m 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT}/sendMessage" \
            --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
            --data-urlencode "text=$1" \
            --data-urlencode "disable_web_page_preview=true" 2>/dev/null)
        case "$resp" in *'"ok":true'*) return 0 ;; esac
        [ "$attempt" -lt 3 ] && sleep 5
    done
    echo "WARNING: telegram send failed after 3 attempts; last response: ${resp:0:200}" >&2
    return 1
}

check_disk_usage() {
    local host; host=$(hostname)
    # -P for POSIX single-line-per-fs output; skip pseudo filesystems that don't reflect real
    # disk (tmpfs/devtmpfs/overlay churn with container lifecycle and mean nothing here).
    df -P -x tmpfs -x devtmpfs -x overlay -x squashfs 2>/dev/null | tail -n +2 | while read -r _fs _blocks _used avail pct mount; do
        local pct_num=${pct%\%}
        case "$pct_num" in ''|*[!0-9]*) continue ;; esac  # skip unparseable lines defensively
        local prev
        prev=$(grep "^${mount}:" "$DISK_STATE" 2>/dev/null | tail -1 | cut -d: -f2)
        # Only record the new state once the alert actually went out, so a failed send is
        # retried on the next tick instead of being silently swallowed.
        if [ "$pct_num" -ge "$DISK_ALERT_PCT" ] && [ "$prev" != "ALERTED" ]; then
            if send_telegram "🔴 disk-monitor: ${mount} on ${host} is ${pct_num}% full (${avail}KB free) — crossed ${DISK_ALERT_PCT}% threshold"; then
                sed -i "\#^${mount}:#d" "$DISK_STATE"
                echo "${mount}:ALERTED" >> "$DISK_STATE"
            fi
        elif [ "$pct_num" -le "$DISK_RECOVER_PCT" ] && [ "$prev" = "ALERTED" ]; then
            if send_telegram "🟢 disk-monitor: ${mount} on ${host} recovered to ${pct_num}% full"; then
                sed -i "\#^${mount}:#d" "$DISK_STATE"
                echo "${mount}:OK" >> "$DISK_STATE"
            fi
        fi
    done
}

snapshot > "$TMPSNAP" 2>/dev/null
if [ "$(tail -n 1 "$TMPSNAP" 2>/dev/null)" != "## end of snapshot" ]; then
    echo "ERROR: snapshot did not complete (no sentinel) — leaving state untouched" >&2
    exit 1
fi
mv -f "$TMPSNAP" "$CURRENT"

check_disk_usage

if [ ! -f "$PREVIOUS" ]; then
    cp "$CURRENT" "$PREVIOUS"
    echo "baseline seeded, no previous snapshot to compare against"
    exit 0
fi

# -U is deliberately larger than the snapshot can plausibly get: with one all-encompassing
# hunk, every "## section" header is present as a context line, which is what lets the awk
# below attribute each changed line to its section.
DIFF=$(diff -U 100000 "$PREVIOUS" "$CURRENT" 2>/dev/null || true)
if [ -n "$DIFF" ]; then
    HOST=$(hostname)
    # Label the first changed line under each section — e.g. "[docker containers (name image)]"
    # — instead of dumping bare +/- lines with no indication of what actually changed.
    # NB: the ---/+++ skip is anchored to the first two lines on purpose. The snapshot itself
    # contains lines like "-- /root/.ssh/authorized_keys --", and when such a line is *removed*
    # the diff renders it as "--- /root/.ssh/authorized_keys --" — an unanchored /^--- / rule
    # silently swallowed exactly the kind of event this script exists to report.
    CHANGED=$(echo "$DIFF" | awk '
        NR <= 2 && /^(--- |\+\+\+ )/ { next }
        /^@@/ { next }
        {
            marker = substr($0, 1, 1)
            text   = substr($0, 2)
        }
        text ~ /^## / {
            section = substr(text, 4)
            if (marker == "+" || marker == "-") {
                print "[" section "]"
                prevsection = section
                print $0
            }
            next
        }
        marker == "+" || marker == "-" {
            if (section != prevsection) { print "[" section "]"; prevsection = section }
            print $0
        }
    ')
    # Summarise up front: the body gets truncated to fit Telegram's limit, and a burst of
    # changes in one section must not hide that a second, quieter section also changed.
    SECTIONS=$(printf '%s\n' "$CHANGED" | sed -n 's/^\[\(.*\)\]$/\1/p' \
        | awk '{ printf "%s%s", (NR > 1 ? "; " : ""), $0 } END { if (NR) print "" }')
    N_ADD=$(printf '%s\n' "$CHANGED" | grep -c '^+' || true)
    N_DEL=$(printf '%s\n' "$CHANGED" | grep -c '^-' || true)
    MSG="⚠️ security-monitor: change detected on ${HOST}
$(date "+%a %Y-%m-%d %H:%M:%S %Z")
sections: ${SECTIONS}
(+${N_ADD} / -${N_DEL} lines)

${CHANGED}"
    if [ "${#MSG}" -gt 3900 ]; then
        MSG="${MSG:0:3880}
… [truncated]"
    fi
    if send_telegram "$MSG"; then
        # Commit the new baseline *before* writing the diff to stdout: if stdout is a pipe
        # that has already closed (someone running this by hand through `head`), the echo
        # takes a SIGPIPE and the script dies — and without this ordering it would die
        # holding the stale baseline and re-alert the identical change on the next run.
        cp "$CURRENT" "$PREVIOUS"
        echo "CHANGE DETECTED — alert sent"
        echo "$CHANGED"
    else
        # Keep the old baseline so the very next run re-detects and re-reports this change,
        # rather than losing it forever to a transient network failure.
        echo "CHANGE DETECTED — alert FAILED to send; keeping previous baseline to retry next run" >&2
        echo "$CHANGED"
    fi
else
    echo "no change"
    cp "$CURRENT" "$PREVIOUS"
fi