399 lines
18 KiB
Bash
Executable File
399 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# Periodic compromise/change monitor for this host.
|
|
#
|
|
# Snapshots a set of security-relevant system facts (kernel/boot id, accounts and groups,
|
|
# UID-0 accounts, SSH authorized keys, sudoers, PAM, sshd config + effective policy, cron/at,
|
|
# enabled systemd units, shell startup files, dynamic-linker preload/config, apt sources,
|
|
# name resolution, listening TCP/UDP sockets, docker containers and their security
|
|
# posture, mounted block devices, SUID/SGID files with hashes, file capabilities, loaded
|
|
# kernel modules, host-level firewall policy, nftables table list, critical binary hashes,
|
|
# and the cumulative set of accepted SSH logins), diffs against the previous run, and pushes
|
|
# a Telegram alert (with a "[section]" label per changed block) when anything changed. Silent
|
|
# (no alert, no output) when nothing changed. First run just seeds the baseline.
|
|
#
|
|
# Deliberately NOT snapshotted, because they'd churn constantly on this host and just be noise:
|
|
# full process list/cmdlines (PIDs and cron-spawned procs change every tick), established
|
|
# network connections (ephemeral client churn), the full Docker-managed firewall ruleset and
|
|
# nftables rule bodies (rewritten on every container start/stop and on every fail2ban ban —
|
|
# see the scoped host-level-only checks instead), SUID files under the container image stores
|
|
# (/var/lib/{docker,containerd,haloy} — image layers carry their own su/mount/passwd and the
|
|
# layer directory names change on every image pull), swap (active devices/usage flap with
|
|
# ordinary memory pressure, not a security signal), a generic "recent /etc mtimes" catch-all
|
|
# (certbot renewals, resolv.conf, apt all touch /etc routinely), and /tmp, /dev/shm, /var/tmp
|
|
# contents (this host runs ad-hoc dev/test work there routinely — dropped after it turned out
|
|
# to be pure noise, see git history).
|
|
#
|
|
# Separately (not part of the diff above — disk % changes constantly and would spam it),
|
|
# checks each real filesystem's usage against a threshold and alerts once on crossing it, with
|
|
# hysteresis so it doesn't flap an alert every tick while hovering right at the line.
|
|
#
|
|
# Install: run once manually to seed the baseline, then drop into cron (e.g. /etc/cron.d,
|
|
# every 15 min). All state lives under $STATE_DIR; nothing else on the host is touched.
|
|
set -uo pipefail
|
|
|
|
# Cron gives system crontabs a bare PATH (/usr/bin:/bin), so tools in /usr/sbin (iptables,
|
|
# nft, getcap...) silently vanish there while working fine in an interactive root shell —
|
|
# which made cron and manual runs produce *different* snapshots and fire false alerts (an
|
|
# empty section that should have had content, etc). Pin PATH so every run sees the same tools.
|
|
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
# Likewise pin the locale: sort(1) collation and date(1) names are locale-dependent, so an
|
|
# interactive UTF-8 root shell and a locale-less cron run would otherwise order/format the
|
|
# snapshot differently and diff against each other.
|
|
export LC_ALL=C
|
|
umask 077 # state files hold key fingerprints and source IPs
|
|
|
|
TELEGRAM_BOT="8602530664:AAFE7tsRY79zhhEjU7sRR7MrS8p3FB0jvXg"
|
|
TELEGRAM_CHAT_ID="108712105"
|
|
|
|
DISK_ALERT_PCT="${DISK_ALERT_PCT:-90}" # alert once usage reaches this
|
|
DISK_RECOVER_PCT="${DISK_RECOVER_PCT:-85}" # ...and don't re-arm until it drops back below this (hysteresis)
|
|
|
|
STATE_DIR=/root/.security-monitor
|
|
CURRENT="$STATE_DIR/current.snapshot"
|
|
PREVIOUS="$STATE_DIR/previous.snapshot"
|
|
DISK_STATE="$STATE_DIR/disk_alert_state"
|
|
SSH_LOGINS="$STATE_DIR/ssh_logins" # cumulative user@ip set, append-only
|
|
SSH_CURSOR="$STATE_DIR/ssh_cursor" # journal cursor so we only read new entries
|
|
LOCK="$STATE_DIR/lock"
|
|
mkdir -p "$STATE_DIR"
|
|
chmod 700 "$STATE_DIR"
|
|
touch "$DISK_STATE" "$SSH_LOGINS"
|
|
# umask only governs newly created files; chmod explicitly so state written by an older
|
|
# version of this script (or by a cp onto an existing file) is tightened too.
|
|
chmod 600 "$DISK_STATE" "$SSH_LOGINS" "$CURRENT" "$PREVIOUS" 2>/dev/null
|
|
|
|
# A slow run (big journal, hung docker) must not overlap the next cron tick and race on state.
|
|
exec 9>"$LOCK"
|
|
if ! flock -n 9; then
|
|
echo "another run is in progress, skipping this tick"
|
|
exit 0
|
|
fi
|
|
|
|
TMPSNAP="$CURRENT.tmp.$$"
|
|
trap 'rm -f "$TMPSNAP"' EXIT
|
|
|
|
# Print "<path> <sha256>" for each argument that is a regular file. Unmatched globs arrive
|
|
# literally and are filtered out by the -f test; IFS-safe and quoting-safe throughout.
|
|
hash_files() {
|
|
local f
|
|
for f in "$@"; do
|
|
[ -f "$f" ] || continue
|
|
echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
|
|
done | sort
|
|
}
|
|
|
|
# Listening sockets, normalised: pid/fd stripped (they change on every restart) and duplicate
|
|
# process entries collapsed (nginx reports one entry per worker, so a worker-count change from
|
|
# a routine reload would otherwise look like a socket change).
|
|
listening_sockets() {
|
|
local flag="$1"
|
|
ss -"$flag"lnp 2>/dev/null | tail -n +2 | awk '
|
|
{
|
|
proc = $6
|
|
gsub(/,?(pid|fd)=[0-9]+/, "", proc)
|
|
gsub(/^users:\(|\)$/, "", proc)
|
|
n = split(proc, parts, /\),\(/)
|
|
out = ""
|
|
delete seen
|
|
for (i = 1; i <= n; i++) {
|
|
gsub(/[()"]/, "", parts[i])
|
|
if (parts[i] != "" && !(parts[i] in seen)) {
|
|
seen[parts[i]] = 1
|
|
out = (out == "" ? parts[i] : out "," parts[i])
|
|
}
|
|
}
|
|
print $1, $4, out
|
|
}' | sort -u
|
|
}
|
|
|
|
# Accepted SSH logins, accumulated into a persistent set rather than re-derived from a rolling
|
|
# journal window. Two reasons: (1) a rolling window makes old-but-legitimate source IPs *fall
|
|
# out* of the snapshot as they age, which shows up as an alarming "-" line for something that
|
|
# never actually changed; (2) scanning 90 days of a multi-GB journal every 15 minutes cost
|
|
# ~10s per run — with a saved cursor we only read entries added since the last run.
|
|
collect_ssh_logins() {
|
|
local raw="" rc newcur
|
|
if [ -s "$SSH_CURSOR" ]; then
|
|
raw=$(journalctl -u ssh --after-cursor "$(cat "$SSH_CURSOR")" --no-pager --show-cursor 2>/dev/null)
|
|
rc=$?
|
|
if [ "$rc" -ne 0 ]; then # cursor no longer in the journal (rotated) — reseed
|
|
raw=""
|
|
rm -f "$SSH_CURSOR"
|
|
fi
|
|
fi
|
|
if [ ! -s "$SSH_CURSOR" ]; then
|
|
raw=$(journalctl -u ssh --since "-90 days" --no-pager --show-cursor 2>/dev/null)
|
|
fi
|
|
if [ -n "$raw" ]; then
|
|
printf '%s\n' "$raw" \
|
|
| grep -oE 'Accepted [a-z-]+ for [^ ]+ from [0-9a-fA-F.:]+' \
|
|
| awk '{print $4"@"$6}' >> "$SSH_LOGINS"
|
|
newcur=$(printf '%s\n' "$raw" | sed -n 's/^-- cursor: //p' | tail -1)
|
|
[ -n "$newcur" ] && printf '%s\n' "$newcur" > "$SSH_CURSOR"
|
|
fi
|
|
[ -s "$SSH_LOGINS" ] && sort -u "$SSH_LOGINS" -o "$SSH_LOGINS"
|
|
cat "$SSH_LOGINS" 2>/dev/null
|
|
}
|
|
|
|
snapshot() {
|
|
echo "## kernel and boot"
|
|
uname -srvm
|
|
echo "boot_id $(cat /proc/sys/kernel/random/boot_id 2>/dev/null)"
|
|
|
|
echo "## users (login shells)"
|
|
awk -F: '$7 !~ /nologin|false/ {print $1":"$3":"$6":"$7}' /etc/passwd | sort
|
|
|
|
echo "## uid 0 accounts"
|
|
awk -F: '$3 == 0 {print $1":"$3":"$6":"$7}' /etc/passwd | sort
|
|
|
|
echo "## all accounts (name:uid:gid:shell)"
|
|
awk -F: '{print $1":"$3":"$4":"$7}' /etc/passwd | sort
|
|
|
|
echo "## groups (name:gid:members)"
|
|
getent group 2>/dev/null | awk -F: '{print $1":"$3":"$4}' | sort
|
|
|
|
echo "## credential db hashes"
|
|
hash_files /etc/shadow /etc/gshadow /etc/securetty
|
|
|
|
echo "## ssh authorized_keys"
|
|
for f in /root/.ssh/authorized_keys /root/.ssh/authorized_keys2 \
|
|
/home/*/.ssh/authorized_keys /home/*/.ssh/authorized_keys2; do
|
|
[ -f "$f" ] || continue
|
|
echo "-- $f --"
|
|
awk '{print $1, $NF}' "$f"
|
|
done
|
|
|
|
echo "## sudoers (path sha256)"
|
|
hash_files /etc/sudoers /etc/sudoers.d/*
|
|
|
|
echo "## pam.d configs (path sha256)"
|
|
hash_files /etc/pam.d/*
|
|
|
|
echo "## ssh daemon config (path sha256)"
|
|
hash_files /etc/ssh/sshd_config /etc/ssh/sshd_config.d/*
|
|
|
|
echo "## sshd effective policy (resolved by sshd -T, catches Include changes)"
|
|
sshd -T 2>/dev/null | grep -E '^(permitrootlogin|passwordauthentication|permitemptypasswords|pubkeyauthentication|kbdinteractiveauthentication|usepam|x11forwarding|allowtcpforwarding|gatewayports|permittunnel|port|listenaddress|authorizedkeysfile|authorizedkeyscommand|allowusers|allowgroups|denyusers|denygroups|forcecommand|permituserenvironment|maxauthtries|logingracetime)\b' | sort
|
|
|
|
echo "## cron files (path sha256)"
|
|
hash_files /etc/crontab /etc/cron.d/* /etc/cron.daily/* /etc/cron.hourly/* \
|
|
/etc/cron.weekly/* /etc/cron.monthly/* /var/spool/cron/crontabs/*
|
|
echo "-- root crontab --"
|
|
crontab -l -u root 2>/dev/null | sha256sum
|
|
echo "-- at jobs --"
|
|
find /var/spool/cron/atjobs -type f 2>/dev/null | sort
|
|
|
|
echo "## systemd enabled services/timers (unit path sha256)"
|
|
systemctl list-unit-files --type=service,timer --state=enabled --no-legend 2>/dev/null \
|
|
| awk '{print $1}' | sort -u | while IFS= read -r u; do
|
|
f=$(systemctl show -p FragmentPath --value "$u" 2>/dev/null)
|
|
if [ -n "$f" ] && [ -f "$f" ]; then
|
|
echo "$u $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
|
|
else
|
|
echo "$u (no fragment)"
|
|
fi
|
|
done
|
|
|
|
echo "## shell startup files (path sha256)"
|
|
hash_files /etc/profile /etc/profile.d/* /etc/bash.bashrc /etc/environment \
|
|
/root/.bashrc /root/.profile /root/.bash_profile /root/.bash_login \
|
|
/root/.bash_logout /home/*/.bashrc /home/*/.profile /home/*/.bash_profile
|
|
|
|
echo "## dynamic linker preload/config (path sha256)"
|
|
if [ -e /etc/ld.so.preload ]; then
|
|
hash_files /etc/ld.so.preload
|
|
else
|
|
echo "/etc/ld.so.preload ABSENT" # its appearance is a classic rootkit signal
|
|
fi
|
|
hash_files /etc/ld.so.conf /etc/ld.so.conf.d/*
|
|
|
|
echo "## apt sources and keyrings (path sha256)"
|
|
hash_files /etc/apt/sources.list /etc/apt/sources.list.d/* \
|
|
/etc/apt/trusted.gpg.d/* /etc/apt/keyrings/* /etc/apt/apt.conf.d/*
|
|
|
|
echo "## name resolution (path sha256)"
|
|
hash_files /etc/hosts /etc/resolv.conf /etc/nsswitch.conf /etc/host.conf
|
|
|
|
echo "## listening tcp sockets"
|
|
listening_sockets t
|
|
|
|
echo "## listening udp sockets"
|
|
listening_sockets u
|
|
|
|
echo "## docker containers (name image)"
|
|
docker ps -a --format "{{.Names}} {{.Image}}" 2>/dev/null | sort
|
|
|
|
echo "## docker security posture (haloy deploy id normalised, so redeploys are silent)"
|
|
local ids
|
|
ids=$(docker ps -q 2>/dev/null)
|
|
if [ -n "$ids" ]; then
|
|
# shellcheck disable=SC2086
|
|
docker inspect --format '{{.Name}} privileged={{.HostConfig.Privileged}} net={{.HostConfig.NetworkMode}} pid={{.HostConfig.PidMode}} user={{.Config.User}} caps={{.HostConfig.CapAdd}} secopt={{.HostConfig.SecurityOpt}} binds={{.HostConfig.Binds}}' $ids 2>/dev/null \
|
|
| sed -E 's#^/##; s/^([^ ]+)-[0-9a-z]{26} /\1-<deploy> /' | sort
|
|
fi
|
|
|
|
echo "## mounted filesystems (real block devices only — excludes overlay/tmpfs/container churn)"
|
|
mount | grep -E '^/dev/' | awk '{print $1, $3, $5, $6}' | sort
|
|
|
|
echo "## suid/sgid regular files (path sha256)"
|
|
# -xdev keeps us off other filesystems; the -prune list keeps us out of the container image
|
|
# stores, which live on / and whose layer dirs (each carrying its own suid su/mount/passwd)
|
|
# are created and destroyed on every image pull — i.e. on every haloy deploy.
|
|
find / -xdev \( -path /var/lib/docker -o -path /var/lib/containerd -o -path /var/lib/haloy \) -prune \
|
|
-o -type f \( -perm -4000 -o -perm -2000 \) -print 2>/dev/null \
|
|
| sort | while IFS= read -r f; do
|
|
echo "$f $(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)"
|
|
done
|
|
|
|
echo "## file capabilities (suid-equivalent privilege grants)"
|
|
getcap -r /usr /opt /srv /etc /root /home 2>/dev/null | sort
|
|
|
|
echo "## kernel modules loaded"
|
|
lsmod | awk 'NR>1{print $1}' | sort
|
|
|
|
echo "## firewall base-chain policies/rules (host-level only — DOCKER-managed nested chains excluded, they churn on every container start/stop)"
|
|
for c in INPUT OUTPUT FORWARD; do
|
|
iptables -S "$c" 2>/dev/null | sed 's/^/v4 /'
|
|
ip6tables -S "$c" 2>/dev/null | sed 's/^/v6 /'
|
|
done
|
|
|
|
echo "## nftables tables (names only — f2b rule bodies churn on every ban)"
|
|
nft list tables 2>/dev/null | sort
|
|
|
|
echo "## critical binary hashes"
|
|
sha256sum /usr/bin/sudo /usr/bin/su /usr/sbin/sshd /bin/bash /bin/sh /usr/bin/docker \
|
|
/usr/bin/dockerd /usr/bin/containerd /usr/bin/runc /usr/bin/systemctl \
|
|
/usr/lib/systemd/systemd /usr/bin/curl /usr/bin/wget /usr/sbin/nft /usr/sbin/iptables \
|
|
/usr/local/bin/haloyd /root/.local/bin/haloy 2>/dev/null | awk '{print $2, $1}' | sort
|
|
|
|
echo "## ssh new logins (user@source — appended once seen, never removed)"
|
|
collect_ssh_logins
|
|
|
|
# Sentinel: proves the snapshot ran to completion. Without it, a run killed mid-snapshot
|
|
# (OOM, cron timeout, hung docker) would write a truncated file, diff it as a huge fake
|
|
# "everything disappeared" alert, and then poison the baseline with the truncated version.
|
|
echo "## end of snapshot"
|
|
}
|
|
|
|
send_telegram() {
|
|
local resp attempt
|
|
for attempt in 1 2 3; do
|
|
resp=$(curl -s -m 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT}/sendMessage" \
|
|
--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
|
|
--data-urlencode "text=$1" \
|
|
--data-urlencode "disable_web_page_preview=true" 2>/dev/null)
|
|
case "$resp" in *'"ok":true'*) return 0 ;; esac
|
|
[ "$attempt" -lt 3 ] && sleep 5
|
|
done
|
|
echo "WARNING: telegram send failed after 3 attempts; last response: ${resp:0:200}" >&2
|
|
return 1
|
|
}
|
|
|
|
check_disk_usage() {
|
|
local host; host=$(hostname)
|
|
# -P for POSIX single-line-per-fs output; skip pseudo filesystems that don't reflect real
|
|
# disk (tmpfs/devtmpfs/overlay churn with container lifecycle and mean nothing here).
|
|
df -P -x tmpfs -x devtmpfs -x overlay -x squashfs 2>/dev/null | tail -n +2 | while read -r _fs _blocks _used avail pct mount; do
|
|
local pct_num=${pct%\%}
|
|
case "$pct_num" in ''|*[!0-9]*) continue ;; esac # skip unparseable lines defensively
|
|
local prev
|
|
prev=$(grep "^${mount}:" "$DISK_STATE" 2>/dev/null | tail -1 | cut -d: -f2)
|
|
# Only record the new state once the alert actually went out, so a failed send is
|
|
# retried on the next tick instead of being silently swallowed.
|
|
if [ "$pct_num" -ge "$DISK_ALERT_PCT" ] && [ "$prev" != "ALERTED" ]; then
|
|
if send_telegram "🔴 disk-monitor: ${mount} on ${host} is ${pct_num}% full (${avail}KB free) — crossed ${DISK_ALERT_PCT}% threshold"; then
|
|
sed -i "\#^${mount}:#d" "$DISK_STATE"
|
|
echo "${mount}:ALERTED" >> "$DISK_STATE"
|
|
fi
|
|
elif [ "$pct_num" -le "$DISK_RECOVER_PCT" ] && [ "$prev" = "ALERTED" ]; then
|
|
if send_telegram "🟢 disk-monitor: ${mount} on ${host} recovered to ${pct_num}% full"; then
|
|
sed -i "\#^${mount}:#d" "$DISK_STATE"
|
|
echo "${mount}:OK" >> "$DISK_STATE"
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
snapshot > "$TMPSNAP" 2>/dev/null
|
|
if [ "$(tail -n 1 "$TMPSNAP" 2>/dev/null)" != "## end of snapshot" ]; then
|
|
echo "ERROR: snapshot did not complete (no sentinel) — leaving state untouched" >&2
|
|
exit 1
|
|
fi
|
|
mv -f "$TMPSNAP" "$CURRENT"
|
|
|
|
check_disk_usage
|
|
|
|
if [ ! -f "$PREVIOUS" ]; then
|
|
cp "$CURRENT" "$PREVIOUS"
|
|
echo "baseline seeded, no previous snapshot to compare against"
|
|
exit 0
|
|
fi
|
|
|
|
# -U is deliberately larger than the snapshot can plausibly get: with one all-encompassing
|
|
# hunk, every "## section" header is present as a context line, which is what lets the awk
|
|
# below attribute each changed line to its section.
|
|
DIFF=$(diff -U 100000 "$PREVIOUS" "$CURRENT" 2>/dev/null || true)
|
|
if [ -n "$DIFF" ]; then
|
|
HOST=$(hostname)
|
|
# Label the first changed line under each section — e.g. "[docker containers (name image)]"
|
|
# — instead of dumping bare +/- lines with no indication of what actually changed.
|
|
# NB: the ---/+++ skip is anchored to the first two lines on purpose. The snapshot itself
|
|
# contains lines like "-- /root/.ssh/authorized_keys --", and when such a line is *removed*
|
|
# the diff renders it as "--- /root/.ssh/authorized_keys --" — an unanchored /^--- / rule
|
|
# silently swallowed exactly the kind of event this script exists to report.
|
|
CHANGED=$(echo "$DIFF" | awk '
|
|
NR <= 2 && /^(--- |\+\+\+ )/ { next }
|
|
/^@@/ { next }
|
|
{
|
|
marker = substr($0, 1, 1)
|
|
text = substr($0, 2)
|
|
}
|
|
text ~ /^## / {
|
|
section = substr(text, 4)
|
|
if (marker == "+" || marker == "-") {
|
|
print "[" section "]"
|
|
prevsection = section
|
|
print $0
|
|
}
|
|
next
|
|
}
|
|
marker == "+" || marker == "-" {
|
|
if (section != prevsection) { print "[" section "]"; prevsection = section }
|
|
print $0
|
|
}
|
|
')
|
|
# Summarise up front: the body gets truncated to fit Telegram's limit, and a burst of
|
|
# changes in one section must not hide that a second, quieter section also changed.
|
|
SECTIONS=$(printf '%s\n' "$CHANGED" | sed -n 's/^\[\(.*\)\]$/\1/p' \
|
|
| awk '{ printf "%s%s", (NR > 1 ? "; " : ""), $0 } END { if (NR) print "" }')
|
|
N_ADD=$(printf '%s\n' "$CHANGED" | grep -c '^+' || true)
|
|
N_DEL=$(printf '%s\n' "$CHANGED" | grep -c '^-' || true)
|
|
MSG="⚠️ security-monitor: change detected on ${HOST}
|
|
$(date "+%a %Y-%m-%d %H:%M:%S %Z")
|
|
sections: ${SECTIONS}
|
|
(+${N_ADD} / -${N_DEL} lines)
|
|
|
|
${CHANGED}"
|
|
if [ "${#MSG}" -gt 3900 ]; then
|
|
MSG="${MSG:0:3880}
|
|
… [truncated]"
|
|
fi
|
|
if send_telegram "$MSG"; then
|
|
# Commit the new baseline *before* writing the diff to stdout: if stdout is a pipe
|
|
# that has already closed (someone running this by hand through `head`), the echo
|
|
# takes a SIGPIPE and the script dies — and without this ordering it would die
|
|
# holding the stale baseline and re-alert the identical change on the next run.
|
|
cp "$CURRENT" "$PREVIOUS"
|
|
echo "CHANGE DETECTED — alert sent"
|
|
echo "$CHANGED"
|
|
else
|
|
# Keep the old baseline so the very next run re-detects and re-reports this change,
|
|
# rather than losing it forever to a transient network failure.
|
|
echo "CHANGE DETECTED — alert FAILED to send; keeping previous baseline to retry next run" >&2
|
|
echo "$CHANGED"
|
|
fi
|
|
else
|
|
echo "no change"
|
|
cp "$CURRENT" "$PREVIOUS"
|
|
fi
|