chore: update search scripts, extra sources and post drafts

Adds the one-off scripts accumulated across recent sessions, removes temp scripts whose searches now live in config.ts as role-family sweeps, and updates fetch-other.ts plus the LinkedIn post drafts.
2026-08-27 09:03:05 +02:00
parent 3f5360f44c
commit b4957025b1
31 changed files with 2008 additions and 283 deletions
@@ -0,0 +1,112 @@
+#!/bin/bash
+# Daily watch on sciCORE's open-positions page.
+# Sends a Telegram message ONLY when a new line mentioning "software" appears.
+# Other changes are logged silently so the snapshot stays current.
+# Credentials come from .env: TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID
+
+set -uo pipefail
+
+REPO="/Users/polux/Projetos/linkme"
+URL="${SCICORE_URL:-https://scicore.unibas.ch/about-scicore/open-positions/}"   # override for testing
+WANTED="software"   # only changes mentioning this word are sent
+STATE="$REPO/results/.scicore-watch"
+SNAP="$STATE/snapshot.txt"
+FAILS="$STATE/consecutive-failures"
+LOG="$STATE/watch.log"
+UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+
+mkdir -p "$STATE"
+
+# shellcheck disable=SC1091
+# accepts either TELEGRAM_BOT_TOKEN= or the shorter TELEGRAM_BOT=
+TELEGRAM_BOT_TOKEN=$(grep -E '^TELEGRAM_BOT(_TOKEN)?=' "$REPO/.env" 2>/dev/null | head -1 | cut -d= -f2-)
+TELEGRAM_CHAT_ID=$(grep -E '^TELEGRAM_CHAT_ID=' "$REPO/.env" 2>/dev/null | cut -d= -f2-)
+
+log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOG"; }
+
+notify() {
+  local text="$1"
+  if [ -z "$TELEGRAM_BOT_TOKEN" ] || [ -z "$TELEGRAM_CHAT_ID" ]; then
+    log "NOTIFY SKIPPED (no telegram creds in .env): $text"
+    return 1
+  fi
+  local code
+  code=$(curl -s -o "$STATE/last-telegram-response.json" -w '%{http_code}' \
+    "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+    --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
+    --data-urlencode "text=${text}" \
+    --data-urlencode "disable_web_page_preview=true")
+  log "telegram http $code"
+  [ "$code" = "200" ]
+}
+
+HTML=$(curl -s --max-time 30 -A "$UA" "$URL")
+if [ -z "$HTML" ]; then
+  n=$(( $(cat "$FAILS" 2>/dev/null || echo 0) + 1 ))
+  echo "$n" > "$FAILS"
+  log "FETCH FAILED (streak $n)"
+  [ "$n" -eq 3 ] && notify "⚠️ sciCORE watcher: 3 days in a row the page failed to load. Check the script.
+$URL"
+  exit 0
+fi
+echo 0 > "$FAILS"
+
+CURRENT=$(printf '%s' "$HTML" | python3 -c '
+import sys, re, html
+h = sys.stdin.read()
+m = re.search(r"<h1>Join the team!</h1>(.*?)<footer", h, re.S | re.I)
+body = m.group(1) if m else h
+body = re.sub(r"<(script|style).*?</\1>", "", body, flags=re.S | re.I)
+body = re.sub(r"<li[^>]*>", "\n- ", body, flags=re.I)
+body = re.sub(r"</(p|div|h[1-6]|ul|li|tr)>", "\n", body, flags=re.I)
+body = re.sub(r"<[^>]+>", " ", body)
+body = html.unescape(body)
+lines = [re.sub(r"\s+", " ", l).strip() for l in body.split("\n")]
+print("\n".join(l for l in lines if l and l not in {"-", " "}))
+')
+
+if [ -z "$CURRENT" ]; then
+  log "PARSE FAILED (empty extraction)"
+  exit 0
+fi
+
+if [ ! -f "$SNAP" ]; then
+  printf '%s\n' "$CURRENT" > "$SNAP"
+  log "BASELINE saved"
+  if printf '%s' "$CURRENT" | grep -qi "$WANTED"; then
+    notify "🚨 sciCORE listing mentions Software:
+
+$CURRENT
+
+$URL"
+  fi
+  exit 0
+fi
+
+if diff -q <(printf '%s\n' "$CURRENT") "$SNAP" >/dev/null; then
+  log "no change"
+  exit 0
+fi
+
+DIFF=$(diff "$SNAP" <(printf '%s\n' "$CURRENT") | sed -n 's/^> /+ /p;s/^< /- /p')
+printf '%s\n' "$CURRENT" > "$SNAP"
+log "CHANGED:
+$DIFF"
+
+# Only notify when a newly ADDED line contains the wanted word.
+# Everything else (sysadmin roles, wording tweaks, removals) is tracked silently.
+ADDED=$(printf '%s\n' "$DIFF" | grep '^+ ' || true)
+if ! printf '%s' "$ADDED" | grep -qi "$WANTED"; then
+  log "change ignored (no \"$WANTED\" in added lines)"
+  exit 0
+fi
+
+notify "🚨 sciCORE posted a Software role
+
+What changed:
+$DIFF
+
+Full listing now:
+$CURRENT
+
+$URL"