chore: update search scripts, extra sources and post drafts

Adds the one-off scripts accumulated across recent sessions, removes temp scripts whose searches now live in config.ts as role-family sweeps, and updates fetch-other.ts plus the LinkedIn post drafts.
2026-08-27 09:03:05 +02:00
parent 3f5360f44c
commit b4957025b1
31 changed files with 2008 additions and 283 deletions
@@ -0,0 +1,145 @@
+#!/usr/bin/env bun
+// Ruby/Rails — GLOBAL remote sweep.
+// Most Rails roles are titled "Backend Engineer", so this runs both a
+// Ruby-title search and a description-level Ruby check over generic titles,
+// across every remote market that is realistic for an EU-citizen in Basel.
+import { NON_ENGLISH_RE, JUNIOR_TITLE_RE } from "../fetch.ts";
+
+const RUBY_KEYWORDS = [
+  "Ruby on Rails", "Rails Engineer", "Ruby Engineer",
+  "Rails Developer", "Ruby Developer", "Ruby", "Rails",
+].join(" OR ");
+
+const RUBY_RE = /\bruby\b|\brails\b|\bsidekiq\b|\bhotwire\b|\bactiverecord\b|\bactive record\b/i;
+const MS_RE = /\b(\.net|c#|sharepoint|dynamics|sccm|intune)\b/i;
+const AGENCY_RE = /\bjobgether\b|\ballps\b/i;
+
+const HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Accept-Language": "en-US,en;q=0.9",
+};
+
+// geoId 92000000 = "Worldwide" on LinkedIn
+const MARKETS: { label: string; geoId?: string; location?: string }[] = [
+  { label: "Worldwide", geoId: "92000000" },
+  { label: "European Union", location: "European Union" },
+  { label: "Portugal", location: "Portugal" },
+  { label: "Germany", location: "Germany" },
+  { label: "United Kingdom", location: "United Kingdom" },
+  { label: "Spain", location: "Spain" },
+  { label: "Netherlands", location: "Netherlands" },
+  { label: "Switzerland", location: "Switzerland" },
+  { label: "Brazil", location: "Brazil" },
+  { label: "United States", location: "United States" },
+];
+
+interface Job { id: string; title: string; company: string; location: string; link: string }
+
+function unescape(s: string) {
+  return s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
+    .replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/\s+/g, " ").trim();
+}
+
+function parseCards(html: string): Job[] {
+  const jobs: Job[] = [];
+  const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
+  const pos: { id: string; start: number }[] = [];
+  let m;
+  while ((m = idRe.exec(html)) !== null) pos.push({ id: m[1], start: m.index });
+
+  for (let i = 0; i < pos.length; i++) {
+    const { id, start } = pos[i];
+    const card = html.slice(start, Math.min(pos[i + 1]?.start ?? Infinity, start + 3000));
+    const titleM = card.match(/base-search-card__title[^>]*>\s*([\s\S]*?)\s*<\/h3>/);
+    if (!titleM) continue;
+    const companyM = card.match(/base-search-card__subtitle[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
+    const locM = card.match(/job-search-card__location[^>]*>\s*([^<]+)/);
+    jobs.push({
+      id,
+      title: unescape(titleM[1].replace(/<[^>]+>/g, "")),
+      company: companyM ? unescape(companyM[1].replace(/<[^>]+>/g, "")) : "",
+      location: locM ? unescape(locM[1]) : "",
+      link: `https://www.linkedin.com/jobs/view/${id}/`,
+    });
+  }
+  return jobs;
+}
+
+const bodyCache = new Map<string, string>();
+async function fetchBody(id: string): Promise<string> {
+  if (bodyCache.has(id)) return bodyCache.get(id)!;
+  let body = "";
+  try {
+    const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
+    const html = await res.text();
+    body = (html.match(/show-more-less-html__markup[^"]*"[^>]*>([\s\S]*?)<\/div>/)?.[1] ?? "")
+      .replace(/<[^>]+>/g, " ").replace(/\s+/g, " ");
+  } catch { /* leave empty */ }
+  bodyCache.set(id, body);
+  await Bun.sleep(350);
+  return body;
+}
+
+type Hit = Job & { market: string; via: string; snippet: string };
+const hits: Hit[] = [];
+const seen = new Set<string>();
+const checked = new Set<string>();
+
+for (const market of MARKETS) {
+  console.log(`\n[${market.label}]`);
+  for (let page = 0; page < 3; page++) {
+    const params = new URLSearchParams({
+      keywords: RUBY_KEYWORDS,
+      f_TPR: "r2592000",
+      f_WT: "2", // remote
+      start: String(page * 25),
+    });
+    if (market.geoId) params.set("geoId", market.geoId);
+    if (market.location) params.set("location", market.location);
+
+    let cards: Job[] = [];
+    try {
+      const res = await fetch(`https://www.linkedin.com/jobs/search/?${params}`, { headers: HEADERS });
+      if (!res.ok) { console.log(`  page ${page + 1}: HTTP ${res.status}`); break; }
+      cards = parseCards(await res.text());
+    } catch (e) { console.log(`  page ${page + 1}: ${e}`); break; }
+
+    if (!cards.length) break;
+    let fresh = 0;
+
+    for (const job of cards) {
+      if (seen.has(job.id) || checked.has(job.id)) continue;
+      if (NON_ENGLISH_RE.test(job.title) || JUNIOR_TITLE_RE.test(job.title)) continue;
+      if (MS_RE.test(job.title) || AGENCY_RE.test(job.company)) continue;
+      checked.add(job.id);
+      fresh++;
+
+      const titleRuby = RUBY_RE.test(job.title);
+      const body = await fetchBody(job.id);
+      const bodyRuby = RUBY_RE.test(body);
+      if (!titleRuby && !bodyRuby) continue;
+      if (MS_RE.test(body)) continue;
+
+      const idx = body.search(RUBY_RE);
+      seen.add(job.id);
+      hits.push({
+        ...job,
+        market: market.label,
+        via: titleRuby ? (bodyRuby ? "title+desc" : "title") : "desc",
+        snippet: idx >= 0 ? body.slice(Math.max(0, idx - 90), idx + 130).trim() : "",
+      });
+      console.log(`  HIT: ${job.title} | ${job.company} | ${job.location}`);
+    }
+    console.log(`  page ${page + 1}: ${cards.length} cards, ${fresh} new`);
+    if (fresh === 0) break;
+    await Bun.sleep(400);
+  }
+}
+
+console.log(`\n=== ${hits.length} RUBY/RAILS REMOTE HITS (global) ===\n`);
+for (const h of hits) {
+  console.log(`${h.title} | ${h.company} | ${h.location} | ${h.market} | via ${h.via}`);
+  console.log(`  ${h.link}`);
+  if (h.snippet) console.log(`  …${h.snippet}…`);
+  console.log();
+}