feat: add data-jobs.ch and arbeitnow.ch sources

fetch-other.ts

@@ -221,6 +221,60 @@ export async function fetchItJobsCh(): Promise<Job[]> {
   return allJobs;
 }
 
+// arbeitnow.ch — Swiss instance of arbeitnow, which pulls postings straight from
+// company ATS feeds (Personio, Greenhouse, …). Free public JSON API, no auth:
+// /api/job-board-api?page=n, 100 per page, newest first, `links.next` null on
+// the last page. It's a rolling ~7-day window (measured 2026-11-03: 195 jobs,
+// oldest 7 days old), so 2 requests cover the whole board. Each item carries
+// the full HTML description, so the DE/FR + requiresGerman check runs here.
+// pilotjob.ch was evaluated first and is NOT a source in its own right: its
+// listing only offers `src=arbeitnow|jobroom`, i.e. it re-serves this feed plus
+// job-room.ch, which we already sweep.
+export async function fetchArbeitnow(): Promise<Job[]> {
+  const allJobs: Job[] = [];
+  const seen = new Set<string>();
+
+  for (let page = 1; page <= 20; page++) {
+    if (page > 1) await Bun.sleep(500);
+    process.stdout.write(`  page ${page} … `);
+    const res = await fetch(`https://www.arbeitnow.ch/api/job-board-api?page=${page}`, {
+      headers: { ...PLAIN_HEADERS, Accept: "application/json" },
+    });
+    if (!res.ok) { console.log(`end (${res.status})`); break; }
+    const data = (await res.json()) as {
+      data: Array<{ slug: string; company_name: string; title: string; description?: string; remote: boolean; url: string; location: string; created_at: number }>;
+      links: { next: string | null };
+    };
+
+    let kept = 0;
+    let langDropped = 0;
+    for (const item of data.data) {
+      const title = decodeHtml(item.title);
+      if (!applyFilters(title) || MS_STACK_RE.test(title) || BIO_DOMAIN_RE.test(title)) continue;
+      const id = `an-${item.slug}`;
+      if (seen.has(id)) continue;
+      seen.add(id);
+      if (!isViableDescription(item.description).ok) { langDropped++; continue; }
+      const loc = decodeHtml(item.location ?? "");
+      allJobs.push({
+        id,
+        title,
+        company: decodeHtml(item.company_name ?? ""),
+        location: /^remote/i.test(loc) || (item.remote && !loc) ? "Remote" : loc || "Switzerland",
+        postedAt: item.created_at ? new Date(item.created_at * 1000).toISOString().slice(0, 10) : "",
+        link: item.url,
+        search: "arbeitnow.ch",
+        source: "arbeitnow.ch",
+        contentChecked: true,
+      });
+      kept++;
+    }
+    console.log(`${data.data.length} items, ${kept} kept, ${langDropped} non-English`);
+    if (!data.links?.next || data.data.length === 0) break;
+  }
+  return allJobs;
+}
+
 // swisscom.com — Swisscom's own Workday career site. Small board (~47 openings,
 // Switzerland-wide), so no keyword sweep — pull the full listing and filter
 // locally, like itjobs.ch. The listing page itself is a bare JS shell (no jobs
@@ -1054,6 +1108,12 @@ export async function fetchStellenBasel(): Promise<Job[]> {
 const DATADOME_CACHE = "results/.datadome.json";
 const CHROME_BIN = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
 const BASELJOBS_ORIGIN = "https://basel-jobs.ch";
+// Sister board on the same jobchannel network: same markup, same DataDome
+// customer (one cookie, rotated or not, clears both hosts), same `/job/<slug>/<id>`
+// path for a posting listed on both. Its inventory is not a subset of
+// basel-jobs.ch, though — data-only postings 404 on the basel-jobs.ch host.
+const DATAJOBS_ORIGIN = "https://data-jobs.ch";
+const DATAJOBS_PROVINCES = "BS,BL,ZH,BE,LU";
 
 interface DataDomeSession {
   cookie: string;
@@ -1258,14 +1318,16 @@ function parseBaselJobsCards(
   html: string,
   seen: Set<string>,
   out: Job[],
-  today: string
+  today: string,
+  host = "basel-jobs.ch",
+  knownTails = new Map<string, string>()
 ): { items: number; added: number } {
   const items = [...html.matchAll(/<li class="item"(.*?)<\/li>/gs)].map((m) => m[1]);
   let added = 0;
 
   for (const card of items) {
     const jid = /data-gtm-jid="(\d+)"/.exec(card)?.[1];
-    const linkM = /href="(https:\/\/basel-jobs\.ch\/job\/[^"]+)"/.exec(card);
+    const linkM = /href="(https:\/\/(?:basel|data)-jobs\.ch\/job\/[^"]+)"/.exec(card);
     const titleM = /<span class="jobtitle">(.*?)<\/span>/s.exec(card);
     if (!jid || !linkM || !titleM) continue;
 
@@ -1276,6 +1338,14 @@ function parseBaselJobsCards(
     seen.add(id);
     added++;
 
+    // Already in the CSV, under either host (a posting carried by both boards
+    // shares its `/job/<slug>/<id>` tail, which index.ts's link dedupe can't
+    // see). Skipping here also spares the detail fetch below: re-checking every
+    // known posting on every run is what got the cookie blocked once
+    // data-jobs.ch's five cantons were added (measured 2026-10-22).
+    const tail = linkM[1].replace(/^https:\/\/[^/]+/, "");
+    if (knownTails.has(tail)) continue;
+
     const title = decodeHtml(titleM[1]);
     if (!applyFilters(title)) continue;
 
@@ -1300,8 +1370,8 @@ function parseBaselJobsCards(
       // best available, same as any board that hides posting age.
       postedAt: today,
       link: linkM[1],
-      search: "basel-jobs.ch",
-      source: "basel-jobs.ch",
+      search: host,
+      source: host,
     });
   }
 
@@ -1463,11 +1533,24 @@ export async function fetchNovartisBasel(): Promise<Job[]> {
   return jobs;
 }
 
+/** `/job/<slug>/<id>` → stored link, for every basel-jobs.ch/data-jobs.ch row in the CSV. */
+async function knownJobchannelTails(): Promise<Map<string, string>> {
+  const tails = new Map<string, string>();
+  try {
+    const text = await Bun.file("results/jobs.csv").text();
+    for (const link of text.match(/https:\/\/(?:basel|data)-jobs\.ch\/job\/[^\s",]+/g) ?? []) {
+      tails.set(link.replace(/^https:\/\/[^/]+/, ""), link);
+    }
+  } catch { /* no CSV yet */ }
+  return tails;
+}
+
 export async function fetchBaselJobs(): Promise<Job[]> {
   const allJobs: Job[] = [];
   const seen = new Set<string>();
   const state = { session: await loadDataDomeSession(), minted: false };
   const today = new Date().toISOString().slice(0, 10);
+  const knownTails = await knownJobchannelTails();
 
   // Pass 1 — the board's IT category, ~300 postings over 3 pages.
   for (let page = 1; page <= 10; page++) {
@@ -1523,11 +1606,40 @@ export async function fetchBaselJobs(): Promise<Job[]> {
       const html = await baselJobsFetch(`${BASELJOBS_ORIGIN}/search?${qs}`, state);
       if (html === null) { console.log("unavailable"); break; }
 
-      const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
+      const { items, added } = parseBaselJobsCards(html, seen, allJobs, today, "basel-jobs.ch", knownTails);
       console.log(`${items} items, ${added} new`);
     }
   }
 
+  // Pass 3 — data-jobs.ch, the network's data/analytics board, filtered to
+  // the cantons Rafael would work in. Runs after the basel-jobs.ch passes and
+  // shares `seen` (vacancy ids are network-wide), so a posting on both boards
+  // keeps its basel-jobs.ch link. "N jobs found" is the board's own listings
+  // plus the "further jobs from the web search" behind `websearch=1`. The own
+  // listings paginate (`page=n`, 100 per page) and past the end still serve a
+  // few sponsored cards, so "no new ids" ends the walk; the web search ignores
+  // `page` and returns everything at once. Measured 2026-10-22 for
+  // BS,BL,ZH,BE,LU: 166 own + 98 web search = 264 unique; for BS,BL alone only
+  // 7 of 24 own listings were already in the CSV via basel-jobs.ch.
+  const dataJobsSearch = `${DATAJOBS_ORIGIN}/search?provinces=${DATAJOBS_PROVINCES}&lang=en-ch`;
+  for (let page = 1; page <= 10; page++) {
+    await Bun.sleep(400);
+    process.stdout.write(`  data-jobs.ch page ${page} … `);
+    const html = await baselJobsFetch(`${dataJobsSearch}&page=${page}`, state);
+    if (html === null) { console.log("unavailable"); break; }
+    const { items, added } = parseBaselJobsCards(html, seen, allJobs, today, "data-jobs.ch", knownTails);
+    console.log(`${items} items, ${added} new`);
+    if (items === 0 || added === 0) break;
+  }
+  await Bun.sleep(400);
+  process.stdout.write(`  data-jobs.ch websearch … `);
+  const webHtml = await baselJobsFetch(`${dataJobsSearch}&page=1&websearch=1`, state);
+  if (webHtml === null) console.log("unavailable");
+  else {
+    const { items, added } = parseBaselJobsCards(webHtml, seen, allJobs, today, "data-jobs.ch", knownTails);
+    console.log(`${items} items, ${added} new`);
+  }
+
   // Detail pages carry the full description inline, so the DE/FR and
   // German-requirement checks run here instead of costing index.ts a fetch that
   // would hit DataDome without the cookie and be dropped as unreachable.
@@ -1536,7 +1648,10 @@ export async function fetchBaselJobs(): Promise<Job[]> {
     await Bun.sleep(300);
     process.stdout.write(`  checking ${job.title} … `);
     const page = await baselJobsFetch(job.link, state);
-    if (page === null) { console.log("unreachable, keeping"); viable.push(job); continue; }
+    // Not kept: a blocked cookie fails every remaining detail page at once, and
+    // keeping those would land them in the CSV unchecked for good. Left out,
+    // they're simply retried on the next run.
+    if (page === null) { console.log("unreachable, skipping"); continue; }
 
     // Only the posting's own text, never the page. basel-jobs.ch is a German
     // site, so its nav/footer/category chrome alone clears the ≥ 6 DE/FR word