From e3368ce161b588b7fb7eab8ba00ae9781f018f86 Mon Sep 17 00:00:00 2001
From: rafapolo <rafael@extrapolo.com>
Date: Sat, 5 Sep 2026 18:04:36 +0200
Subject: [PATCH] fix: stop title filters from truncating a LinkedIn search
 early
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fetchJobs ended pagination on `newOnPage === 0`, but newOnPage counts
jobs that survived the title filters. A page whose 25 cards all happened
to be German/Java/junior/non-software read as end-of-results and killed
the whole search, discarding every later page. The end condition is now
new raw data-entity-urn card ids, pre-filter, so what gets filtered can
no longer decide where the crawl stops.

Also add a name-only "Basel (broad)" search alongside the three geoId
ones. LinkedIn's guest search serves a different slice per query shape:
the Basel-city geoId query exhausts at 76 unique cards and never returns
some genuinely Basel-Stadt postings a name-resolved query does. PAS01's
"Full-Stack Software Engineer (60-80%)" in Basel (4461764782) was
invisible to all six Basel searches and reached the digest only because
the Bern query — also name-only — happened to return it. Union across
the Basel searches: 126 unique.
---
 config.ts | 16 ++++++++++++++++
 fetch.ts  | 18 ++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/config.ts b/config.ts
index fb02fd3..8572423 100644
--- a/config.ts
+++ b/config.ts
@@ -121,6 +121,22 @@ export const searches: SearchConfig[] = [
     workType: ["onsite", "hybrid"],
     timePosted: "month",
   },
+  {
+    // Name-only, no geoId — deliberately alongside the three geoId searches
+    // above, not instead of them. LinkedIn's guest search serves a *different*
+    // slice per query shape: measured 2026-09-05, the Basel-city geoId query
+    // exhausts at 76 unique cards and never returns some genuinely Basel-Stadt
+    // postings that a name-resolved query does. The PAS01 "Full-Stack Software
+    // Engineer (60-80%)" in Basel (4461764782) was invisible to all six Basel
+    // searches and reached the digest only because the *Bern* query — also
+    // name-only — happened to return it. Overlap with the geoId slice is high
+    // but not total, so this is cheap insurance against exactly that miss.
+    label: "Basel (broad)",
+    keywords: KEYWORDS,
+    location: "Basel, Switzerland",
+    workType: ["onsite", "hybrid"],
+    timePosted: "month",
+  },
   {
     label: "Bern",
     keywords: KEYWORDS,
diff --git a/fetch.ts b/fetch.ts
index 91e0171..5fd63f9 100644
--- a/fetch.ts
+++ b/fetch.ts
@@ -431,6 +431,7 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
   const baseUrl = buildUrl(config);
   const allJobs: Job[] = [];
   const seen = new Set<string>();
+  const seenCards = new Set<string>(); // every card id served, pre-filter — drives the end condition
 
   for (let page = 0; page < MAX_PAGES; page++) {
     if (page > 0) await Bun.sleep(400);
@@ -452,6 +453,19 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
     }
 
     const html = await res.text();
+
+    // End-of-results is decided on the raw card ids LinkedIn served, never on
+    // how many survived the title filters. A page whose cards all happen to be
+    // German/Java/junior/non-software read as "no new jobs" and truncated the
+    // whole search, discarding every later page — pages that do carry keepers.
+    let newCards = 0;
+    for (const m of html.matchAll(/data-entity-urn="urn:li:jobPosting:(\d+)"/g)) {
+      if (!seenCards.has(m[1])) {
+        seenCards.add(m[1]);
+        newCards++;
+      }
+    }
+
     const jobs = parseJobs(html, config.label)
       .filter((j) => !NON_ENGLISH_RE.test(j.title))
       .filter((j) => !JUNIOR_TITLE_RE.test(j.title))
@@ -469,8 +483,8 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
       }
     }
 
-    console.log(`${jobs.length} jobs, ${newOnPage} new`);
-    if (newOnPage === 0) break;
+    console.log(`${jobs.length} jobs, ${newOnPage} new (${newCards} new cards)`);
+    if (newCards === 0) break;
   }
 
   return allJobs;