fix: stop title filters from truncating a LinkedIn search early
fetchJobs ended pagination on `newOnPage === 0`, but newOnPage counts jobs that survived the title filters. A page whose 25 cards all happened to be German/Java/junior/non-software read as end-of-results and killed the whole search, discarding every later page. The end condition is now new raw data-entity-urn card ids, pre-filter, so what gets filtered can no longer decide where the crawl stops. Also add a name-only "Basel (broad)" search alongside the three geoId ones. LinkedIn's guest search serves a different slice per query shape: the Basel-city geoId query exhausts at 76 unique cards and never returns some genuinely Basel-Stadt postings a name-resolved query does. PAS01's "Full-Stack Software Engineer (60-80%)" in Basel (4461764782) was invisible to all six Basel searches and reached the digest only because the Bern query — also name-only — happened to return it. Union across the Basel searches: 126 unique.
This commit is contained in:
@@ -121,6 +121,22 @@ export const searches: SearchConfig[] = [
|
|||||||
workType: ["onsite", "hybrid"],
|
workType: ["onsite", "hybrid"],
|
||||||
timePosted: "month",
|
timePosted: "month",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
// Name-only, no geoId — deliberately alongside the three geoId searches
|
||||||
|
// above, not instead of them. LinkedIn's guest search serves a *different*
|
||||||
|
// slice per query shape: measured 2026-09-05, the Basel-city geoId query
|
||||||
|
// exhausts at 76 unique cards and never returns some genuinely Basel-Stadt
|
||||||
|
// postings that a name-resolved query does. The PAS01 "Full-Stack Software
|
||||||
|
// Engineer (60-80%)" in Basel (4461764782) was invisible to all six Basel
|
||||||
|
// searches and reached the digest only because the *Bern* query — also
|
||||||
|
// name-only — happened to return it. Overlap with the geoId slice is high
|
||||||
|
// but not total, so this is cheap insurance against exactly that miss.
|
||||||
|
label: "Basel (broad)",
|
||||||
|
keywords: KEYWORDS,
|
||||||
|
location: "Basel, Switzerland",
|
||||||
|
workType: ["onsite", "hybrid"],
|
||||||
|
timePosted: "month",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
label: "Bern",
|
label: "Bern",
|
||||||
keywords: KEYWORDS,
|
keywords: KEYWORDS,
|
||||||
|
|||||||
@@ -431,6 +431,7 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
|
|||||||
const baseUrl = buildUrl(config);
|
const baseUrl = buildUrl(config);
|
||||||
const allJobs: Job[] = [];
|
const allJobs: Job[] = [];
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
|
const seenCards = new Set<string>(); // every card id served, pre-filter — drives the end condition
|
||||||
|
|
||||||
for (let page = 0; page < MAX_PAGES; page++) {
|
for (let page = 0; page < MAX_PAGES; page++) {
|
||||||
if (page > 0) await Bun.sleep(400);
|
if (page > 0) await Bun.sleep(400);
|
||||||
@@ -452,6 +453,19 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
|
|||||||
}
|
}
|
||||||
|
|
||||||
const html = await res.text();
|
const html = await res.text();
|
||||||
|
|
||||||
|
// End-of-results is decided on the raw card ids LinkedIn served, never on
|
||||||
|
// how many survived the title filters. A page whose cards all happen to be
|
||||||
|
// German/Java/junior/non-software read as "no new jobs" and truncated the
|
||||||
|
// whole search, discarding every later page — pages that do carry keepers.
|
||||||
|
let newCards = 0;
|
||||||
|
for (const m of html.matchAll(/data-entity-urn="urn:li:jobPosting:(\d+)"/g)) {
|
||||||
|
if (!seenCards.has(m[1])) {
|
||||||
|
seenCards.add(m[1]);
|
||||||
|
newCards++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const jobs = parseJobs(html, config.label)
|
const jobs = parseJobs(html, config.label)
|
||||||
.filter((j) => !NON_ENGLISH_RE.test(j.title))
|
.filter((j) => !NON_ENGLISH_RE.test(j.title))
|
||||||
.filter((j) => !JUNIOR_TITLE_RE.test(j.title))
|
.filter((j) => !JUNIOR_TITLE_RE.test(j.title))
|
||||||
@@ -469,8 +483,8 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`${jobs.length} jobs, ${newOnPage} new`);
|
console.log(`${jobs.length} jobs, ${newOnPage} new (${newCards} new cards)`);
|
||||||
if (newOnPage === 0) break;
|
if (newCards === 0) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
return allJobs;
|
return allJobs;
|
||||||
|
|||||||
Reference in New Issue
Block a user