diff --git a/fetch-other.ts b/fetch-other.ts index cf1ed2f..8b68ec6 100644 --- a/fetch-other.ts +++ b/fetch-other.ts @@ -291,6 +291,16 @@ function stripTracking(url: string): string { export async function fetchJobRoom(): Promise { const ads = new Map(); + // The search API returns a short keyword-anchored teaser per hit, not the + // full posting — the same ad re-appears across keyword sweeps with a + // different teaser each time, and `ads.set()` above only keeps the latest. + // A genuinely German ad (Consult & Pepper's recurring "Senior Fullstack + // Software Engineer" — full text: "In Deutsch bist du verhandlungssicher") + // slipped through repeatedly because whichever keyword ran last happened to + // anchor on an English-reading fragment of it. Collecting every variant and + // checking them together means one clean-looking teaser can no longer + // override another teaser that actually shows the German. + const descVariants = new Map>(); for (const keyword of JOBROOM_KEYWORDS) { process.stdout.write(` ${keyword} … `); @@ -313,7 +323,11 @@ export async function fetchJobRoom(): Promise { const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>; for (const h of hits) { - if (h.jobAdvertisement?.id) ads.set(h.jobAdvertisement.id, h.jobAdvertisement); + const id = h.jobAdvertisement?.id; + if (!id) continue; + ads.set(id, h.jobAdvertisement!); + const d = h.jobAdvertisement!.jobContent?.jobDescriptions?.[0]?.description; + if (d) (descVariants.get(id) ?? descVariants.set(id, new Set()).get(id)!).add(d); } total += hits.length; if (hits.length < 100) break; @@ -341,8 +355,11 @@ export async function fetchJobRoom(): Promise { // A handful of ads ship without a description. isViableDescription passes // those through by design, so leave contentChecked unset and let index.ts - // fall back to fetching the page. - const body = desc.description ?? ""; + // fall back to fetching the page. When we do have variants, join every + // one seen across keyword sweeps rather than just the last teaser stored + // on `ad` (see comment above) — more text only adds detection signal, it + // never removes any. + const body = [...(descVariants.get(ad.id) ?? [])].join(" \n ") || desc.description || ""; const hasBody = body.trim().length > 40; if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; } diff --git a/fetch.ts b/fetch.ts index 7fc6338..f74a7e4 100644 --- a/fetch.ts +++ b/fetch.ts @@ -295,10 +295,21 @@ export async function fetchDescription(id: string): Promise { return null; } -/** English-language AND not demanding German above B1. */ -export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> { +/** + * English-language AND not demanding German above B1. + * + * `retry: true` means the description couldn't be read at all (rate-limited, + * stub page) — index.ts leaves the job out of the CSV entirely rather than + * defaulting to "keep", so it's picked up fresh and re-checked next run + * instead of ever being judged sight-unseen. The old default-to-keep here is + * what let a fully-German HRM Systems/Consult & Pepper posting ("In Deutsch + * bist du verhandlungssicher") through on 2026-09-10 — same failure mode as + * the CLEEVEN case above, just on the "keep" side of isViableContent instead + * of isEnglishContent. + */ +export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> { const text = await fetchDescription(id); - if (text === null) return { ok: true }; // keep when we genuinely can't read it + if (text === null) return { ok: false, reason: "unreadable — retrying next run", retry: true }; if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" }; const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; @@ -344,11 +355,13 @@ export async function isEnglishUrl(url: string): Promise { } /** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */ -export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> { +export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> { try { await Bun.sleep(350); const res = await fetch(url, { headers: HEADERS }); - if (!res.ok) return { ok: true }; + // Same reasoning as isViableContent: an unreadable page shouldn't default + // to "keep" — defer it and let next run's fresh fetch actually judge it. + if (!res.ok) return { ok: false, reason: "unreadable — retrying next run", retry: true }; const text = (await res.text()).replace(/<[^>]+>/g, " "); if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" }; const de = (text.match(DE_CONTENT_RE) ?? []).length; @@ -357,7 +370,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" }; return { ok: true }; } catch { - return { ok: true }; + return { ok: false, reason: "unreadable — retrying next run", retry: true }; } }