fix: stop letting unreadable or truncated descriptions slip past the German filter

isViableContent/isViableUrl defaulted to keep a job when its description
couldn't be fetched at all -- that's what let a fully-German HRM
Systems/Consult & Pepper posting ("In Deutsch bist du verhandlungssicher")
through on 2026-09-10. They now report retry:true instead, and index.ts
leaves the job out of the CSV so a fresh fetch judges it next run rather
than defaulting to keep.

job-room.ch's search API also returns a different keyword-anchored teaser
per hit for the same ad, and only the last one was kept -- a genuinely
German posting could slip through whenever the last keyword's teaser
happened to read as English. Every teaser variant is now collected and
checked together.
This commit is contained in:
2026-09-14 12:12:43 +02:00
parent c7b86e06f4
commit 0249800278
2 changed files with 39 additions and 9 deletions
+19 -6
View File
@@ -295,10 +295,21 @@ export async function fetchDescription(id: string): Promise<string | null> {
return null;
}
/** English-language AND not demanding German above B1. */
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
/**
* English-language AND not demanding German above B1.
*
* `retry: true` means the description couldn't be read at all (rate-limited,
* stub page) — index.ts leaves the job out of the CSV entirely rather than
* defaulting to "keep", so it's picked up fresh and re-checked next run
* instead of ever being judged sight-unseen. The old default-to-keep here is
* what let a fully-German HRM Systems/Consult & Pepper posting ("In Deutsch
* bist du verhandlungssicher") through on 2026-09-10 — same failure mode as
* the CLEEVEN case above, just on the "keep" side of isViableContent instead
* of isEnglishContent.
*/
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
const text = await fetchDescription(id);
if (text === null) return { ok: true }; // keep when we genuinely can't read it
if (text === null) return { ok: false, reason: "unreadable — retrying next run", retry: true };
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
const de = (text.match(DE_CONTENT_RE) ?? []).length;
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
@@ -344,11 +355,13 @@ export async function isEnglishUrl(url: string): Promise<boolean> {
}
/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> {
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
try {
await Bun.sleep(350);
const res = await fetch(url, { headers: HEADERS });
if (!res.ok) return { ok: true };
// Same reasoning as isViableContent: an unreadable page shouldn't default
// to "keep" — defer it and let next run's fresh fetch actually judge it.
if (!res.ok) return { ok: false, reason: "unreadable — retrying next run", retry: true };
const text = (await res.text()).replace(/<[^>]+>/g, " ");
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
const de = (text.match(DE_CONTENT_RE) ?? []).length;
@@ -357,7 +370,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?:
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
return { ok: true };
} catch {
return { ok: true };
return { ok: false, reason: "unreadable — retrying next run", retry: true };
}
}