fix: stop letting unreadable or truncated descriptions slip past the German filter
isViableContent/isViableUrl defaulted to keep a job when its description
couldn't be fetched at all -- that's what let a fully-German HRM
Systems/Consult & Pepper posting ("In Deutsch bist du verhandlungssicher")
through on 2026-09-10. They now report retry:true instead, and index.ts
leaves the job out of the CSV so a fresh fetch judges it next run rather
than defaulting to keep.
job-room.ch's search API also returns a different keyword-anchored teaser
per hit for the same ad, and only the last one was kept -- a genuinely
German posting could slip through whenever the last keyword's teaser
happened to read as English. Every teaser variant is now collected and
checked together.
This commit is contained in:
@@ -295,10 +295,21 @@ export async function fetchDescription(id: string): Promise<string | null> {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** English-language AND not demanding German above B1. */
|
||||
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
|
||||
/**
|
||||
* English-language AND not demanding German above B1.
|
||||
*
|
||||
* `retry: true` means the description couldn't be read at all (rate-limited,
|
||||
* stub page) — index.ts leaves the job out of the CSV entirely rather than
|
||||
* defaulting to "keep", so it's picked up fresh and re-checked next run
|
||||
* instead of ever being judged sight-unseen. The old default-to-keep here is
|
||||
* what let a fully-German HRM Systems/Consult & Pepper posting ("In Deutsch
|
||||
* bist du verhandlungssicher") through on 2026-09-10 — same failure mode as
|
||||
* the CLEEVEN case above, just on the "keep" side of isViableContent instead
|
||||
* of isEnglishContent.
|
||||
*/
|
||||
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
|
||||
const text = await fetchDescription(id);
|
||||
if (text === null) return { ok: true }; // keep when we genuinely can't read it
|
||||
if (text === null) return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||||
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||||
@@ -344,11 +355,13 @@ export async function isEnglishUrl(url: string): Promise<boolean> {
|
||||
}
|
||||
|
||||
/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
|
||||
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> {
|
||||
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
|
||||
try {
|
||||
await Bun.sleep(350);
|
||||
const res = await fetch(url, { headers: HEADERS });
|
||||
if (!res.ok) return { ok: true };
|
||||
// Same reasoning as isViableContent: an unreadable page shouldn't default
|
||||
// to "keep" — defer it and let next run's fresh fetch actually judge it.
|
||||
if (!res.ok) return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||||
const text = (await res.text()).replace(/<[^>]+>/g, " ");
|
||||
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||
@@ -357,7 +370,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?:
|
||||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||||
return { ok: true };
|
||||
} catch {
|
||||
return { ok: true };
|
||||
return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user