From 2b7bf3b5f7a91620c20b680a24a3bb76bae865a7 Mon Sep 17 00:00:00 2001 From: rafapolo Date: Sat, 5 Sep 2026 18:04:22 +0200 Subject: [PATCH] fix: catch German descriptions by umlaut, not just DE word count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The >=6 DE_CONTENT_RE word threshold needs a reasonably long snippet to trip. An Expleo (Bern) posting slipped through because a concurrent fetch under load returned a truncated description with too few matches, even though the full text (72 matches) was entirely German. A bare umlaut or eszett is a near-certain German signal and survives truncation — one German sentence almost always carries one. Zurich/ Zurcher is excluded as the one umlaut word that shows up constantly in genuinely English-language Swiss postings, as a place name. Applied in isViableContent, isEnglishContent, isViableUrl and the fetch-other.ts sources, so every description path gets the same check. --- fetch-other.ts | 3 ++- fetch.ts | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/fetch-other.ts b/fetch-other.ts index d040807..cd87ccc 100644 --- a/fetch-other.ts +++ b/fetch-other.ts @@ -1,4 +1,4 @@ -import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman } from "./fetch.ts"; +import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts"; import type { Job } from "./fetch.ts"; const HEADERS = { @@ -60,6 +60,7 @@ const DE_FR_WORDS_RE = function isViableDescription(description?: string): { ok: boolean; reason?: string } { if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " "); + if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" }; if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" }; if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" }; return { ok: true }; diff --git a/fetch.ts b/fetch.ts index 427c88e..91e0171 100644 --- a/fetch.ts +++ b/fetch.ts @@ -166,6 +166,20 @@ export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devseco const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g; const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g; +// A bare umlaut/ß is a near-certain German signal, and unlike the ≥6 word-count +// threshold above it, it can catch even a short or truncated snippet — one +// German sentence almost always has one ("für", "können", "möchten"...). Added +// 2026-09-05 after an Expleo (Bern) posting slipped through: a concurrent fetch +// under load returned a version of the description with too few DE_CONTENT_RE +// word matches to trip ≥6, even though the full text (72 matches) was entirely +// German. "Zürich"/"Zürcher" is excluded — the one umlaut word that legitimately +// shows up constantly in genuinely English-language Swiss postings, just as a +// place name. +const UMLAUT_RE = /[äöüÄÖÜß]/; +export function hasNonEnglishUmlaut(text: string): boolean { + return UMLAUT_RE.test(text.replace(/zürich|zürcher/gi, "")); +} + // --- German-requirement detection ----------------------------------------- // // Distinct from isEnglishContent: that drops postings *written* in German. @@ -279,6 +293,7 @@ export async function fetchDescription(id: string): Promise { export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> { const text = await fetchDescription(id); if (text === null) return { ok: true }; // keep when we genuinely can't read it + if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" }; const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" }; @@ -300,11 +315,13 @@ export async function isEnglishContent(id: string): Promise { if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry // Last resort: scan the whole stripped page (still catches DE/FR body text) const text = html.replace(/<[^>]+>/g, " "); + if (hasNonEnglishUmlaut(text)) return false; const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; return de < 6 && fr < 6; } const text = m[1].replace(/<[^>]+>/g, " "); + if (hasNonEnglishUmlaut(text)) return false; const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; return de < 6 && fr < 6; @@ -326,6 +343,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: const res = await fetch(url, { headers: HEADERS }); if (!res.ok) return { ok: true }; const text = (await res.text()).replace(/<[^>]+>/g, " "); + if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" }; const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };