rodado/linkme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: catch German descriptions by umlaut, not just DE word count

Browse Source 
The >=6 DE_CONTENT_RE word threshold needs a reasonably long snippet to
trip. An Expleo (Bern) posting slipped through because a concurrent fetch
under load returned a truncated description with too few matches, even
though the full text (72 matches) was entirely German.

A bare umlaut or eszett is a near-certain German signal and survives
truncation — one German sentence almost always carries one. Zurich/
Zurcher is excluded as the one umlaut word that shows up constantly in
genuinely English-language Swiss postings, as a place name.

Applied in isViableContent, isEnglishContent, isViableUrl and the
fetch-other.ts sources, so every description path gets the same check.
This commit is contained in:
rodado
2026-09-05 18:04:22 +02:00
parent afddbe496d
commit 2b7bf3b5f7
2 changed files with 20 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fetch.ts
+18
View File
@@ -166,6 +166,20 @@ export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devseco
const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;
// A bare umlaut/ß is a near-certain German signal, and unlike the ≥6 word-count
// threshold above it, it can catch even a short or truncated snippet — one
// German sentence almost always has one ("für", "können", "möchten"...). Added
// 2026-09-05 after an Expleo (Bern) posting slipped through: a concurrent fetch
// under load returned a version of the description with too few DE_CONTENT_RE
// word matches to trip ≥6, even though the full text (72 matches) was entirely
// German. "Zürich"/"Zürcher" is excluded — the one umlaut word that legitimately
// shows up constantly in genuinely English-language Swiss postings, just as a
// place name.
const UMLAUT_RE = /[äöüÄÖÜß]/;
export function hasNonEnglishUmlaut(text: string): boolean {
return UMLAUT_RE.test(text.replace(/zürich|zürcher/gi, ""));
}
// --- German-requirement detection -----------------------------------------
//
// Distinct from isEnglishContent: that drops postings *written* in German.
@@ -279,6 +293,7 @@ export async function fetchDescription(id: string): Promise<string | null> {
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
const text = await fetchDescription(id);
if (text === null) return { ok: true }; // keep when we genuinely can't read it
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
const de = (text.match(DE_CONTENT_RE) ?? []).length;
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
@@ -300,11 +315,13 @@ export async function isEnglishContent(id: string): Promise<boolean> {
if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
// Last resort: scan the whole stripped page (still catches DE/FR body text)
const text = html.replace(/<[^>]+>/g, " ");
if (hasNonEnglishUmlaut(text)) return false;
const de = (text.match(DE_CONTENT_RE) ?? []).length;
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
return de < 6 && fr < 6;
}
const text = m[1].replace(/<[^>]+>/g, " ");
if (hasNonEnglishUmlaut(text)) return false;
const de = (text.match(DE_CONTENT_RE) ?? []).length;
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
return de < 6 && fr < 6;
@@ -326,6 +343,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?:
const res = await fetch(url, { headers: HEADERS });
if (!res.ok) return { ok: true };
const text = (await res.text()).replace(/<[^>]+>/g, " ");
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
const de = (text.match(DE_CONTENT_RE) ?? []).length;
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };