fix: catch German descriptions by umlaut, not just DE word count
The >=6 DE_CONTENT_RE word threshold needs a reasonably long snippet to trip. An Expleo (Bern) posting slipped through because a concurrent fetch under load returned a truncated description with too few matches, even though the full text (72 matches) was entirely German. A bare umlaut or eszett is a near-certain German signal and survives truncation — one German sentence almost always carries one. Zurich/ Zurcher is excluded as the one umlaut word that shows up constantly in genuinely English-language Swiss postings, as a place name. Applied in isViableContent, isEnglishContent, isViableUrl and the fetch-other.ts sources, so every description path gets the same check.
This commit is contained in:
+2
-1
@@ -1,4 +1,4 @@
|
|||||||
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman } from "./fetch.ts";
|
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
||||||
import type { Job } from "./fetch.ts";
|
import type { Job } from "./fetch.ts";
|
||||||
|
|
||||||
const HEADERS = {
|
const HEADERS = {
|
||||||
@@ -60,6 +60,7 @@ const DE_FR_WORDS_RE =
|
|||||||
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
|
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
|
||||||
if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
|
if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
|
||||||
const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
|
const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
|
||||||
|
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||||||
if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" };
|
if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" };
|
||||||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
|
|||||||
@@ -166,6 +166,20 @@ export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devseco
|
|||||||
const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
|
const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
|
||||||
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;
|
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;
|
||||||
|
|
||||||
|
// A bare umlaut/ß is a near-certain German signal, and unlike the ≥6 word-count
|
||||||
|
// threshold above it, it can catch even a short or truncated snippet — one
|
||||||
|
// German sentence almost always has one ("für", "können", "möchten"...). Added
|
||||||
|
// 2026-09-05 after an Expleo (Bern) posting slipped through: a concurrent fetch
|
||||||
|
// under load returned a version of the description with too few DE_CONTENT_RE
|
||||||
|
// word matches to trip ≥6, even though the full text (72 matches) was entirely
|
||||||
|
// German. "Zürich"/"Zürcher" is excluded — the one umlaut word that legitimately
|
||||||
|
// shows up constantly in genuinely English-language Swiss postings, just as a
|
||||||
|
// place name.
|
||||||
|
const UMLAUT_RE = /[äöüÄÖÜß]/;
|
||||||
|
export function hasNonEnglishUmlaut(text: string): boolean {
|
||||||
|
return UMLAUT_RE.test(text.replace(/zürich|zürcher/gi, ""));
|
||||||
|
}
|
||||||
|
|
||||||
// --- German-requirement detection -----------------------------------------
|
// --- German-requirement detection -----------------------------------------
|
||||||
//
|
//
|
||||||
// Distinct from isEnglishContent: that drops postings *written* in German.
|
// Distinct from isEnglishContent: that drops postings *written* in German.
|
||||||
@@ -279,6 +293,7 @@ export async function fetchDescription(id: string): Promise<string | null> {
|
|||||||
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
|
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
|
||||||
const text = await fetchDescription(id);
|
const text = await fetchDescription(id);
|
||||||
if (text === null) return { ok: true }; // keep when we genuinely can't read it
|
if (text === null) return { ok: true }; // keep when we genuinely can't read it
|
||||||
|
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||||||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||||||
@@ -300,11 +315,13 @@ export async function isEnglishContent(id: string): Promise<boolean> {
|
|||||||
if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
|
if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
|
||||||
// Last resort: scan the whole stripped page (still catches DE/FR body text)
|
// Last resort: scan the whole stripped page (still catches DE/FR body text)
|
||||||
const text = html.replace(/<[^>]+>/g, " ");
|
const text = html.replace(/<[^>]+>/g, " ");
|
||||||
|
if (hasNonEnglishUmlaut(text)) return false;
|
||||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||||||
return de < 6 && fr < 6;
|
return de < 6 && fr < 6;
|
||||||
}
|
}
|
||||||
const text = m[1].replace(/<[^>]+>/g, " ");
|
const text = m[1].replace(/<[^>]+>/g, " ");
|
||||||
|
if (hasNonEnglishUmlaut(text)) return false;
|
||||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||||||
return de < 6 && fr < 6;
|
return de < 6 && fr < 6;
|
||||||
@@ -326,6 +343,7 @@ export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?:
|
|||||||
const res = await fetch(url, { headers: HEADERS });
|
const res = await fetch(url, { headers: HEADERS });
|
||||||
if (!res.ok) return { ok: true };
|
if (!res.ok) return { ok: true };
|
||||||
const text = (await res.text()).replace(/<[^>]+>/g, " ");
|
const text = (await res.text()).replace(/<[^>]+>/g, " ");
|
||||||
|
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||||||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||||||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||||||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||||||
|
|||||||
Reference in New Issue
Block a user