feat: add role-family sweeps and domain/language filters

Broaden the search beyond engineer/developer titles and stop three classes of bad result from ever reaching the CSV. config.ts - PLATFORM_KEYWORDS (architect, SRE, infra, security, MLOps), RESEARCH_KEYWORDS (research/scientific software, HPC, data platform) and FIELD_KEYWORDS (forward-deployed, customer/sales, founding). Separate OR-strings, not folded into KEYWORDS: LinkedIn ranks a focused query far better than one 50-term disjunction. - sweep() generates one search per area per family. 7 searches -> 17. fetch.ts - BIO_DOMAIN_RE drops life-science roles (bioinformatics, biostats, GMP/MSAT, genomics, pathology, drug discovery). Filters on the role, never the employer: software jobs at Roche/Novartis/sciCORE stay. - requiresGerman() drops English postings demanding German above B1. Two rules, both from real false results: the nearest marker wins ("Infrastructure skills are a plus" sat 95 chars before "Good knowledge of German"), and an intervening language name breaks the binding ("Fluent in English; German and/or French advantageous"). - GERMAN_TITLE_RE catches "(German-speaking)" without a page fetch. - isViableContent/isViableUrl run the DE/FR and German-requirement checks on a single fetch; isEnglishContent/isEnglishUrl kept for recheck.ts and temp scripts. - SOFTWARE_TITLE_RE widened so the new families survive it. index.ts - both content-check paths use the combined check and report which filter dropped a job.
2026-08-27 09:02:50 +02:00
parent fb67c38edd
commit 3aa4365fb9
4 changed files with 327 additions and 24 deletions
@@ -9,6 +9,13 @@ export interface Job {
  link: string;
  search: string;
  source: string;
+  /**
+   * Set when the source already language-checked the real job description.
+   * `isEnglishUrl` counts DE/FR words across the whole page, so boards with
+   * German site chrome (itjobs.ch, stellen-basel.ch) fail it even when the
+   * posting itself is English — skip the re-check for those.
+   */
+  contentChecked?: boolean;
 }

 const WORK_TYPE_MAP: Record<WorkType, string> = {
@@ -62,6 +69,8 @@ export const NON_ENGLISH_RE = new RegExp(
    String.raw`\(w\/m[^)]*\)`,
    String.raw`\(d\/w[^)]*\)`,
    String.raw`\(mwd\)`,
+    // m/w without parens (e.g. "Developer m/w")
+    String.raw`\bm\/w\b`,
    // Swiss neutral (a) suffix: " (a)" before %, digits, or end
    String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
    // German/Swiss umlauts and ß
@@ -69,7 +78,9 @@ export const NON_ENGLISH_RE = new RegExp(
    // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
    String.raw`[a-zäöü]\/in\b`,
    // German-only words (case-insensitive handled by flag)
-    String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`,
+    String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale|Ingenieur|Spezialist|Perspektive|Stellvertretung|Leitung|Beratung|Entwicklung|Betreuung|Verwaltung)\b`,
+    // German prepositions/articles in title context (must appear as whole words)
+    String.raw`\bmit\b`,
    // French gender markers
    String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
    // French-only accented characters
@@ -92,25 +103,185 @@ export const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:s

 export const JAVA_TITLE_RE = /\bjava\b/i;

-export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;
+// Roles whose *domain* is biology / chemistry / the lab bench. Rafael has no
+// life-science background, so these need expertise he can't offer.
+//
+// This filters on the ROLE, never the employer: software, platform, data and
+// infrastructure jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope —
+// Basel pharma is a core target market. Only the science itself is excluded.
+export const BIO_DOMAIN_RE = new RegExp(
+  [
+    // bench / domain science job families
+    String.raw`\b(bio)?statistician\b`,
+    String.raw`\bbioinformatic\w*`,
+    String.raw`\bcomputational\s+(?:biolog\w*|chemist\w*)`,
+    String.raw`\b(?:cheminformatic\w*|chemometric\w*)`,
+    String.raw`\b(?:micro)?biolog(?:y|ist)\b`,
+    String.raw`\b(?:bio)?chemist(?:ry|s)?\b`,
+    String.raw`\bpharmacolog\w*|\btoxicolog\w*|\bimmunolog\w*|\bvirolog\w*|\bepidemiolog\w*`,
+    String.raw`\bgenomic\w*|\bproteomic\w*|\bmetabolomic\w*|\bmolecular\s+(?:biolog\w*|patholog\w*)`,
+    // manufacturing / QC / regulated-lab functions
+    String.raw`\bGMP\b|\bGxP\b|\bMSAT\b|\bQC\s+analyst\b`,
+    String.raw`\b(?:bio)?process\s+(?:development|engineer)|\bupstream\b|\bdownstream\s+process`,
+    String.raw`\bassay\b|\bbioassay\b|\bwet.?lab\b|\blaboratory\s+(?:technician|assistant|manager)`,
+    // clinical / trial work
+    String.raw`\bclinical\s+(?:trial|research|data\s+manager|operations)`,
+    String.raw`\bbiomarker\b|\bpharmacovigilance\b|\bregulatory\s+affairs\b`,
+    // life-science domains that make any title (Scientist, Engineer, Analyst) a domain role
+    String.raw`\bbiologic\w*|\bpatholog\w*|\bhistolog\w*`,
+    String.raw`\bdrug\s+(?:discovery|design|development)\b`,
+    String.raw`\breal.world\s+(?:data|evidence)\b`,
+  ].join("|"),
+  "i"
+);
+
+export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devsecops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|infrastructure|mlops|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|scientific\s*(?:software|comput\w*|programmer)|research\s*(?:software|engineer)|hpc|forward\s*deployed|field\s*engineer|customer\s*engineer|sales\s*engineer|pre.?sales|integration\s*engineer|api\s*engineer|founding\s*engineer|product\s*engineer|automation\s*engineer|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;

 const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
 const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;

+// --- German-requirement detection -----------------------------------------
+//
+// Distinct from isEnglishContent: that drops postings *written* in German.
+// This drops postings written in English that *require* German above B1 —
+// e.g. Helvetia's "Good knowledge of German and English", which reads as a
+// clean English posting and sails through every other filter.
+//
+// Rafael is German B1 (TELC). B1 or "basic"/"a plus" is fine; B2+ is not.
+
+/** Any mention of the language, in either language. */
+const GERMAN_MENTION_RE = /\b(german|deutsch\w*|allemand)\b/gi;
+
+/** Within a mention's window: signals the requirement is optional. */
+const GERMAN_OPTIONAL_RE =
+  /\b(?:a\s+plus|an?\s+advantage|advantageous|beneficial|nice[\s-]to[\s-]have|desirable|preferred|welcome|bonus|not\s+required|no\s+german|optional|willing(?:ness)?\s+to\s+learn|basic|beginner|elementary|grundkenntnisse|von\s+vorteil|wünschenswert|A1|A2|B1)\b/i;
+
+/** Within a mention's window: signals it is genuinely required at B2+. */
+const GERMAN_REQUIRED_RE =
+  /\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluent|fluency|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend|fließend|muttersprach\w*|sehr\s+gute?|gute?\s+Deutschkenntnisse|voraussetzung|zwingend)\b/i;
+
+/** Title-level marker — no description fetch needed. */
+export const GERMAN_TITLE_RE =
+  /\bgerman[\s-]speaking\b|\bdeutschsprachig\w*|\(\s*de\s*\)\s*speaking|\bgerman\s+(?:required|mandatory)\b/i;
+
+/**
+ * True when the text demands German above B1.
+ *
+ * Presence of a marker in the window isn't enough — an unrelated "a plus" from
+ * a neighbouring bullet would cancel a real requirement (Helvetia's posting
+ * puts "Infrastructure skills are a plus" ~95 chars before "Good knowledge of
+ * German"). So for each mention we take whichever marker sits *closest*.
+ */
+export function requiresGerman(text: string): boolean {
+  const flat = text.replace(/\s+/g, " ");
+  const optional = new RegExp(GERMAN_OPTIONAL_RE.source, "gi");
+  const required = new RegExp(GERMAN_REQUIRED_RE.source, "gi");
+
+  // A marker binds to the German mention only if no OTHER language name sits
+  // between them: in "Fluent in English; German ... advantageous", "Fluent"
+  // qualifies English, not German.
+  const OTHER_LANG = /\b(english|french|italian|spanish|portuguese|anglais|français|englisch|franz|italienisch)\b/i;
+
+  const nearest = (re: RegExp, window: string, at: number, bind: boolean): number => {
+    let best = Infinity;
+    for (const m of window.matchAll(re)) {
+      const i = m.index ?? 0;
+      const between = i < at ? window.slice(i + m[0].length, at) : window.slice(at, i);
+      if (bind && OTHER_LANG.test(between)) continue; // belongs to a different language
+      const d = Math.abs(i - at);
+      if (d < best) best = d;
+    }
+    return best;
+  };
+
+  for (const m of flat.matchAll(GERMAN_MENTION_RE)) {
+    const i = m.index ?? 0;
+    const from = Math.max(0, i - 110);
+    const window = flat.slice(from, i + 110);
+    const at = i - from; // mention's offset inside the window
+    const req = nearest(required, window, at, true);
+    const opt = nearest(optional, window, at, false);
+    if (req === Infinity) continue;      // nothing demanding it
+    if (req < opt) return true;          // the closer marker wins
+  }
+  return false;
+}
+
+export async function fetchDescription(id: string): Promise<string | null> {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      await Bun.sleep(350 * (attempt + 1));
+      const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
+      const html = await res.text();
+      const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
+      if (m) return m[1].replace(/<[^>]+>/g, " ");
+      if (!res.ok || attempt < 2) continue;
+      return html.replace(/<[^>]+>/g, " ");
+    } catch {
+      // transient — retry
+    }
+  }
+  return null;
+}
+
+/** English-language AND not demanding German above B1. */
+export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
+  const text = await fetchDescription(id);
+  if (text === null) return { ok: true }; // keep when we genuinely can't read it
+  const de = (text.match(DE_CONTENT_RE) ?? []).length;
+  const fr = (text.match(FR_CONTENT_RE) ?? []).length;
+  if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
+  if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
+  return { ok: true };
+}
+
 export async function isEnglishContent(id: string): Promise<boolean> {
+  // Retry when the description block is missing — LinkedIn serves a stub page
+  // under rate limiting, and the English UI chrome would otherwise pass as a
+  // false "English" result. Back off and try again before falling back.
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      await Bun.sleep(350 * (attempt + 1));
+      const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
+      const html = await res.text();
+      const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
+      if (!m) {
+        if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
+        // Last resort: scan the whole stripped page (still catches DE/FR body text)
+        const text = html.replace(/<[^>]+>/g, " ");
+        const de = (text.match(DE_CONTENT_RE) ?? []).length;
+        const fr = (text.match(FR_CONTENT_RE) ?? []).length;
+        return de < 6 && fr < 6;
+      }
+      const text = m[1].replace(/<[^>]+>/g, " ");
+      const de = (text.match(DE_CONTENT_RE) ?? []).length;
+      const fr = (text.match(FR_CONTENT_RE) ?? []).length;
+      return de < 6 && fr < 6;
+    } catch {
+      // transient fetch error — retry
+    }
+  }
+  return true; // keep when we genuinely can't fetch the description
+}
+
+export async function isEnglishUrl(url: string): Promise<boolean> {
+  return (await isViableUrl(url)).ok;
+}
+
+/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
+export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> {
  try {
    await Bun.sleep(350);
-    const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
-    const html = await res.text();
-    const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
-    const text = m
-      ? m[1].replace(/<[^>]+>/g, " ")
-      : html.replace(/<[^>]+>/g, " ").slice(0, 5000);
+    const res = await fetch(url, { headers: HEADERS });
+    if (!res.ok) return { ok: true };
+    const text = (await res.text()).replace(/<[^>]+>/g, " ");
    const de = (text.match(DE_CONTENT_RE) ?? []).length;
    const fr = (text.match(FR_CONTENT_RE) ?? []).length;
-    return de < 6 && fr < 6;
+    if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
+    if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
+    return { ok: true };
  } catch {
-    return true; // keep on fetch error
+    return { ok: true };
  }
 }

@@ -179,7 +350,9 @@ function parseJobs(html: string, label: string): Job[] {
    const company = companyM ? clean(companyM[1]) : "";
    const postedAt = timeM ? timeM[1] : "";

-    jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" });
+    // Prefer the location printed on the card over the search label — recruiters
+    // and multi-site companies often post under a different area than the job sits in.
+    jobs.push({ id, title, company, location: jobLocation || label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" });
  }

  return jobs;
@@ -206,6 +379,8 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro
      .filter((j) => !NON_ENGLISH_RE.test(j.title))
      .filter((j) => !JUNIOR_TITLE_RE.test(j.title))
      .filter((j) => !JAVA_TITLE_RE.test(j.title))
+      .filter((j) => !BIO_DOMAIN_RE.test(j.title))
+      .filter((j) => !GERMAN_TITLE_RE.test(j.title))
      .filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title));

    let newOnPage = 0;