linkme/fetch.ts

import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";

export interface Job {
  id: string;
  title: string;
  company: string;
  location: string;
  postedAt: string;
  link: string;
  search: string;
  source: string;
  /**
   * Set when the source already language-checked the real job description.
   * `isEnglishUrl` counts DE/FR words across the whole page, so boards with
   * German site chrome (itjobs.ch, stellen-basel.ch) fail it even when the
   * posting itself is English — skip the re-check for those.
   */
  contentChecked?: boolean;
}

const WORK_TYPE_MAP: Record<WorkType, string> = {
  onsite: "1",
  remote: "2",
  hybrid: "3",
};

const JOB_TYPE_MAP: Record<JobType, string> = {
  "full-time": "F",
  "part-time": "P",
  contract: "C",
  temporary: "T",
  internship: "I",
};

const TIME_MAP: Record<TimePosted, string> = {
  day: "r86400",
  week: "r604800",
  month: "r2592000",
};

const SENIORITY_MAP: Record<Seniority, string> = {
  internship: "1",
  entry: "2",
  associate: "3",
  "mid-senior": "4",
  director: "5",
  executive: "6",
};

// Detect non-English (German / French / Swiss-German) job titles.
//
// German signals:
//   - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
//     but NOT (m/f/d) / (f/m/d) which are used in English international posts
//   - Swiss neutral suffix: standalone (a) before % or end of string
//   - German umlauts / ß
//   - /in word-ending (Pflegehelfer/in, Köchin/Koch)
//   - German-only vocabulary
//   - Swiss German: EFZ (apprenticeship cert), Schweiz
//
// French signals:
//   - gender markers: (H/F), (F/H), (h/f)
//   - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
//   - French-only vocabulary
export const NON_ENGLISH_RE = new RegExp(
  [
    // German gender markers (w = weiblich; must not match m/f/d)
    String.raw`\(m\/w[^)]*\)`,
    String.raw`\(w\/m[^)]*\)`,
    String.raw`\(d\/w[^)]*\)`,
    String.raw`\(mwd\)`,
    // m/w without parens (e.g. "Developer m/w")
    String.raw`\bm\/w\b`,
    // Swiss neutral (a) suffix: " (a)" before %, digits, or end
    String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
    // German/Swiss umlauts and ß
    String.raw`[äöüÄÖÜß]`,
    // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
    String.raw`[a-zäöü]\/in\b`,
    // German-only words (case-insensitive handled by flag)
    String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale|Ingenieur|Spezialist|Perspektive|Stellvertretung|Leitung|Beratung|Entwicklung|Betreuung|Verwaltung)\b`,
    // German prepositions/articles in title context (must appear as whole words)
    String.raw`\bmit\b`,
    // French gender markers
    String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
    // French-only accented characters
    String.raw`[éèêëàâçîïôùûÿœæ]`,
    // French-only words
    String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
  ].join("|"),
  "i"
);

const HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept-Language": "en-US,en;q=0.9",
  Accept:
    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
};

export const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:ship)?|trainee|apprentice|werkstudent)\b/i;

export const JAVA_TITLE_RE = /\bjava\b/i;

// Roles whose *domain* is biology / chemistry / the lab bench. Rafael has no
// life-science background, so these need expertise he can't offer.
//
// This filters on the ROLE, never the employer: software, platform, data and
// infrastructure jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope —
// Basel pharma is a core target market. Only the science itself is excluded.
export const BIO_DOMAIN_RE = new RegExp(
  [
    // bench / domain science job families
    String.raw`\b(bio)?statistician\b`,
    String.raw`\bbioinformatic\w*`,
    String.raw`\bcomputational\s+(?:biolog\w*|chemist\w*)`,
    String.raw`\b(?:cheminformatic\w*|chemometric\w*)`,
    String.raw`\b(?:micro)?biolog(?:y|ist)\b`,
    String.raw`\b(?:bio)?chemist(?:ry|s)?\b`,
    String.raw`\bpharmacolog\w*|\btoxicolog\w*|\bimmunolog\w*|\bvirolog\w*|\bepidemiolog\w*`,
    String.raw`\bgenomic\w*|\bproteomic\w*|\bmetabolomic\w*|\bmolecular\s+(?:biolog\w*|patholog\w*)`,
    // manufacturing / QC / regulated-lab functions
    String.raw`\bGMP\b|\bGxP\b|\bMSAT\b|\bQC\s+analyst\b`,
    String.raw`\b(?:bio)?process\s+(?:development|engineer)|\bupstream\b|\bdownstream\s+process`,
    String.raw`\bassay\b|\bbioassay\b|\bwet.?lab\b|\blaboratory\s+(?:technician|assistant|manager)`,
    // clinical / trial work
    String.raw`\bclinical\s+(?:trial|research|data\s+manager|operations)`,
    String.raw`\bbiomarker\b|\bpharmacovigilance\b|\bregulatory\s+affairs\b`,
    // life-science domains that make any title (Scientist, Engineer, Analyst) a domain role
    String.raw`\bbiologic\w*|\bpatholog\w*|\bhistolog\w*`,
    String.raw`\bdrug\s+(?:discovery|design|development)\b`,
    String.raw`\breal.world\s+(?:data|evidence)\b`,
  ].join("|"),
  "i"
);

export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devsecops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|infrastructure|mlops|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|scientific\s*(?:software|comput\w*|programmer)|research\s*(?:software|engineer)|hpc|forward\s*deployed|field\s*engineer|customer\s*engineer|sales\s*engineer|pre.?sales|integration\s*engineer|api\s*engineer|founding\s*engineer|product\s*engineer|automation\s*engineer|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;

const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;

// --- German-requirement detection -----------------------------------------
//
// Distinct from isEnglishContent: that drops postings *written* in German.
// This drops postings written in English that *require* German above B1 —
// e.g. Helvetia's "Good knowledge of German and English", which reads as a
// clean English posting and sails through every other filter.
//
// Rafael is German B1 (TELC). B1 or "basic"/"a plus" is fine; B2+ is not.

/** Any mention of the language, in either language. */
const GERMAN_MENTION_RE = /\b(german|deutsch\w*|allemand)\b/gi;

/** Within a mention's window: signals the requirement is optional. */
const GERMAN_OPTIONAL_RE =
  /\b(?:a\s+plus|an?\s+advantage|advantageous|beneficial|nice[\s-]to[\s-]have|desirable|preferred|welcome|bonus|not\s+required|no\s+german|optional|willing(?:ness)?\s+to\s+learn|basic|beginner|elementary|grundkenntnisse|von\s+vorteil|wünschenswert|A1|A2|B1)\b/i;

/** Within a mention's window: signals it is genuinely required at B2+. */
const GERMAN_REQUIRED_RE =
  /\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluen\w*|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend\w*|fließend\w*|muttersprach\w*|sehr\s+gut\w*|gut\w*\s+Deutschkenntnisse|Deutschkenntnisse|voraussetzung|zwingend|erforderlich)\b/i;

/**
 * Spec-sheet style language line ("Languages: German & English"), common on
 * contractor postings. There is no adjective for the nearest-marker logic to
 * bind to, so the label itself counts as a requirement — but it goes through
 * the same nearest-marker comparison, so "Language: English and German (bonus)"
 * still loses to the "bonus" sitting closer to the mention.
 */
const GERMAN_SPEC_LABEL_RE = /\b(?:languages?|sprachen|sprache)\s*[::]/i;

/** Title-level marker — no description fetch needed. */
export const GERMAN_TITLE_RE =
  /\bgerman[\s-]speaking\b|\bdeutschsprachig\w*|\(\s*de\s*\)\s*speaking|\bgerman\s+(?:required|mandatory)\b/i;

/**
 * True when the text demands German above B1.
 *
 * Presence of a marker in the window isn't enough — an unrelated "a plus" from
 * a neighbouring bullet would cancel a real requirement (Helvetia's posting
 * puts "Infrastructure skills are a plus" ~95 chars before "Good knowledge of
 * German"). So for each mention we take whichever marker sits *closest*.
 */
export function requiresGerman(text: string): boolean {
  const flat = text.replace(/\s+/g, " ");
  const optional = new RegExp(GERMAN_OPTIONAL_RE.source, "gi");
  const required = new RegExp(
    `${GERMAN_REQUIRED_RE.source}|${GERMAN_SPEC_LABEL_RE.source}`,
    "gi"
  );

  // A marker binds to the German mention only if no OTHER language name sits
  // between them: in "Fluent in English; German ... advantageous", "Fluent"
  // qualifies English, not German.
  const OTHER_LANG = /\b(english|french|italian|spanish|portuguese|anglais|français|englisch|franz|italienisch)\b/i;

  // …except when that other language is *coordinated* with German, where the
  // marker governs both: "Excellent communication skills in both English and
  // German" (Karakun) and "Fluency in English and German" (J. Safra Sarasin)
  // are hard requirements that the bare rule above waved through — the entire
  // reason 24 basel-jobs.ch postings mentioning German were caught 0 times.
  // Deliberately excludes "and/or" and a bare "or": "Very good English and/or
  // German" (BBF) is satisfied by English alone, so it must stay optional.
  const CONJOINED = /\b(?:and|und|sowie|as\s+well\s+as|&)\s*$/i;
  const ANDOR = /\b(?:and\s*\/\s*or|or)\s*$/i;

  const nearest = (re: RegExp, window: string, at: number, bind: boolean): number => {
    let best = Infinity;
    for (const m of window.matchAll(re)) {
      const i = m.index ?? 0;
      const between = i < at ? window.slice(i + m[0].length, at) : window.slice(at, i);
      // "…English and German" keeps the marker; "…English; German" or
      // "…English and/or German" hands it to the other language.
      const conjoined = CONJOINED.test(between) && !ANDOR.test(between);
      if (bind && OTHER_LANG.test(between) && !conjoined) continue;
      const d = Math.abs(i - at);
      if (d < best) best = d;
    }
    return best;
  };

  for (const m of flat.matchAll(GERMAN_MENTION_RE)) {
    const i = m.index ?? 0;
    const from = Math.max(0, i - 110);
    const window = flat.slice(from, i + 110);
    const at = i - from; // mention's offset inside the window
    const req = nearest(required, window, at, true);
    const opt = nearest(optional, window, at, false);
    if (req === Infinity) continue;      // nothing demanding it
    if (req < opt) return true;          // the closer marker wins
  }
  return false;
}

export async function fetchDescription(id: string): Promise<string | null> {
  for (let attempt = 0; attempt < 3; attempt++) {
    try {
      await Bun.sleep(350 * (attempt + 1));
      const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
      const html = await res.text();
      const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
      if (m) return m[1].replace(/<[^>]+>/g, " ");
      if (!res.ok || attempt < 2) continue;
      return html.replace(/<[^>]+>/g, " ");
    } catch {
      // transient — retry
    }
  }
  return null;
}

/** English-language AND not demanding German above B1. */
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
  const text = await fetchDescription(id);
  if (text === null) return { ok: true }; // keep when we genuinely can't read it
  const de = (text.match(DE_CONTENT_RE) ?? []).length;
  const fr = (text.match(FR_CONTENT_RE) ?? []).length;
  if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
  if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
  return { ok: true };
}

export async function isEnglishContent(id: string): Promise<boolean> {
  // Retry when the description block is missing — LinkedIn serves a stub page
  // under rate limiting, and the English UI chrome would otherwise pass as a
  // false "English" result. Back off and try again before falling back.
  for (let attempt = 0; attempt < 3; attempt++) {
    try {
      await Bun.sleep(350 * (attempt + 1));
      const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
      const html = await res.text();
      const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
      if (!m) {
        if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
        // Last resort: scan the whole stripped page (still catches DE/FR body text)
        const text = html.replace(/<[^>]+>/g, " ");
        const de = (text.match(DE_CONTENT_RE) ?? []).length;
        const fr = (text.match(FR_CONTENT_RE) ?? []).length;
        return de < 6 && fr < 6;
      }
      const text = m[1].replace(/<[^>]+>/g, " ");
      const de = (text.match(DE_CONTENT_RE) ?? []).length;
      const fr = (text.match(FR_CONTENT_RE) ?? []).length;
      return de < 6 && fr < 6;
    } catch {
      // transient fetch error — retry
    }
  }
  return true; // keep when we genuinely can't fetch the description
}

export async function isEnglishUrl(url: string): Promise<boolean> {
  return (await isViableUrl(url)).ok;
}

/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> {
  try {
    await Bun.sleep(350);
    const res = await fetch(url, { headers: HEADERS });
    if (!res.ok) return { ok: true };
    const text = (await res.text()).replace(/<[^>]+>/g, " ");
    const de = (text.match(DE_CONTENT_RE) ?? []).length;
    const fr = (text.match(FR_CONTENT_RE) ?? []).length;
    if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
    if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
    return { ok: true };
  } catch {
    return { ok: true };
  }
}

const PAGE_SIZE = 25;
const MAX_PAGES = 6;

export function buildUrl(config: SearchConfig): string {
  if (config.url) return config.url;
  const params = new URLSearchParams();
  params.set("keywords", config.keywords ?? "");
  if (config.location) params.set("location", config.location);
  if (config.geoId) params.set("geoId", config.geoId);
  if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
  if (config.workType?.length)
    params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
  if (config.jobType?.length)
    params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
  if (config.seniority?.length)
    params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
  return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
}

function clean(raw: string): string {
  return raw
    .replace(/<[^>]+>/g, "")
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&ndash;/g, "–")
    .replace(/&mdash;/g, "—")
    .replace(/\s+/g, " ")
    .trim();
}

function parseJobs(html: string, label: string): Job[] {
  const jobs: Job[] = [];

  // Locate every card boundary by job ID position
  const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
  const positions: { id: string; start: number }[] = [];
  let m;
  while ((m = idRe.exec(html)) !== null) {
    positions.push({ id: m[1], start: m.index });
  }

  for (let i = 0; i < positions.length; i++) {
    const { id, start } = positions[i];
    // Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
    const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));

    const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
    if (!titleM) continue;

    const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
    const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);

    const locM = card.match(/job-search-card__location[^>]*>\s*([^<]+)/);
    const jobLocation = locM ? locM[1].trim() : "";

    // Drop jobs not physically in Switzerland — LinkedIn ignores geoId without auth
    if (jobLocation && !jobLocation.includes("Switzerland")) continue;

    const title = clean(titleM[1]);
    const company = companyM ? clean(companyM[1]) : "";
    const postedAt = timeM ? timeM[1] : "";

    // Prefer the location printed on the card over the search label — recruiters
    // and multi-site companies often post under a different area than the job sits in.
    jobs.push({ id, title, company, location: jobLocation || label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" });
  }

  return jobs;
}

export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Promise<Job[]> {
  const baseUrl = buildUrl(config);
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 0; page < MAX_PAGES; page++) {
    if (page > 0) await Bun.sleep(400);
    const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
    process.stdout.write(`  page ${page + 1} … `);

    // 429s became common once index.ts started running several searches
    // concurrently — a transient rate-limit, not a real end-of-results
    // signal, so retry with backoff instead of silently truncating the
    // search (losing every page after this one).
    let res = await fetch(url, { headers: HEADERS });
    for (let attempt = 0; res.status === 429 && attempt < 3; attempt++) {
      await Bun.sleep(1000 * 2 ** attempt); // 1s, 2s, 4s
      res = await fetch(url, { headers: HEADERS });
    }
    if (!res.ok) {
      console.error(`failed (${res.status})`);
      break;
    }

    const html = await res.text();
    const jobs = parseJobs(html, config.label)
      .filter((j) => !NON_ENGLISH_RE.test(j.title))
      .filter((j) => !JUNIOR_TITLE_RE.test(j.title))
      .filter((j) => !JAVA_TITLE_RE.test(j.title))
      .filter((j) => !BIO_DOMAIN_RE.test(j.title))
      .filter((j) => !GERMAN_TITLE_RE.test(j.title))
      .filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title));

    let newOnPage = 0;
    for (const job of jobs) {
      if (!seen.has(job.id)) {
        seen.add(job.id);
        allJobs.push(job);
        newOnPage++;
      }
    }

    console.log(`${jobs.length} jobs, ${newOnPage} new`);
    if (newOnPage === 0) break;
  }

  return allJobs;
}