linkme/fetch.ts

import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";

export interface Job {
  id: string;
  title: string;
  company: string;
  location: string;
  postedAt: string;
  link: string;
  search: string;
}

const WORK_TYPE_MAP: Record<WorkType, string> = {
  onsite: "1",
  remote: "2",
  hybrid: "3",
};

const JOB_TYPE_MAP: Record<JobType, string> = {
  "full-time": "F",
  "part-time": "P",
  contract: "C",
  temporary: "T",
  internship: "I",
};

const TIME_MAP: Record<TimePosted, string> = {
  day: "r86400",
  week: "r604800",
  month: "r2592000",
};

const SENIORITY_MAP: Record<Seniority, string> = {
  internship: "1",
  entry: "2",
  associate: "3",
  "mid-senior": "4",
  director: "5",
  executive: "6",
};

// Detect non-English (German / French / Swiss-German) job titles.
//
// German signals:
//   - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
//     but NOT (m/f/d) / (f/m/d) which are used in English international posts
//   - Swiss neutral suffix: standalone (a) before % or end of string
//   - German umlauts / ß
//   - /in word-ending (Pflegehelfer/in, Köchin/Koch)
//   - German-only vocabulary
//   - Swiss German: EFZ (apprenticeship cert), Schweiz
//
// French signals:
//   - gender markers: (H/F), (F/H), (h/f)
//   - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
//   - French-only vocabulary
const NON_ENGLISH_RE = new RegExp(
  [
    // German gender markers (w = weiblich; must not match m/f/d)
    String.raw`\(m\/w[^)]*\)`,
    String.raw`\(w\/m[^)]*\)`,
    String.raw`\(d\/w[^)]*\)`,
    String.raw`\(mwd\)`,
    // Swiss neutral (a) suffix: " (a)" before %, digits, or end
    String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
    // German/Swiss umlauts and ß
    String.raw`[äöüÄÖÜß]`,
    // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
    String.raw`[a-zäöü]\/in\b`,
    // German-only words (case-insensitive handled by flag)
    String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`,
    // French gender markers
    String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
    // French-only accented characters
    String.raw`[éèêëàâçîïôùûÿœæ]`,
    // French-only words
    String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
  ].join("|"),
  "i"
);

const HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept-Language": "en-US,en;q=0.9",
  Accept:
    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
};

const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:ship)?|trainee|apprentice|werkstudent)\b/i;

export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;

const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;

export async function isEnglishContent(id: string): Promise<boolean> {
  try {
    await Bun.sleep(350);
    const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
    const html = await res.text();
    const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
    const text = m
      ? m[1].replace(/<[^>]+>/g, " ")
      : html.replace(/<[^>]+>/g, " ").slice(0, 5000);
    const de = (text.match(DE_CONTENT_RE) ?? []).length;
    const fr = (text.match(FR_CONTENT_RE) ?? []).length;
    return de < 6 && fr < 6;
  } catch {
    return true; // keep on fetch error
  }
}

const PAGE_SIZE = 25;
const MAX_PAGES = 6;

export function buildUrl(config: SearchConfig): string {
  if (config.url) return config.url;
  const params = new URLSearchParams();
  params.set("keywords", config.keywords ?? "");
  if (config.location) params.set("location", config.location);
  if (config.geoId) params.set("geoId", config.geoId);
  if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
  if (config.workType?.length)
    params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
  if (config.jobType?.length)
    params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
  if (config.seniority?.length)
    params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
  return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
}

function clean(raw: string): string {
  return raw
    .replace(/<[^>]+>/g, "")
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&ndash;/g, "–")
    .replace(/&mdash;/g, "—")
    .replace(/\s+/g, " ")
    .trim();
}

function parseJobs(html: string, label: string): Job[] {
  const jobs: Job[] = [];

  // Locate every card boundary by job ID position
  const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
  const positions: { id: string; start: number }[] = [];
  let m;
  while ((m = idRe.exec(html)) !== null) {
    positions.push({ id: m[1], start: m.index });
  }

  for (let i = 0; i < positions.length; i++) {
    const { id, start } = positions[i];
    // Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
    const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));

    const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
    if (!titleM) continue;

    const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
    const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);

    const title = clean(titleM[1]);
    const company = companyM ? clean(companyM[1]) : "";
    const postedAt = timeM ? timeM[1] : "";

    jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label });
  }

  return jobs;
}

export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Promise<Job[]> {
  const baseUrl = buildUrl(config);
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 0; page < MAX_PAGES; page++) {
    if (page > 0) await Bun.sleep(400);
    const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
    process.stdout.write(`  page ${page + 1} … `);

    const res = await fetch(url, { headers: HEADERS });
    if (!res.ok) {
      console.error(`failed (${res.status})`);
      break;
    }

    const html = await res.text();
    const jobs = parseJobs(html, config.label)
      .filter((j) => !NON_ENGLISH_RE.test(j.title))
      .filter((j) => !JUNIOR_TITLE_RE.test(j.title))
      .filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title));

    let newOnPage = 0;
    for (const job of jobs) {
      if (!seen.has(job.id)) {
        seen.add(job.id);
        allJobs.push(job);
        newOnPage++;
      }
    }

    console.log(`${jobs.length} jobs, ${newOnPage} new`);
    if (newOnPage === 0) break;
  }

  return allJobs;
}