import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman } from "./fetch.ts";
import type { Job } from "./fetch.ts";

const HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept-Language": "en-US,en;q=0.9",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
};

// datacareer.ch inverts the usual bot rule: its nginx 403s anything claiming to
// be a browser (a spoofed Chrome UA is the one thing guaranteed to fail) while
// letting honest clients — no UA, curl, Googlebot — straight through. Presumably
// real browser traffic arrives via a path that also carries matching TLS/CH
// fingerprints, so a bare Chrome UA over plain HTTP reads as a scraper. Sending
// an honest identifier is both what unblocks it and the polite thing to do.
// This is what killed the source for months; it was never a hard block.
const PLAIN_HEADERS = {
  "User-Agent": "linkme-jobbot/1.0 (personal job search; contact via basel)",
  "Accept-Language": "en-US,en;q=0.9",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
};

function decodeHtml(s: string): string {
  return s
    .replace(/&/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
    .replace(/\s+/g, " ")
    .trim();
}

function applyFilters(title: string): boolean {
  if (NON_ENGLISH_RE.test(title)) return false;
  if (JUNIOR_TITLE_RE.test(title)) return false;
  if (!SOFTWARE_TITLE_RE.test(title)) return false;
  return true;
}

const MS_STACK_RE = /\.net\b|\bc#|sharepoint|dynamics 365|power platform|\bsccm\b|intune|successfactors|\bsap\b/i;

// DE/FR indicator words, counted against a description we already have in hand.
// Mirrors the ≥ 6 threshold isEnglishContent uses on fetched pages.
const DE_FR_WORDS_RE =
  /\b(und|oder|wir|Sie|Ihre|Ihrer|Kenntnisse|Erfahrung|Aufgaben|Anforderungen|Stelle|Unternehmen|Mitarbeiter|Bereich|nous|vous|votre|pour|avec|des|les)\b/g;

// English-language AND not demanding German above B1 — mirrors isViableContent/
// isViableUrl in fetch.ts, but against a description we already have in hand
// instead of a page fetch. The two checks catch different things: the word
// count drops postings *written* in German/French; requiresGerman() drops
// postings *written in English* that still demand German above B1 (e.g.
// PRODYNA: "you can communicate fluently in both German and English" — clean
// English text, would sail through the word count untouched). Skipping the
// requiresGerman() half was a real gap in every source below until 2026-08-30
// — it let German-fluency requirements slip past whenever an item's inline
// description read as English.
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
  if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
  const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
  if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" };
  if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
  return { ok: true };
}

// datacareer.ch — HTML listing at /jobs/?city=Basel
export async function fetchDataCareer(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 1; page <= 10; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  page ${page} … `);

    const res = await fetch(
      `https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
      { headers: PLAIN_HEADERS }
    );
    if (!res.ok) { console.error(`failed (${res.status})`); break; }

    const html = await res.text();
    const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
    if (articles.length === 0) { console.log("0 items"); break; }

    let newOnPage = 0;
    for (const [, card] of articles) {
      const titleM = card.match(/class="link"[^>]*>([^<]+)/);
      const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
      if (!titleM || !hrefM) continue;

      const title = decodeHtml(titleM[1]);
      if (!applyFilters(title)) continue;

      const id = `dc-${hrefM[2]}`;
      if (seen.has(id)) continue;
      seen.add(id);

      const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
      const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
      const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);

      // Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
      let postedAt = "";
      const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
      if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;

      allJobs.push({
        id,
        title,
        company: companyM ? decodeHtml(companyM[1].trim()) : "",
        location: locM ? locM[1].trim() : "Basel",
        postedAt,
        link: hrefM[1],
        search: "datacareer.ch",
        source: "datacareer.ch",
      });
      newOnPage++;
    }

    console.log(`${articles.length} items, ${newOnPage} new`);
    if (articles.length < 8) break;
  }

  return allJobs;
}

// itjobs.ch — HTML scrape, jobs embedded as window.jobsList in page.
// Swept Switzerland-wide (no location filter): the board is small, ~210 jobs
// across 7 pages, and locking it to Basel returned only ~9.
// Each item carries its full description, so the DE/FR content check runs here
// instead of costing an extra page fetch per job in index.ts.
export async function fetchItJobsCh(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 1; page <= 25; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  page ${page} … `);

    const res = await fetch(
      `https://www.itjobs.ch/jobs?page=${page}`,
      { headers: HEADERS }
    );
    // The board 404s one page past the end rather than serving an empty list.
    if (!res.ok) { console.log(`end (${res.status})`); break; }

    const html = await res.text();
    const blocks = [...html.matchAll(/window\.jobsList\s*=\s*window\.jobsList\.concat\((\[[\s\S]*?\])\);/g)];
    if (!blocks.length) { console.log("0 items"); break; }

    const items: Array<{
      id: number;
      title: string;
      description?: string;
      posted_at: string;
      job_details_path: string;
      employer: { name: string };
      location?: string;
      job_location?: { name?: string };
    }> = [];
    for (const [, json] of blocks) {
      try { items.push(...JSON.parse(json)); } catch { /* skip malformed block */ }
    }

    if (!items.length) { console.log("0 items"); break; }

    let newOnPage = 0;
    let langDropped = 0;
    for (const item of items) {
      const title = decodeHtml(item.title);
      if (!applyFilters(title)) continue;
      if (MS_STACK_RE.test(title)) continue;

      const id = `ij-${item.id}`;
      if (seen.has(id)) continue;
      seen.add(id);

      if (!isViableDescription(item.description).ok) { langDropped++; continue; }

      allJobs.push({
        id,
        title,
        company: item.employer?.name ?? "",
        // job_location.name is the English form ("Aarau, Aargau, Switzerland");
        // the bare location field is German ("Aarau, Kanton Aargau, Schweiz").
        location: item.job_location?.name ?? item.location ?? "Switzerland",
        postedAt: item.posted_at?.slice(0, 10) ?? "",
        link: `https://www.itjobs.ch${item.job_details_path}`,
        search: "itjobs.ch",
        source: "itjobs.ch",
        contentChecked: true, // isViableDescription already ran on the real description
      });
      newOnPage++;
    }

    console.log(`${items.length} items, ${newOnPage} new, ${langDropped} non-English`);
    if (items.length < 30) break;
  }

  return allJobs;
}

// job-room.ch — the federal employment service board (RAV / Arbeitsvermittlung).
// Roles under the Stellenmeldepflicht must be posted here before anywhere else,
// so it often carries a job days ahead of LinkedIn.
//
// Quirks worth knowing:
//  - Each hit is wrapped: [{ jobAdvertisement: {...} }, …]
//  - Titles come back with <em> highlighting around the matched keyword.
//  - `languageIsoCode` on a description is unreliable — plenty of records tagged
//    "de" carry English text (Roche, Novartis, Bachem all do). The DE/FR word
//    count is what actually decides, so we ignore the tag.
//  - An unfiltered canton sweep returns thousands of ads across every industry,
//    so we narrow by keyword instead and union the results.
//  - Descriptions arrive inline, so the language check runs here rather than
//    costing a page fetch per job in index.ts.
const JOBROOM_API = "https://www.job-room.ch/jobadservice/api/jobAdvertisements/_search";
const JOBROOM_CANTONS = ["BS", "BL", "BE", "ZH"];
const JOBROOM_KEYWORDS = [
  "software", "engineer", "developer", "devops", "data", "python",
  "cloud", "platform", "typescript", "backend", "full stack", "ai",
];

// job-room carries every industry; pharma/lab titles slip past SOFTWARE_TITLE_RE
// via words like "data", "director" and "technical".
const PHARMA_ROLE_RE =
  /\b(clinical|pharmaceutical|drug\s+(substance|product)|ha?ematology|oncology|formulation|laborator|lab\s+technician|chemist|chemistry|biolog|preclinical|regulatory\s+affairs|medical\s+affairs|commercial\s+launch|veterinary|pathologist)\b/i;

interface JobRoomAd {
  id: string;
  createdTime?: string;
  jobContent?: {
    externalUrl?: string | null;
    jobDescriptions?: Array<{ title: string; description?: string }>;
    company?: { name?: string; city?: string };
    location?: { city?: string; cantonCode?: string };
    languageSkills?: JobRoomLanguageSkill[];
  };
}

interface JobRoomLanguageSkill {
  languageIsoCode?: string;
  spokenLevel?: string;
  writtenLevel?: string;
}

// job-room's language requirement is a structured field, not prose — it never
// touches jobDescriptions[].description, so isViableDescription()/requiresGerman()
// can't see it. The UI renders it as e.g. "Deutsch (Fliessend)"; PROFICIENT is
// that "Fliessend" (fluent/C1+) tier, well above Rafael's German B1. BASIC and
// INTERMEDIATE stay in scope.
function jobRoomRequiresGerman(skills?: JobRoomLanguageSkill[]): boolean {
  if (!skills) return false;
  return skills.some(
    (s) => s.languageIsoCode === "de" && (s.spokenLevel === "PROFICIENT" || s.writtenLevel === "PROFICIENT")
  );
}

// jobs.ch links arrive with a tracking query string that would break dedup
// across runs if it ever changes; the path alone identifies the posting.
function stripTracking(url: string): string {
  try {
    const u = new URL(url);
    for (const k of [...u.searchParams.keys()]) {
      if (k.startsWith("utm_")) u.searchParams.delete(k);
    }
    return u.toString().replace(/\?$/, "");
  } catch {
    return url;
  }
}

export async function fetchJobRoom(): Promise<Job[]> {
  const ads = new Map<string, JobRoomAd>();

  for (const keyword of JOBROOM_KEYWORDS) {
    process.stdout.write(`  ${keyword} … `);
    let total = 0;
    for (let page = 0; page < 8; page++) {
      if (page > 0) await Bun.sleep(250);
      const res = await fetch(`${JOBROOM_API}?page=${page}&size=100`, {
        method: "POST",
        headers: { ...HEADERS, "Content-Type": "application/json", Accept: "application/json" },
        body: JSON.stringify({
          onlineSince: 30,
          displayRestricted: false,
          keywords: [keyword],
          cantonCodes: JOBROOM_CANTONS,
          workloadPercentageMin: 0,
          workloadPercentageMax: 100,
        }),
      });
      if (!res.ok) { console.log(`failed (${res.status})`); break; }

      const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>;
      for (const h of hits) {
        if (h.jobAdvertisement?.id) ads.set(h.jobAdvertisement.id, h.jobAdvertisement);
      }
      total += hits.length;
      if (hits.length < 100) break;
    }
    console.log(`${total} ads (${ads.size} unique)`);
    await Bun.sleep(250);
  }

  const jobs: Job[] = [];
  let langDropped = 0;
  for (const ad of ads.values()) {
    const content = ad.jobContent;
    const desc = content?.jobDescriptions?.[0];
    if (!desc) continue;

    // Strip the <em> highlighting the search API wraps around matched terms.
    const title = decodeHtml(desc.title.replace(/<[^>]+>/g, ""));
    if (!applyFilters(title)) continue;
    if (MS_STACK_RE.test(title)) continue;
    if (PHARMA_ROLE_RE.test(title)) continue;

    // Structured language requirement — separate from the free-text description
    // entirely, so it must be checked even when the description reads clean.
    if (jobRoomRequiresGerman(content?.languageSkills)) { langDropped++; continue; }

    // A handful of ads ship without a description. isViableDescription passes
    // those through by design, so leave contentChecked unset and let index.ts
    // fall back to fetching the page.
    const body = desc.description ?? "";
    const hasBody = body.trim().length > 40;
    if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }

    const city = content?.location?.city ?? content?.company?.city ?? "";
    const canton = content?.location?.cantonCode ?? "";

    jobs.push({
      id: `jr-${ad.id}`,
      title,
      company: content?.company?.name ?? "",
      location: [city, canton].filter(Boolean).join(", ") || "Switzerland",
      postedAt: ad.createdTime?.slice(0, 10) ?? "",
      link: content?.externalUrl
        ? stripTracking(content.externalUrl)
        : `https://www.job-room.ch/job-search/${ad.id}`,
      search: "job-room.ch",
      source: "job-room.ch",
      contentChecked: hasBody, // only skip the re-check when we really judged a description
    });
  }

  console.log(`  ${ads.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
  return jobs;
}

// International organisations in the Basel region (BIS, Swiss TPH, Basel
// Institute on Governance). Each board carries only a handful of postings, so
// they're swept together under one label rather than one entry apiece.
//
// These are mission-driven employers whose tech roles use UN-flavoured titles
// ("Information Management Officer", "ICT Specialist") that SOFTWARE_TITLE_RE
// alone would drop, so INTL_TECH_RE widens the net for these boards only.
// Boards this small can afford the extra noise; missing a rare opening can't.
const INTL_TECH_RE =
  /\b(information\s*(management|systems?|technology)|ict\b|digital|geospatial|\bmis\b|data\s*(management|steward))\b/i;

function applyIntlFilters(title: string): boolean {
  if (NON_ENGLISH_RE.test(title)) return false;
  if (JUNIOR_TITLE_RE.test(title)) return false;
  if (MS_STACK_RE.test(title)) return false;
  return SOFTWARE_TITLE_RE.test(title) || INTL_TECH_RE.test(title);
}

// BIS runs a JS-rendered career site with no usable API, so we read the UNjobs
// mirror of its Basel office instead. Titles arrive as "Role, City, Country".
async function fetchBis(): Promise<Job[]> {
  const jobs: Job[] = [];
  const res = await fetch(
    "https://unjobs.org/offices/bis_182320855689338880",
    { headers: HEADERS }
  );
  if (!res.ok) { console.log(`  BIS … failed (${res.status})`); return jobs; }

  const html = await res.text();
  const cards = [...html.matchAll(
    /<a[^>]+href="(https:\/\/unjobs\.org\/vacancies\/(\d+))"[\s\S]*?<span\s+class="jtitle">([^<]+)<\/span>([\s\S]*?)<\/a>/g
  )];

  let kept = 0;
  for (const [, link, id, rawTitle, rest] of cards) {
    // "Senior Platform Engineer, Basel, Switzerland" → title + location
    const split = decodeHtml(rawTitle).match(/^(.*?),\s*([^,]+,\s*[^,]+)$/);
    const title = split ? split[1] : decodeHtml(rawTitle);
    const location = split ? split[2] : "Basel, Switzerland";
    if (!applyIntlFilters(title)) continue;

    jobs.push({
      id: `bis-${id}`,
      title,
      company: "Bank for International Settlements",
      location,
      postedAt: rest.match(/datetime="(\d{4}-\d{2}-\d{2})/)?.[1] ?? "",
      link,
      search: "bis.org",
      source: "bis.org",
      contentChecked: true, // BIS publishes in English only
    });
    kept++;
  }

  console.log(`  BIS … ${cards.length} ads, ${kept} kept`);
  return jobs;
}

// Swiss TPH runs Umantis. Site chrome is German even on English postings, so
// isEnglishUrl would reject the lot — the title filters decide instead.
async function fetchSwissTph(): Promise<Job[]> {
  const jobs: Job[] = [];
  const res = await fetch("https://jobs.swisstph.ch/Jobs/All", { headers: HEADERS });
  if (!res.ok) { console.log(`  Swiss TPH … failed (${res.status})`); return jobs; }

  const html = await res.text();
  // One block per posting, opened by the subtitle link that carries the title.
  const blocks = html.split(/(?=<a\s[^>]*class="HSTableLinkSubTitle")/).slice(1);

  let kept = 0;
  for (const block of blocks) {
    const hrefM = block.match(/href="(\/Vacancies\/(\d+)\/Description\/\d+)"/);
    const titleM = block.match(/class="HSTableLinkSubTitle"[^>]*>([^<]+)</);
    if (!hrefM || !titleM) continue;

    const title = decodeHtml(titleM[1]);
    if (!applyIntlFilters(title)) continue;

    // "Online seit" label and its value sit in separate spans sharing an id suffix.
    const dateM = block.match(/Online seit<\/span>[\s\S]{0,300}?>\s*(\d{2})\.(\d{2})\.(\d{4})/);
    const locM = block.match(/>\s*Schweiz\s*-\s*([^<]+)</);

    jobs.push({
      id: `tph-${hrefM[2]}`,
      title,
      company: "Swiss TPH",
      location: locM ? `${locM[1].trim()}, Switzerland` : "Allschwil, Switzerland",
      postedAt: dateM ? `${dateM[3]}-${dateM[2]}-${dateM[1]}` : "",
      link: `https://jobs.swisstph.ch${hrefM[1]}`,
      search: "swisstph.ch",
      source: "swisstph.ch",
      contentChecked: true, // German site chrome would fail the page-level check
    });
    kept++;
  }

  console.log(`  Swiss TPH … ${blocks.length} ads, ${kept} kept`);
  return jobs;
}

// Basel Institute on Governance publishes each vacancy as a linked PDF, so
// there is no HTML description to language-check downstream.
async function fetchBaselGovernance(): Promise<Job[]> {
  const jobs: Job[] = [];
  const res = await fetch("https://baselgovernance.org/about/careers", { headers: HEADERS });
  if (!res.ok) { console.log(`  Basel Governance … failed (${res.status})`); return jobs; }

  const html = await res.text();
  // The `rounded-3xl` card class is what separates vacancy tiles from other
  // asset links on the page (the annual report PDF sits in a plain anchor).
  const cards = [...html.matchAll(
    /<a\s+href="(https:\/\/baselgovernance\.org\/cms\/api\/assets\/([^\/]+)\/[^"]+)"[^>]*class="[^"]*rounded-3xl[^"]*"[\s\S]{0,400}?<h2[^>]*>([^<]+)<\/h2>([\s\S]{0,900})/g
  )];

  let kept = 0;
  for (const [, link, assetId, rawTitle, rest] of cards) {
    const title = decodeHtml(rawTitle);
    if (!applyIntlFilters(title)) continue;

    const locM = rest.match(/map-pin[\s\S]{0,200}?<\/span>\s*([^<]+)</);

    jobs.push({
      id: `big-${assetId}`,
      title,
      company: "Basel Institute on Governance",
      location: locM ? decodeHtml(locM[1]) : "Basel, Switzerland",
      postedAt: "", // board shows the closing date only
      link,
      search: "baselgovernance.org",
      source: "baselgovernance.org",
      contentChecked: true, // postings are PDFs, and the institute publishes in English
    });
    kept++;
  }

  console.log(`  Basel Governance … ${cards.length} ads, ${kept} kept`);
  return jobs;
}

export async function fetchBaselIntlOrgs(): Promise<Job[]> {
  const jobs: Job[] = [];
  for (const fetchOrg of [fetchBis, fetchSwissTph, fetchBaselGovernance]) {
    try {
      jobs.push(...(await fetchOrg()));
    } catch (e) {
      console.log(`  error: ${e}`);
    }
    await Bun.sleep(400);
  }
  return jobs;
}

// jobbasel.ch — the page shell is a React/Vite SPA (no jobs in the raw HTML
// on the homepage), but the search results route is server-rendered: fetching
// `/job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__`
// blob already containing the matched vacancies as JSON. No headless browser
// needed. Pagination is a URL suffix (`-seite-{n}`), not a query param — a
// bare `?page=2` is silently ignored by the SPA route.
//
// The board (Basel-Stadt + Basel-Landschaft, ~4000 openings) has no keyword
// union endpoint, so — like job-room.ch — we sweep a fixed keyword list and
// dedupe by vacancy id. Each item carries its full German description
// (activity/requirements/offer), so the DE/FR check runs here instead of
// costing a page fetch per job in index.ts.
const JOBBASEL_KEYWORDS = [
  "software", "engineer", "developer", "devops", "data", "python",
  "cloud", "platform", "typescript", "backend", "full stack", "ai",
];

interface JobBaselItem {
  id: number;
  title: string;
  activity?: string;
  requirements?: string;
  offer?: string;
  workplaceCity?: string;
  urlApplication?: string;
  dateFirstPublished?: string;
  company?: { name?: string };
}

// Balanced-brace extraction — the blob can itself contain "};" inside string
// values (job descriptions), so a regex cutoff would truncate it.
function extractPreloadedState(html: string): any | null {
  const marker = "window.__PRELOADED_STATE__";
  const markerIdx = html.indexOf(marker);
  if (markerIdx === -1) return null;
  const start = html.indexOf("{", markerIdx);
  if (start === -1) return null;

  let depth = 0, inStr = false, esc = false, i = start;
  for (; i < html.length; i++) {
    const c = html[i];
    if (inStr) {
      if (esc) esc = false;
      else if (c === "\\") esc = true;
      else if (c === '"') inStr = false;
    } else {
      if (c === '"') inStr = true;
      else if (c === "{") depth++;
      else if (c === "}") { depth--; if (depth === 0) { i++; break; } }
    }
  }

  try { return JSON.parse(html.slice(start, i)); } catch { return null; }
}

export async function fetchJobBasel(): Promise<Job[]> {
  const items = new Map<number, JobBaselItem>();

  for (const keyword of JOBBASEL_KEYWORDS) {
    process.stdout.write(`  ${keyword} … `);
    const slug = keyword.toLowerCase().replace(/\s+/g, "-");
    let total = 0;

    for (let page = 1; page <= 5; page++) {
      if (page > 1) await Bun.sleep(300);
      const suffix = page === 1 ? "" : `-seite-${page}`;
      const res = await fetch(
        `https://www.jobbasel.ch/job/suche-${slug}${suffix}`,
        { headers: HEADERS }
      );
      if (!res.ok) { console.log(`failed (${res.status})`); break; }

      const state = extractPreloadedState(await res.text());
      const pageItems: JobBaselItem[] = state?.vacancySearch?.data?.items ?? [];
      if (!pageItems.length) break;

      for (const item of pageItems) items.set(item.id, item);
      total += pageItems.length;
      if (pageItems.length < 10) break; // last page (pageSize is 10)
    }
    console.log(`${total} ads (${items.size} unique)`);
    await Bun.sleep(300);
  }

  const jobs: Job[] = [];
  let langDropped = 0;
  for (const item of items.values()) {
    const title = decodeHtml(item.title.replace(/<[^>]+>/g, ""));
    if (!applyFilters(title)) continue;
    if (MS_STACK_RE.test(title)) continue;

    const body = [item.activity, item.requirements, item.offer]
      .filter(Boolean)
      .join(" ")
      .replace(/<[^>]+>/g, " ");
    const hasBody = body.trim().length > 40;
    if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }

    jobs.push({
      id: `jb-${item.id}`,
      title,
      company: item.company?.name ?? "",
      location: item.workplaceCity ? `${item.workplaceCity}, Switzerland` : "Basel, Switzerland",
      postedAt: item.dateFirstPublished?.slice(0, 10) ?? "",
      link: item.urlApplication || `https://www.jobbasel.ch/job/${item.id}`,
      search: "jobbasel.ch",
      source: "jobbasel.ch",
      contentChecked: hasBody,
    });
  }

  console.log(`  ${items.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
  return jobs;
}

// englishjobsearch.ch — plain SSR HTML, no JS needed, but reached through a
// non-obvious lever: any page on the site accepts `?format=markdown` and
// returns the same content pre-converted to clean markdown text (job title,
// company, city, date, short description all present). Without that param
// the canton pages ship the job cards as plain text nodes too — no JSON blob,
// no client-side rendering — so this works with a bare `curl`-equivalent
// fetch either way; `?format=markdown` just makes parsing trivial.
//
// Two catches: (1) the directory root groups by canton, so we hit the
// Basel-Stadt and Basel-Landschaft canton pages directly rather than the
// generic `/in/basel/` landing page, which only lists canton counts.
// (2) every job link is a `/clickout/<hash>` tracking redirect, not the real
// posting URL — there is no way to recover the underlying employer link
// without following the redirect, so the clickout link is stored as-is (it
// still resolves for a human clicking it, and content-checking follows it
// server-side the same as any other link).
const ENGLISHJOBSEARCH_CANTONS = ["canton-basel-stadt", "canton-basel-landschaft"];

export async function fetchEnglishJobSearch(): Promise<Job[]> {
  const jobs: Job[] = [];
  const seen = new Set<string>();

  for (const canton of ENGLISHJOBSEARCH_CANTONS) {
    process.stdout.write(`  ${canton} … `);
    const res = await fetch(
      `https://englishjobsearch.ch/in/${canton}?format=markdown`,
      { headers: HEADERS }
    );
    if (!res.ok) { console.log(`failed (${res.status})`); continue; }

    const md = await res.text();
    // Each posting: "### Title](/clickout/<hash>?...)" heading, then on
    // separate lines "* Company", "* City", "* Date", then a description
    // paragraph before the next "### " heading or "report probem" footer.
    const cards = [...md.matchAll(
      /### ([^\]]+)\]\((\/clickout\/[^)]+)\)\n\n\* ([^\n]+)\n\* ([^\n]+)\n\* ([^\n]+)\n\n([\s\S]*?)\n\nreport probem/g
    )];

    let kept = 0;
    for (const [, rawTitle, clickout, company, city, dateStr, desc] of cards) {
      const title = rawTitle.trim();
      if (!applyFilters(title)) continue;
      if (MS_STACK_RE.test(title)) continue;

      const id = clickout.match(/\/clickout\/([a-f0-9]+)/)?.[1] ?? clickout;
      if (seen.has(id)) continue;
      seen.add(id);

      const description = desc.trim();
      const hasBody = description.length > 40;
      if (hasBody && !isViableDescription(description).ok) continue;

      jobs.push({
        id: `ejs-${id}`,
        title,
        company: company.trim(),
        location: `${city.trim()}, Switzerland`,
        postedAt: "", // "August 30" style, no year given — not worth mis-parsing
        link: `https://englishjobsearch.ch${clickout}`,
        search: "englishjobsearch.ch",
        source: "englishjobsearch.ch",
        contentChecked: hasBody,
      });
      kept++;
    }
    console.log(`${cards.length} ads, ${kept} kept`);
    await Bun.sleep(300);
  }

  return jobs;
}

// stellen-basel.ch — WordPress REST API
export async function fetchStellenBasel(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 1; page <= 30; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  page ${page} … `);

    const res = await fetch(
      `https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
      { headers: { ...HEADERS, Accept: "application/json" } }
    );
    if (!res.ok) { console.log("done"); break; }

    const data = (await res.json()) as Array<{
      id: number;
      title: { rendered: string };
      date: string;
      link: string;
      meta: { _company_name?: string; _job_location?: string };
    }>;

    if (!data.length) { console.log("0 items"); break; }

    let newOnPage = 0;
    for (const item of data) {
      const title = decodeHtml(item.title.rendered);
      if (!applyFilters(title)) continue;

      const id = `sb-${item.id}`;
      if (seen.has(id)) continue;
      seen.add(id);

      allJobs.push({
        id,
        title,
        company: item.meta._company_name ?? "",
        location: item.meta._job_location ?? "Basel",
        postedAt: item.date.slice(0, 10),
        link: item.link,
        search: "stellen-basel.ch",
        source: "stellen-basel.ch",
      });
      newOnPage++;
    }

    console.log(`${data.length} items, ${newOnPage} new`);
    if (data.length < 100) break;
  }

  return allJobs;
}

// basel-jobs.ch — regional board on the JobChannel network (same syndication
// network whose ?source=jobchannel links show up on Siemens/jobs.ch postings).
//
// The board is protected by DataDome, which is why this source sat documented
// as "not integrable" for months. The block is real but narrower than it looked:
//
//   - Plain curl / bun fetch → 403 on every UA, browser-like or not.
//   - Chrome --headless=new, driven over CDP → 403. Headless is detected.
//   - Playwright's own chromium.launch(), even headful → 403. The CDP-launch
//     path leaves automation artefacts DataDome fingerprints.
//   - A real Chrome launched *normally* (with --remote-debugging-port so we can
//     attach afterwards) → 200. Attaching over CDP after the fact is invisible.
//
// The unlock is that DataDome's clearance lives entirely in a `datadome` cookie
// (Max-Age one year) which replays fine over plain fetch with a matching
// User-Agent. So we mint a cookie once with a real browser window, cache it, and
// scrape normally thereafter — a browser is touched only when the cookie is
// missing or has been rejected, not on every run. An earlier note in CLAUDE.md
// claimed the cookie "can't be extracted for replay"; that was true of the
// browser-extension tooling, not of CDP, which hands it over directly.
//
// Operational caveat: minting needs a *visible* Chrome window for ~10s, so an
// unattended `bun daily` under launchd cannot mint. When the cookie is stale in
// that context the source logs and yields nothing rather than failing the run;
// refresh it deliberately with `bun run scripts/mint-datadome.ts`.

const DATADOME_CACHE = "results/.datadome.json";
const CHROME_BIN = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
const BASELJOBS_ORIGIN = "https://basel-jobs.ch";

interface DataDomeSession {
  cookie: string;
  ua: string;
  mintedAt: string;
}

async function loadDataDomeSession(): Promise<DataDomeSession | null> {
  try {
    const raw = await Bun.file(DATADOME_CACHE).text();
    const s = JSON.parse(raw) as DataDomeSession;
    return s.cookie && s.ua ? s : null;
  } catch {
    return null;
  }
}

async function waitForCdp(port: number, timeoutMs: number): Promise<boolean> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    try {
      const r = await fetch(`http://127.0.0.1:${port}/json/version`);
      if (r.ok) return true;
    } catch { /* not up yet */ }
    await Bun.sleep(300);
  }
  return false;
}

/**
 * Launch a real, visible Chrome, let it clear DataDome, then attach over CDP and
 * lift the resulting cookie. Headless and Playwright-launched browsers are both
 * detected — only a normally-launched window gets through, which is the whole
 * reason this is a separate, rarely-run step instead of the fetch path itself.
 */
export async function mintDataDomeSession(): Promise<DataDomeSession | null> {
  if (process.env.LINKME_NO_BROWSER === "1") {
    console.log("  LINKME_NO_BROWSER=1 — skipping DataDome mint");
    return null;
  }
  if (!(await Bun.file(CHROME_BIN).exists())) {
    console.error(`  Chrome not found at ${CHROME_BIN} — cannot mint DataDome cookie`);
    return null;
  }

  let chromium: typeof import("playwright-core").chromium;
  try {
    ({ chromium } = await import("playwright-core"));
  } catch {
    console.error("  playwright-core not installed — run `bun add playwright-core`");
    return null;
  }

  // A port unlikely to collide with a debugging session the user opened.
  const port = 9333;
  const profile = "results/.datadome-profile";
  const proc = Bun.spawn(
    [
      CHROME_BIN,
      `--remote-debugging-port=${port}`,
      `--user-data-dir=${profile}`,
      "--no-first-run",
      "--no-default-browser-check",
      "--window-size=1200,860",
      BASELJOBS_ORIGIN,
    ],
    { stdout: "ignore", stderr: "ignore" }
  );

  try {
    if (!(await waitForCdp(port, 20_000))) {
      console.error("  Chrome CDP endpoint never came up");
      return null;
    }
    const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
    const ctx = browser.contexts()[0];
    const page =
      ctx.pages().find((p) => p.url().includes("basel-jobs")) ?? ctx.pages()[0];

    // The interstitial resolves itself; give it room, then confirm we're through.
    let cookie = "";
    let ua = "";
    for (let attempt = 0; attempt < 12; attempt++) {
      await page.waitForTimeout(1500);
      const html = await page.content();
      if (/Please enable JS|geo\.captcha/.test(html)) continue;
      const cookies = await ctx.cookies(BASELJOBS_ORIGIN);
      const dd = cookies.find((c) => c.name === "datadome");
      if (!dd) continue;
      cookie = dd.value;
      ua = await page.evaluate(() => navigator.userAgent);
      break;
    }
    await browser.close();

    if (!cookie || !ua) {
      console.error("  DataDome challenge did not clear — no cookie minted");
      return null;
    }

    const session: DataDomeSession = { cookie, ua, mintedAt: new Date().toISOString() };
    await Bun.write(DATADOME_CACHE, JSON.stringify(session, null, 2));
    console.log(`  minted DataDome cookie (${cookie.length} chars)`);
    return session;
  } catch (e) {
    console.error(`  DataDome mint failed: ${e}`);
    return null;
  } finally {
    proc.kill();
  }
}

function baselJobsHeaders(s: DataDomeSession): Record<string, string> {
  return {
    "User-Agent": s.ua,
    "Accept-Language": "de-CH,de;q=0.9,en;q=0.8",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    Cookie: `datadome=${s.cookie}`,
  };
}

/**
 * Fetch with the cached cookie, re-minting once if the cookie has gone stale.
 * DataDome rotates the cookie on ordinary responses too, so a Set-Cookie on the
 * way past is folded back into the session rather than ignored.
 */
async function baselJobsFetch(
  url: string,
  state: { session: DataDomeSession | null; minted: boolean }
): Promise<string | null> {
  for (let attempt = 0; attempt < 2; attempt++) {
    if (!state.session) {
      if (state.minted) return null;
      state.minted = true;
      state.session = await mintDataDomeSession();
      if (!state.session) return null;
    }

    const res = await fetch(url, { headers: baselJobsHeaders(state.session) });

    const rotated = res.headers
      .getSetCookie?.()
      .map((c) => /(?:^|;\s*)datadome=([^;]+)/.exec(c)?.[1])
      .find(Boolean);
    if (rotated && state.session) {
      state.session = { ...state.session, cookie: rotated };
      await Bun.write(DATADOME_CACHE, JSON.stringify(state.session, null, 2));
    }

    if (res.ok) return res.text();

    if (res.status === 403 && !state.minted) {
      console.log("  cookie rejected — re-minting");
      state.session = null;
      continue;
    }
    console.error(`  failed (${res.status}) ${url}`);
    return null;
  }
  return null;
}

// Terms swept through the board's search. No keyword-union endpoint exists, so
// — like job-room.ch and jobbasel.ch — this is a fixed list, one request each
// per mode, deduped by vacancy id. German stems are in deliberately: the search
// matches description text, so "entwickler" surfaces postings whose English
// title the category never filed under IT. The title and description filters
// drop the German-language ones afterwards.
//
// Role words and stack words behave differently across the two search modes
// (see fetchBaselJobs), which is why both modes get the whole list rather than
// a tuned subset — `websearch=1` finds no stack terms at all, and the bare
// search is the only mode that matches "react"/"frontend"/"backend".
const BASELJOBS_TERMS = [
  // role / domain words — these carry the `websearch=1` pass
  "software", "engineer", "developer", "entwickler", "informatik", "data",
  "cloud", "devops", "platform", "architect", "security", "ai",
  // stack words — only the bare search matches these
  "python", "javascript", "typescript", "react", "node",
  "fullstack", "full stack", "backend", "frontend", "kubernetes", "linux",
];

function buildBaselJobsLocation(city: string, canton: string): string {
  if (!city) return canton || "Basel";
  if (!canton) return city;
  // data-gtm-jp is one or more codes ("BS", "BL,BS"); keep only what the city
  // string doesn't already name.
  const missing = canton
    .split(",")
    .map((c) => c.trim())
    .filter((c) => c && !new RegExp(`\\b${c}\\b`).test(city));
  return missing.length ? `${city}, ${missing.join(",")}` : city;
}

/**
 * Pull every vacancy card out of a listing page. Returns how many ids were
 * new so callers can use "no new postings" as their end condition — past the
 * last real page the board still serves its two sponsored slots, so an empty
 * list never arrives.
 */
function parseBaselJobsCards(
  html: string,
  seen: Set<string>,
  out: Job[],
  today: string
): { items: number; added: number } {
  const items = [...html.matchAll(/<li class="item"(.*?)<\/li>/gs)].map((m) => m[1]);
  let added = 0;

  for (const card of items) {
    const jid = /data-gtm-jid="(\d+)"/.exec(card)?.[1];
    const linkM = /href="(https:\/\/basel-jobs\.ch\/job\/[^"]+)"/.exec(card);
    const titleM = /<span class="jobtitle">(.*?)<\/span>/s.exec(card);
    if (!jid || !linkM || !titleM) continue;

    // Sponsored slots repeat the same posting within a page, between pages,
    // and across the category and search sweeps.
    const id = `bj-${jid}`;
    if (seen.has(id)) continue;
    seen.add(id);
    added++;

    const title = decodeHtml(titleM[1]);
    if (!applyFilters(title)) continue;

    const companyM =
      /<span class="company">\s*<a[^>]*>(.*?)<\/a>/s.exec(card) ??
      /<span class="company">(.*?)<\/span>/s.exec(card);
    const locationM = /<span class="location">(.*?)<\/span>/s.exec(card);
    const canton = /data-gtm-jp="([^"]*)"/.exec(card)?.[1] ?? "";

    out.push({
      id,
      title,
      company: companyM ? decodeHtml(companyM[1]) : "",
      // Cards carry a city; the canton attribute disambiguates BS vs BL,
      // which is what daily.ts groups on. Some cards already end in the
      // canton code ("Basel, BS, 4030") — don't append it twice.
      location: buildBaselJobsLocation(
        locationM ? decodeHtml(locationM[1]) : "",
        canton
      ),
      // No date anywhere on the card or detail page — discovery date is the
      // best available, same as any board that hides posting age.
      postedAt: today,
      link: linkM[1],
      search: "basel-jobs.ch",
      source: "basel-jobs.ch",
    });
  }

  return { items: items.length, added };
}

export async function fetchBaselJobs(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();
  const state = { session: await loadDataDomeSession(), minted: false };
  const today = new Date().toISOString().slice(0, 10);

  // Pass 1 — the board's IT category, ~300 postings over 3 pages.
  for (let page = 1; page <= 10; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  informatik page ${page} … `);

    const html = await baselJobsFetch(
      `${BASELJOBS_ORIGIN}/informatik-jobs?page=${page}`,
      state
    );
    if (html === null) { console.log("unavailable"); break; }

    const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
    if (items === 0) { console.log("0 items"); break; }
    console.log(`${items} items, ${added} new`);
    if (added === 0) break;
  }

  // Pass 2 — full-text search. The category is a *curated taxonomy*, not the
  // whole board: it only holds what the board filed under IT, so tech roles
  // parked under Industrie/Pharma/Bau, and roles whose relevance lives in the
  // description rather than the title, never appear in pass 1. Measured on
  // 2026-09-01 this pass added **147 postings the category missed**, including
  // "Senior AI Infrastructure & Platform Engineer (Basel)" and "Senior DevOps /
  // Cloud Engineer (Kubernetes, CI/CD, Linux)". The reverse is also true (the
  // category held 257 the search missed), so both passes are needed.
  //
  // `websearch=1` is the mode that does this — the bare `?terms=` search
  // returned nothing that wasn't already in the category. It ignores `page`
  // entirely and returns the whole result set on one response, so there is no
  // pagination to walk.
  // Both search modes are swept, because they query different indexes and each
  // is blind to what the other finds:
  //
  //   term        websearch=1   bare
  //   engineer         90        260 found
  //   frontend          0          2
  //   backend           0          4
  //   react             0          2
  //
  // `websearch=1` matches role/domain words and reaches postings filed outside
  // the IT category; the bare search is the *only* mode that matches stack
  // terms. Neither paginates — both ignore `page` and return their whole result
  // set in one response (bare caps around 100 cards). Measured 2026-09-01
  // against the category: websearch added 147, bare added a further 139 raw
  // (23 surviving the title filters).
  for (const mode of ["websearch", "bare"] as const) {
    for (const term of BASELJOBS_TERMS) {
      await Bun.sleep(400);
      process.stdout.write(`  ${mode} "${term}" … `);

      const qs = `terms=${encodeURIComponent(term)}${mode === "websearch" ? "&page=1&websearch=1" : ""}`;
      const html = await baselJobsFetch(`${BASELJOBS_ORIGIN}/search?${qs}`, state);
      if (html === null) { console.log("unavailable"); break; }

      const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
      console.log(`${items} items, ${added} new`);
    }
  }

  // Detail pages carry the full description inline, so the DE/FR and
  // German-requirement checks run here instead of costing index.ts a fetch that
  // would hit DataDome without the cookie and be dropped as unreachable.
  const viable: Job[] = [];
  for (const job of allJobs) {
    await Bun.sleep(300);
    process.stdout.write(`  checking ${job.title} … `);
    const page = await baselJobsFetch(job.link, state);
    if (page === null) { console.log("unreachable, keeping"); viable.push(job); continue; }

    // Only the posting's own text, never the page. basel-jobs.ch is a German
    // site, so its nav/footer/category chrome alone clears the ≥ 6 DE/FR word
    // threshold — checking the whole page dropped *every* job, English ones
    // included. The mobile `plain-content` block is the description as plain
    // text; the desktop view loads the same thing into an iframe, so this is
    // the only copy present in the server-rendered HTML.
    const descM = /<div class="box plain-content[^"]*">(.*?)<\/div>/s.exec(page);
    if (!descM) { console.log("no description, keeping"); viable.push(job); continue; }
    const check = isViableDescription(descM[1].replace(/<br\s*\/?>/gi, " "));
    if (!check.ok) { console.log(`dropped (${check.reason})`); continue; }
    console.log("ok");
    viable.push({ ...job, contentChecked: true });
  }

  return viable;
}