#!/usr/bin/env bun
import { searches } from "./config.ts";
import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts";
import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch, fetchBaselJobs, fetchSciCore, fetchSwisscom, fetchNovartisBasel, fetchMoreJobs } from "./fetch-other.ts";

const CSV = "./results/jobs.csv";
const OLD_HEADER = "date_found,posted_at,search,title,company,link";
const HEADER_NO_LOCATION = "date_found,posted_at,search,title,company,link,source";
const HEADER = "date_found,posted_at,search,title,company,link,source,location";

// Migrate CSV: add source column if the old header is present, then location
// (the posting's real location, as parsed off its own card — distinct from
// `search`, which is just the query that found it. See CLAUDE.md.)
const csvExists = await Bun.file(CSV).exists();
if (csvExists) {
  const raw = await Bun.file(CSV).text();
  if (raw.startsWith(OLD_HEADER + "\n") || raw === OLD_HEADER) {
    const lines = raw.split("\n");
    const migrated = [
      HEADER_NO_LOCATION,
      ...lines.slice(1).map((l) => (l.trim() ? l + ',"linkedin.com"' : "")),
    ]
      .join("\n")
      .replace(/\n+$/, "") + "\n";
    await Bun.write(CSV, migrated);
  }
}
{
  const raw = await Bun.file(CSV).text().catch(() => "");
  if (raw.startsWith(HEADER_NO_LOCATION + "\n") || raw === HEADER_NO_LOCATION) {
    const lines = raw.split("\n");
    const migrated = [
      HEADER,
      // Unknown for pre-existing rows — daily.ts falls back to `search` when empty.
      ...lines.slice(1).map((l) => (l.trim() ? l + ',""' : "")),
    ]
      .join("\n")
      .replace(/\n+$/, "") + "\n";
    await Bun.write(CSV, migrated);
  }
}

// Load existing links for deduplication. The link column (6th field) is
// pulled by position, not by an http(s):// match — job-room.ch/jobs.ch links
// are stored without a protocol (e.g. "www.jobs.ch/de/..."), so a URL-pattern
// match silently skipped every one of them and re-added the same jobs.ch
// posting on every run (found 2026-09-08: 4 duplicate rows for one link,
// added on both 09-07 and 09-08).
const existingLinks = new Set<string>();
// Normalized title+company of every row already in the CSV, for aggregator
// sources that repost jobs other sources carry (see `dedupeByTitle`).
const knownJobKeys = new Set<string>();
try {
  const text = await Bun.file(CSV).text();
  for (const line of text.split("\n").slice(1)) {
    if (!line.trim()) continue;
    const cells = line.replace(/^"|"$/g, "").split('","');
    const link = cells[5];
    if (link) existingLinks.add(link);
    if (cells[3]) knownJobKeys.add(langKey(cells[3], cells[4] ?? ""));
  }
} catch {
  // first run
}

const date = new Date().toISOString().slice(0, 10);
const newRows: string[] = [];
let totalLangDropped = 0; // jobs dropped for German/French content — never reach the CSV, so daily.ts can't see them on its own

// Persisted across runs so the Telegram digest's "Ignored (German/French)"
// count reflects genuinely new drops, not the same posting getting re-flagged
// every run (dropped jobs never reach the CSV, so nothing else remembers
// them) or the same underlying job counted again each time it resurfaces
// under a different link/provider (e.g. Consult & Pepper's "Senior Fullstack
// Software Engineer" reposted on job-room.ch every few days, and separately
// on LinkedIn) — keyed on normalized title+company rather than the link.
const LANG_SEEN = "./results/.lang-dropped-seen.json";
const langSeen = new Set<string>(
  await Bun.file(LANG_SEEN)
    .json()
    .catch(() => [])
);
function langKey(title: string, company: string): string {
  const t = title
    .replace(/[([][^)\]]*[)\]]/g, "") // drop "(all genders)", "(m/w/d)", "(a)", "(80 – 100%)" noise
    .replace(/\s+/g, " ")
    .trim()
    .toLowerCase();
  return `${t}|${company.trim().toLowerCase()}`;
}
// Only counts (and remembers) a drop the first time this title+company is
// seen; still runs for every dropped job so `langSeen` stays complete.
function countLangDrop(title: string, company: string): void {
  const key = langKey(title, company);
  if (langSeen.has(key)) return;
  langSeen.add(key);
  totalLangDropped++;
}

function csvCell(val: string): string {
  return `"${val.replace(/"/g, '""')}"`;
}

// Every search/source below hits a different host (LinkedIn, job-room.ch,
// jobbasel.ch, itjobs.ch, unjobs.org, ...), so there's no reason they need to
// run one after another — that's what made a full run take 30-40+ minutes.
// Run them concurrently instead, bounded per group so we don't burst LinkedIn
// (unauthenticated scraping, sensitive to anti-bot heuristics — see the
// auth warning in CLAUDE.md) or open too many simultaneous connections to
// the smaller extra sources at once.
//
// `existingLinks` is claimed synchronously (`.add()` right after the `.has()`
// check, before any `await`) rather than after the content-check — since Bun
// is single-threaded and only yields at `await`, that ordering is what makes
// concurrent workers safe: two sources racing on the same link can't both
// slip past the check before either claims it.
//
// Trade-off: each source still logs its own page-by-page progress from
// inside fetch.ts/fetch-other.ts, so concurrent sources' output interleaves
// in the terminal instead of printing in clean sequential blocks. Cutting
// total runtime was the actual complaint; log readability is secondary and
// still traceable (grep the source label).
async function runWithConcurrency<T>(items: T[], limit: number, worker: (item: T) => Promise<void>): Promise<void> {
  const queue = [...items];
  async function runner() {
    while (queue.length) {
      const item = queue.shift()!;
      await worker(item);
    }
  }
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runner));
}

async function processLinkedInSearch(config: (typeof searches)[number]): Promise<void> {
  console.log(`\n[${config.label}]`);
  const jobs = await fetchJobs(config);

  if (jobs.length === 0) {
    console.log(`  [${config.label}] No jobs found.`);
    return;
  }

  let added = 0;
  let langDropped = 0;
  for (const job of jobs) {
    if (existingLinks.has(job.link)) continue;
    existingLinks.add(job.link); // claim before the await — see note above

    process.stdout.write(`  [${config.label}] checking content: ${job.title} … `);
    // One fetch, two checks: written in German/French, or written in English
    // but demanding German above B1.
    const viable = await isViableContent(job.id);
    if (viable.retry) {
      // Couldn't read the description at all — leave it out of the CSV so
      // next run's fresh fetch judges it properly, instead of defaulting to
      // "keep" and risking a German-requiring posting slipping through.
      console.log(viable.reason);
      continue;
    }
    if (!viable.ok) {
      console.log(`dropped (${viable.reason})`);
      langDropped++;
      countLangDrop(job.title, job.company);
      continue;
    }
    console.log("ok");
    knownJobKeys.add(langKey(job.title, job.company));
    newRows.push(
      [date, job.postedAt, config.label, job.title, job.company, job.link, "linkedin.com", job.location]
        .map(csvCell)
        .join(",")
    );
    added++;
  }
  console.log(`  [${config.label}] ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
}

const extraSources: Array<{ label: string; fetch: () => ReturnType<typeof fetchDataCareer>; dedupeByTitle?: boolean }> = [
  { label: "datacareer.ch", fetch: fetchDataCareer },
  { label: "stellen-basel.ch", fetch: fetchStellenBasel },
  { label: "itjobs.ch", fetch: fetchItJobsCh },
  { label: "job-room.ch", fetch: fetchJobRoom },
  { label: "basel intl orgs", fetch: fetchBaselIntlOrgs },
  { label: "jobbasel.ch", fetch: fetchJobBasel },
  { label: "englishjobsearch.ch", fetch: fetchEnglishJobSearch },
  { label: "basel-jobs.ch", fetch: fetchBaselJobs },
  { label: "scicore.unibas.ch", fetch: fetchSciCore },
  { label: "swisscom.com", fetch: fetchSwisscom },
  { label: "novartis.com", fetch: fetchNovartisBasel },
  // Aggregator: reposts what LinkedIn/jobs.ch/etc. already carry, so also skip
  // anything whose normalized title+company is already in the CSV.
  { label: "more-jobs.ch", fetch: fetchMoreJobs, dedupeByTitle: true },
];

async function processExtraSource({ label, fetch: fetchFn, dedupeByTitle }: (typeof extraSources)[number]): Promise<void> {
  console.log(`\n[${label}]`);
  let jobs;
  try {
    jobs = await fetchFn();
  } catch (e) {
    console.error(`  [${label}] error: ${e}`);
    return;
  }

  if (jobs.length === 0) {
    console.log(`  [${label}] No jobs found.`);
    return;
  }

  let added = 0;
  let langDropped = 0;
  for (const job of jobs) {
    if (existingLinks.has(job.link)) continue;
    existingLinks.add(job.link); // claim before the await — see note above
    if (dedupeByTitle) {
      const key = langKey(job.title, job.company);
      if (knownJobKeys.has(key)) continue;
      knownJobKeys.add(key); // same synchronous-claim rule as the link
    }

    if (!job.contentChecked) {
      process.stdout.write(`  [${label}] checking content: ${job.title} … `);
      const viable = await isViableUrl(job.link);
      if (viable.retry) {
        console.log(viable.reason);
        continue;
      }
      if (!viable.ok) {
        console.log(`dropped (${viable.reason})`);
        langDropped++;
        countLangDrop(job.title, job.company);
        continue;
      }
      console.log("ok");
    }
    knownJobKeys.add(langKey(job.title, job.company));
    newRows.push(
      [date, job.postedAt, job.search, job.title, job.company, job.link, job.source, job.location]
        .map(csvCell)
        .join(",")
    );
    added++;
  }
  console.log(`  [${label}] ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
}

const LINKEDIN_CONCURRENCY = 3;
const EXTRA_SOURCE_CONCURRENCY = 3;

await Promise.all([
  runWithConcurrency(searches, LINKEDIN_CONCURRENCY, processLinkedInSearch),
  runWithConcurrency(extraSources, EXTRA_SOURCE_CONCURRENCY, processExtraSource),
]);

// Jobs dropped for German/French content never reach the CSV, so this is the
// only record of them — daily.ts reads it back to show a count in the digest
// without listing rejects that were never even parsed into a Row.
const LANG_DROPPED = "./results/.lang-dropped.json";
await Bun.write(LANG_DROPPED, JSON.stringify({ date, count: totalLangDropped }));
await Bun.write(LANG_SEEN, JSON.stringify([...langSeen]));

if (newRows.length === 0) {
  console.log("\nNo new jobs.");
  process.exit(0);
}

// Append to CSV
const existing = await Bun.file(CSV).exists();
const append = existing
  ? await Bun.file(CSV).text()
  : HEADER + "\n";

await Bun.write(CSV, append + newRows.join("\n") + "\n");

console.log(`\n${newRows.length} new jobs → ${CSV}`);