linkme/index.ts

#!/usr/bin/env bun
import { searches } from "./config.ts";
import { fetchJobs, isEnglishContent } from "./fetch.ts";
import { fetchDataCareer, fetchStellenBasel } from "./fetch-other.ts";

const CSV = "./results/jobs.csv";
const OLD_HEADER = "date_found,posted_at,search,title,company,link";
const HEADER = "date_found,posted_at,search,title,company,link,source";

// Migrate CSV: add source column if the old header is present
const csvExists = await Bun.file(CSV).exists();
if (csvExists) {
  const raw = await Bun.file(CSV).text();
  if (raw.startsWith(OLD_HEADER + "\n") || raw === OLD_HEADER) {
    const lines = raw.split("\n");
    const migrated = [
      HEADER,
      ...lines.slice(1).map((l) => (l.trim() ? l + ',"linkedin.com"' : "")),
    ]
      .join("\n")
      .replace(/\n+$/, "") + "\n";
    await Bun.write(CSV, migrated);
  }
}

// Load existing links for deduplication
const existingLinks = new Set<string>();
try {
  const text = await Bun.file(CSV).text();
  for (const line of text.split("\n").slice(1)) {
    const url = line.match(/"(https?:\/\/[^"]+)"/)?.[1];
    if (url) existingLinks.add(url);
  }
} catch {
  // first run
}

const date = new Date().toISOString().slice(0, 10);
const newRows: string[] = [];

function csvCell(val: string): string {
  return `"${val.replace(/"/g, '""')}"`;
}

// LinkedIn searches
for (const config of searches) {
  console.log(`\n[${config.label}]`);
  const jobs = await fetchJobs(config);

  if (jobs.length === 0) {
    console.log("  No jobs found.");
    continue;
  }

  let added = 0;
  let langDropped = 0;
  for (const job of jobs) {
    if (!existingLinks.has(job.link)) {
      process.stdout.write(`  checking content: ${job.title} … `);
      const english = await isEnglishContent(job.id);
      if (!english) {
        console.log("dropped (non-English)");
        langDropped++;
        continue;
      }
      console.log("ok");
      existingLinks.add(job.link);
      newRows.push(
        [date, job.postedAt, config.label, job.title, job.company, job.link, "linkedin.com"]
          .map(csvCell)
          .join(",")
      );
      added++;
    }
  }
  console.log(`  ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
}

// Extra sources (no isEnglishContent — title filters already applied)
const extraSources: Array<{ label: string; fetch: () => Promise<ReturnType<typeof fetchDataCareer>> }> = [
  { label: "datacareer.ch", fetch: fetchDataCareer },
  { label: "stellen-basel.ch", fetch: fetchStellenBasel },
];

for (const { label, fetch: fetchFn } of extraSources) {
  console.log(`\n[${label}]`);
  let jobs;
  try {
    jobs = await fetchFn();
  } catch (e) {
    console.error(`  error: ${e}`);
    continue;
  }

  if (jobs.length === 0) {
    console.log("  No jobs found.");
    continue;
  }

  let added = 0;
  for (const job of jobs) {
    if (!existingLinks.has(job.link)) {
      existingLinks.add(job.link);
      newRows.push(
        [date, job.postedAt, job.search, job.title, job.company, job.link, job.source]
          .map(csvCell)
          .join(",")
      );
      added++;
    }
  }
  console.log(`  ${jobs.length} found, ${added} new`);
}

if (newRows.length === 0) {
  console.log("\nNo new jobs.");
  process.exit(0);
}

// Append to CSV
const existing = await Bun.file(CSV).exists();
const append = existing
  ? await Bun.file(CSV).text()
  : HEADER + "\n";

await Bun.write(CSV, append + newRows.join("\n") + "\n");

console.log(`\n${newRows.length} new jobs → ${CSV}`);