#!/usr/bin/env bun
import { searches } from "./config.ts";
import { fetchJobs, SOFTWARE_TITLE_RE } from "./fetch.ts";

const CSV = "./jobs.csv";

const existingIds = new Set<string>();
try {
  const text = await Bun.file(CSV).text();
  for (const line of text.split("\n").slice(1)) {
    const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
    if (id) existingIds.add(id);
  }
} catch { /* first run */ }

const missed: { title: string; company: string; link: string }[] = [];
const seenIds = new Set<string>();

for (const config of searches) {
  console.log(`\n[${config.label}]`);
  const all = await fetchJobs(config, { raw: true });
  for (const job of all) {
    if (!seenIds.has(job.id) && !existingIds.has(job.id) && !SOFTWARE_TITLE_RE.test(job.title)) {
      seenIds.add(job.id);
      missed.push({ title: job.title, company: job.company, link: job.link });
    }
  }
}

if (missed.length === 0) {
  console.log("\nNo missed jobs — all new titles pass the filter.");
  process.exit(0);
}

console.log(`\n── ${missed.length} new jobs currently filtered out ──\n`);
for (const j of missed) {
  console.log(`  ${j.title}`);
  console.log(`    ${j.company}  ${j.link}`);
}

// Word frequency analysis to suggest keywords
const STOP = new Set([
  "and", "or", "of", "the", "a", "an", "in", "for", "to", "at", "by",
  "with", "on", "as", "is", "its", "be", "are", "was", "were", "our",
  "your", "this", "that", "we", "you", "all", "new", "use",
]);

const wordCounts = new Map<string, number>();
for (const { title } of missed) {
  const words = title.match(/\b[a-zA-Z]{3,}\b/g) ?? [];
  for (const w of words) {
    const key = w.toLowerCase();
    if (!STOP.has(key)) wordCounts.set(key, (wordCounts.get(key) ?? 0) + 1);
  }
}

const suggestions = [...wordCounts.entries()]
  .sort((a, b) => b[1] - a[1])
  .filter(([, n]) => n >= 1)
  .slice(0, 20);

console.log("\n── Suggested keywords to consider ──\n");
for (const [word, count] of suggestions) {
  console.log(`  "${word}"  — ${count} title${count > 1 ? "s" : ""}`);
}