rodado
bcb6ae5ea3
- isEnglishContent() fetches each job page and drops descriptions with ≥6 German or French indicator words; called for every new job in index.ts before writing to CSV - New `bun recheck` command (recheck.ts) runs searches in raw mode, lists filtered-out jobs, and prints word-frequency suggestions for SOFTWARE_TITLE_RE - Expanded KEYWORDS and SOFTWARE_TITLE_RE to cover ML/AI, system/applications engineer, director, informatics, and more title variants - Added `raw` flag to fetchJobs() to bypass the software-role filter - Add CLAUDE.md with project workflow docs; add CV files and city temp scripts
66 lines
2.0 KiB
TypeScript
66 lines
2.0 KiB
TypeScript
#!/usr/bin/env bun
|
|
import { searches } from "./config.ts";
|
|
import { fetchJobs, SOFTWARE_TITLE_RE } from "./fetch.ts";
|
|
|
|
const CSV = "./jobs.csv";
|
|
|
|
const existingIds = new Set<string>();
|
|
try {
|
|
const text = await Bun.file(CSV).text();
|
|
for (const line of text.split("\n").slice(1)) {
|
|
const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
|
|
if (id) existingIds.add(id);
|
|
}
|
|
} catch { /* first run */ }
|
|
|
|
const missed: { title: string; company: string; link: string }[] = [];
|
|
const seenIds = new Set<string>();
|
|
|
|
for (const config of searches) {
|
|
console.log(`\n[${config.label}]`);
|
|
const all = await fetchJobs(config, { raw: true });
|
|
for (const job of all) {
|
|
if (!seenIds.has(job.id) && !existingIds.has(job.id) && !SOFTWARE_TITLE_RE.test(job.title)) {
|
|
seenIds.add(job.id);
|
|
missed.push({ title: job.title, company: job.company, link: job.link });
|
|
}
|
|
}
|
|
}
|
|
|
|
if (missed.length === 0) {
|
|
console.log("\nNo missed jobs — all new titles pass the filter.");
|
|
process.exit(0);
|
|
}
|
|
|
|
console.log(`\n── ${missed.length} new jobs currently filtered out ──\n`);
|
|
for (const j of missed) {
|
|
console.log(` ${j.title}`);
|
|
console.log(` ${j.company} ${j.link}`);
|
|
}
|
|
|
|
// Word frequency analysis to suggest keywords
|
|
const STOP = new Set([
|
|
"and", "or", "of", "the", "a", "an", "in", "for", "to", "at", "by",
|
|
"with", "on", "as", "is", "its", "be", "are", "was", "were", "our",
|
|
"your", "this", "that", "we", "you", "all", "new", "use",
|
|
]);
|
|
|
|
const wordCounts = new Map<string, number>();
|
|
for (const { title } of missed) {
|
|
const words = title.match(/\b[a-zA-Z]{3,}\b/g) ?? [];
|
|
for (const w of words) {
|
|
const key = w.toLowerCase();
|
|
if (!STOP.has(key)) wordCounts.set(key, (wordCounts.get(key) ?? 0) + 1);
|
|
}
|
|
}
|
|
|
|
const suggestions = [...wordCounts.entries()]
|
|
.sort((a, b) => b[1] - a[1])
|
|
.filter(([, n]) => n >= 1)
|
|
.slice(0, 20);
|
|
|
|
console.log("\n── Suggested keywords to consider ──\n");
|
|
for (const [word, count] of suggestions) {
|
|
console.log(` "${word}" — ${count} title${count > 1 ? "s" : ""}`);
|
|
}
|