Add content-level language filter, recheck command, expanded keywords
- isEnglishContent() fetches each job page and drops descriptions with ≥6 German or French indicator words; called for every new job in index.ts before writing to CSV - New `bun recheck` command (recheck.ts) runs searches in raw mode, lists filtered-out jobs, and prints word-frequency suggestions for SOFTWARE_TITLE_RE - Expanded KEYWORDS and SOFTWARE_TITLE_RE to cover ML/AI, system/applications engineer, director, informatics, and more title variants - Added `raw` flag to fetchJobs() to bypass the software-role filter - Add CLAUDE.md with project workflow docs; add CV files and city temp scripts
This commit is contained in:
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bun
|
||||
import { searches } from "./config.ts";
|
||||
import { fetchJobs, SOFTWARE_TITLE_RE } from "./fetch.ts";
|
||||
|
||||
const CSV = "./jobs.csv";
|
||||
|
||||
const existingIds = new Set<string>();
|
||||
try {
|
||||
const text = await Bun.file(CSV).text();
|
||||
for (const line of text.split("\n").slice(1)) {
|
||||
const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
|
||||
if (id) existingIds.add(id);
|
||||
}
|
||||
} catch { /* first run */ }
|
||||
|
||||
const missed: { title: string; company: string; link: string }[] = [];
|
||||
const seenIds = new Set<string>();
|
||||
|
||||
for (const config of searches) {
|
||||
console.log(`\n[${config.label}]`);
|
||||
const all = await fetchJobs(config, { raw: true });
|
||||
for (const job of all) {
|
||||
if (!seenIds.has(job.id) && !existingIds.has(job.id) && !SOFTWARE_TITLE_RE.test(job.title)) {
|
||||
seenIds.add(job.id);
|
||||
missed.push({ title: job.title, company: job.company, link: job.link });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (missed.length === 0) {
|
||||
console.log("\nNo missed jobs — all new titles pass the filter.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log(`\n── ${missed.length} new jobs currently filtered out ──\n`);
|
||||
for (const j of missed) {
|
||||
console.log(` ${j.title}`);
|
||||
console.log(` ${j.company} ${j.link}`);
|
||||
}
|
||||
|
||||
// Word frequency analysis to suggest keywords
|
||||
const STOP = new Set([
|
||||
"and", "or", "of", "the", "a", "an", "in", "for", "to", "at", "by",
|
||||
"with", "on", "as", "is", "its", "be", "are", "was", "were", "our",
|
||||
"your", "this", "that", "we", "you", "all", "new", "use",
|
||||
]);
|
||||
|
||||
const wordCounts = new Map<string, number>();
|
||||
for (const { title } of missed) {
|
||||
const words = title.match(/\b[a-zA-Z]{3,}\b/g) ?? [];
|
||||
for (const w of words) {
|
||||
const key = w.toLowerCase();
|
||||
if (!STOP.has(key)) wordCounts.set(key, (wordCounts.get(key) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const suggestions = [...wordCounts.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.filter(([, n]) => n >= 1)
|
||||
.slice(0, 20);
|
||||
|
||||
console.log("\n── Suggested keywords to consider ──\n");
|
||||
for (const [word, count] of suggestions) {
|
||||
console.log(` "${word}" — ${count} title${count > 1 ? "s" : ""}`);
|
||||
}
|
||||
Reference in New Issue
Block a user