Add content-level language filter, recheck command, expanded keywords
- isEnglishContent() fetches each job page and drops descriptions with ≥6 German or French indicator words; called for every new job in index.ts before writing to CSV - New `bun recheck` command (recheck.ts) runs searches in raw mode, lists filtered-out jobs, and prints word-frequency suggestions for SOFTWARE_TITLE_RE - Expanded KEYWORDS and SOFTWARE_TITLE_RE to cover ML/AI, system/applications engineer, director, informatics, and more title variants - Added `raw` flag to fetchJobs() to bypass the software-role filter - Add CLAUDE.md with project workflow docs; add CV files and city temp scripts
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bun
|
||||
import { searches } from "./config.ts";
|
||||
import { fetchJobs } from "./fetch.ts";
|
||||
import { fetchJobs, isEnglishContent } from "./fetch.ts";
|
||||
|
||||
const CSV = "./jobs.csv";
|
||||
const HEADER = "date_found,posted_at,search,title,company,link";
|
||||
@@ -34,8 +34,17 @@ for (const config of searches) {
|
||||
}
|
||||
|
||||
let added = 0;
|
||||
let langDropped = 0;
|
||||
for (const job of jobs) {
|
||||
if (!existingIds.has(job.id)) {
|
||||
process.stdout.write(` checking content: ${job.title} … `);
|
||||
const english = await isEnglishContent(job.id);
|
||||
if (!english) {
|
||||
console.log("dropped (non-English)");
|
||||
langDropped++;
|
||||
continue;
|
||||
}
|
||||
console.log("ok");
|
||||
existingIds.add(job.id);
|
||||
newRows.push(
|
||||
[date, job.postedAt, config.label, job.title, job.company, job.link]
|
||||
@@ -45,7 +54,7 @@ for (const config of searches) {
|
||||
added++;
|
||||
}
|
||||
}
|
||||
console.log(` ${jobs.length} found, ${added} new`);
|
||||
console.log(` ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
|
||||
}
|
||||
|
||||
if (newRows.length === 0) {
|
||||
|
||||
Reference in New Issue
Block a user