feat: add role-family sweeps and domain/language filters
Broaden the search beyond engineer/developer titles and stop three
classes of bad result from ever reaching the CSV.
config.ts
- PLATFORM_KEYWORDS (architect, SRE, infra, security, MLOps),
RESEARCH_KEYWORDS (research/scientific software, HPC, data platform)
and FIELD_KEYWORDS (forward-deployed, customer/sales, founding).
Separate OR-strings, not folded into KEYWORDS: LinkedIn ranks a
focused query far better than one 50-term disjunction.
- sweep() generates one search per area per family. 7 searches -> 17.
fetch.ts
- BIO_DOMAIN_RE drops life-science roles (bioinformatics, biostats,
GMP/MSAT, genomics, pathology, drug discovery). Filters on the role,
never the employer: software jobs at Roche/Novartis/sciCORE stay.
- requiresGerman() drops English postings demanding German above B1.
Two rules, both from real false results: the nearest marker wins
("Infrastructure skills are a plus" sat 95 chars before "Good
knowledge of German"), and an intervening language name breaks the
binding ("Fluent in English; German and/or French advantageous").
- GERMAN_TITLE_RE catches "(German-speaking)" without a page fetch.
- isViableContent/isViableUrl run the DE/FR and German-requirement
checks on a single fetch; isEnglishContent/isEnglishUrl kept for
recheck.ts and temp scripts.
- SOFTWARE_TITLE_RE widened so the new families survive it.
index.ts
- both content-check paths use the combined check and report which
filter dropped a job.
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bun
|
||||
import { searches } from "./config.ts";
|
||||
import { fetchJobs, isEnglishContent } from "./fetch.ts";
|
||||
import { fetchDataCareer, fetchStellenBasel } from "./fetch-other.ts";
|
||||
import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts";
|
||||
import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs } from "./fetch-other.ts";
|
||||
|
||||
const CSV = "./results/jobs.csv";
|
||||
const OLD_HEADER = "date_found,posted_at,search,title,company,link";
|
||||
@@ -57,9 +57,11 @@ for (const config of searches) {
|
||||
for (const job of jobs) {
|
||||
if (!existingLinks.has(job.link)) {
|
||||
process.stdout.write(` checking content: ${job.title} … `);
|
||||
const english = await isEnglishContent(job.id);
|
||||
if (!english) {
|
||||
console.log("dropped (non-English)");
|
||||
// One fetch, two checks: written in German/French, or written in English
|
||||
// but demanding German above B1.
|
||||
const viable = await isViableContent(job.id);
|
||||
if (!viable.ok) {
|
||||
console.log(`dropped (${viable.reason})`);
|
||||
langDropped++;
|
||||
continue;
|
||||
}
|
||||
@@ -76,10 +78,12 @@ for (const config of searches) {
|
||||
console.log(` ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
|
||||
}
|
||||
|
||||
// Extra sources (no isEnglishContent — title filters already applied)
|
||||
const extraSources: Array<{ label: string; fetch: () => Promise<ReturnType<typeof fetchDataCareer>> }> = [
|
||||
{ label: "datacareer.ch", fetch: fetchDataCareer },
|
||||
{ label: "stellen-basel.ch", fetch: fetchStellenBasel },
|
||||
{ label: "itjobs.ch", fetch: fetchItJobsCh },
|
||||
{ label: "job-room.ch", fetch: fetchJobRoom },
|
||||
{ label: "basel intl orgs", fetch: fetchBaselIntlOrgs },
|
||||
];
|
||||
|
||||
for (const { label, fetch: fetchFn } of extraSources) {
|
||||
@@ -98,8 +102,19 @@ for (const { label, fetch: fetchFn } of extraSources) {
|
||||
}
|
||||
|
||||
let added = 0;
|
||||
let langDropped = 0;
|
||||
for (const job of jobs) {
|
||||
if (!existingLinks.has(job.link)) {
|
||||
if (!job.contentChecked) {
|
||||
process.stdout.write(` checking content: ${job.title} … `);
|
||||
const viable = await isViableUrl(job.link);
|
||||
if (!viable.ok) {
|
||||
console.log(`dropped (${viable.reason})`);
|
||||
langDropped++;
|
||||
continue;
|
||||
}
|
||||
console.log("ok");
|
||||
}
|
||||
existingLinks.add(job.link);
|
||||
newRows.push(
|
||||
[date, job.postedAt, job.search, job.title, job.company, job.link, job.source]
|
||||
@@ -109,7 +124,7 @@ for (const { label, fetch: fetchFn } of extraSources) {
|
||||
added++;
|
||||
}
|
||||
}
|
||||
console.log(` ${jobs.length} found, ${added} new`);
|
||||
console.log(` ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
|
||||
}
|
||||
|
||||
if (newRows.length === 0) {
|
||||
|
||||
Reference in New Issue
Block a user