rodado/linkme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add novartis.com and more-jobs.ch as job sources

Browse Source 
more-jobs.ch lists foreign postings tagged as CH, so drop locations naming
another country/city or a 5-digit postcode, and dedupe its reposts by
title+company.
This commit is contained in:
rodado
2026-10-03 12:47:31 +02:00
parent 3875f13713
commit dfba00ae91
5 changed files with 289 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.ts
+20 -4
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bun
import { searches } from "./config.ts";
import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts";
import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch, fetchBaselJobs, fetchSciCore, fetchSwisscom } from "./fetch-other.ts";
import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch, fetchBaselJobs, fetchSciCore, fetchSwisscom, fetchNovartisBasel, fetchMoreJobs } from "./fetch-other.ts";
const CSV = "./results/jobs.csv";
const OLD_HEADER = "date_found,posted_at,search,title,company,link";
@@ -47,12 +47,17 @@ if (csvExists) {
// posting on every run (found 2026-09-08: 4 duplicate rows for one link,
// added on both 09-07 and 09-08).
const existingLinks = new Set<string>();
// Normalized title+company of every row already in the CSV, for aggregator
// sources that repost jobs other sources carry (see `dedupeByTitle`).
const knownJobKeys = new Set<string>();
try {
const text = await Bun.file(CSV).text();
for (const line of text.split("\n").slice(1)) {
if (!line.trim()) continue;
const link = line.replace(/^"|"$/g, "").split('","')[5];
const cells = line.replace(/^"|"$/g, "").split('","');
const link = cells[5];
if (link) existingLinks.add(link);
if (cells[3]) knownJobKeys.add(langKey(cells[3], cells[4] ?? ""));
}
} catch {
// first run
@@ -159,6 +164,7 @@ async function processLinkedInSearch(config: (typeof searches)[number]): Promise
continue;
}
console.log("ok");
knownJobKeys.add(langKey(job.title, job.company));
newRows.push(
[date, job.postedAt, config.label, job.title, job.company, job.link, "linkedin.com", job.location]
.map(csvCell)
@@ -169,7 +175,7 @@ async function processLinkedInSearch(config: (typeof searches)[number]): Promise
console.log(` [${config.label}] ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`);
}
const extraSources: Array<{ label: string; fetch: () => ReturnType<typeof fetchDataCareer> }> = [
const extraSources: Array<{ label: string; fetch: () => ReturnType<typeof fetchDataCareer>; dedupeByTitle?: boolean }> = [
{ label: "datacareer.ch", fetch: fetchDataCareer },
{ label: "stellen-basel.ch", fetch: fetchStellenBasel },
{ label: "itjobs.ch", fetch: fetchItJobsCh },
@@ -180,9 +186,13 @@ const extraSources: Array<{ label: string; fetch: () => ReturnType<typeof fetchD
{ label: "basel-jobs.ch", fetch: fetchBaselJobs },
{ label: "scicore.unibas.ch", fetch: fetchSciCore },
{ label: "swisscom.com", fetch: fetchSwisscom },
{ label: "novartis.com", fetch: fetchNovartisBasel },
// Aggregator: reposts what LinkedIn/jobs.ch/etc. already carry, so also skip
// anything whose normalized title+company is already in the CSV.
{ label: "more-jobs.ch", fetch: fetchMoreJobs, dedupeByTitle: true },
];
async function processExtraSource({ label, fetch: fetchFn }: (typeof extraSources)[number]): Promise<void> {
async function processExtraSource({ label, fetch: fetchFn, dedupeByTitle }: (typeof extraSources)[number]): Promise<void> {
console.log(`\n[${label}]`);
let jobs;
try {
@@ -202,6 +212,11 @@ async function processExtraSource({ label, fetch: fetchFn }: (typeof extraSource
for (const job of jobs) {
if (existingLinks.has(job.link)) continue;
existingLinks.add(job.link); // claim before the await — see note above
if (dedupeByTitle) {
const key = langKey(job.title, job.company);
if (knownJobKeys.has(key)) continue;
knownJobKeys.add(key); // same synchronous-claim rule as the link
}
if (!job.contentChecked) {
process.stdout.write(` [${label}] checking content: ${job.title} … `);
@@ -218,6 +233,7 @@ async function processExtraSource({ label, fetch: fetchFn }: (typeof extraSource
}
console.log("ok");
}
knownJobKeys.add(langKey(job.title, job.company));
newRows.push(
[date, job.postedAt, job.search, job.title, job.company, job.link, job.source, job.location]
.map(csvCell)