rodado
0249800278
isViableContent/isViableUrl defaulted to keep a job when its description
couldn't be fetched at all -- that's what let a fully-German HRM
Systems/Consult & Pepper posting ("In Deutsch bist du verhandlungssicher")
through on 2026-09-10. They now report retry:true instead, and index.ts
leaves the job out of the CSV so a fresh fetch judges it next run rather
than defaulting to keep.
job-room.ch's search API also returns a different keyword-anchored teaser
per hit for the same ad, and only the last one was kept -- a genuinely
German posting could slip through whenever the last keyword's teaser
happened to read as English. Every teaser variant is now collected and
checked together.
1156 lines
46 KiB
TypeScript
1156 lines
46 KiB
TypeScript
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
|
import type { Job } from "./fetch.ts";
|
|
|
|
const HEADERS = {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
};
|
|
|
|
// datacareer.ch inverts the usual bot rule: its nginx 403s anything claiming to
|
|
// be a browser (a spoofed Chrome UA is the one thing guaranteed to fail) while
|
|
// letting honest clients — no UA, curl, Googlebot — straight through. Presumably
|
|
// real browser traffic arrives via a path that also carries matching TLS/CH
|
|
// fingerprints, so a bare Chrome UA over plain HTTP reads as a scraper. Sending
|
|
// an honest identifier is both what unblocks it and the polite thing to do.
|
|
// This is what killed the source for months; it was never a hard block.
|
|
const PLAIN_HEADERS = {
|
|
"User-Agent": "linkme-jobbot/1.0 (personal job search; contact via basel)",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
};
|
|
|
|
function decodeHtml(s: string): string {
|
|
return s
|
|
.replace(/&/g, "&")
|
|
.replace(/</g, "<")
|
|
.replace(/>/g, ">")
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
function applyFilters(title: string): boolean {
|
|
if (NON_ENGLISH_RE.test(title)) return false;
|
|
if (JUNIOR_TITLE_RE.test(title)) return false;
|
|
if (!SOFTWARE_TITLE_RE.test(title)) return false;
|
|
return true;
|
|
}
|
|
|
|
const MS_STACK_RE = /\.net\b|\bc#|sharepoint|dynamics 365|power platform|\bsccm\b|intune|successfactors|\bsap\b/i;
|
|
|
|
// DE/FR indicator words, counted against a description we already have in hand.
|
|
// Mirrors the ≥ 6 threshold isEnglishContent uses on fetched pages — but see
|
|
// the adaptive threshold below for why short snippets need a lower bar.
|
|
// du/dein*/dich/dir/einem/einer/eines/der/dem added 2026-09-08: job-room's
|
|
// search API "Senior Fullstack Software Engineer" (Consult & Pepper) teaser —
|
|
// "Du arbeitest in einem agilen Umfeld nach Scrum mit folgendem Tech-Stack:
|
|
// ... TypeScript, Angular, HTML, CSS, PostgreSQL, ..." — is entirely German
|
|
// but has zero umlauts and only one hit ("mit") on the original list, so it
|
|
// sailed through as English despite jobs.ch itself showing "Deutsch
|
|
// (Fliessend)" on the full posting. These are unambiguous German/French
|
|
// function words with no common English collision.
|
|
const DE_FR_WORDS_RE =
|
|
/\b(und|oder|wir|Sie|Ihre|Ihrer|Kenntnisse|Erfahrung|Aufgaben|Anforderungen|Stelle|Unternehmen|Mitarbeiter|Bereich|nous|vous|votre|pour|avec|des|les|du|dein\w*|dich|dir|einem|einer|eines|der|dem)\b/g;
|
|
|
|
// English-language AND not demanding German above B1 — mirrors isViableContent/
|
|
// isViableUrl in fetch.ts, but against a description we already have in hand
|
|
// instead of a page fetch. The two checks catch different things: the word
|
|
// count drops postings *written* in German/French; requiresGerman() drops
|
|
// postings *written in English* that still demand German above B1 (e.g.
|
|
// PRODYNA: "you can communicate fluently in both German and English" — clean
|
|
// English text, would sail through the word count untouched). Skipping the
|
|
// requiresGerman() half was a real gap in every source below until 2026-08-30
|
|
// — it let German-fluency requirements slip past whenever an item's inline
|
|
// description read as English.
|
|
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
|
|
if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
|
|
const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
|
|
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
|
// job-room's search API returns a short keyword-anchored teaser (measured:
|
|
// always < 500 chars), not the full posting — ≥6 is calibrated against full
|
|
// page text and is unreachable on a couple of sentences, so a genuinely
|
|
// German teaser can clear it on word count alone. Lower the bar for short
|
|
// text; leave the full-page threshold (≥6) alone for longer descriptions,
|
|
// where it was tuned to avoid false positives on English text.
|
|
const threshold = text.length < 500 ? 3 : 6;
|
|
if ((text.match(DE_FR_WORDS_RE) ?? []).length >= threshold) return { ok: false, reason: "non-English" };
|
|
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
|
return { ok: true };
|
|
}
|
|
|
|
// datacareer.ch — HTML listing at /jobs/?city=Basel
|
|
export async function fetchDataCareer(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 10; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
|
|
{ headers: PLAIN_HEADERS }
|
|
);
|
|
if (!res.ok) { console.error(`failed (${res.status})`); break; }
|
|
|
|
const html = await res.text();
|
|
const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
|
|
if (articles.length === 0) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
for (const [, card] of articles) {
|
|
const titleM = card.match(/class="link"[^>]*>([^<]+)/);
|
|
const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
|
|
if (!titleM || !hrefM) continue;
|
|
|
|
const title = decodeHtml(titleM[1]);
|
|
if (!applyFilters(title)) continue;
|
|
|
|
const id = `dc-${hrefM[2]}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
|
|
const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
|
|
const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);
|
|
|
|
// Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
|
|
let postedAt = "";
|
|
const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
|
|
if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: companyM ? decodeHtml(companyM[1].trim()) : "",
|
|
location: locM ? locM[1].trim() : "Basel",
|
|
postedAt,
|
|
link: hrefM[1],
|
|
search: "datacareer.ch",
|
|
source: "datacareer.ch",
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${articles.length} items, ${newOnPage} new`);
|
|
if (articles.length < 8) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|
|
|
|
// itjobs.ch — HTML scrape, jobs embedded as window.jobsList in page.
|
|
// Swept Switzerland-wide (no location filter): the board is small, ~210 jobs
|
|
// across 7 pages, and locking it to Basel returned only ~9.
|
|
// Each item carries its full description, so the DE/FR content check runs here
|
|
// instead of costing an extra page fetch per job in index.ts.
|
|
export async function fetchItJobsCh(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 25; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.itjobs.ch/jobs?page=${page}`,
|
|
{ headers: HEADERS }
|
|
);
|
|
// The board 404s one page past the end rather than serving an empty list.
|
|
if (!res.ok) { console.log(`end (${res.status})`); break; }
|
|
|
|
const html = await res.text();
|
|
const blocks = [...html.matchAll(/window\.jobsList\s*=\s*window\.jobsList\.concat\((\[[\s\S]*?\])\);/g)];
|
|
if (!blocks.length) { console.log("0 items"); break; }
|
|
|
|
const items: Array<{
|
|
id: number;
|
|
title: string;
|
|
description?: string;
|
|
posted_at: string;
|
|
job_details_path: string;
|
|
employer: { name: string };
|
|
location?: string;
|
|
job_location?: { name?: string };
|
|
}> = [];
|
|
for (const [, json] of blocks) {
|
|
try { items.push(...JSON.parse(json)); } catch { /* skip malformed block */ }
|
|
}
|
|
|
|
if (!items.length) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
let langDropped = 0;
|
|
for (const item of items) {
|
|
const title = decodeHtml(item.title);
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const id = `ij-${item.id}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
if (!isViableDescription(item.description).ok) { langDropped++; continue; }
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: item.employer?.name ?? "",
|
|
// job_location.name is the English form ("Aarau, Aargau, Switzerland");
|
|
// the bare location field is German ("Aarau, Kanton Aargau, Schweiz").
|
|
location: item.job_location?.name ?? item.location ?? "Switzerland",
|
|
postedAt: item.posted_at?.slice(0, 10) ?? "",
|
|
link: `https://www.itjobs.ch${item.job_details_path}`,
|
|
search: "itjobs.ch",
|
|
source: "itjobs.ch",
|
|
contentChecked: true, // isViableDescription already ran on the real description
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${items.length} items, ${newOnPage} new, ${langDropped} non-English`);
|
|
if (items.length < 30) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|
|
|
|
// job-room.ch — the federal employment service board (RAV / Arbeitsvermittlung).
|
|
// Roles under the Stellenmeldepflicht must be posted here before anywhere else,
|
|
// so it often carries a job days ahead of LinkedIn.
|
|
//
|
|
// Quirks worth knowing:
|
|
// - Each hit is wrapped: [{ jobAdvertisement: {...} }, …]
|
|
// - Titles come back with <em> highlighting around the matched keyword.
|
|
// - `languageIsoCode` on a description is unreliable — plenty of records tagged
|
|
// "de" carry English text (Roche, Novartis, Bachem all do). The DE/FR word
|
|
// count is what actually decides, so we ignore the tag.
|
|
// - An unfiltered canton sweep returns thousands of ads across every industry,
|
|
// so we narrow by keyword instead and union the results.
|
|
// - Descriptions arrive inline, so the language check runs here rather than
|
|
// costing a page fetch per job in index.ts.
|
|
const JOBROOM_API = "https://www.job-room.ch/jobadservice/api/jobAdvertisements/_search";
|
|
const JOBROOM_CANTONS = ["BS", "BL", "BE", "ZH"];
|
|
const JOBROOM_KEYWORDS = [
|
|
"software", "engineer", "developer", "devops", "data", "python",
|
|
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
|
];
|
|
|
|
// job-room carries every industry; pharma/lab titles slip past SOFTWARE_TITLE_RE
|
|
// via words like "data", "director" and "technical".
|
|
const PHARMA_ROLE_RE =
|
|
/\b(clinical|pharmaceutical|drug\s+(substance|product)|ha?ematology|oncology|formulation|laborator|lab\s+technician|chemist|chemistry|biolog|preclinical|regulatory\s+affairs|medical\s+affairs|commercial\s+launch|veterinary|pathologist)\b/i;
|
|
|
|
interface JobRoomAd {
|
|
id: string;
|
|
createdTime?: string;
|
|
jobContent?: {
|
|
externalUrl?: string | null;
|
|
jobDescriptions?: Array<{ title: string; description?: string }>;
|
|
company?: { name?: string; city?: string };
|
|
location?: { city?: string; cantonCode?: string };
|
|
languageSkills?: JobRoomLanguageSkill[];
|
|
};
|
|
}
|
|
|
|
interface JobRoomLanguageSkill {
|
|
languageIsoCode?: string;
|
|
spokenLevel?: string;
|
|
writtenLevel?: string;
|
|
}
|
|
|
|
// job-room's language requirement is a structured field, not prose — it never
|
|
// touches jobDescriptions[].description, so isViableDescription()/requiresGerman()
|
|
// can't see it. The UI renders it as e.g. "Deutsch (Fliessend)"; PROFICIENT is
|
|
// that "Fliessend" (fluent/C1+) tier, well above Rafael's German B1. BASIC and
|
|
// INTERMEDIATE stay in scope.
|
|
function jobRoomRequiresGerman(skills?: JobRoomLanguageSkill[]): boolean {
|
|
if (!skills) return false;
|
|
return skills.some(
|
|
(s) => s.languageIsoCode === "de" && (s.spokenLevel === "PROFICIENT" || s.writtenLevel === "PROFICIENT")
|
|
);
|
|
}
|
|
|
|
// jobs.ch links arrive with a tracking query string that would break dedup
|
|
// across runs if it ever changes; the path alone identifies the posting.
|
|
function stripTracking(url: string): string {
|
|
try {
|
|
const u = new URL(url);
|
|
for (const k of [...u.searchParams.keys()]) {
|
|
if (k.startsWith("utm_")) u.searchParams.delete(k);
|
|
}
|
|
return u.toString().replace(/\?$/, "");
|
|
} catch {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
export async function fetchJobRoom(): Promise<Job[]> {
|
|
const ads = new Map<string, JobRoomAd>();
|
|
// The search API returns a short keyword-anchored teaser per hit, not the
|
|
// full posting — the same ad re-appears across keyword sweeps with a
|
|
// different teaser each time, and `ads.set()` above only keeps the latest.
|
|
// A genuinely German ad (Consult & Pepper's recurring "Senior Fullstack
|
|
// Software Engineer" — full text: "In Deutsch bist du verhandlungssicher")
|
|
// slipped through repeatedly because whichever keyword ran last happened to
|
|
// anchor on an English-reading fragment of it. Collecting every variant and
|
|
// checking them together means one clean-looking teaser can no longer
|
|
// override another teaser that actually shows the German.
|
|
const descVariants = new Map<string, Set<string>>();
|
|
|
|
for (const keyword of JOBROOM_KEYWORDS) {
|
|
process.stdout.write(` ${keyword} … `);
|
|
let total = 0;
|
|
for (let page = 0; page < 8; page++) {
|
|
if (page > 0) await Bun.sleep(250);
|
|
const res = await fetch(`${JOBROOM_API}?page=${page}&size=100`, {
|
|
method: "POST",
|
|
headers: { ...HEADERS, "Content-Type": "application/json", Accept: "application/json" },
|
|
body: JSON.stringify({
|
|
onlineSince: 30,
|
|
displayRestricted: false,
|
|
keywords: [keyword],
|
|
cantonCodes: JOBROOM_CANTONS,
|
|
workloadPercentageMin: 0,
|
|
workloadPercentageMax: 100,
|
|
}),
|
|
});
|
|
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
|
|
|
const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>;
|
|
for (const h of hits) {
|
|
const id = h.jobAdvertisement?.id;
|
|
if (!id) continue;
|
|
ads.set(id, h.jobAdvertisement!);
|
|
const d = h.jobAdvertisement!.jobContent?.jobDescriptions?.[0]?.description;
|
|
if (d) (descVariants.get(id) ?? descVariants.set(id, new Set()).get(id)!).add(d);
|
|
}
|
|
total += hits.length;
|
|
if (hits.length < 100) break;
|
|
}
|
|
console.log(`${total} ads (${ads.size} unique)`);
|
|
await Bun.sleep(250);
|
|
}
|
|
|
|
const jobs: Job[] = [];
|
|
let langDropped = 0;
|
|
for (const ad of ads.values()) {
|
|
const content = ad.jobContent;
|
|
const desc = content?.jobDescriptions?.[0];
|
|
if (!desc) continue;
|
|
|
|
// Strip the <em> highlighting the search API wraps around matched terms.
|
|
const title = decodeHtml(desc.title.replace(/<[^>]+>/g, ""));
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
if (PHARMA_ROLE_RE.test(title)) continue;
|
|
|
|
// Structured language requirement — separate from the free-text description
|
|
// entirely, so it must be checked even when the description reads clean.
|
|
if (jobRoomRequiresGerman(content?.languageSkills)) { langDropped++; continue; }
|
|
|
|
// A handful of ads ship without a description. isViableDescription passes
|
|
// those through by design, so leave contentChecked unset and let index.ts
|
|
// fall back to fetching the page. When we do have variants, join every
|
|
// one seen across keyword sweeps rather than just the last teaser stored
|
|
// on `ad` (see comment above) — more text only adds detection signal, it
|
|
// never removes any.
|
|
const body = [...(descVariants.get(ad.id) ?? [])].join(" \n ") || desc.description || "";
|
|
const hasBody = body.trim().length > 40;
|
|
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
|
|
|
const city = content?.location?.city ?? content?.company?.city ?? "";
|
|
const canton = content?.location?.cantonCode ?? "";
|
|
|
|
jobs.push({
|
|
id: `jr-${ad.id}`,
|
|
title,
|
|
company: content?.company?.name ?? "",
|
|
location: [city, canton].filter(Boolean).join(", ") || "Switzerland",
|
|
postedAt: ad.createdTime?.slice(0, 10) ?? "",
|
|
link: content?.externalUrl
|
|
? stripTracking(content.externalUrl)
|
|
: `https://www.job-room.ch/job-search/${ad.id}`,
|
|
search: "job-room.ch",
|
|
source: "job-room.ch",
|
|
contentChecked: hasBody, // only skip the re-check when we really judged a description
|
|
});
|
|
}
|
|
|
|
console.log(` ${ads.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
|
return jobs;
|
|
}
|
|
|
|
// International organisations in the Basel region (BIS, Swiss TPH, Basel
|
|
// Institute on Governance). Each board carries only a handful of postings, so
|
|
// they're swept together under one label rather than one entry apiece.
|
|
//
|
|
// These are mission-driven employers whose tech roles use UN-flavoured titles
|
|
// ("Information Management Officer", "ICT Specialist") that SOFTWARE_TITLE_RE
|
|
// alone would drop, so INTL_TECH_RE widens the net for these boards only.
|
|
// Boards this small can afford the extra noise; missing a rare opening can't.
|
|
const INTL_TECH_RE =
|
|
/\b(information\s*(management|systems?|technology)|ict\b|digital|geospatial|\bmis\b|data\s*(management|steward))\b/i;
|
|
|
|
function applyIntlFilters(title: string): boolean {
|
|
if (NON_ENGLISH_RE.test(title)) return false;
|
|
if (JUNIOR_TITLE_RE.test(title)) return false;
|
|
if (MS_STACK_RE.test(title)) return false;
|
|
return SOFTWARE_TITLE_RE.test(title) || INTL_TECH_RE.test(title);
|
|
}
|
|
|
|
// BIS runs a JS-rendered career site with no usable API, so we read the UNjobs
|
|
// mirror of its Basel office instead. Titles arrive as "Role, City, Country".
|
|
async function fetchBis(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch(
|
|
"https://unjobs.org/offices/bis_182320855689338880",
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(` BIS … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
const cards = [...html.matchAll(
|
|
/<a[^>]+href="(https:\/\/unjobs\.org\/vacancies\/(\d+))"[\s\S]*?<span\s+class="jtitle">([^<]+)<\/span>([\s\S]*?)<\/a>/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, link, id, rawTitle, rest] of cards) {
|
|
// "Senior Platform Engineer, Basel, Switzerland" → title + location
|
|
const split = decodeHtml(rawTitle).match(/^(.*?),\s*([^,]+,\s*[^,]+)$/);
|
|
const title = split ? split[1] : decodeHtml(rawTitle);
|
|
const location = split ? split[2] : "Basel, Switzerland";
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
jobs.push({
|
|
id: `bis-${id}`,
|
|
title,
|
|
company: "Bank for International Settlements",
|
|
location,
|
|
postedAt: rest.match(/datetime="(\d{4}-\d{2}-\d{2})/)?.[1] ?? "",
|
|
link,
|
|
search: "bis.org",
|
|
source: "bis.org",
|
|
contentChecked: true, // BIS publishes in English only
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` BIS … ${cards.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
// Swiss TPH runs Umantis. Site chrome is German even on English postings, so
|
|
// isEnglishUrl would reject the lot — the title filters decide instead.
|
|
async function fetchSwissTph(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch("https://jobs.swisstph.ch/Jobs/All", { headers: HEADERS });
|
|
if (!res.ok) { console.log(` Swiss TPH … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
// One block per posting, opened by the subtitle link that carries the title.
|
|
const blocks = html.split(/(?=<a\s[^>]*class="HSTableLinkSubTitle")/).slice(1);
|
|
|
|
let kept = 0;
|
|
for (const block of blocks) {
|
|
const hrefM = block.match(/href="(\/Vacancies\/(\d+)\/Description\/\d+)"/);
|
|
const titleM = block.match(/class="HSTableLinkSubTitle"[^>]*>([^<]+)</);
|
|
if (!hrefM || !titleM) continue;
|
|
|
|
const title = decodeHtml(titleM[1]);
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
// "Online seit" label and its value sit in separate spans sharing an id suffix.
|
|
const dateM = block.match(/Online seit<\/span>[\s\S]{0,300}?>\s*(\d{2})\.(\d{2})\.(\d{4})/);
|
|
const locM = block.match(/>\s*Schweiz\s*-\s*([^<]+)</);
|
|
|
|
jobs.push({
|
|
id: `tph-${hrefM[2]}`,
|
|
title,
|
|
company: "Swiss TPH",
|
|
location: locM ? `${locM[1].trim()}, Switzerland` : "Allschwil, Switzerland",
|
|
postedAt: dateM ? `${dateM[3]}-${dateM[2]}-${dateM[1]}` : "",
|
|
link: `https://jobs.swisstph.ch${hrefM[1]}`,
|
|
search: "swisstph.ch",
|
|
source: "swisstph.ch",
|
|
contentChecked: true, // German site chrome would fail the page-level check
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` Swiss TPH … ${blocks.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
// Basel Institute on Governance publishes each vacancy as a linked PDF, so
|
|
// there is no HTML description to language-check downstream.
|
|
async function fetchBaselGovernance(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch("https://baselgovernance.org/about/careers", { headers: HEADERS });
|
|
if (!res.ok) { console.log(` Basel Governance … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
// The `rounded-3xl` card class is what separates vacancy tiles from other
|
|
// asset links on the page (the annual report PDF sits in a plain anchor).
|
|
const cards = [...html.matchAll(
|
|
/<a\s+href="(https:\/\/baselgovernance\.org\/cms\/api\/assets\/([^\/]+)\/[^"]+)"[^>]*class="[^"]*rounded-3xl[^"]*"[\s\S]{0,400}?<h2[^>]*>([^<]+)<\/h2>([\s\S]{0,900})/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, link, assetId, rawTitle, rest] of cards) {
|
|
const title = decodeHtml(rawTitle);
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
const locM = rest.match(/map-pin[\s\S]{0,200}?<\/span>\s*([^<]+)</);
|
|
|
|
jobs.push({
|
|
id: `big-${assetId}`,
|
|
title,
|
|
company: "Basel Institute on Governance",
|
|
location: locM ? decodeHtml(locM[1]) : "Basel, Switzerland",
|
|
postedAt: "", // board shows the closing date only
|
|
link,
|
|
search: "baselgovernance.org",
|
|
source: "baselgovernance.org",
|
|
contentChecked: true, // postings are PDFs, and the institute publishes in English
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` Basel Governance … ${cards.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
export async function fetchBaselIntlOrgs(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
for (const fetchOrg of [fetchBis, fetchSwissTph, fetchBaselGovernance]) {
|
|
try {
|
|
jobs.push(...(await fetchOrg()));
|
|
} catch (e) {
|
|
console.log(` error: ${e}`);
|
|
}
|
|
await Bun.sleep(400);
|
|
}
|
|
return jobs;
|
|
}
|
|
|
|
// jobbasel.ch — the page shell is a React/Vite SPA (no jobs in the raw HTML
|
|
// on the homepage), but the search results route is server-rendered: fetching
|
|
// `/job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__`
|
|
// blob already containing the matched vacancies as JSON. No headless browser
|
|
// needed. Pagination is a URL suffix (`-seite-{n}`), not a query param — a
|
|
// bare `?page=2` is silently ignored by the SPA route.
|
|
//
|
|
// The board (Basel-Stadt + Basel-Landschaft, ~4000 openings) has no keyword
|
|
// union endpoint, so — like job-room.ch — we sweep a fixed keyword list and
|
|
// dedupe by vacancy id. Each item carries its full German description
|
|
// (activity/requirements/offer), so the DE/FR check runs here instead of
|
|
// costing a page fetch per job in index.ts.
|
|
const JOBBASEL_KEYWORDS = [
|
|
"software", "engineer", "developer", "devops", "data", "python",
|
|
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
|
];
|
|
|
|
interface JobBaselItem {
|
|
id: number;
|
|
title: string;
|
|
activity?: string;
|
|
requirements?: string;
|
|
offer?: string;
|
|
workplaceCity?: string;
|
|
urlApplication?: string;
|
|
dateFirstPublished?: string;
|
|
company?: { name?: string };
|
|
}
|
|
|
|
// Balanced-brace extraction — the blob can itself contain "};" inside string
|
|
// values (job descriptions), so a regex cutoff would truncate it.
|
|
function extractPreloadedState(html: string): any | null {
|
|
const marker = "window.__PRELOADED_STATE__";
|
|
const markerIdx = html.indexOf(marker);
|
|
if (markerIdx === -1) return null;
|
|
const start = html.indexOf("{", markerIdx);
|
|
if (start === -1) return null;
|
|
|
|
let depth = 0, inStr = false, esc = false, i = start;
|
|
for (; i < html.length; i++) {
|
|
const c = html[i];
|
|
if (inStr) {
|
|
if (esc) esc = false;
|
|
else if (c === "\\") esc = true;
|
|
else if (c === '"') inStr = false;
|
|
} else {
|
|
if (c === '"') inStr = true;
|
|
else if (c === "{") depth++;
|
|
else if (c === "}") { depth--; if (depth === 0) { i++; break; } }
|
|
}
|
|
}
|
|
|
|
try { return JSON.parse(html.slice(start, i)); } catch { return null; }
|
|
}
|
|
|
|
export async function fetchJobBasel(): Promise<Job[]> {
|
|
const items = new Map<number, JobBaselItem>();
|
|
|
|
for (const keyword of JOBBASEL_KEYWORDS) {
|
|
process.stdout.write(` ${keyword} … `);
|
|
const slug = keyword.toLowerCase().replace(/\s+/g, "-");
|
|
let total = 0;
|
|
|
|
for (let page = 1; page <= 5; page++) {
|
|
if (page > 1) await Bun.sleep(300);
|
|
const suffix = page === 1 ? "" : `-seite-${page}`;
|
|
const res = await fetch(
|
|
`https://www.jobbasel.ch/job/suche-${slug}${suffix}`,
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
|
|
|
const state = extractPreloadedState(await res.text());
|
|
const pageItems: JobBaselItem[] = state?.vacancySearch?.data?.items ?? [];
|
|
if (!pageItems.length) break;
|
|
|
|
for (const item of pageItems) items.set(item.id, item);
|
|
total += pageItems.length;
|
|
if (pageItems.length < 10) break; // last page (pageSize is 10)
|
|
}
|
|
console.log(`${total} ads (${items.size} unique)`);
|
|
await Bun.sleep(300);
|
|
}
|
|
|
|
const jobs: Job[] = [];
|
|
let langDropped = 0;
|
|
for (const item of items.values()) {
|
|
const title = decodeHtml(item.title.replace(/<[^>]+>/g, ""));
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const body = [item.activity, item.requirements, item.offer]
|
|
.filter(Boolean)
|
|
.join(" ")
|
|
.replace(/<[^>]+>/g, " ");
|
|
const hasBody = body.trim().length > 40;
|
|
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
|
|
|
jobs.push({
|
|
id: `jb-${item.id}`,
|
|
title,
|
|
company: item.company?.name ?? "",
|
|
location: item.workplaceCity ? `${item.workplaceCity}, Switzerland` : "Basel, Switzerland",
|
|
postedAt: item.dateFirstPublished?.slice(0, 10) ?? "",
|
|
link: item.urlApplication || `https://www.jobbasel.ch/job/${item.id}`,
|
|
search: "jobbasel.ch",
|
|
source: "jobbasel.ch",
|
|
contentChecked: hasBody,
|
|
});
|
|
}
|
|
|
|
console.log(` ${items.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
|
return jobs;
|
|
}
|
|
|
|
// englishjobsearch.ch — plain SSR HTML, no JS needed, but reached through a
|
|
// non-obvious lever: any page on the site accepts `?format=markdown` and
|
|
// returns the same content pre-converted to clean markdown text (job title,
|
|
// company, city, date, short description all present). Without that param
|
|
// the canton pages ship the job cards as plain text nodes too — no JSON blob,
|
|
// no client-side rendering — so this works with a bare `curl`-equivalent
|
|
// fetch either way; `?format=markdown` just makes parsing trivial.
|
|
//
|
|
// Two catches: (1) the directory root groups by canton, so we hit the
|
|
// Basel-Stadt and Basel-Landschaft canton pages directly rather than the
|
|
// generic `/in/basel/` landing page, which only lists canton counts.
|
|
// (2) every job link is a `/clickout/<hash>` tracking redirect, not the real
|
|
// posting URL — there is no way to recover the underlying employer link
|
|
// without following the redirect, so the clickout link is stored as-is (it
|
|
// still resolves for a human clicking it, and content-checking follows it
|
|
// server-side the same as any other link).
|
|
const ENGLISHJOBSEARCH_CANTONS = ["canton-basel-stadt", "canton-basel-landschaft"];
|
|
|
|
export async function fetchEnglishJobSearch(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (const canton of ENGLISHJOBSEARCH_CANTONS) {
|
|
process.stdout.write(` ${canton} … `);
|
|
const res = await fetch(
|
|
`https://englishjobsearch.ch/in/${canton}?format=markdown`,
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(`failed (${res.status})`); continue; }
|
|
|
|
const md = await res.text();
|
|
// Each posting: "### Title](/clickout/<hash>?...)" heading, then on
|
|
// separate lines "* Company", "* City", "* Date", then a description
|
|
// paragraph before the next "### " heading or "report probem" footer.
|
|
const cards = [...md.matchAll(
|
|
/### ([^\]]+)\]\((\/clickout\/[^)]+)\)\n\n\* ([^\n]+)\n\* ([^\n]+)\n\* ([^\n]+)\n\n([\s\S]*?)\n\nreport probem/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, rawTitle, clickout, company, city, dateStr, desc] of cards) {
|
|
const title = rawTitle.trim();
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const id = clickout.match(/\/clickout\/([a-f0-9]+)/)?.[1] ?? clickout;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
const description = desc.trim();
|
|
const hasBody = description.length > 40;
|
|
if (hasBody && !isViableDescription(description).ok) continue;
|
|
|
|
jobs.push({
|
|
id: `ejs-${id}`,
|
|
title,
|
|
company: company.trim(),
|
|
location: `${city.trim()}, Switzerland`,
|
|
postedAt: "", // "August 30" style, no year given — not worth mis-parsing
|
|
link: `https://englishjobsearch.ch${clickout}`,
|
|
search: "englishjobsearch.ch",
|
|
source: "englishjobsearch.ch",
|
|
contentChecked: hasBody,
|
|
});
|
|
kept++;
|
|
}
|
|
console.log(`${cards.length} ads, ${kept} kept`);
|
|
await Bun.sleep(300);
|
|
}
|
|
|
|
return jobs;
|
|
}
|
|
|
|
// stellen-basel.ch — WordPress REST API
|
|
export async function fetchStellenBasel(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 30; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
|
|
{ headers: { ...HEADERS, Accept: "application/json" } }
|
|
);
|
|
if (!res.ok) { console.log("done"); break; }
|
|
|
|
const data = (await res.json()) as Array<{
|
|
id: number;
|
|
title: { rendered: string };
|
|
date: string;
|
|
link: string;
|
|
meta: { _company_name?: string; _job_location?: string };
|
|
}>;
|
|
|
|
if (!data.length) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
for (const item of data) {
|
|
const title = decodeHtml(item.title.rendered);
|
|
if (!applyFilters(title)) continue;
|
|
|
|
const id = `sb-${item.id}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: item.meta._company_name ?? "",
|
|
location: item.meta._job_location ?? "Basel",
|
|
postedAt: item.date.slice(0, 10),
|
|
link: item.link,
|
|
search: "stellen-basel.ch",
|
|
source: "stellen-basel.ch",
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${data.length} items, ${newOnPage} new`);
|
|
if (data.length < 100) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|
|
|
|
// basel-jobs.ch — regional board on the JobChannel network (same syndication
|
|
// network whose ?source=jobchannel links show up on Siemens/jobs.ch postings).
|
|
//
|
|
// The board is protected by DataDome, which is why this source sat documented
|
|
// as "not integrable" for months. The block is real but narrower than it looked:
|
|
//
|
|
// - Plain curl / bun fetch → 403 on every UA, browser-like or not.
|
|
// - Chrome --headless=new, driven over CDP → 403. Headless is detected.
|
|
// - Playwright's own chromium.launch(), even headful → 403. The CDP-launch
|
|
// path leaves automation artefacts DataDome fingerprints.
|
|
// - A real Chrome launched *normally* (with --remote-debugging-port so we can
|
|
// attach afterwards) → 200. Attaching over CDP after the fact is invisible.
|
|
//
|
|
// The unlock is that DataDome's clearance lives entirely in a `datadome` cookie
|
|
// (Max-Age one year) which replays fine over plain fetch with a matching
|
|
// User-Agent. So we mint a cookie once with a real browser window, cache it, and
|
|
// scrape normally thereafter — a browser is touched only when the cookie is
|
|
// missing or has been rejected, not on every run. An earlier note in CLAUDE.md
|
|
// claimed the cookie "can't be extracted for replay"; that was true of the
|
|
// browser-extension tooling, not of CDP, which hands it over directly.
|
|
//
|
|
// Operational caveat: minting needs a *visible* Chrome window for ~10s, so an
|
|
// unattended `bun daily` under launchd cannot mint. When the cookie is stale in
|
|
// that context the source logs and yields nothing rather than failing the run;
|
|
// refresh it deliberately with `bun run scripts/mint-datadome.ts`.
|
|
|
|
const DATADOME_CACHE = "results/.datadome.json";
|
|
const CHROME_BIN = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
|
|
const BASELJOBS_ORIGIN = "https://basel-jobs.ch";
|
|
|
|
interface DataDomeSession {
|
|
cookie: string;
|
|
ua: string;
|
|
mintedAt: string;
|
|
}
|
|
|
|
async function loadDataDomeSession(): Promise<DataDomeSession | null> {
|
|
try {
|
|
const raw = await Bun.file(DATADOME_CACHE).text();
|
|
const s = JSON.parse(raw) as DataDomeSession;
|
|
return s.cookie && s.ua ? s : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function waitForCdp(port: number, timeoutMs: number): Promise<boolean> {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const r = await fetch(`http://127.0.0.1:${port}/json/version`);
|
|
if (r.ok) return true;
|
|
} catch { /* not up yet */ }
|
|
await Bun.sleep(300);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Launch a real, visible Chrome, let it clear DataDome, then attach over CDP and
|
|
* lift the resulting cookie. Headless and Playwright-launched browsers are both
|
|
* detected — only a normally-launched window gets through, which is the whole
|
|
* reason this is a separate, rarely-run step instead of the fetch path itself.
|
|
*/
|
|
export async function mintDataDomeSession(): Promise<DataDomeSession | null> {
|
|
if (process.env.LINKME_NO_BROWSER === "1") {
|
|
console.log(" LINKME_NO_BROWSER=1 — skipping DataDome mint");
|
|
return null;
|
|
}
|
|
if (!(await Bun.file(CHROME_BIN).exists())) {
|
|
console.error(` Chrome not found at ${CHROME_BIN} — cannot mint DataDome cookie`);
|
|
return null;
|
|
}
|
|
|
|
let chromium: typeof import("playwright-core").chromium;
|
|
try {
|
|
({ chromium } = await import("playwright-core"));
|
|
} catch {
|
|
console.error(" playwright-core not installed — run `bun add playwright-core`");
|
|
return null;
|
|
}
|
|
|
|
// A port unlikely to collide with a debugging session the user opened.
|
|
const port = 9333;
|
|
const profile = "results/.datadome-profile";
|
|
const proc = Bun.spawn(
|
|
[
|
|
CHROME_BIN,
|
|
`--remote-debugging-port=${port}`,
|
|
`--user-data-dir=${profile}`,
|
|
"--no-first-run",
|
|
"--no-default-browser-check",
|
|
"--window-size=1200,860",
|
|
BASELJOBS_ORIGIN,
|
|
],
|
|
{ stdout: "ignore", stderr: "ignore" }
|
|
);
|
|
|
|
try {
|
|
if (!(await waitForCdp(port, 20_000))) {
|
|
console.error(" Chrome CDP endpoint never came up");
|
|
return null;
|
|
}
|
|
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
|
|
const ctx = browser.contexts()[0];
|
|
const page =
|
|
ctx.pages().find((p) => p.url().includes("basel-jobs")) ?? ctx.pages()[0];
|
|
|
|
// The interstitial resolves itself; give it room, then confirm we're through.
|
|
let cookie = "";
|
|
let ua = "";
|
|
for (let attempt = 0; attempt < 12; attempt++) {
|
|
await page.waitForTimeout(1500);
|
|
const html = await page.content();
|
|
if (/Please enable JS|geo\.captcha/.test(html)) continue;
|
|
const cookies = await ctx.cookies(BASELJOBS_ORIGIN);
|
|
const dd = cookies.find((c) => c.name === "datadome");
|
|
if (!dd) continue;
|
|
cookie = dd.value;
|
|
ua = await page.evaluate(() => navigator.userAgent);
|
|
break;
|
|
}
|
|
await browser.close();
|
|
|
|
if (!cookie || !ua) {
|
|
console.error(" DataDome challenge did not clear — no cookie minted");
|
|
return null;
|
|
}
|
|
|
|
const session: DataDomeSession = { cookie, ua, mintedAt: new Date().toISOString() };
|
|
await Bun.write(DATADOME_CACHE, JSON.stringify(session, null, 2));
|
|
console.log(` minted DataDome cookie (${cookie.length} chars)`);
|
|
return session;
|
|
} catch (e) {
|
|
console.error(` DataDome mint failed: ${e}`);
|
|
return null;
|
|
} finally {
|
|
proc.kill();
|
|
}
|
|
}
|
|
|
|
function baselJobsHeaders(s: DataDomeSession): Record<string, string> {
|
|
return {
|
|
"User-Agent": s.ua,
|
|
"Accept-Language": "de-CH,de;q=0.9,en;q=0.8",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
Cookie: `datadome=${s.cookie}`,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Fetch with the cached cookie, re-minting once if the cookie has gone stale.
|
|
* DataDome rotates the cookie on ordinary responses too, so a Set-Cookie on the
|
|
* way past is folded back into the session rather than ignored.
|
|
*/
|
|
async function baselJobsFetch(
|
|
url: string,
|
|
state: { session: DataDomeSession | null; minted: boolean }
|
|
): Promise<string | null> {
|
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
if (!state.session) {
|
|
if (state.minted) return null;
|
|
state.minted = true;
|
|
state.session = await mintDataDomeSession();
|
|
if (!state.session) return null;
|
|
}
|
|
|
|
const res = await fetch(url, { headers: baselJobsHeaders(state.session) });
|
|
|
|
const rotated = res.headers
|
|
.getSetCookie?.()
|
|
.map((c) => /(?:^|;\s*)datadome=([^;]+)/.exec(c)?.[1])
|
|
.find(Boolean);
|
|
if (rotated && state.session) {
|
|
state.session = { ...state.session, cookie: rotated };
|
|
await Bun.write(DATADOME_CACHE, JSON.stringify(state.session, null, 2));
|
|
}
|
|
|
|
if (res.ok) return res.text();
|
|
|
|
if (res.status === 403 && !state.minted) {
|
|
console.log(" cookie rejected — re-minting");
|
|
state.session = null;
|
|
continue;
|
|
}
|
|
console.error(` failed (${res.status}) ${url}`);
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// Terms swept through the board's search. No keyword-union endpoint exists, so
|
|
// — like job-room.ch and jobbasel.ch — this is a fixed list, one request each
|
|
// per mode, deduped by vacancy id. German stems are in deliberately: the search
|
|
// matches description text, so "entwickler" surfaces postings whose English
|
|
// title the category never filed under IT. The title and description filters
|
|
// drop the German-language ones afterwards.
|
|
//
|
|
// Role words and stack words behave differently across the two search modes
|
|
// (see fetchBaselJobs), which is why both modes get the whole list rather than
|
|
// a tuned subset — `websearch=1` finds no stack terms at all, and the bare
|
|
// search is the only mode that matches "react"/"frontend"/"backend".
|
|
const BASELJOBS_TERMS = [
|
|
// role / domain words — these carry the `websearch=1` pass
|
|
"software", "engineer", "developer", "entwickler", "informatik", "data",
|
|
"cloud", "devops", "platform", "architect", "security", "ai",
|
|
// stack words — only the bare search matches these
|
|
"python", "javascript", "typescript", "react", "node",
|
|
"fullstack", "full stack", "backend", "frontend", "kubernetes", "linux",
|
|
];
|
|
|
|
function buildBaselJobsLocation(city: string, canton: string): string {
|
|
if (!city) return canton || "Basel";
|
|
if (!canton) return city;
|
|
// data-gtm-jp is one or more codes ("BS", "BL,BS"); keep only what the city
|
|
// string doesn't already name.
|
|
const missing = canton
|
|
.split(",")
|
|
.map((c) => c.trim())
|
|
.filter((c) => c && !new RegExp(`\\b${c}\\b`).test(city));
|
|
return missing.length ? `${city}, ${missing.join(",")}` : city;
|
|
}
|
|
|
|
/**
|
|
* Pull every vacancy card out of a listing page. Returns how many ids were
|
|
* new so callers can use "no new postings" as their end condition — past the
|
|
* last real page the board still serves its two sponsored slots, so an empty
|
|
* list never arrives.
|
|
*/
|
|
function parseBaselJobsCards(
|
|
html: string,
|
|
seen: Set<string>,
|
|
out: Job[],
|
|
today: string
|
|
): { items: number; added: number } {
|
|
const items = [...html.matchAll(/<li class="item"(.*?)<\/li>/gs)].map((m) => m[1]);
|
|
let added = 0;
|
|
|
|
for (const card of items) {
|
|
const jid = /data-gtm-jid="(\d+)"/.exec(card)?.[1];
|
|
const linkM = /href="(https:\/\/basel-jobs\.ch\/job\/[^"]+)"/.exec(card);
|
|
const titleM = /<span class="jobtitle">(.*?)<\/span>/s.exec(card);
|
|
if (!jid || !linkM || !titleM) continue;
|
|
|
|
// Sponsored slots repeat the same posting within a page, between pages,
|
|
// and across the category and search sweeps.
|
|
const id = `bj-${jid}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
added++;
|
|
|
|
const title = decodeHtml(titleM[1]);
|
|
if (!applyFilters(title)) continue;
|
|
|
|
const companyM =
|
|
/<span class="company">\s*<a[^>]*>(.*?)<\/a>/s.exec(card) ??
|
|
/<span class="company">(.*?)<\/span>/s.exec(card);
|
|
const locationM = /<span class="location">(.*?)<\/span>/s.exec(card);
|
|
const canton = /data-gtm-jp="([^"]*)"/.exec(card)?.[1] ?? "";
|
|
|
|
out.push({
|
|
id,
|
|
title,
|
|
company: companyM ? decodeHtml(companyM[1]) : "",
|
|
// Cards carry a city; the canton attribute disambiguates BS vs BL,
|
|
// which is what daily.ts groups on. Some cards already end in the
|
|
// canton code ("Basel, BS, 4030") — don't append it twice.
|
|
location: buildBaselJobsLocation(
|
|
locationM ? decodeHtml(locationM[1]) : "",
|
|
canton
|
|
),
|
|
// No date anywhere on the card or detail page — discovery date is the
|
|
// best available, same as any board that hides posting age.
|
|
postedAt: today,
|
|
link: linkM[1],
|
|
search: "basel-jobs.ch",
|
|
source: "basel-jobs.ch",
|
|
});
|
|
}
|
|
|
|
return { items: items.length, added };
|
|
}
|
|
|
|
export async function fetchBaselJobs(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
const state = { session: await loadDataDomeSession(), minted: false };
|
|
const today = new Date().toISOString().slice(0, 10);
|
|
|
|
// Pass 1 — the board's IT category, ~300 postings over 3 pages.
|
|
for (let page = 1; page <= 10; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` informatik page ${page} … `);
|
|
|
|
const html = await baselJobsFetch(
|
|
`${BASELJOBS_ORIGIN}/informatik-jobs?page=${page}`,
|
|
state
|
|
);
|
|
if (html === null) { console.log("unavailable"); break; }
|
|
|
|
const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
|
|
if (items === 0) { console.log("0 items"); break; }
|
|
console.log(`${items} items, ${added} new`);
|
|
if (added === 0) break;
|
|
}
|
|
|
|
// Pass 2 — full-text search. The category is a *curated taxonomy*, not the
|
|
// whole board: it only holds what the board filed under IT, so tech roles
|
|
// parked under Industrie/Pharma/Bau, and roles whose relevance lives in the
|
|
// description rather than the title, never appear in pass 1. Measured on
|
|
// 2026-09-01 this pass added **147 postings the category missed**, including
|
|
// "Senior AI Infrastructure & Platform Engineer (Basel)" and "Senior DevOps /
|
|
// Cloud Engineer (Kubernetes, CI/CD, Linux)". The reverse is also true (the
|
|
// category held 257 the search missed), so both passes are needed.
|
|
//
|
|
// `websearch=1` is the mode that does this — the bare `?terms=` search
|
|
// returned nothing that wasn't already in the category. It ignores `page`
|
|
// entirely and returns the whole result set on one response, so there is no
|
|
// pagination to walk.
|
|
// Both search modes are swept, because they query different indexes and each
|
|
// is blind to what the other finds:
|
|
//
|
|
// term websearch=1 bare
|
|
// engineer 90 260 found
|
|
// frontend 0 2
|
|
// backend 0 4
|
|
// react 0 2
|
|
//
|
|
// `websearch=1` matches role/domain words and reaches postings filed outside
|
|
// the IT category; the bare search is the *only* mode that matches stack
|
|
// terms. Neither paginates — both ignore `page` and return their whole result
|
|
// set in one response (bare caps around 100 cards). Measured 2026-09-01
|
|
// against the category: websearch added 147, bare added a further 139 raw
|
|
// (23 surviving the title filters).
|
|
for (const mode of ["websearch", "bare"] as const) {
|
|
for (const term of BASELJOBS_TERMS) {
|
|
await Bun.sleep(400);
|
|
process.stdout.write(` ${mode} "${term}" … `);
|
|
|
|
const qs = `terms=${encodeURIComponent(term)}${mode === "websearch" ? "&page=1&websearch=1" : ""}`;
|
|
const html = await baselJobsFetch(`${BASELJOBS_ORIGIN}/search?${qs}`, state);
|
|
if (html === null) { console.log("unavailable"); break; }
|
|
|
|
const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
|
|
console.log(`${items} items, ${added} new`);
|
|
}
|
|
}
|
|
|
|
// Detail pages carry the full description inline, so the DE/FR and
|
|
// German-requirement checks run here instead of costing index.ts a fetch that
|
|
// would hit DataDome without the cookie and be dropped as unreachable.
|
|
const viable: Job[] = [];
|
|
for (const job of allJobs) {
|
|
await Bun.sleep(300);
|
|
process.stdout.write(` checking ${job.title} … `);
|
|
const page = await baselJobsFetch(job.link, state);
|
|
if (page === null) { console.log("unreachable, keeping"); viable.push(job); continue; }
|
|
|
|
// Only the posting's own text, never the page. basel-jobs.ch is a German
|
|
// site, so its nav/footer/category chrome alone clears the ≥ 6 DE/FR word
|
|
// threshold — checking the whole page dropped *every* job, English ones
|
|
// included. The mobile `plain-content` block is the description as plain
|
|
// text; the desktop view loads the same thing into an iframe, so this is
|
|
// the only copy present in the server-rendered HTML.
|
|
const descM = /<div class="box plain-content[^"]*">(.*?)<\/div>/s.exec(page);
|
|
if (!descM) { console.log("no description, keeping"); viable.push(job); continue; }
|
|
const check = isViableDescription(descM[1].replace(/<br\s*\/?>/gi, " "));
|
|
if (!check.ok) { console.log(`dropped (${check.reason})`); continue; }
|
|
console.log("ok");
|
|
viable.push({ ...job, contentChecked: true });
|
|
}
|
|
|
|
return viable;
|
|
}
|