1786 lines
73 KiB
TypeScript
1786 lines
73 KiB
TypeScript
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, BIO_DOMAIN_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
||
import type { Job } from "./fetch.ts";
|
||
import { tmpdir } from "os";
|
||
import { rm } from "fs/promises";
|
||
|
||
const HEADERS = {
|
||
"User-Agent":
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
};
|
||
|
||
// datacareer.ch inverts the usual bot rule: its nginx 403s anything claiming to
|
||
// be a browser (a spoofed Chrome UA is the one thing guaranteed to fail) while
|
||
// letting honest clients — no UA, curl, Googlebot — straight through. Presumably
|
||
// real browser traffic arrives via a path that also carries matching TLS/CH
|
||
// fingerprints, so a bare Chrome UA over plain HTTP reads as a scraper. Sending
|
||
// an honest identifier is both what unblocks it and the polite thing to do.
|
||
// This is what killed the source for months; it was never a hard block.
|
||
const PLAIN_HEADERS = {
|
||
"User-Agent": "linkme-jobbot/1.0 (personal job search; contact via basel)",
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
};
|
||
|
||
function decodeHtml(s: string): string {
|
||
return s
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
function applyFilters(title: string): boolean {
|
||
if (NON_ENGLISH_RE.test(title)) return false;
|
||
if (JUNIOR_TITLE_RE.test(title)) return false;
|
||
if (!SOFTWARE_TITLE_RE.test(title)) return false;
|
||
return true;
|
||
}
|
||
|
||
const MS_STACK_RE = /\.net\b|\bc#|sharepoint|dynamics 365|power platform|\bsccm\b|intune|successfactors|\bsap\b/i;
|
||
|
||
// DE/FR indicator words, counted against a description we already have in hand.
|
||
// Mirrors the ≥ 6 threshold isEnglishContent uses on fetched pages — but see
|
||
// the adaptive threshold below for why short snippets need a lower bar.
|
||
// du/dein*/dich/dir/einem/einer/eines/der/dem added 2026-09-08: job-room's
|
||
// search API "Senior Fullstack Software Engineer" (Consult & Pepper) teaser —
|
||
// "Du arbeitest in einem agilen Umfeld nach Scrum mit folgendem Tech-Stack:
|
||
// ... TypeScript, Angular, HTML, CSS, PostgreSQL, ..." — is entirely German
|
||
// but has zero umlauts and only one hit ("mit") on the original list, so it
|
||
// sailed through as English despite jobs.ch itself showing "Deutsch
|
||
// (Fliessend)" on the full posting. These are unambiguous German/French
|
||
// function words with no common English collision.
|
||
const DE_FR_WORDS_RE =
|
||
/\b(und|oder|wir|Sie|Ihre|Ihrer|Kenntnisse|Erfahrung|Aufgaben|Anforderungen|Stelle|Unternehmen|Mitarbeiter|Bereich|nous|vous|votre|pour|avec|des|les|du|dein\w*|dich|dir|einem|einer|eines|der|dem)\b/g;
|
||
|
||
// English-language AND not demanding German above B1 — mirrors isViableContent/
|
||
// isViableUrl in fetch.ts, but against a description we already have in hand
|
||
// instead of a page fetch. The two checks catch different things: the word
|
||
// count drops postings *written* in German/French; requiresGerman() drops
|
||
// postings *written in English* that still demand German above B1 (e.g.
|
||
// PRODYNA: "you can communicate fluently in both German and English" — clean
|
||
// English text, would sail through the word count untouched). Skipping the
|
||
// requiresGerman() half was a real gap in every source below until 2026-08-30
|
||
// — it let German-fluency requirements slip past whenever an item's inline
|
||
// description read as English.
|
||
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
|
||
if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
|
||
const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
|
||
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||
// job-room's search API returns a short keyword-anchored teaser (measured:
|
||
// always < 500 chars), not the full posting — ≥6 is calibrated against full
|
||
// page text and is unreachable on a couple of sentences, so a genuinely
|
||
// German teaser can clear it on word count alone. Lower the bar for short
|
||
// text; leave the full-page threshold (≥6) alone for longer descriptions,
|
||
// where it was tuned to avoid false positives on English text.
|
||
const threshold = text.length < 500 ? 3 : 6;
|
||
if ((text.match(DE_FR_WORDS_RE) ?? []).length >= threshold) return { ok: false, reason: "non-English" };
|
||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||
return { ok: true };
|
||
}
|
||
|
||
// datacareer.ch — HTML listing at /jobs/?city=Basel
|
||
export async function fetchDataCareer(): Promise<Job[]> {
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (let page = 1; page <= 10; page++) {
|
||
if (page > 1) await Bun.sleep(400);
|
||
process.stdout.write(` page ${page} … `);
|
||
|
||
const res = await fetch(
|
||
`https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
|
||
{ headers: PLAIN_HEADERS }
|
||
);
|
||
if (!res.ok) { console.error(`failed (${res.status})`); break; }
|
||
|
||
const html = await res.text();
|
||
const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
|
||
if (articles.length === 0) { console.log("0 items"); break; }
|
||
|
||
let newOnPage = 0;
|
||
for (const [, card] of articles) {
|
||
const titleM = card.match(/class="link"[^>]*>([^<]+)/);
|
||
const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
|
||
if (!titleM || !hrefM) continue;
|
||
|
||
const title = decodeHtml(titleM[1]);
|
||
if (!applyFilters(title)) continue;
|
||
|
||
const id = `dc-${hrefM[2]}`;
|
||
if (seen.has(id)) continue;
|
||
seen.add(id);
|
||
|
||
const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
|
||
const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
|
||
const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);
|
||
|
||
// Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
|
||
let postedAt = "";
|
||
const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
|
||
if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;
|
||
|
||
allJobs.push({
|
||
id,
|
||
title,
|
||
company: companyM ? decodeHtml(companyM[1].trim()) : "",
|
||
location: locM ? locM[1].trim() : "Basel",
|
||
postedAt,
|
||
link: hrefM[1],
|
||
search: "datacareer.ch",
|
||
source: "datacareer.ch",
|
||
});
|
||
newOnPage++;
|
||
}
|
||
|
||
console.log(`${articles.length} items, ${newOnPage} new`);
|
||
if (articles.length < 8) break;
|
||
}
|
||
|
||
return allJobs;
|
||
}
|
||
|
||
// itjobs.ch — HTML scrape, jobs embedded as window.jobsList in page.
|
||
// Swept Switzerland-wide (no location filter): the board is small, ~210 jobs
|
||
// across 7 pages, and locking it to Basel returned only ~9.
|
||
// Each item carries its full description, so the DE/FR content check runs here
|
||
// instead of costing an extra page fetch per job in index.ts.
|
||
export async function fetchItJobsCh(): Promise<Job[]> {
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (let page = 1; page <= 25; page++) {
|
||
if (page > 1) await Bun.sleep(400);
|
||
process.stdout.write(` page ${page} … `);
|
||
|
||
const res = await fetch(
|
||
`https://www.itjobs.ch/jobs?page=${page}`,
|
||
{ headers: HEADERS }
|
||
);
|
||
// The board 404s one page past the end rather than serving an empty list.
|
||
if (!res.ok) { console.log(`end (${res.status})`); break; }
|
||
|
||
const html = await res.text();
|
||
const blocks = [...html.matchAll(/window\.jobsList\s*=\s*window\.jobsList\.concat\((\[[\s\S]*?\])\);/g)];
|
||
if (!blocks.length) { console.log("0 items"); break; }
|
||
|
||
const items: Array<{
|
||
id: number;
|
||
title: string;
|
||
description?: string;
|
||
posted_at: string;
|
||
job_details_path: string;
|
||
employer: { name: string };
|
||
location?: string;
|
||
job_location?: { name?: string };
|
||
}> = [];
|
||
for (const [, json] of blocks) {
|
||
try { items.push(...JSON.parse(json)); } catch { /* skip malformed block */ }
|
||
}
|
||
|
||
if (!items.length) { console.log("0 items"); break; }
|
||
|
||
let newOnPage = 0;
|
||
let langDropped = 0;
|
||
for (const item of items) {
|
||
const title = decodeHtml(item.title);
|
||
if (!applyFilters(title)) continue;
|
||
if (MS_STACK_RE.test(title)) continue;
|
||
|
||
const id = `ij-${item.id}`;
|
||
if (seen.has(id)) continue;
|
||
seen.add(id);
|
||
|
||
if (!isViableDescription(item.description).ok) { langDropped++; continue; }
|
||
|
||
allJobs.push({
|
||
id,
|
||
title,
|
||
company: item.employer?.name ?? "",
|
||
// job_location.name is the English form ("Aarau, Aargau, Switzerland");
|
||
// the bare location field is German ("Aarau, Kanton Aargau, Schweiz").
|
||
location: item.job_location?.name ?? item.location ?? "Switzerland",
|
||
postedAt: item.posted_at?.slice(0, 10) ?? "",
|
||
link: `https://www.itjobs.ch${item.job_details_path}`,
|
||
search: "itjobs.ch",
|
||
source: "itjobs.ch",
|
||
contentChecked: true, // isViableDescription already ran on the real description
|
||
});
|
||
newOnPage++;
|
||
}
|
||
|
||
console.log(`${items.length} items, ${newOnPage} new, ${langDropped} non-English`);
|
||
if (items.length < 30) break;
|
||
}
|
||
|
||
return allJobs;
|
||
}
|
||
|
||
// swisscom.com — Swisscom's own Workday career site. Small board (~47 openings,
|
||
// Switzerland-wide), so no keyword sweep — pull the full listing and filter
|
||
// locally, like itjobs.ch. The listing page itself is a bare JS shell (no jobs
|
||
// in the raw HTML); postings live behind the standard Workday CXS JSON API:
|
||
// POST /wday/cxs/swisscom/SwisscomExternalCareers/jobs {appliedFacets, limit, offset, searchText}
|
||
// `total` on the first page (offset 0) is authoritative — on later pages it
|
||
// comes back as 0 (a quirk of this tenant), and an offset past the true total
|
||
// silently wraps around and re-serves page 1 instead of an empty list. So the
|
||
// loop's stopping condition is the first page's total, never a later one's.
|
||
// Listing rows carry no description, so each title-filtered candidate costs
|
||
// one extra fetch against the job detail endpoint (`/job{externalPath}`),
|
||
// which returns the full HTML description — the DE/FR + requiresGerman check
|
||
// runs on that here, same as job-room.ch/jobbasel.ch.
|
||
const SWISSCOM_API_BASE = "https://swisscom.wd103.myworkdayjobs.com/wday/cxs/swisscom/SwisscomExternalCareers";
|
||
|
||
interface SwisscomListing {
|
||
title: string;
|
||
externalPath: string;
|
||
postedOn: string;
|
||
bulletFields: string[];
|
||
}
|
||
|
||
function parseSwisscomPostedOn(text: string): string {
|
||
const iso = (d: Date) => d.toISOString().slice(0, 10);
|
||
const now = new Date();
|
||
if (/posted today/i.test(text)) return iso(now);
|
||
if (/posted yesterday/i.test(text)) {
|
||
const d = new Date(now);
|
||
d.setDate(d.getDate() - 1);
|
||
return iso(d);
|
||
}
|
||
const m = text.match(/posted (\d+) days? ago/i);
|
||
if (m) {
|
||
const d = new Date(now);
|
||
d.setDate(d.getDate() - Number(m[1]));
|
||
return iso(d);
|
||
}
|
||
return ""; // "Posted 30+ Days Ago" — no exact date available
|
||
}
|
||
|
||
export async function fetchSwisscom(): Promise<Job[]> {
|
||
const listings: SwisscomListing[] = [];
|
||
let total = Infinity;
|
||
|
||
for (let offset = 0; offset < total; offset += 20) {
|
||
if (offset > 0) await Bun.sleep(400);
|
||
const res = await fetch(`${SWISSCOM_API_BASE}/jobs`, {
|
||
method: "POST",
|
||
headers: { ...HEADERS, "Content-Type": "application/json" },
|
||
body: JSON.stringify({ appliedFacets: {}, limit: 20, offset, searchText: "" }),
|
||
});
|
||
if (!res.ok) { console.log(` Swisscom … failed (${res.status})`); break; }
|
||
|
||
const data = (await res.json()) as { total: number; jobPostings: SwisscomListing[] };
|
||
if (offset === 0) total = data.total;
|
||
if (!data.jobPostings.length) break;
|
||
listings.push(...data.jobPostings);
|
||
}
|
||
|
||
const jobs: Job[] = [];
|
||
let kept = 0;
|
||
let langDropped = 0;
|
||
for (const item of listings) {
|
||
const title = decodeHtml(item.title);
|
||
if (!applyFilters(title)) continue;
|
||
if (MS_STACK_RE.test(title)) continue;
|
||
|
||
await Bun.sleep(300);
|
||
const detailRes = await fetch(`${SWISSCOM_API_BASE}${item.externalPath}`, { headers: HEADERS });
|
||
if (!detailRes.ok) continue;
|
||
|
||
const detail = (await detailRes.json()) as {
|
||
jobPostingInfo?: { location?: string; jobDescription?: string; externalUrl?: string };
|
||
};
|
||
const info = detail.jobPostingInfo;
|
||
if (!isViableDescription(info?.jobDescription).ok) { langDropped++; continue; }
|
||
|
||
jobs.push({
|
||
id: `sc-${item.bulletFields?.[0] ?? item.externalPath}`,
|
||
title,
|
||
company: "Swisscom",
|
||
location: info?.location ?? "Switzerland",
|
||
postedAt: parseSwisscomPostedOn(item.postedOn),
|
||
link: info?.externalUrl ?? `https://swisscom.wd103.myworkdayjobs.com/SwisscomExternalCareers${item.externalPath}`,
|
||
search: "swisscom.com",
|
||
source: "swisscom.com",
|
||
contentChecked: true, // isViableDescription already ran on the real description
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` Swisscom … ${listings.length} items, ${kept} kept, ${langDropped} non-English`);
|
||
return jobs;
|
||
}
|
||
|
||
// job-room.ch — the federal employment service board (RAV / Arbeitsvermittlung).
|
||
// Roles under the Stellenmeldepflicht must be posted here before anywhere else,
|
||
// so it often carries a job days ahead of LinkedIn.
|
||
//
|
||
// Quirks worth knowing:
|
||
// - Each hit is wrapped: [{ jobAdvertisement: {...} }, …]
|
||
// - Titles come back with <em> highlighting around the matched keyword.
|
||
// - `languageIsoCode` on a description is unreliable — plenty of records tagged
|
||
// "de" carry English text (Roche, Novartis, Bachem all do). The DE/FR word
|
||
// count is what actually decides, so we ignore the tag.
|
||
// - An unfiltered canton sweep returns thousands of ads across every industry,
|
||
// so we narrow by keyword instead and union the results.
|
||
// - Descriptions arrive inline, so the language check runs here rather than
|
||
// costing a page fetch per job in index.ts.
|
||
const JOBROOM_API = "https://www.job-room.ch/jobadservice/api/jobAdvertisements/_search";
|
||
const JOBROOM_CANTONS = ["BS", "BL", "BE", "ZH"];
|
||
const JOBROOM_KEYWORDS = [
|
||
"software", "engineer", "developer", "devops", "data", "python",
|
||
"cloud", "platform", "typescript", "backend", "full stack", "ai", "agentic",
|
||
];
|
||
|
||
// job-room carries every industry; pharma/lab titles slip past SOFTWARE_TITLE_RE
|
||
// via words like "data", "director" and "technical".
|
||
const PHARMA_ROLE_RE =
|
||
/\b(clinical|pharmaceutical|drug\s+(substance|product)|ha?ematology|oncology|formulation|laborator|lab\s+technician|chemist|chemistry|biolog|preclinical|regulatory\s+affairs|medical\s+affairs|commercial\s+launch|veterinary|pathologist)\b/i;
|
||
|
||
interface JobRoomAd {
|
||
id: string;
|
||
createdTime?: string;
|
||
jobContent?: {
|
||
externalUrl?: string | null;
|
||
jobDescriptions?: Array<{ title: string; description?: string }>;
|
||
company?: { name?: string; city?: string };
|
||
location?: { city?: string; cantonCode?: string };
|
||
languageSkills?: JobRoomLanguageSkill[];
|
||
};
|
||
}
|
||
|
||
interface JobRoomLanguageSkill {
|
||
languageIsoCode?: string;
|
||
spokenLevel?: string;
|
||
writtenLevel?: string;
|
||
}
|
||
|
||
// job-room's language requirement is a structured field, not prose — it never
|
||
// touches jobDescriptions[].description, so isViableDescription()/requiresGerman()
|
||
// can't see it. The UI renders it as e.g. "Deutsch (Fliessend)"; PROFICIENT is
|
||
// that "Fliessend" (fluent/C1+) tier, well above Rafael's German B1. BASIC and
|
||
// INTERMEDIATE stay in scope.
|
||
function jobRoomRequiresGerman(skills?: JobRoomLanguageSkill[]): boolean {
|
||
if (!skills) return false;
|
||
return skills.some(
|
||
(s) => s.languageIsoCode === "de" && (s.spokenLevel === "PROFICIENT" || s.writtenLevel === "PROFICIENT")
|
||
);
|
||
}
|
||
|
||
// jobs.ch links arrive with a tracking query string that would break dedup
|
||
// across runs if it ever changes; the path alone identifies the posting.
|
||
function stripTracking(url: string): string {
|
||
try {
|
||
const u = new URL(url);
|
||
for (const k of [...u.searchParams.keys()]) {
|
||
if (k.startsWith("utm_")) u.searchParams.delete(k);
|
||
}
|
||
return u.toString().replace(/\?$/, "");
|
||
} catch {
|
||
return url;
|
||
}
|
||
}
|
||
|
||
export async function fetchJobRoom(): Promise<Job[]> {
|
||
const ads = new Map<string, JobRoomAd>();
|
||
// The search API returns a short keyword-anchored teaser per hit, not the
|
||
// full posting — the same ad re-appears across keyword sweeps with a
|
||
// different teaser each time, and `ads.set()` above only keeps the latest.
|
||
// A genuinely German ad (Consult & Pepper's recurring "Senior Fullstack
|
||
// Software Engineer" — full text: "In Deutsch bist du verhandlungssicher")
|
||
// slipped through repeatedly because whichever keyword ran last happened to
|
||
// anchor on an English-reading fragment of it. Collecting every variant and
|
||
// checking them together means one clean-looking teaser can no longer
|
||
// override another teaser that actually shows the German.
|
||
const descVariants = new Map<string, Set<string>>();
|
||
|
||
for (const keyword of JOBROOM_KEYWORDS) {
|
||
process.stdout.write(` ${keyword} … `);
|
||
let total = 0;
|
||
for (let page = 0; page < 8; page++) {
|
||
if (page > 0) await Bun.sleep(250);
|
||
const res = await fetch(`${JOBROOM_API}?page=${page}&size=100`, {
|
||
method: "POST",
|
||
headers: { ...HEADERS, "Content-Type": "application/json", Accept: "application/json" },
|
||
body: JSON.stringify({
|
||
onlineSince: 30,
|
||
displayRestricted: false,
|
||
keywords: [keyword],
|
||
cantonCodes: JOBROOM_CANTONS,
|
||
workloadPercentageMin: 0,
|
||
workloadPercentageMax: 100,
|
||
}),
|
||
});
|
||
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
||
|
||
const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>;
|
||
for (const h of hits) {
|
||
const id = h.jobAdvertisement?.id;
|
||
if (!id) continue;
|
||
ads.set(id, h.jobAdvertisement!);
|
||
const d = h.jobAdvertisement!.jobContent?.jobDescriptions?.[0]?.description;
|
||
if (d) (descVariants.get(id) ?? descVariants.set(id, new Set()).get(id)!).add(d);
|
||
}
|
||
total += hits.length;
|
||
if (hits.length < 100) break;
|
||
}
|
||
console.log(`${total} ads (${ads.size} unique)`);
|
||
await Bun.sleep(250);
|
||
}
|
||
|
||
const jobs: Job[] = [];
|
||
let langDropped = 0;
|
||
for (const ad of ads.values()) {
|
||
const content = ad.jobContent;
|
||
const desc = content?.jobDescriptions?.[0];
|
||
if (!desc) continue;
|
||
|
||
// Strip the <em> highlighting the search API wraps around matched terms.
|
||
const title = decodeHtml(desc.title.replace(/<[^>]+>/g, ""));
|
||
if (!applyFilters(title)) continue;
|
||
if (MS_STACK_RE.test(title)) continue;
|
||
if (PHARMA_ROLE_RE.test(title)) continue;
|
||
|
||
// Structured language requirement — separate from the free-text description
|
||
// entirely, so it must be checked even when the description reads clean.
|
||
if (jobRoomRequiresGerman(content?.languageSkills)) { langDropped++; continue; }
|
||
|
||
// A handful of ads ship without a description. isViableDescription passes
|
||
// those through by design, so leave contentChecked unset and let index.ts
|
||
// fall back to fetching the page. When we do have variants, join every
|
||
// one seen across keyword sweeps rather than just the last teaser stored
|
||
// on `ad` (see comment above) — more text only adds detection signal, it
|
||
// never removes any.
|
||
const body = [...(descVariants.get(ad.id) ?? [])].join(" \n ") || desc.description || "";
|
||
const hasBody = body.trim().length > 40;
|
||
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
||
|
||
const city = content?.location?.city ?? content?.company?.city ?? "";
|
||
const canton = content?.location?.cantonCode ?? "";
|
||
|
||
jobs.push({
|
||
id: `jr-${ad.id}`,
|
||
title,
|
||
company: content?.company?.name ?? "",
|
||
location: [city, canton].filter(Boolean).join(", ") || "Switzerland",
|
||
postedAt: ad.createdTime?.slice(0, 10) ?? "",
|
||
link: content?.externalUrl
|
||
? stripTracking(content.externalUrl)
|
||
: `https://www.job-room.ch/job-search/${ad.id}`,
|
||
search: "job-room.ch",
|
||
source: "job-room.ch",
|
||
contentChecked: hasBody, // only skip the re-check when we really judged a description
|
||
});
|
||
}
|
||
|
||
console.log(` ${ads.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
||
return jobs;
|
||
}
|
||
|
||
// International organisations in the Basel region (BIS, Swiss TPH, Basel
|
||
// Institute on Governance). Each board carries only a handful of postings, so
|
||
// they're swept together under one label rather than one entry apiece.
|
||
//
|
||
// These are mission-driven employers whose tech roles use UN-flavoured titles
|
||
// ("Information Management Officer", "ICT Specialist") that SOFTWARE_TITLE_RE
|
||
// alone would drop, so INTL_TECH_RE widens the net for these boards only.
|
||
// Boards this small can afford the extra noise; missing a rare opening can't.
|
||
const INTL_TECH_RE =
|
||
/\b(information\s*(management|systems?|technology)|ict\b|digital|geospatial|\bmis\b|data\s*(management|steward))\b/i;
|
||
|
||
function applyIntlFilters(title: string): boolean {
|
||
if (NON_ENGLISH_RE.test(title)) return false;
|
||
if (JUNIOR_TITLE_RE.test(title)) return false;
|
||
if (MS_STACK_RE.test(title)) return false;
|
||
return SOFTWARE_TITLE_RE.test(title) || INTL_TECH_RE.test(title);
|
||
}
|
||
|
||
// BIS runs a JS-rendered career site with no usable API, so we read the UNjobs
|
||
// mirror of its Basel office instead. Titles arrive as "Role, City, Country".
|
||
async function fetchBis(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
const res = await fetch(
|
||
"https://unjobs.org/offices/bis_182320855689338880",
|
||
{ headers: HEADERS }
|
||
);
|
||
if (!res.ok) { console.log(` BIS … failed (${res.status})`); return jobs; }
|
||
|
||
const html = await res.text();
|
||
const cards = [...html.matchAll(
|
||
/<a[^>]+href="(https:\/\/unjobs\.org\/vacancies\/(\d+))"[\s\S]*?<span\s+class="jtitle">([^<]+)<\/span>([\s\S]*?)<\/a>/g
|
||
)];
|
||
|
||
let kept = 0;
|
||
for (const [, link, id, rawTitle, rest] of cards) {
|
||
// "Senior Platform Engineer, Basel, Switzerland" → title + location
|
||
const split = decodeHtml(rawTitle).match(/^(.*?),\s*([^,]+,\s*[^,]+)$/);
|
||
const title = split ? split[1] : decodeHtml(rawTitle);
|
||
const location = split ? split[2] : "Basel, Switzerland";
|
||
if (!applyIntlFilters(title)) continue;
|
||
|
||
jobs.push({
|
||
id: `bis-${id}`,
|
||
title,
|
||
company: "Bank for International Settlements",
|
||
location,
|
||
postedAt: rest.match(/datetime="(\d{4}-\d{2}-\d{2})/)?.[1] ?? "",
|
||
link,
|
||
search: "bis.org",
|
||
source: "bis.org",
|
||
contentChecked: true, // BIS publishes in English only
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` BIS … ${cards.length} ads, ${kept} kept`);
|
||
return jobs;
|
||
}
|
||
|
||
// Swiss TPH runs Umantis. Site chrome is German even on English postings, so
|
||
// isEnglishUrl would reject the lot — the title filters decide instead.
|
||
async function fetchSwissTph(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
const res = await fetch("https://jobs.swisstph.ch/Jobs/All", { headers: HEADERS });
|
||
if (!res.ok) { console.log(` Swiss TPH … failed (${res.status})`); return jobs; }
|
||
|
||
const html = await res.text();
|
||
// One block per posting, opened by the subtitle link that carries the title.
|
||
const blocks = html.split(/(?=<a\s[^>]*class="HSTableLinkSubTitle")/).slice(1);
|
||
|
||
let kept = 0;
|
||
for (const block of blocks) {
|
||
const hrefM = block.match(/href="(\/Vacancies\/(\d+)\/Description\/\d+)"/);
|
||
const titleM = block.match(/class="HSTableLinkSubTitle"[^>]*>([^<]+)</);
|
||
if (!hrefM || !titleM) continue;
|
||
|
||
const title = decodeHtml(titleM[1]);
|
||
if (!applyIntlFilters(title)) continue;
|
||
|
||
// "Online seit" label and its value sit in separate spans sharing an id suffix.
|
||
const dateM = block.match(/Online seit<\/span>[\s\S]{0,300}?>\s*(\d{2})\.(\d{2})\.(\d{4})/);
|
||
const locM = block.match(/>\s*Schweiz\s*-\s*([^<]+)</);
|
||
|
||
jobs.push({
|
||
id: `tph-${hrefM[2]}`,
|
||
title,
|
||
company: "Swiss TPH",
|
||
location: locM ? `${locM[1].trim()}, Switzerland` : "Allschwil, Switzerland",
|
||
postedAt: dateM ? `${dateM[3]}-${dateM[2]}-${dateM[1]}` : "",
|
||
link: `https://jobs.swisstph.ch${hrefM[1]}`,
|
||
search: "swisstph.ch",
|
||
source: "swisstph.ch",
|
||
contentChecked: true, // German site chrome would fail the page-level check
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` Swiss TPH … ${blocks.length} ads, ${kept} kept`);
|
||
return jobs;
|
||
}
|
||
|
||
// Basel Institute on Governance publishes each vacancy as a linked PDF, so
|
||
// there is no HTML description to language-check downstream.
|
||
async function fetchBaselGovernance(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
const res = await fetch("https://baselgovernance.org/about/careers", { headers: HEADERS });
|
||
if (!res.ok) { console.log(` Basel Governance … failed (${res.status})`); return jobs; }
|
||
|
||
const html = await res.text();
|
||
// The `rounded-3xl` card class is what separates vacancy tiles from other
|
||
// asset links on the page (the annual report PDF sits in a plain anchor).
|
||
const cards = [...html.matchAll(
|
||
/<a\s+href="(https:\/\/baselgovernance\.org\/cms\/api\/assets\/([^\/]+)\/[^"]+)"[^>]*class="[^"]*rounded-3xl[^"]*"[\s\S]{0,400}?<h2[^>]*>([^<]+)<\/h2>([\s\S]{0,900})/g
|
||
)];
|
||
|
||
let kept = 0;
|
||
for (const [, link, assetId, rawTitle, rest] of cards) {
|
||
const title = decodeHtml(rawTitle);
|
||
if (!applyIntlFilters(title)) continue;
|
||
|
||
const locM = rest.match(/map-pin[\s\S]{0,200}?<\/span>\s*([^<]+)</);
|
||
|
||
jobs.push({
|
||
id: `big-${assetId}`,
|
||
title,
|
||
company: "Basel Institute on Governance",
|
||
location: locM ? decodeHtml(locM[1]) : "Basel, Switzerland",
|
||
postedAt: "", // board shows the closing date only
|
||
link,
|
||
search: "baselgovernance.org",
|
||
source: "baselgovernance.org",
|
||
contentChecked: true, // postings are PDFs, and the institute publishes in English
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` Basel Governance … ${cards.length} ads, ${kept} kept`);
|
||
return jobs;
|
||
}
|
||
|
||
export async function fetchBaselIntlOrgs(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
for (const fetchOrg of [fetchBis, fetchSwissTph, fetchBaselGovernance]) {
|
||
try {
|
||
jobs.push(...(await fetchOrg()));
|
||
} catch (e) {
|
||
console.log(` error: ${e}`);
|
||
}
|
||
await Bun.sleep(400);
|
||
}
|
||
return jobs;
|
||
}
|
||
|
||
// sciCORE's open-positions page (formerly polled separately by
|
||
// scripts/scicore-watch.sh) is a plain-text announcement list, not a job
|
||
// board: each <li> is either an active posting linked to careers.unibas.ch
|
||
// or a closed/in-review notice with no link at all. Skip the latter — "the
|
||
// interview process is still ongoing, but the submission deadline has
|
||
// passed" is the site's own wording for "not accepting applications" — since
|
||
// they're not something Rafael could still apply to.
|
||
export async function fetchSciCore(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
const url = "https://scicore.unibas.ch/about-scicore/open-positions/";
|
||
const res = await fetch(url, { headers: HEADERS });
|
||
if (!res.ok) { console.log(` sciCORE … failed (${res.status})`); return jobs; }
|
||
|
||
const html = await res.text();
|
||
const sectionM = html.match(/<h1>Join the team!<\/h1>([\s\S]*?)<footer/i);
|
||
const items = sectionM ? [...sectionM[1].matchAll(/<li[^>]*>([\s\S]*?)<\/li>/gi)] : [];
|
||
|
||
let kept = 0;
|
||
for (const [, raw] of items) {
|
||
if (/deadline has passed|interview process is (?:still\s+)?ongoing/i.test(raw)) continue;
|
||
|
||
const linkM = raw.match(/<a\s[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
const title = decodeHtml((linkM ? linkM[2] : raw).replace(/<[^>]+>/g, " "));
|
||
if (!title || !applyFilters(title)) continue;
|
||
|
||
const link = linkM ? linkM[1] : url;
|
||
jobs.push({
|
||
id: `scicore-${link.replace(/\W+/g, "").slice(-24)}`,
|
||
title,
|
||
company: "sciCORE",
|
||
location: "Basel, Switzerland",
|
||
postedAt: "",
|
||
link,
|
||
search: "scicore.unibas.ch",
|
||
source: "scicore.unibas.ch",
|
||
contentChecked: true, // listing is site-authored English; no separate description to fetch
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` sciCORE … ${items.length} items, ${kept} kept`);
|
||
return jobs;
|
||
}
|
||
|
||
// jobbasel.ch — the page shell is a React/Vite SPA (no jobs in the raw HTML
|
||
// on the homepage), but the search results route is server-rendered: fetching
|
||
// `/job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__`
|
||
// blob already containing the matched vacancies as JSON. No headless browser
|
||
// needed. Pagination is a URL suffix (`-seite-{n}`), not a query param — a
|
||
// bare `?page=2` is silently ignored by the SPA route.
|
||
//
|
||
// The board (Basel-Stadt + Basel-Landschaft, ~4000 openings) has no keyword
|
||
// union endpoint, so — like job-room.ch — we sweep a fixed keyword list and
|
||
// dedupe by vacancy id. Each item carries its full German description
|
||
// (activity/requirements/offer), so the DE/FR check runs here instead of
|
||
// costing a page fetch per job in index.ts.
|
||
const JOBBASEL_KEYWORDS = [
|
||
"software", "engineer", "developer", "devops", "data", "python",
|
||
"cloud", "platform", "typescript", "backend", "full stack", "ai", "agentic",
|
||
];
|
||
|
||
interface JobBaselItem {
|
||
id: number;
|
||
title: string;
|
||
activity?: string;
|
||
requirements?: string;
|
||
offer?: string;
|
||
workplaceCity?: string;
|
||
urlApplication?: string;
|
||
dateFirstPublished?: string;
|
||
company?: { name?: string };
|
||
}
|
||
|
||
// Balanced-brace extraction — the blob can itself contain "};" inside string
|
||
// values (job descriptions), so a regex cutoff would truncate it.
|
||
function extractPreloadedState(html: string): any | null {
|
||
const marker = "window.__PRELOADED_STATE__";
|
||
const markerIdx = html.indexOf(marker);
|
||
if (markerIdx === -1) return null;
|
||
const start = html.indexOf("{", markerIdx);
|
||
if (start === -1) return null;
|
||
|
||
let depth = 0, inStr = false, esc = false, i = start;
|
||
for (; i < html.length; i++) {
|
||
const c = html[i];
|
||
if (inStr) {
|
||
if (esc) esc = false;
|
||
else if (c === "\\") esc = true;
|
||
else if (c === '"') inStr = false;
|
||
} else {
|
||
if (c === '"') inStr = true;
|
||
else if (c === "{") depth++;
|
||
else if (c === "}") { depth--; if (depth === 0) { i++; break; } }
|
||
}
|
||
}
|
||
|
||
try { return JSON.parse(html.slice(start, i)); } catch { return null; }
|
||
}
|
||
|
||
export async function fetchJobBasel(): Promise<Job[]> {
|
||
const items = new Map<number, JobBaselItem>();
|
||
|
||
for (const keyword of JOBBASEL_KEYWORDS) {
|
||
process.stdout.write(` ${keyword} … `);
|
||
const slug = keyword.toLowerCase().replace(/\s+/g, "-");
|
||
let total = 0;
|
||
|
||
for (let page = 1; page <= 5; page++) {
|
||
if (page > 1) await Bun.sleep(300);
|
||
const suffix = page === 1 ? "" : `-seite-${page}`;
|
||
const res = await fetch(
|
||
`https://www.jobbasel.ch/job/suche-${slug}${suffix}`,
|
||
{ headers: HEADERS }
|
||
);
|
||
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
||
|
||
const state = extractPreloadedState(await res.text());
|
||
const pageItems: JobBaselItem[] = state?.vacancySearch?.data?.items ?? [];
|
||
if (!pageItems.length) break;
|
||
|
||
for (const item of pageItems) items.set(item.id, item);
|
||
total += pageItems.length;
|
||
if (pageItems.length < 10) break; // last page (pageSize is 10)
|
||
}
|
||
console.log(`${total} ads (${items.size} unique)`);
|
||
await Bun.sleep(300);
|
||
}
|
||
|
||
const jobs: Job[] = [];
|
||
let langDropped = 0;
|
||
for (const item of items.values()) {
|
||
const title = decodeHtml(item.title.replace(/<[^>]+>/g, ""));
|
||
if (!applyFilters(title)) continue;
|
||
if (MS_STACK_RE.test(title)) continue;
|
||
|
||
const body = [item.activity, item.requirements, item.offer]
|
||
.filter(Boolean)
|
||
.join(" ")
|
||
.replace(/<[^>]+>/g, " ");
|
||
const hasBody = body.trim().length > 40;
|
||
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
||
|
||
jobs.push({
|
||
id: `jb-${item.id}`,
|
||
title,
|
||
company: item.company?.name ?? "",
|
||
location: item.workplaceCity ? `${item.workplaceCity}, Switzerland` : "Basel, Switzerland",
|
||
postedAt: item.dateFirstPublished?.slice(0, 10) ?? "",
|
||
link: item.urlApplication || `https://www.jobbasel.ch/job/${item.id}`,
|
||
search: "jobbasel.ch",
|
||
source: "jobbasel.ch",
|
||
contentChecked: hasBody,
|
||
});
|
||
}
|
||
|
||
console.log(` ${items.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
||
return jobs;
|
||
}
|
||
|
||
// englishjobsearch.ch — plain SSR HTML, no JS needed, but reached through a
|
||
// non-obvious lever: any page on the site accepts `?format=markdown` and
|
||
// returns the same content pre-converted to clean markdown text (job title,
|
||
// company, city, date, short description all present). Without that param
|
||
// the canton pages ship the job cards as plain text nodes too — no JSON blob,
|
||
// no client-side rendering — so this works with a bare `curl`-equivalent
|
||
// fetch either way; `?format=markdown` just makes parsing trivial.
|
||
//
|
||
// Two catches: (1) the directory root groups by canton, so we hit the
|
||
// Basel-Stadt and Basel-Landschaft canton pages directly rather than the
|
||
// generic `/in/basel/` landing page, which only lists canton counts.
|
||
// (2) every job link is a `/clickout/<hash>` tracking redirect, not the real
|
||
// posting URL — there is no way to recover the underlying employer link
|
||
// without following the redirect, so the clickout link is stored as-is (it
|
||
// still resolves for a human clicking it, and content-checking follows it
|
||
// server-side the same as any other link).
|
||
const ENGLISHJOBSEARCH_CANTONS = ["canton-basel-stadt", "canton-basel-landschaft"];
|
||
|
||
export async function fetchEnglishJobSearch(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (const canton of ENGLISHJOBSEARCH_CANTONS) {
|
||
process.stdout.write(` ${canton} … `);
|
||
const res = await fetch(
|
||
`https://englishjobsearch.ch/in/${canton}?format=markdown`,
|
||
{ headers: HEADERS }
|
||
);
|
||
if (!res.ok) { console.log(`failed (${res.status})`); continue; }
|
||
|
||
const md = await res.text();
|
||
// Each posting: "### Title](/clickout/<hash>?...)" heading, then on
|
||
// separate lines "* Company", "* City", "* Date", then a description
|
||
// paragraph before the next "### " heading or "report probem" footer.
|
||
const cards = [...md.matchAll(
|
||
/### ([^\]]+)\]\((\/clickout\/[^)]+)\)\n\n\* ([^\n]+)\n\* ([^\n]+)\n\* ([^\n]+)\n\n([\s\S]*?)\n\nreport probem/g
|
||
)];
|
||
|
||
let kept = 0;
|
||
for (const [, rawTitle, clickout, company, city, dateStr, desc] of cards) {
|
||
const title = rawTitle.trim();
|
||
if (!applyFilters(title)) continue;
|
||
if (MS_STACK_RE.test(title)) continue;
|
||
|
||
const id = clickout.match(/\/clickout\/([a-f0-9]+)/)?.[1] ?? clickout;
|
||
if (seen.has(id)) continue;
|
||
seen.add(id);
|
||
|
||
const description = desc.trim();
|
||
const hasBody = description.length > 40;
|
||
if (hasBody && !isViableDescription(description).ok) continue;
|
||
|
||
jobs.push({
|
||
id: `ejs-${id}`,
|
||
title,
|
||
company: company.trim(),
|
||
location: `${city.trim()}, Switzerland`,
|
||
postedAt: "", // "August 30" style, no year given — not worth mis-parsing
|
||
link: `https://englishjobsearch.ch${clickout}`,
|
||
search: "englishjobsearch.ch",
|
||
source: "englishjobsearch.ch",
|
||
contentChecked: hasBody,
|
||
});
|
||
kept++;
|
||
}
|
||
console.log(`${cards.length} ads, ${kept} kept`);
|
||
await Bun.sleep(300);
|
||
}
|
||
|
||
return jobs;
|
||
}
|
||
|
||
// more-jobs.ch — server-rendered PHP board, an aggregator (the same posting
|
||
// often also sits on LinkedIn/jobs.ch, deduped by title+company in index.ts).
|
||
// The site is Swiss-wide with ~20 cards per page; /recherche?q= is fast on the
|
||
// first pages but deep pages take 20s+ and out-of-range pages clamp to the last
|
||
// one instead of going empty, so "no new ids" ends a keyword's sweep, not an
|
||
// empty list. Cards link to /emploi/<slug>-<id>.html; the listing carries no
|
||
// description, so each title-filtered candidate costs one detail fetch, which
|
||
// also gives the real publication date (dd.mm.yyyy) and runs the DE/FR check.
|
||
const MOREJOBS_TERMS = ["software", "engineer", "developer", "devops", "platform", "architect", "data", "python", "agentic"];
|
||
const MOREJOBS_MAX_PAGES = 5;
|
||
// The board is Swiss but lists foreign postings too ("Eastern Europe, Poland",
|
||
// "95050 Santa Clara", "London"), and its JSON-LD stamps addressCountry "CH" on
|
||
// all of them — so the card's location text is the only signal. Drop anything
|
||
// naming another country/region/foreign city, or carrying a 5-digit postcode
|
||
// (Swiss ones are 4 digits).
|
||
const MOREJOBS_FOREIGN_RE = new RegExp(
|
||
"\\b(?:usa|united states|canada|mexico|brazil|india|china|japan|singapore|thailand|australia|" +
|
||
"united kingdom|uk|england|scotland|ireland|london|cambridge|cambridgeshire|manchester|" +
|
||
"germany|deutschland|berlin|munich|m(?:ü|u)nchen|frankfurt|hamburg|stuttgart|cologne|k(?:ö|o)ln|" +
|
||
"france|paris|lyon|annecy|italy|italia|milan|milano|rome|spain|madrid|barcelona|portugal|porto|lisbon|lisboa|" +
|
||
"austria|(?:ö|o)sterreich|vienna|wien|netherlands|amsterdam|belgium|brussels|luxembourg|denmark|sweden|norway|finland|" +
|
||
"poland|warsaw|krakow|romania|bucharest|czech|prague|hungary|budapest|bulgaria|sofia|serbia|belgrade|croatia|greece|" +
|
||
"slovakia|slovenia|ukraine|turkey|israel|dubai|uae)\\b|\\b\\d{5}\\b",
|
||
"i"
|
||
);
|
||
|
||
function moreJobsLocation(raw: string): string {
|
||
// Some cards repeat the location twice ("Zurich, Switzerland Zurich, Switzerland").
|
||
const half = raw.slice(0, raw.length / 2).trim();
|
||
const loc = raw.length % 2 === 1 && raw === `${half} ${half}` ? half : raw;
|
||
return loc.replace(/(?:,\s*Switzerland)+$/i, "").replace(/^[,\s]+|[,\s]+$/g, "");
|
||
}
|
||
|
||
async function fetchMoreJobsPage(url: string): Promise<string | null> {
|
||
for (let attempt = 0; attempt < 2; attempt++) {
|
||
try {
|
||
const res = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20_000) });
|
||
if (res.ok) return await res.text();
|
||
} catch {}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
export async function fetchMoreJobs(): Promise<Job[]> {
|
||
const candidates = new Map<string, { title: string; company: string; location: string; path: string }>();
|
||
|
||
for (const term of MOREJOBS_TERMS) {
|
||
process.stdout.write(` ${term} … `);
|
||
let found = 0;
|
||
for (let page = 1; page <= MOREJOBS_MAX_PAGES; page++) {
|
||
const html = await fetchMoreJobsPage(
|
||
`https://www.more-jobs.ch/recherche?q=${encodeURIComponent(term)}&page=${page}`
|
||
);
|
||
if (!html) break;
|
||
const cards = [...html.matchAll(
|
||
/<a class="positions-list-item[^"]*" href="(\/emploi\/[^"]*-(\d+)\.html)"[^>]*>\s*<h2><span>([^<]*)<\/span><\/h2>[\s\S]*?<h3><span[^>]*>([^<]*)<\/span> - <span>([^<]*)<\/span>/g
|
||
)];
|
||
let fresh = 0;
|
||
for (const [, path, id, title, company, location] of cards) {
|
||
if (candidates.has(id)) continue;
|
||
fresh++;
|
||
candidates.set(id, {
|
||
title: decodeHtml(title.trim()),
|
||
company: decodeHtml(company.trim()),
|
||
location: moreJobsLocation(decodeHtml(location.trim())),
|
||
path,
|
||
});
|
||
}
|
||
found += fresh;
|
||
if (fresh === 0 || cards.length < 20) break;
|
||
await Bun.sleep(300);
|
||
}
|
||
console.log(`${found} new`);
|
||
}
|
||
|
||
const jobs: Job[] = [];
|
||
const queue = [...candidates].filter(([, c]) =>
|
||
applyFilters(c.title) && !MS_STACK_RE.test(c.title) && !MOREJOBS_FOREIGN_RE.test(c.location)
|
||
);
|
||
console.log(` ${candidates.size} unique, ${queue.length} pass the title filters — fetching details`);
|
||
async function worker() {
|
||
while (queue.length) {
|
||
const [id, c] = queue.shift()!;
|
||
const html = await fetchMoreJobsPage(`https://www.more-jobs.ch${c.path}`);
|
||
if (!html) continue; // next run retries — nothing claimed yet
|
||
const body = html.match(/<div class="job-content"[^>]*>([\s\S]*?)<\/div>\s*(?:<!--|<div class="clearfix)/)?.[1] ?? "";
|
||
if (body && !isViableDescription(body).ok) continue;
|
||
const d = html.match(/Publication date<\/dt>\s*<dd>(\d{2})\.(\d{2})\.(\d{4})/);
|
||
jobs.push({
|
||
id: `morejobs-${id}`,
|
||
title: c.title,
|
||
company: c.company,
|
||
location: c.location ? `${c.location}, Switzerland` : "Switzerland",
|
||
postedAt: d ? `${d[3]}-${d[2]}-${d[1]}` : "",
|
||
link: `https://www.more-jobs.ch${c.path}`,
|
||
search: "more-jobs.ch",
|
||
source: "more-jobs.ch",
|
||
contentChecked: body.length > 40,
|
||
});
|
||
}
|
||
}
|
||
await Promise.all(Array.from({ length: 4 }, worker));
|
||
return jobs;
|
||
}
|
||
|
||
// stellen-basel.ch — WordPress REST API
|
||
export async function fetchStellenBasel(): Promise<Job[]> {
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (let page = 1; page <= 30; page++) {
|
||
if (page > 1) await Bun.sleep(400);
|
||
process.stdout.write(` page ${page} … `);
|
||
|
||
const res = await fetch(
|
||
`https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
|
||
{ headers: { ...HEADERS, Accept: "application/json" } }
|
||
);
|
||
if (!res.ok) { console.log("done"); break; }
|
||
|
||
const data = (await res.json()) as Array<{
|
||
id: number;
|
||
title: { rendered: string };
|
||
date: string;
|
||
link: string;
|
||
meta: { _company_name?: string; _job_location?: string };
|
||
}>;
|
||
|
||
if (!data.length) { console.log("0 items"); break; }
|
||
|
||
let newOnPage = 0;
|
||
for (const item of data) {
|
||
const title = decodeHtml(item.title.rendered);
|
||
if (!applyFilters(title)) continue;
|
||
|
||
const id = `sb-${item.id}`;
|
||
if (seen.has(id)) continue;
|
||
seen.add(id);
|
||
|
||
allJobs.push({
|
||
id,
|
||
title,
|
||
company: item.meta._company_name ?? "",
|
||
location: item.meta._job_location ?? "Basel",
|
||
postedAt: item.date.slice(0, 10),
|
||
link: item.link,
|
||
search: "stellen-basel.ch",
|
||
source: "stellen-basel.ch",
|
||
});
|
||
newOnPage++;
|
||
}
|
||
|
||
console.log(`${data.length} items, ${newOnPage} new`);
|
||
if (data.length < 100) break;
|
||
}
|
||
|
||
return allJobs;
|
||
}
|
||
|
||
// basel-jobs.ch — regional board on the JobChannel network (same syndication
|
||
// network whose ?source=jobchannel links show up on Siemens/jobs.ch postings).
|
||
//
|
||
// The board is protected by DataDome, which is why this source sat documented
|
||
// as "not integrable" for months. The block is real but narrower than it looked:
|
||
//
|
||
// - Plain curl / bun fetch → 403 on every UA, browser-like or not.
|
||
// - Chrome --headless=new, driven over CDP → 403. Headless is detected.
|
||
// - Playwright's own chromium.launch(), even headful → 403. The CDP-launch
|
||
// path leaves automation artefacts DataDome fingerprints.
|
||
// - A real Chrome launched *normally* (with --remote-debugging-port so we can
|
||
// attach afterwards) → 200. Attaching over CDP after the fact is invisible.
|
||
//
|
||
// The unlock is that DataDome's clearance lives entirely in a `datadome` cookie
|
||
// (Max-Age one year) which replays fine over plain fetch with a matching
|
||
// User-Agent. So we mint a cookie once with a real browser window, cache it, and
|
||
// scrape normally thereafter — a browser is touched only when the cookie is
|
||
// missing or has been rejected, not on every run. An earlier note in CLAUDE.md
|
||
// claimed the cookie "can't be extracted for replay"; that was true of the
|
||
// browser-extension tooling, not of CDP, which hands it over directly.
|
||
//
|
||
// Operational caveat: minting needs a *visible* Chrome window for ~10s, so an
|
||
// unattended `bun daily` under launchd cannot mint. When the cookie is stale in
|
||
// that context the source logs and yields nothing rather than failing the run;
|
||
// refresh it deliberately with `bun run scripts/mint-datadome.ts`.
|
||
|
||
const DATADOME_CACHE = "results/.datadome.json";
|
||
const CHROME_BIN = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
|
||
const BASELJOBS_ORIGIN = "https://basel-jobs.ch";
|
||
|
||
interface DataDomeSession {
|
||
cookie: string;
|
||
ua: string;
|
||
mintedAt: string;
|
||
}
|
||
|
||
async function loadDataDomeSession(): Promise<DataDomeSession | null> {
|
||
try {
|
||
const raw = await Bun.file(DATADOME_CACHE).text();
|
||
const s = JSON.parse(raw) as DataDomeSession;
|
||
return s.cookie && s.ua ? s : null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function waitForCdp(port: number, timeoutMs: number): Promise<boolean> {
|
||
const deadline = Date.now() + timeoutMs;
|
||
while (Date.now() < deadline) {
|
||
try {
|
||
const r = await fetch(`http://127.0.0.1:${port}/json/version`);
|
||
if (r.ok) return true;
|
||
} catch { /* not up yet */ }
|
||
await Bun.sleep(300);
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Launch a real, visible Chrome, let it clear DataDome, then attach over CDP and
|
||
* lift the resulting cookie. Headless and Playwright-launched browsers are both
|
||
* detected — only a normally-launched window gets through, which is the whole
|
||
* reason this is a separate, rarely-run step instead of the fetch path itself.
|
||
*/
|
||
export async function mintDataDomeSession(): Promise<DataDomeSession | null> {
|
||
if (process.env.LINKME_NO_BROWSER === "1") {
|
||
console.log(" LINKME_NO_BROWSER=1 — skipping DataDome mint");
|
||
return null;
|
||
}
|
||
if (!(await Bun.file(CHROME_BIN).exists())) {
|
||
console.error(` Chrome not found at ${CHROME_BIN} — cannot mint DataDome cookie`);
|
||
return null;
|
||
}
|
||
|
||
let chromium: typeof import("playwright-core").chromium;
|
||
try {
|
||
({ chromium } = await import("playwright-core"));
|
||
} catch {
|
||
console.error(" playwright-core not installed — run `bun add playwright-core`");
|
||
return null;
|
||
}
|
||
|
||
// A port unlikely to collide with a debugging session the user opened.
|
||
const port = 9333;
|
||
const profile = "results/.datadome-profile";
|
||
const proc = Bun.spawn(
|
||
[
|
||
CHROME_BIN,
|
||
`--remote-debugging-port=${port}`,
|
||
`--user-data-dir=${profile}`,
|
||
"--no-first-run",
|
||
"--no-default-browser-check",
|
||
"--window-size=1200,860",
|
||
BASELJOBS_ORIGIN,
|
||
],
|
||
{ stdout: "ignore", stderr: "ignore" }
|
||
);
|
||
|
||
try {
|
||
if (!(await waitForCdp(port, 20_000))) {
|
||
console.error(" Chrome CDP endpoint never came up");
|
||
return null;
|
||
}
|
||
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
|
||
const ctx = browser.contexts()[0];
|
||
const page =
|
||
ctx.pages().find((p) => p.url().includes("basel-jobs")) ?? ctx.pages()[0];
|
||
|
||
// The interstitial resolves itself; give it room, then confirm we're through.
|
||
let cookie = "";
|
||
let ua = "";
|
||
for (let attempt = 0; attempt < 12; attempt++) {
|
||
await page.waitForTimeout(1500);
|
||
const html = await page.content();
|
||
if (/Please enable JS|geo\.captcha/.test(html)) continue;
|
||
const cookies = await ctx.cookies(BASELJOBS_ORIGIN);
|
||
const dd = cookies.find((c) => c.name === "datadome");
|
||
if (!dd) continue;
|
||
cookie = dd.value;
|
||
ua = await page.evaluate(() => navigator.userAgent);
|
||
break;
|
||
}
|
||
await browser.close();
|
||
|
||
if (!cookie || !ua) {
|
||
console.error(" DataDome challenge did not clear — no cookie minted");
|
||
return null;
|
||
}
|
||
|
||
const session: DataDomeSession = { cookie, ua, mintedAt: new Date().toISOString() };
|
||
await Bun.write(DATADOME_CACHE, JSON.stringify(session, null, 2));
|
||
console.log(` minted DataDome cookie (${cookie.length} chars)`);
|
||
return session;
|
||
} catch (e) {
|
||
console.error(` DataDome mint failed: ${e}`);
|
||
return null;
|
||
} finally {
|
||
proc.kill();
|
||
}
|
||
}
|
||
|
||
function baselJobsHeaders(s: DataDomeSession): Record<string, string> {
|
||
return {
|
||
"User-Agent": s.ua,
|
||
"Accept-Language": "de-CH,de;q=0.9,en;q=0.8",
|
||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
Cookie: `datadome=${s.cookie}`,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Fetch with the cached cookie, re-minting once if the cookie has gone stale.
|
||
* DataDome rotates the cookie on ordinary responses too, so a Set-Cookie on the
|
||
* way past is folded back into the session rather than ignored.
|
||
*/
|
||
async function baselJobsFetch(
|
||
url: string,
|
||
state: { session: DataDomeSession | null; minted: boolean }
|
||
): Promise<string | null> {
|
||
for (let attempt = 0; attempt < 2; attempt++) {
|
||
if (!state.session) {
|
||
if (state.minted) return null;
|
||
state.minted = true;
|
||
state.session = await mintDataDomeSession();
|
||
if (!state.session) return null;
|
||
}
|
||
|
||
const res = await fetch(url, { headers: baselJobsHeaders(state.session) });
|
||
|
||
const rotated = res.headers
|
||
.getSetCookie?.()
|
||
.map((c) => /(?:^|;\s*)datadome=([^;]+)/.exec(c)?.[1])
|
||
.find(Boolean);
|
||
if (rotated && state.session) {
|
||
state.session = { ...state.session, cookie: rotated };
|
||
await Bun.write(DATADOME_CACHE, JSON.stringify(state.session, null, 2));
|
||
}
|
||
|
||
if (res.ok) return res.text();
|
||
|
||
if (res.status === 403 && !state.minted) {
|
||
console.log(" cookie rejected — re-minting");
|
||
state.session = null;
|
||
continue;
|
||
}
|
||
console.error(` failed (${res.status}) ${url}`);
|
||
return null;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// Terms swept through the board's search. No keyword-union endpoint exists, so
|
||
// — like job-room.ch and jobbasel.ch — this is a fixed list, one request each
|
||
// per mode, deduped by vacancy id. German stems are in deliberately: the search
|
||
// matches description text, so "entwickler" surfaces postings whose English
|
||
// title the category never filed under IT. The title and description filters
|
||
// drop the German-language ones afterwards.
|
||
//
|
||
// Role words and stack words behave differently across the two search modes
|
||
// (see fetchBaselJobs), which is why both modes get the whole list rather than
|
||
// a tuned subset — `websearch=1` finds no stack terms at all, and the bare
|
||
// search is the only mode that matches "react"/"frontend"/"backend".
|
||
const BASELJOBS_TERMS = [
|
||
// role / domain words — these carry the `websearch=1` pass
|
||
"software", "engineer", "developer", "entwickler", "informatik", "data",
|
||
"cloud", "devops", "platform", "architect", "security", "ai", "agentic",
|
||
// stack words — only the bare search matches these
|
||
"python", "javascript", "typescript", "react", "node",
|
||
"fullstack", "full stack", "backend", "frontend", "kubernetes", "linux",
|
||
];
|
||
|
||
function buildBaselJobsLocation(city: string, canton: string): string {
|
||
if (!city) return canton || "Basel";
|
||
if (!canton) return city;
|
||
// data-gtm-jp is one or more codes ("BS", "BL,BS"); keep only what the city
|
||
// string doesn't already name.
|
||
const missing = canton
|
||
.split(",")
|
||
.map((c) => c.trim())
|
||
.filter((c) => c && !new RegExp(`\\b${c}\\b`).test(city));
|
||
return missing.length ? `${city}, ${missing.join(",")}` : city;
|
||
}
|
||
|
||
/**
|
||
* Pull every vacancy card out of a listing page. Returns how many ids were
|
||
* new so callers can use "no new postings" as their end condition — past the
|
||
* last real page the board still serves its two sponsored slots, so an empty
|
||
* list never arrives.
|
||
*/
|
||
function parseBaselJobsCards(
|
||
html: string,
|
||
seen: Set<string>,
|
||
out: Job[],
|
||
today: string
|
||
): { items: number; added: number } {
|
||
const items = [...html.matchAll(/<li class="item"(.*?)<\/li>/gs)].map((m) => m[1]);
|
||
let added = 0;
|
||
|
||
for (const card of items) {
|
||
const jid = /data-gtm-jid="(\d+)"/.exec(card)?.[1];
|
||
const linkM = /href="(https:\/\/basel-jobs\.ch\/job\/[^"]+)"/.exec(card);
|
||
const titleM = /<span class="jobtitle">(.*?)<\/span>/s.exec(card);
|
||
if (!jid || !linkM || !titleM) continue;
|
||
|
||
// Sponsored slots repeat the same posting within a page, between pages,
|
||
// and across the category and search sweeps.
|
||
const id = `bj-${jid}`;
|
||
if (seen.has(id)) continue;
|
||
seen.add(id);
|
||
added++;
|
||
|
||
const title = decodeHtml(titleM[1]);
|
||
if (!applyFilters(title)) continue;
|
||
|
||
const companyM =
|
||
/<span class="company">\s*<a[^>]*>(.*?)<\/a>/s.exec(card) ??
|
||
/<span class="company">(.*?)<\/span>/s.exec(card);
|
||
const locationM = /<span class="location">(.*?)<\/span>/s.exec(card);
|
||
const canton = /data-gtm-jp="([^"]*)"/.exec(card)?.[1] ?? "";
|
||
|
||
out.push({
|
||
id,
|
||
title,
|
||
company: companyM ? decodeHtml(companyM[1]) : "",
|
||
// Cards carry a city; the canton attribute disambiguates BS vs BL,
|
||
// which is what daily.ts groups on. Some cards already end in the
|
||
// canton code ("Basel, BS, 4030") — don't append it twice.
|
||
location: buildBaselJobsLocation(
|
||
locationM ? decodeHtml(locationM[1]) : "",
|
||
canton
|
||
),
|
||
// No date anywhere on the card or detail page — discovery date is the
|
||
// best available, same as any board that hides posting age.
|
||
postedAt: today,
|
||
link: linkM[1],
|
||
search: "basel-jobs.ch",
|
||
source: "basel-jobs.ch",
|
||
});
|
||
}
|
||
|
||
return { items: items.length, added };
|
||
}
|
||
|
||
// novartis.com — Rafael has an internal contact at Novartis Basel who can refer
|
||
// him, so this source watches specifically for roles that would be worth
|
||
// flagging to that contact rather than applying cold.
|
||
//
|
||
// The career-search page (Drupal Views) is server-rendered HTML, no headless
|
||
// browser needed — but only at the locale-less path (`/careers/career-search`,
|
||
// English). The `ch-de` path serves the same listing in German and its
|
||
// "job/details" links 302 back to the search page for both locales when hit
|
||
// without a referrer/session, so job detail pages are only readable through
|
||
// the English path (`/careers/career-search/job/details/req-...`), which
|
||
// resolves fine and returns the full posting.
|
||
//
|
||
// Pagination is 0-indexed (`page=0` is implicit/omitted, `page=1` is the
|
||
// second page of 10) and the page's own "Showing N results" header gives the
|
||
// exact total, so the loop bound is computed rather than probed.
|
||
//
|
||
// Novartis is 100% pharma, and its Basel HQ titling skews heavily to
|
||
// non-technical "Director, <business function>" roles (Portfolio Analytics,
|
||
// Therapeutic Area Strategy, Policy Communications, ...) plus device/hardware
|
||
// "Engineering" roles (autoinjector/drug-delivery hardware, not software).
|
||
// SOFTWARE_TITLE_RE's bare "director"/"analytics"/"ai\b" branches — tuned for
|
||
// smaller, less title-inflated boards — matched every one of those in an
|
||
// initial run (measured 2026-09-21: 13 of 47 CH listings "kept", none of them
|
||
// an actual engineering-team role). A dedicated, stricter title match is used
|
||
// here instead: it requires an explicit software/IT engineering job-family
|
||
// term, not just a seniority or business-function word BIO_DOMAIN_RE and
|
||
// PHARMA_ROLE_RE happen to miss.
|
||
const NOVARTIS_TITLE_RE =
|
||
/\b(software\s+(engineer|developer|architect)|solutions?\s+architect|platform\s+engineer|cloud\s+engineer|devops(?:\s+engineer)?|data\s+engineer|site\s+reliability\s+engineer|\bsre\b|backend\s+(?:engineer|developer)|frontend\s+(?:engineer|developer)|full.?stack\s+(?:developer|engineer)|machine\s+learning\s+engineer|\bmlops\b|infrastructure\s+engineer|integration\s+engineer|automation\s+engineer|it\s+engineer|systems?\s+engineer|application[s]?\s+engineer|security\s+engineer|ai\s+engineer|agentic|engineering\s+manager|technical\s+lead|tech\s+lead|principal\s+(?:software\s+)?engineer|staff\s+engineer|product\s+engineer|programmer|it\s+architect|enterprise\s+architect|data\s+scientist|data\s+platform|forward\s+deployed|\bfde\b|palantir|foundry\s+(?:engineer|developer)|digital\s+product\s+(?:engineer|manager)|it\s+project\s+manager|it\s+manager)\b/i;
|
||
|
||
// Narrowed to the Basel region specifically (per the source's whole purpose —
|
||
// the inside contact is there), not all of Switzerland: Novartis CH postings
|
||
// also list Stein (Aargau) and a handful of remote/blank locations that are a
|
||
// different commute and not what the contact could refer into.
|
||
const NOVARTIS_BASE = "https://www.novartis.com";
|
||
const NOVARTIS_BASEL_RE = /\bbasel\b|\bmuttenz\b/i;
|
||
|
||
interface NovartisListing {
|
||
title: string;
|
||
href: string;
|
||
businessUnit: string;
|
||
location: string;
|
||
postedAt: string;
|
||
}
|
||
|
||
function parseNovartisDate(text: string): string {
|
||
const d = new Date(text.trim());
|
||
return isNaN(d.getTime()) ? "" : d.toISOString().slice(0, 10);
|
||
}
|
||
|
||
function parseNovartisPage(html: string): { total: number; listings: NovartisListing[] } {
|
||
const totalM = /Showing\s+(\d+)\s+results/i.exec(html);
|
||
const total = totalM ? Number(totalM[1]) : 0;
|
||
|
||
const listings: NovartisListing[] = [];
|
||
const rows = [...html.matchAll(/<tr class="(?:odd|even)"[^>]*>([\s\S]*?)<\/tr>/g)];
|
||
for (const [, row] of rows) {
|
||
const titleM = /views-field-field-job-title"><a href="([^"]+)">([^<]*)<\/a>/.exec(row);
|
||
const buM = /views-field-field-job-business-unit">([^<]*)<\/td>/.exec(row);
|
||
const locM = /views-field-field-job-work-location">([^<]*)<\/td>/.exec(row);
|
||
const dateM = /views-field-field-job-posted-date">([^<]*)<\/td>/.exec(row);
|
||
if (!titleM) continue;
|
||
|
||
listings.push({
|
||
href: titleM[1],
|
||
title: decodeHtml(titleM[2]),
|
||
businessUnit: buM ? decodeHtml(buM[1]) : "",
|
||
location: locM ? decodeHtml(locM[1]) : "",
|
||
postedAt: dateM ? parseNovartisDate(dateM[1]) : "",
|
||
});
|
||
}
|
||
return { total, listings };
|
||
}
|
||
|
||
function applyNovartisFilters(title: string): boolean {
|
||
if (NON_ENGLISH_RE.test(title)) return false;
|
||
if (JUNIOR_TITLE_RE.test(title)) return false;
|
||
if (MS_STACK_RE.test(title)) return false;
|
||
if (BIO_DOMAIN_RE.test(title)) return false;
|
||
if (PHARMA_ROLE_RE.test(title)) return false;
|
||
return NOVARTIS_TITLE_RE.test(title);
|
||
}
|
||
|
||
export async function fetchNovartisBasel(): Promise<Job[]> {
|
||
const jobs: Job[] = [];
|
||
// The listing has been observed to repeat rows across pages within one
|
||
// sweep (measured 2026-09-21: several titles appeared twice across the 5
|
||
// pages of one run) — dedupe by req id rather than trusting the pager.
|
||
const byReqId = new Map<string, NovartisListing>();
|
||
|
||
const firstHtml = await (
|
||
await fetch(`${NOVARTIS_BASE}/careers/career-search?search_api_fulltext=&country%5B0%5D=LOC_CH&field_job_posted_date=All&op=Submit`, { headers: HEADERS })
|
||
).text();
|
||
const { total, listings: firstListings } = parseNovartisPage(firstHtml);
|
||
const totalPages = Math.max(1, Math.ceil(total / 10));
|
||
console.log(` Novartis … ${total} CH results, ${totalPages} pages`);
|
||
for (const item of firstListings) {
|
||
const reqId = /req-(\d+)/.exec(item.href)?.[1];
|
||
if (reqId) byReqId.set(reqId, item);
|
||
}
|
||
|
||
for (let page = 1; page < totalPages; page++) {
|
||
await Bun.sleep(400);
|
||
const res = await fetch(
|
||
`${NOVARTIS_BASE}/careers/career-search?search_api_fulltext=&country%5B0%5D=LOC_CH&field_job_posted_date=All&op=Submit&page=${page}`,
|
||
{ headers: HEADERS }
|
||
);
|
||
if (!res.ok) { console.log(` Novartis … page ${page} failed (${res.status})`); break; }
|
||
const { listings } = parseNovartisPage(await res.text());
|
||
for (const item of listings) {
|
||
const reqId = /req-(\d+)/.exec(item.href)?.[1];
|
||
if (reqId) byReqId.set(reqId, item);
|
||
}
|
||
}
|
||
|
||
let kept = 0;
|
||
let langDropped = 0;
|
||
for (const [reqId, item] of byReqId) {
|
||
if (!NOVARTIS_BASEL_RE.test(item.location)) continue;
|
||
if (!applyNovartisFilters(item.title)) continue;
|
||
|
||
const link = `${NOVARTIS_BASE}${item.href}`;
|
||
await Bun.sleep(300);
|
||
const detailRes = await fetch(link, { headers: HEADERS });
|
||
let contentChecked = false;
|
||
if (detailRes.ok) {
|
||
const detailHtml = await detailRes.text();
|
||
const start = detailHtml.indexOf("job_details_content_center");
|
||
const end = detailHtml.indexOf("job_details_content_bottom");
|
||
if (start !== -1 && end !== -1) {
|
||
const description = detailHtml.slice(start, end).replace(/<[^>]+>/g, " ");
|
||
const check = isViableDescription(description);
|
||
if (!check.ok) { langDropped++; continue; }
|
||
contentChecked = true;
|
||
}
|
||
}
|
||
|
||
jobs.push({
|
||
id: `nov-${reqId}`,
|
||
title: item.title,
|
||
company: "Novartis",
|
||
location: `${item.location}, Switzerland`,
|
||
postedAt: item.postedAt,
|
||
link,
|
||
search: "novartis.com",
|
||
source: "novartis.com",
|
||
contentChecked,
|
||
});
|
||
kept++;
|
||
}
|
||
|
||
console.log(` Novartis … ${byReqId.size} unique CH listings, ${kept} kept (Basel-area + profile match), ${langDropped} non-English/German-required dropped`);
|
||
return jobs;
|
||
}
|
||
|
||
export async function fetchBaselJobs(): Promise<Job[]> {
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
const state = { session: await loadDataDomeSession(), minted: false };
|
||
const today = new Date().toISOString().slice(0, 10);
|
||
|
||
// Pass 1 — the board's IT category, ~300 postings over 3 pages.
|
||
for (let page = 1; page <= 10; page++) {
|
||
if (page > 1) await Bun.sleep(400);
|
||
process.stdout.write(` informatik page ${page} … `);
|
||
|
||
const html = await baselJobsFetch(
|
||
`${BASELJOBS_ORIGIN}/informatik-jobs?page=${page}`,
|
||
state
|
||
);
|
||
if (html === null) { console.log("unavailable"); break; }
|
||
|
||
const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
|
||
if (items === 0) { console.log("0 items"); break; }
|
||
console.log(`${items} items, ${added} new`);
|
||
if (added === 0) break;
|
||
}
|
||
|
||
// Pass 2 — full-text search. The category is a *curated taxonomy*, not the
|
||
// whole board: it only holds what the board filed under IT, so tech roles
|
||
// parked under Industrie/Pharma/Bau, and roles whose relevance lives in the
|
||
// description rather than the title, never appear in pass 1. Measured on
|
||
// 2026-09-01 this pass added **147 postings the category missed**, including
|
||
// "Senior AI Infrastructure & Platform Engineer (Basel)" and "Senior DevOps /
|
||
// Cloud Engineer (Kubernetes, CI/CD, Linux)". The reverse is also true (the
|
||
// category held 257 the search missed), so both passes are needed.
|
||
//
|
||
// `websearch=1` is the mode that does this — the bare `?terms=` search
|
||
// returned nothing that wasn't already in the category. It ignores `page`
|
||
// entirely and returns the whole result set on one response, so there is no
|
||
// pagination to walk.
|
||
// Both search modes are swept, because they query different indexes and each
|
||
// is blind to what the other finds:
|
||
//
|
||
// term websearch=1 bare
|
||
// engineer 90 260 found
|
||
// frontend 0 2
|
||
// backend 0 4
|
||
// react 0 2
|
||
//
|
||
// `websearch=1` matches role/domain words and reaches postings filed outside
|
||
// the IT category; the bare search is the *only* mode that matches stack
|
||
// terms. Neither paginates — both ignore `page` and return their whole result
|
||
// set in one response (bare caps around 100 cards). Measured 2026-09-01
|
||
// against the category: websearch added 147, bare added a further 139 raw
|
||
// (23 surviving the title filters).
|
||
for (const mode of ["websearch", "bare"] as const) {
|
||
for (const term of BASELJOBS_TERMS) {
|
||
await Bun.sleep(400);
|
||
process.stdout.write(` ${mode} "${term}" … `);
|
||
|
||
const qs = `terms=${encodeURIComponent(term)}${mode === "websearch" ? "&page=1&websearch=1" : ""}`;
|
||
const html = await baselJobsFetch(`${BASELJOBS_ORIGIN}/search?${qs}`, state);
|
||
if (html === null) { console.log("unavailable"); break; }
|
||
|
||
const { items, added } = parseBaselJobsCards(html, seen, allJobs, today);
|
||
console.log(`${items} items, ${added} new`);
|
||
}
|
||
}
|
||
|
||
// Detail pages carry the full description inline, so the DE/FR and
|
||
// German-requirement checks run here instead of costing index.ts a fetch that
|
||
// would hit DataDome without the cookie and be dropped as unreachable.
|
||
const viable: Job[] = [];
|
||
for (const job of allJobs) {
|
||
await Bun.sleep(300);
|
||
process.stdout.write(` checking ${job.title} … `);
|
||
const page = await baselJobsFetch(job.link, state);
|
||
if (page === null) { console.log("unreachable, keeping"); viable.push(job); continue; }
|
||
|
||
// Only the posting's own text, never the page. basel-jobs.ch is a German
|
||
// site, so its nav/footer/category chrome alone clears the ≥ 6 DE/FR word
|
||
// threshold — checking the whole page dropped *every* job, English ones
|
||
// included. The mobile `plain-content` block is the description as plain
|
||
// text; the desktop view loads the same thing into an iframe, so this is
|
||
// the only copy present in the server-rendered HTML.
|
||
const descM = /<div class="box plain-content[^"]*">(.*?)<\/div>/s.exec(page);
|
||
if (!descM) { console.log("no description, keeping"); viable.push(job); continue; }
|
||
const check = isViableDescription(descM[1].replace(/<br\s*\/?>/gi, " "));
|
||
if (!check.ok) { console.log(`dropped (${check.reason})`); continue; }
|
||
console.log("ok");
|
||
viable.push({ ...job, contentChecked: true });
|
||
}
|
||
|
||
return viable;
|
||
}
|
||
|
||
// startup.jobs — global startup board, swept Switzerland-wide. Every path on
|
||
// the domain (listing, detail pages, feeds, even /robots-adjacent API routes)
|
||
// sits behind a Cloudflare managed challenge that 403s plain HTTP on any UA.
|
||
// Unlike basel-jobs.ch's DataDome cookie, the `cf_clearance` cookie does NOT
|
||
// replay over Bun's fetch (measured: 403 with the exact cookie + UA) — it is
|
||
// bound to Chrome's TLS fingerprint. What gets through:
|
||
// - Chrome `--headless=new` with a fresh profile passes the challenge, but
|
||
// only with the "HeadlessChrome" token stripped from the UA (403 otherwise);
|
||
// - the listings themselves come from Algolia (app 4CQMTMMK73, index
|
||
// Post_production), queried with a *secured* search key the page embeds in
|
||
// `meta[name="current-algolia-api-key-search"]`. algolia.net is not behind
|
||
// Cloudflare, so once the key is in hand the queries run over plain fetch.
|
||
// The key is minted per page load with `validUntil` ~3–5h out, so it is read
|
||
// fresh every run, never cached. That makes this source need a (headless)
|
||
// browser on every run — which, unlike basel-jobs.ch's visible-window mint,
|
||
// works fine under launchd.
|
||
//
|
||
// The secured key rejects every country filter (`country_code`, `country_id`
|
||
// both return 0), so the sweep is a geo bounding box around Switzerland with
|
||
// `country_code === "CH"` checked locally (the box also catches DE/FR/AT/LI/IT).
|
||
// Algolia caps a query at 500 retrievable hits (100 × 5 pages) while the box
|
||
// holds ~1400, so a box reporting > 500 is split into quadrants until each fits.
|
||
// Records carry no description; each title-filtered candidate's detail page is
|
||
// fetched through the browser (`page.evaluate(fetch)`, same TLS) and its
|
||
// JSON-LD `JobPosting.description` runs through isViableDescription in-process.
|
||
// Links already in results/jobs.csv are skipped before that fetch — daily runs
|
||
// 5×/day and the detail fetches are the only expensive part.
|
||
const STARTUPJOBS_ORIGIN = "https://startup.jobs";
|
||
const STARTUPJOBS_ALGOLIA = "https://4CQMTMMK73-dsn.algolia.net/1/indexes/Post_production/query";
|
||
const STARTUPJOBS_CH_BOX: [number, number, number, number] = [45.8, 5.9, 47.85, 10.5];
|
||
const STARTUPJOBS_MAX_AGE_DAYS = 60;
|
||
|
||
interface StartupJobsHit {
|
||
objectID: string;
|
||
title: string;
|
||
company_name: string;
|
||
country_code?: string;
|
||
location?: string;
|
||
path: string;
|
||
published_at_iso8601?: string;
|
||
}
|
||
|
||
function chromeUserAgent(): string | null {
|
||
const out = Bun.spawnSync([CHROME_BIN, "--version"]).stdout.toString();
|
||
const version = /(\d+\.\d+\.\d+\.\d+)/.exec(out)?.[1];
|
||
if (!version) return null;
|
||
return `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version} Safari/537.36`;
|
||
}
|
||
|
||
async function startupJobsQuery(
|
||
key: string,
|
||
box: [number, number, number, number],
|
||
since: number,
|
||
page: number
|
||
): Promise<{ nbHits: number; nbPages: number; hits: StartupJobsHit[] }> {
|
||
const res = await fetch(STARTUPJOBS_ALGOLIA, {
|
||
method: "POST",
|
||
headers: { "X-Algolia-Application-Id": "4CQMTMMK73", "X-Algolia-API-Key": key },
|
||
body: JSON.stringify({
|
||
query: "",
|
||
insideBoundingBox: [box],
|
||
numericFilters: [`published_at_i>${since}`],
|
||
hitsPerPage: 100,
|
||
page,
|
||
attributesToRetrieve: ["title", "company_name", "country_code", "location", "path", "published_at_iso8601"],
|
||
attributesToHighlight: [],
|
||
}),
|
||
});
|
||
if (!res.ok) throw new Error(`algolia ${res.status}`);
|
||
return res.json();
|
||
}
|
||
|
||
async function startupJobsSweep(
|
||
key: string,
|
||
box: [number, number, number, number],
|
||
since: number,
|
||
out: Map<string, StartupJobsHit>,
|
||
depth = 0
|
||
): Promise<void> {
|
||
const first = await startupJobsQuery(key, box, since, 0);
|
||
if (first.nbHits > 500 && depth < 6) {
|
||
const [s, w, n, e] = box;
|
||
const midLat = (s + n) / 2;
|
||
const midLng = (w + e) / 2;
|
||
for (const q of [
|
||
[s, w, midLat, midLng], [s, midLng, midLat, e],
|
||
[midLat, w, n, midLng], [midLat, midLng, n, e],
|
||
] as [number, number, number, number][]) {
|
||
await startupJobsSweep(key, q, since, out, depth + 1);
|
||
}
|
||
return;
|
||
}
|
||
if (first.nbHits > 500) console.log(` warning: box ${box.join(",")} still holds ${first.nbHits} hits — truncated at 500`);
|
||
for (const h of first.hits) out.set(h.objectID, h);
|
||
for (let p = 1; p < first.nbPages; p++) {
|
||
const r = await startupJobsQuery(key, box, since, p);
|
||
for (const h of r.hits) out.set(h.objectID, h);
|
||
}
|
||
}
|
||
|
||
async function knownCsvLinks(): Promise<Set<string>> {
|
||
try {
|
||
const text = await Bun.file("results/jobs.csv").text();
|
||
return new Set(text.match(/https:\/\/startup\.jobs\/[^\s",]+/g) ?? []);
|
||
} catch {
|
||
return new Set();
|
||
}
|
||
}
|
||
|
||
export async function fetchStartupJobs(): Promise<Job[]> {
|
||
if (process.env.LINKME_NO_BROWSER === "1") {
|
||
console.log(" LINKME_NO_BROWSER=1 — startup.jobs unavailable");
|
||
return [];
|
||
}
|
||
const ua = (await Bun.file(CHROME_BIN).exists()) ? chromeUserAgent() : null;
|
||
if (!ua) {
|
||
console.error(` Chrome not found at ${CHROME_BIN} — startup.jobs unavailable`);
|
||
return [];
|
||
}
|
||
|
||
let chromium: typeof import("playwright-core").chromium;
|
||
try {
|
||
({ chromium } = await import("playwright-core"));
|
||
} catch {
|
||
console.error(" playwright-core not installed — run `bun add playwright-core`");
|
||
return [];
|
||
}
|
||
|
||
// Own port so it never collides with the DataDome mint (9333) running alongside.
|
||
const port = 9334;
|
||
const profile = `${tmpdir()}/linkme-startupjobs-${process.pid}`;
|
||
const proc = Bun.spawn(
|
||
[
|
||
CHROME_BIN,
|
||
"--headless=new",
|
||
`--user-agent=${ua}`,
|
||
`--remote-debugging-port=${port}`,
|
||
`--user-data-dir=${profile}`,
|
||
"--no-first-run",
|
||
"--no-default-browser-check",
|
||
"about:blank",
|
||
],
|
||
{ stdout: "ignore", stderr: "ignore" }
|
||
);
|
||
|
||
const jobs: Job[] = [];
|
||
try {
|
||
if (!(await waitForCdp(port, 20_000))) {
|
||
console.error(" Chrome CDP endpoint never came up — startup.jobs unavailable");
|
||
return [];
|
||
}
|
||
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
|
||
try {
|
||
const page = browser.contexts()[0].pages()[0] ?? (await browser.contexts()[0].newPage());
|
||
await page.goto(STARTUPJOBS_ORIGIN, { timeout: 30_000 });
|
||
|
||
let key: string | null = null;
|
||
for (let i = 0; i < 15 && !key; i++) {
|
||
key = await page
|
||
.evaluate(() => document.querySelector('meta[name="current-algolia-api-key-search"]')?.getAttribute("content") ?? null)
|
||
.catch(() => null); // the challenge navigates mid-evaluate
|
||
if (!key) await page.waitForTimeout(1000);
|
||
}
|
||
if (!key) {
|
||
console.error(" Cloudflare challenge did not clear (no Algolia key) — startup.jobs unavailable");
|
||
return [];
|
||
}
|
||
|
||
const since = Math.floor(Date.now() / 1000) - STARTUPJOBS_MAX_AGE_DAYS * 86400;
|
||
const hits = new Map<string, StartupJobsHit>();
|
||
await startupJobsSweep(key, STARTUPJOBS_CH_BOX, since, hits);
|
||
|
||
const known = await knownCsvLinks();
|
||
const candidates = [...hits.values()].filter((h) => {
|
||
if (h.country_code !== "CH") return false;
|
||
const title = decodeHtml(h.title);
|
||
if (!applyFilters(title)) return false;
|
||
if (MS_STACK_RE.test(title) || BIO_DOMAIN_RE.test(title)) return false;
|
||
return !known.has(`${STARTUPJOBS_ORIGIN}${h.path}`);
|
||
});
|
||
console.log(` ${hits.size} in CH box, ${candidates.length} new title-matched candidates`);
|
||
|
||
let langDropped = 0;
|
||
for (const h of candidates) {
|
||
const description = await page
|
||
.evaluate(async (path) => {
|
||
const r = await fetch(path);
|
||
if (!r.ok) return null;
|
||
const doc = new DOMParser().parseFromString(await r.text(), "text/html");
|
||
for (const s of doc.querySelectorAll('script[type="application/ld+json"]')) {
|
||
try {
|
||
const data = JSON.parse(s.textContent ?? "");
|
||
for (const d of Array.isArray(data) ? data : [data]) {
|
||
if (d?.["@type"] === "JobPosting") return String(d.description ?? "");
|
||
}
|
||
} catch { /* malformed block */ }
|
||
}
|
||
return null;
|
||
}, h.path)
|
||
.catch(() => null);
|
||
if (description === null) continue; // detail page failed — retry next run
|
||
if (!isViableDescription(description).ok) { langDropped++; continue; }
|
||
|
||
jobs.push({
|
||
id: `sj-${h.objectID}`,
|
||
title: decodeHtml(h.title),
|
||
company: h.company_name ?? "",
|
||
location: h.location ?? "Switzerland",
|
||
postedAt: h.published_at_iso8601?.slice(0, 10) ?? "",
|
||
link: `${STARTUPJOBS_ORIGIN}${h.path}`,
|
||
search: "startup.jobs",
|
||
source: "startup.jobs",
|
||
contentChecked: true, // isViableDescription already ran on the JSON-LD description
|
||
});
|
||
await Bun.sleep(300);
|
||
}
|
||
console.log(` ${jobs.length} kept, ${langDropped} non-English / German-required`);
|
||
} finally {
|
||
await browser.close();
|
||
}
|
||
} catch (e) {
|
||
console.error(` startup.jobs failed: ${e}`);
|
||
} finally {
|
||
proc.kill();
|
||
await rm(profile, { recursive: true, force: true }).catch(() => {});
|
||
}
|
||
return jobs;
|
||
}
|