rodado
d195d67c2a
jobbasel.ch's search route (/job/suche-{slug}) is server-rendered with
a window.__PRELOADED_STATE__ JSON blob, no headless browser needed.
englishjobsearch.ch works the same way via its ?format=markdown param.
Both swept with the same fixed keyword list job-room.ch already uses.
basel-jobs.ch was evaluated too but left out: it's DataDome-protected,
which blocks plain curl outright, and this project's fetchers are
intentionally browser-free.
While wiring these in, found that isEnglishDescription (now
isViableDescription) only ever ran the DE/FR word-count check against
inline descriptions, never requiresGerman() - so an English-written
posting demanding fluent German (e.g. PRODYNA's "you can communicate
fluently in both German and English") sailed straight through. That
gap existed in itjobs.ch and job-room.ch too, not just the two new
sources, so the fix applies to all four.
711 lines
26 KiB
TypeScript
711 lines
26 KiB
TypeScript
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman } from "./fetch.ts";
|
|
import type { Job } from "./fetch.ts";
|
|
|
|
const HEADERS = {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
};
|
|
|
|
function decodeHtml(s: string): string {
|
|
return s
|
|
.replace(/&/g, "&")
|
|
.replace(/</g, "<")
|
|
.replace(/>/g, ">")
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
function applyFilters(title: string): boolean {
|
|
if (NON_ENGLISH_RE.test(title)) return false;
|
|
if (JUNIOR_TITLE_RE.test(title)) return false;
|
|
if (!SOFTWARE_TITLE_RE.test(title)) return false;
|
|
return true;
|
|
}
|
|
|
|
const MS_STACK_RE = /\.net\b|\bc#|sharepoint|dynamics 365|power platform|\bsccm\b|intune|successfactors|\bsap\b/i;
|
|
|
|
// DE/FR indicator words, counted against a description we already have in hand.
|
|
// Mirrors the ≥ 6 threshold isEnglishContent uses on fetched pages.
|
|
const DE_FR_WORDS_RE =
|
|
/\b(und|oder|wir|Sie|Ihre|Ihrer|Kenntnisse|Erfahrung|Aufgaben|Anforderungen|Stelle|Unternehmen|Mitarbeiter|Bereich|nous|vous|votre|pour|avec|des|les)\b/g;
|
|
|
|
// English-language AND not demanding German above B1 — mirrors isViableContent/
|
|
// isViableUrl in fetch.ts, but against a description we already have in hand
|
|
// instead of a page fetch. The two checks catch different things: the word
|
|
// count drops postings *written* in German/French; requiresGerman() drops
|
|
// postings *written in English* that still demand German above B1 (e.g.
|
|
// PRODYNA: "you can communicate fluently in both German and English" — clean
|
|
// English text, would sail through the word count untouched). Skipping the
|
|
// requiresGerman() half was a real gap in every source below until 2026-08-30
|
|
// — it let German-fluency requirements slip past whenever an item's inline
|
|
// description read as English.
|
|
function isViableDescription(description?: string): { ok: boolean; reason?: string } {
|
|
if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide
|
|
const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " ");
|
|
if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" };
|
|
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
|
return { ok: true };
|
|
}
|
|
|
|
// datacareer.ch — HTML listing at /jobs/?city=Basel
|
|
export async function fetchDataCareer(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 10; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.error(`failed (${res.status})`); break; }
|
|
|
|
const html = await res.text();
|
|
const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
|
|
if (articles.length === 0) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
for (const [, card] of articles) {
|
|
const titleM = card.match(/class="link"[^>]*>([^<]+)/);
|
|
const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
|
|
if (!titleM || !hrefM) continue;
|
|
|
|
const title = decodeHtml(titleM[1]);
|
|
if (!applyFilters(title)) continue;
|
|
|
|
const id = `dc-${hrefM[2]}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
|
|
const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
|
|
const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);
|
|
|
|
// Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
|
|
let postedAt = "";
|
|
const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
|
|
if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: companyM ? decodeHtml(companyM[1].trim()) : "",
|
|
location: locM ? locM[1].trim() : "Basel",
|
|
postedAt,
|
|
link: hrefM[1],
|
|
search: "datacareer.ch",
|
|
source: "datacareer.ch",
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${articles.length} items, ${newOnPage} new`);
|
|
if (articles.length < 8) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|
|
|
|
// itjobs.ch — HTML scrape, jobs embedded as window.jobsList in page.
|
|
// Swept Switzerland-wide (no location filter): the board is small, ~210 jobs
|
|
// across 7 pages, and locking it to Basel returned only ~9.
|
|
// Each item carries its full description, so the DE/FR content check runs here
|
|
// instead of costing an extra page fetch per job in index.ts.
|
|
export async function fetchItJobsCh(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 25; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.itjobs.ch/jobs?page=${page}`,
|
|
{ headers: HEADERS }
|
|
);
|
|
// The board 404s one page past the end rather than serving an empty list.
|
|
if (!res.ok) { console.log(`end (${res.status})`); break; }
|
|
|
|
const html = await res.text();
|
|
const blocks = [...html.matchAll(/window\.jobsList\s*=\s*window\.jobsList\.concat\((\[[\s\S]*?\])\);/g)];
|
|
if (!blocks.length) { console.log("0 items"); break; }
|
|
|
|
const items: Array<{
|
|
id: number;
|
|
title: string;
|
|
description?: string;
|
|
posted_at: string;
|
|
job_details_path: string;
|
|
employer: { name: string };
|
|
location?: string;
|
|
job_location?: { name?: string };
|
|
}> = [];
|
|
for (const [, json] of blocks) {
|
|
try { items.push(...JSON.parse(json)); } catch { /* skip malformed block */ }
|
|
}
|
|
|
|
if (!items.length) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
let langDropped = 0;
|
|
for (const item of items) {
|
|
const title = decodeHtml(item.title);
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const id = `ij-${item.id}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
if (!isViableDescription(item.description).ok) { langDropped++; continue; }
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: item.employer?.name ?? "",
|
|
// job_location.name is the English form ("Aarau, Aargau, Switzerland");
|
|
// the bare location field is German ("Aarau, Kanton Aargau, Schweiz").
|
|
location: item.job_location?.name ?? item.location ?? "Switzerland",
|
|
postedAt: item.posted_at?.slice(0, 10) ?? "",
|
|
link: `https://www.itjobs.ch${item.job_details_path}`,
|
|
search: "itjobs.ch",
|
|
source: "itjobs.ch",
|
|
contentChecked: true, // isViableDescription already ran on the real description
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${items.length} items, ${newOnPage} new, ${langDropped} non-English`);
|
|
if (items.length < 30) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|
|
|
|
// job-room.ch — the federal employment service board (RAV / Arbeitsvermittlung).
|
|
// Roles under the Stellenmeldepflicht must be posted here before anywhere else,
|
|
// so it often carries a job days ahead of LinkedIn.
|
|
//
|
|
// Quirks worth knowing:
|
|
// - Each hit is wrapped: [{ jobAdvertisement: {...} }, …]
|
|
// - Titles come back with <em> highlighting around the matched keyword.
|
|
// - `languageIsoCode` on a description is unreliable — plenty of records tagged
|
|
// "de" carry English text (Roche, Novartis, Bachem all do). The DE/FR word
|
|
// count is what actually decides, so we ignore the tag.
|
|
// - An unfiltered canton sweep returns thousands of ads across every industry,
|
|
// so we narrow by keyword instead and union the results.
|
|
// - Descriptions arrive inline, so the language check runs here rather than
|
|
// costing a page fetch per job in index.ts.
|
|
const JOBROOM_API = "https://www.job-room.ch/jobadservice/api/jobAdvertisements/_search";
|
|
const JOBROOM_CANTONS = ["BS", "BL", "BE", "ZH"];
|
|
const JOBROOM_KEYWORDS = [
|
|
"software", "engineer", "developer", "devops", "data", "python",
|
|
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
|
];
|
|
|
|
// job-room carries every industry; pharma/lab titles slip past SOFTWARE_TITLE_RE
|
|
// via words like "data", "director" and "technical".
|
|
const PHARMA_ROLE_RE =
|
|
/\b(clinical|pharmaceutical|drug\s+(substance|product)|ha?ematology|oncology|formulation|laborator|lab\s+technician|chemist|chemistry|biolog|preclinical|regulatory\s+affairs|medical\s+affairs|commercial\s+launch|veterinary|pathologist)\b/i;
|
|
|
|
interface JobRoomAd {
|
|
id: string;
|
|
createdTime?: string;
|
|
jobContent?: {
|
|
externalUrl?: string | null;
|
|
jobDescriptions?: Array<{ title: string; description?: string }>;
|
|
company?: { name?: string; city?: string };
|
|
location?: { city?: string; cantonCode?: string };
|
|
};
|
|
}
|
|
|
|
// jobs.ch links arrive with a tracking query string that would break dedup
|
|
// across runs if it ever changes; the path alone identifies the posting.
|
|
function stripTracking(url: string): string {
|
|
try {
|
|
const u = new URL(url);
|
|
for (const k of [...u.searchParams.keys()]) {
|
|
if (k.startsWith("utm_")) u.searchParams.delete(k);
|
|
}
|
|
return u.toString().replace(/\?$/, "");
|
|
} catch {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
export async function fetchJobRoom(): Promise<Job[]> {
|
|
const ads = new Map<string, JobRoomAd>();
|
|
|
|
for (const keyword of JOBROOM_KEYWORDS) {
|
|
process.stdout.write(` ${keyword} … `);
|
|
let total = 0;
|
|
for (let page = 0; page < 8; page++) {
|
|
if (page > 0) await Bun.sleep(250);
|
|
const res = await fetch(`${JOBROOM_API}?page=${page}&size=100`, {
|
|
method: "POST",
|
|
headers: { ...HEADERS, "Content-Type": "application/json", Accept: "application/json" },
|
|
body: JSON.stringify({
|
|
onlineSince: 30,
|
|
displayRestricted: false,
|
|
keywords: [keyword],
|
|
cantonCodes: JOBROOM_CANTONS,
|
|
workloadPercentageMin: 0,
|
|
workloadPercentageMax: 100,
|
|
}),
|
|
});
|
|
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
|
|
|
const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>;
|
|
for (const h of hits) {
|
|
if (h.jobAdvertisement?.id) ads.set(h.jobAdvertisement.id, h.jobAdvertisement);
|
|
}
|
|
total += hits.length;
|
|
if (hits.length < 100) break;
|
|
}
|
|
console.log(`${total} ads (${ads.size} unique)`);
|
|
await Bun.sleep(250);
|
|
}
|
|
|
|
const jobs: Job[] = [];
|
|
let langDropped = 0;
|
|
for (const ad of ads.values()) {
|
|
const content = ad.jobContent;
|
|
const desc = content?.jobDescriptions?.[0];
|
|
if (!desc) continue;
|
|
|
|
// Strip the <em> highlighting the search API wraps around matched terms.
|
|
const title = decodeHtml(desc.title.replace(/<[^>]+>/g, ""));
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
if (PHARMA_ROLE_RE.test(title)) continue;
|
|
|
|
// A handful of ads ship without a description. isViableDescription passes
|
|
// those through by design, so leave contentChecked unset and let index.ts
|
|
// fall back to fetching the page.
|
|
const body = desc.description ?? "";
|
|
const hasBody = body.trim().length > 40;
|
|
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
|
|
|
const city = content?.location?.city ?? content?.company?.city ?? "";
|
|
const canton = content?.location?.cantonCode ?? "";
|
|
|
|
jobs.push({
|
|
id: `jr-${ad.id}`,
|
|
title,
|
|
company: content?.company?.name ?? "",
|
|
location: [city, canton].filter(Boolean).join(", ") || "Switzerland",
|
|
postedAt: ad.createdTime?.slice(0, 10) ?? "",
|
|
link: content?.externalUrl
|
|
? stripTracking(content.externalUrl)
|
|
: `https://www.job-room.ch/job-search/${ad.id}`,
|
|
search: "job-room.ch",
|
|
source: "job-room.ch",
|
|
contentChecked: hasBody, // only skip the re-check when we really judged a description
|
|
});
|
|
}
|
|
|
|
console.log(` ${ads.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
|
return jobs;
|
|
}
|
|
|
|
// International organisations in the Basel region (BIS, Swiss TPH, Basel
|
|
// Institute on Governance). Each board carries only a handful of postings, so
|
|
// they're swept together under one label rather than one entry apiece.
|
|
//
|
|
// These are mission-driven employers whose tech roles use UN-flavoured titles
|
|
// ("Information Management Officer", "ICT Specialist") that SOFTWARE_TITLE_RE
|
|
// alone would drop, so INTL_TECH_RE widens the net for these boards only.
|
|
// Boards this small can afford the extra noise; missing a rare opening can't.
|
|
const INTL_TECH_RE =
|
|
/\b(information\s*(management|systems?|technology)|ict\b|digital|geospatial|\bmis\b|data\s*(management|steward))\b/i;
|
|
|
|
function applyIntlFilters(title: string): boolean {
|
|
if (NON_ENGLISH_RE.test(title)) return false;
|
|
if (JUNIOR_TITLE_RE.test(title)) return false;
|
|
if (MS_STACK_RE.test(title)) return false;
|
|
return SOFTWARE_TITLE_RE.test(title) || INTL_TECH_RE.test(title);
|
|
}
|
|
|
|
// BIS runs a JS-rendered career site with no usable API, so we read the UNjobs
|
|
// mirror of its Basel office instead. Titles arrive as "Role, City, Country".
|
|
async function fetchBis(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch(
|
|
"https://unjobs.org/offices/bis_182320855689338880",
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(` BIS … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
const cards = [...html.matchAll(
|
|
/<a[^>]+href="(https:\/\/unjobs\.org\/vacancies\/(\d+))"[\s\S]*?<span\s+class="jtitle">([^<]+)<\/span>([\s\S]*?)<\/a>/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, link, id, rawTitle, rest] of cards) {
|
|
// "Senior Platform Engineer, Basel, Switzerland" → title + location
|
|
const split = decodeHtml(rawTitle).match(/^(.*?),\s*([^,]+,\s*[^,]+)$/);
|
|
const title = split ? split[1] : decodeHtml(rawTitle);
|
|
const location = split ? split[2] : "Basel, Switzerland";
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
jobs.push({
|
|
id: `bis-${id}`,
|
|
title,
|
|
company: "Bank for International Settlements",
|
|
location,
|
|
postedAt: rest.match(/datetime="(\d{4}-\d{2}-\d{2})/)?.[1] ?? "",
|
|
link,
|
|
search: "bis.org",
|
|
source: "bis.org",
|
|
contentChecked: true, // BIS publishes in English only
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` BIS … ${cards.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
// Swiss TPH runs Umantis. Site chrome is German even on English postings, so
|
|
// isEnglishUrl would reject the lot — the title filters decide instead.
|
|
async function fetchSwissTph(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch("https://jobs.swisstph.ch/Jobs/All", { headers: HEADERS });
|
|
if (!res.ok) { console.log(` Swiss TPH … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
// One block per posting, opened by the subtitle link that carries the title.
|
|
const blocks = html.split(/(?=<a\s[^>]*class="HSTableLinkSubTitle")/).slice(1);
|
|
|
|
let kept = 0;
|
|
for (const block of blocks) {
|
|
const hrefM = block.match(/href="(\/Vacancies\/(\d+)\/Description\/\d+)"/);
|
|
const titleM = block.match(/class="HSTableLinkSubTitle"[^>]*>([^<]+)</);
|
|
if (!hrefM || !titleM) continue;
|
|
|
|
const title = decodeHtml(titleM[1]);
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
// "Online seit" label and its value sit in separate spans sharing an id suffix.
|
|
const dateM = block.match(/Online seit<\/span>[\s\S]{0,300}?>\s*(\d{2})\.(\d{2})\.(\d{4})/);
|
|
const locM = block.match(/>\s*Schweiz\s*-\s*([^<]+)</);
|
|
|
|
jobs.push({
|
|
id: `tph-${hrefM[2]}`,
|
|
title,
|
|
company: "Swiss TPH",
|
|
location: locM ? `${locM[1].trim()}, Switzerland` : "Allschwil, Switzerland",
|
|
postedAt: dateM ? `${dateM[3]}-${dateM[2]}-${dateM[1]}` : "",
|
|
link: `https://jobs.swisstph.ch${hrefM[1]}`,
|
|
search: "swisstph.ch",
|
|
source: "swisstph.ch",
|
|
contentChecked: true, // German site chrome would fail the page-level check
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` Swiss TPH … ${blocks.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
// Basel Institute on Governance publishes each vacancy as a linked PDF, so
|
|
// there is no HTML description to language-check downstream.
|
|
async function fetchBaselGovernance(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const res = await fetch("https://baselgovernance.org/about/careers", { headers: HEADERS });
|
|
if (!res.ok) { console.log(` Basel Governance … failed (${res.status})`); return jobs; }
|
|
|
|
const html = await res.text();
|
|
// The `rounded-3xl` card class is what separates vacancy tiles from other
|
|
// asset links on the page (the annual report PDF sits in a plain anchor).
|
|
const cards = [...html.matchAll(
|
|
/<a\s+href="(https:\/\/baselgovernance\.org\/cms\/api\/assets\/([^\/]+)\/[^"]+)"[^>]*class="[^"]*rounded-3xl[^"]*"[\s\S]{0,400}?<h2[^>]*>([^<]+)<\/h2>([\s\S]{0,900})/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, link, assetId, rawTitle, rest] of cards) {
|
|
const title = decodeHtml(rawTitle);
|
|
if (!applyIntlFilters(title)) continue;
|
|
|
|
const locM = rest.match(/map-pin[\s\S]{0,200}?<\/span>\s*([^<]+)</);
|
|
|
|
jobs.push({
|
|
id: `big-${assetId}`,
|
|
title,
|
|
company: "Basel Institute on Governance",
|
|
location: locM ? decodeHtml(locM[1]) : "Basel, Switzerland",
|
|
postedAt: "", // board shows the closing date only
|
|
link,
|
|
search: "baselgovernance.org",
|
|
source: "baselgovernance.org",
|
|
contentChecked: true, // postings are PDFs, and the institute publishes in English
|
|
});
|
|
kept++;
|
|
}
|
|
|
|
console.log(` Basel Governance … ${cards.length} ads, ${kept} kept`);
|
|
return jobs;
|
|
}
|
|
|
|
export async function fetchBaselIntlOrgs(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
for (const fetchOrg of [fetchBis, fetchSwissTph, fetchBaselGovernance]) {
|
|
try {
|
|
jobs.push(...(await fetchOrg()));
|
|
} catch (e) {
|
|
console.log(` error: ${e}`);
|
|
}
|
|
await Bun.sleep(400);
|
|
}
|
|
return jobs;
|
|
}
|
|
|
|
// jobbasel.ch — the page shell is a React/Vite SPA (no jobs in the raw HTML
|
|
// on the homepage), but the search results route is server-rendered: fetching
|
|
// `/job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__`
|
|
// blob already containing the matched vacancies as JSON. No headless browser
|
|
// needed. Pagination is a URL suffix (`-seite-{n}`), not a query param — a
|
|
// bare `?page=2` is silently ignored by the SPA route.
|
|
//
|
|
// The board (Basel-Stadt + Basel-Landschaft, ~4000 openings) has no keyword
|
|
// union endpoint, so — like job-room.ch — we sweep a fixed keyword list and
|
|
// dedupe by vacancy id. Each item carries its full German description
|
|
// (activity/requirements/offer), so the DE/FR check runs here instead of
|
|
// costing a page fetch per job in index.ts.
|
|
const JOBBASEL_KEYWORDS = [
|
|
"software", "engineer", "developer", "devops", "data", "python",
|
|
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
|
];
|
|
|
|
interface JobBaselItem {
|
|
id: number;
|
|
title: string;
|
|
activity?: string;
|
|
requirements?: string;
|
|
offer?: string;
|
|
workplaceCity?: string;
|
|
urlApplication?: string;
|
|
dateFirstPublished?: string;
|
|
company?: { name?: string };
|
|
}
|
|
|
|
// Balanced-brace extraction — the blob can itself contain "};" inside string
|
|
// values (job descriptions), so a regex cutoff would truncate it.
|
|
function extractPreloadedState(html: string): any | null {
|
|
const marker = "window.__PRELOADED_STATE__";
|
|
const markerIdx = html.indexOf(marker);
|
|
if (markerIdx === -1) return null;
|
|
const start = html.indexOf("{", markerIdx);
|
|
if (start === -1) return null;
|
|
|
|
let depth = 0, inStr = false, esc = false, i = start;
|
|
for (; i < html.length; i++) {
|
|
const c = html[i];
|
|
if (inStr) {
|
|
if (esc) esc = false;
|
|
else if (c === "\\") esc = true;
|
|
else if (c === '"') inStr = false;
|
|
} else {
|
|
if (c === '"') inStr = true;
|
|
else if (c === "{") depth++;
|
|
else if (c === "}") { depth--; if (depth === 0) { i++; break; } }
|
|
}
|
|
}
|
|
|
|
try { return JSON.parse(html.slice(start, i)); } catch { return null; }
|
|
}
|
|
|
|
export async function fetchJobBasel(): Promise<Job[]> {
|
|
const items = new Map<number, JobBaselItem>();
|
|
|
|
for (const keyword of JOBBASEL_KEYWORDS) {
|
|
process.stdout.write(` ${keyword} … `);
|
|
const slug = keyword.toLowerCase().replace(/\s+/g, "-");
|
|
let total = 0;
|
|
|
|
for (let page = 1; page <= 5; page++) {
|
|
if (page > 1) await Bun.sleep(300);
|
|
const suffix = page === 1 ? "" : `-seite-${page}`;
|
|
const res = await fetch(
|
|
`https://www.jobbasel.ch/job/suche-${slug}${suffix}`,
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(`failed (${res.status})`); break; }
|
|
|
|
const state = extractPreloadedState(await res.text());
|
|
const pageItems: JobBaselItem[] = state?.vacancySearch?.data?.items ?? [];
|
|
if (!pageItems.length) break;
|
|
|
|
for (const item of pageItems) items.set(item.id, item);
|
|
total += pageItems.length;
|
|
if (pageItems.length < 10) break; // last page (pageSize is 10)
|
|
}
|
|
console.log(`${total} ads (${items.size} unique)`);
|
|
await Bun.sleep(300);
|
|
}
|
|
|
|
const jobs: Job[] = [];
|
|
let langDropped = 0;
|
|
for (const item of items.values()) {
|
|
const title = decodeHtml(item.title.replace(/<[^>]+>/g, ""));
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const body = [item.activity, item.requirements, item.offer]
|
|
.filter(Boolean)
|
|
.join(" ")
|
|
.replace(/<[^>]+>/g, " ");
|
|
const hasBody = body.trim().length > 40;
|
|
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
|
|
|
jobs.push({
|
|
id: `jb-${item.id}`,
|
|
title,
|
|
company: item.company?.name ?? "",
|
|
location: item.workplaceCity ? `${item.workplaceCity}, Switzerland` : "Basel, Switzerland",
|
|
postedAt: item.dateFirstPublished?.slice(0, 10) ?? "",
|
|
link: item.urlApplication || `https://www.jobbasel.ch/job/${item.id}`,
|
|
search: "jobbasel.ch",
|
|
source: "jobbasel.ch",
|
|
contentChecked: hasBody,
|
|
});
|
|
}
|
|
|
|
console.log(` ${items.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`);
|
|
return jobs;
|
|
}
|
|
|
|
// englishjobsearch.ch — plain SSR HTML, no JS needed, but reached through a
|
|
// non-obvious lever: any page on the site accepts `?format=markdown` and
|
|
// returns the same content pre-converted to clean markdown text (job title,
|
|
// company, city, date, short description all present). Without that param
|
|
// the canton pages ship the job cards as plain text nodes too — no JSON blob,
|
|
// no client-side rendering — so this works with a bare `curl`-equivalent
|
|
// fetch either way; `?format=markdown` just makes parsing trivial.
|
|
//
|
|
// Two catches: (1) the directory root groups by canton, so we hit the
|
|
// Basel-Stadt and Basel-Landschaft canton pages directly rather than the
|
|
// generic `/in/basel/` landing page, which only lists canton counts.
|
|
// (2) every job link is a `/clickout/<hash>` tracking redirect, not the real
|
|
// posting URL — there is no way to recover the underlying employer link
|
|
// without following the redirect, so the clickout link is stored as-is (it
|
|
// still resolves for a human clicking it, and content-checking follows it
|
|
// server-side the same as any other link).
|
|
const ENGLISHJOBSEARCH_CANTONS = ["canton-basel-stadt", "canton-basel-landschaft"];
|
|
|
|
export async function fetchEnglishJobSearch(): Promise<Job[]> {
|
|
const jobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (const canton of ENGLISHJOBSEARCH_CANTONS) {
|
|
process.stdout.write(` ${canton} … `);
|
|
const res = await fetch(
|
|
`https://englishjobsearch.ch/in/${canton}?format=markdown`,
|
|
{ headers: HEADERS }
|
|
);
|
|
if (!res.ok) { console.log(`failed (${res.status})`); continue; }
|
|
|
|
const md = await res.text();
|
|
// Each posting: "### Title](/clickout/<hash>?...)" heading, then on
|
|
// separate lines "* Company", "* City", "* Date", then a description
|
|
// paragraph before the next "### " heading or "report probem" footer.
|
|
const cards = [...md.matchAll(
|
|
/### ([^\]]+)\]\((\/clickout\/[^)]+)\)\n\n\* ([^\n]+)\n\* ([^\n]+)\n\* ([^\n]+)\n\n([\s\S]*?)\n\nreport probem/g
|
|
)];
|
|
|
|
let kept = 0;
|
|
for (const [, rawTitle, clickout, company, city, dateStr, desc] of cards) {
|
|
const title = rawTitle.trim();
|
|
if (!applyFilters(title)) continue;
|
|
if (MS_STACK_RE.test(title)) continue;
|
|
|
|
const id = clickout.match(/\/clickout\/([a-f0-9]+)/)?.[1] ?? clickout;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
const description = desc.trim();
|
|
const hasBody = description.length > 40;
|
|
if (hasBody && !isViableDescription(description).ok) continue;
|
|
|
|
jobs.push({
|
|
id: `ejs-${id}`,
|
|
title,
|
|
company: company.trim(),
|
|
location: `${city.trim()}, Switzerland`,
|
|
postedAt: "", // "August 30" style, no year given — not worth mis-parsing
|
|
link: `https://englishjobsearch.ch${clickout}`,
|
|
search: "englishjobsearch.ch",
|
|
source: "englishjobsearch.ch",
|
|
contentChecked: hasBody,
|
|
});
|
|
kept++;
|
|
}
|
|
console.log(`${cards.length} ads, ${kept} kept`);
|
|
await Bun.sleep(300);
|
|
}
|
|
|
|
return jobs;
|
|
}
|
|
|
|
// stellen-basel.ch — WordPress REST API
|
|
export async function fetchStellenBasel(): Promise<Job[]> {
|
|
const allJobs: Job[] = [];
|
|
const seen = new Set<string>();
|
|
|
|
for (let page = 1; page <= 30; page++) {
|
|
if (page > 1) await Bun.sleep(400);
|
|
process.stdout.write(` page ${page} … `);
|
|
|
|
const res = await fetch(
|
|
`https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
|
|
{ headers: { ...HEADERS, Accept: "application/json" } }
|
|
);
|
|
if (!res.ok) { console.log("done"); break; }
|
|
|
|
const data = (await res.json()) as Array<{
|
|
id: number;
|
|
title: { rendered: string };
|
|
date: string;
|
|
link: string;
|
|
meta: { _company_name?: string; _job_location?: string };
|
|
}>;
|
|
|
|
if (!data.length) { console.log("0 items"); break; }
|
|
|
|
let newOnPage = 0;
|
|
for (const item of data) {
|
|
const title = decodeHtml(item.title.rendered);
|
|
if (!applyFilters(title)) continue;
|
|
|
|
const id = `sb-${item.id}`;
|
|
if (seen.has(id)) continue;
|
|
seen.add(id);
|
|
|
|
allJobs.push({
|
|
id,
|
|
title,
|
|
company: item.meta._company_name ?? "",
|
|
location: item.meta._job_location ?? "Basel",
|
|
postedAt: item.date.slice(0, 10),
|
|
link: item.link,
|
|
search: "stellen-basel.ch",
|
|
source: "stellen-basel.ch",
|
|
});
|
|
newOnPage++;
|
|
}
|
|
|
|
console.log(`${data.length} items, ${newOnPage} new`);
|
|
if (data.length < 100) break;
|
|
}
|
|
|
|
return allJobs;
|
|
}
|