From d195d67c2a549928d733bd2f807e93e66bc3e6b8 Mon Sep 17 00:00:00 2001 From: rafapolo Date: Sun, 30 Aug 2026 13:07:16 +0200 Subject: [PATCH] feat: add jobbasel.ch and englishjobsearch.ch job sources jobbasel.ch's search route (/job/suche-{slug}) is server-rendered with a window.__PRELOADED_STATE__ JSON blob, no headless browser needed. englishjobsearch.ch works the same way via its ?format=markdown param. Both swept with the same fixed keyword list job-room.ch already uses. basel-jobs.ch was evaluated too but left out: it's DataDome-protected, which blocks plain curl outright, and this project's fetchers are intentionally browser-free. While wiring these in, found that isEnglishDescription (now isViableDescription) only ever ran the DE/FR word-count check against inline descriptions, never requiresGerman() - so an English-written posting demanding fluent German (e.g. PRODYNA's "you can communicate fluently in both German and English") sailed straight through. That gap existed in itjobs.ch and job-room.ch too, not just the two new sources, so the fix applies to all four. --- CLAUDE.md | 7 +- fetch-other.ts | 215 +++++++++++++++++++++++++++++++++++++++++++++++-- fetch.ts | 2 +- 3 files changed, 213 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4a04885..edff1af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ Temp scripts: `bun run scripts/-temp.ts` — never edit `config.ts` for on ``` config.ts # search keywords and areas fetch.ts # scraper, HTML parser, all filters -fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel intl orgs) +fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel intl orgs, jobbasel.ch, englishjobsearch.ch) index.ts # orchestrator: runs searches, writes CSV recheck.ts # raw-mode fetch, lists filtered-out jobs daily.ts # digest: fetch → match against profile.ts → send to Telegram @@ -42,8 +42,11 @@ plans/ # LinkedIn post drafts - Also exports `PLATFORM_KEYWORDS` (architect/SRE/infra/security/MLOps), `RESEARCH_KEYWORDS` (research & scientific software, HPC, data platform) and `FIELD_KEYWORDS` (forward-deployed, customer/sales/integration, founding/product). Kept as separate OR-strings rather than folded into `KEYWORDS` — LinkedIn ranks a focused query far better than one 50-term disjunction. The `sweep()` helper generates one search per area per family; Research skips Bern, Platform and Field add a Switzerland-wide remote pass. - `RESEARCH_KEYWORDS` deliberately contains no bioinformatics/computational-biology terms — see `BIO_DOMAIN_RE`. - **`fetch.ts`** — HTTP scraping, HTML parsing, and all filters. Exports `fetchJobs(config, { raw? })` and `isEnglishContent(id)`. -- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel international orgs). Imported by `index.ts`. +- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel international orgs, jobbasel.ch, englishjobsearch.ch). Imported by `index.ts`. - **datacareer.ch** currently returns **403** on every page — the source is dead and yields nothing. Needs new headers or a different entry point. + - **jobbasel.ch** looks like a JS-only React/Vite SPA (no jobs in the homepage's raw HTML), but its search route is server-rendered: `GET /job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__` blob containing the matched vacancies as JSON — no headless browser needed. Pagination is a URL suffix (`-seite-{n}`); a bare `?page=` query param is silently ignored. Swept with the same fixed keyword list as job-room.ch (no keyword-union endpoint exists) and deduped by vacancy id. Each item carries its full German description, so the DE/FR check runs in-process. + - **englishjobsearch.ch** also looks JS-only at first glance, but any page accepts `?format=markdown` and returns clean server-rendered markdown with full job cards — title, company, city, date, description. Hit the canton pages directly (`/in/canton-basel-stadt`, `/in/canton-basel-landschaft`) rather than the `/in/basel/` landing page, which only lists canton counts. Every job link is a `/clickout/` tracking redirect rather than the real posting URL — there's no way to recover the underlying employer link without following the redirect, so the clickout link is stored as-is. + - **basel-jobs.ch** was evaluated but is **not integrated**: it's protected by DataDome (not basic Cloudflare), which blocks plain `curl` outright. A real Chrome session gets through, confirming the content is genuine, but that only proves the block isn't automatable within this project's curl-only architecture — `bun daily`/`bun fetch` run headless via launchd with no browser available, and the session cookie that lets a real browser through can't be extracted for replay (browser tooling blocks cookie access). Would need persistent Playwright + stealth infra to chase further, which isn't worth it for what's likely a small regional board already covered by jobbasel.ch and stellen-basel.ch. - **Basel international orgs** (`fetchBaselIntlOrgs`) sweeps three boards under one label, each only a handful of postings: - **BIS** — the career site is JS-rendered with no usable API, so we read the UNjobs mirror at `unjobs.org/offices/bis_182320855689338880`. Titles arrive as `Role, City, Country` and get split back apart. BIS also cycles Senior Software / Data / Data & Analytics Engineer roles through this board. - **Swiss TPH** — Umantis at `jobs.swisstph.ch/Jobs/All`. Site chrome is German even on English postings, so `isEnglishUrl` would reject everything; `contentChecked: true` and the title filters decide instead. diff --git a/fetch-other.ts b/fetch-other.ts index ed00153..e1c182d 100644 --- a/fetch-other.ts +++ b/fetch-other.ts @@ -1,4 +1,4 @@ -import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE } from "./fetch.ts"; +import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman } from "./fetch.ts"; import type { Job } from "./fetch.ts"; const HEADERS = { @@ -34,10 +34,22 @@ const MS_STACK_RE = /\.net\b|\bc#|sharepoint|dynamics 365|power platform|\bsccm\ const DE_FR_WORDS_RE = /\b(und|oder|wir|Sie|Ihre|Ihrer|Kenntnisse|Erfahrung|Aufgaben|Anforderungen|Stelle|Unternehmen|Mitarbeiter|Bereich|nous|vous|votre|pour|avec|des|les)\b/g; -function isEnglishDescription(description?: string): boolean { - if (!description) return true; // nothing to judge — let the URL check in index.ts decide +// English-language AND not demanding German above B1 — mirrors isViableContent/ +// isViableUrl in fetch.ts, but against a description we already have in hand +// instead of a page fetch. The two checks catch different things: the word +// count drops postings *written* in German/French; requiresGerman() drops +// postings *written in English* that still demand German above B1 (e.g. +// PRODYNA: "you can communicate fluently in both German and English" — clean +// English text, would sail through the word count untouched). Skipping the +// requiresGerman() half was a real gap in every source below until 2026-08-30 +// — it let German-fluency requirements slip past whenever an item's inline +// description read as English. +function isViableDescription(description?: string): { ok: boolean; reason?: string } { + if (!description) return { ok: true }; // nothing to judge — let the URL check in index.ts decide const text = description.replace(/<[^>]+>/g, " ").replace(/&[a-z]+;/g, " "); - return (text.match(DE_FR_WORDS_RE) ?? []).length < 6; + if ((text.match(DE_FR_WORDS_RE) ?? []).length >= 6) return { ok: false, reason: "non-English" }; + if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" }; + return { ok: true }; } // datacareer.ch — HTML listing at /jobs/?city=Basel @@ -152,7 +164,7 @@ export async function fetchItJobsCh(): Promise { if (seen.has(id)) continue; seen.add(id); - if (!isEnglishDescription(item.description)) { langDropped++; continue; } + if (!isViableDescription(item.description).ok) { langDropped++; continue; } allJobs.push({ id, @@ -165,7 +177,7 @@ export async function fetchItJobsCh(): Promise { link: `https://www.itjobs.ch${item.job_details_path}`, search: "itjobs.ch", source: "itjobs.ch", - contentChecked: true, // isEnglishDescription already ran on the real description + contentChecked: true, // isViableDescription already ran on the real description }); newOnPage++; } @@ -274,12 +286,12 @@ export async function fetchJobRoom(): Promise { if (MS_STACK_RE.test(title)) continue; if (PHARMA_ROLE_RE.test(title)) continue; - // A handful of ads ship without a description. isEnglishDescription passes + // A handful of ads ship without a description. isViableDescription passes // those through by design, so leave contentChecked unset and let index.ts // fall back to fetching the page. const body = desc.description ?? ""; const hasBody = body.trim().length > 40; - if (hasBody && !isEnglishDescription(body)) { langDropped++; continue; } + if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; } const city = content?.location?.city ?? content?.company?.city ?? ""; const canton = content?.location?.cantonCode ?? ""; @@ -456,6 +468,193 @@ export async function fetchBaselIntlOrgs(): Promise { return jobs; } +// jobbasel.ch — the page shell is a React/Vite SPA (no jobs in the raw HTML +// on the homepage), but the search results route is server-rendered: fetching +// `/job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__` +// blob already containing the matched vacancies as JSON. No headless browser +// needed. Pagination is a URL suffix (`-seite-{n}`), not a query param — a +// bare `?page=2` is silently ignored by the SPA route. +// +// The board (Basel-Stadt + Basel-Landschaft, ~4000 openings) has no keyword +// union endpoint, so — like job-room.ch — we sweep a fixed keyword list and +// dedupe by vacancy id. Each item carries its full German description +// (activity/requirements/offer), so the DE/FR check runs here instead of +// costing a page fetch per job in index.ts. +const JOBBASEL_KEYWORDS = [ + "software", "engineer", "developer", "devops", "data", "python", + "cloud", "platform", "typescript", "backend", "full stack", "ai", +]; + +interface JobBaselItem { + id: number; + title: string; + activity?: string; + requirements?: string; + offer?: string; + workplaceCity?: string; + urlApplication?: string; + dateFirstPublished?: string; + company?: { name?: string }; +} + +// Balanced-brace extraction — the blob can itself contain "};" inside string +// values (job descriptions), so a regex cutoff would truncate it. +function extractPreloadedState(html: string): any | null { + const marker = "window.__PRELOADED_STATE__"; + const markerIdx = html.indexOf(marker); + if (markerIdx === -1) return null; + const start = html.indexOf("{", markerIdx); + if (start === -1) return null; + + let depth = 0, inStr = false, esc = false, i = start; + for (; i < html.length; i++) { + const c = html[i]; + if (inStr) { + if (esc) esc = false; + else if (c === "\\") esc = true; + else if (c === '"') inStr = false; + } else { + if (c === '"') inStr = true; + else if (c === "{") depth++; + else if (c === "}") { depth--; if (depth === 0) { i++; break; } } + } + } + + try { return JSON.parse(html.slice(start, i)); } catch { return null; } +} + +export async function fetchJobBasel(): Promise { + const items = new Map(); + + for (const keyword of JOBBASEL_KEYWORDS) { + process.stdout.write(` ${keyword} … `); + const slug = keyword.toLowerCase().replace(/\s+/g, "-"); + let total = 0; + + for (let page = 1; page <= 5; page++) { + if (page > 1) await Bun.sleep(300); + const suffix = page === 1 ? "" : `-seite-${page}`; + const res = await fetch( + `https://www.jobbasel.ch/job/suche-${slug}${suffix}`, + { headers: HEADERS } + ); + if (!res.ok) { console.log(`failed (${res.status})`); break; } + + const state = extractPreloadedState(await res.text()); + const pageItems: JobBaselItem[] = state?.vacancySearch?.data?.items ?? []; + if (!pageItems.length) break; + + for (const item of pageItems) items.set(item.id, item); + total += pageItems.length; + if (pageItems.length < 10) break; // last page (pageSize is 10) + } + console.log(`${total} ads (${items.size} unique)`); + await Bun.sleep(300); + } + + const jobs: Job[] = []; + let langDropped = 0; + for (const item of items.values()) { + const title = decodeHtml(item.title.replace(/<[^>]+>/g, "")); + if (!applyFilters(title)) continue; + if (MS_STACK_RE.test(title)) continue; + + const body = [item.activity, item.requirements, item.offer] + .filter(Boolean) + .join(" ") + .replace(/<[^>]+>/g, " "); + const hasBody = body.trim().length > 40; + if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; } + + jobs.push({ + id: `jb-${item.id}`, + title, + company: item.company?.name ?? "", + location: item.workplaceCity ? `${item.workplaceCity}, Switzerland` : "Basel, Switzerland", + postedAt: item.dateFirstPublished?.slice(0, 10) ?? "", + link: item.urlApplication || `https://www.jobbasel.ch/job/${item.id}`, + search: "jobbasel.ch", + source: "jobbasel.ch", + contentChecked: hasBody, + }); + } + + console.log(` ${items.size} ads scanned, ${jobs.length} kept, ${langDropped} non-English`); + return jobs; +} + +// englishjobsearch.ch — plain SSR HTML, no JS needed, but reached through a +// non-obvious lever: any page on the site accepts `?format=markdown` and +// returns the same content pre-converted to clean markdown text (job title, +// company, city, date, short description all present). Without that param +// the canton pages ship the job cards as plain text nodes too — no JSON blob, +// no client-side rendering — so this works with a bare `curl`-equivalent +// fetch either way; `?format=markdown` just makes parsing trivial. +// +// Two catches: (1) the directory root groups by canton, so we hit the +// Basel-Stadt and Basel-Landschaft canton pages directly rather than the +// generic `/in/basel/` landing page, which only lists canton counts. +// (2) every job link is a `/clickout/` tracking redirect, not the real +// posting URL — there is no way to recover the underlying employer link +// without following the redirect, so the clickout link is stored as-is (it +// still resolves for a human clicking it, and content-checking follows it +// server-side the same as any other link). +const ENGLISHJOBSEARCH_CANTONS = ["canton-basel-stadt", "canton-basel-landschaft"]; + +export async function fetchEnglishJobSearch(): Promise { + const jobs: Job[] = []; + const seen = new Set(); + + for (const canton of ENGLISHJOBSEARCH_CANTONS) { + process.stdout.write(` ${canton} … `); + const res = await fetch( + `https://englishjobsearch.ch/in/${canton}?format=markdown`, + { headers: HEADERS } + ); + if (!res.ok) { console.log(`failed (${res.status})`); continue; } + + const md = await res.text(); + // Each posting: "### Title](/clickout/?...)" heading, then on + // separate lines "* Company", "* City", "* Date", then a description + // paragraph before the next "### " heading or "report probem" footer. + const cards = [...md.matchAll( + /### ([^\]]+)\]\((\/clickout\/[^)]+)\)\n\n\* ([^\n]+)\n\* ([^\n]+)\n\* ([^\n]+)\n\n([\s\S]*?)\n\nreport probem/g + )]; + + let kept = 0; + for (const [, rawTitle, clickout, company, city, dateStr, desc] of cards) { + const title = rawTitle.trim(); + if (!applyFilters(title)) continue; + if (MS_STACK_RE.test(title)) continue; + + const id = clickout.match(/\/clickout\/([a-f0-9]+)/)?.[1] ?? clickout; + if (seen.has(id)) continue; + seen.add(id); + + const description = desc.trim(); + const hasBody = description.length > 40; + if (hasBody && !isViableDescription(description).ok) continue; + + jobs.push({ + id: `ejs-${id}`, + title, + company: company.trim(), + location: `${city.trim()}, Switzerland`, + postedAt: "", // "August 30" style, no year given — not worth mis-parsing + link: `https://englishjobsearch.ch${clickout}`, + search: "englishjobsearch.ch", + source: "englishjobsearch.ch", + contentChecked: hasBody, + }); + kept++; + } + console.log(`${cards.length} ads, ${kept} kept`); + await Bun.sleep(300); + } + + return jobs; +} + // stellen-basel.ch — WordPress REST API export async function fetchStellenBasel(): Promise { const allJobs: Job[] = []; diff --git a/fetch.ts b/fetch.ts index 694d5b9..919fb4a 100644 --- a/fetch.ts +++ b/fetch.ts @@ -158,7 +158,7 @@ const GERMAN_OPTIONAL_RE = /** Within a mention's window: signals it is genuinely required at B2+. */ const GERMAN_REQUIRED_RE = - /\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluent|fluency|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend|fließend|muttersprach\w*|sehr\s+gute?|gute?\s+Deutschkenntnisse|voraussetzung|zwingend)\b/i; + /\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluen\w*|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend|fließend|muttersprach\w*|sehr\s+gute?|gute?\s+Deutschkenntnisse|voraussetzung|zwingend)\b/i; /** Title-level marker — no description fetch needed. */ export const GERMAN_TITLE_RE =