From d5e6605176647853d676b2f0cf09975bf61e1988 Mon Sep 17 00:00:00 2001 From: rafapolo Date: Tue, 1 Sep 2026 12:23:59 +0200 Subject: [PATCH] feat: revive datacareer.ch and integrate basel-jobs.ch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit datacareer.ch had been returning 403 on every page for months. The cause was our own User-Agent: its nginx inverts the usual bot rule and blocks browser-like UAs while serving honest ones (no UA, curl, Googlebot). Give it PLAIN_HEADERS instead of the shared spoofed-Chrome HEADERS. Basel has 8 jobs on a single page with no pagination, so page 2 returning 0 items is correct rather than truncation. basel-jobs.ch was documented as un-integrable because DataDome blocks plain curl. Only half of that was true. Measured: plain fetch 403s on any UA, headless Chrome over CDP 403s, Playwright's own launch 403s even headful — but a real Chrome launched normally with --remote-debugging-port and attached over CDP afterwards gets 200. Clearance then lives entirely in a datadome cookie that replays fine over plain fetch with a matching UA, so a browser is touched only to mint, never to scrape. The earlier "cookie can't be extracted" note was true of the browser extension, not of CDP. Minting needs a visible window, so launchd runs can't do it; a stale cookie makes the source log and yield nothing instead of failing the run. Refresh by hand with `bun run mint-datadome`, or set LINKME_NO_BROWSER=1. Three passes, all needed — they cover different sets. The informatik category is a curated taxonomy, not the whole board, and the two search modes query different indexes: websearch=1 finds role words and postings filed outside IT (+147 over the category), while the bare search is the only mode matching stack terms — react, frontend and backend return zero under websearch=1. Language checks run against div.plain-content only; against the whole page the German site chrome alone clears the DE/FR threshold and drops every job, English ones included. Also fix a pre-existing double-wrapped Promise in the extraSources type. --- CLAUDE.md | 36 +++- fetch-other.ts | 390 ++++++++++++++++++++++++++++++++++++++- index.ts | 5 +- package.json | 10 +- scripts/mint-datadome.ts | 15 ++ 5 files changed, 446 insertions(+), 10 deletions(-) create mode 100644 scripts/mint-datadome.ts diff --git a/CLAUDE.md b/CLAUDE.md index 6def690..f76534c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co | `bun daily --dry-run` | same, but print the message instead of sending | | `bun daily --resend` | skip fetching, re-send today's jobs from the CSV (`--date=YYYY-MM-DD` to pick a day) | | `bun run telegram.ts --setup` | print the chat ID for `.env` after messaging the bot | +| `bun run mint-datadome` | open a real Chrome window to refresh the basel-jobs.ch DataDome cookie | Temp scripts: `bun run scripts/-temp.ts` — never edit `config.ts` for one-off searches. @@ -20,7 +21,7 @@ Temp scripts: `bun run scripts/-temp.ts` — never edit `config.ts` for on ``` config.ts # search keywords and areas fetch.ts # scraper, HTML parser, all filters -fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel intl orgs, jobbasel.ch, englishjobsearch.ch) +fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel intl orgs, jobbasel.ch, englishjobsearch.ch, basel-jobs.ch) index.ts # orchestrator: runs searches, writes CSV recheck.ts # raw-mode fetch, lists filtered-out jobs daily.ts # digest: fetch → match against profile.ts → send to Telegram @@ -42,11 +43,38 @@ plans/ # LinkedIn post drafts - Also exports `PLATFORM_KEYWORDS` (architect/SRE/infra/security/MLOps), `RESEARCH_KEYWORDS` (research & scientific software, HPC, data platform) and `FIELD_KEYWORDS` (forward-deployed, customer/sales/integration, founding/product). Kept as separate OR-strings rather than folded into `KEYWORDS` — LinkedIn ranks a focused query far better than one 50-term disjunction. The `sweep()` helper generates one search per area per family; Research skips Bern, Platform and Field add a Switzerland-wide remote pass. - `RESEARCH_KEYWORDS` deliberately contains no bioinformatics/computational-biology terms — see `BIO_DOMAIN_RE`. - **`fetch.ts`** — HTTP scraping, HTML parsing, and all filters. Exports `fetchJobs(config, { raw? })` and `isEnglishContent(id)`. -- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel international orgs, jobbasel.ch, englishjobsearch.ch). Imported by `index.ts`. - - **datacareer.ch** currently returns **403** on every page — the source is dead and yields nothing. Needs new headers or a different entry point. +- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel international orgs, jobbasel.ch, englishjobsearch.ch, basel-jobs.ch). Imported by `index.ts`. + - **datacareer.ch** inverts the usual bot rule: its nginx **403s browser-like User-Agents** and serves honest ones (no UA, `curl/8.0`, Googlebot) fine. The spoofed Chrome UA in the shared `HEADERS` was the entire cause of the long-standing 403 — it uses `PLAIN_HEADERS` instead. Small board: Basel returns **8 jobs on one page with no pagination**, so `?page=2` legitimately yields 0 items. - **jobbasel.ch** looks like a JS-only React/Vite SPA (no jobs in the homepage's raw HTML), but its search route is server-rendered: `GET /job/suche-{slug}` returns full HTML with a `window.__PRELOADED_STATE__` blob containing the matched vacancies as JSON — no headless browser needed. Pagination is a URL suffix (`-seite-{n}`); a bare `?page=` query param is silently ignored. Swept with the same fixed keyword list as job-room.ch (no keyword-union endpoint exists) and deduped by vacancy id. Each item carries its full German description, so the DE/FR check runs in-process. - **englishjobsearch.ch** also looks JS-only at first glance, but any page accepts `?format=markdown` and returns clean server-rendered markdown with full job cards — title, company, city, date, description. Hit the canton pages directly (`/in/canton-basel-stadt`, `/in/canton-basel-landschaft`) rather than the `/in/basel/` landing page, which only lists canton counts. Every job link is a `/clickout/` tracking redirect rather than the real posting URL — there's no way to recover the underlying employer link without following the redirect, so the clickout link is stored as-is. - - **basel-jobs.ch** was evaluated but is **not integrated**: it's protected by DataDome (not basic Cloudflare), which blocks plain `curl` outright. A real Chrome session gets through, confirming the content is genuine, but that only proves the block isn't automatable within this project's curl-only architecture — `bun daily`/`bun fetch` run headless via launchd with no browser available, and the session cookie that lets a real browser through can't be extracted for replay (browser tooling blocks cookie access). Would need persistent Playwright + stealth infra to chase further, which isn't worth it for what's likely a small regional board already covered by jobbasel.ch and stellen-basel.ch. + - **basel-jobs.ch** is behind DataDome and **is now integrated** (`fetchBaselJobs`), contrary to the earlier note that it couldn't be. What the block actually permits, tested: + - plain `curl`/`fetch` → 403 on **every** UA, honest or spoofed; + - Chrome `--headless=new` over CDP → 403 (headless is detected); + - Playwright's own `chromium.launch()`, even headful → 403 (launch-path artefacts); + - a **real Chrome launched normally** with `--remote-debugging-port`, attached over CDP afterwards → **200**. + + Clearance lives entirely in a `datadome` cookie (Max-Age 1 year) that **replays fine over plain `fetch`** with a matching User-Agent — the old claim that it "can't be extracted" was true of the browser-extension tooling, not of CDP. So `mintDataDomeSession()` opens a real window once, caches `{cookie, ua}` to `results/.datadome.json` (gitignored), and every run after that is plain HTTP. A rotated `Set-Cookie` is folded back into the cache; a 403 triggers one re-mint. + + **Minting needs a visible window, so launchd runs can't do it.** With a stale cookie an unattended run logs `unavailable`, returns `[]`, and the rest of the fetch continues — it never fails the run. Refresh by hand with `bun run mint-datadome`, or set `LINKME_NO_BROWSER=1` to suppress minting entirely. + + **Two passes, both needed.** Pass 1 walks the `informatik-jobs` category (~300 postings over 3 pages). Pass 2 sweeps `BASELJOBS_TERMS` through `/search?terms={kw}&page=1&websearch=1`. The category is a *curated taxonomy, not the whole board* — it holds only what the board filed under IT, so tech roles parked under Industrie/Pharma/Bau and roles whose relevance lives in the description never show up in pass 1. Measured 2026-09-01: pass 2 added **147 postings the category missed** (incl. "Senior AI Infrastructure & Platform Engineer (Basel)", "Senior DevOps / Cloud Engineer (Kubernetes, CI/CD, Linux)"), while the category held **257 the search missed**. 296 → 442 unique, 48 → 68 viable. + + **Both search modes are swept — they query different indexes and each is blind to the other.** Do not drop one as redundant: + + | term | `websearch=1` | bare `?terms=` | + |---|---|---| + | engineer | 90 | 260 found | + | frontend | **0** | 2 | + | backend | **0** | 4 | + | react | **0** | 2 | + + `websearch=1` matches role/domain words and reaches postings filed outside the IT category. The bare search is the **only** mode that matches stack terms — `react`, `frontend`, `backend`, `javascript` return literally zero under `websearch=1`. Neither paginates: both ignore `page` and return the whole result set in one response (bare caps around 100 cards). Measured 2026-09-01 against the category: websearch added 147, bare a further 139 raw (23 past the title filters). 48 → 68 → **88 viable**. + + German stems (`entwickler`, `informatik`) are in the term list deliberately: search matches *description* text, so they surface English-titled postings the category never filed under IT; the language filters drop the German-language ones afterwards. The bare search on `architect`/`security` drags in a lot of German noise (*Architekt* = building architect, *Sicherheitsassistent*, *Gerüstbauer*) — that's expected and `NON_ENGLISH_RE`/`SOFTWARE_TITLE_RE` handle it, 139 raw → 23. + + Pagination/dedup: page 4+ of the category still serves 2 sponsored slots, so **"no new ids" is the end condition**, not an empty list. Sponsored cards repeat within a page, across pages, and across both passes — dedupe on `data-gtm-jid`. Cards carry no date, so `postedAt` is the discovery date. Canton comes from `data-gtm-jp` (sometimes `"BL,BS"`). + + **Language-check the `div.plain-content` block only, never the whole page.** The site chrome is German and on its own clears the ≥ 6 DE/FR word threshold — checking the full page dropped *every* job, English ones included. That block is the mobile rendering of the description; the desktop view iframes the same text, so it's the only copy in the server-rendered HTML. - **Basel international orgs** (`fetchBaselIntlOrgs`) sweeps three boards under one label, each only a handful of postings: - **BIS** — the career site is JS-rendered with no usable API, so we read the UNjobs mirror at `unjobs.org/offices/bis_182320855689338880`. Titles arrive as `Role, City, Country` and get split back apart. BIS also cycles Senior Software / Data / Data & Analytics Engineer roles through this board. - **Swiss TPH** — Umantis at `jobs.swisstph.ch/Jobs/All`. Site chrome is German even on English postings, so `isEnglishUrl` would reject everything; `contentChecked: true` and the title filters decide instead. diff --git a/fetch-other.ts b/fetch-other.ts index e1c182d..99b035a 100644 --- a/fetch-other.ts +++ b/fetch-other.ts @@ -8,6 +8,19 @@ const HEADERS = { Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", }; +// datacareer.ch inverts the usual bot rule: its nginx 403s anything claiming to +// be a browser (a spoofed Chrome UA is the one thing guaranteed to fail) while +// letting honest clients — no UA, curl, Googlebot — straight through. Presumably +// real browser traffic arrives via a path that also carries matching TLS/CH +// fingerprints, so a bare Chrome UA over plain HTTP reads as a scraper. Sending +// an honest identifier is both what unblocks it and the polite thing to do. +// This is what killed the source for months; it was never a hard block. +const PLAIN_HEADERS = { + "User-Agent": "linkme-jobbot/1.0 (personal job search; contact via basel)", + "Accept-Language": "en-US,en;q=0.9", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +}; + function decodeHtml(s: string): string { return s .replace(/&/g, "&") @@ -63,7 +76,7 @@ export async function fetchDataCareer(): Promise { const res = await fetch( `https://www.datacareer.ch/jobs/?city=Basel&page=${page}`, - { headers: HEADERS } + { headers: PLAIN_HEADERS } ); if (!res.ok) { console.error(`failed (${res.status})`); break; } @@ -708,3 +721,378 @@ export async function fetchStellenBasel(): Promise { return allJobs; } + +// basel-jobs.ch — regional board on the JobChannel network (same syndication +// network whose ?source=jobchannel links show up on Siemens/jobs.ch postings). +// +// The board is protected by DataDome, which is why this source sat documented +// as "not integrable" for months. The block is real but narrower than it looked: +// +// - Plain curl / bun fetch → 403 on every UA, browser-like or not. +// - Chrome --headless=new, driven over CDP → 403. Headless is detected. +// - Playwright's own chromium.launch(), even headful → 403. The CDP-launch +// path leaves automation artefacts DataDome fingerprints. +// - A real Chrome launched *normally* (with --remote-debugging-port so we can +// attach afterwards) → 200. Attaching over CDP after the fact is invisible. +// +// The unlock is that DataDome's clearance lives entirely in a `datadome` cookie +// (Max-Age one year) which replays fine over plain fetch with a matching +// User-Agent. So we mint a cookie once with a real browser window, cache it, and +// scrape normally thereafter — a browser is touched only when the cookie is +// missing or has been rejected, not on every run. An earlier note in CLAUDE.md +// claimed the cookie "can't be extracted for replay"; that was true of the +// browser-extension tooling, not of CDP, which hands it over directly. +// +// Operational caveat: minting needs a *visible* Chrome window for ~10s, so an +// unattended `bun daily` under launchd cannot mint. When the cookie is stale in +// that context the source logs and yields nothing rather than failing the run; +// refresh it deliberately with `bun run scripts/mint-datadome.ts`. + +const DATADOME_CACHE = "results/.datadome.json"; +const CHROME_BIN = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; +const BASELJOBS_ORIGIN = "https://basel-jobs.ch"; + +interface DataDomeSession { + cookie: string; + ua: string; + mintedAt: string; +} + +async function loadDataDomeSession(): Promise { + try { + const raw = await Bun.file(DATADOME_CACHE).text(); + const s = JSON.parse(raw) as DataDomeSession; + return s.cookie && s.ua ? s : null; + } catch { + return null; + } +} + +async function waitForCdp(port: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const r = await fetch(`http://127.0.0.1:${port}/json/version`); + if (r.ok) return true; + } catch { /* not up yet */ } + await Bun.sleep(300); + } + return false; +} + +/** + * Launch a real, visible Chrome, let it clear DataDome, then attach over CDP and + * lift the resulting cookie. Headless and Playwright-launched browsers are both + * detected — only a normally-launched window gets through, which is the whole + * reason this is a separate, rarely-run step instead of the fetch path itself. + */ +export async function mintDataDomeSession(): Promise { + if (process.env.LINKME_NO_BROWSER === "1") { + console.log(" LINKME_NO_BROWSER=1 — skipping DataDome mint"); + return null; + } + if (!(await Bun.file(CHROME_BIN).exists())) { + console.error(` Chrome not found at ${CHROME_BIN} — cannot mint DataDome cookie`); + return null; + } + + let chromium: typeof import("playwright-core").chromium; + try { + ({ chromium } = await import("playwright-core")); + } catch { + console.error(" playwright-core not installed — run `bun add playwright-core`"); + return null; + } + + // A port unlikely to collide with a debugging session the user opened. + const port = 9333; + const profile = "results/.datadome-profile"; + const proc = Bun.spawn( + [ + CHROME_BIN, + `--remote-debugging-port=${port}`, + `--user-data-dir=${profile}`, + "--no-first-run", + "--no-default-browser-check", + "--window-size=1200,860", + BASELJOBS_ORIGIN, + ], + { stdout: "ignore", stderr: "ignore" } + ); + + try { + if (!(await waitForCdp(port, 20_000))) { + console.error(" Chrome CDP endpoint never came up"); + return null; + } + const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`); + const ctx = browser.contexts()[0]; + const page = + ctx.pages().find((p) => p.url().includes("basel-jobs")) ?? ctx.pages()[0]; + + // The interstitial resolves itself; give it room, then confirm we're through. + let cookie = ""; + let ua = ""; + for (let attempt = 0; attempt < 12; attempt++) { + await page.waitForTimeout(1500); + const html = await page.content(); + if (/Please enable JS|geo\.captcha/.test(html)) continue; + const cookies = await ctx.cookies(BASELJOBS_ORIGIN); + const dd = cookies.find((c) => c.name === "datadome"); + if (!dd) continue; + cookie = dd.value; + ua = await page.evaluate(() => navigator.userAgent); + break; + } + await browser.close(); + + if (!cookie || !ua) { + console.error(" DataDome challenge did not clear — no cookie minted"); + return null; + } + + const session: DataDomeSession = { cookie, ua, mintedAt: new Date().toISOString() }; + await Bun.write(DATADOME_CACHE, JSON.stringify(session, null, 2)); + console.log(` minted DataDome cookie (${cookie.length} chars)`); + return session; + } catch (e) { + console.error(` DataDome mint failed: ${e}`); + return null; + } finally { + proc.kill(); + } +} + +function baselJobsHeaders(s: DataDomeSession): Record { + return { + "User-Agent": s.ua, + "Accept-Language": "de-CH,de;q=0.9,en;q=0.8", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + Cookie: `datadome=${s.cookie}`, + }; +} + +/** + * Fetch with the cached cookie, re-minting once if the cookie has gone stale. + * DataDome rotates the cookie on ordinary responses too, so a Set-Cookie on the + * way past is folded back into the session rather than ignored. + */ +async function baselJobsFetch( + url: string, + state: { session: DataDomeSession | null; minted: boolean } +): Promise { + for (let attempt = 0; attempt < 2; attempt++) { + if (!state.session) { + if (state.minted) return null; + state.minted = true; + state.session = await mintDataDomeSession(); + if (!state.session) return null; + } + + const res = await fetch(url, { headers: baselJobsHeaders(state.session) }); + + const rotated = res.headers + .getSetCookie?.() + .map((c) => /(?:^|;\s*)datadome=([^;]+)/.exec(c)?.[1]) + .find(Boolean); + if (rotated && state.session) { + state.session = { ...state.session, cookie: rotated }; + await Bun.write(DATADOME_CACHE, JSON.stringify(state.session, null, 2)); + } + + if (res.ok) return res.text(); + + if (res.status === 403 && !state.minted) { + console.log(" cookie rejected — re-minting"); + state.session = null; + continue; + } + console.error(` failed (${res.status}) ${url}`); + return null; + } + return null; +} + +// Terms swept through the board's search. No keyword-union endpoint exists, so +// — like job-room.ch and jobbasel.ch — this is a fixed list, one request each +// per mode, deduped by vacancy id. German stems are in deliberately: the search +// matches description text, so "entwickler" surfaces postings whose English +// title the category never filed under IT. The title and description filters +// drop the German-language ones afterwards. +// +// Role words and stack words behave differently across the two search modes +// (see fetchBaselJobs), which is why both modes get the whole list rather than +// a tuned subset — `websearch=1` finds no stack terms at all, and the bare +// search is the only mode that matches "react"/"frontend"/"backend". +const BASELJOBS_TERMS = [ + // role / domain words — these carry the `websearch=1` pass + "software", "engineer", "developer", "entwickler", "informatik", "data", + "cloud", "devops", "platform", "architect", "security", "ai", + // stack words — only the bare search matches these + "python", "javascript", "typescript", "react", "node", + "fullstack", "full stack", "backend", "frontend", "kubernetes", "linux", +]; + +function buildBaselJobsLocation(city: string, canton: string): string { + if (!city) return canton || "Basel"; + if (!canton) return city; + // data-gtm-jp is one or more codes ("BS", "BL,BS"); keep only what the city + // string doesn't already name. + const missing = canton + .split(",") + .map((c) => c.trim()) + .filter((c) => c && !new RegExp(`\\b${c}\\b`).test(city)); + return missing.length ? `${city}, ${missing.join(",")}` : city; +} + +/** + * Pull every vacancy card out of a listing page. Returns how many ids were + * new so callers can use "no new postings" as their end condition — past the + * last real page the board still serves its two sponsored slots, so an empty + * list never arrives. + */ +function parseBaselJobsCards( + html: string, + seen: Set, + out: Job[], + today: string +): { items: number; added: number } { + const items = [...html.matchAll(/
  • /gs)].map((m) => m[1]); + let added = 0; + + for (const card of items) { + const jid = /data-gtm-jid="(\d+)"/.exec(card)?.[1]; + const linkM = /href="(https:\/\/basel-jobs\.ch\/job\/[^"]+)"/.exec(card); + const titleM = /(.*?)<\/span>/s.exec(card); + if (!jid || !linkM || !titleM) continue; + + // Sponsored slots repeat the same posting within a page, between pages, + // and across the category and search sweeps. + const id = `bj-${jid}`; + if (seen.has(id)) continue; + seen.add(id); + added++; + + const title = decodeHtml(titleM[1]); + if (!applyFilters(title)) continue; + + const companyM = + /\s*]*>(.*?)<\/a>/s.exec(card) ?? + /(.*?)<\/span>/s.exec(card); + const locationM = /(.*?)<\/span>/s.exec(card); + const canton = /data-gtm-jp="([^"]*)"/.exec(card)?.[1] ?? ""; + + out.push({ + id, + title, + company: companyM ? decodeHtml(companyM[1]) : "", + // Cards carry a city; the canton attribute disambiguates BS vs BL, + // which is what daily.ts groups on. Some cards already end in the + // canton code ("Basel, BS, 4030") — don't append it twice. + location: buildBaselJobsLocation( + locationM ? decodeHtml(locationM[1]) : "", + canton + ), + // No date anywhere on the card or detail page — discovery date is the + // best available, same as any board that hides posting age. + postedAt: today, + link: linkM[1], + search: "basel-jobs.ch", + source: "basel-jobs.ch", + }); + } + + return { items: items.length, added }; +} + +export async function fetchBaselJobs(): Promise { + const allJobs: Job[] = []; + const seen = new Set(); + const state = { session: await loadDataDomeSession(), minted: false }; + const today = new Date().toISOString().slice(0, 10); + + // Pass 1 — the board's IT category, ~300 postings over 3 pages. + for (let page = 1; page <= 10; page++) { + if (page > 1) await Bun.sleep(400); + process.stdout.write(` informatik page ${page} … `); + + const html = await baselJobsFetch( + `${BASELJOBS_ORIGIN}/informatik-jobs?page=${page}`, + state + ); + if (html === null) { console.log("unavailable"); break; } + + const { items, added } = parseBaselJobsCards(html, seen, allJobs, today); + if (items === 0) { console.log("0 items"); break; } + console.log(`${items} items, ${added} new`); + if (added === 0) break; + } + + // Pass 2 — full-text search. The category is a *curated taxonomy*, not the + // whole board: it only holds what the board filed under IT, so tech roles + // parked under Industrie/Pharma/Bau, and roles whose relevance lives in the + // description rather than the title, never appear in pass 1. Measured on + // 2026-09-01 this pass added **147 postings the category missed**, including + // "Senior AI Infrastructure & Platform Engineer (Basel)" and "Senior DevOps / + // Cloud Engineer (Kubernetes, CI/CD, Linux)". The reverse is also true (the + // category held 257 the search missed), so both passes are needed. + // + // `websearch=1` is the mode that does this — the bare `?terms=` search + // returned nothing that wasn't already in the category. It ignores `page` + // entirely and returns the whole result set on one response, so there is no + // pagination to walk. + // Both search modes are swept, because they query different indexes and each + // is blind to what the other finds: + // + // term websearch=1 bare + // engineer 90 260 found + // frontend 0 2 + // backend 0 4 + // react 0 2 + // + // `websearch=1` matches role/domain words and reaches postings filed outside + // the IT category; the bare search is the *only* mode that matches stack + // terms. Neither paginates — both ignore `page` and return their whole result + // set in one response (bare caps around 100 cards). Measured 2026-09-01 + // against the category: websearch added 147, bare added a further 139 raw + // (23 surviving the title filters). + for (const mode of ["websearch", "bare"] as const) { + for (const term of BASELJOBS_TERMS) { + await Bun.sleep(400); + process.stdout.write(` ${mode} "${term}" … `); + + const qs = `terms=${encodeURIComponent(term)}${mode === "websearch" ? "&page=1&websearch=1" : ""}`; + const html = await baselJobsFetch(`${BASELJOBS_ORIGIN}/search?${qs}`, state); + if (html === null) { console.log("unavailable"); break; } + + const { items, added } = parseBaselJobsCards(html, seen, allJobs, today); + console.log(`${items} items, ${added} new`); + } + } + + // Detail pages carry the full description inline, so the DE/FR and + // German-requirement checks run here instead of costing index.ts a fetch that + // would hit DataDome without the cookie and be dropped as unreachable. + const viable: Job[] = []; + for (const job of allJobs) { + await Bun.sleep(300); + process.stdout.write(` checking ${job.title} … `); + const page = await baselJobsFetch(job.link, state); + if (page === null) { console.log("unreachable, keeping"); viable.push(job); continue; } + + // Only the posting's own text, never the page. basel-jobs.ch is a German + // site, so its nav/footer/category chrome alone clears the ≥ 6 DE/FR word + // threshold — checking the whole page dropped *every* job, English ones + // included. The mobile `plain-content` block is the description as plain + // text; the desktop view loads the same thing into an iframe, so this is + // the only copy present in the server-rendered HTML. + const descM = /
    (.*?)<\/div>/s.exec(page); + if (!descM) { console.log("no description, keeping"); viable.push(job); continue; } + const check = isViableDescription(descM[1].replace(//gi, " ")); + if (!check.ok) { console.log(`dropped (${check.reason})`); continue; } + console.log("ok"); + viable.push({ ...job, contentChecked: true }); + } + + return viable; +} diff --git a/index.ts b/index.ts index b18c160..d6bf45f 100644 --- a/index.ts +++ b/index.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { searches } from "./config.ts"; import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts"; -import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch } from "./fetch-other.ts"; +import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch, fetchBaselJobs } from "./fetch-other.ts"; const CSV = "./results/jobs.csv"; const OLD_HEADER = "date_found,posted_at,search,title,company,link"; @@ -124,7 +124,7 @@ async function processLinkedInSearch(config: (typeof searches)[number]): Promise console.log(` [${config.label}] ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`); } -const extraSources: Array<{ label: string; fetch: () => Promise> }> = [ +const extraSources: Array<{ label: string; fetch: () => ReturnType }> = [ { label: "datacareer.ch", fetch: fetchDataCareer }, { label: "stellen-basel.ch", fetch: fetchStellenBasel }, { label: "itjobs.ch", fetch: fetchItJobsCh }, @@ -132,6 +132,7 @@ const extraSources: Array<{ label: string; fetch: () => Promise { diff --git a/package.json b/package.json index db45cc9..bd70178 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,11 @@ "scripts": { "fetch": "bun run index.ts", "recheck": "bun run recheck.ts", - "daily": "bun run daily.ts" + "daily": "bun run daily.ts", + "mint-datadome": "bun run scripts/mint-datadome.ts" }, - "type": "module" -} \ No newline at end of file + "type": "module", + "dependencies": { + "playwright-core": "^1.60.0" + } +} diff --git a/scripts/mint-datadome.ts b/scripts/mint-datadome.ts new file mode 100644 index 0000000..1ef7641 --- /dev/null +++ b/scripts/mint-datadome.ts @@ -0,0 +1,15 @@ +/** + * Refresh the basel-jobs.ch DataDome cookie. + * + * Opens a real Chrome window for a few seconds — headless and Playwright-launched + * browsers are both fingerprinted and blocked, so this cannot run unattended. + * Run it by hand when `bun fetch` reports basel-jobs.ch as unavailable. + */ +import { mintDataDomeSession } from "../fetch-other.ts"; + +const session = await mintDataDomeSession(); +if (!session) { + console.error("failed to mint a DataDome cookie"); + process.exit(1); +} +console.log(`ok — minted ${session.mintedAt}`);