feat: add startup.jobs as a switzerland-wide job source
This commit is contained in:
+236
-5
@@ -1,5 +1,7 @@
|
||||
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, BIO_DOMAIN_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
||||
import type { Job } from "./fetch.ts";
|
||||
import { tmpdir } from "os";
|
||||
import { rm } from "fs/promises";
|
||||
|
||||
const HEADERS = {
|
||||
"User-Agent":
|
||||
@@ -332,7 +334,7 @@ const JOBROOM_API = "https://www.job-room.ch/jobadservice/api/jobAdvertisements/
|
||||
const JOBROOM_CANTONS = ["BS", "BL", "BE", "ZH"];
|
||||
const JOBROOM_KEYWORDS = [
|
||||
"software", "engineer", "developer", "devops", "data", "python",
|
||||
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
||||
"cloud", "platform", "typescript", "backend", "full stack", "ai", "agentic",
|
||||
];
|
||||
|
||||
// job-room carries every industry; pharma/lab titles slip past SOFTWARE_TITLE_RE
|
||||
@@ -691,7 +693,7 @@ export async function fetchSciCore(): Promise<Job[]> {
|
||||
// costing a page fetch per job in index.ts.
|
||||
const JOBBASEL_KEYWORDS = [
|
||||
"software", "engineer", "developer", "devops", "data", "python",
|
||||
"cloud", "platform", "typescript", "backend", "full stack", "ai",
|
||||
"cloud", "platform", "typescript", "backend", "full stack", "ai", "agentic",
|
||||
];
|
||||
|
||||
interface JobBaselItem {
|
||||
@@ -872,7 +874,7 @@ export async function fetchEnglishJobSearch(): Promise<Job[]> {
|
||||
// empty list. Cards link to /emploi/<slug>-<id>.html; the listing carries no
|
||||
// description, so each title-filtered candidate costs one detail fetch, which
|
||||
// also gives the real publication date (dd.mm.yyyy) and runs the DE/FR check.
|
||||
const MOREJOBS_TERMS = ["software", "engineer", "developer", "devops", "platform", "architect", "data", "python"];
|
||||
const MOREJOBS_TERMS = ["software", "engineer", "developer", "devops", "platform", "architect", "data", "python", "agentic"];
|
||||
const MOREJOBS_MAX_PAGES = 5;
|
||||
// The board is Swiss but lists foreign postings too ("Eastern Europe, Poland",
|
||||
// "95050 Santa Clara", "London"), and its JSON-LD stamps addressCountry "CH" on
|
||||
@@ -1228,7 +1230,7 @@ async function baselJobsFetch(
|
||||
const BASELJOBS_TERMS = [
|
||||
// role / domain words — these carry the `websearch=1` pass
|
||||
"software", "engineer", "developer", "entwickler", "informatik", "data",
|
||||
"cloud", "devops", "platform", "architect", "security", "ai",
|
||||
"cloud", "devops", "platform", "architect", "security", "ai", "agentic",
|
||||
// stack words — only the bare search matches these
|
||||
"python", "javascript", "typescript", "react", "node",
|
||||
"fullstack", "full stack", "backend", "frontend", "kubernetes", "linux",
|
||||
@@ -1334,7 +1336,7 @@ function parseBaselJobsCards(
|
||||
// term, not just a seniority or business-function word BIO_DOMAIN_RE and
|
||||
// PHARMA_ROLE_RE happen to miss.
|
||||
const NOVARTIS_TITLE_RE =
|
||||
/\b(software\s+(engineer|developer|architect)|solutions?\s+architect|platform\s+engineer|cloud\s+engineer|devops(?:\s+engineer)?|data\s+engineer|site\s+reliability\s+engineer|\bsre\b|backend\s+(?:engineer|developer)|frontend\s+(?:engineer|developer)|full.?stack\s+(?:developer|engineer)|machine\s+learning\s+engineer|\bmlops\b|infrastructure\s+engineer|integration\s+engineer|automation\s+engineer|it\s+engineer|systems?\s+engineer|application[s]?\s+engineer|security\s+engineer|ai\s+engineer|engineering\s+manager|technical\s+lead|tech\s+lead|principal\s+(?:software\s+)?engineer|staff\s+engineer|product\s+engineer|programmer|it\s+architect|enterprise\s+architect|data\s+scientist|data\s+platform|digital\s+product\s+(?:engineer|manager)|it\s+project\s+manager|it\s+manager)\b/i;
|
||||
/\b(software\s+(engineer|developer|architect)|solutions?\s+architect|platform\s+engineer|cloud\s+engineer|devops(?:\s+engineer)?|data\s+engineer|site\s+reliability\s+engineer|\bsre\b|backend\s+(?:engineer|developer)|frontend\s+(?:engineer|developer)|full.?stack\s+(?:developer|engineer)|machine\s+learning\s+engineer|\bmlops\b|infrastructure\s+engineer|integration\s+engineer|automation\s+engineer|it\s+engineer|systems?\s+engineer|application[s]?\s+engineer|security\s+engineer|ai\s+engineer|agentic|engineering\s+manager|technical\s+lead|tech\s+lead|principal\s+(?:software\s+)?engineer|staff\s+engineer|product\s+engineer|programmer|it\s+architect|enterprise\s+architect|data\s+scientist|data\s+platform|forward\s+deployed|\bfde\b|palantir|foundry\s+(?:engineer|developer)|digital\s+product\s+(?:engineer|manager)|it\s+project\s+manager|it\s+manager)\b/i;
|
||||
|
||||
// Narrowed to the Basel region specifically (per the source's whole purpose —
|
||||
// the inside contact is there), not all of Switzerland: Novartis CH postings
|
||||
@@ -1552,3 +1554,232 @@ export async function fetchBaselJobs(): Promise<Job[]> {
|
||||
|
||||
return viable;
|
||||
}
|
||||
|
||||
// startup.jobs — global startup board, swept Switzerland-wide. Every path on
|
||||
// the domain (listing, detail pages, feeds, even /robots-adjacent API routes)
|
||||
// sits behind a Cloudflare managed challenge that 403s plain HTTP on any UA.
|
||||
// Unlike basel-jobs.ch's DataDome cookie, the `cf_clearance` cookie does NOT
|
||||
// replay over Bun's fetch (measured: 403 with the exact cookie + UA) — it is
|
||||
// bound to Chrome's TLS fingerprint. What gets through:
|
||||
// - Chrome `--headless=new` with a fresh profile passes the challenge, but
|
||||
// only with the "HeadlessChrome" token stripped from the UA (403 otherwise);
|
||||
// - the listings themselves come from Algolia (app 4CQMTMMK73, index
|
||||
// Post_production), queried with a *secured* search key the page embeds in
|
||||
// `meta[name="current-algolia-api-key-search"]`. algolia.net is not behind
|
||||
// Cloudflare, so once the key is in hand the queries run over plain fetch.
|
||||
// The key is minted per page load with `validUntil` ~3–5h out, so it is read
|
||||
// fresh every run, never cached. That makes this source need a (headless)
|
||||
// browser on every run — which, unlike basel-jobs.ch's visible-window mint,
|
||||
// works fine under launchd.
|
||||
//
|
||||
// The secured key rejects every country filter (`country_code`, `country_id`
|
||||
// both return 0), so the sweep is a geo bounding box around Switzerland with
|
||||
// `country_code === "CH"` checked locally (the box also catches DE/FR/AT/LI/IT).
|
||||
// Algolia caps a query at 500 retrievable hits (100 × 5 pages) while the box
|
||||
// holds ~1400, so a box reporting > 500 is split into quadrants until each fits.
|
||||
// Records carry no description; each title-filtered candidate's detail page is
|
||||
// fetched through the browser (`page.evaluate(fetch)`, same TLS) and its
|
||||
// JSON-LD `JobPosting.description` runs through isViableDescription in-process.
|
||||
// Links already in results/jobs.csv are skipped before that fetch — daily runs
|
||||
// 5×/day and the detail fetches are the only expensive part.
|
||||
const STARTUPJOBS_ORIGIN = "https://startup.jobs";
|
||||
const STARTUPJOBS_ALGOLIA = "https://4CQMTMMK73-dsn.algolia.net/1/indexes/Post_production/query";
|
||||
const STARTUPJOBS_CH_BOX: [number, number, number, number] = [45.8, 5.9, 47.85, 10.5];
|
||||
const STARTUPJOBS_MAX_AGE_DAYS = 60;
|
||||
|
||||
interface StartupJobsHit {
|
||||
objectID: string;
|
||||
title: string;
|
||||
company_name: string;
|
||||
country_code?: string;
|
||||
location?: string;
|
||||
path: string;
|
||||
published_at_iso8601?: string;
|
||||
}
|
||||
|
||||
function chromeUserAgent(): string | null {
|
||||
const out = Bun.spawnSync([CHROME_BIN, "--version"]).stdout.toString();
|
||||
const version = /(\d+\.\d+\.\d+\.\d+)/.exec(out)?.[1];
|
||||
if (!version) return null;
|
||||
return `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${version} Safari/537.36`;
|
||||
}
|
||||
|
||||
async function startupJobsQuery(
|
||||
key: string,
|
||||
box: [number, number, number, number],
|
||||
since: number,
|
||||
page: number
|
||||
): Promise<{ nbHits: number; nbPages: number; hits: StartupJobsHit[] }> {
|
||||
const res = await fetch(STARTUPJOBS_ALGOLIA, {
|
||||
method: "POST",
|
||||
headers: { "X-Algolia-Application-Id": "4CQMTMMK73", "X-Algolia-API-Key": key },
|
||||
body: JSON.stringify({
|
||||
query: "",
|
||||
insideBoundingBox: [box],
|
||||
numericFilters: [`published_at_i>${since}`],
|
||||
hitsPerPage: 100,
|
||||
page,
|
||||
attributesToRetrieve: ["title", "company_name", "country_code", "location", "path", "published_at_iso8601"],
|
||||
attributesToHighlight: [],
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`algolia ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function startupJobsSweep(
|
||||
key: string,
|
||||
box: [number, number, number, number],
|
||||
since: number,
|
||||
out: Map<string, StartupJobsHit>,
|
||||
depth = 0
|
||||
): Promise<void> {
|
||||
const first = await startupJobsQuery(key, box, since, 0);
|
||||
if (first.nbHits > 500 && depth < 6) {
|
||||
const [s, w, n, e] = box;
|
||||
const midLat = (s + n) / 2;
|
||||
const midLng = (w + e) / 2;
|
||||
for (const q of [
|
||||
[s, w, midLat, midLng], [s, midLng, midLat, e],
|
||||
[midLat, w, n, midLng], [midLat, midLng, n, e],
|
||||
] as [number, number, number, number][]) {
|
||||
await startupJobsSweep(key, q, since, out, depth + 1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (first.nbHits > 500) console.log(` warning: box ${box.join(",")} still holds ${first.nbHits} hits — truncated at 500`);
|
||||
for (const h of first.hits) out.set(h.objectID, h);
|
||||
for (let p = 1; p < first.nbPages; p++) {
|
||||
const r = await startupJobsQuery(key, box, since, p);
|
||||
for (const h of r.hits) out.set(h.objectID, h);
|
||||
}
|
||||
}
|
||||
|
||||
async function knownCsvLinks(): Promise<Set<string>> {
|
||||
try {
|
||||
const text = await Bun.file("results/jobs.csv").text();
|
||||
return new Set(text.match(/https:\/\/startup\.jobs\/[^\s",]+/g) ?? []);
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchStartupJobs(): Promise<Job[]> {
|
||||
if (process.env.LINKME_NO_BROWSER === "1") {
|
||||
console.log(" LINKME_NO_BROWSER=1 — startup.jobs unavailable");
|
||||
return [];
|
||||
}
|
||||
const ua = (await Bun.file(CHROME_BIN).exists()) ? chromeUserAgent() : null;
|
||||
if (!ua) {
|
||||
console.error(` Chrome not found at ${CHROME_BIN} — startup.jobs unavailable`);
|
||||
return [];
|
||||
}
|
||||
|
||||
let chromium: typeof import("playwright-core").chromium;
|
||||
try {
|
||||
({ chromium } = await import("playwright-core"));
|
||||
} catch {
|
||||
console.error(" playwright-core not installed — run `bun add playwright-core`");
|
||||
return [];
|
||||
}
|
||||
|
||||
// Own port so it never collides with the DataDome mint (9333) running alongside.
|
||||
const port = 9334;
|
||||
const profile = `${tmpdir()}/linkme-startupjobs-${process.pid}`;
|
||||
const proc = Bun.spawn(
|
||||
[
|
||||
CHROME_BIN,
|
||||
"--headless=new",
|
||||
`--user-agent=${ua}`,
|
||||
`--remote-debugging-port=${port}`,
|
||||
`--user-data-dir=${profile}`,
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"about:blank",
|
||||
],
|
||||
{ stdout: "ignore", stderr: "ignore" }
|
||||
);
|
||||
|
||||
const jobs: Job[] = [];
|
||||
try {
|
||||
if (!(await waitForCdp(port, 20_000))) {
|
||||
console.error(" Chrome CDP endpoint never came up — startup.jobs unavailable");
|
||||
return [];
|
||||
}
|
||||
const browser = await chromium.connectOverCDP(`http://127.0.0.1:${port}`);
|
||||
try {
|
||||
const page = browser.contexts()[0].pages()[0] ?? (await browser.contexts()[0].newPage());
|
||||
await page.goto(STARTUPJOBS_ORIGIN, { timeout: 30_000 });
|
||||
|
||||
let key: string | null = null;
|
||||
for (let i = 0; i < 15 && !key; i++) {
|
||||
key = await page
|
||||
.evaluate(() => document.querySelector('meta[name="current-algolia-api-key-search"]')?.getAttribute("content") ?? null)
|
||||
.catch(() => null); // the challenge navigates mid-evaluate
|
||||
if (!key) await page.waitForTimeout(1000);
|
||||
}
|
||||
if (!key) {
|
||||
console.error(" Cloudflare challenge did not clear (no Algolia key) — startup.jobs unavailable");
|
||||
return [];
|
||||
}
|
||||
|
||||
const since = Math.floor(Date.now() / 1000) - STARTUPJOBS_MAX_AGE_DAYS * 86400;
|
||||
const hits = new Map<string, StartupJobsHit>();
|
||||
await startupJobsSweep(key, STARTUPJOBS_CH_BOX, since, hits);
|
||||
|
||||
const known = await knownCsvLinks();
|
||||
const candidates = [...hits.values()].filter((h) => {
|
||||
if (h.country_code !== "CH") return false;
|
||||
const title = decodeHtml(h.title);
|
||||
if (!applyFilters(title)) return false;
|
||||
if (MS_STACK_RE.test(title) || BIO_DOMAIN_RE.test(title)) return false;
|
||||
return !known.has(`${STARTUPJOBS_ORIGIN}${h.path}`);
|
||||
});
|
||||
console.log(` ${hits.size} in CH box, ${candidates.length} new title-matched candidates`);
|
||||
|
||||
let langDropped = 0;
|
||||
for (const h of candidates) {
|
||||
const description = await page
|
||||
.evaluate(async (path) => {
|
||||
const r = await fetch(path);
|
||||
if (!r.ok) return null;
|
||||
const doc = new DOMParser().parseFromString(await r.text(), "text/html");
|
||||
for (const s of doc.querySelectorAll('script[type="application/ld+json"]')) {
|
||||
try {
|
||||
const data = JSON.parse(s.textContent ?? "");
|
||||
for (const d of Array.isArray(data) ? data : [data]) {
|
||||
if (d?.["@type"] === "JobPosting") return String(d.description ?? "");
|
||||
}
|
||||
} catch { /* malformed block */ }
|
||||
}
|
||||
return null;
|
||||
}, h.path)
|
||||
.catch(() => null);
|
||||
if (description === null) continue; // detail page failed — retry next run
|
||||
if (!isViableDescription(description).ok) { langDropped++; continue; }
|
||||
|
||||
jobs.push({
|
||||
id: `sj-${h.objectID}`,
|
||||
title: decodeHtml(h.title),
|
||||
company: h.company_name ?? "",
|
||||
location: h.location ?? "Switzerland",
|
||||
postedAt: h.published_at_iso8601?.slice(0, 10) ?? "",
|
||||
link: `${STARTUPJOBS_ORIGIN}${h.path}`,
|
||||
search: "startup.jobs",
|
||||
source: "startup.jobs",
|
||||
contentChecked: true, // isViableDescription already ran on the JSON-LD description
|
||||
});
|
||||
await Bun.sleep(300);
|
||||
}
|
||||
console.log(` ${jobs.length} kept, ${langDropped} non-English / German-required`);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(` startup.jobs failed: ${e}`);
|
||||
} finally {
|
||||
proc.kill();
|
||||
await rm(profile, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user