feat: add novartis.com and more-jobs.ch as job sources
more-jobs.ch lists foreign postings tagged as CH, so drop locations naming another country/city or a 5-digit postcode, and dedupe its reposts by title+company.
This commit is contained in:
+261
-1
@@ -1,4 +1,4 @@
|
||||
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
||||
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE, BIO_DOMAIN_RE, requiresGerman, hasNonEnglishUmlaut } from "./fetch.ts";
|
||||
import type { Job } from "./fetch.ts";
|
||||
|
||||
const HEADERS = {
|
||||
@@ -864,6 +864,111 @@ export async function fetchEnglishJobSearch(): Promise<Job[]> {
|
||||
return jobs;
|
||||
}
|
||||
|
||||
// more-jobs.ch — server-rendered PHP board, an aggregator (the same posting
|
||||
// often also sits on LinkedIn/jobs.ch, deduped by title+company in index.ts).
|
||||
// The site is Swiss-wide with ~20 cards per page; /recherche?q= is fast on the
|
||||
// first pages but deep pages take 20s+ and out-of-range pages clamp to the last
|
||||
// one instead of going empty, so "no new ids" ends a keyword's sweep, not an
|
||||
// empty list. Cards link to /emploi/<slug>-<id>.html; the listing carries no
|
||||
// description, so each title-filtered candidate costs one detail fetch, which
|
||||
// also gives the real publication date (dd.mm.yyyy) and runs the DE/FR check.
|
||||
const MOREJOBS_TERMS = ["software", "engineer", "developer", "devops", "platform", "architect", "data", "python"];
|
||||
const MOREJOBS_MAX_PAGES = 5;
|
||||
// The board is Swiss but lists foreign postings too ("Eastern Europe, Poland",
|
||||
// "95050 Santa Clara", "London"), and its JSON-LD stamps addressCountry "CH" on
|
||||
// all of them — so the card's location text is the only signal. Drop anything
|
||||
// naming another country/region/foreign city, or carrying a 5-digit postcode
|
||||
// (Swiss ones are 4 digits).
|
||||
const MOREJOBS_FOREIGN_RE = new RegExp(
|
||||
"\\b(?:usa|united states|canada|mexico|brazil|india|china|japan|singapore|thailand|australia|" +
|
||||
"united kingdom|uk|england|scotland|ireland|london|cambridge|cambridgeshire|manchester|" +
|
||||
"germany|deutschland|berlin|munich|m(?:ü|u)nchen|frankfurt|hamburg|stuttgart|cologne|k(?:ö|o)ln|" +
|
||||
"france|paris|lyon|annecy|italy|italia|milan|milano|rome|spain|madrid|barcelona|portugal|porto|lisbon|lisboa|" +
|
||||
"austria|(?:ö|o)sterreich|vienna|wien|netherlands|amsterdam|belgium|brussels|luxembourg|denmark|sweden|norway|finland|" +
|
||||
"poland|warsaw|krakow|romania|bucharest|czech|prague|hungary|budapest|bulgaria|sofia|serbia|belgrade|croatia|greece|" +
|
||||
"slovakia|slovenia|ukraine|turkey|israel|dubai|uae)\\b|\\b\\d{5}\\b",
|
||||
"i"
|
||||
);
|
||||
|
||||
function moreJobsLocation(raw: string): string {
|
||||
// Some cards repeat the location twice ("Zurich, Switzerland Zurich, Switzerland").
|
||||
const half = raw.slice(0, raw.length / 2).trim();
|
||||
const loc = raw.length % 2 === 1 && raw === `${half} ${half}` ? half : raw;
|
||||
return loc.replace(/(?:,\s*Switzerland)+$/i, "").replace(/^[,\s]+|[,\s]+$/g, "");
|
||||
}
|
||||
|
||||
async function fetchMoreJobsPage(url: string): Promise<string | null> {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20_000) });
|
||||
if (res.ok) return await res.text();
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function fetchMoreJobs(): Promise<Job[]> {
|
||||
const candidates = new Map<string, { title: string; company: string; location: string; path: string }>();
|
||||
|
||||
for (const term of MOREJOBS_TERMS) {
|
||||
process.stdout.write(` ${term} … `);
|
||||
let found = 0;
|
||||
for (let page = 1; page <= MOREJOBS_MAX_PAGES; page++) {
|
||||
const html = await fetchMoreJobsPage(
|
||||
`https://www.more-jobs.ch/recherche?q=${encodeURIComponent(term)}&page=${page}`
|
||||
);
|
||||
if (!html) break;
|
||||
const cards = [...html.matchAll(
|
||||
/<a class="positions-list-item[^"]*" href="(\/emploi\/[^"]*-(\d+)\.html)"[^>]*>\s*<h2><span>([^<]*)<\/span><\/h2>[\s\S]*?<h3><span[^>]*>([^<]*)<\/span> - <span>([^<]*)<\/span>/g
|
||||
)];
|
||||
let fresh = 0;
|
||||
for (const [, path, id, title, company, location] of cards) {
|
||||
if (candidates.has(id)) continue;
|
||||
fresh++;
|
||||
candidates.set(id, {
|
||||
title: decodeHtml(title.trim()),
|
||||
company: decodeHtml(company.trim()),
|
||||
location: moreJobsLocation(decodeHtml(location.trim())),
|
||||
path,
|
||||
});
|
||||
}
|
||||
found += fresh;
|
||||
if (fresh === 0 || cards.length < 20) break;
|
||||
await Bun.sleep(300);
|
||||
}
|
||||
console.log(`${found} new`);
|
||||
}
|
||||
|
||||
const jobs: Job[] = [];
|
||||
const queue = [...candidates].filter(([, c]) =>
|
||||
applyFilters(c.title) && !MS_STACK_RE.test(c.title) && !MOREJOBS_FOREIGN_RE.test(c.location)
|
||||
);
|
||||
console.log(` ${candidates.size} unique, ${queue.length} pass the title filters — fetching details`);
|
||||
async function worker() {
|
||||
while (queue.length) {
|
||||
const [id, c] = queue.shift()!;
|
||||
const html = await fetchMoreJobsPage(`https://www.more-jobs.ch${c.path}`);
|
||||
if (!html) continue; // next run retries — nothing claimed yet
|
||||
const body = html.match(/<div class="job-content"[^>]*>([\s\S]*?)<\/div>\s*(?:<!--|<div class="clearfix)/)?.[1] ?? "";
|
||||
if (body && !isViableDescription(body).ok) continue;
|
||||
const d = html.match(/Publication date<\/dt>\s*<dd>(\d{2})\.(\d{2})\.(\d{4})/);
|
||||
jobs.push({
|
||||
id: `morejobs-${id}`,
|
||||
title: c.title,
|
||||
company: c.company,
|
||||
location: c.location ? `${c.location}, Switzerland` : "Switzerland",
|
||||
postedAt: d ? `${d[3]}-${d[2]}-${d[1]}` : "",
|
||||
link: `https://www.more-jobs.ch${c.path}`,
|
||||
search: "more-jobs.ch",
|
||||
source: "more-jobs.ch",
|
||||
contentChecked: body.length > 40,
|
||||
});
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: 4 }, worker));
|
||||
return jobs;
|
||||
}
|
||||
|
||||
// stellen-basel.ch — WordPress REST API
|
||||
export async function fetchStellenBasel(): Promise<Job[]> {
|
||||
const allJobs: Job[] = [];
|
||||
@@ -1201,6 +1306,161 @@ function parseBaselJobsCards(
|
||||
return { items: items.length, added };
|
||||
}
|
||||
|
||||
// novartis.com — Rafael has an internal contact at Novartis Basel who can refer
|
||||
// him, so this source watches specifically for roles that would be worth
|
||||
// flagging to that contact rather than applying cold.
|
||||
//
|
||||
// The career-search page (Drupal Views) is server-rendered HTML, no headless
|
||||
// browser needed — but only at the locale-less path (`/careers/career-search`,
|
||||
// English). The `ch-de` path serves the same listing in German and its
|
||||
// "job/details" links 302 back to the search page for both locales when hit
|
||||
// without a referrer/session, so job detail pages are only readable through
|
||||
// the English path (`/careers/career-search/job/details/req-...`), which
|
||||
// resolves fine and returns the full posting.
|
||||
//
|
||||
// Pagination is 0-indexed (`page=0` is implicit/omitted, `page=1` is the
|
||||
// second page of 10) and the page's own "Showing N results" header gives the
|
||||
// exact total, so the loop bound is computed rather than probed.
|
||||
//
|
||||
// Novartis is 100% pharma, and its Basel HQ titling skews heavily to
|
||||
// non-technical "Director, <business function>" roles (Portfolio Analytics,
|
||||
// Therapeutic Area Strategy, Policy Communications, ...) plus device/hardware
|
||||
// "Engineering" roles (autoinjector/drug-delivery hardware, not software).
|
||||
// SOFTWARE_TITLE_RE's bare "director"/"analytics"/"ai\b" branches — tuned for
|
||||
// smaller, less title-inflated boards — matched every one of those in an
|
||||
// initial run (measured 2026-09-21: 13 of 47 CH listings "kept", none of them
|
||||
// an actual engineering-team role). A dedicated, stricter title match is used
|
||||
// here instead: it requires an explicit software/IT engineering job-family
|
||||
// term, not just a seniority or business-function word BIO_DOMAIN_RE and
|
||||
// PHARMA_ROLE_RE happen to miss.
|
||||
const NOVARTIS_TITLE_RE =
|
||||
/\b(software\s+(engineer|developer|architect)|solutions?\s+architect|platform\s+engineer|cloud\s+engineer|devops(?:\s+engineer)?|data\s+engineer|site\s+reliability\s+engineer|\bsre\b|backend\s+(?:engineer|developer)|frontend\s+(?:engineer|developer)|full.?stack\s+(?:developer|engineer)|machine\s+learning\s+engineer|\bmlops\b|infrastructure\s+engineer|integration\s+engineer|automation\s+engineer|it\s+engineer|systems?\s+engineer|application[s]?\s+engineer|security\s+engineer|ai\s+engineer|engineering\s+manager|technical\s+lead|tech\s+lead|principal\s+(?:software\s+)?engineer|staff\s+engineer|product\s+engineer|programmer|it\s+architect|enterprise\s+architect|data\s+scientist|data\s+platform|digital\s+product\s+(?:engineer|manager)|it\s+project\s+manager|it\s+manager)\b/i;
|
||||
|
||||
// Narrowed to the Basel region specifically (per the source's whole purpose —
|
||||
// the inside contact is there), not all of Switzerland: Novartis CH postings
|
||||
// also list Stein (Aargau) and a handful of remote/blank locations that are a
|
||||
// different commute and not what the contact could refer into.
|
||||
const NOVARTIS_BASE = "https://www.novartis.com";
|
||||
const NOVARTIS_BASEL_RE = /\bbasel\b|\bmuttenz\b/i;
|
||||
|
||||
interface NovartisListing {
|
||||
title: string;
|
||||
href: string;
|
||||
businessUnit: string;
|
||||
location: string;
|
||||
postedAt: string;
|
||||
}
|
||||
|
||||
function parseNovartisDate(text: string): string {
|
||||
const d = new Date(text.trim());
|
||||
return isNaN(d.getTime()) ? "" : d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function parseNovartisPage(html: string): { total: number; listings: NovartisListing[] } {
|
||||
const totalM = /Showing\s+(\d+)\s+results/i.exec(html);
|
||||
const total = totalM ? Number(totalM[1]) : 0;
|
||||
|
||||
const listings: NovartisListing[] = [];
|
||||
const rows = [...html.matchAll(/<tr class="(?:odd|even)"[^>]*>([\s\S]*?)<\/tr>/g)];
|
||||
for (const [, row] of rows) {
|
||||
const titleM = /views-field-field-job-title"><a href="([^"]+)">([^<]*)<\/a>/.exec(row);
|
||||
const buM = /views-field-field-job-business-unit">([^<]*)<\/td>/.exec(row);
|
||||
const locM = /views-field-field-job-work-location">([^<]*)<\/td>/.exec(row);
|
||||
const dateM = /views-field-field-job-posted-date">([^<]*)<\/td>/.exec(row);
|
||||
if (!titleM) continue;
|
||||
|
||||
listings.push({
|
||||
href: titleM[1],
|
||||
title: decodeHtml(titleM[2]),
|
||||
businessUnit: buM ? decodeHtml(buM[1]) : "",
|
||||
location: locM ? decodeHtml(locM[1]) : "",
|
||||
postedAt: dateM ? parseNovartisDate(dateM[1]) : "",
|
||||
});
|
||||
}
|
||||
return { total, listings };
|
||||
}
|
||||
|
||||
function applyNovartisFilters(title: string): boolean {
|
||||
if (NON_ENGLISH_RE.test(title)) return false;
|
||||
if (JUNIOR_TITLE_RE.test(title)) return false;
|
||||
if (MS_STACK_RE.test(title)) return false;
|
||||
if (BIO_DOMAIN_RE.test(title)) return false;
|
||||
if (PHARMA_ROLE_RE.test(title)) return false;
|
||||
return NOVARTIS_TITLE_RE.test(title);
|
||||
}
|
||||
|
||||
export async function fetchNovartisBasel(): Promise<Job[]> {
|
||||
const jobs: Job[] = [];
|
||||
// The listing has been observed to repeat rows across pages within one
|
||||
// sweep (measured 2026-09-21: several titles appeared twice across the 5
|
||||
// pages of one run) — dedupe by req id rather than trusting the pager.
|
||||
const byReqId = new Map<string, NovartisListing>();
|
||||
|
||||
const firstHtml = await (
|
||||
await fetch(`${NOVARTIS_BASE}/careers/career-search?search_api_fulltext=&country%5B0%5D=LOC_CH&field_job_posted_date=All&op=Submit`, { headers: HEADERS })
|
||||
).text();
|
||||
const { total, listings: firstListings } = parseNovartisPage(firstHtml);
|
||||
const totalPages = Math.max(1, Math.ceil(total / 10));
|
||||
console.log(` Novartis … ${total} CH results, ${totalPages} pages`);
|
||||
for (const item of firstListings) {
|
||||
const reqId = /req-(\d+)/.exec(item.href)?.[1];
|
||||
if (reqId) byReqId.set(reqId, item);
|
||||
}
|
||||
|
||||
for (let page = 1; page < totalPages; page++) {
|
||||
await Bun.sleep(400);
|
||||
const res = await fetch(
|
||||
`${NOVARTIS_BASE}/careers/career-search?search_api_fulltext=&country%5B0%5D=LOC_CH&field_job_posted_date=All&op=Submit&page=${page}`,
|
||||
{ headers: HEADERS }
|
||||
);
|
||||
if (!res.ok) { console.log(` Novartis … page ${page} failed (${res.status})`); break; }
|
||||
const { listings } = parseNovartisPage(await res.text());
|
||||
for (const item of listings) {
|
||||
const reqId = /req-(\d+)/.exec(item.href)?.[1];
|
||||
if (reqId) byReqId.set(reqId, item);
|
||||
}
|
||||
}
|
||||
|
||||
let kept = 0;
|
||||
let langDropped = 0;
|
||||
for (const [reqId, item] of byReqId) {
|
||||
if (!NOVARTIS_BASEL_RE.test(item.location)) continue;
|
||||
if (!applyNovartisFilters(item.title)) continue;
|
||||
|
||||
const link = `${NOVARTIS_BASE}${item.href}`;
|
||||
await Bun.sleep(300);
|
||||
const detailRes = await fetch(link, { headers: HEADERS });
|
||||
let contentChecked = false;
|
||||
if (detailRes.ok) {
|
||||
const detailHtml = await detailRes.text();
|
||||
const start = detailHtml.indexOf("job_details_content_center");
|
||||
const end = detailHtml.indexOf("job_details_content_bottom");
|
||||
if (start !== -1 && end !== -1) {
|
||||
const description = detailHtml.slice(start, end).replace(/<[^>]+>/g, " ");
|
||||
const check = isViableDescription(description);
|
||||
if (!check.ok) { langDropped++; continue; }
|
||||
contentChecked = true;
|
||||
}
|
||||
}
|
||||
|
||||
jobs.push({
|
||||
id: `nov-${reqId}`,
|
||||
title: item.title,
|
||||
company: "Novartis",
|
||||
location: `${item.location}, Switzerland`,
|
||||
postedAt: item.postedAt,
|
||||
link,
|
||||
search: "novartis.com",
|
||||
source: "novartis.com",
|
||||
contentChecked,
|
||||
});
|
||||
kept++;
|
||||
}
|
||||
|
||||
console.log(` Novartis … ${byReqId.size} unique CH listings, ${kept} kept (Basel-area + profile match), ${langDropped} non-English/German-required dropped`);
|
||||
return jobs;
|
||||
}
|
||||
|
||||
export async function fetchBaselJobs(): Promise<Job[]> {
|
||||
const allJobs: Job[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
Reference in New Issue
Block a user