rodado
274f946bd2
"Software Engineer auf dem Weg zum Data / AI Engineer" (ti&m, via job-room.ch) sailed through: none of the German-word list matched 'auf dem Weg zum', and the job-room description was too short to clear the separate description-level DE/FR word-count threshold. auf/zum/zur/vom/für/beim don't exist as English words, so they're safe unambiguous markers — verified against the full 2858-row jobs.csv history: only this one title now flags.
439 lines
20 KiB
TypeScript
439 lines
20 KiB
TypeScript
import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";
|
||
|
||
export interface Job {
|
||
id: string;
|
||
title: string;
|
||
company: string;
|
||
location: string;
|
||
postedAt: string;
|
||
link: string;
|
||
search: string;
|
||
source: string;
|
||
/**
|
||
* Set when the source already language-checked the real job description.
|
||
* `isEnglishUrl` counts DE/FR words across the whole page, so boards with
|
||
* German site chrome (itjobs.ch, stellen-basel.ch) fail it even when the
|
||
* posting itself is English — skip the re-check for those.
|
||
*/
|
||
contentChecked?: boolean;
|
||
}
|
||
|
||
const WORK_TYPE_MAP: Record<WorkType, string> = {
|
||
onsite: "1",
|
||
remote: "2",
|
||
hybrid: "3",
|
||
};
|
||
|
||
const JOB_TYPE_MAP: Record<JobType, string> = {
|
||
"full-time": "F",
|
||
"part-time": "P",
|
||
contract: "C",
|
||
temporary: "T",
|
||
internship: "I",
|
||
};
|
||
|
||
const TIME_MAP: Record<TimePosted, string> = {
|
||
day: "r86400",
|
||
week: "r604800",
|
||
month: "r2592000",
|
||
};
|
||
|
||
const SENIORITY_MAP: Record<Seniority, string> = {
|
||
internship: "1",
|
||
entry: "2",
|
||
associate: "3",
|
||
"mid-senior": "4",
|
||
director: "5",
|
||
executive: "6",
|
||
};
|
||
|
||
// Detect non-English (German / French / Swiss-German) job titles.
|
||
//
|
||
// German signals:
|
||
// - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
|
||
// but NOT (m/f/d) / (f/m/d) which are used in English international posts
|
||
// - Swiss neutral suffix: standalone (a) before % or end of string
|
||
// - German umlauts / ß
|
||
// - /in word-ending (Pflegehelfer/in, Köchin/Koch)
|
||
// - German-only vocabulary
|
||
// - Swiss German: EFZ (apprenticeship cert), Schweiz
|
||
//
|
||
// French signals:
|
||
// - gender markers: (H/F), (F/H), (h/f)
|
||
// - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
|
||
// - French-only vocabulary
|
||
export const NON_ENGLISH_RE = new RegExp(
|
||
[
|
||
// German gender markers (w = weiblich; must not match m/f/d)
|
||
String.raw`\(m\/w[^)]*\)`,
|
||
String.raw`\(w\/m[^)]*\)`,
|
||
String.raw`\(d\/w[^)]*\)`,
|
||
String.raw`\(mwd\)`,
|
||
// m/w without parens (e.g. "Developer m/w")
|
||
String.raw`\bm\/w\b`,
|
||
// Swiss neutral (a) suffix: " (a)" before %, digits, or end
|
||
String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
|
||
// German/Swiss umlauts and ß
|
||
String.raw`[äöüÄÖÜß]`,
|
||
// /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
|
||
String.raw`[a-zäöü]\/in\b`,
|
||
// German-only words (case-insensitive handled by flag)
|
||
String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale|Ingenieur|Spezialist|Perspektive|Stellvertretung|Leitung|Beratung|Entwicklung|Betreuung|Verwaltung)\b`,
|
||
// German prepositions/articles in title context (must appear as whole words) —
|
||
// none of these exist as English words, so they're safe unambiguous markers.
|
||
// Added after "Software Engineer auf dem Weg zum Data / AI Engineer" (ti&m,
|
||
// via job-room.ch) sailed through: none of the German-word list above
|
||
// matched, and the job-room description was too short (~1 hit) to clear the
|
||
// separate ≥6 DE/FR word-count threshold either.
|
||
String.raw`\bmit\b|\bauf\b|\bzum\b|\bzur\b|\bvom\b|\bfür\b|\bbeim\b`,
|
||
// French gender markers
|
||
String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
|
||
// French-only accented characters
|
||
String.raw`[éèêëàâçîïôùûÿœæ]`,
|
||
// French-only words
|
||
String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
|
||
].join("|"),
|
||
"i"
|
||
);
|
||
|
||
const HEADERS = {
|
||
"User-Agent":
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
Accept:
|
||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||
};
|
||
|
||
export const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:ship)?|trainee|apprentice|werkstudent)\b/i;
|
||
|
||
export const JAVA_TITLE_RE = /\bjava\b/i;
|
||
|
||
// Roles whose *domain* is biology / chemistry / the lab bench. Rafael has no
|
||
// life-science background, so these need expertise he can't offer.
|
||
//
|
||
// This filters on the ROLE, never the employer: software, platform, data and
|
||
// infrastructure jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope —
|
||
// Basel pharma is a core target market. Only the science itself is excluded.
|
||
export const BIO_DOMAIN_RE = new RegExp(
|
||
[
|
||
// bench / domain science job families
|
||
String.raw`\b(bio)?statistician\b`,
|
||
String.raw`\bbioinformatic\w*`,
|
||
String.raw`\bcomputational\s+(?:biolog\w*|chemist\w*)`,
|
||
String.raw`\b(?:cheminformatic\w*|chemometric\w*)`,
|
||
String.raw`\b(?:micro)?biolog(?:y|ist)\b`,
|
||
String.raw`\b(?:bio)?chemist(?:ry|s)?\b`,
|
||
String.raw`\bpharmacolog\w*|\btoxicolog\w*|\bimmunolog\w*|\bvirolog\w*|\bepidemiolog\w*`,
|
||
String.raw`\bgenomic\w*|\bproteomic\w*|\bmetabolomic\w*|\bmolecular\s+(?:biolog\w*|patholog\w*)`,
|
||
// manufacturing / QC / regulated-lab functions
|
||
String.raw`\bGMP\b|\bGxP\b|\bMSAT\b|\bQC\s+analyst\b`,
|
||
String.raw`\b(?:bio)?process\s+(?:development|engineer)|\bupstream\b|\bdownstream\s+process`,
|
||
String.raw`\bassay\b|\bbioassay\b|\bwet.?lab\b|\blaboratory\s+(?:technician|assistant|manager)`,
|
||
// clinical / trial work
|
||
String.raw`\bclinical\s+(?:trial|research|data\s+manager|operations)`,
|
||
String.raw`\bbiomarker\b|\bpharmacovigilance\b|\bregulatory\s+affairs\b`,
|
||
// life-science domains that make any title (Scientist, Engineer, Analyst) a domain role
|
||
String.raw`\bbiologic\w*|\bpatholog\w*|\bhistolog\w*`,
|
||
String.raw`\bdrug\s+(?:discovery|design|development)\b`,
|
||
String.raw`\breal.world\s+(?:data|evidence)\b`,
|
||
].join("|"),
|
||
"i"
|
||
);
|
||
|
||
export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devsecops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|infrastructure|mlops|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|scientific\s*(?:software|comput\w*|programmer)|research\s*(?:software|engineer)|hpc|forward\s*deployed|field\s*engineer|customer\s*engineer|sales\s*engineer|pre.?sales|integration\s*engineer|api\s*engineer|founding\s*engineer|product\s*engineer|automation\s*engineer|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;
|
||
|
||
const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
|
||
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;
|
||
|
||
// --- German-requirement detection -----------------------------------------
|
||
//
|
||
// Distinct from isEnglishContent: that drops postings *written* in German.
|
||
// This drops postings written in English that *require* German above B1 —
|
||
// e.g. Helvetia's "Good knowledge of German and English", which reads as a
|
||
// clean English posting and sails through every other filter.
|
||
//
|
||
// Rafael is German B1 (TELC). B1 or "basic"/"a plus" is fine; B2+ is not.
|
||
|
||
/** Any mention of the language, in either language. */
|
||
const GERMAN_MENTION_RE = /\b(german|deutsch\w*|allemand)\b/gi;
|
||
|
||
/** Within a mention's window: signals the requirement is optional. */
|
||
const GERMAN_OPTIONAL_RE =
|
||
/\b(?:a\s+plus|an?\s+advantage|advantageous|beneficial|nice[\s-]to[\s-]have|desirable|preferred|welcome|bonus|not\s+required|no\s+german|optional|willing(?:ness)?\s+to\s+learn|basic|beginner|elementary|grundkenntnisse|von\s+vorteil|wünschenswert|A1|A2|B1)\b/i;
|
||
|
||
/** Within a mention's window: signals it is genuinely required at B2+. */
|
||
const GERMAN_REQUIRED_RE =
|
||
/\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluen\w*|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend\w*|fließend\w*|muttersprach\w*|sehr\s+gut\w*|gut\w*\s+Deutschkenntnisse|Deutschkenntnisse|voraussetzung|zwingend|erforderlich)\b/i;
|
||
|
||
/**
|
||
* Spec-sheet style language line ("Languages: German & English"), common on
|
||
* contractor postings. There is no adjective for the nearest-marker logic to
|
||
* bind to, so the label itself counts as a requirement — but it goes through
|
||
* the same nearest-marker comparison, so "Language: English and German (bonus)"
|
||
* still loses to the "bonus" sitting closer to the mention.
|
||
*/
|
||
const GERMAN_SPEC_LABEL_RE = /\b(?:languages?|sprachen|sprache)\s*[::]/i;
|
||
|
||
/** Title-level marker — no description fetch needed. */
|
||
export const GERMAN_TITLE_RE =
|
||
/\bgerman[\s-]speaking\b|\bdeutschsprachig\w*|\(\s*de\s*\)\s*speaking|\bgerman\s+(?:required|mandatory)\b/i;
|
||
|
||
/**
|
||
* True when the text demands German above B1.
|
||
*
|
||
* Presence of a marker in the window isn't enough — an unrelated "a plus" from
|
||
* a neighbouring bullet would cancel a real requirement (Helvetia's posting
|
||
* puts "Infrastructure skills are a plus" ~95 chars before "Good knowledge of
|
||
* German"). So for each mention we take whichever marker sits *closest*.
|
||
*/
|
||
export function requiresGerman(text: string): boolean {
|
||
const flat = text.replace(/\s+/g, " ");
|
||
const optional = new RegExp(GERMAN_OPTIONAL_RE.source, "gi");
|
||
const required = new RegExp(
|
||
`${GERMAN_REQUIRED_RE.source}|${GERMAN_SPEC_LABEL_RE.source}`,
|
||
"gi"
|
||
);
|
||
|
||
// A marker binds to the German mention only if no OTHER language name sits
|
||
// between them: in "Fluent in English; German ... advantageous", "Fluent"
|
||
// qualifies English, not German.
|
||
const OTHER_LANG = /\b(english|french|italian|spanish|portuguese|anglais|français|englisch|franz|italienisch)\b/i;
|
||
|
||
// …except when that other language is *coordinated* with German, where the
|
||
// marker governs both: "Excellent communication skills in both English and
|
||
// German" (Karakun) and "Fluency in English and German" (J. Safra Sarasin)
|
||
// are hard requirements that the bare rule above waved through — the entire
|
||
// reason 24 basel-jobs.ch postings mentioning German were caught 0 times.
|
||
// Deliberately excludes "and/or" and a bare "or": "Very good English and/or
|
||
// German" (BBF) is satisfied by English alone, so it must stay optional.
|
||
const CONJOINED = /\b(?:and|und|sowie|as\s+well\s+as|&)\s*$/i;
|
||
const ANDOR = /\b(?:and\s*\/\s*or|or)\s*$/i;
|
||
|
||
const nearest = (re: RegExp, window: string, at: number, bind: boolean): number => {
|
||
let best = Infinity;
|
||
for (const m of window.matchAll(re)) {
|
||
const i = m.index ?? 0;
|
||
const between = i < at ? window.slice(i + m[0].length, at) : window.slice(at, i);
|
||
// "…English and German" keeps the marker; "…English; German" or
|
||
// "…English and/or German" hands it to the other language.
|
||
const conjoined = CONJOINED.test(between) && !ANDOR.test(between);
|
||
if (bind && OTHER_LANG.test(between) && !conjoined) continue;
|
||
const d = Math.abs(i - at);
|
||
if (d < best) best = d;
|
||
}
|
||
return best;
|
||
};
|
||
|
||
for (const m of flat.matchAll(GERMAN_MENTION_RE)) {
|
||
const i = m.index ?? 0;
|
||
const from = Math.max(0, i - 110);
|
||
const window = flat.slice(from, i + 110);
|
||
const at = i - from; // mention's offset inside the window
|
||
const req = nearest(required, window, at, true);
|
||
const opt = nearest(optional, window, at, false);
|
||
if (req === Infinity) continue; // nothing demanding it
|
||
if (req < opt) return true; // the closer marker wins
|
||
}
|
||
return false;
|
||
}
|
||
|
||
export async function fetchDescription(id: string): Promise<string | null> {
|
||
for (let attempt = 0; attempt < 3; attempt++) {
|
||
try {
|
||
await Bun.sleep(350 * (attempt + 1));
|
||
const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
|
||
const html = await res.text();
|
||
const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
|
||
if (m) return m[1].replace(/<[^>]+>/g, " ");
|
||
if (!res.ok || attempt < 2) continue;
|
||
return html.replace(/<[^>]+>/g, " ");
|
||
} catch {
|
||
// transient — retry
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/** English-language AND not demanding German above B1. */
|
||
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> {
|
||
const text = await fetchDescription(id);
|
||
if (text === null) return { ok: true }; // keep when we genuinely can't read it
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||
return { ok: true };
|
||
}
|
||
|
||
export async function isEnglishContent(id: string): Promise<boolean> {
|
||
// Retry when the description block is missing — LinkedIn serves a stub page
|
||
// under rate limiting, and the English UI chrome would otherwise pass as a
|
||
// false "English" result. Back off and try again before falling back.
|
||
for (let attempt = 0; attempt < 3; attempt++) {
|
||
try {
|
||
await Bun.sleep(350 * (attempt + 1));
|
||
const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
|
||
const html = await res.text();
|
||
const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
|
||
if (!m) {
|
||
if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
|
||
// Last resort: scan the whole stripped page (still catches DE/FR body text)
|
||
const text = html.replace(/<[^>]+>/g, " ");
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
return de < 6 && fr < 6;
|
||
}
|
||
const text = m[1].replace(/<[^>]+>/g, " ");
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
return de < 6 && fr < 6;
|
||
} catch {
|
||
// transient fetch error — retry
|
||
}
|
||
}
|
||
return true; // keep when we genuinely can't fetch the description
|
||
}
|
||
|
||
export async function isEnglishUrl(url: string): Promise<boolean> {
|
||
return (await isViableUrl(url)).ok;
|
||
}
|
||
|
||
/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
|
||
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> {
|
||
try {
|
||
await Bun.sleep(350);
|
||
const res = await fetch(url, { headers: HEADERS });
|
||
if (!res.ok) return { ok: true };
|
||
const text = (await res.text()).replace(/<[^>]+>/g, " ");
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||
return { ok: true };
|
||
} catch {
|
||
return { ok: true };
|
||
}
|
||
}
|
||
|
||
const PAGE_SIZE = 25;
|
||
const MAX_PAGES = 6;
|
||
|
||
export function buildUrl(config: SearchConfig): string {
|
||
if (config.url) return config.url;
|
||
const params = new URLSearchParams();
|
||
params.set("keywords", config.keywords ?? "");
|
||
if (config.location) params.set("location", config.location);
|
||
if (config.geoId) params.set("geoId", config.geoId);
|
||
if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
|
||
if (config.workType?.length)
|
||
params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
|
||
if (config.jobType?.length)
|
||
params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
|
||
if (config.seniority?.length)
|
||
params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
|
||
return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
|
||
}
|
||
|
||
function clean(raw: string): string {
|
||
return raw
|
||
.replace(/<[^>]+>/g, "")
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/–/g, "–")
|
||
.replace(/—/g, "—")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
function parseJobs(html: string, label: string): Job[] {
|
||
const jobs: Job[] = [];
|
||
|
||
// Locate every card boundary by job ID position
|
||
const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
|
||
const positions: { id: string; start: number }[] = [];
|
||
let m;
|
||
while ((m = idRe.exec(html)) !== null) {
|
||
positions.push({ id: m[1], start: m.index });
|
||
}
|
||
|
||
for (let i = 0; i < positions.length; i++) {
|
||
const { id, start } = positions[i];
|
||
// Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
|
||
const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));
|
||
|
||
const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
|
||
if (!titleM) continue;
|
||
|
||
const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
|
||
const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);
|
||
|
||
const locM = card.match(/job-search-card__location[^>]*>\s*([^<]+)/);
|
||
const jobLocation = locM ? locM[1].trim() : "";
|
||
|
||
// Drop jobs not physically in Switzerland — LinkedIn ignores geoId without auth
|
||
if (jobLocation && !jobLocation.includes("Switzerland")) continue;
|
||
|
||
const title = clean(titleM[1]);
|
||
const company = companyM ? clean(companyM[1]) : "";
|
||
const postedAt = timeM ? timeM[1] : "";
|
||
|
||
// Prefer the location printed on the card over the search label — recruiters
|
||
// and multi-site companies often post under a different area than the job sits in.
|
||
jobs.push({ id, title, company, location: jobLocation || label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" });
|
||
}
|
||
|
||
return jobs;
|
||
}
|
||
|
||
export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Promise<Job[]> {
|
||
const baseUrl = buildUrl(config);
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
for (let page = 0; page < MAX_PAGES; page++) {
|
||
if (page > 0) await Bun.sleep(400);
|
||
const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
|
||
process.stdout.write(` page ${page + 1} … `);
|
||
|
||
// 429s became common once index.ts started running several searches
|
||
// concurrently — a transient rate-limit, not a real end-of-results
|
||
// signal, so retry with backoff instead of silently truncating the
|
||
// search (losing every page after this one).
|
||
let res = await fetch(url, { headers: HEADERS });
|
||
for (let attempt = 0; res.status === 429 && attempt < 3; attempt++) {
|
||
await Bun.sleep(1000 * 2 ** attempt); // 1s, 2s, 4s
|
||
res = await fetch(url, { headers: HEADERS });
|
||
}
|
||
if (!res.ok) {
|
||
console.error(`failed (${res.status})`);
|
||
break;
|
||
}
|
||
|
||
const html = await res.text();
|
||
const jobs = parseJobs(html, config.label)
|
||
.filter((j) => !NON_ENGLISH_RE.test(j.title))
|
||
.filter((j) => !JUNIOR_TITLE_RE.test(j.title))
|
||
.filter((j) => !JAVA_TITLE_RE.test(j.title))
|
||
.filter((j) => !BIO_DOMAIN_RE.test(j.title))
|
||
.filter((j) => !GERMAN_TITLE_RE.test(j.title))
|
||
.filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title));
|
||
|
||
let newOnPage = 0;
|
||
for (const job of jobs) {
|
||
if (!seen.has(job.id)) {
|
||
seen.add(job.id);
|
||
allJobs.push(job);
|
||
newOnPage++;
|
||
}
|
||
}
|
||
|
||
console.log(`${jobs.length} jobs, ${newOnPage} new`);
|
||
if (newOnPage === 0) break;
|
||
}
|
||
|
||
return allJobs;
|
||
}
|