538 lines
26 KiB
TypeScript
538 lines
26 KiB
TypeScript
import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";
|
||
|
||
export interface Job {
|
||
id: string;
|
||
title: string;
|
||
company: string;
|
||
location: string;
|
||
postedAt: string;
|
||
link: string;
|
||
search: string;
|
||
source: string;
|
||
/**
|
||
* Set when the source already language-checked the real job description.
|
||
* `isEnglishUrl` counts DE/FR words across the whole page, so boards with
|
||
* German site chrome (itjobs.ch, stellen-basel.ch) fail it even when the
|
||
* posting itself is English — skip the re-check for those.
|
||
*/
|
||
contentChecked?: boolean;
|
||
}
|
||
|
||
const WORK_TYPE_MAP: Record<WorkType, string> = {
|
||
onsite: "1",
|
||
remote: "2",
|
||
hybrid: "3",
|
||
};
|
||
|
||
const JOB_TYPE_MAP: Record<JobType, string> = {
|
||
"full-time": "F",
|
||
"part-time": "P",
|
||
contract: "C",
|
||
temporary: "T",
|
||
internship: "I",
|
||
};
|
||
|
||
const TIME_MAP: Record<TimePosted, string> = {
|
||
day: "r86400",
|
||
week: "r604800",
|
||
month: "r2592000",
|
||
};
|
||
|
||
const SENIORITY_MAP: Record<Seniority, string> = {
|
||
internship: "1",
|
||
entry: "2",
|
||
associate: "3",
|
||
"mid-senior": "4",
|
||
director: "5",
|
||
executive: "6",
|
||
};
|
||
|
||
// Detect non-English (German / French / Swiss-German) job titles.
|
||
//
|
||
// German signals:
|
||
// - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
|
||
// but NOT (m/f/d) / (f/m/d) which are used in English international posts
|
||
// - Swiss neutral suffix: standalone (a) before % or end of string
|
||
// - German umlauts / ß
|
||
// - /in word-ending (Pflegehelfer/in, Köchin/Koch)
|
||
// - German-only vocabulary
|
||
// - Swiss German: EFZ (apprenticeship cert), Schweiz
|
||
//
|
||
// French signals:
|
||
// - gender markers: (H/F), (F/H), (h/f)
|
||
// - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
|
||
// - French-only vocabulary
|
||
export const NON_ENGLISH_RE = new RegExp(
|
||
[
|
||
// German gender markers (w = weiblich; must not match m/f/d)
|
||
String.raw`\(m\/w[^)]*\)`,
|
||
String.raw`\(w\/m[^)]*\)`,
|
||
String.raw`\(d\/w[^)]*\)`,
|
||
String.raw`\(mwd\)`,
|
||
// m/w without parens (e.g. "Developer m/w")
|
||
String.raw`\bm\/w\b`,
|
||
// Swiss neutral (a) suffix: " (a)" before %, digits, or end
|
||
String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
|
||
// German/Swiss umlauts and ß
|
||
String.raw`[äöüÄÖÜß]`,
|
||
// Gender-neutral suffix, in all four notations Swiss postings use
|
||
// (Mitarbeiter/in, Entwickler:in, Engineer*in, Mitarbeiter_in) — excludes
|
||
// m/f, n/a, etc. Also covers the adjectival-noun form, which appends "r"
|
||
// instead of "in" (Verantwortliche:r, Angestellte:r, Studierende:r) since
|
||
// the base word already ends in "-e". ZHAW's "Senior Full Stack
|
||
// Entwickler:in 80-100 %" used ":in" and slipped past both this rule
|
||
// (slash-only, at the time) and the word list ("Entwickler" wasn't on it
|
||
// either); the audit below found the *in/_in/:r variants were equally
|
||
// unhandled (OT-System Engineer*in, Applikationsverantwortliche:r).
|
||
String.raw`[a-zäöü][\/:*_](in|r)\b`,
|
||
// German-only words (case-insensitive handled by flag)
|
||
String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale|Ingenieur|Spezialist|Perspektive|Stellvertretung|Leitung|Beratung|Entwicklung|Betreuung|Verwaltung|unbefristet|befristet)\b`,
|
||
// German occupational suffixes — matched with NO leading \b, deliberately,
|
||
// because German compounds glue words together with no separator
|
||
// (Aussendienstmitarbeiter, Vertriebsingenieur, Applikationsverantwortliche)
|
||
// and a leading \b would only ever match the bare word. Found by auditing
|
||
// jobs.csv for titles containing these strings that the rest of the regex
|
||
// still let through: "Software Berater Fokus Lohn/HR in Bern",
|
||
// "Leiter Business Applikationen, CRM & Analytics", "Leiterin Trading
|
||
// Data, Integration & Reporting", "ICT System Engineer / Informatiker
|
||
// Systemtechnik". All are German-only strings with no English overlap.
|
||
String.raw`(mitarbeiter|ingenieur|berater|leiterin|leiter|informatiker|verantwortliche)\b`,
|
||
// German prepositions/articles/conjunctions in title context (must appear
|
||
// as whole words) — none of these exist as English words, so they're safe
|
||
// unambiguous markers. Added after "Software Engineer auf dem Weg zum
|
||
// Data / AI Engineer" (ti&m, via job-room.ch) sailed through: none of the
|
||
// German-word list above matched, and the job-room description was too
|
||
// short (~1 hit) to clear the separate ≥6 DE/FR word-count threshold
|
||
// either. "und"/"oder" added after the same audit found "System Engineer
|
||
// Microsoft und OnPrem Infrastruktur" slipping through — even the most
|
||
// basic German conjunctions weren't covered.
|
||
String.raw`\bmit\b|\bauf\b|\bzum\b|\bzur\b|\bvom\b|\bfür\b|\bbeim\b|\bund\b|\boder\b`,
|
||
// French gender markers
|
||
String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
|
||
// French-only accented characters
|
||
String.raw`[éèêëàâçîïôùûÿœæ]`,
|
||
// French-only words
|
||
String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
|
||
].join("|"),
|
||
"i"
|
||
);
|
||
|
||
const HEADERS = {
|
||
"User-Agent":
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
Accept:
|
||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||
};
|
||
|
||
export const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:ship)?|trainee|apprentice|werkstudent)\b/i;
|
||
|
||
export const JAVA_TITLE_RE = /\bjava\b/i;
|
||
|
||
// Roles whose *domain* is biology / chemistry / the lab bench. Rafael has no
|
||
// life-science background, so these need expertise he can't offer.
|
||
//
|
||
// This filters on the ROLE, never the employer: software, platform, data and
|
||
// infrastructure jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope —
|
||
// Basel pharma is a core target market. Only the science itself is excluded.
|
||
export const BIO_DOMAIN_RE = new RegExp(
|
||
[
|
||
// bench / domain science job families
|
||
String.raw`\b(bio)?statistician\b`,
|
||
String.raw`\bbioinformatic\w*`,
|
||
String.raw`\bcomputational\s+(?:biolog\w*|chemist\w*)`,
|
||
String.raw`\b(?:cheminformatic\w*|chemometric\w*)`,
|
||
String.raw`\b(?:micro)?biolog(?:y|ist)\b`,
|
||
String.raw`\b(?:bio)?chemist(?:ry|s)?\b`,
|
||
String.raw`\bpharmacolog\w*|\btoxicolog\w*|\bimmunolog\w*|\bvirolog\w*|\bepidemiolog\w*`,
|
||
String.raw`\bgenomic\w*|\bproteomic\w*|\bmetabolomic\w*|\bmolecular\s+(?:biolog\w*|patholog\w*)`,
|
||
// manufacturing / QC / regulated-lab functions
|
||
String.raw`\bGMP\b|\bGxP\b|\bMSAT\b|\bQC\s+analyst\b`,
|
||
String.raw`\b(?:bio)?process\s+(?:development|engineer)|\bupstream\b|\bdownstream\s+process`,
|
||
String.raw`\bassay\b|\bbioassay\b|\bwet.?lab\b|\blaboratory\s+(?:technician|assistant|manager)`,
|
||
// clinical / trial work
|
||
String.raw`\bclinical\s+(?:trial|research|data\s+manager|operations)`,
|
||
String.raw`\bbiomarker\b|\bpharmacovigilance\b|\bregulatory\s+affairs\b`,
|
||
// life-science domains that make any title (Scientist, Engineer, Analyst) a domain role
|
||
String.raw`\bbiologic\w*|\bpatholog\w*|\bhistolog\w*`,
|
||
String.raw`\bdrug\s+(?:discovery|design|development)\b`,
|
||
String.raw`\breal.world\s+(?:data|evidence)\b`,
|
||
].join("|"),
|
||
"i"
|
||
);
|
||
|
||
export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devsecops|platform|data|backend|frontend|front.end|fullstack|full.stack|react|web\s*developer|gis|geospatial|visuali[sz]ation|webgl|graphics|creative\s*technologist|cartograph\w*|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|agentic|llm|machine\s*learning|cloud|security|analytics|sre|reliability|infrastructure|mlops|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|scientific\s*(?:software|comput\w*|programmer)|research\s*(?:software|engineer)|hpc|forward\s*deployed|fde|palantir|field\s*engineer|customer\s*engineer|sales\s*engineer|pre.?sales|integration\s*engineer|api\s*engineer|founding\s*engineer|product\s*engineer|automation\s*engineer|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i;
|
||
|
||
const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g;
|
||
const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g;
|
||
|
||
// A bare umlaut/ß is a near-certain German signal, and unlike the ≥6 word-count
|
||
// threshold above it, it can catch even a short or truncated snippet — one
|
||
// German sentence almost always has one ("für", "können", "möchten"...). Added
|
||
// 2026-09-05 after an Expleo (Bern) posting slipped through: a concurrent fetch
|
||
// under load returned a version of the description with too few DE_CONTENT_RE
|
||
// word matches to trip ≥6, even though the full text (72 matches) was entirely
|
||
// German. "Zürich"/"Zürcher" is excluded — the one umlaut word that legitimately
|
||
// shows up constantly in genuinely English-language Swiss postings, just as a
|
||
// place name.
|
||
const UMLAUT_RE = /[äöüÄÖÜß]/;
|
||
export function hasNonEnglishUmlaut(text: string): boolean {
|
||
return UMLAUT_RE.test(text.replace(/zürich|zürcher/gi, ""));
|
||
}
|
||
|
||
// --- German-requirement detection -----------------------------------------
|
||
//
|
||
// Distinct from isEnglishContent: that drops postings *written* in German.
|
||
// This drops postings written in English that *require* German above B1 —
|
||
// e.g. Helvetia's "Good knowledge of German and English", which reads as a
|
||
// clean English posting and sails through every other filter.
|
||
//
|
||
// Rafael is German B1 (TELC). B1 or "basic"/"a plus" is fine; B2+ is not.
|
||
|
||
/** Any mention of the language, in either language. */
|
||
const GERMAN_MENTION_RE = /\b(german|deutsch\w*|allemand)\b/gi;
|
||
|
||
/** Within a mention's window: signals the requirement is optional. */
|
||
const GERMAN_OPTIONAL_RE =
|
||
/\b(?:a\s+plus|an?\s+advantage|advantageous|beneficial|nice[\s-]to[\s-]have|desirable|preferred|welcome|bonus|not\s+required|no\s+german|optional|willing(?:ness)?\s+to\s+learn|basic|beginner|elementary|grundkenntnisse|von\s+vorteil|wünschenswert|A1|A2|B1)\b/i;
|
||
|
||
/** Within a mention's window: signals it is genuinely required at B2+. */
|
||
const GERMAN_REQUIRED_RE =
|
||
/\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluen\w*|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend\w*|fließend\w*|muttersprach\w*|sehr\s+gut\w*|gut\w*\s+Deutschkenntnisse|Deutschkenntnisse|voraussetzung|zwingend|erforderlich)\b/i;
|
||
|
||
/**
|
||
* "German is <verdict>" / "proficiency in German is considered <verdict>" —
|
||
* the copula directly after the mention names the mention's own predicate,
|
||
* which should decide the case outright rather than compete on raw distance
|
||
* with unrelated markers elsewhere in the window. Swisscom's stock line
|
||
* "Fluency in English is required; proficiency in German is considered an
|
||
* advantage" put a required-marker word ("proficiency") right next to
|
||
* "German" and an optional-marker word ("advantage") further away — nearest()
|
||
* picked "proficiency" and wrongly called it required. The predicate right
|
||
* after the mention ("is considered an advantage") is what actually decides.
|
||
* Only fires when a copula immediately follows; "Work proficiency in German,
|
||
* English and French" (no copula after "German") correctly falls through to
|
||
* the general nearest-marker logic below.
|
||
*/
|
||
const GERMAN_LOCAL_PREDICATE_RE = /^\s*,?\s*(?:is|are)\s+(?:considered\s+|deemed\s+)?/i;
|
||
|
||
/**
|
||
* Spec-sheet style language line ("Languages: German & English"), common on
|
||
* contractor postings. There is no adjective for the nearest-marker logic to
|
||
* bind to, so the label itself counts as a requirement — but it goes through
|
||
* the same nearest-marker comparison, so "Language: English and German (bonus)"
|
||
* still loses to the "bonus" sitting closer to the mention.
|
||
*/
|
||
const GERMAN_SPEC_LABEL_RE = /\b(?:languages?|sprachen|sprache)\s*[::]/i;
|
||
|
||
/** Title-level marker — no description fetch needed. */
|
||
export const GERMAN_TITLE_RE =
|
||
/\bgerman[\s-]speaking\b|\bdeutschsprachig\w*|\(\s*de\s*\)\s*speaking|\bgerman\s+(?:required|mandatory)\b/i;
|
||
|
||
/**
|
||
* True when the text demands German above B1.
|
||
*
|
||
* Presence of a marker in the window isn't enough — an unrelated "a plus" from
|
||
* a neighbouring bullet would cancel a real requirement (Helvetia's posting
|
||
* puts "Infrastructure skills are a plus" ~95 chars before "Good knowledge of
|
||
* German"). So for each mention we take whichever marker sits *closest*.
|
||
*/
|
||
export function requiresGerman(text: string): boolean {
|
||
const flat = text.replace(/\s+/g, " ");
|
||
const optional = new RegExp(GERMAN_OPTIONAL_RE.source, "gi");
|
||
const required = new RegExp(
|
||
`${GERMAN_REQUIRED_RE.source}|${GERMAN_SPEC_LABEL_RE.source}`,
|
||
"gi"
|
||
);
|
||
|
||
// A marker binds to the German mention only if no OTHER language name sits
|
||
// between them: in "Fluent in English; German ... advantageous", "Fluent"
|
||
// qualifies English, not German.
|
||
const OTHER_LANG = /\b(english|french|italian|spanish|portuguese|anglais|français|englisch|franz|italienisch)\b/i;
|
||
|
||
// …except when that other language is *coordinated* with German, where the
|
||
// marker governs both: "Excellent communication skills in both English and
|
||
// German" (Karakun) and "Fluency in English and German" (J. Safra Sarasin)
|
||
// are hard requirements that the bare rule above waved through — the entire
|
||
// reason 24 basel-jobs.ch postings mentioning German were caught 0 times.
|
||
// Deliberately excludes "and/or" and a bare "or": "Very good English and/or
|
||
// German" (BBF) is satisfied by English alone, so it must stay optional.
|
||
const CONJOINED = /\b(?:and|und|sowie|as\s+well\s+as|&)\s*$/i;
|
||
const ANDOR = /\b(?:and\s*\/\s*or|or)\s*$/i;
|
||
|
||
const nearest = (re: RegExp, window: string, at: number, bind: boolean): number => {
|
||
let best = Infinity;
|
||
for (const m of window.matchAll(re)) {
|
||
const i = m.index ?? 0;
|
||
const between = i < at ? window.slice(i + m[0].length, at) : window.slice(at, i);
|
||
// "…English and German" keeps the marker; "…English; German" or
|
||
// "…English and/or German" hands it to the other language.
|
||
const conjoined = CONJOINED.test(between) && !ANDOR.test(between);
|
||
if (bind && OTHER_LANG.test(between) && !conjoined) continue;
|
||
const d = Math.abs(i - at);
|
||
if (d < best) best = d;
|
||
}
|
||
return best;
|
||
};
|
||
|
||
for (const m of flat.matchAll(GERMAN_MENTION_RE)) {
|
||
const i = m.index ?? 0;
|
||
|
||
// The mention's own predicate ("German is <verdict>") overrides distance.
|
||
const after = flat.slice(i + m[0].length, i + m[0].length + 60);
|
||
const predicateM = after.match(GERMAN_LOCAL_PREDICATE_RE);
|
||
if (predicateM) {
|
||
const rest = after.slice(predicateM[0].length);
|
||
if (new RegExp(`^${GERMAN_OPTIONAL_RE.source}`, "i").test(rest)) continue;
|
||
if (new RegExp(`^${GERMAN_REQUIRED_RE.source}`, "i").test(rest)) return true;
|
||
}
|
||
|
||
const from = Math.max(0, i - 110);
|
||
const window = flat.slice(from, i + 110);
|
||
const at = i - from; // mention's offset inside the window
|
||
const req = nearest(required, window, at, true);
|
||
const opt = nearest(optional, window, at, false);
|
||
if (req === Infinity) continue; // nothing demanding it
|
||
if (req < opt) return true; // the closer marker wins
|
||
}
|
||
return false;
|
||
}
|
||
|
||
export async function fetchDescription(id: string): Promise<string | null> {
|
||
for (let attempt = 0; attempt < 3; attempt++) {
|
||
try {
|
||
// 429s need the same backoff as the listing-page retry (1s/2s/4s) — the
|
||
// old 350/700/1050ms wait was too short under concurrent load, so this
|
||
// exhausted its retries while still rate-limited and fell through to the
|
||
// "can't read it, so keep" default below, letting German postings (e.g.
|
||
// CLEEVEN, job 4465075383, all-German with zero English content) slip
|
||
// past isViableContent entirely on 2026-09-10.
|
||
await Bun.sleep(1000 * 2 ** attempt);
|
||
const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
|
||
const html = await res.text();
|
||
const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
|
||
if (m) return m[1].replace(/<[^>]+>/g, " ");
|
||
if (!res.ok || attempt < 2) continue;
|
||
return html.replace(/<[^>]+>/g, " ");
|
||
} catch {
|
||
// transient — retry
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* English-language AND not demanding German above B1.
|
||
*
|
||
* `retry: true` means the description couldn't be read at all (rate-limited,
|
||
* stub page) — index.ts leaves the job out of the CSV entirely rather than
|
||
* defaulting to "keep", so it's picked up fresh and re-checked next run
|
||
* instead of ever being judged sight-unseen. The old default-to-keep here is
|
||
* what let a fully-German HRM Systems/Consult & Pepper posting ("In Deutsch
|
||
* bist du verhandlungssicher") through on 2026-09-10 — same failure mode as
|
||
* the CLEEVEN case above, just on the "keep" side of isViableContent instead
|
||
* of isEnglishContent.
|
||
*/
|
||
export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
|
||
const text = await fetchDescription(id);
|
||
if (text === null) return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||
return { ok: true };
|
||
}
|
||
|
||
export async function isEnglishContent(id: string): Promise<boolean> {
|
||
// Retry when the description block is missing — LinkedIn serves a stub page
|
||
// under rate limiting, and the English UI chrome would otherwise pass as a
|
||
// false "English" result. Back off and try again before falling back.
|
||
for (let attempt = 0; attempt < 3; attempt++) {
|
||
try {
|
||
// Same 1s/2s/4s backoff as fetchDescription — see its comment.
|
||
await Bun.sleep(1000 * 2 ** attempt);
|
||
const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS });
|
||
const html = await res.text();
|
||
const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);
|
||
if (!m) {
|
||
if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry
|
||
// Last resort: scan the whole stripped page (still catches DE/FR body text)
|
||
const text = html.replace(/<[^>]+>/g, " ");
|
||
if (hasNonEnglishUmlaut(text)) return false;
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
return de < 6 && fr < 6;
|
||
}
|
||
const text = m[1].replace(/<[^>]+>/g, " ");
|
||
if (hasNonEnglishUmlaut(text)) return false;
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
return de < 6 && fr < 6;
|
||
} catch {
|
||
// transient fetch error — retry
|
||
}
|
||
}
|
||
return true; // keep when we genuinely can't fetch the description
|
||
}
|
||
|
||
export async function isEnglishUrl(url: string): Promise<boolean> {
|
||
return (await isViableUrl(url)).ok;
|
||
}
|
||
|
||
/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */
|
||
export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string; retry?: boolean }> {
|
||
try {
|
||
await Bun.sleep(350);
|
||
const res = await fetch(url, { headers: HEADERS });
|
||
// Same reasoning as isViableContent: an unreadable page shouldn't default
|
||
// to "keep" — defer it and let next run's fresh fetch actually judge it.
|
||
if (!res.ok) return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||
const text = (await res.text()).replace(/<[^>]+>/g, " ");
|
||
if (hasNonEnglishUmlaut(text)) return { ok: false, reason: "non-English (umlaut)" };
|
||
const de = (text.match(DE_CONTENT_RE) ?? []).length;
|
||
const fr = (text.match(FR_CONTENT_RE) ?? []).length;
|
||
if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" };
|
||
if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" };
|
||
return { ok: true };
|
||
} catch {
|
||
return { ok: false, reason: "unreadable — retrying next run", retry: true };
|
||
}
|
||
}
|
||
|
||
const PAGE_SIZE = 25;
|
||
const MAX_PAGES = 6;
|
||
|
||
export function buildUrl(config: SearchConfig): string {
|
||
if (config.url) return config.url;
|
||
const params = new URLSearchParams();
|
||
params.set("keywords", config.keywords ?? "");
|
||
if (config.location) params.set("location", config.location);
|
||
if (config.geoId) params.set("geoId", config.geoId);
|
||
if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
|
||
if (config.workType?.length)
|
||
params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
|
||
if (config.jobType?.length)
|
||
params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
|
||
if (config.seniority?.length)
|
||
params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
|
||
return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
|
||
}
|
||
|
||
function clean(raw: string): string {
|
||
return raw
|
||
.replace(/<[^>]+>/g, "")
|
||
.replace(/&/g, "&")
|
||
.replace(/</g, "<")
|
||
.replace(/>/g, ">")
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, "'")
|
||
.replace(/–/g, "–")
|
||
.replace(/—/g, "—")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
function parseJobs(html: string, label: string): Job[] {
|
||
const jobs: Job[] = [];
|
||
|
||
// Locate every card boundary by job ID position
|
||
const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
|
||
const positions: { id: string; start: number }[] = [];
|
||
let m;
|
||
while ((m = idRe.exec(html)) !== null) {
|
||
positions.push({ id: m[1], start: m.index });
|
||
}
|
||
|
||
for (let i = 0; i < positions.length; i++) {
|
||
const { id, start } = positions[i];
|
||
// Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
|
||
const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));
|
||
|
||
const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
|
||
if (!titleM) continue;
|
||
|
||
const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
|
||
const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);
|
||
|
||
const locM = card.match(/job-search-card__location[^>]*>\s*([^<]+)/);
|
||
const jobLocation = locM ? locM[1].trim() : "";
|
||
|
||
// Drop jobs not physically in Switzerland — LinkedIn ignores geoId without auth
|
||
if (jobLocation && !jobLocation.includes("Switzerland")) continue;
|
||
|
||
const title = clean(titleM[1]);
|
||
const company = companyM ? clean(companyM[1]) : "";
|
||
const postedAt = timeM ? timeM[1] : "";
|
||
|
||
// Prefer the location printed on the card over the search label — recruiters
|
||
// and multi-site companies often post under a different area than the job sits in.
|
||
jobs.push({ id, title, company, location: jobLocation || label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" });
|
||
}
|
||
|
||
return jobs;
|
||
}
|
||
|
||
export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Promise<Job[]> {
|
||
const baseUrl = buildUrl(config);
|
||
const allJobs: Job[] = [];
|
||
const seen = new Set<string>();
|
||
const seenCards = new Set<string>(); // every card id served, pre-filter — drives the end condition
|
||
|
||
for (let page = 0; page < MAX_PAGES; page++) {
|
||
if (page > 0) await Bun.sleep(400);
|
||
const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
|
||
process.stdout.write(` page ${page + 1} … `);
|
||
|
||
// 429s became common once index.ts started running several searches
|
||
// concurrently — a transient rate-limit, not a real end-of-results
|
||
// signal, so retry with backoff instead of silently truncating the
|
||
// search (losing every page after this one).
|
||
let res = await fetch(url, { headers: HEADERS });
|
||
for (let attempt = 0; res.status === 429 && attempt < 3; attempt++) {
|
||
await Bun.sleep(1000 * 2 ** attempt); // 1s, 2s, 4s
|
||
res = await fetch(url, { headers: HEADERS });
|
||
}
|
||
if (!res.ok) {
|
||
console.error(`failed (${res.status})`);
|
||
break;
|
||
}
|
||
|
||
const html = await res.text();
|
||
|
||
// End-of-results is decided on the raw card ids LinkedIn served, never on
|
||
// how many survived the title filters. A page whose cards all happen to be
|
||
// German/Java/junior/non-software read as "no new jobs" and truncated the
|
||
// whole search, discarding every later page — pages that do carry keepers.
|
||
let newCards = 0;
|
||
for (const m of html.matchAll(/data-entity-urn="urn:li:jobPosting:(\d+)"/g)) {
|
||
if (!seenCards.has(m[1])) {
|
||
seenCards.add(m[1]);
|
||
newCards++;
|
||
}
|
||
}
|
||
|
||
const jobs = parseJobs(html, config.label)
|
||
.filter((j) => !NON_ENGLISH_RE.test(j.title))
|
||
.filter((j) => !JUNIOR_TITLE_RE.test(j.title))
|
||
.filter((j) => !JAVA_TITLE_RE.test(j.title))
|
||
.filter((j) => !BIO_DOMAIN_RE.test(j.title))
|
||
.filter((j) => !GERMAN_TITLE_RE.test(j.title))
|
||
.filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title));
|
||
|
||
let newOnPage = 0;
|
||
for (const job of jobs) {
|
||
if (!seen.has(job.id)) {
|
||
seen.add(job.id);
|
||
allJobs.push(job);
|
||
newOnPage++;
|
||
}
|
||
}
|
||
|
||
console.log(`${jobs.length} jobs, ${newOnPage} new (${newCards} new cards)`);
|
||
if (newCards === 0) break;
|
||
}
|
||
|
||
return allJobs;
|
||
}
|