diff --git a/CLAUDE.md b/CLAUDE.md index e0ce77f..0b817a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ Temp scripts: `bun run scripts/-temp.ts` — never edit `config.ts` for on ``` config.ts # search keywords and areas fetch.ts # scraper, HTML parser, all filters -fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch) +fetch-other.ts # extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel intl orgs) index.ts # orchestrator: runs searches, writes CSV recheck.ts # raw-mode fetch, lists filtered-out jobs scripts/ # one-off search scripts; import from ../config.ts and ../fetch.ts @@ -30,8 +30,17 @@ plans/ # LinkedIn post drafts ## Architecture - **`config.ts`** — `KEYWORDS` string (joined with ` OR `) and `searches: SearchConfig[]`. Edit here to add/remove search areas or keywords. + - Also exports `PLATFORM_KEYWORDS` (architect/SRE/infra/security/MLOps), `RESEARCH_KEYWORDS` (research & scientific software, HPC, data platform) and `FIELD_KEYWORDS` (forward-deployed, customer/sales/integration, founding/product). Kept as separate OR-strings rather than folded into `KEYWORDS` — LinkedIn ranks a focused query far better than one 50-term disjunction. The `sweep()` helper generates one search per area per family; Research skips Bern, Platform and Field add a Switzerland-wide remote pass. + - `RESEARCH_KEYWORDS` deliberately contains no bioinformatics/computational-biology terms — see `BIO_DOMAIN_RE`. - **`fetch.ts`** — HTTP scraping, HTML parsing, and all filters. Exports `fetchJobs(config, { raw? })` and `isEnglishContent(id)`. -- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch). Imported by `index.ts`. +- **`fetch-other.ts`** — scrapes extra sources (datacareer.ch, stellen-basel.ch, itjobs.ch, job-room.ch, Basel international orgs). Imported by `index.ts`. + - **datacareer.ch** currently returns **403** on every page — the source is dead and yields nothing. Needs new headers or a different entry point. + - **Basel international orgs** (`fetchBaselIntlOrgs`) sweeps three boards under one label, each only a handful of postings: + - **BIS** — the career site is JS-rendered with no usable API, so we read the UNjobs mirror at `unjobs.org/offices/bis_182320855689338880`. Titles arrive as `Role, City, Country` and get split back apart. BIS also cycles Senior Software / Data / Data & Analytics Engineer roles through this board. + - **Swiss TPH** — Umantis at `jobs.swisstph.ch/Jobs/All`. Site chrome is German even on English postings, so `isEnglishUrl` would reject everything; `contentChecked: true` and the title filters decide instead. + - **Basel Institute on Governance** — `baselgovernance.org/about/careers`. Each vacancy is a linked PDF, so there's no HTML description to language-check. The `rounded-3xl` card class is what separates vacancy tiles from the annual-report PDF link. + - These boards use UN-flavoured titles ("Information Management Officer", "ICT Specialist") that `SOFTWARE_TITLE_RE` alone drops, so `INTL_TECH_RE` widens the net **for these sources only**. Boards this small can afford the noise; missing a rare opening can't. + - **itjobs.ch** is swept Switzerland-wide (no `location` param) — the board is small, ~210 jobs over 7 pages, and locking it to Basel returned only ~9. It 404s one page past the end instead of serving an empty list. Each item embeds its full `description`, so the DE/FR check runs in-process via `isEnglishDescription` rather than costing a page fetch per job. Use `job_location.name` for location — it's the English form; the bare `location` field is German. - **`index.ts`** — orchestrator: loads existing IDs from `results/jobs.csv`, calls `fetchJobs`, calls `isEnglishContent` for each new job, appends survivors to CSV. - **`recheck.ts`** — calls `fetchJobs(..., { raw: true })` to bypass `SOFTWARE_TITLE_RE`, then lists every new job that *would* be filtered out and prints word-frequency suggestions. @@ -41,8 +50,14 @@ plans/ # LinkedIn post drafts 2. **`NON_ENGLISH_RE`** (title) — drops German gender markers, umlauts, `/in` suffix, French accents/markers. Runs inside `fetchJobs`. 3. **`JUNIOR_TITLE_RE`** (title) — drops junior/intern/trainee/werkstudent. Runs inside `fetchJobs`. 4. **`JAVA_TITLE_RE`** (title) — drops Java-titled roles. Runs inside `fetchJobs`. -5. **`SOFTWARE_TITLE_RE`** (title) — requires a relevant tech/role term. Skipped when `raw: true`. Runs inside `fetchJobs`. -6. **`isEnglishContent`** (description page) — fetches the job page, counts DE/FR indicator words, drops if ≥ 6 matches. Runs in `index.ts` only (not `recheck.ts`). Adds 350 ms delay per job. +5. **`BIO_DOMAIN_RE`** (title) — drops roles whose *domain* is biology/chemistry/the lab bench (bioinformatics, biostatistics, GMP/MSAT, genomics, pathology, drug discovery, bench Scientist titles). Filters on the **role, not the employer** — software/platform/data jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope. Runs inside `fetchJobs`. +6. **`SOFTWARE_TITLE_RE`** (title) — requires a relevant tech/role term. Skipped when `raw: true`. Runs inside `fetchJobs`. +6.5. **`GERMAN_TITLE_RE`** (title) — drops "(German-speaking)" / "deutschsprachig" titles without a description fetch. Runs inside `fetchJobs`. +7. **`isViableContent`** / **`isViableUrl`** (description page) — one fetch, two checks: (a) counts DE/FR indicator words, drops if ≥ 6; (b) `requiresGerman()` drops English postings that demand German above B1. Runs in `index.ts` only (not `recheck.ts`). Adds 350 ms delay per job. `isEnglishContent` / `isEnglishUrl` remain exported for `recheck.ts` and temp scripts. + +**`requiresGerman(text)`** is deliberately more than a keyword match — Rafael is German B1 (TELC), so B1/"basic"/"a plus" is fine and B2+ is not. For each mention of German it takes the *nearest* qualifying marker, and a marker is ignored when another language name sits between it and the mention. Both rules come from real false results: +- *"Infrastructure skills are **a plus** … Good knowledge of **German**"* (Helvetia) — presence-based matching let an unrelated "a plus" 95 chars away cancel a real requirement. +- *"Fluent in **English**; **German** and/or French advantageous"* (albedis) — "Fluent" is nearer to "German" than "advantageous" is, but qualifies English. Deduplication: job IDs are extracted from `results/jobs.csv` on startup; any ID already seen is skipped before content-checking. @@ -110,12 +125,16 @@ Read `cv/polo.md` for the full profile. Key signals: - **Core stack**: Python · TypeScript · Ruby on Rails · React · Node.js - **Domain strengths**: platform engineering · data pipelines · AI/LLM integrations · DevOps (Docker, Kubernetes, CI/CD, AWS) - **Background edges**: fintech/CBDC (Moedax), civic tech (Tactical Tech / GFOSS), independent consulting (ExtraPolo, 40+ projects) -- **Languages**: English (full) · German B1 TELC certified · Spanish (full) — flag roles requiring C1+ German +- **Languages**: English (full) · German B1 TELC certified · Spanish (full) — **drop** roles requiring German above B1; `requiresGerman()` does this automatically at fetch time +- **Hard no: biological domains** — drop bioinformatics, computational biology, biostatistics, lab/bench Scientist, GMP/MSAT, clinical/assay and drug-discovery-science roles; he has no life-science background. Software roles *at* pharma companies are still strong targets +- **Hard no: Microsoft stack** — drop .NET/C#, Azure-first, Windows/SCCM/Intune, SharePoint, Dynamics, Power Platform roles entirely; don't present them even as weak fits - **Location**: based in Basel, open to Basel / Bern / Zürich; EU citizen, no permit needed ## Output format -Present jobs as a markdown table: `#`, `Title`, `Company`, `Why it fits`, `Link`. +**Always** present jobs as a markdown table — never as a bullet list or prose: `#`, `Title`, `Company`, `Canton`, `Why it fits`, `Link`. + +`Canton` is mandatory on every table, including single-city searches and the list of dropped/rejected jobs (there `Why dropped` replaces `Why it fits`). Derive it from the actual location on the posting, not from the search label — recruiters and multi-site companies often post under a different area than the job sits in. Use the two-letter code: BS, BL, ZH, BE, ZG, LU, VD, GE, TI, AG, SG, TG. For remote roles write `Remote` (add scope if known, e.g. `Remote (EU)`). Links use just the job ID as anchor text: `[4192954199](https://www.linkedin.com/jobs/view/4192954199/)` diff --git a/config.ts b/config.ts index 4e0f61a..fb02fd3 100644 --- a/config.ts +++ b/config.ts @@ -46,6 +46,55 @@ export const KEYWORDS = [ "Solution Engineer", ].join(" OR "); +// --- Role families beyond the core engineer/developer titles --- +// Kept as separate OR-strings rather than folded into KEYWORDS: LinkedIn ranks +// a focused query far better than one 50-term disjunction, and each family +// needs a different set of areas (research is Basel-heavy, field/remote is not). + +/** Platform, reliability, cloud and security — the DevOps/Swisscom lane. */ +export const PLATFORM_KEYWORDS = [ + "Solutions Architect", + "Cloud Architect", + "Site Reliability Engineer", + "Infrastructure Engineer", + "Cloud Engineer", + "Security Engineer", + "DevSecOps", + "Application Security Engineer", + "Cloud Security Engineer", + "MLOps Engineer", + "AI Platform Engineer", + "ML Platform Engineer", + "AI Infrastructure", +].join(" OR "); + +/** Research software engineering — Basel pharma/university cluster. + * Deliberately no bioinformatics/computational-biology terms: those are + * domain-science roles, filtered out by BIO_DOMAIN_RE in fetch.ts. */ +export const RESEARCH_KEYWORDS = [ + "Research Software Engineer", + "Scientific Software Engineer", + "Scientific Programmer", + "Scientific Computing", + "Research Engineer", + "HPC Engineer", + "Data Platform Engineer", +].join(" OR "); + +/** Customer-facing and early-stage product roles — the consulting lane. */ +export const FIELD_KEYWORDS = [ + "Forward Deployed Engineer", + "Field Engineer", + "Customer Engineer", + "Sales Engineer", + "Pre-Sales Engineer", + "Integration Engineer", + "API Engineer", + "Analytics Engineer", + "Founding Engineer", + "Product Engineer", +].join(" OR "); + // --- Define your searches here --- export const searches: SearchConfig[] = [ { @@ -104,3 +153,48 @@ export const searches: SearchConfig[] = [ timePosted: "month", }, ]; + +// --- Role-family sweeps --------------------------------------------------- +// Same filters as the core searches, run across a narrower set of areas. +// Research is Basel/Zürich only (that's where the institutes are); platform +// and field roles also sweep Switzerland-wide remote. + +const CORE_AREAS: { label: string; geoId?: string; location: string }[] = [ + { label: "Basel Metropolitan Area", geoId: "90009885", location: "Basel Metropolitan Area" }, + { label: "Zürich Metropolitan Area", geoId: "107814425", location: "Zürich Metropolitan Area" }, + { label: "Bern", location: "Bern, Switzerland" }, +]; + +const RESEARCH_AREAS = CORE_AREAS.filter((a) => a.label !== "Bern"); + +function sweep( + family: string, + keywords: string, + areas: typeof CORE_AREAS, + opts: { remote?: boolean } = {} +): SearchConfig[] { + const out: SearchConfig[] = areas.map((a) => ({ + label: `${family} — ${a.label}`, + keywords, + geoId: a.geoId, + location: a.location, + workType: ["onsite", "hybrid"] as WorkType[], + timePosted: "month" as TimePosted, + })); + if (opts.remote) { + out.push({ + label: `${family} — Switzerland Remote`, + keywords, + location: "Switzerland", + workType: ["remote"], + timePosted: "month", + }); + } + return out; +} + +searches.push( + ...sweep("Platform", PLATFORM_KEYWORDS, CORE_AREAS, { remote: true }), + ...sweep("Research", RESEARCH_KEYWORDS, RESEARCH_AREAS), + ...sweep("Field", FIELD_KEYWORDS, CORE_AREAS, { remote: true }) +); diff --git a/fetch.ts b/fetch.ts index 101985e..694d5b9 100644 --- a/fetch.ts +++ b/fetch.ts @@ -9,6 +9,13 @@ export interface Job { link: string; search: string; source: string; + /** + * Set when the source already language-checked the real job description. + * `isEnglishUrl` counts DE/FR words across the whole page, so boards with + * German site chrome (itjobs.ch, stellen-basel.ch) fail it even when the + * posting itself is English — skip the re-check for those. + */ + contentChecked?: boolean; } const WORK_TYPE_MAP: Record = { @@ -62,6 +69,8 @@ export const NON_ENGLISH_RE = new RegExp( String.raw`\(w\/m[^)]*\)`, String.raw`\(d\/w[^)]*\)`, String.raw`\(mwd\)`, + // m/w without parens (e.g. "Developer m/w") + String.raw`\bm\/w\b`, // Swiss neutral (a) suffix: " (a)" before %, digits, or end String.raw`\s\(a\)(\s*[\d%]|\s*$)`, // German/Swiss umlauts and ß @@ -69,7 +78,9 @@ export const NON_ENGLISH_RE = new RegExp( // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc. String.raw`[a-zäöü]\/in\b`, // German-only words (case-insensitive handled by flag) - String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`, + String.raw`\b(Deine|Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale|Ingenieur|Spezialist|Perspektive|Stellvertretung|Leitung|Beratung|Entwicklung|Betreuung|Verwaltung)\b`, + // German prepositions/articles in title context (must appear as whole words) + String.raw`\bmit\b`, // French gender markers String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`, // French-only accented characters @@ -92,25 +103,185 @@ export const JUNIOR_TITLE_RE = /\b(junior|jr\.?|entry.?level|graduate|intern(?:s export const JAVA_TITLE_RE = /\bjava\b/i; -export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i; +// Roles whose *domain* is biology / chemistry / the lab bench. Rafael has no +// life-science background, so these need expertise he can't offer. +// +// This filters on the ROLE, never the employer: software, platform, data and +// infrastructure jobs at Roche, Novartis, Swiss TPH and sciCORE stay in scope — +// Basel pharma is a core target market. Only the science itself is excluded. +export const BIO_DOMAIN_RE = new RegExp( + [ + // bench / domain science job families + String.raw`\b(bio)?statistician\b`, + String.raw`\bbioinformatic\w*`, + String.raw`\bcomputational\s+(?:biolog\w*|chemist\w*)`, + String.raw`\b(?:cheminformatic\w*|chemometric\w*)`, + String.raw`\b(?:micro)?biolog(?:y|ist)\b`, + String.raw`\b(?:bio)?chemist(?:ry|s)?\b`, + String.raw`\bpharmacolog\w*|\btoxicolog\w*|\bimmunolog\w*|\bvirolog\w*|\bepidemiolog\w*`, + String.raw`\bgenomic\w*|\bproteomic\w*|\bmetabolomic\w*|\bmolecular\s+(?:biolog\w*|patholog\w*)`, + // manufacturing / QC / regulated-lab functions + String.raw`\bGMP\b|\bGxP\b|\bMSAT\b|\bQC\s+analyst\b`, + String.raw`\b(?:bio)?process\s+(?:development|engineer)|\bupstream\b|\bdownstream\s+process`, + String.raw`\bassay\b|\bbioassay\b|\bwet.?lab\b|\blaboratory\s+(?:technician|assistant|manager)`, + // clinical / trial work + String.raw`\bclinical\s+(?:trial|research|data\s+manager|operations)`, + String.raw`\bbiomarker\b|\bpharmacovigilance\b|\bregulatory\s+affairs\b`, + // life-science domains that make any title (Scientist, Engineer, Analyst) a domain role + String.raw`\bbiologic\w*|\bpatholog\w*|\bhistolog\w*`, + String.raw`\bdrug\s+(?:discovery|design|development)\b`, + String.raw`\breal.world\s+(?:data|evidence)\b`, + ].join("|"), + "i" +); + +export const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|devsecops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|technical|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|infrastructure|mlops|solutions?|consultant|applications?\s*engineer|system[\s-]*engineer|informatics|scientific\s*(?:software|comput\w*|programmer)|research\s*(?:software|engineer)|hpc|forward\s*deployed|field\s*engineer|customer\s*engineer|sales\s*engineer|pre.?sales|integration\s*engineer|api\s*engineer|founding\s*engineer|product\s*engineer|automation\s*engineer|director|product\s*manager|engineering\s*manager|it\s*manager|it\s*project|staff\s*engineer|principal\s*engineer|cto|vp\s*eng)\b/i; const DE_CONTENT_RE = /\b(Sie|Ihre?|Ihnen|Wir|wir|und\s|für|mit\s|von\s|bei\s|das\s|ist\s|sind\s|haben\s|werden\s|können|suchen|bieten|Kenntnisse|Erfahrung|Anforderungen|Aufgaben|Stellenbeschreibung|Stellenangebot|Bewerbung|Ausbildung|Berufserfahrung|Teamarbeit|Vollzeit|Teilzeit|unbefristet|Abschluss|gesucht|Deutsch|Englisch|Schweizer)\b/g; const FR_CONTENT_RE = /\b(vous|nous|votre|notre|pour\s|avec\s|profil|poste|emploi|entreprise|compétences|expérience|formation|équipe|rejoindre|recherchons|offrons|missions|candidature|CDI|CDD|temps\s+plein|temps\s+partiel)\b/g; +// --- German-requirement detection ----------------------------------------- +// +// Distinct from isEnglishContent: that drops postings *written* in German. +// This drops postings written in English that *require* German above B1 — +// e.g. Helvetia's "Good knowledge of German and English", which reads as a +// clean English posting and sails through every other filter. +// +// Rafael is German B1 (TELC). B1 or "basic"/"a plus" is fine; B2+ is not. + +/** Any mention of the language, in either language. */ +const GERMAN_MENTION_RE = /\b(german|deutsch\w*|allemand)\b/gi; + +/** Within a mention's window: signals the requirement is optional. */ +const GERMAN_OPTIONAL_RE = + /\b(?:a\s+plus|an?\s+advantage|advantageous|beneficial|nice[\s-]to[\s-]have|desirable|preferred|welcome|bonus|not\s+required|no\s+german|optional|willing(?:ness)?\s+to\s+learn|basic|beginner|elementary|grundkenntnisse|von\s+vorteil|wünschenswert|A1|A2|B1)\b/i; + +/** Within a mention's window: signals it is genuinely required at B2+. */ +const GERMAN_REQUIRED_RE = + /\b(?:required|require[sd]?|must|mandatory|essential|necessary|fluent|fluency|proficien\w*|excellent|strong|very\s+good|good\s+(?:knowledge|command|level)|native|mother\s*tongue|business\s+level|negotiat\w*|B2|C1|C2|verhandlungssicher\w*|fliessend|fließend|muttersprach\w*|sehr\s+gute?|gute?\s+Deutschkenntnisse|voraussetzung|zwingend)\b/i; + +/** Title-level marker — no description fetch needed. */ +export const GERMAN_TITLE_RE = + /\bgerman[\s-]speaking\b|\bdeutschsprachig\w*|\(\s*de\s*\)\s*speaking|\bgerman\s+(?:required|mandatory)\b/i; + +/** + * True when the text demands German above B1. + * + * Presence of a marker in the window isn't enough — an unrelated "a plus" from + * a neighbouring bullet would cancel a real requirement (Helvetia's posting + * puts "Infrastructure skills are a plus" ~95 chars before "Good knowledge of + * German"). So for each mention we take whichever marker sits *closest*. + */ +export function requiresGerman(text: string): boolean { + const flat = text.replace(/\s+/g, " "); + const optional = new RegExp(GERMAN_OPTIONAL_RE.source, "gi"); + const required = new RegExp(GERMAN_REQUIRED_RE.source, "gi"); + + // A marker binds to the German mention only if no OTHER language name sits + // between them: in "Fluent in English; German ... advantageous", "Fluent" + // qualifies English, not German. + const OTHER_LANG = /\b(english|french|italian|spanish|portuguese|anglais|français|englisch|franz|italienisch)\b/i; + + const nearest = (re: RegExp, window: string, at: number, bind: boolean): number => { + let best = Infinity; + for (const m of window.matchAll(re)) { + const i = m.index ?? 0; + const between = i < at ? window.slice(i + m[0].length, at) : window.slice(at, i); + if (bind && OTHER_LANG.test(between)) continue; // belongs to a different language + const d = Math.abs(i - at); + if (d < best) best = d; + } + return best; + }; + + for (const m of flat.matchAll(GERMAN_MENTION_RE)) { + const i = m.index ?? 0; + const from = Math.max(0, i - 110); + const window = flat.slice(from, i + 110); + const at = i - from; // mention's offset inside the window + const req = nearest(required, window, at, true); + const opt = nearest(optional, window, at, false); + if (req === Infinity) continue; // nothing demanding it + if (req < opt) return true; // the closer marker wins + } + return false; +} + +export async function fetchDescription(id: string): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + try { + await Bun.sleep(350 * (attempt + 1)); + const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); + const html = await res.text(); + const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/); + if (m) return m[1].replace(/<[^>]+>/g, " "); + if (!res.ok || attempt < 2) continue; + return html.replace(/<[^>]+>/g, " "); + } catch { + // transient — retry + } + } + return null; +} + +/** English-language AND not demanding German above B1. */ +export async function isViableContent(id: string): Promise<{ ok: boolean; reason?: string }> { + const text = await fetchDescription(id); + if (text === null) return { ok: true }; // keep when we genuinely can't read it + const de = (text.match(DE_CONTENT_RE) ?? []).length; + const fr = (text.match(FR_CONTENT_RE) ?? []).length; + if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" }; + if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" }; + return { ok: true }; +} + export async function isEnglishContent(id: string): Promise { + // Retry when the description block is missing — LinkedIn serves a stub page + // under rate limiting, and the English UI chrome would otherwise pass as a + // false "English" result. Back off and try again before falling back. + for (let attempt = 0; attempt < 3; attempt++) { + try { + await Bun.sleep(350 * (attempt + 1)); + const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); + const html = await res.text(); + const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/); + if (!m) { + if (!res.ok || attempt < 2) continue; // rate-limited / stub — retry + // Last resort: scan the whole stripped page (still catches DE/FR body text) + const text = html.replace(/<[^>]+>/g, " "); + const de = (text.match(DE_CONTENT_RE) ?? []).length; + const fr = (text.match(FR_CONTENT_RE) ?? []).length; + return de < 6 && fr < 6; + } + const text = m[1].replace(/<[^>]+>/g, " "); + const de = (text.match(DE_CONTENT_RE) ?? []).length; + const fr = (text.match(FR_CONTENT_RE) ?? []).length; + return de < 6 && fr < 6; + } catch { + // transient fetch error — retry + } + } + return true; // keep when we genuinely can't fetch the description +} + +export async function isEnglishUrl(url: string): Promise { + return (await isViableUrl(url)).ok; +} + +/** As isViableContent, but for an arbitrary job URL (non-LinkedIn sources). */ +export async function isViableUrl(url: string): Promise<{ ok: boolean; reason?: string }> { try { await Bun.sleep(350); - const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); - const html = await res.text(); - const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/); - const text = m - ? m[1].replace(/<[^>]+>/g, " ") - : html.replace(/<[^>]+>/g, " ").slice(0, 5000); + const res = await fetch(url, { headers: HEADERS }); + if (!res.ok) return { ok: true }; + const text = (await res.text()).replace(/<[^>]+>/g, " "); const de = (text.match(DE_CONTENT_RE) ?? []).length; const fr = (text.match(FR_CONTENT_RE) ?? []).length; - return de < 6 && fr < 6; + if (de >= 6 || fr >= 6) return { ok: false, reason: "non-English" }; + if (requiresGerman(text)) return { ok: false, reason: "requires German > B1" }; + return { ok: true }; } catch { - return true; // keep on fetch error + return { ok: true }; } } @@ -179,7 +350,9 @@ function parseJobs(html: string, label: string): Job[] { const company = companyM ? clean(companyM[1]) : ""; const postedAt = timeM ? timeM[1] : ""; - jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" }); + // Prefer the location printed on the card over the search label — recruiters + // and multi-site companies often post under a different area than the job sits in. + jobs.push({ id, title, company, location: jobLocation || label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label, source: "linkedin.com" }); } return jobs; @@ -206,6 +379,8 @@ export async function fetchJobs(config: SearchConfig, { raw = false } = {}): Pro .filter((j) => !NON_ENGLISH_RE.test(j.title)) .filter((j) => !JUNIOR_TITLE_RE.test(j.title)) .filter((j) => !JAVA_TITLE_RE.test(j.title)) + .filter((j) => !BIO_DOMAIN_RE.test(j.title)) + .filter((j) => !GERMAN_TITLE_RE.test(j.title)) .filter((j) => raw || SOFTWARE_TITLE_RE.test(j.title)); let newOnPage = 0; diff --git a/index.ts b/index.ts index cd5af47..ccfed76 100644 --- a/index.ts +++ b/index.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { searches } from "./config.ts"; -import { fetchJobs, isEnglishContent } from "./fetch.ts"; -import { fetchDataCareer, fetchStellenBasel } from "./fetch-other.ts"; +import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts"; +import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs } from "./fetch-other.ts"; const CSV = "./results/jobs.csv"; const OLD_HEADER = "date_found,posted_at,search,title,company,link"; @@ -57,9 +57,11 @@ for (const config of searches) { for (const job of jobs) { if (!existingLinks.has(job.link)) { process.stdout.write(` checking content: ${job.title} … `); - const english = await isEnglishContent(job.id); - if (!english) { - console.log("dropped (non-English)"); + // One fetch, two checks: written in German/French, or written in English + // but demanding German above B1. + const viable = await isViableContent(job.id); + if (!viable.ok) { + console.log(`dropped (${viable.reason})`); langDropped++; continue; } @@ -76,10 +78,12 @@ for (const config of searches) { console.log(` ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`); } -// Extra sources (no isEnglishContent — title filters already applied) const extraSources: Array<{ label: string; fetch: () => Promise> }> = [ { label: "datacareer.ch", fetch: fetchDataCareer }, { label: "stellen-basel.ch", fetch: fetchStellenBasel }, + { label: "itjobs.ch", fetch: fetchItJobsCh }, + { label: "job-room.ch", fetch: fetchJobRoom }, + { label: "basel intl orgs", fetch: fetchBaselIntlOrgs }, ]; for (const { label, fetch: fetchFn } of extraSources) { @@ -98,8 +102,19 @@ for (const { label, fetch: fetchFn } of extraSources) { } let added = 0; + let langDropped = 0; for (const job of jobs) { if (!existingLinks.has(job.link)) { + if (!job.contentChecked) { + process.stdout.write(` checking content: ${job.title} … `); + const viable = await isViableUrl(job.link); + if (!viable.ok) { + console.log(`dropped (${viable.reason})`); + langDropped++; + continue; + } + console.log("ok"); + } existingLinks.add(job.link); newRows.push( [date, job.postedAt, job.search, job.title, job.company, job.link, job.source] @@ -109,7 +124,7 @@ for (const { label, fetch: fetchFn } of extraSources) { added++; } } - console.log(` ${jobs.length} found, ${added} new`); + console.log(` ${jobs.length} found, ${added} new, ${langDropped} non-English dropped`); } if (newRows.length === 0) {