From b5a6f58fe326c4cb57b4ce1bd527da7aede37582 Mon Sep 17 00:00:00 2001 From: rafapolo Date: Tue, 5 May 2026 13:43:17 +0200 Subject: [PATCH] Initial commit: LinkedIn job scraper for Basel area - Paginated HTML scraper (up to 6 pages, 400ms delay between requests) - Card-boundary-safe parsing to prevent field bleed across cards - Language filter: drops German (m/w/d, umlauts) and French (H/F, accents) titles - Software-role filter: requires title to contain a relevant tech term - Global deduplication by job ID across runs - 30-day recency filter via LinkedIn f_TPR parameter - CSV output: date_found, posted_at, search, title, company, link --- .gitignore | 4 ++ README.md | 69 ++++++++++++++++++ config.ts | 64 +++++++++++++++++ fetch.ts | 192 +++++++++++++++++++++++++++++++++++++++++++++++++++ index.ts | 64 +++++++++++++++++ package.json | 8 +++ 6 files changed, 401 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config.ts create mode 100644 fetch.ts create mode 100644 index.ts create mode 100644 package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d3c5f24 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +jobs/ +jobs.csv +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..9200a9c --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# linkme + +Scrapes LinkedIn job listings into a local CSV. Run it daily, delete rows you've applied to. + +## Usage + +```bash +bun fetch +``` + +No setup needed. Results are appended to `jobs.csv` — reruns are safe, already-seen jobs are skipped by ID. + +## Output + +`jobs.csv` — one row per job: + +| column | description | +|---|---| +| `date_found` | date you ran the fetch | +| `posted_at` | date the job was posted on LinkedIn | +| `search` | which search config matched it | +| `title` | job title | +| `company` | hiring company | +| `link` | canonical LinkedIn job URL | + +Open in any spreadsheet app. Delete rows as you apply or dismiss. + +## How it works + +- **Paginated scraping** — fetches up to 6 pages per search (150 jobs) with a 400 ms delay between pages to avoid rate limiting +- **Card-boundary-safe parsing** — each job card is isolated before extracting fields, preventing content from adjacent cards bleeding into company or date values +- **Language filter** — drops German (`m/w/d`, umlauts, gender-marked vocabulary) and French (`H/F`, accented characters) job titles automatically +- **Software-role filter** — requires the title to contain a relevant term (software, developer, data, AI, backend, devops, platform, architect, etc.) — eliminates aviation, mechanical, and pharma-process roles that match "Engineer" broadly +- **Global deduplication** — job IDs are indexed from the existing CSV on startup; jobs you've already seen are never re-added, even if you delete and re-run +- **30-day recency** — only fetches jobs posted in the last month (`f_TPR=r2592000`) + +## Configure searches + +Edit `config.ts`: + +```ts +const KEYWORDS = [ + "Software Engineer", + "Full Stack Engineer", + "Data Engineer", + "AI Engineer", + "Ruby on Rails", + "Python", + // add or remove freely +].join(" OR "); + +export const searches: SearchConfig[] = [ + { + label: "Basel (city)", + keywords: KEYWORDS, + geoId: "103383283", + location: "Basel", + workType: ["onsite", "hybrid"], + timePosted: "month", + }, +]; +``` + +`geoId` values for the Basel area: +- `103383283` — Basel (city) +- `90009885` — Basel Metropolitan Area +- `107117530` — Basel-Country + +You can also paste any LinkedIn search URL directly as `url:` instead of building from filters. diff --git a/config.ts b/config.ts new file mode 100644 index 0000000..0bdb99a --- /dev/null +++ b/config.ts @@ -0,0 +1,64 @@ +export type WorkType = "onsite" | "remote" | "hybrid"; +export type JobType = "full-time" | "part-time" | "contract" | "temporary" | "internship"; +export type TimePosted = "day" | "week" | "month"; +export type Seniority = "internship" | "entry" | "associate" | "mid-senior" | "director" | "executive"; + +export interface SearchConfig { + label: string; + /** Use a raw LinkedIn URL instead of building one from filters. */ + url?: string; + keywords?: string; + geoId?: string; + location?: string; + workType?: WorkType[]; + jobType?: JobType[]; + timePosted?: TimePosted; + seniority?: Seniority[]; +} + +const KEYWORDS = [ + "Software Engineer", + "Solutions Engineer", + "Full Stack Engineer", + "Backend Engineer", + "Data Engineer", + "AI Engineer", + "Platform Engineer", + "DevOps Engineer", + "Staff Engineer", + "Principal Engineer", + "Tech Lead", + "Developer", + "Ruby on Rails", + "Python", + "Node.js", + "TypeScript", +].join(" OR "); + +// --- Define your searches here --- +export const searches: SearchConfig[] = [ + { + label: "Basel (city)", + keywords: KEYWORDS, + geoId: "103383283", + location: "Basel", + workType: ["onsite", "hybrid"], + timePosted: "month", + }, + { + label: "Basel Metropolitan Area", + keywords: KEYWORDS, + geoId: "90009885", + location: "Basel Metropolitan Area", + workType: ["onsite", "hybrid"], + timePosted: "month", + }, + { + label: "Basel-Country", + keywords: KEYWORDS, + geoId: "107117530", + location: "Basel-Country", + workType: ["onsite", "hybrid"], + timePosted: "month", + }, +]; diff --git a/fetch.ts b/fetch.ts new file mode 100644 index 0000000..31b15e9 --- /dev/null +++ b/fetch.ts @@ -0,0 +1,192 @@ +import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts"; + +export interface Job { + id: string; + title: string; + company: string; + location: string; + postedAt: string; + link: string; + search: string; +} + +const WORK_TYPE_MAP: Record = { + onsite: "1", + remote: "2", + hybrid: "3", +}; + +const JOB_TYPE_MAP: Record = { + "full-time": "F", + "part-time": "P", + contract: "C", + temporary: "T", + internship: "I", +}; + +const TIME_MAP: Record = { + day: "r86400", + week: "r604800", + month: "r2592000", +}; + +const SENIORITY_MAP: Record = { + internship: "1", + entry: "2", + associate: "3", + "mid-senior": "4", + director: "5", + executive: "6", +}; + +// Detect non-English (German / French / Swiss-German) job titles. +// +// German signals: +// - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc. +// but NOT (m/f/d) / (f/m/d) which are used in English international posts +// - Swiss neutral suffix: standalone (a) before % or end of string +// - German umlauts / ß +// - /in word-ending (Pflegehelfer/in, Köchin/Koch) +// - German-only vocabulary +// - Swiss German: EFZ (apprenticeship cert), Schweiz +// +// French signals: +// - gender markers: (H/F), (F/H), (h/f) +// - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ +// - French-only vocabulary +const NON_ENGLISH_RE = new RegExp( + [ + // German gender markers (w = weiblich; must not match m/f/d) + String.raw`\(m\/w[^)]*\)`, + String.raw`\(w\/m[^)]*\)`, + String.raw`\(d\/w[^)]*\)`, + String.raw`\(mwd\)`, + // Swiss neutral (a) suffix: " (a)" before %, digits, or end + String.raw`\s\(a\)(\s*[\d%]|\s*$)`, + // German/Swiss umlauts and ß + String.raw`[äöüÄÖÜß]`, + // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc. + String.raw`[a-zäöü]\/in\b`, + // German-only words (case-insensitive handled by flag) + String.raw`\b(Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`, + // French gender markers + String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`, + // French-only accented characters + String.raw`[éèêëàâçîïôùûÿœæ]`, + // French-only words + String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`, + ].join("|"), + "i" +); + +const HEADERS = { + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept-Language": "en-US,en;q=0.9", + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", +}; + +const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|engineering\s*lead|solutions\s*engineer|staff\s*engineer|principal\s*engineer)\b/i; + +const PAGE_SIZE = 25; +const MAX_PAGES = 6; + +export function buildUrl(config: SearchConfig): string { + if (config.url) return config.url; + const params = new URLSearchParams(); + params.set("keywords", config.keywords ?? ""); + if (config.location) params.set("location", config.location); + if (config.geoId) params.set("geoId", config.geoId); + if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]); + if (config.workType?.length) + params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(",")); + if (config.jobType?.length) + params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(",")); + if (config.seniority?.length) + params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(",")); + return `https://www.linkedin.com/jobs/search/?${params.toString()}`; +} + +function clean(raw: string): string { + return raw + .replace(/<[^>]+>/g, "") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/–/g, "–") + .replace(/—/g, "—") + .replace(/\s+/g, " ") + .trim(); +} + +function parseJobs(html: string, label: string): Job[] { + const jobs: Job[] = []; + + // Locate every card boundary by job ID position + const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g; + const positions: { id: string; start: number }[] = []; + let m; + while ((m = idRe.exec(html)) !== null) { + positions.push({ id: m[1], start: m.index }); + } + + for (let i = 0; i < positions.length; i++) { + const { id, start } = positions[i]; + // Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards + const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000)); + + const titleM = card.match(/

\s*([\s\S]*?)\s*<\/h3>/); + if (!titleM) continue; + + const companyM = card.match(/

[\s\S]{0,300}?]*>\s*([\s\S]*?)\s*<\/a>/); + const timeM = card.match(/]+datetime="([^"]+)"/); + + const title = clean(titleM[1]); + const company = companyM ? clean(companyM[1]) : ""; + const postedAt = timeM ? timeM[1] : ""; + + jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label }); + } + + return jobs; +} + +export async function fetchJobs(config: SearchConfig): Promise { + const baseUrl = buildUrl(config); + const allJobs: Job[] = []; + const seen = new Set(); + + for (let page = 0; page < MAX_PAGES; page++) { + if (page > 0) await Bun.sleep(400); + const url = `${baseUrl}&start=${page * PAGE_SIZE}`; + process.stdout.write(` page ${page + 1} … `); + + const res = await fetch(url, { headers: HEADERS }); + if (!res.ok) { + console.error(`failed (${res.status})`); + break; + } + + const html = await res.text(); + const jobs = parseJobs(html, config.label) + .filter((j) => !NON_ENGLISH_RE.test(j.title)) + .filter((j) => SOFTWARE_TITLE_RE.test(j.title)); + + let newOnPage = 0; + for (const job of jobs) { + if (!seen.has(job.id)) { + seen.add(job.id); + allJobs.push(job); + newOnPage++; + } + } + + console.log(`${jobs.length} jobs, ${newOnPage} new`); + if (newOnPage === 0) break; + } + + return allJobs; +} diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..7d24926 --- /dev/null +++ b/index.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env bun +import { searches } from "./config.ts"; +import { fetchJobs } from "./fetch.ts"; + +const CSV = "./jobs.csv"; +const HEADER = "date_found,posted_at,search,title,company,link"; + +// Load existing IDs from CSV to deduplicate across runs +const existingIds = new Set(); +try { + const text = await Bun.file(CSV).text(); + for (const line of text.split("\n").slice(1)) { + const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1]; + if (id) existingIds.add(id); + } +} catch { + // first run — file doesn't exist yet +} + +const date = new Date().toISOString().slice(0, 10); +const newRows: string[] = []; + +function csvCell(val: string): string { + return `"${val.replace(/"/g, '""')}"`; +} + +for (const config of searches) { + console.log(`\n[${config.label}]`); + const jobs = await fetchJobs(config); + + if (jobs.length === 0) { + console.log(" No jobs found."); + continue; + } + + let added = 0; + for (const job of jobs) { + if (!existingIds.has(job.id)) { + existingIds.add(job.id); + newRows.push( + [date, job.postedAt, config.label, job.title, job.company, job.link] + .map(csvCell) + .join(",") + ); + added++; + } + } + console.log(` ${jobs.length} found, ${added} new`); +} + +if (newRows.length === 0) { + console.log("\nNo new jobs."); + process.exit(0); +} + +// Append to CSV (write header if first run) +const existing = await Bun.file(CSV).exists(); +const append = existing + ? await Bun.file(CSV).text() + : HEADER + "\n"; + +await Bun.write(CSV, append + newRows.join("\n") + "\n"); + +console.log(`\n${newRows.length} new jobs → ${CSV}`); diff --git a/package.json b/package.json new file mode 100644 index 0000000..42b9f0e --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "linkme", + "version": "1.0.0", + "type": "module", + "scripts": { + "fetch": "bun run index.ts" + } +}