Initial commit: LinkedIn job scraper for Basel area

- Paginated HTML scraper (up to 6 pages, 400ms delay between requests) - Card-boundary-safe parsing to prevent field bleed across cards - Language filter: drops German (m/w/d, umlauts) and French (H/F, accents) titles - Software-role filter: requires title to contain a relevant tech term - Global deduplication by job ID across runs - 30-day recency filter via LinkedIn f_TPR parameter - CSV output: date_found, posted_at, search, title, company, link
2026-05-05 13:43:17 +02:00
commit b5a6f58fe3
6 changed files with 401 additions and 0 deletions
@@ -0,0 +1,4 @@
+node_modules/
+jobs/
+jobs.csv
+.env
@@ -0,0 +1,69 @@
+# linkme
+
+Scrapes LinkedIn job listings into a local CSV. Run it daily, delete rows you've applied to.
+
+## Usage
+
+```bash
+bun fetch
+```
+
+No setup needed. Results are appended to `jobs.csv` — reruns are safe, already-seen jobs are skipped by ID.
+
+## Output
+
+`jobs.csv` — one row per job:
+
+| column | description |
+|---|---|
+| `date_found` | date you ran the fetch |
+| `posted_at` | date the job was posted on LinkedIn |
+| `search` | which search config matched it |
+| `title` | job title |
+| `company` | hiring company |
+| `link` | canonical LinkedIn job URL |
+
+Open in any spreadsheet app. Delete rows as you apply or dismiss.
+
+## How it works
+
+- **Paginated scraping** — fetches up to 6 pages per search (150 jobs) with a 400 ms delay between pages to avoid rate limiting
+- **Card-boundary-safe parsing** — each job card is isolated before extracting fields, preventing content from adjacent cards bleeding into company or date values
+- **Language filter** — drops German (`m/w/d`, umlauts, gender-marked vocabulary) and French (`H/F`, accented characters) job titles automatically
+- **Software-role filter** — requires the title to contain a relevant term (software, developer, data, AI, backend, devops, platform, architect, etc.) — eliminates aviation, mechanical, and pharma-process roles that match "Engineer" broadly
+- **Global deduplication** — job IDs are indexed from the existing CSV on startup; jobs you've already seen are never re-added, even if you delete and re-run
+- **30-day recency** — only fetches jobs posted in the last month (`f_TPR=r2592000`)
+
+## Configure searches
+
+Edit `config.ts`:
+
+```ts
+const KEYWORDS = [
+  "Software Engineer",
+  "Full Stack Engineer",
+  "Data Engineer",
+  "AI Engineer",
+  "Ruby on Rails",
+  "Python",
+  // add or remove freely
+].join(" OR ");
+
+export const searches: SearchConfig[] = [
+  {
+    label: "Basel (city)",
+    keywords: KEYWORDS,
+    geoId: "103383283",
+    location: "Basel",
+    workType: ["onsite", "hybrid"],
+    timePosted: "month",
+  },
+];
+```
+
+`geoId` values for the Basel area:
+- `103383283` — Basel (city)
+- `90009885` — Basel Metropolitan Area
+- `107117530` — Basel-Country
+
+You can also paste any LinkedIn search URL directly as `url:` instead of building from filters.
@@ -0,0 +1,64 @@
+export type WorkType = "onsite" | "remote" | "hybrid";
+export type JobType = "full-time" | "part-time" | "contract" | "temporary" | "internship";
+export type TimePosted = "day" | "week" | "month";
+export type Seniority = "internship" | "entry" | "associate" | "mid-senior" | "director" | "executive";
+
+export interface SearchConfig {
+  label: string;
+  /** Use a raw LinkedIn URL instead of building one from filters. */
+  url?: string;
+  keywords?: string;
+  geoId?: string;
+  location?: string;
+  workType?: WorkType[];
+  jobType?: JobType[];
+  timePosted?: TimePosted;
+  seniority?: Seniority[];
+}
+
+const KEYWORDS = [
+  "Software Engineer",
+  "Solutions Engineer",
+  "Full Stack Engineer",
+  "Backend Engineer",
+  "Data Engineer",
+  "AI Engineer",
+  "Platform Engineer",
+  "DevOps Engineer",
+  "Staff Engineer",
+  "Principal Engineer",
+  "Tech Lead",
+  "Developer",
+  "Ruby on Rails",
+  "Python",
+  "Node.js",
+  "TypeScript",
+].join(" OR ");
+
+// --- Define your searches here ---
+export const searches: SearchConfig[] = [
+  {
+    label: "Basel (city)",
+    keywords: KEYWORDS,
+    geoId: "103383283",
+    location: "Basel",
+    workType: ["onsite", "hybrid"],
+    timePosted: "month",
+  },
+  {
+    label: "Basel Metropolitan Area",
+    keywords: KEYWORDS,
+    geoId: "90009885",
+    location: "Basel Metropolitan Area",
+    workType: ["onsite", "hybrid"],
+    timePosted: "month",
+  },
+  {
+    label: "Basel-Country",
+    keywords: KEYWORDS,
+    geoId: "107117530",
+    location: "Basel-Country",
+    workType: ["onsite", "hybrid"],
+    timePosted: "month",
+  },
+];
@@ -0,0 +1,192 @@
+import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";
+
+export interface Job {
+  id: string;
+  title: string;
+  company: string;
+  location: string;
+  postedAt: string;
+  link: string;
+  search: string;
+}
+
+const WORK_TYPE_MAP: Record<WorkType, string> = {
+  onsite: "1",
+  remote: "2",
+  hybrid: "3",
+};
+
+const JOB_TYPE_MAP: Record<JobType, string> = {
+  "full-time": "F",
+  "part-time": "P",
+  contract: "C",
+  temporary: "T",
+  internship: "I",
+};
+
+const TIME_MAP: Record<TimePosted, string> = {
+  day: "r86400",
+  week: "r604800",
+  month: "r2592000",
+};
+
+const SENIORITY_MAP: Record<Seniority, string> = {
+  internship: "1",
+  entry: "2",
+  associate: "3",
+  "mid-senior": "4",
+  director: "5",
+  executive: "6",
+};
+
+// Detect non-English (German / French / Swiss-German) job titles.
+//
+// German signals:
+//   - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
+//     but NOT (m/f/d) / (f/m/d) which are used in English international posts
+//   - Swiss neutral suffix: standalone (a) before % or end of string
+//   - German umlauts / ß
+//   - /in word-ending (Pflegehelfer/in, Köchin/Koch)
+//   - German-only vocabulary
+//   - Swiss German: EFZ (apprenticeship cert), Schweiz
+//
+// French signals:
+//   - gender markers: (H/F), (F/H), (h/f)
+//   - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
+//   - French-only vocabulary
+const NON_ENGLISH_RE = new RegExp(
+  [
+    // German gender markers (w = weiblich; must not match m/f/d)
+    String.raw`\(m\/w[^)]*\)`,
+    String.raw`\(w\/m[^)]*\)`,
+    String.raw`\(d\/w[^)]*\)`,
+    String.raw`\(mwd\)`,
+    // Swiss neutral (a) suffix: " (a)" before %, digits, or end
+    String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
+    // German/Swiss umlauts and ß
+    String.raw`[äöüÄÖÜß]`,
+    // /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
+    String.raw`[a-zäöü]\/in\b`,
+    // German-only words (case-insensitive handled by flag)
+    String.raw`\b(Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`,
+    // French gender markers
+    String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
+    // French-only accented characters
+    String.raw`[éèêëàâçîïôùûÿœæ]`,
+    // French-only words
+    String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
+  ].join("|"),
+  "i"
+);
+
+const HEADERS = {
+  "User-Agent":
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+  "Accept-Language": "en-US,en;q=0.9",
+  Accept:
+    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+};
+
+const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|engineering\s*lead|solutions\s*engineer|staff\s*engineer|principal\s*engineer)\b/i;
+
+const PAGE_SIZE = 25;
+const MAX_PAGES = 6;
+
+export function buildUrl(config: SearchConfig): string {
+  if (config.url) return config.url;
+  const params = new URLSearchParams();
+  params.set("keywords", config.keywords ?? "");
+  if (config.location) params.set("location", config.location);
+  if (config.geoId) params.set("geoId", config.geoId);
+  if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
+  if (config.workType?.length)
+    params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
+  if (config.jobType?.length)
+    params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
+  if (config.seniority?.length)
+    params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
+  return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
+}
+
+function clean(raw: string): string {
+  return raw
+    .replace(/<[^>]+>/g, "")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&ndash;/g, "–")
+    .replace(/&mdash;/g, "—")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function parseJobs(html: string, label: string): Job[] {
+  const jobs: Job[] = [];
+
+  // Locate every card boundary by job ID position
+  const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
+  const positions: { id: string; start: number }[] = [];
+  let m;
+  while ((m = idRe.exec(html)) !== null) {
+    positions.push({ id: m[1], start: m.index });
+  }
+
+  for (let i = 0; i < positions.length; i++) {
+    const { id, start } = positions[i];
+    // Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
+    const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));
+
+    const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
+    if (!titleM) continue;
+
+    const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
+    const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);
+
+    const title = clean(titleM[1]);
+    const company = companyM ? clean(companyM[1]) : "";
+    const postedAt = timeM ? timeM[1] : "";
+
+    jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label });
+  }
+
+  return jobs;
+}
+
+export async function fetchJobs(config: SearchConfig): Promise<Job[]> {
+  const baseUrl = buildUrl(config);
+  const allJobs: Job[] = [];
+  const seen = new Set<string>();
+
+  for (let page = 0; page < MAX_PAGES; page++) {
+    if (page > 0) await Bun.sleep(400);
+    const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
+    process.stdout.write(`  page ${page + 1} … `);
+
+    const res = await fetch(url, { headers: HEADERS });
+    if (!res.ok) {
+      console.error(`failed (${res.status})`);
+      break;
+    }
+
+    const html = await res.text();
+    const jobs = parseJobs(html, config.label)
+      .filter((j) => !NON_ENGLISH_RE.test(j.title))
+      .filter((j) => SOFTWARE_TITLE_RE.test(j.title));
+
+    let newOnPage = 0;
+    for (const job of jobs) {
+      if (!seen.has(job.id)) {
+        seen.add(job.id);
+        allJobs.push(job);
+        newOnPage++;
+      }
+    }
+
+    console.log(`${jobs.length} jobs, ${newOnPage} new`);
+    if (newOnPage === 0) break;
+  }
+
+  return allJobs;
+}
@@ -0,0 +1,64 @@
+#!/usr/bin/env bun
+import { searches } from "./config.ts";
+import { fetchJobs } from "./fetch.ts";
+
+const CSV = "./jobs.csv";
+const HEADER = "date_found,posted_at,search,title,company,link";
+
+// Load existing IDs from CSV to deduplicate across runs
+const existingIds = new Set<string>();
+try {
+  const text = await Bun.file(CSV).text();
+  for (const line of text.split("\n").slice(1)) {
+    const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
+    if (id) existingIds.add(id);
+  }
+} catch {
+  // first run — file doesn't exist yet
+}
+
+const date = new Date().toISOString().slice(0, 10);
+const newRows: string[] = [];
+
+function csvCell(val: string): string {
+  return `"${val.replace(/"/g, '""')}"`;
+}
+
+for (const config of searches) {
+  console.log(`\n[${config.label}]`);
+  const jobs = await fetchJobs(config);
+
+  if (jobs.length === 0) {
+    console.log("  No jobs found.");
+    continue;
+  }
+
+  let added = 0;
+  for (const job of jobs) {
+    if (!existingIds.has(job.id)) {
+      existingIds.add(job.id);
+      newRows.push(
+        [date, job.postedAt, config.label, job.title, job.company, job.link]
+          .map(csvCell)
+          .join(",")
+      );
+      added++;
+    }
+  }
+  console.log(`  ${jobs.length} found, ${added} new`);
+}
+
+if (newRows.length === 0) {
+  console.log("\nNo new jobs.");
+  process.exit(0);
+}
+
+// Append to CSV (write header if first run)
+const existing = await Bun.file(CSV).exists();
+const append = existing
+  ? await Bun.file(CSV).text()
+  : HEADER + "\n";
+
+await Bun.write(CSV, append + newRows.join("\n") + "\n");
+
+console.log(`\n${newRows.length} new jobs → ${CSV}`);
@@ -0,0 +1,8 @@
+{
+  "name": "linkme",
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "fetch": "bun run index.ts"
+  }
+}