Initial commit: LinkedIn job scraper for Basel area
- Paginated HTML scraper (up to 6 pages, 400ms delay between requests) - Card-boundary-safe parsing to prevent field bleed across cards - Language filter: drops German (m/w/d, umlauts) and French (H/F, accents) titles - Software-role filter: requires title to contain a relevant tech term - Global deduplication by job ID across runs - 30-day recency filter via LinkedIn f_TPR parameter - CSV output: date_found, posted_at, search, title, company, link
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
node_modules/
|
||||||
|
jobs/
|
||||||
|
jobs.csv
|
||||||
|
.env
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
# linkme
|
||||||
|
|
||||||
|
Scrapes LinkedIn job listings into a local CSV. Run it daily, delete rows you've applied to.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun fetch
|
||||||
|
```
|
||||||
|
|
||||||
|
No setup needed. Results are appended to `jobs.csv` — reruns are safe, already-seen jobs are skipped by ID.
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
`jobs.csv` — one row per job:
|
||||||
|
|
||||||
|
| column | description |
|
||||||
|
|---|---|
|
||||||
|
| `date_found` | date you ran the fetch |
|
||||||
|
| `posted_at` | date the job was posted on LinkedIn |
|
||||||
|
| `search` | which search config matched it |
|
||||||
|
| `title` | job title |
|
||||||
|
| `company` | hiring company |
|
||||||
|
| `link` | canonical LinkedIn job URL |
|
||||||
|
|
||||||
|
Open in any spreadsheet app. Delete rows as you apply or dismiss.
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
- **Paginated scraping** — fetches up to 6 pages per search (150 jobs) with a 400 ms delay between pages to avoid rate limiting
|
||||||
|
- **Card-boundary-safe parsing** — each job card is isolated before extracting fields, preventing content from adjacent cards bleeding into company or date values
|
||||||
|
- **Language filter** — drops German (`m/w/d`, umlauts, gender-marked vocabulary) and French (`H/F`, accented characters) job titles automatically
|
||||||
|
- **Software-role filter** — requires the title to contain a relevant term (software, developer, data, AI, backend, devops, platform, architect, etc.) — eliminates aviation, mechanical, and pharma-process roles that match "Engineer" broadly
|
||||||
|
- **Global deduplication** — job IDs are indexed from the existing CSV on startup; jobs you've already seen are never re-added, even if you delete and re-run
|
||||||
|
- **30-day recency** — only fetches jobs posted in the last month (`f_TPR=r2592000`)
|
||||||
|
|
||||||
|
## Configure searches
|
||||||
|
|
||||||
|
Edit `config.ts`:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const KEYWORDS = [
|
||||||
|
"Software Engineer",
|
||||||
|
"Full Stack Engineer",
|
||||||
|
"Data Engineer",
|
||||||
|
"AI Engineer",
|
||||||
|
"Ruby on Rails",
|
||||||
|
"Python",
|
||||||
|
// add or remove freely
|
||||||
|
].join(" OR ");
|
||||||
|
|
||||||
|
export const searches: SearchConfig[] = [
|
||||||
|
{
|
||||||
|
label: "Basel (city)",
|
||||||
|
keywords: KEYWORDS,
|
||||||
|
geoId: "103383283",
|
||||||
|
location: "Basel",
|
||||||
|
workType: ["onsite", "hybrid"],
|
||||||
|
timePosted: "month",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
```
|
||||||
|
|
||||||
|
`geoId` values for the Basel area:
|
||||||
|
- `103383283` — Basel (city)
|
||||||
|
- `90009885` — Basel Metropolitan Area
|
||||||
|
- `107117530` — Basel-Country
|
||||||
|
|
||||||
|
You can also paste any LinkedIn search URL directly as `url:` instead of building from filters.
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
export type WorkType = "onsite" | "remote" | "hybrid";
|
||||||
|
export type JobType = "full-time" | "part-time" | "contract" | "temporary" | "internship";
|
||||||
|
export type TimePosted = "day" | "week" | "month";
|
||||||
|
export type Seniority = "internship" | "entry" | "associate" | "mid-senior" | "director" | "executive";
|
||||||
|
|
||||||
|
export interface SearchConfig {
|
||||||
|
label: string;
|
||||||
|
/** Use a raw LinkedIn URL instead of building one from filters. */
|
||||||
|
url?: string;
|
||||||
|
keywords?: string;
|
||||||
|
geoId?: string;
|
||||||
|
location?: string;
|
||||||
|
workType?: WorkType[];
|
||||||
|
jobType?: JobType[];
|
||||||
|
timePosted?: TimePosted;
|
||||||
|
seniority?: Seniority[];
|
||||||
|
}
|
||||||
|
|
||||||
|
const KEYWORDS = [
|
||||||
|
"Software Engineer",
|
||||||
|
"Solutions Engineer",
|
||||||
|
"Full Stack Engineer",
|
||||||
|
"Backend Engineer",
|
||||||
|
"Data Engineer",
|
||||||
|
"AI Engineer",
|
||||||
|
"Platform Engineer",
|
||||||
|
"DevOps Engineer",
|
||||||
|
"Staff Engineer",
|
||||||
|
"Principal Engineer",
|
||||||
|
"Tech Lead",
|
||||||
|
"Developer",
|
||||||
|
"Ruby on Rails",
|
||||||
|
"Python",
|
||||||
|
"Node.js",
|
||||||
|
"TypeScript",
|
||||||
|
].join(" OR ");
|
||||||
|
|
||||||
|
// --- Define your searches here ---
|
||||||
|
export const searches: SearchConfig[] = [
|
||||||
|
{
|
||||||
|
label: "Basel (city)",
|
||||||
|
keywords: KEYWORDS,
|
||||||
|
geoId: "103383283",
|
||||||
|
location: "Basel",
|
||||||
|
workType: ["onsite", "hybrid"],
|
||||||
|
timePosted: "month",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Basel Metropolitan Area",
|
||||||
|
keywords: KEYWORDS,
|
||||||
|
geoId: "90009885",
|
||||||
|
location: "Basel Metropolitan Area",
|
||||||
|
workType: ["onsite", "hybrid"],
|
||||||
|
timePosted: "month",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: "Basel-Country",
|
||||||
|
keywords: KEYWORDS,
|
||||||
|
geoId: "107117530",
|
||||||
|
location: "Basel-Country",
|
||||||
|
workType: ["onsite", "hybrid"],
|
||||||
|
timePosted: "month",
|
||||||
|
},
|
||||||
|
];
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
import type { SearchConfig, WorkType, JobType, TimePosted, Seniority } from "./config.ts";
|
||||||
|
|
||||||
|
export interface Job {
|
||||||
|
id: string;
|
||||||
|
title: string;
|
||||||
|
company: string;
|
||||||
|
location: string;
|
||||||
|
postedAt: string;
|
||||||
|
link: string;
|
||||||
|
search: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const WORK_TYPE_MAP: Record<WorkType, string> = {
|
||||||
|
onsite: "1",
|
||||||
|
remote: "2",
|
||||||
|
hybrid: "3",
|
||||||
|
};
|
||||||
|
|
||||||
|
const JOB_TYPE_MAP: Record<JobType, string> = {
|
||||||
|
"full-time": "F",
|
||||||
|
"part-time": "P",
|
||||||
|
contract: "C",
|
||||||
|
temporary: "T",
|
||||||
|
internship: "I",
|
||||||
|
};
|
||||||
|
|
||||||
|
const TIME_MAP: Record<TimePosted, string> = {
|
||||||
|
day: "r86400",
|
||||||
|
week: "r604800",
|
||||||
|
month: "r2592000",
|
||||||
|
};
|
||||||
|
|
||||||
|
const SENIORITY_MAP: Record<Seniority, string> = {
|
||||||
|
internship: "1",
|
||||||
|
entry: "2",
|
||||||
|
associate: "3",
|
||||||
|
"mid-senior": "4",
|
||||||
|
director: "5",
|
||||||
|
executive: "6",
|
||||||
|
};
|
||||||
|
|
||||||
|
// Detect non-English (German / French / Swiss-German) job titles.
|
||||||
|
//
|
||||||
|
// German signals:
|
||||||
|
// - gender markers: (m/w/d), (w/m/d), (d/w/m), (mwd), (M/W/D) etc.
|
||||||
|
// but NOT (m/f/d) / (f/m/d) which are used in English international posts
|
||||||
|
// - Swiss neutral suffix: standalone (a) before % or end of string
|
||||||
|
// - German umlauts / ß
|
||||||
|
// - /in word-ending (Pflegehelfer/in, Köchin/Koch)
|
||||||
|
// - German-only vocabulary
|
||||||
|
// - Swiss German: EFZ (apprenticeship cert), Schweiz
|
||||||
|
//
|
||||||
|
// French signals:
|
||||||
|
// - gender markers: (H/F), (F/H), (h/f)
|
||||||
|
// - French-only accented chars: é è ê ë à â ç î ï ô ù û ÿ œ æ
|
||||||
|
// - French-only vocabulary
|
||||||
|
const NON_ENGLISH_RE = new RegExp(
|
||||||
|
[
|
||||||
|
// German gender markers (w = weiblich; must not match m/f/d)
|
||||||
|
String.raw`\(m\/w[^)]*\)`,
|
||||||
|
String.raw`\(w\/m[^)]*\)`,
|
||||||
|
String.raw`\(d\/w[^)]*\)`,
|
||||||
|
String.raw`\(mwd\)`,
|
||||||
|
// Swiss neutral (a) suffix: " (a)" before %, digits, or end
|
||||||
|
String.raw`\s\(a\)(\s*[\d%]|\s*$)`,
|
||||||
|
// German/Swiss umlauts and ß
|
||||||
|
String.raw`[äöüÄÖÜß]`,
|
||||||
|
// /in word-ending (e.g. Mitarbeiter/in) — excludes m/f, n/a, etc.
|
||||||
|
String.raw`[a-zäöü]\/in\b`,
|
||||||
|
// German-only words (case-insensitive handled by flag)
|
||||||
|
String.raw`\b(Mitarbeiter|Pflegehelfer|Köchin|Koch EFZ|Kellner|Sozialpädagog|Lehrperson|Produktionsmitarbeiter|Produktionsplaner|Geschäftsführer|Spontanbewerbung|Arztsekretär|Physiotherapeut|Empfangsmitarbeiter|Automobilmechatroniker|Verkäufer|Projektleiter|Crewmitarbeiter|Crewmember|Systemgastronomie|Studienteilnehmende|Teilnehmende|Bauingenieur|Tiefbau|Hochbau|Elektroingenieur|Maschinenbau|gesucht|EFZ|Schweiz|Filiale)\b`,
|
||||||
|
// French gender markers
|
||||||
|
String.raw`\(H\/F\)|\(F\/H\)|\(h\/f\)|\(f\/h\)`,
|
||||||
|
// French-only accented characters
|
||||||
|
String.raw`[éèêëàâçîïôùûÿœæ]`,
|
||||||
|
// French-only words
|
||||||
|
String.raw`\b(Responsable|Directeur|Directrice|Adjoint|Chargé|Ingénieur|Technicien|Gestionnaire|Coordinateur|Analyste)\b`,
|
||||||
|
].join("|"),
|
||||||
|
"i"
|
||||||
|
);
|
||||||
|
|
||||||
|
const HEADERS = {
|
||||||
|
"User-Agent":
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||||
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
|
Accept:
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||||
|
};
|
||||||
|
|
||||||
|
const SOFTWARE_TITLE_RE = /\b(software|developer|architect|devops|platform|data|backend|frontend|fullstack|full.stack|tech\s*lead|typescript|python|rails|node\.?js|ai\b|machine\s*learning|cloud|security|analytics|sre|reliability|engineering\s*lead|solutions\s*engineer|staff\s*engineer|principal\s*engineer)\b/i;
|
||||||
|
|
||||||
|
const PAGE_SIZE = 25;
|
||||||
|
const MAX_PAGES = 6;
|
||||||
|
|
||||||
|
export function buildUrl(config: SearchConfig): string {
|
||||||
|
if (config.url) return config.url;
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
params.set("keywords", config.keywords ?? "");
|
||||||
|
if (config.location) params.set("location", config.location);
|
||||||
|
if (config.geoId) params.set("geoId", config.geoId);
|
||||||
|
if (config.timePosted) params.set("f_TPR", TIME_MAP[config.timePosted]);
|
||||||
|
if (config.workType?.length)
|
||||||
|
params.set("f_WT", config.workType.map((w) => WORK_TYPE_MAP[w]).join(","));
|
||||||
|
if (config.jobType?.length)
|
||||||
|
params.set("f_JT", config.jobType.map((j) => JOB_TYPE_MAP[j]).join(","));
|
||||||
|
if (config.seniority?.length)
|
||||||
|
params.set("f_E", config.seniority.map((s) => SENIORITY_MAP[s]).join(","));
|
||||||
|
return `https://www.linkedin.com/jobs/search/?${params.toString()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clean(raw: string): string {
|
||||||
|
return raw
|
||||||
|
.replace(/<[^>]+>/g, "")
|
||||||
|
.replace(/&/g, "&")
|
||||||
|
.replace(/</g, "<")
|
||||||
|
.replace(/>/g, ">")
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, "'")
|
||||||
|
.replace(/–/g, "–")
|
||||||
|
.replace(/—/g, "—")
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseJobs(html: string, label: string): Job[] {
|
||||||
|
const jobs: Job[] = [];
|
||||||
|
|
||||||
|
// Locate every card boundary by job ID position
|
||||||
|
const idRe = /data-entity-urn="urn:li:jobPosting:(\d+)"/g;
|
||||||
|
const positions: { id: string; start: number }[] = [];
|
||||||
|
let m;
|
||||||
|
while ((m = idRe.exec(html)) !== null) {
|
||||||
|
positions.push({ id: m[1], start: m.index });
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < positions.length; i++) {
|
||||||
|
const { id, start } = positions[i];
|
||||||
|
// Slice only up to the next card (or 3 000 chars max) so field regexes can't bleed across cards
|
||||||
|
const card = html.slice(start, Math.min(positions[i + 1]?.start ?? Infinity, start + 3000));
|
||||||
|
|
||||||
|
const titleM = card.match(/<h3 class="base-search-card__title">\s*([\s\S]*?)\s*<\/h3>/);
|
||||||
|
if (!titleM) continue;
|
||||||
|
|
||||||
|
const companyM = card.match(/<h4 class="base-search-card__subtitle">[\s\S]{0,300}?<a[^>]*>\s*([\s\S]*?)\s*<\/a>/);
|
||||||
|
const timeM = card.match(/<time[^>]+datetime="([^"]+)"/);
|
||||||
|
|
||||||
|
const title = clean(titleM[1]);
|
||||||
|
const company = companyM ? clean(companyM[1]) : "";
|
||||||
|
const postedAt = timeM ? timeM[1] : "";
|
||||||
|
|
||||||
|
jobs.push({ id, title, company, location: label, postedAt, link: `https://www.linkedin.com/jobs/view/${id}/`, search: label });
|
||||||
|
}
|
||||||
|
|
||||||
|
return jobs;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchJobs(config: SearchConfig): Promise<Job[]> {
|
||||||
|
const baseUrl = buildUrl(config);
|
||||||
|
const allJobs: Job[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
|
||||||
|
for (let page = 0; page < MAX_PAGES; page++) {
|
||||||
|
if (page > 0) await Bun.sleep(400);
|
||||||
|
const url = `${baseUrl}&start=${page * PAGE_SIZE}`;
|
||||||
|
process.stdout.write(` page ${page + 1} … `);
|
||||||
|
|
||||||
|
const res = await fetch(url, { headers: HEADERS });
|
||||||
|
if (!res.ok) {
|
||||||
|
console.error(`failed (${res.status})`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await res.text();
|
||||||
|
const jobs = parseJobs(html, config.label)
|
||||||
|
.filter((j) => !NON_ENGLISH_RE.test(j.title))
|
||||||
|
.filter((j) => SOFTWARE_TITLE_RE.test(j.title));
|
||||||
|
|
||||||
|
let newOnPage = 0;
|
||||||
|
for (const job of jobs) {
|
||||||
|
if (!seen.has(job.id)) {
|
||||||
|
seen.add(job.id);
|
||||||
|
allJobs.push(job);
|
||||||
|
newOnPage++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`${jobs.length} jobs, ${newOnPage} new`);
|
||||||
|
if (newOnPage === 0) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return allJobs;
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
import { searches } from "./config.ts";
|
||||||
|
import { fetchJobs } from "./fetch.ts";
|
||||||
|
|
||||||
|
const CSV = "./jobs.csv";
|
||||||
|
const HEADER = "date_found,posted_at,search,title,company,link";
|
||||||
|
|
||||||
|
// Load existing IDs from CSV to deduplicate across runs
|
||||||
|
const existingIds = new Set<string>();
|
||||||
|
try {
|
||||||
|
const text = await Bun.file(CSV).text();
|
||||||
|
for (const line of text.split("\n").slice(1)) {
|
||||||
|
const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
|
||||||
|
if (id) existingIds.add(id);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// first run — file doesn't exist yet
|
||||||
|
}
|
||||||
|
|
||||||
|
const date = new Date().toISOString().slice(0, 10);
|
||||||
|
const newRows: string[] = [];
|
||||||
|
|
||||||
|
function csvCell(val: string): string {
|
||||||
|
return `"${val.replace(/"/g, '""')}"`;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const config of searches) {
|
||||||
|
console.log(`\n[${config.label}]`);
|
||||||
|
const jobs = await fetchJobs(config);
|
||||||
|
|
||||||
|
if (jobs.length === 0) {
|
||||||
|
console.log(" No jobs found.");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let added = 0;
|
||||||
|
for (const job of jobs) {
|
||||||
|
if (!existingIds.has(job.id)) {
|
||||||
|
existingIds.add(job.id);
|
||||||
|
newRows.push(
|
||||||
|
[date, job.postedAt, config.label, job.title, job.company, job.link]
|
||||||
|
.map(csvCell)
|
||||||
|
.join(",")
|
||||||
|
);
|
||||||
|
added++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(` ${jobs.length} found, ${added} new`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newRows.length === 0) {
|
||||||
|
console.log("\nNo new jobs.");
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Append to CSV (write header if first run)
|
||||||
|
const existing = await Bun.file(CSV).exists();
|
||||||
|
const append = existing
|
||||||
|
? await Bun.file(CSV).text()
|
||||||
|
: HEADER + "\n";
|
||||||
|
|
||||||
|
await Bun.write(CSV, append + newRows.join("\n") + "\n");
|
||||||
|
|
||||||
|
console.log(`\n${newRows.length} new jobs → ${CSV}`);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"name": "linkme",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"fetch": "bun run index.ts"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user