Initial commit: LinkedIn job scraper for Basel area
- Paginated HTML scraper (up to 6 pages, 400ms delay between requests) - Card-boundary-safe parsing to prevent field bleed across cards - Language filter: drops German (m/w/d, umlauts) and French (H/F, accents) titles - Software-role filter: requires title to contain a relevant tech term - Global deduplication by job ID across runs - 30-day recency filter via LinkedIn f_TPR parameter - CSV output: date_found, posted_at, search, title, company, link
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bun
|
||||
import { searches } from "./config.ts";
|
||||
import { fetchJobs } from "./fetch.ts";
|
||||
|
||||
const CSV = "./jobs.csv";
|
||||
const HEADER = "date_found,posted_at,search,title,company,link";
|
||||
|
||||
// Load existing IDs from CSV to deduplicate across runs
|
||||
const existingIds = new Set<string>();
|
||||
try {
|
||||
const text = await Bun.file(CSV).text();
|
||||
for (const line of text.split("\n").slice(1)) {
|
||||
const id = line.match(/\/jobs\/view\/(\d+)\//)?.[1];
|
||||
if (id) existingIds.add(id);
|
||||
}
|
||||
} catch {
|
||||
// first run — file doesn't exist yet
|
||||
}
|
||||
|
||||
const date = new Date().toISOString().slice(0, 10);
|
||||
const newRows: string[] = [];
|
||||
|
||||
function csvCell(val: string): string {
|
||||
return `"${val.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
for (const config of searches) {
|
||||
console.log(`\n[${config.label}]`);
|
||||
const jobs = await fetchJobs(config);
|
||||
|
||||
if (jobs.length === 0) {
|
||||
console.log(" No jobs found.");
|
||||
continue;
|
||||
}
|
||||
|
||||
let added = 0;
|
||||
for (const job of jobs) {
|
||||
if (!existingIds.has(job.id)) {
|
||||
existingIds.add(job.id);
|
||||
newRows.push(
|
||||
[date, job.postedAt, config.label, job.title, job.company, job.link]
|
||||
.map(csvCell)
|
||||
.join(",")
|
||||
);
|
||||
added++;
|
||||
}
|
||||
}
|
||||
console.log(` ${jobs.length} found, ${added} new`);
|
||||
}
|
||||
|
||||
if (newRows.length === 0) {
|
||||
console.log("\nNo new jobs.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Append to CSV (write header if first run)
|
||||
const existing = await Bun.file(CSV).exists();
|
||||
const append = existing
|
||||
? await Bun.file(CSV).text()
|
||||
: HEADER + "\n";
|
||||
|
||||
await Bun.write(CSV, append + newRows.join("\n") + "\n");
|
||||
|
||||
console.log(`\n${newRows.length} new jobs → ${CSV}`);
|
||||
Reference in New Issue
Block a user