chore: reorganize repo structure and update docs
- move temp scripts to scripts/, cv files to cv/, jobs.csv to results/ - update index.ts and recheck.ts CSV path to results/jobs.csv - fix imports in scripts/ to use ../config.ts and ../fetch.ts - update CLAUDE.md and README.md to reflect new structure - add plans/ with LinkedIn post drafts - update .gitignore: ignore results/, jobs/, cv binaries, .DS_Store
This commit is contained in:
+143
@@ -0,0 +1,143 @@
|
||||
import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE } from "./fetch.ts";
|
||||
import type { Job } from "./fetch.ts";
|
||||
|
||||
const HEADERS = {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
};
|
||||
|
||||
function decodeHtml(s: string): string {
|
||||
return s
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function applyFilters(title: string): boolean {
|
||||
if (NON_ENGLISH_RE.test(title)) return false;
|
||||
if (JUNIOR_TITLE_RE.test(title)) return false;
|
||||
if (!SOFTWARE_TITLE_RE.test(title)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// datacareer.ch — HTML listing at /jobs/?city=Basel
|
||||
export async function fetchDataCareer(): Promise<Job[]> {
|
||||
const allJobs: Job[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
if (page > 1) await Bun.sleep(400);
|
||||
process.stdout.write(` page ${page} … `);
|
||||
|
||||
const res = await fetch(
|
||||
`https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
|
||||
{ headers: HEADERS }
|
||||
);
|
||||
if (!res.ok) { console.error(`failed (${res.status})`); break; }
|
||||
|
||||
const html = await res.text();
|
||||
const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
|
||||
if (articles.length === 0) { console.log("0 items"); break; }
|
||||
|
||||
let newOnPage = 0;
|
||||
for (const [, card] of articles) {
|
||||
const titleM = card.match(/class="link"[^>]*>([^<]+)/);
|
||||
const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
|
||||
if (!titleM || !hrefM) continue;
|
||||
|
||||
const title = decodeHtml(titleM[1]);
|
||||
if (!applyFilters(title)) continue;
|
||||
|
||||
const id = `dc-${hrefM[2]}`;
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
|
||||
const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
|
||||
const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
|
||||
const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);
|
||||
|
||||
// Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
|
||||
let postedAt = "";
|
||||
const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
|
||||
if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;
|
||||
|
||||
allJobs.push({
|
||||
id,
|
||||
title,
|
||||
company: companyM ? decodeHtml(companyM[1].trim()) : "",
|
||||
location: locM ? locM[1].trim() : "Basel",
|
||||
postedAt,
|
||||
link: hrefM[1],
|
||||
search: "datacareer.ch",
|
||||
source: "datacareer.ch",
|
||||
});
|
||||
newOnPage++;
|
||||
}
|
||||
|
||||
console.log(`${articles.length} items, ${newOnPage} new`);
|
||||
if (articles.length < 8) break;
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
}
|
||||
|
||||
// stellen-basel.ch — WordPress REST API
|
||||
export async function fetchStellenBasel(): Promise<Job[]> {
|
||||
const allJobs: Job[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (let page = 1; page <= 30; page++) {
|
||||
if (page > 1) await Bun.sleep(400);
|
||||
process.stdout.write(` page ${page} … `);
|
||||
|
||||
const res = await fetch(
|
||||
`https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
|
||||
{ headers: { ...HEADERS, Accept: "application/json" } }
|
||||
);
|
||||
if (!res.ok) { console.log("done"); break; }
|
||||
|
||||
const data = (await res.json()) as Array<{
|
||||
id: number;
|
||||
title: { rendered: string };
|
||||
date: string;
|
||||
link: string;
|
||||
meta: { _company_name?: string; _job_location?: string };
|
||||
}>;
|
||||
|
||||
if (!data.length) { console.log("0 items"); break; }
|
||||
|
||||
let newOnPage = 0;
|
||||
for (const item of data) {
|
||||
const title = decodeHtml(item.title.rendered);
|
||||
if (!applyFilters(title)) continue;
|
||||
|
||||
const id = `sb-${item.id}`;
|
||||
if (seen.has(id)) continue;
|
||||
seen.add(id);
|
||||
|
||||
allJobs.push({
|
||||
id,
|
||||
title,
|
||||
company: item.meta._company_name ?? "",
|
||||
location: item.meta._job_location ?? "Basel",
|
||||
postedAt: item.date.slice(0, 10),
|
||||
link: item.link,
|
||||
search: "stellen-basel.ch",
|
||||
source: "stellen-basel.ch",
|
||||
});
|
||||
newOnPage++;
|
||||
}
|
||||
|
||||
console.log(`${data.length} items, ${newOnPage} new`);
|
||||
if (data.length < 100) break;
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
}
|
||||
Reference in New Issue
Block a user