linkme/fetch-other.ts

import { NON_ENGLISH_RE, JUNIOR_TITLE_RE, SOFTWARE_TITLE_RE } from "./fetch.ts";
import type { Job } from "./fetch.ts";

const HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
  "Accept-Language": "en-US,en;q=0.9",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
};

function decodeHtml(s: string): string {
  return s
    .replace(/&/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
    .replace(/\s+/g, " ")
    .trim();
}

function applyFilters(title: string): boolean {
  if (NON_ENGLISH_RE.test(title)) return false;
  if (JUNIOR_TITLE_RE.test(title)) return false;
  if (!SOFTWARE_TITLE_RE.test(title)) return false;
  return true;
}

// datacareer.ch — HTML listing at /jobs/?city=Basel
export async function fetchDataCareer(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 1; page <= 10; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  page ${page} … `);

    const res = await fetch(
      `https://www.datacareer.ch/jobs/?city=Basel&page=${page}`,
      { headers: HEADERS }
    );
    if (!res.ok) { console.error(`failed (${res.status})`); break; }

    const html = await res.text();
    const articles = [...html.matchAll(/<article[^>]*listing-item[^>]*>(.*?)<\/article>/gs)];
    if (articles.length === 0) { console.log("0 items"); break; }

    let newOnPage = 0;
    for (const [, card] of articles) {
      const titleM = card.match(/class="link"[^>]*>([^<]+)/);
      const hrefM = card.match(/href="(https:\/\/www\.datacareer\.ch\/job\/(\d+)\/[^"]*)"\s+class="link"/);
      if (!titleM || !hrefM) continue;

      const title = decodeHtml(titleM[1]);
      if (!applyFilters(title)) continue;

      const id = `dc-${hrefM[2]}`;
      if (seen.has(id)) continue;
      seen.add(id);

      const companyM = card.match(/listing-item__info--item-company[^>]*>\s*([^<\s][^<]*)/);
      const locM = card.match(/listing-item__info--item-location[^>]*>\s*([^<\s][^<]*)/);
      const dateM = card.match(/listing-item__date[^>]*>\s*([\d\/]+)/);

      // Date comes as DD/MM/YYYY → reformat to YYYY-MM-DD
      let postedAt = "";
      const dm = dateM?.[1].match(/(\d{2})\/(\d{2})\/(\d{4})/);
      if (dm) postedAt = `${dm[3]}-${dm[2]}-${dm[1]}`;

      allJobs.push({
        id,
        title,
        company: companyM ? decodeHtml(companyM[1].trim()) : "",
        location: locM ? locM[1].trim() : "Basel",
        postedAt,
        link: hrefM[1],
        search: "datacareer.ch",
        source: "datacareer.ch",
      });
      newOnPage++;
    }

    console.log(`${articles.length} items, ${newOnPage} new`);
    if (articles.length < 8) break;
  }

  return allJobs;
}

// stellen-basel.ch — WordPress REST API
export async function fetchStellenBasel(): Promise<Job[]> {
  const allJobs: Job[] = [];
  const seen = new Set<string>();

  for (let page = 1; page <= 30; page++) {
    if (page > 1) await Bun.sleep(400);
    process.stdout.write(`  page ${page} … `);

    const res = await fetch(
      `https://www.stellen-basel.ch/wp-json/wp/v2/job-listings?per_page=100&page=${page}&_fields=id,title,date,link,meta`,
      { headers: { ...HEADERS, Accept: "application/json" } }
    );
    if (!res.ok) { console.log("done"); break; }

    const data = (await res.json()) as Array<{
      id: number;
      title: { rendered: string };
      date: string;
      link: string;
      meta: { _company_name?: string; _job_location?: string };
    }>;

    if (!data.length) { console.log("0 items"); break; }

    let newOnPage = 0;
    for (const item of data) {
      const title = decodeHtml(item.title.rendered);
      if (!applyFilters(title)) continue;

      const id = `sb-${item.id}`;
      if (seen.has(id)) continue;
      seen.add(id);

      allJobs.push({
        id,
        title,
        company: item.meta._company_name ?? "",
        location: item.meta._job_location ?? "Basel",
        postedAt: item.date.slice(0, 10),
        link: item.link,
        search: "stellen-basel.ch",
        source: "stellen-basel.ch",
      });
      newOnPage++;
    }

    console.log(`${data.length} items, ${newOnPage} new`);
    if (data.length < 100) break;
  }

  return allJobs;
}