fix: stop letting unreadable or truncated descriptions slip past the German filter
isViableContent/isViableUrl defaulted to keep a job when its description
couldn't be fetched at all -- that's what let a fully-German HRM
Systems/Consult & Pepper posting ("In Deutsch bist du verhandlungssicher")
through on 2026-09-10. They now report retry:true instead, and index.ts
leaves the job out of the CSV so a fresh fetch judges it next run rather
than defaulting to keep.
job-room.ch's search API also returns a different keyword-anchored teaser
per hit for the same ad, and only the last one was kept -- a genuinely
German posting could slip through whenever the last keyword's teaser
happened to read as English. Every teaser variant is now collected and
checked together.
This commit is contained in:
+20
-3
@@ -291,6 +291,16 @@ function stripTracking(url: string): string {
|
||||
|
||||
export async function fetchJobRoom(): Promise<Job[]> {
|
||||
const ads = new Map<string, JobRoomAd>();
|
||||
// The search API returns a short keyword-anchored teaser per hit, not the
|
||||
// full posting — the same ad re-appears across keyword sweeps with a
|
||||
// different teaser each time, and `ads.set()` above only keeps the latest.
|
||||
// A genuinely German ad (Consult & Pepper's recurring "Senior Fullstack
|
||||
// Software Engineer" — full text: "In Deutsch bist du verhandlungssicher")
|
||||
// slipped through repeatedly because whichever keyword ran last happened to
|
||||
// anchor on an English-reading fragment of it. Collecting every variant and
|
||||
// checking them together means one clean-looking teaser can no longer
|
||||
// override another teaser that actually shows the German.
|
||||
const descVariants = new Map<string, Set<string>>();
|
||||
|
||||
for (const keyword of JOBROOM_KEYWORDS) {
|
||||
process.stdout.write(` ${keyword} … `);
|
||||
@@ -313,7 +323,11 @@ export async function fetchJobRoom(): Promise<Job[]> {
|
||||
|
||||
const hits = (await res.json()) as Array<{ jobAdvertisement?: JobRoomAd }>;
|
||||
for (const h of hits) {
|
||||
if (h.jobAdvertisement?.id) ads.set(h.jobAdvertisement.id, h.jobAdvertisement);
|
||||
const id = h.jobAdvertisement?.id;
|
||||
if (!id) continue;
|
||||
ads.set(id, h.jobAdvertisement!);
|
||||
const d = h.jobAdvertisement!.jobContent?.jobDescriptions?.[0]?.description;
|
||||
if (d) (descVariants.get(id) ?? descVariants.set(id, new Set()).get(id)!).add(d);
|
||||
}
|
||||
total += hits.length;
|
||||
if (hits.length < 100) break;
|
||||
@@ -341,8 +355,11 @@ export async function fetchJobRoom(): Promise<Job[]> {
|
||||
|
||||
// A handful of ads ship without a description. isViableDescription passes
|
||||
// those through by design, so leave contentChecked unset and let index.ts
|
||||
// fall back to fetching the page.
|
||||
const body = desc.description ?? "";
|
||||
// fall back to fetching the page. When we do have variants, join every
|
||||
// one seen across keyword sweeps rather than just the last teaser stored
|
||||
// on `ad` (see comment above) — more text only adds detection signal, it
|
||||
// never removes any.
|
||||
const body = [...(descVariants.get(ad.id) ?? [])].join(" \n ") || desc.description || "";
|
||||
const hasBody = body.trim().length > 40;
|
||||
if (hasBody && !isViableDescription(body).ok) { langDropped++; continue; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user