rodado/linkme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: dedupe the digest's matched jobs across links/providers

Browse Source 
englishjobsearch.ch listed the same posting twice under different
canton pages, once as 'Swiss AviationSoftware' and once as 'Swiss-AS' --
different clickout links and slightly different company text for the
same opening, so it showed up twice in one digest. Matched jobs are now
deduped on normalized title+company, plus a title-only pass for titles
specific enough (5+ words) to make a same-source coincidental match
implausible.
This commit is contained in:
rodado
2026-09-14 12:25:06 +02:00
parent d401dd5f9a
commit 29b00cd91e
1 changed files with 33 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daily.ts
+33 -1
View File
@@ -62,7 +62,39 @@ const langDropped = await Bun.file(LANG_DROPPED)
.catch(() => 0);
// Only leads that actually match the CV (profile.ts) and haven't been sent before.
const matched = fresh.filter((r) => !sent.has(r.link) && matchesProfile(r.title, r.company).ok);
const preDedupe = fresh.filter((r) => !sent.has(r.link) && matchesProfile(r.title, r.company).ok);
// Sources can surface the same underlying opening twice under different
// links/providers — e.g. englishjobsearch.ch listing the same posting on two
// canton pages with the company rendered as "Swiss AviationSoftware" on one
// and "Swiss-AS" on the other. Exact title+company catches the common case
// (same employer, identical spelling); the title-only pass catches spelling
// drift too, but only for titles specific enough (5+ words) that a
// coincidental match between two genuinely different postings is implausible.
const normTitle = (title: string) =>
title
.replace(/[([][^)\]]*[)\]]/g, "") // drop "(all genders)", "(m/w/d)", "(a)", "80-100%" noise
.replace(/[^a-z0-9 ]/gi, " ")
.replace(/\s+/g, " ")
.trim()
.toLowerCase();
const normCompany = (company: string) =>
company
.replace(/\b(ag|gmbh|ltd|llc|inc|sa|se|plc|co)\b\.?/gi, "")
.replace(/[^a-z0-9]/gi, "")
.toLowerCase();
const seenExact = new Set<string>();
const seenTitleOnly = new Set<string>();
const matched = preDedupe.filter((r) => {
const t = normTitle(r.title);
const exactKey = `${t}|${normCompany(r.company)}`;
if (seenExact.has(exactKey)) return false;
const specific = t.split(" ").filter(Boolean).length >= 5;
if (specific && seenTitleOnly.has(t)) return false;
seenExact.add(exactKey);
if (specific) seenTitleOnly.add(t);
return true;
});
console.log(`${fresh.length} new · ${matched.length} match profile · ${fresh.length - matched.length} filtered or already sent`);
if (!matched.length) {