From 29b00cd91e13ed04d83302c7fa5bb10957533df1 Mon Sep 17 00:00:00 2001
From: rafapolo <rafael@extrapolo.com>
Date: Mon, 14 Sep 2026 12:25:06 +0200
Subject: [PATCH] fix: dedupe the digest's matched jobs across links/providers

englishjobsearch.ch listed the same posting twice under different
canton pages, once as 'Swiss AviationSoftware' and once as 'Swiss-AS' --
different clickout links and slightly different company text for the
same opening, so it showed up twice in one digest. Matched jobs are now
deduped on normalized title+company, plus a title-only pass for titles
specific enough (5+ words) to make a same-source coincidental match
implausible.
---
 daily.ts | 34 +++++++++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/daily.ts b/daily.ts
index 0f6b740..86a7545 100644
--- a/daily.ts
+++ b/daily.ts
@@ -62,7 +62,39 @@ const langDropped = await Bun.file(LANG_DROPPED)
   .catch(() => 0);
 
 // Only leads that actually match the CV (profile.ts) and haven't been sent before.
-const matched = fresh.filter((r) => !sent.has(r.link) && matchesProfile(r.title, r.company).ok);
+const preDedupe = fresh.filter((r) => !sent.has(r.link) && matchesProfile(r.title, r.company).ok);
+
+// Sources can surface the same underlying opening twice under different
+// links/providers — e.g. englishjobsearch.ch listing the same posting on two
+// canton pages with the company rendered as "Swiss AviationSoftware" on one
+// and "Swiss-AS" on the other. Exact title+company catches the common case
+// (same employer, identical spelling); the title-only pass catches spelling
+// drift too, but only for titles specific enough (5+ words) that a
+// coincidental match between two genuinely different postings is implausible.
+const normTitle = (title: string) =>
+  title
+    .replace(/[([][^)\]]*[)\]]/g, "") // drop "(all genders)", "(m/w/d)", "(a)", "80-100%" noise
+    .replace(/[^a-z0-9 ]/gi, " ")
+    .replace(/\s+/g, " ")
+    .trim()
+    .toLowerCase();
+const normCompany = (company: string) =>
+  company
+    .replace(/\b(ag|gmbh|ltd|llc|inc|sa|se|plc|co)\b\.?/gi, "")
+    .replace(/[^a-z0-9]/gi, "")
+    .toLowerCase();
+const seenExact = new Set<string>();
+const seenTitleOnly = new Set<string>();
+const matched = preDedupe.filter((r) => {
+  const t = normTitle(r.title);
+  const exactKey = `${t}|${normCompany(r.company)}`;
+  if (seenExact.has(exactKey)) return false;
+  const specific = t.split(" ").filter(Boolean).length >= 5;
+  if (specific && seenTitleOnly.has(t)) return false;
+  seenExact.add(exactKey);
+  if (specific) seenTitleOnly.add(t);
+  return true;
+});
 console.log(`${fresh.length} new · ${matched.length} match profile · ${fresh.length - matched.length} filtered or already sent`);
 
 if (!matched.length) {