fix: dedupe digest against all previously-sent jobs, not just this run

This commit is contained in:
2026-09-14 20:12:09 +02:00
parent 29b00cd91e
commit 9cbc525cf5
+16
View File
@@ -85,6 +85,22 @@ const normCompany = (company: string) =>
.toLowerCase(); .toLowerCase();
const seenExact = new Set<string>(); const seenExact = new Set<string>();
const seenTitleOnly = new Set<string>(); const seenTitleOnly = new Set<string>();
// Seed both sets from everything already sent (any past day), keyed the same
// way — not just today's batch. Without this, a job re-dedupes only against
// jobs sent in the *same run*: LinkedIn reposts the same opening under a new
// job ID every few weeks (Optiml, LocalBini, hexagonab, nicollcurtin, etc. —
// confirmed repeats in sent.txt across different days), each repost lands
// under a fresh link that passes the `sent.has(link)` check untouched, and it
// got renotified as if new. `all` (the full CSV) still has every past row, so
// each previously-sent link resolves back to a title+company to seed with.
for (const r of all) {
if (!sent.has(r.link)) continue;
const t = normTitle(r.title);
seenExact.add(`${t}|${normCompany(r.company)}`);
if (t.split(" ").filter(Boolean).length >= 5) seenTitleOnly.add(t);
}
const matched = preDedupe.filter((r) => { const matched = preDedupe.filter((r) => {
const t = normTitle(r.title); const t = normTitle(r.title);
const exactKey = `${t}|${normCompany(r.company)}`; const exactKey = `${t}|${normCompany(r.company)}`;