fix: dedupe digest against all previously-sent jobs, not just this run
This commit is contained in:
@@ -85,6 +85,22 @@ const normCompany = (company: string) =>
|
||||
.toLowerCase();
|
||||
const seenExact = new Set<string>();
|
||||
const seenTitleOnly = new Set<string>();
|
||||
|
||||
// Seed both sets from everything already sent (any past day), keyed the same
|
||||
// way — not just today's batch. Without this, a job re-dedupes only against
|
||||
// jobs sent in the *same run*: LinkedIn reposts the same opening under a new
|
||||
// job ID every few weeks (Optiml, LocalBini, hexagonab, nicollcurtin, etc. —
|
||||
// confirmed repeats in sent.txt across different days), each repost lands
|
||||
// under a fresh link that passes the `sent.has(link)` check untouched, and it
|
||||
// got renotified as if new. `all` (the full CSV) still has every past row, so
|
||||
// each previously-sent link resolves back to a title+company to seed with.
|
||||
for (const r of all) {
|
||||
if (!sent.has(r.link)) continue;
|
||||
const t = normTitle(r.title);
|
||||
seenExact.add(`${t}|${normCompany(r.company)}`);
|
||||
if (t.split(" ").filter(Boolean).length >= 5) seenTitleOnly.add(t);
|
||||
}
|
||||
|
||||
const matched = preDedupe.filter((r) => {
|
||||
const t = normTitle(r.title);
|
||||
const exactKey = `${t}|${normCompany(r.company)}`;
|
||||
|
||||
Reference in New Issue
Block a user