fix: dedupe digest against all previously-sent jobs, not just this run
This commit is contained in:
@@ -85,6 +85,22 @@ const normCompany = (company: string) =>
|
|||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
const seenExact = new Set<string>();
|
const seenExact = new Set<string>();
|
||||||
const seenTitleOnly = new Set<string>();
|
const seenTitleOnly = new Set<string>();
|
||||||
|
|
||||||
|
// Seed both sets from everything already sent (any past day), keyed the same
|
||||||
|
// way — not just today's batch. Without this, a job re-dedupes only against
|
||||||
|
// jobs sent in the *same run*: LinkedIn reposts the same opening under a new
|
||||||
|
// job ID every few weeks (Optiml, LocalBini, hexagonab, nicollcurtin, etc. —
|
||||||
|
// confirmed repeats in sent.txt across different days), each repost lands
|
||||||
|
// under a fresh link that passes the `sent.has(link)` check untouched, and it
|
||||||
|
// got renotified as if new. `all` (the full CSV) still has every past row, so
|
||||||
|
// each previously-sent link resolves back to a title+company to seed with.
|
||||||
|
for (const r of all) {
|
||||||
|
if (!sent.has(r.link)) continue;
|
||||||
|
const t = normTitle(r.title);
|
||||||
|
seenExact.add(`${t}|${normCompany(r.company)}`);
|
||||||
|
if (t.split(" ").filter(Boolean).length >= 5) seenTitleOnly.add(t);
|
||||||
|
}
|
||||||
|
|
||||||
const matched = preDedupe.filter((r) => {
|
const matched = preDedupe.filter((r) => {
|
||||||
const t = normTitle(r.title);
|
const t = normTitle(r.title);
|
||||||
const exactKey = `${t}|${normCompany(r.company)}`;
|
const exactKey = `${t}|${normCompany(r.company)}`;
|
||||||
|
|||||||
Reference in New Issue
Block a user