From d401dd5f9aff452826d180b57697404fd8b2bdd2 Mon Sep 17 00:00:00 2001
From: rafapolo <rafael@extrapolo.com>
Date: Mon, 14 Sep 2026 12:12:49 +0200
Subject: [PATCH] feat: dedupe the Ignored (German/French) digest count across
 runs and sources

Dropped jobs never reach the CSV, so nothing remembered them -- the same
posting got re-flagged and re-counted every single bun daily run (5x/day),
and separately every time it resurfaced under a different link/provider
(e.g. Consult & Pepper's recurring Senior Fullstack Software Engineer
reposted on job-room.ch every few days and again on LinkedIn). A persisted
results/.lang-dropped-seen.json now keys on normalized title+company so
each underlying job only counts once, ever.
---
 index.ts | 46 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/index.ts b/index.ts
index f63ccfa..8b838f2 100644
--- a/index.ts
+++ b/index.ts
@@ -62,6 +62,36 @@ const date = new Date().toISOString().slice(0, 10);
 const newRows: string[] = [];
 let totalLangDropped = 0; // jobs dropped for German/French content — never reach the CSV, so daily.ts can't see them on its own
 
+// Persisted across runs so the Telegram digest's "Ignored (German/French)"
+// count reflects genuinely new drops, not the same posting getting re-flagged
+// every run (dropped jobs never reach the CSV, so nothing else remembers
+// them) or the same underlying job counted again each time it resurfaces
+// under a different link/provider (e.g. Consult & Pepper's "Senior Fullstack
+// Software Engineer" reposted on job-room.ch every few days, and separately
+// on LinkedIn) — keyed on normalized title+company rather than the link.
+const LANG_SEEN = "./results/.lang-dropped-seen.json";
+const langSeen = new Set<string>(
+  await Bun.file(LANG_SEEN)
+    .json()
+    .catch(() => [])
+);
+function langKey(title: string, company: string): string {
+  const t = title
+    .replace(/[([][^)\]]*[)\]]/g, "") // drop "(all genders)", "(m/w/d)", "(a)", "(80 – 100%)" noise
+    .replace(/\s+/g, " ")
+    .trim()
+    .toLowerCase();
+  return `${t}|${company.trim().toLowerCase()}`;
+}
+// Only counts (and remembers) a drop the first time this title+company is
+// seen; still runs for every dropped job so `langSeen` stays complete.
+function countLangDrop(title: string, company: string): void {
+  const key = langKey(title, company);
+  if (langSeen.has(key)) return;
+  langSeen.add(key);
+  totalLangDropped++;
+}
+
 function csvCell(val: string): string {
   return `"${val.replace(/"/g, '""')}"`;
 }
@@ -115,10 +145,17 @@ async function processLinkedInSearch(config: (typeof searches)[number]): Promise
     // One fetch, two checks: written in German/French, or written in English
     // but demanding German above B1.
     const viable = await isViableContent(job.id);
+    if (viable.retry) {
+      // Couldn't read the description at all — leave it out of the CSV so
+      // next run's fresh fetch judges it properly, instead of defaulting to
+      // "keep" and risking a German-requiring posting slipping through.
+      console.log(viable.reason);
+      continue;
+    }
     if (!viable.ok) {
       console.log(`dropped (${viable.reason})`);
       langDropped++;
-      totalLangDropped++;
+      countLangDrop(job.title, job.company);
       continue;
     }
     console.log("ok");
@@ -167,10 +204,14 @@ async function processExtraSource({ label, fetch: fetchFn }: (typeof extraSource
     if (!job.contentChecked) {
       process.stdout.write(`  [${label}] checking content: ${job.title} … `);
       const viable = await isViableUrl(job.link);
+      if (viable.retry) {
+        console.log(viable.reason);
+        continue;
+      }
       if (!viable.ok) {
         console.log(`dropped (${viable.reason})`);
         langDropped++;
-        totalLangDropped++;
+        countLangDrop(job.title, job.company);
         continue;
       }
       console.log("ok");
@@ -198,6 +239,7 @@ await Promise.all([
 // without listing rejects that were never even parsed into a Row.
 const LANG_DROPPED = "./results/.lang-dropped.json";
 await Bun.write(LANG_DROPPED, JSON.stringify({ date, count: totalLangDropped }));
+await Bun.write(LANG_SEEN, JSON.stringify([...langSeen]));
 
 if (newRows.length === 0) {
   console.log("\nNo new jobs.");