From c7b86e06f46fff83209e35cc3f6e4ccec4de6d48 Mon Sep 17 00:00:00 2001 From: rafapolo Date: Thu, 10 Sep 2026 12:15:58 +0200 Subject: [PATCH] fix: back off longer on 429s in description fetch fetchDescription and isEnglishContent retried rate-limited (429) requests after only 350/700/1050ms, too short under concurrent load, so retries kept hitting 429, exhausted, and fell back to the "can't read it, so keep" default. That let an all-German CLEEVEN posting (job 4465075383) through undetected and out to Telegram. Match the 1s/2s/4s backoff already used for listing-page 429 retries. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011zgfzuFy3bAbs8GTh9apFc --- fetch.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/fetch.ts b/fetch.ts index 5fd63f9..7fc6338 100644 --- a/fetch.ts +++ b/fetch.ts @@ -275,7 +275,13 @@ export function requiresGerman(text: string): boolean { export async function fetchDescription(id: string): Promise { for (let attempt = 0; attempt < 3; attempt++) { try { - await Bun.sleep(350 * (attempt + 1)); + // 429s need the same backoff as the listing-page retry (1s/2s/4s) — the + // old 350/700/1050ms wait was too short under concurrent load, so this + // exhausted its retries while still rate-limited and fell through to the + // "can't read it, so keep" default below, letting German postings (e.g. + // CLEEVEN, job 4465075383, all-German with zero English content) slip + // past isViableContent entirely on 2026-09-10. + await Bun.sleep(1000 * 2 ** attempt); const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); const html = await res.text(); const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/); @@ -307,7 +313,8 @@ export async function isEnglishContent(id: string): Promise { // false "English" result. Back off and try again before falling back. for (let attempt = 0; attempt < 3; attempt++) { try { - await Bun.sleep(350 * (attempt + 1)); + // Same 1s/2s/4s backoff as fetchDescription — see its comment. + await Bun.sleep(1000 * 2 ** attempt); const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); const html = await res.text(); const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);