diff --git a/fetch.ts b/fetch.ts index 5fd63f9..7fc6338 100644 --- a/fetch.ts +++ b/fetch.ts @@ -275,7 +275,13 @@ export function requiresGerman(text: string): boolean { export async function fetchDescription(id: string): Promise { for (let attempt = 0; attempt < 3; attempt++) { try { - await Bun.sleep(350 * (attempt + 1)); + // 429s need the same backoff as the listing-page retry (1s/2s/4s) — the + // old 350/700/1050ms wait was too short under concurrent load, so this + // exhausted its retries while still rate-limited and fell through to the + // "can't read it, so keep" default below, letting German postings (e.g. + // CLEEVEN, job 4465075383, all-German with zero English content) slip + // past isViableContent entirely on 2026-09-10. + await Bun.sleep(1000 * 2 ** attempt); const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); const html = await res.text(); const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/); @@ -307,7 +313,8 @@ export async function isEnglishContent(id: string): Promise { // false "English" result. Back off and try again before falling back. for (let attempt = 0; attempt < 3; attempt++) { try { - await Bun.sleep(350 * (attempt + 1)); + // Same 1s/2s/4s backoff as fetchDescription — see its comment. + await Bun.sleep(1000 * 2 ** attempt); const res = await fetch(`https://www.linkedin.com/jobs/view/${id}/`, { headers: HEADERS }); const html = await res.text(); const m = html.match(/class="show-more-less-html__markup[^"]*">([\s\S]{0,8000}?)<\/div>/);