fix: parse minimally-quoted csv rows when loading jobs.csv
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
// Parse one jobs.csv line. The file holds both fully-quoted rows (what
|
||||
// index.ts writes) and minimally-quoted ones (any tool that rewrites the CSV,
|
||||
// e.g. Python's csv module, only quotes fields containing a comma). Splitting
|
||||
// on '","' only understands the first form — every minimally-quoted row came
|
||||
// back with no link, so index.ts re-added old jobs as new and daily.ts
|
||||
// couldn't dedupe against them.
|
||||
export function parseCsvLine(line: string): string[] {
|
||||
const out: string[] = [];
|
||||
let cur = "";
|
||||
let quoted = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (quoted) {
|
||||
if (ch === '"' && line[i + 1] === '"') { cur += '"'; i++; }
|
||||
else if (ch === '"') quoted = false;
|
||||
else cur += ch;
|
||||
} else if (ch === '"') quoted = true;
|
||||
else if (ch === ",") { out.push(cur); cur = ""; }
|
||||
else cur += ch;
|
||||
}
|
||||
out.push(cur);
|
||||
return out;
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
// bun daily --resend skip fetching, re-send today's jobs from the CSV
|
||||
import { send, escapeHtml } from "./telegram.ts";
|
||||
import { matchesProfile } from "./profile.ts";
|
||||
import { parseCsvLine } from "./csv.ts";
|
||||
|
||||
const CSV = "./results/jobs.csv";
|
||||
const SENT = "./results/sent.txt"; // links already pushed — never resend
|
||||
@@ -20,7 +21,7 @@ function parse(text: string): Row[] {
|
||||
.slice(1)
|
||||
.filter(Boolean)
|
||||
.map((l) => {
|
||||
const c = l.replace(/^"|"$/g, "").split('","');
|
||||
const c = parseCsvLine(l);
|
||||
return { fetched: c[0], posted: c[1], search: c[2], title: c[3], company: c[4], link: c[5], source: c[6], location: c[7] ?? "" };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env bun
|
||||
import { searches } from "./config.ts";
|
||||
import { parseCsvLine } from "./csv.ts";
|
||||
import { fetchJobs, isViableContent, isViableUrl } from "./fetch.ts";
|
||||
import { fetchDataCareer, fetchStellenBasel, fetchItJobsCh, fetchJobRoom, fetchBaselIntlOrgs, fetchJobBasel, fetchEnglishJobSearch, fetchBaselJobs, fetchSciCore, fetchSwisscom, fetchNovartisBasel, fetchMoreJobs } from "./fetch-other.ts";
|
||||
|
||||
@@ -54,7 +55,7 @@ try {
|
||||
const text = await Bun.file(CSV).text();
|
||||
for (const line of text.split("\n").slice(1)) {
|
||||
if (!line.trim()) continue;
|
||||
const cells = line.replace(/^"|"$/g, "").split('","');
|
||||
const cells = parseCsvLine(line);
|
||||
const link = cells[5];
|
||||
if (link) existingLinks.add(link);
|
||||
if (cells[3]) knownJobKeys.add(langKey(cells[3], cells[4] ?? ""));
|
||||
|
||||
Reference in New Issue
Block a user