Captura integral do acervo da Brasiliana Fotográfica (Biblioteca Nacional) para s3://brasiliana: 13.060 itens, 12.985 com imagem de preservação, 13.059 com miniatura derivada, 2.137 cópias legadas e 636 posts do blog. 41.444 objetos no ledger. Os 75 itens sem preservação estão em errors.md: a página do item responde 200, mas a origem serve o bitstream com HTTP 500. Nenhum fica fora do mural, porque todos têm thumbnail. Não versionado, por viver no bucket ou ser reconstruível: catálogo, posts .jsonl, atlas .webp, sqlite de estado e logs. Credenciais ficam de fora pelo .gitignore.
559 lines
23 KiB
Python
559 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Brasiliana Fotografica -> acervo arquivistico em S3.
|
|
|
|
Coleta os 13.036 itens do DSpace da Brasiliana Fotografica e deposita, no
|
|
bucket S3, um acervo autodescritivo: metadados Dublin Core, a pagina original
|
|
como evidencia, o arquivo de preservacao, a copia legada quando existe, e um
|
|
manifesto de checksums por item.
|
|
|
|
TRANSPORTE -- por que Chrome de verdade
|
|
O Cloudflare do site barra por fingerprint TLS, nao por IP. Medido na mesma
|
|
maquina, mesmo IP, mesmo instante: Chrome 200, curl/httpx 403. E medido
|
|
de novo aqui: Playwright headless 403, Playwright headed 200.
|
|
Logo: a janela do Chrome precisa ficar ABERTA durante a coleta.
|
|
Proxy nao resolveria -- o que denuncia e o handshake, nao a origem.
|
|
|
|
QUAL E O ARQUIVO DE PRESERVACAO
|
|
O bitstream do DSpace (medido: 5533x4065 / 7,4 MB, ate 8679x6484 / 34,7 MB).
|
|
O link em objdigital.bn.br, apesar de vir no DC.identifier, e uma copia
|
|
MENOR (~2600x2100, ~330 KB): vale como procedencia, nao como master.
|
|
Excecao: alguns PDFs de album no objdigital sao outro objeto, e sao
|
|
guardados a parte.
|
|
|
|
export $(grep -v '^#' /Users/polux/Projetos/tocador/.env | xargs)
|
|
python3 censo.py discover # ~5 min enumera por comunidade
|
|
python3 censo.py items # ~7 h Dublin Core + pagina original
|
|
python3 censo.py sizes -n 300 # ~40 min amostra p/ projetar espaco
|
|
python3 censo.py fetch # o grosso: imagens -> S3
|
|
python3 censo.py push # catalogo + manifesto -> S3
|
|
python3 censo.py report
|
|
|
|
Requer: playwright (+ Chrome instalado), lxml, aws-cli
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
from lxml import html as lxhtml
|
|
|
|
# --- Identifique-se: vai no cabecalho From (RFC 9110), que existe exatamente
|
|
# --- para dizer quem opera o robo. E o que faz a instituicao escrever em vez
|
|
# --- de bloquear.
|
|
CONTACT = os.environ.get("ARCHIVE_CONTACT", "SEU-EMAIL-AQUI@exemplo.com")
|
|
|
|
CANON = "https://brasilianafotografica.bn.gov.br"
|
|
DSPACE = f"{CANON}/brasiliana"
|
|
PREFIX = "20.500.12156.1"
|
|
RPP = 100
|
|
N_MAPS = 18
|
|
|
|
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "https://hel1.your-objectstorage.com")
|
|
S3_BUCKET = os.environ.get("ARCHIVE_BUCKET", "brasiliana")
|
|
|
|
COMMUNITIES = {
|
|
"2": "Biblioteca Nacional",
|
|
"1753": "Instituto Moreira Salles",
|
|
"3540": "Arquivo Geral da Cidade do Rio de Janeiro",
|
|
"3541": "Diretoria do Patrimonio Historico e Documentacao da Marinha",
|
|
"4342": "Leibniz-Institut fuer Laenderkunde, Leipzig",
|
|
"4884": "Arquivo Nacional",
|
|
"4886": "Museu da Republica",
|
|
"5018": "Fiocruz",
|
|
"6142": "Museu Historico Nacional",
|
|
"6660": "Fundacao Joaquim Nabuco",
|
|
"7072": "Museu Aeroespacial",
|
|
"11464": "Escola de Ciencias Sociais (FGV CPDOC)",
|
|
"12796": "Instituto Historico e Geografico Brasileiro",
|
|
"13099": "Instituto Brasileiro de Geografia e Estatistica",
|
|
"14087": "Museu Paraense Emilio Goeldi",
|
|
}
|
|
ROOT_COMMUNITY = "1"
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
DB_PATH = ROOT / "censo.sqlite"
|
|
ITEMS_DIR = ROOT / "itens"
|
|
PROFILE_DIR = ROOT / ".chrome-profile"
|
|
STAGE_DIR = ROOT / ".stage" # buffer local; esvaziado apos subir
|
|
|
|
DELAYS = {"brasilianafotografica.bn.gov.br": 2.0, "objdigital.bn.br": 0.7}
|
|
DEFAULT_DELAY = 2.0
|
|
|
|
HANDLE_RE = re.compile(rf"handle/{re.escape(PREFIX)}/(\d+)")
|
|
TOTAL_RE = re.compile(
|
|
r"(?:no momento|currently)\D{0,24}[\d.,]+\s*-\s*[\d.,]+\s*(?:de|of)\s*([\d.,]+)", re.I)
|
|
|
|
|
|
def canon(url):
|
|
"""Normaliza para o host canonico (os sitemaps emitem o dominio legado)."""
|
|
if not url:
|
|
return url
|
|
p = urlparse(url if "://" in url else "https://" + url)
|
|
h = p.netloc.lower().removeprefix("www.")
|
|
if h in ("brasilianafotografica.bn.br", "brasilianafotografica.bn.gov.br"):
|
|
p = p._replace(scheme="https", netloc="brasilianafotografica.bn.gov.br")
|
|
elif h == "objdigital.bn.br":
|
|
p = p._replace(scheme="https", netloc="objdigital.bn.br")
|
|
return urlunparse(p)
|
|
|
|
|
|
class Fetcher:
|
|
"""Chrome real, uma requisicao por vez, com espera por host e backoff."""
|
|
|
|
def __init__(self, headless=False):
|
|
from playwright.sync_api import sync_playwright
|
|
self._pw = sync_playwright().start()
|
|
self.ctx = self._pw.chromium.launch_persistent_context(
|
|
user_data_dir=str(PROFILE_DIR), channel="chrome", headless=headless,
|
|
viewport={"width": 1280, "height": 900},
|
|
extra_http_headers={"From": CONTACT})
|
|
self.page = self.ctx.pages[0] if self.ctx.pages else self.ctx.new_page()
|
|
self.last = {}
|
|
|
|
def _wait(self, host):
|
|
delay = DELAYS.get(host, DEFAULT_DELAY)
|
|
prev = self.last.get(host)
|
|
if prev is not None and (gap := time.monotonic() - prev) < delay:
|
|
time.sleep(delay - gap)
|
|
self.last[host] = time.monotonic()
|
|
|
|
def _goto(self, url, tries=4):
|
|
host = urlparse(url).netloc.lower()
|
|
backoff = 6.0
|
|
r = None
|
|
for attempt in range(1, tries + 1):
|
|
self._wait(host)
|
|
try:
|
|
r = self.page.goto(url, wait_until="domcontentloaded", timeout=120000)
|
|
except Exception as e:
|
|
if attempt == tries:
|
|
raise
|
|
print(f" ! {type(e).__name__}, espera {backoff:.0f}s", flush=True)
|
|
time.sleep(backoff); backoff *= 2
|
|
continue
|
|
if r is not None and r.status in (403, 429, 500, 502, 503, 504):
|
|
if attempt == tries:
|
|
return r
|
|
print(f" ! HTTP {r.status}, espera {backoff:.0f}s", flush=True)
|
|
time.sleep(backoff); backoff *= 2
|
|
continue
|
|
return r
|
|
return r
|
|
|
|
def text(self, url):
|
|
r = self._goto(url)
|
|
if r is None:
|
|
return 0, ""
|
|
return r.status, (self.page.content() if r.ok else "")
|
|
|
|
def blob(self, url):
|
|
r = self._goto(url)
|
|
if r is None:
|
|
return 0, b""
|
|
try:
|
|
return r.status, (r.body() if r.ok else b"")
|
|
except Exception:
|
|
return r.status, b""
|
|
|
|
def close(self):
|
|
try:
|
|
self.ctx.close()
|
|
finally:
|
|
self._pw.stop()
|
|
|
|
|
|
# ---------------------------------------------------------------- S3
|
|
|
|
def s3(*args, capture=False):
|
|
cmd = ["aws", "--endpoint-url", S3_ENDPOINT, "s3", *args]
|
|
if capture:
|
|
return subprocess.run(cmd, capture_output=True, text=True).stdout
|
|
return subprocess.run(cmd, capture_output=True, text=True).returncode == 0
|
|
|
|
|
|
def s3_put(local, key):
|
|
return s3("cp", str(local), f"s3://{S3_BUCKET}/{key}", "--only-show-errors")
|
|
|
|
|
|
# ---------------------------------------------------------------- estado
|
|
|
|
def db_connect():
|
|
con = sqlite3.connect(DB_PATH)
|
|
con.execute("PRAGMA journal_mode=WAL")
|
|
con.executescript("""
|
|
CREATE TABLE IF NOT EXISTS items (
|
|
handle TEXT PRIMARY KEY,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
http_status INTEGER, title TEXT, community TEXT,
|
|
bitstream_url TEXT, bitstream_bytes INTEGER, bitstream_sha TEXT,
|
|
legacy_url TEXT, legacy_bytes INTEGER, legacy_sha TEXT,
|
|
rights TEXT, source TEXT, fetched_at TEXT,
|
|
stored INTEGER DEFAULT 0, error TEXT);
|
|
CREATE TABLE IF NOT EXISTS meta (handle TEXT, name TEXT, value TEXT);
|
|
CREATE INDEX IF NOT EXISTS meta_handle ON meta(handle);
|
|
CREATE INDEX IF NOT EXISTS items_status ON items(status);
|
|
CREATE INDEX IF NOT EXISTS items_stored ON items(stored);
|
|
""")
|
|
return con
|
|
|
|
|
|
def browse_page(f, scope, offset):
|
|
base = f"{DSPACE}/handle/{PREFIX}/{scope}/browse" if scope else f"{DSPACE}/browse"
|
|
return f.text(f"{base}?type=title&rpp={RPP}&offset={offset}")
|
|
|
|
|
|
# ---------------------------------------------------------------- comandos
|
|
|
|
def cmd_discover(args):
|
|
"""Enumera comunidade por comunidade.
|
|
|
|
O htmlmap/sitemap do DSpace e de 2019 e lista ~5.300 de 13.036 itens; usa-lo
|
|
como fonte perderia 59% do acervo sem erro aparente. O indice de browse e
|
|
vivo, e percorre-lo por comunidade ja etiqueta a instituicao de origem.
|
|
"""
|
|
con, f = db_connect(), Fetcher(args.headless)
|
|
try:
|
|
for handle, name in COMMUNITIES.items():
|
|
st, html = browse_page(f, handle, 0)
|
|
if st != 200:
|
|
print(f" {name}: HTTP {st} -- pulando", flush=True)
|
|
continue
|
|
m = TOTAL_RE.search(lxhtml.fromstring(html).text_content())
|
|
total = int(re.sub(r"[.,]", "", m.group(1))) if m else None
|
|
|
|
seen, offset = set(), 0
|
|
while True:
|
|
if offset:
|
|
st, html = browse_page(f, handle, offset)
|
|
if st != 200:
|
|
break
|
|
hs = set(HANDLE_RE.findall(html)) - {handle}
|
|
if not (hs - seen) and offset:
|
|
break
|
|
seen |= hs
|
|
con.executemany(
|
|
"INSERT INTO items(handle,community,source) VALUES (?,?,'browse') "
|
|
"ON CONFLICT(handle) DO UPDATE SET community=excluded.community",
|
|
[(f"{PREFIX}/{h}", name) for h in hs])
|
|
con.commit()
|
|
offset += RPP
|
|
if total and offset >= total:
|
|
break
|
|
print(f" {name[:46]:<46} {len(seen):>6}"
|
|
+ (f" (indice diz {total})" if total and total != len(seen) else ""),
|
|
flush=True)
|
|
|
|
got = con.execute("SELECT COUNT(*) FROM items").fetchone()[0]
|
|
st, html = browse_page(f, ROOT_COMMUNITY, 0)
|
|
m = TOTAL_RE.search(lxhtml.fromstring(html).text_content()) if st == 200 else None
|
|
print(f"\n soma das comunidades: {got}")
|
|
if m:
|
|
root = int(re.sub(r"[.,]", "", m.group(1)))
|
|
print(f" raiz declara .......: {root}")
|
|
if root != got:
|
|
print(f" >> divergencia de {abs(root-got)}; varrendo indice global")
|
|
off = 0
|
|
while off < root:
|
|
st, html = browse_page(f, None, off)
|
|
if st != 200:
|
|
break
|
|
con.executemany(
|
|
"INSERT OR IGNORE INTO items(handle,source) VALUES (?,'global')",
|
|
[(f"{PREFIX}/{h}",) for h in set(HANDLE_RE.findall(html))])
|
|
con.commit()
|
|
off += RPP
|
|
n = con.execute("SELECT COUNT(*) FROM items").fetchone()[0]
|
|
print(f"\nAcervo enumerado: {n} itens")
|
|
finally:
|
|
f.close()
|
|
|
|
|
|
def cmd_items(args):
|
|
"""Colhe Dublin Core + guarda a pagina original como evidencia."""
|
|
con, f = db_connect(), Fetcher(args.headless)
|
|
ITEMS_DIR.mkdir(exist_ok=True)
|
|
rows = con.execute("SELECT handle FROM items WHERE status IN ('pending','error') "
|
|
"ORDER BY CAST(substr(handle,instr(handle,'/')+1) AS INTEGER)").fetchall()
|
|
if args.limit:
|
|
rows = rows[: args.limit]
|
|
print(f"{len(rows)} itens (~{len(rows)*2.0/3600:.1f} h)\n")
|
|
try:
|
|
for i, (handle,) in enumerate(rows, 1):
|
|
url = f"{DSPACE}/handle/{handle}?show=full"
|
|
try:
|
|
st, html = f.text(url)
|
|
except Exception as e:
|
|
con.execute("UPDATE items SET status='error', error=? WHERE handle=?",
|
|
(str(e)[:200], handle)); con.commit()
|
|
continue
|
|
if st != 200:
|
|
con.execute("UPDATE items SET status='error', http_status=?, error=? "
|
|
"WHERE handle=?", (st, f"HTTP {st}", handle)); con.commit()
|
|
print(f"[{i}/{len(rows)}] {handle}: HTTP {st}", flush=True)
|
|
continue
|
|
|
|
doc = lxhtml.fromstring(html)
|
|
dc = [(e.get("name"), e.get("content")) for e in doc.xpath("//meta[@name and @content]")
|
|
if e.get("name").startswith(("DC", "citation"))]
|
|
table = [(c[0].text_content().strip(), c[1].text_content().strip())
|
|
for c in (r.xpath("./td") for r in
|
|
doc.xpath("//table[contains(@class,'itemDisplayTable')]//tr"))
|
|
if len(c) >= 2]
|
|
title = next((v for k, v in dc if k == "DC.title"), None)
|
|
bit = next((canon(v) for k, v in dc if k == "citation_pdf_url"), None)
|
|
leg = next((canon(v) for k, v in dc
|
|
if k == "DC.identifier" and v and "objdigital" in v), None)
|
|
rights = next((v for k, v in table if k.lower().startswith("dc.rights")), None)
|
|
|
|
d = ITEMS_DIR / handle.replace("/", "_")
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
(d / "item.html").write_text(html, encoding="utf-8")
|
|
(d / "metadata.json").write_text(json.dumps({
|
|
"handle": handle, "source_url": url,
|
|
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"title": title,
|
|
"community": con.execute("SELECT community FROM items WHERE handle=?",
|
|
(handle,)).fetchone()[0],
|
|
"preservation_file": bit, "legacy_copy": leg, "rights": rights,
|
|
"dublin_core": [{"name": k, "value": v} for k, v in dc],
|
|
"full_table": [{"field": k, "value": v} for k, v in table],
|
|
"sha256_item_html": hashlib.sha256(html.encode()).hexdigest(),
|
|
}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
con.execute("DELETE FROM meta WHERE handle=?", (handle,))
|
|
con.executemany("INSERT INTO meta(handle,name,value) VALUES (?,?,?)",
|
|
[(handle, k, v) for k, v in dc])
|
|
con.execute("""UPDATE items SET status='done', http_status=?, title=?,
|
|
bitstream_url=?, legacy_url=?, rights=?, fetched_at=?, error=NULL
|
|
WHERE handle=?""",
|
|
(st, title, bit, leg, rights,
|
|
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), handle))
|
|
con.commit()
|
|
print(f"[{i}/{len(rows)}] {handle.split('/')[-1]} "
|
|
f"[{'L' if leg else '-'}] {(title or '')[:52]}", flush=True)
|
|
finally:
|
|
f.close()
|
|
|
|
|
|
def _grab(f, con, handle, url, kind, key_dir):
|
|
"""Baixa um binario, confere sha256, sobe pro S3 e apaga o local."""
|
|
st, data = f.blob(url)
|
|
if st != 200 or not data:
|
|
return None, None, st
|
|
STAGE_DIR.mkdir(exist_ok=True)
|
|
name = url.split("/")[-1] or f"{kind}.bin"
|
|
tmp = STAGE_DIR / name
|
|
tmp.write_bytes(data)
|
|
sha = hashlib.sha256(data).hexdigest()
|
|
ok = s3_put(tmp, f"itens/{handle.replace('/','_')}/{key_dir}/{name}")
|
|
tmp.unlink(missing_ok=True)
|
|
return (len(data), sha, st) if ok else (None, None, -1)
|
|
|
|
|
|
def cmd_fetch(args):
|
|
"""Baixa preservacao (+ copia legada) e deposita no bucket."""
|
|
con, f = db_connect(), Fetcher(args.headless)
|
|
rows = con.execute("SELECT handle,bitstream_url,legacy_url FROM items "
|
|
"WHERE status='done' AND bitstream_url IS NOT NULL AND stored=0"
|
|
).fetchall()
|
|
if args.limit:
|
|
rows = rows[: args.limit]
|
|
print(f"{len(rows)} itens a depositar em s3://{S3_BUCKET}/\n")
|
|
try:
|
|
for i, (handle, bit, leg) in enumerate(rows, 1):
|
|
d = ITEMS_DIR / handle.replace("/", "_")
|
|
nb, shab, st = _grab(f, con, handle, bit, "preservacao", "preservacao")
|
|
if nb is None:
|
|
print(f"[{i}/{len(rows)}] {handle}: falhou preservacao (HTTP {st})", flush=True)
|
|
con.execute("UPDATE items SET error=? WHERE handle=?",
|
|
(f"preservacao HTTP {st}", handle)); con.commit()
|
|
continue
|
|
nl = shal = None
|
|
if leg and not args.skip_legacy:
|
|
nl, shal, _ = _grab(f, con, handle, leg, "legado", "legado")
|
|
|
|
lines = [f"{shab} preservacao/{bit.split('/')[-1]}"]
|
|
if shal:
|
|
lines.append(f"{shal} legado/{leg.split('/')[-1]}")
|
|
for fn in ("metadata.json", "item.html"):
|
|
p = d / fn
|
|
if p.exists():
|
|
lines.append(f"{hashlib.sha256(p.read_bytes()).hexdigest()} {fn}")
|
|
s3_put(p, f"itens/{handle.replace('/','_')}/{fn}")
|
|
man = d / "manifest-sha256.txt"
|
|
man.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
s3_put(man, f"itens/{handle.replace('/','_')}/manifest-sha256.txt")
|
|
|
|
con.execute("""UPDATE items SET bitstream_bytes=?, bitstream_sha=?,
|
|
legacy_bytes=?, legacy_sha=?, stored=1 WHERE handle=?""",
|
|
(nb, shab, nl, shal, handle))
|
|
con.commit()
|
|
print(f"[{i}/{len(rows)}] {handle.split('/')[-1]} "
|
|
f"{nb/1048576:.1f} MB{' +legado' if nl else ''}", flush=True)
|
|
finally:
|
|
f.close()
|
|
|
|
|
|
def cmd_sizes(args):
|
|
"""Amostra o peso dos bitstreams para projetar o espaco total."""
|
|
con, f = db_connect(), Fetcher(args.headless)
|
|
rows = con.execute("SELECT handle,bitstream_url FROM items WHERE bitstream_url IS NOT NULL "
|
|
"AND bitstream_bytes IS NULL ORDER BY RANDOM() LIMIT ?",
|
|
(args.n,)).fetchall()
|
|
print(f"{len(rows)} bitstreams a medir\n")
|
|
try:
|
|
for i, (handle, url) in enumerate(rows, 1):
|
|
st, data = f.blob(url)
|
|
if st == 200 and data:
|
|
con.execute("UPDATE items SET bitstream_bytes=? WHERE handle=?",
|
|
(len(data), handle))
|
|
con.commit()
|
|
if i % 10 == 0 or i == len(rows):
|
|
print(f" {i}/{len(rows)}", flush=True)
|
|
finally:
|
|
f.close()
|
|
|
|
|
|
def cmd_push(args):
|
|
"""Sobe catalogo e manifesto geral -- o que torna o acervo autodescritivo."""
|
|
con = db_connect()
|
|
STAGE_DIR.mkdir(exist_ok=True)
|
|
cat = STAGE_DIR / "catalogo.jsonl"
|
|
n = 0
|
|
with cat.open("w", encoding="utf-8") as fh:
|
|
for r in con.execute("""SELECT handle,title,community,rights,bitstream_url,
|
|
bitstream_bytes,bitstream_sha,legacy_url,legacy_sha,fetched_at
|
|
FROM items WHERE status='done' ORDER BY handle"""):
|
|
fh.write(json.dumps(dict(zip(
|
|
"handle title community rights preservation_url preservation_bytes "
|
|
"preservation_sha256 legacy_url legacy_sha256 fetched_at".split(), r)),
|
|
ensure_ascii=False) + "\n")
|
|
n += 1
|
|
total, stored, byt = con.execute(
|
|
"SELECT COUNT(*), SUM(stored), COALESCE(SUM(bitstream_bytes),0) FROM items").fetchone()
|
|
|
|
man = STAGE_DIR / "MANIFEST.md"
|
|
man.write_text(f"""# Brasiliana Fotografica -- copia de arquivo
|
|
|
|
Colhido de <{DSPACE}> em {time.strftime('%Y-%m-%d', time.gmtime())}.
|
|
Repositorio de origem: DSpace 4.2, handles com prefixo `{PREFIX}`.
|
|
Acervo consorciado de 15 instituicoes brasileiras (e o Leibniz-Institut, Leipzig).
|
|
|
|
## Numeros
|
|
- Itens enumerados: {total}
|
|
- Itens depositados: {stored or 0}
|
|
- Bytes de preservacao: {byt}
|
|
|
|
## Estrutura
|
|
itens/<handle>/metadata.json Dublin Core completo
|
|
itens/<handle>/item.html pagina original, como evidencia da fonte
|
|
itens/<handle>/preservacao/<arq> arquivo de preservacao (bitstream DSpace)
|
|
itens/<handle>/legado/<arq> copia menor do objdigital, quando existe
|
|
itens/<handle>/manifest-sha256.txt checksum de cada arquivo do item
|
|
catalogo.jsonl uma linha por item
|
|
|
|
## Procedencia
|
|
`metadata.json` registra a URL de origem e a data/hora UTC de cada captura.
|
|
Todo arquivo tem SHA-256 no manifesto do item.
|
|
|
|
## Nota sobre qualidade
|
|
O arquivo de preservacao e o bitstream do DSpace (medidos entre 2 e 35 MB,
|
|
ate 8679x6484 px). O link para `objdigital.bn.br` que aparece no campo
|
|
`DC.identifier` e uma copia MENOR (~2600x2100, ~330 KB), guardada em
|
|
`legado/` apenas como referencia de procedencia.
|
|
|
|
## Direitos
|
|
Acervo majoritariamente em dominio publico por antiguidade, mas as instituicoes
|
|
participantes tem politicas distintas. O campo `rights` e a instituicao de
|
|
origem estao preservados por item: a decisao de redistribuir e item a item.
|
|
""", encoding="utf-8")
|
|
|
|
ok1 = s3_put(cat, "catalogo.jsonl")
|
|
ok2 = s3_put(man, "MANIFEST.md")
|
|
print(f"catalogo.jsonl ({n} linhas): {'ok' if ok1 else 'FALHOU'}")
|
|
print(f"MANIFEST.md: {'ok' if ok2 else 'FALHOU'}")
|
|
print(f"metadados dos itens -> sync")
|
|
subprocess.run(["aws", "--endpoint-url", S3_ENDPOINT, "s3", "sync", str(ITEMS_DIR),
|
|
f"s3://{S3_BUCKET}/itens/", "--exclude", "*", "--include", "*.json",
|
|
"--include", "*.html", "--include", "*.txt", "--only-show-errors"])
|
|
print("pronto")
|
|
|
|
|
|
def human(n):
|
|
if not n:
|
|
return "?"
|
|
for u in ("B", "KB", "MB", "GB", "TB"):
|
|
if n < 1024:
|
|
return f"{n:.1f} {u}"
|
|
n /= 1024
|
|
return f"{n:.1f} PB"
|
|
|
|
|
|
def cmd_report(args):
|
|
con = db_connect()
|
|
q = lambda s, *a: con.execute(s, a).fetchone()[0]
|
|
total = q("SELECT COUNT(*) FROM items")
|
|
done = q("SELECT COUNT(*) FROM items WHERE status='done'")
|
|
stored = q("SELECT COALESCE(SUM(stored),0) FROM items")
|
|
err = q("SELECT COUNT(*) FROM items WHERE status='error'")
|
|
print(f"\n{'='*64}\n BRASILIANA FOTOGRAFICA -- s3://{S3_BUCKET}\n{'='*64}")
|
|
print(f" Enumerados ........ {total}")
|
|
print(f" Metadados ......... {done} (erros: {err})")
|
|
print(f" Depositados no S3 . {stored}")
|
|
n, avg, tot = con.execute("SELECT COUNT(*), AVG(bitstream_bytes), SUM(bitstream_bytes) "
|
|
"FROM items WHERE bitstream_bytes IS NOT NULL").fetchone()
|
|
if n:
|
|
print(f"\n Medidos ........... {n} media {human(avg)}")
|
|
print(f" Baixado ........... {human(tot)}")
|
|
if done:
|
|
print(f" Projecao total .... {human(avg*done)}")
|
|
rows = con.execute("SELECT COALESCE(community,'(sem)'), COUNT(*), SUM(stored) "
|
|
"FROM items GROUP BY 1 ORDER BY 2 DESC").fetchall()
|
|
if rows:
|
|
print(f"\n Por instituicao:\n {'-'*60}")
|
|
for nm, c, s_ in rows:
|
|
print(f" {nm[:44]:<44} {c:>6} ({s_ or 0} no S3)")
|
|
print()
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--headless", action="store_true",
|
|
help="NAO use: medido, o Cloudflare devolve 403 em headless")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
sub.add_parser("discover")
|
|
p = sub.add_parser("items"); p.add_argument("--limit", type=int, default=0)
|
|
p = sub.add_parser("fetch"); p.add_argument("--limit", type=int, default=0)
|
|
p.add_argument("--skip-legacy", action="store_true")
|
|
p = sub.add_parser("sizes"); p.add_argument("-n", type=int, default=300)
|
|
sub.add_parser("push")
|
|
sub.add_parser("report")
|
|
args = ap.parse_args()
|
|
|
|
if args.cmd != "report":
|
|
if CONTACT.startswith("SEU-EMAIL"):
|
|
sys.exit("Defina ARCHIVE_CONTACT com um e-mail real (vai no cabecalho From).")
|
|
if not os.environ.get("AWS_ACCESS_KEY_ID"):
|
|
sys.exit("Credenciais S3 ausentes. Rode:\n"
|
|
" export $(grep -v '^#' /Users/polux/Projetos/tocador/.env | xargs)")
|
|
|
|
{"discover": cmd_discover, "items": cmd_items, "fetch": cmd_fetch,
|
|
"sizes": cmd_sizes, "push": cmd_push, "report": cmd_report}[args.cmd](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print("\nInterrompido. Estado salvo -- rode o mesmo comando para continuar.")
|