rodado
fbcc201e43
- move index.html/docs.json/people.json para web/ - resumo de cada documento ganha botão "ver resumo do processo", que busca e renderiza o markdown de parte1/ ou parte2/ - corrige hashJitter (módulo negativo em JS jogava pontos pra raia errada, quebrando desenho e hover) - corrige overflow horizontal no mobile (align-items:flex-start virava constraint de largura no layout empilhado) - recupera data de 366 dos 615 documentos antes sem data, lendo a linha de fechamento do texto original (dateInferred); script em pipeline/recover_dates.py - adiciona parte1/ e parte2/ (resumos dos 53 processos vinculados a Pet16704) e demais arquivos pendentes
108 lines
4.8 KiB
Python
108 lines
4.8 KiB
Python
# Recovers a "date" for docs.json entries that TIMELINE.md left undated, by
|
|
# reading each document's original text (in originals/, not committed — see
|
|
# .gitignore) and looking for its own closing signature date, e.g.
|
|
# "Brasília, 10 de abril de 2026." Reference script; not run automatically.
|
|
#
|
|
# Of the 615 originally undated documents, 366 got a recovered date this way
|
|
# (flagged `dateInferred: true` in docs.json); 249 had no recognizable
|
|
# signature date in the text and were left undated.
|
|
#
|
|
# Precision note — two looser versions were tried and rejected by manual
|
|
# sampling before landing on the rule below:
|
|
# 1. Matching the LAST "DD de mês de YYYY" anywhere in the text picked up
|
|
# legal citations phrased the same way as a signature date ("da Lei
|
|
# 11.419, de 19 de dezembro de 2006") — a citation-keyword filter cut
|
|
# most of these, but not phrasings a keyword list can't anticipate
|
|
# ("será exigido a partir de 1º de janeiro de 2017", a compliance
|
|
# deadline quoted from cited legislation, with no "Lei"/"Decreto" right
|
|
# before it).
|
|
# 2. Adding a numeric dd.mm.yyyy fallback for docs with no written-form
|
|
# date matched unrelated numbers inside long technical attachments
|
|
# (a solar-plant inspection report, a realtor's license expiry date
|
|
# inside a property appraisal) — dropped entirely, not worth the risk.
|
|
# What actually distinguishes a genuine signature date in these documents,
|
|
# reliably, regardless of how long or dense the attachment is: Brazilian
|
|
# legal documents close with "<Cidade[/UF]>, DD de mês de YYYY." — so this
|
|
# version requires that a place name and a comma sit immediately before the
|
|
# date (PLACE_RE below), and takes the LAST such occurrence. Verified
|
|
# against 30+ random samples with zero remaining false positives, including
|
|
# on multi-hundred-KB attachments that defeated the earlier heuristics.
|
|
# Even so, this is heuristic text extraction, not a curated field — treat
|
|
# recovered dates as a reasonable placement on the timeline, not as the
|
|
# authoritative date of juntada in the docket.
|
|
|
|
import json, os, re, collections
|
|
|
|
MONTHS = {
|
|
"janeiro": 1, "fevereiro": 2, "março": 3, "marco": 3, "abril": 4, "maio": 5, "junho": 6,
|
|
"julho": 7, "agosto": 8, "setembro": 9, "outubro": 10, "novembro": 11, "dezembro": 12,
|
|
}
|
|
MONTH_ALT = "|".join(MONTHS.keys())
|
|
WRITTEN_RE = re.compile(r"\b(\d{1,2})\s*(?:º|°)?\s*de\s+(" + MONTH_ALT + r")\s+de\s+(\d{4})\b", re.IGNORECASE)
|
|
|
|
CITATION_RE = re.compile(
|
|
r"(lei|decreto|resolu[çc][ãa]o|portaria|emenda|provimento|instru[çc][ãa]o\s+normativa|"
|
|
r"medida\s+provis[óo]ria|s[úu]mula|c[óo]digo|ato\s+normativo|normativa)\b", re.IGNORECASE)
|
|
CITATION_NUM_RE = re.compile(r"n[ºo°]\.?\s*[\d./-]+\s*,?\s*$", re.IGNORECASE)
|
|
|
|
# "<Cidade[/UF]>, " immediately before the date — the actual shape of a
|
|
# Brazilian legal document's closing line
|
|
PLACE_RE = re.compile(r"[A-ZÀ-Ú][A-Za-zà-ú\.]+(?:\s[A-ZÀ-Ú][A-Za-zà-ú\.]+){0,3}(?:\s?[/\-]\s?[A-Z]{2})?,\s*$")
|
|
|
|
YEAR_MIN, YEAR_MAX = 2015, 2026
|
|
|
|
|
|
def is_citation(text, start):
|
|
window = text[max(0, start - 70):start]
|
|
return bool(CITATION_RE.search(window)) or bool(CITATION_NUM_RE.search(window))
|
|
|
|
|
|
def looks_like_signature(text, start):
|
|
if is_citation(text, start):
|
|
return False
|
|
window = text[max(0, start - 45):start]
|
|
return bool(PLACE_RE.search(window))
|
|
|
|
|
|
def signature_dates(text):
|
|
out = []
|
|
for m in WRITTEN_RE.finditer(text):
|
|
day, mon, year = int(m.group(1)), MONTHS[m.group(2).lower()], int(m.group(3))
|
|
if 1 <= day <= 31 and YEAR_MIN <= year <= YEAR_MAX and looks_like_signature(text, m.start()):
|
|
out.append((m.start(), day, mon, year))
|
|
return out
|
|
|
|
|
|
def recover(docs_json_path="web/docs.json", originals_dir="originals/Parte2"):
|
|
docs = json.load(open(docs_json_path, encoding="utf-8"))
|
|
undated = [r for r in docs if not r.get("date")]
|
|
|
|
results = {}
|
|
for r in undated:
|
|
path = os.path.join(originals_dir, r["file"])
|
|
try:
|
|
text = open(path, encoding="utf-8", errors="replace").read()
|
|
except OSError:
|
|
continue
|
|
cands = signature_dates(text)
|
|
if cands:
|
|
_, day, mon, year = cands[-1]
|
|
results[r["id"]] = f"{year:04d}-{mon:02d}-{day:02d}"
|
|
return results
|
|
|
|
|
|
if __name__ == "__main__":
|
|
recovered = recover()
|
|
print(f"recovered {len(recovered)} of the originally-undated documents")
|
|
|
|
docs_path = "web/docs.json"
|
|
docs = json.load(open(docs_path, encoding="utf-8"))
|
|
for r in docs:
|
|
if not r.get("date") and r["id"] in recovered:
|
|
r["date"] = recovered[r["id"]]
|
|
r["dateInferred"] = True
|
|
json.dump(docs, open(docs_path, "w", encoding="utf-8"), ensure_ascii=False)
|
|
|
|
still_missing = sum(1 for r in docs if not r.get("date"))
|
|
print(f"still undated: {still_missing}")
|