feat: site em web/, resumo de processo ao clicar, autos vinculados e recuperação de datas
- move index.html/docs.json/people.json para web/ - resumo de cada documento ganha botão "ver resumo do processo", que busca e renderiza o markdown de parte1/ ou parte2/ - corrige hashJitter (módulo negativo em JS jogava pontos pra raia errada, quebrando desenho e hover) - corrige overflow horizontal no mobile (align-items:flex-start virava constraint de largura no layout empilhado) - recupera data de 366 dos 615 documentos antes sem data, lendo a linha de fechamento do texto original (dateInferred); script em pipeline/recover_dates.py - adiciona parte1/ e parte2/ (resumos dos 53 processos vinculados a Pet16704) e demais arquivos pendentes
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
# Recovers a "date" for docs.json entries that TIMELINE.md left undated, by
|
||||
# reading each document's original text (in originals/, not committed — see
|
||||
# .gitignore) and looking for its own closing signature date, e.g.
|
||||
# "Brasília, 10 de abril de 2026." Reference script; not run automatically.
|
||||
#
|
||||
# Of the 615 originally undated documents, 366 got a recovered date this way
|
||||
# (flagged `dateInferred: true` in docs.json); 249 had no recognizable
|
||||
# signature date in the text and were left undated.
|
||||
#
|
||||
# Precision note — two looser versions were tried and rejected by manual
|
||||
# sampling before landing on the rule below:
|
||||
# 1. Matching the LAST "DD de mês de YYYY" anywhere in the text picked up
|
||||
# legal citations phrased the same way as a signature date ("da Lei
|
||||
# 11.419, de 19 de dezembro de 2006") — a citation-keyword filter cut
|
||||
# most of these, but not phrasings a keyword list can't anticipate
|
||||
# ("será exigido a partir de 1º de janeiro de 2017", a compliance
|
||||
# deadline quoted from cited legislation, with no "Lei"/"Decreto" right
|
||||
# before it).
|
||||
# 2. Adding a numeric dd.mm.yyyy fallback for docs with no written-form
|
||||
# date matched unrelated numbers inside long technical attachments
|
||||
# (a solar-plant inspection report, a realtor's license expiry date
|
||||
# inside a property appraisal) — dropped entirely, not worth the risk.
|
||||
# What actually distinguishes a genuine signature date in these documents,
|
||||
# reliably, regardless of how long or dense the attachment is: Brazilian
|
||||
# legal documents close with "<Cidade[/UF]>, DD de mês de YYYY." — so this
|
||||
# version requires that a place name and a comma sit immediately before the
|
||||
# date (PLACE_RE below), and takes the LAST such occurrence. Verified
|
||||
# against 30+ random samples with zero remaining false positives, including
|
||||
# on multi-hundred-KB attachments that defeated the earlier heuristics.
|
||||
# Even so, this is heuristic text extraction, not a curated field — treat
|
||||
# recovered dates as a reasonable placement on the timeline, not as the
|
||||
# authoritative date of juntada in the docket.
|
||||
|
||||
import json, os, re, collections
|
||||
|
||||
MONTHS = {
|
||||
"janeiro": 1, "fevereiro": 2, "março": 3, "marco": 3, "abril": 4, "maio": 5, "junho": 6,
|
||||
"julho": 7, "agosto": 8, "setembro": 9, "outubro": 10, "novembro": 11, "dezembro": 12,
|
||||
}
|
||||
MONTH_ALT = "|".join(MONTHS.keys())
|
||||
WRITTEN_RE = re.compile(r"\b(\d{1,2})\s*(?:º|°)?\s*de\s+(" + MONTH_ALT + r")\s+de\s+(\d{4})\b", re.IGNORECASE)
|
||||
|
||||
CITATION_RE = re.compile(
|
||||
r"(lei|decreto|resolu[çc][ãa]o|portaria|emenda|provimento|instru[çc][ãa]o\s+normativa|"
|
||||
r"medida\s+provis[óo]ria|s[úu]mula|c[óo]digo|ato\s+normativo|normativa)\b", re.IGNORECASE)
|
||||
CITATION_NUM_RE = re.compile(r"n[ºo°]\.?\s*[\d./-]+\s*,?\s*$", re.IGNORECASE)
|
||||
|
||||
# "<Cidade[/UF]>, " immediately before the date — the actual shape of a
|
||||
# Brazilian legal document's closing line
|
||||
PLACE_RE = re.compile(r"[A-ZÀ-Ú][A-Za-zà-ú\.]+(?:\s[A-ZÀ-Ú][A-Za-zà-ú\.]+){0,3}(?:\s?[/\-]\s?[A-Z]{2})?,\s*$")
|
||||
|
||||
YEAR_MIN, YEAR_MAX = 2015, 2026
|
||||
|
||||
|
||||
def is_citation(text, start):
|
||||
window = text[max(0, start - 70):start]
|
||||
return bool(CITATION_RE.search(window)) or bool(CITATION_NUM_RE.search(window))
|
||||
|
||||
|
||||
def looks_like_signature(text, start):
|
||||
if is_citation(text, start):
|
||||
return False
|
||||
window = text[max(0, start - 45):start]
|
||||
return bool(PLACE_RE.search(window))
|
||||
|
||||
|
||||
def signature_dates(text):
|
||||
out = []
|
||||
for m in WRITTEN_RE.finditer(text):
|
||||
day, mon, year = int(m.group(1)), MONTHS[m.group(2).lower()], int(m.group(3))
|
||||
if 1 <= day <= 31 and YEAR_MIN <= year <= YEAR_MAX and looks_like_signature(text, m.start()):
|
||||
out.append((m.start(), day, mon, year))
|
||||
return out
|
||||
|
||||
|
||||
def recover(docs_json_path="web/docs.json", originals_dir="originals/Parte2"):
|
||||
docs = json.load(open(docs_json_path, encoding="utf-8"))
|
||||
undated = [r for r in docs if not r.get("date")]
|
||||
|
||||
results = {}
|
||||
for r in undated:
|
||||
path = os.path.join(originals_dir, r["file"])
|
||||
try:
|
||||
text = open(path, encoding="utf-8", errors="replace").read()
|
||||
except OSError:
|
||||
continue
|
||||
cands = signature_dates(text)
|
||||
if cands:
|
||||
_, day, mon, year = cands[-1]
|
||||
results[r["id"]] = f"{year:04d}-{mon:02d}-{day:02d}"
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
recovered = recover()
|
||||
print(f"recovered {len(recovered)} of the originally-undated documents")
|
||||
|
||||
docs_path = "web/docs.json"
|
||||
docs = json.load(open(docs_path, encoding="utf-8"))
|
||||
for r in docs:
|
||||
if not r.get("date") and r["id"] in recovered:
|
||||
r["date"] = recovered[r["id"]]
|
||||
r["dateInferred"] = True
|
||||
json.dump(docs, open(docs_path, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
|
||||
still_missing = sum(1 for r in docs if not r.get("date"))
|
||||
print(f"still undated: {still_missing}")
|
||||
Reference in New Issue
Block a user