Rihards
b9f854cbf7
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016679RwmHsuTFfxt26wP6rk
102 lines
4.6 KiB
Python
102 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Ielādē Ministru kabineta sēdes lapas no TAP portāla kešā (.cache/, netiek glabāts git).
|
|
|
|
python3 tools/fetch.py # jaunākā MK sēde sēžu sarakstā
|
|
python3 tools/fetch.py 2026-10-13 # sēde šajā datumā (sēžu saraksta 1. lapā)
|
|
python3 tools/fetch.py --uuid 776813c2-… # sēde pēc TAP identifikatora
|
|
|
|
Ielādē: sēžu saraksta rindu (datums, laiks, nosaukums, protokola numurs), darba kārtību, katra jautājuma lēmumu tekstu
|
|
(MK sēdes protokollēmuma projekts, rīkojuma projekts) un TAP API ierakstu par tiesību akta projektu (veids, atbildīgā institūcija).
|
|
Pieprasījumi — pa vienam ar pauzi. Kešā — manifest.json ar katras lapas URL, sha256 un ielādes laiku.
|
|
"""
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
from lxml import html as H
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import tap # noqa: E402
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
LIST = tap.BASE + "/meetings/cabinet_ministers"
|
|
PAUSE = 0.5
|
|
|
|
|
|
def meeting_rows(raw):
|
|
"""Sēžu saraksta lapa → [{'uuid', 'start', 'title', 'protocol', 'protocol_url'}]."""
|
|
doc = H.fromstring(raw)
|
|
out = []
|
|
for r in doc.xpath('//*[starts-with(@data-url, "/meetings/cabinet_ministers/")]'):
|
|
m = re.match(r"/meetings/cabinet_ministers/([0-9a-f-]{36})$", r.get("data-url"))
|
|
if not m:
|
|
continue
|
|
t = tap.norm(r.text_content())
|
|
d = re.search(r"(\d{2})\.(\d{2})\.(\d{4})\.\s*(\d{1,2}):(\d{2})", t)
|
|
p = re.search(r"Nr\.\s*(\d+)", t)
|
|
a = r.xpath('.//a[starts-with(@href, "/meetings/protocols/")]')
|
|
title = re.sub(r"^\d{2}\.\d{2}\.\d{4}\.\s*\d{1,2}:\d{2}\s*", "", t)
|
|
title = re.sub(r"\s*Nr\.\s*\d+\s*$", "", title)
|
|
out.append({"uuid": m.group(1), "title": title, "protocol": int(p.group(1)) if p else None,
|
|
"protocol_url": tap.BASE + a[0].get("href") if a else None,
|
|
"start": f"{d.group(3)}-{d.group(2)}-{d.group(1)}T{int(d.group(4)):02d}:{d.group(5)}:00" if d else None})
|
|
return out
|
|
|
|
|
|
def main(argv):
|
|
rows = meeting_rows(tap.get(LIST))
|
|
if "--uuid" in argv:
|
|
uuid = argv[argv.index("--uuid") + 1]
|
|
row = next((r for r in rows if r["uuid"] == uuid), {"uuid": uuid, "title": None, "protocol": None, "start": None})
|
|
else:
|
|
want = next((a for a in argv if re.fullmatch(r"\d{4}-\d{2}-\d{2}", a)), None)
|
|
cand = [r for r in rows if r["start"] and (not want or r["start"].startswith(want))]
|
|
if not cand:
|
|
sys.exit(f"sēde nav atrasta sēžu saraksta 1. lapā: {want or 'jaunākā'}")
|
|
row = cand[0]
|
|
cache = os.path.join(ROOT, ".cache", row["uuid"])
|
|
os.makedirs(os.path.join(cache, "nodes"), exist_ok=True)
|
|
os.makedirs(os.path.join(cache, "api"), exist_ok=True)
|
|
files = {}
|
|
|
|
def save(rel, url, data):
|
|
with open(os.path.join(cache, rel), "wb") as f:
|
|
f.write(data)
|
|
files[rel] = {"url": url, "sha256": hashlib.sha256(data).hexdigest(),
|
|
"retrieved": dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()}
|
|
|
|
url = f"{LIST}/{row['uuid']}"
|
|
raw = tap.get(url)
|
|
save("meeting.html", url, raw)
|
|
meeting = tap.parse_meeting(raw)
|
|
n_docs = n_api = 0
|
|
for sec in meeting["sections"]:
|
|
for it in sec["items"]:
|
|
for label, href in it["documents"]:
|
|
m = re.search(r"/structuralizer/data/nodes/([0-9a-f-]{36})/preview", href or "")
|
|
if label in tap.DECISION_DOCS and m:
|
|
time.sleep(PAUSE)
|
|
save(f"nodes/{m.group(1)}.html", tap.BASE + href, tap.get(tap.BASE + href))
|
|
n_docs += 1
|
|
if it["legal_act"]:
|
|
time.sleep(PAUSE)
|
|
api = f"{tap.BASE}/api/v1/legal_acts/{it['legal_act']}?include=legal_act_type,policy_areas"
|
|
try:
|
|
save(f"api/{it['legal_act']}.json", api, tap.get(api, accept="application/vnd.api+json"))
|
|
n_api += 1
|
|
except Exception as e: # noqa: BLE001 — slēgtiem projektiem API var neatbildēt
|
|
print(f"API {it['tap']}: {e}", file=sys.stderr)
|
|
with open(os.path.join(cache, "manifest.json"), "w", encoding="utf-8") as f:
|
|
json.dump({"meeting": row, "files": files}, f, ensure_ascii=False, indent=1)
|
|
items = sum(len(s["items"]) for s in meeting["sections"])
|
|
print(f"{row['start']} {row['title']}: {items} jautājumi, {n_docs} lēmumu teksti, {n_api} API ieraksti -> {cache}")
|
|
print(row["uuid"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|