MK-Lemumi-as-Code/tools/convert.py

#!/usr/bin/env python3
"""TAP sēdes lapas (kešs no tools/fetch.py) → data/lv-mk-sede-GGGG-MM-DD.xml un sources/<sēde>/*.txt

  python3 tools/convert.py .cache/<sēdes uuid> [--registry organizacijas.xml] [--pasvaldibas pasvaldibas-atvk.xml] [--valdibas valdibas.xml]

Reģistri pēc noklusējuma — no ProcessGit (Organizations_register, Pasvaldibu-registrs, Valdibas-Deklaracija-as-Code).
Ja sēdes datne jau ir un saturs mainījies (piemēram, publicēts protokols vai mainīts statuss), datu versija palielinās.
"""
import datetime as dt
import hashlib
import json
import os
import re
import sys
import urllib.request
from xml.sax.saxutils import escape, quoteattr

from lxml import etree

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import derive  # noqa: E402
import tap  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
NS = "urn:pppa:vpk:mk-lemumi:0.1"
PG = "https://processgit.org/Valsts-Pirmkods"
REG = {"registry": f"{PG}/Organizations_register/raw/branch/main/organizacijas.xml",
       "pasvaldibas": f"{PG}/Pasvaldibu-registrs/raw/branch/main/data/pasvaldibas-atvk.xml",
       "valdibas": f"{PG}/Valdibas-Deklaracija-as-Code/raw/branch/main/data/valdibas.xml"}
ROLE = {"org": "institution", "minister": "minister", "atvk": "municipality"}


def load(src):
    if os.path.exists(src):
        return open(src, "rb").read()
    return urllib.request.urlopen(urllib.request.Request(src, headers=tap.UA), timeout=60).read()


def a(name, value):
    return f" {name}={quoteattr(str(value))}" if value not in (None, "", False) else ""


def el(tag, text, **attrs):
    return f"<{tag}{''.join(a(k, v) for k, v in attrs.items())}>{escape(text)}</{tag}>" if text else ""


def government(valdibas_xml, date):
    govs = []
    for g in etree.fromstring(valdibas_xml):
        if isinstance(g.tag, str) and etree.QName(g).localname == "Government":
            cv = next(c for c in g if etree.QName(c).localname == "ConfidenceVote").get("date")
            govs.append((cv, g.get("id")))
    return max(x for x in govs if x[0] <= date)[1]


def org_by_name(org_xml):
    out = {}
    for o in etree.fromstring(org_xml):
        if isinstance(o.tag, str) and etree.QName(o).localname == "Organization" and not o.get("validTo"):
            name = next((c.text for c in o if etree.QName(c).localname == "Name"), "") or ""
            out[re.sub(r"^latvijas republikas ", "", name.lower())] = o.get("id")
    return out


def decision_suffix(label, used):
    s = "pl" if "protokollēmum" in label else "rik" if "Rīkojum" in label else None
    if not s or s in used:
        s = f"doc{len(used) + 1}"
    used.add(s)
    return s


def point_ids(points, prefix, parent_num=None):
    """Punktu identifikatori: „2.” → p2, „2.1.” → p2.1; „1)” zem p2 → p2.1; nenumurēts → u1, u2 …"""
    seen, u = set(), 0
    for p in points:
        digits = re.findall(r"\d+", p["n"].replace("¹", "").replace("²", ""))
        if digits:
            num = ".".join(digits) if (parent_num is None or ".".join(digits).startswith(parent_num + ".")) else \
                f"{parent_num}.{'.'.join(digits)}"
            pid = f"{prefix}.p{num}"
        else:
            u += 1
            num, pid = parent_num, f"{prefix}.u{u}"
        while pid in seen:
            u += 1
            pid = f"{prefix}.u{u}"
        seen.add(pid)
        p["id"], p["num"] = pid, num
        point_ids(p["children"], prefix, num)


def write_points(points, R, depth=0):
    out = []
    for p in points:
        parties = R.parties(p["text"])
        date, printed = derive.deadline(p["text"])
        kind = derive.kind_of(p["text"], parties, depth > 0)
        party_xml = "".join(el("Party", printed_, role=ROLE[k], org=i if k != "atvk" else None, atvk=i if k == "atvk" else None)
                            for k, i, printed_ in parties)
        out.append(f'<Point id="{p["id"]}"{a("n", p["n"]) or " n=" + quoteattr("")} kind="{kind}">' + el("Text", p["text"])
                   + "".join(el("Quote", q) for q in p["quotes"] if q) + party_xml + (el("Deadline", printed, date=date) if date else "")
                   + "".join(write_points(p["children"], R, depth + 1)) + "</Point>")
    return out


def heading_numbers(heading):
    h = " ".join(heading)
    act = re.search(r"rīkojums Nr\.\s*(\d+)", h)
    prot = re.search(r"prot\.\s*Nr\.\s*(\d+)\s+(\d+)\.\s*§", h)
    return (act.group(1) if act else None, prot.group(1) if prot else None, prot.group(2) if prot else None)


def source_text(d):
    lines = list(d["heading"]) + [d["title"] or d["header"]]
    for depth, p in tap.flatten(d["points"]):
        lines.append("  " * depth + (p["n"] + " " if p["n"] else "") + p["text"])
        lines += ["  " * depth + "  " + q for q in p["quotes"] if q]
    for x in d["annexes"]:
        lines.append(f"[pielikums {x['n']}] {x['text']}")
    lines += d["signatures"]
    return "\n".join(lines) + "\n"


def main(argv):
    cache = argv[0]
    opt = {k: (argv[argv.index("--" + k) + 1] if "--" + k in argv else v) for k, v in REG.items()}
    man = json.load(open(os.path.join(cache, "manifest.json"), encoding="utf-8"))
    row, files = man["meeting"], man["files"]
    R = derive.Registers(load(opt["registry"]), load(opt["pasvaldibas"]))
    orgs = org_by_name(load(opt["registry"]))
    date = row["start"][:10]
    mid = f"lv-mk-sede-{date}"
    os.makedirs(os.path.join(ROOT, "data"), exist_ok=True)
    existing = os.path.join(ROOT, "data", mid + ".xml")
    if os.path.exists(existing) and row["uuid"] not in open(existing, encoding="utf-8").read():
        mid += "-2"
        existing = os.path.join(ROOT, "data", mid + ".xml")
    gov = government(load(opt["valdibas"]), date)
    meeting = tap.parse_meeting(open(os.path.join(cache, "meeting.html"), "rb").read())
    title = row["title"] or meeting["title"]
    src_dir = os.path.join("sources", mid)
    os.makedirs(os.path.join(ROOT, src_dir), exist_ok=True)

    sections, n_points, n_dec = [], 0, 0
    for s in meeting["sections"]:
        items = []
        for it in s["items"]:
            iid = f"{mid}.q{it['n']}"
            api = os.path.join(cache, "api", f"{it['legal_act']}.json") if it["legal_act"] else None
            act_type = resp = None
            if api and os.path.exists(api):
                j = json.load(open(api, encoding="utf-8"))
                resp = tap.norm(j["data"]["attributes"].get("responsible_institution_name"))
                act_type = next((x["attributes"]["name"] for x in j.get("included", []) if x["type"] == "legal_act_types"), None)
            used, decisions = set(), []
            for label, href in it["documents"]:
                m = re.search(r"/structuralizer/data/nodes/([0-9a-f-]{36})/preview", href or "")
                rel = f"nodes/{m.group(1)}.html" if m else None
                if label not in tap.DECISION_DOCS or not rel or rel not in files:
                    continue
                d = tap.parse_node(open(os.path.join(cache, rel), "rb").read())
                if not d["points"]:
                    continue
                did = f"{iid}.{decision_suffix(label, used)}"
                point_ids(d["points"], did)
                fname = f"q{it['n']}.{did.rsplit('.', 1)[1]}.txt"
                with open(os.path.join(ROOT, src_dir, fname), "w", encoding="utf-8") as f:
                    f.write(source_text(d))
                n_points += sum(1 for _ in tap.flatten(d["points"]))
                n_dec += 1
                act_no, prot, par = heading_numbers(d["heading"])
                decisions.append(
                    f'<Decision id="{did}" document={quoteattr(label)} textStatus="{"draft" if "projekts" in label else "final"}"'
                    f'{a("actNumber", act_no)}{a("protocol", prot)}{a("paragraph", par)}>'
                    + "".join(el("Heading", h) for h in d["heading"]) + el("Title", d["title"]) + el("Url", files[rel]["url"]) + el("SourceFile", f"{src_dir}/{fname}")
                    + el("SHA256", files[rel]["sha256"]) + "".join(write_points(d["points"], R))
                    + "".join(el("Annex", x["text"], n=x["n"]) for x in d["annexes"] if x["text"])
                    + "".join(el("Signatory", x) for x in d["signatures"]) + "</Decision>")
            items.append(
                f'<Item id="{iid}" n="{it["n"]}"{a("tap", it["tap"])}{a("restricted", "true" if it["restricted"] else "")}'
                f' status={quoteattr(it["status"] or "nav norādīts")}'
                + "".join(a(f, "true") for f in ("addedLate", "movedToA", "postponed") if f in it["flags"]) + ">"
                + el("Title", it["title"]) + el("ActType", act_type)
                + (el("Responsible", resp, org=orgs.get(resp.lower().replace("latvijas republikas ", ""))) if resp else "")
                + el("Presenter", it["presenter"]) + el("Invited", it["invited"])
                + (f'<LegalAct uuid="{it["legal_act"]}">' + el("Url", f"{tap.BASE}/legal_acts/{it['legal_act']}") + "</LegalAct>"
                   if it["legal_act"] else "")
                + "".join(el("Document", lab, url=(tap.BASE + h if h and h.startswith("/") else h)) for lab, h in it["documents"] if lab and h)
                + "".join(el("Approval", x) for x in it["approvals"] if x)
                + el("PublicParticipation", it["participation"]) + "".join(decisions) + "</Item>")
        sections.append(f'<Section n="{s["n"]}" closed="{"true" if "slēgtā daļa" in s["title"] else "false"}">'
                        + el("Title", s["title"]) + "".join(items) + "</Section>")

    n_items = sum(len(s["items"]) for s in meeting["sections"])
    changes, number = [f"Darba kārtība no TAP: {len(meeting['sections'])} sadaļas, {n_items} jautājumi; "
                       f"{n_dec} lēmumu teksti ({n_points} punkti) — projekti, kā pievienoti darba kārtībai."], 1
    prev_versions = ""
    body = "".join(sections)
    if os.path.exists(existing):
        old = etree.parse(existing).getroot()
        N = {"m": NS}
        old_body = "".join(etree.tostring(x, encoding="unicode") for x in old.findall("m:Section", N))
        new_body = "".join(etree.tostring(x, encoding="unicode") for x in
                           etree.fromstring(f'<x xmlns="{NS}">{body}</x>').findall("m:Section", N))
        vers = old.findall("m:Metadata/m:DataVersion", N)
        prev_versions = "".join(etree.tostring(v, encoding="unicode").replace(f' xmlns="{NS}"', "") for v in vers)
        old_proto = old.find("m:Metadata/m:Protocol", N)
        if old_body == new_body and (old_proto is not None) == bool(row["protocol"]):
            print(f"{mid}: nav izmaiņu")
            return
        number = max(int(v.get("number")) for v in vers) + 1
        changes = []
        if row["protocol"] and old_proto is None:
            changes.append(f"Publicēts sēdes protokols Nr. {row['protocol']}.")
        if old_body != new_body:
            changes.append("Darba kārtība vai lēmumu teksts atjaunots no TAP.")
    today = dt.date.today().isoformat()
    meta = ("<Metadata>" + el("Title", title) + el("Start", row["start"])
            + el("Kind", "extraordinary" if "ārkārtas" in (title or "") else "regular")
            + (f'<Protocol number="{row["protocol"]}">' + el("Url", row.get("protocol_url")) + "</Protocol>" if row["protocol"] else "")
            + "<Source>" + el("Url", files["meeting.html"]["url"]) + el("SHA256", files["meeting.html"]["sha256"])
            + el("Retrieved", files["meeting.html"]["retrieved"][:19]) + "</Source>"
            + f'<DataVersion number="{number}" date="{today}">' + "".join(el("Change", c) for c in changes) + "</DataVersion>"
            + prev_versions + "</Metadata>")
    xml = (f'<CabinetMeeting xmlns="{NS}" id="{mid}" government="{gov}" schemaVersion="0.1">' + meta + body + "</CabinetMeeting>")
    root = etree.fromstring(xml.encode())
    etree.indent(root, space="  ")
    with open(existing, "wb") as f:
        f.write(etree.tostring(root, xml_declaration=True, encoding="UTF-8") + b"\n")
    viewer = os.path.join(ROOT, "data", "processgit.viewer.json")   # ProcessGit skatītājs: jaunā datne sarakstā edit_allow
    if os.path.exists(viewer):
        v = json.load(open(viewer, encoding="utf-8"))
        allow = v["viewers"][0]["edit_allow"]
        if mid + ".xml" not in allow:
            v["viewers"][0]["edit_allow"] = sorted(set(allow + [mid + ".xml"]) - {"lv-mk-sede-*.xml"})
            with open(viewer, "w", encoding="utf-8") as f:
                json.dump(v, f, ensure_ascii=False, indent=2)
                f.write("\n")
    print(f"{existing}: {n_items} jautājumi, {n_dec} lēmumi, {n_points} punkti, datu versija {number}")


if __name__ == "__main__":
    main(sys.argv[1:])