FDK-Registrs-as-Code/tools/build.py

"""FDK reģistrs: Nolikumi-as-Code → iestades/<VPK ID>/{funkcijas,darbi,kompetence}.xml → registrs/{…}.xml.

Noteikumi (vienādi visiem nolikumiem):
  • Funkcija  — nolikuma punkts ar kind="function" (vai tā apakšpunkts sarakstā, kas sākas ar „… funkcijas:”).
  • Darbs     — punkts ar kind="task" (vai apakšpunkts sarakstā „… uzdevumus:”).
  • Kompetence — punkts ar kind="right" (veids = tiesības) un kompetences nodaļas saraksti, kuru priekšmets ir pati iestāde
                 („Ministrija:”, „Birojs atbilstoši kompetencei:”, „Dienests ir tiesīgs:” …); vadītāja, personāla un
                 struktūras saraksti netiek ņemti.
  • Formulējums — ievadteksti no ārējā saraksta ievada uz leju + pats punkts (ārējais ievads, piemēram, „Ministrijai ir šādas
                 funkcijas:”, tiek izlaists, jo to izsaka ieraksta veids). statuss="auto".
                 sources/formulejumi.csv var aizstāt formulējumu ar gramatiski saskaņotu (ai_proposed / confirmed).
  • ID — F|D|K-<VPK ID>-NNN, pastāvīgs: sources/ids.csv glabā avots → id; jauni punkti saņem nākamo brīvo numuru.
  • Svītrotie punkti netiek iekļauti.

python3 tools/build.py --nolikumi ../Nolikumi-as-Code [--only 21-0000 …]
"""
import argparse
import csv
import datetime as dt
import os
import re
import subprocess
import sys
import xml.etree.ElementTree as ET

NS = "{urn:pppa:vpk:nolikums:0.1}"
OUT_NS = "urn:pppa:vpk:fdk:0.1"
TYPES = {"F": ("funkcijas.xml", "Funkcijas", "Funkcija"),
         "D": ("darbi.xml", "Darbi", "Darbs"),
         "K": ("kompetence.xml", "Kompetences", "Kompetence")}
# kompetences nodaļas saraksti, kas apraksta nevis iestādi, bet tās vadītāju, personālu vai uzbūvi
NOT_INSTITUTION = re.compile(
    r"^(ministrs|valsts sekretār|parlamentār|[^:]*priekšniek|[^:]*vadītāj|[^:]*direktor|[^:]*personāl|[^:]*struktūrvienīb|"
    r"[^:]*sastāvā|[^:]*finanšu resurs|[^:]*izglītojam|izglītības process|akadēmisk|[^:]*pienākumi|[^:]*komandier)", re.I)
RIGHTS = re.compile(r"tiesīb|tiesīg|\bvar:\s*$", re.I)
DATE = re.compile(r"\b(\d{2})\.(\d{2})\.(\d{4})\.")


def clean(t):
    t = " ".join((t or "").split())
    t = re.sub(r"\s*[;,]\s*(un|vai)?\s*$", "", t)
    return t.rstrip(" .;")


def list_type(lead):
    """Record type of a list by its outermost introductory text."""
    s = (lead or "").lower()
    if "funkcij" in s:
        return "F"
    if "uzdevum" in s:
        return "D"
    if RIGHTS.search(s):
        return "Kt"
    return None


def read_csv(path):
    return list(csv.DictReader(open(path, encoding="utf-8"))) if os.path.exists(path) else []


def git_commit(path):
    try:
        return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"], text=True).strip()
    except Exception:
        return os.environ.get("NOLIKUMI_COMMIT", "unknown")


def records_of(root):
    """Yield (type, veids, point, chain_texts, parent_point) for every record in one nolikums."""
    out = []

    def walk(p, chain, chapter_kind, outer_type, comp_ok, parent_rec, inside):
        k = p.get("kind")
        text = (p.findtext(NS + "Text") or "").strip()
        if k == "deleted" or text.lower().startswith("(svītrots"):
            return
        rtype = veids = None
        if k == "function":
            rtype = "F"
        elif k == "task":
            rtype = "D"
        elif k == "right":
            rtype, veids = "K", "tiesības"
        elif k == "provision":
            if outer_type == "F":
                rtype = "F"
            elif outer_type == "D":
                rtype = "D"
            elif outer_type == "Kt":
                rtype, veids = "K", "tiesības"
            elif outer_type == "Kd":
                rtype, veids = "K", "darbība"
        if rtype:
            out.append((rtype, veids, p, list(chain), parent_rec if parent_rec and parent_rec[0] == rtype else None))
        kids = p.findall(NS + "Point")
        if not kids:
            return
        # a point with children introduces a list: its text becomes part of the children's formulation
        if not inside and k in ("lead", "provision"):
            # outermost introductory text — defines the list type, is not repeated in formulations
            lt = list_type(text)
            if lt is None and chapter_kind == "competence" and text.rstrip().endswith(":") and not NOT_INSTITUTION.match(text):
                lt = "Kd"  # „Ministrija:”, „Birojs atbilstoši kompetencei:” — iestādes vispārējā kompetence
            for c in kids:
                walk(c, [], chapter_kind, lt or outer_type, False, None, True)
            return
        sub = chain + [text]
        rec = (rtype, p) if rtype else parent_rec
        for c in kids:
            walk(c, sub, chapter_kind, outer_type, comp_ok, rec, True)

    for ch in root.findall(NS + "Chapter"):
        for p in ch.findall(NS + "Point"):
            walk(p, [], ch.get("kind"), None, False, None, False)
    return out


def formulate(rtype, veids, chain, text, outer_rights):
    parts = [" ".join(c.split()) for c in chain] + [clean(text)]
    s = " ".join(x for x in parts if x)
    s = re.sub(r"\s+", " ", s).strip()
    if rtype == "K" and veids == "tiesības" and outer_rights and not s.lower().startswith("tiesīb"):
        s = "tiesības " + s
    return s


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--nolikumi", required=True)
    ap.add_argument("--only", nargs="*")
    a = ap.parse_args()
    names = {r["vpk_id"]: r["nosaukums"] for r in read_csv("sources/iestades.csv")}
    ids = {r["avots"]: r["id"] for r in read_csv("sources/ids.csv")}
    over = {r["avots"]: r for r in read_csv("sources/formulejumi.csv")}
    used_over = set()
    src = f"Nolikumi-as-Code@{git_commit(a.nolikumi)}"
    today = dt.date.today().isoformat()
    ET.register_namespace("", OUT_NS)
    stats = []
    files = sorted(f for f in os.listdir(os.path.join(a.nolikumi, "data")) if f.endswith(".xml"))
    for fn in files:
        root = ET.parse(os.path.join(a.nolikumi, "data", fn)).getroot()
        org = root.find(NS + "Institution").get("org")
        if a.only and org not in a.only:
            continue
        if org not in names:
            sys.exit(f"{org}: nav sources/iestades.csv")
        act = root.find(NS + "Act")
        nol = f"MK Nr. {act.findtext(NS + 'Number')}" if act.findtext(NS + "Number") else (act.findtext(NS + "Title") or "")
        dates = [act.findtext(NS + "Adopted")]
        for note in root.iter(NS + "Note"):
            for d, m, y in DATE.findall(note.text or ""):
                dates.append(f"{y}-{m}-{d}")
        red = max(d for d in dates if d)
        recs = records_of(root)
        # outermost list intro per record, for the "tiesības" prefix
        counters = {}
        for v in ids.values():
            t, o, n = v[0], v[2:9], int(v[10:])
            if o == org:
                counters[t] = max(counters.get(t, 0), n)
        per = {t: [] for t in TYPES}
        rec_id = {}
        for rtype, veids, p, chain, parent in recs:
            avots = p.get("id")
            if avots not in ids:
                counters[rtype] = counters.get(rtype, 0) + 1
                ids[avots] = f"{rtype}-{org}-{counters[rtype]:03d}"
            rid = ids[avots]
            if not rid.startswith(rtype + "-"):
                sys.exit(f"{avots}: ids.csv saka {rid}, bet punkts tagad ir {rtype} — jāpārskata")
            rec_id[p] = rid
            text = p.findtext(NS + "Text") or ""
            f = formulate(rtype, veids, chain, text, veids == "tiesības" and not chain)
            statuss = "auto"
            if avots in over:
                f, statuss = over[avots]["formulejums"], over[avots]["statuss"]
                used_over.add(avots)
            attrs = {"id": rid, "iestāde": org, "iestādesNosaukums": names[org], "avots": avots,
                     "nolikums": nol, "redakcija": red, "statuss": statuss}
            if rtype == "K":
                attrs = {"id": rid, "veids": veids, **{k: v for k, v in attrs.items() if k != "id"}}
            if parent and parent[1] in rec_id:
                attrs["vecāks"] = rec_id[parent[1]]
            per[rtype].append((attrs, f))
        os.makedirs(f"iestades/{org}", exist_ok=True)
        for t, (fname, rootname, elname) in TYPES.items():
            write(f"iestades/{org}/{fname}", rootname, elname, per[t], src, today, org)
        stats.append((org, len(per["F"]), len(per["D"]), len(per["K"])))
    # registry = all institution files on disk (also those not rebuilt in this run)
    os.makedirs("registrs", exist_ok=True)
    for t, (fname, rootname, elname) in TYPES.items():
        rows = []
        for d in sorted(os.listdir("iestades")):
            pth = f"iestades/{d}/{fname}"
            if os.path.exists(pth):
                for e in ET.parse(pth).getroot():
                    rows.append(({k: v for k, v in e.attrib.items()}, e.findtext(f"{{{OUT_NS}}}Formulējums")))
        write(f"registrs/{fname}", rootname, elname, sorted(rows, key=lambda r: r[0]["id"]), src, today, None)
    with open("sources/ids.csv", "w", encoding="utf-8", newline="") as fh:
        w = csv.writer(fh)
        w.writerow(["avots", "id"])
        for k in sorted(ids, key=lambda k: ids[k]):
            w.writerow([k, ids[k]])
    unused = set(over) - used_over
    if unused and not a.only:
        sys.exit("formulejumi.csv: avots nav atrasts: " + ", ".join(sorted(unused)))
    tot = [sum(s[i] for s in stats) for i in (1, 2, 3)]
    print(f"{src}: {len(stats)} iestādes — funkcijas {tot[0]}, darbi {tot[1]}, kompetence {tot[2]}")
    for s in stats:
        if not (s[1] or s[2]):
            print("  bez funkcijām un darbiem:", s[0])


def write(path, rootname, elname, rows, src, today, org):
    r = ET.Element(f"{{{OUT_NS}}}{rootname}", {"schemaVersion": "0.1", "avots": src, "ģenerēts": today, **({"iestāde": org} if org else {})})
    for attrs, f in rows:
        e = ET.SubElement(r, f"{{{OUT_NS}}}{elname}", attrs)
        ET.SubElement(e, f"{{{OUT_NS}}}Formulējums").text = f
    ET.indent(r, "  ")
    ET.ElementTree(r).write(path, encoding="UTF-8", xml_declaration=True)
    with open(path, "a", encoding="utf-8") as fh:
        fh.write("\n")


if __name__ == "__main__":
    main()