MK-Lemumi-as-Code/tools/verify.py

#!/usr/bin/env python3
"""Sēdes datnes pārbaude pret TAP lapām (kešs no tools/fetch.py), ar citu nolasīšanas veidu nekā tools/tap.py.

  python3 tools/verify.py .cache/<sēdes uuid> data/lv-mk-sede-….xml

tools/tap.py nolasa punktus pa HTML elementiem. Šī pārbaude ņem visu lapas teksta bloku kā vienu virkni un pārbauda:
  1. darba kārtība — katra jautājuma numurs, TAP identifikators un statuss ir lapas tekstā šajā secībā;
  2. katra lēmuma lapa — sha256 sakrīt ar datnē norādīto;
  3. katra punkta numurs un teksts ir lapas teksta blokā burtiski un tādā pašā secībā kā datnē;
  4. pārklājums — lapas teksta bloks bez datnes punktiem, numuriem un virsraksta ir tukšs (nekas nav izlaists).
Atskaite — verification/<sēde>.verify.json.
"""
import hashlib
import json
import os
import re
import sys

from lxml import etree
from lxml import html as H

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
NS = "urn:pppa:vpk:mk-lemumi:0.1"
N = {"m": NS}


def flat(s):
    return re.sub(r"\s+", " ", s.replace("\xa0", " ")).strip()


def main(cache, path):
    man = json.load(open(os.path.join(cache, "manifest.json"), encoding="utf-8"))
    r = etree.parse(path).getroot()
    rep = {"file": os.path.relpath(path, ROOT), "meeting": r.get("id"), "agenda": {}, "decisions": {}, "passed": True}
    page = flat(H.parse(os.path.join(cache, "meeting.html")).getroot().text_content())
    pos, miss = 0, []
    for it in r.iterfind("m:Section/m:Item", N):
        for token in (it.get("n"), it.get("tap") or "", it.get("status")):
            i = page.find(token, pos) if token else pos
            if i < 0:
                miss.append(f"{it.get('id')}: {token}")
            else:
                pos = i
    rep["agenda"] = {"items": len(r.findall("m:Section/m:Item", N)), "missing": miss}
    rep["passed"] &= not miss
    by_url = {v["url"]: k for k, v in man["files"].items()}
    for d in r.iter(f"{{{NS}}}Decision"):
        rel = by_url.get(d.findtext("m:Url", namespaces=N))
        res = {"sha256_ok": False, "points": 0, "missing": [], "leftover": ""}
        if rel:
            raw = open(os.path.join(cache, rel), "rb").read()
            res["sha256_ok"] = hashlib.sha256(raw).hexdigest() == d.findtext("m:SHA256", namespaces=N)
            tree = H.fromstring(raw).xpath('//div[contains(@class,"structuralizer-tree--preview")]')[0]
            for x in tree.xpath('.//div[contains(@class,"node-preview__actions")]'):
                x.getparent().remove(x)
            text = flat(tree.text_content())
            rest, pos = text, 0
            for p in d.iter(f"{{{NS}}}Point"):
                res["points"] += 1
                pat = (re.escape(p.get("n")) + r"\s*" if p.get("n") else "") + re.escape(flat(p.findtext("m:Text", namespaces=N)))
                m = re.compile(pat).search(text, pos)
                if not m:
                    res["missing"].append(p.get("id"))
                else:
                    pos = m.end()
                rest = re.compile(pat).sub(" ", rest, count=1)
                for q in p.iterfind("m:Quote", N):
                    rest = rest.replace(flat(q.text), " ", 1)
            extras = ([x.text for x in d.iterfind("m:Heading", N)] + [d.findtext("m:Title", namespaces=N) or ""]
                      + [x.text for x in d.iterfind("m:Annex", N)] + [x.text for x in d.iterfind("m:Signatory", N)])
            for extra in extras:
                rest = rest.replace(flat(extra), " ", 1)
            rest = flat(re.sub(r"\b\d{2}-TA-\d+[A-Za-z]*\b", " ", rest))
            res["leftover"] = rest
        res["passed"] = res["sha256_ok"] and not res["missing"] and not res["leftover"]
        rep["passed"] &= res["passed"]
        rep["decisions"][d.get("id")] = res
    out = os.path.join(ROOT, "verification", r.get("id") + ".verify.json")
    os.makedirs(os.path.dirname(out), exist_ok=True)
    with open(out, "w", encoding="utf-8") as f:
        json.dump(rep, f, ensure_ascii=False, indent=1)
    bad = {k: v for k, v in rep["decisions"].items() if not v["passed"]}
    print(f"{rep['meeting']}: jautājumi {rep['agenda']['items']} (trūkst {len(miss)}), lēmumi {len(rep['decisions'])}, "
          f"neizturēti {len(bad)} -> {out}")
    for k, v in list(bad.items())[:10]:
        print("  ", k, "sha" if not v["sha256_ok"] else "", v["missing"][:3], v["leftover"][:160])
    return 0 if rep["passed"] else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1], sys.argv[2]))