Likums-as-Code/tools/html_source.py

"""html_source.py — likumi.lv likuma lapa (HTML) → rindkopu plūsma tools/convert.py.

Word eksports no likumi.lv nav pieejams automātiski, tāpēc tas pats konvertors var lasīt
arī likumi.lv lapu. Šis modulis no lapas uztaisa tādu pašu rindkopu secību, kādu dod Word
fails: galvene (Izdevējs, Veids, Publicēts, Attēlotā redakcija, Grozījumi), likumdevēja
formula, nosaukums, nodaļas, panti, daļas, punkti, piezīmes, pārejas noteikumi, ES atsauce,
noslēgums. Numuri lapā ir drukāti tekstā, tāpēc tie tiek nodoti tieši (nevis skaitīti kā Word).

  python3 tools/html_source.py fetch <likumi.lv id> <kešs>   # lapa + grozījumu lapas
  python3 tools/convert.py <kešs>/<id>.html data/<id>.xml        # konvertē
"""
import copy
import datetime
import os
import re
import sys
import time
import types
import urllib.request

SUP = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹")
SUPC = "⁰¹²³⁴⁵⁶⁷⁸⁹"
UA = {"User-Agent": "Mozilla/5.0 (likums-as-code/0.2)"}
PART = re.compile(r"^\((\d+)([" + SUPC + r"]*)\)\s*(.*)$")
POINT = re.compile(r"^(\d+)([" + SUPC + r"]*)\)\s*(.*)$")
LETTER = re.compile(r"^([a-zāčēģīķļņšūž])\)\s*(.*)$")
LETTERS = "abcdefghijklmnopqrstuvwxyz"
TPNUM = re.compile(r"^(\d+)([" + SUPC + r"]*)\.\s*(?=\S)(.*)$")
ART = re.compile(r"^(\d+)\.?([" + SUPC + r"]*)([A-Z]?)\.?\s*pants(?:\.|(?=\s|$))")
DEL_TAIL = re.compile(r"\s(\((?:Izslēgts|izslēgts)\b.*\))\s*$")


class Para:
    """Rindkopa ar to pašu saskarni, ko convert.py lieto no python-docx."""
    def __init__(self, text, style="Normal", num=None):
        self.text = text
        self.style = types.SimpleNamespace(name=style)
        self._num = num  # (vērtība, līmenis) vai None


class Explicit:
    """Numerācija no drukātā teksta (Word numerācijas skaitītāja vietā)."""
    explicit = True

    def next(self, value, ilvl):
        return value


def _get(url):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=60) as r:
        return r.read()


def amendment_ids(page):
    """Grozījumu likumi, uz kuriem atsaucas šā likuma redakcijas piezīmes (p.labojumu_pamats)."""
    notes = re.findall(r"(?s)<p class=['\"]labojumu_pamats['\"][^>]*>(.*?)</p>", page)
    return sorted(set(re.findall(r"/ta/id/(\d+)-grozijum", " ".join(notes))), key=int)


def fetch(likumi_id, cache):
    os.makedirs(cache, exist_ok=True)
    path = os.path.join(cache, f"{likumi_id}.html")
    raw = _get(f"https://likumi.lv/ta/id/{likumi_id}")
    open(path, "wb").write(raw)
    page = raw.decode("utf-8")
    ids = amendment_ids(page)
    for a in ids:
        p = os.path.join(cache, f"{a}.html")
        if not os.path.exists(p):
            time.sleep(1)
            open(p, "wb").write(_get(f"https://likumi.lv/ta/id/{a}"))
    print(f"{likumi_id}: lapa {len(raw)} B, grozījumu lapas {len(ids)}")


def flat(page):
    page = re.sub(r"(?s)<(script|style)\b.*?</\1>", " ", page)
    import html as H
    return " ".join(H.unescape(re.sub(r"<[^>]+>", " ", page)).split())


def pase(page):
    t = flat(page)
    f = {}
    for k in ["Nosaukums", "Izdevējs", "Veids", "Pieņemts", "Stājas spēkā"]:
        m = re.search(re.escape(k) + r":\s*(.*?)\s+(?=Statuss:|Izdevējs:|Veids:|Pieņemts:|Stājas spēkā:|Tēma:|Zaudē spēku:|Publicēts:)", t)
        if m:
            f[k] = m.group(1).strip()
    m = re.search(r"Publicēts:\s*(.*?)\s+(?=Saistītie dokumenti|Dokumenta valoda|OP numurs|Satura rādītājs)", t)
    if m:
        f["Publicēts"] = re.sub(r"\s+,", ",", m.group(1).strip())
    return f


def short_pub(s):
    s = s.replace("Latvijas Republikas Saeimas un Ministru Kabineta Ziņotājs", "Ziņotājs")
    return s.replace("Latvijas Vēstnesis", "LV")


def as_of(page, likumi_id):
    today = datetime.date.today().strftime("%Y/%m/%d")
    ds = sorted(d for d in set(re.findall(rf"/ta/id/{likumi_id}[^'\"]*/redakcijas-datums/(\d{{4}}/\d\d/\d\d)", page)) if d <= today)
    if ds:
        y, m, d = ds[-1].split("/")
        return f"{d}.{m}.{y}"
    p = pase(page)
    return p.get("Stājas spēkā", "").rstrip(".")


def ptext(el):
    e = copy.deepcopy(el)
    for junk in e.xpath(".//*[contains(@class,'info-icon-wrapper') or contains(@class,'panta-doc-npk')]"):
        junk.getparent().remove(junk)
    for s in e.iter("sup"):
        s.text = (s.text or "").translate(SUP)
    for br in e.iter("br"):
        br.tail = "\n" + (br.tail or "")
    return e.text_content()


def norm(s):
    return re.sub(r"\s+", " ", s).strip()


ORD = ["pirm", "otr", "treš", "ceturt", "piekt", "sest", "septīt", "astot", "devīt", "desmit",
       "vienpadsmit", "divpadsmit", "trīspadsmit", "četrpadsmit", "piecpadsmit", "sešpadsmit",
       "septiņpadsmit", "astoņpadsmit", "deviņpadsmit", "divdesmit"]
ORDINAL = re.compile(r"^([A-ZĀČĒĢĪĶĻŅŠŪŽ][a-zāčēģīķļņšūž]+ā)(\s+[A-C])?\s+(nodaļa\b.*)$")


def to_roman(n):
    out = ""
    for v, s in ((50, "L"), (40, "XL"), (10, "X"), (9, "IX"), (5, "V"), (4, "IV"), (1, "I")):
        while n >= v:
            out += s; n -= v
    return out


def chapter_roman(t):
    """'Piektā A nodaļa X' → 'V A nodaļa X' (convert.py CHAP saprot romiešu ciparus)."""
    m = ORDINAL.match(t)
    if not m:
        return t
    w = m.group(1).lower()[:-1]  # bez galotnes ā
    for k in sorted(range(len(ORD)), key=lambda k: -len(ORD[k])):
        if w == ORD[k]:
            return f"{to_roman(k + 1)}{m.group(2) or ''} {m.group(3)}"
    return t


def classify(t, xc, tp, notes):
    """Panta rindkopa → Para ar numuru (daļa, punkts, apakšpunkts) vai bez tā."""
    lvl = 2 if "limenis3" in xc else 1
    m = PART.match(t)
    if m and not tp:
        if m.group(2):  # 4¹ — convert.py INLINE_PART
            return [Para(t)]
        return [Para(m.group(3), num=(int(m.group(1)), 0))]
    m = POINT.match(t)
    if m and not tp:
        return [Para(m.group(3), num=(m.group(1) + m.group(2), lvl))]
    m = LETTER.match(t)
    if m and m.group(1) in LETTERS:  # a), b) … → apakšpunkts n="1", "2" … (kā Word eksportā)
        return [Para(m.group(2), num=(str(LETTERS.index(m.group(1)) + 1), 2))]
    return [Para(t)]  # pārejas noteikumu apakšpunkti "1) …" → convert.py INLINE_POINT


def load(path):
    """→ (rindkopas, numerācija, avota baiti, avota nosaukums, piezīmes)"""
    import lxml.html
    raw = open(path, "rb").read()
    page = raw.decode("utf-8")
    cache = os.path.dirname(path)
    likumi_id = os.path.basename(path)[:-5]
    notes = []
    p = pase(page)
    title = norm(ptext(lxml.html.fromstring(page).xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' TV207 ')]")[0]))

    # --- galvene tādā pašā formā kā likumi.lv Word eksportā ---
    ps = [Para(f"Izdevējs: {p.get('Izdevējs', '')}"), Para(f"Veids: {p.get('Veids', '')}"),
          Para(f"Publicēts: {p.get('Publicēts', '')}"), Para(f"Attēlotā redakcija: {as_of(page, likumi_id)}.")]
    asof = as_of(page, likumi_id)
    asof_iso = "-".join(reversed(asof.split(".")))
    amends = []
    for a in amendment_ids(page):
        ap = os.path.join(cache, f"{a}.html")
        if not os.path.exists(ap):
            notes.append(f"Grozījumu lapa {a} nav kešā — palaid html_source.py fetch")
            continue
        apage = open(ap, encoding="utf-8").read()
        q = pase(apage)
        if not q.get("Nosaukums", "").startswith("Grozījum"):
            notes.append(f"Lapa {a} nav grozījumu likums: {q.get('Nosaukums', '')[:60]}")
            continue
        if not q.get("Pieņemts") or not q.get("Stājas spēkā"):
            notes.append(f"Grozījumu lapai {a} trūkst datumu")
            continue
        inf = q["Stājas spēkā"].rstrip(".")
        if "-".join(reversed(inf.split("."))) > asof_iso:
            continue  # vēl nav attēlotajā redakcijā
        amends.append((q["Pieņemts"].rstrip("."), short_pub(q.get("Publicēts", "")), inf))
    amends.sort(key=lambda x: "-".join(reversed(x[0].split("."))))
    if amends:
        ps.append(Para("Grozījumi:"))
        for ad, pub, inf in amends:
            ps.append(Para(f"{ad}. likums / {pub} / Stājas spēkā {inf}."))
    ps.append(Para(""))

    # --- korpuss ---
    r = lxml.html.fromstring(page)
    body = r.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' TV207 ')]")[0].getparent()
    tp = False
    annexes, ann = [], None
    for el in body:
        if not isinstance(el.tag, str) or el.tag == "style":
            continue
        cls = (el.get("class") or "").split()
        c = cls[0] if cls else ""
        if c == "TV218":  # pielikums: virsraksts + redakcijas piezīme; viss tālākais pieder pielikumam
            e = copy.deepcopy(el)
            pn = [norm(ptext(x)) for x in e.findall("p")]
            for x in e.findall("p"):
                e.remove(x)
            ann = {"heading": norm(ptext(e)), "title": None, "lines": [t for t in pn if t]}
            annexes.append(ann); continue
        if ann is not None:
            if c == "TV212" and ann["title"] is None and not ann["lines"][1:]:
                e = copy.deepcopy(el)
                pn = [norm(ptext(x)) for x in e.findall("p")]
                for x in e.findall("p"):
                    e.remove(x)
                ann["title"] = norm(ptext(e)) or None
                ann["lines"].extend(t for t in pn if t)  # piem. "(Pielikums izslēgts ar …)"
                continue
            rows = el.xpath(".//tr")
            if rows:
                for tr in rows:
                    cells = [norm(ptext(td)) for td in tr if td.tag in ("td", "th")]
                    if any(cells): ann["lines"].append(" | ".join(x for x in cells if x))
                continue
            ps_ = el.findall("p")
            for t in ([norm(ptext(x)) for x in ps_] if ps_ else [norm(ptext(el))]):
                if t: ann["lines"].append(t)
            continue
        if c == "TV206":
            ps.append(Para(norm(ptext(el))))
        elif c == "TV207":
            ps.append(Para(norm(ptext(el))))
        elif c in ("TV212", "TV214"):
            # nodaļa / "Pārejas noteikumi": virsraksts + atsevišķas piezīmes (p.labojumu_pamats)
            e = copy.deepcopy(el)
            pn = [norm(ptext(x)) for x in e.findall("p")]
            for x in e.findall("p"):
                e.remove(x)
            head = norm(ptext(e))
            if c == "TV212":
                ps.append(Para(chapter_roman(head), "Heading 1"))
            else:
                ps.append(Para(head)); tp = True
            ps.extend(Para(t) for t in pn if t)
        elif c in ("TV213", "TV208"):
            eu = c == "TV208"
            for x in el:
                if x.tag != "p":
                    continue
                xc = (x.get("class") or "").split()
                t = norm(ptext(x))
                if not t:
                    continue
                if "labojumu_pamats" in xc:
                    ps.append(Para(t)); continue
                if "TVP" in xc and not eu:
                    m = ART.match(t)
                    if m:
                        rest = t[m.end():].strip()
                        # vecāka stila panti bez nosaukuma: "2.pants. (1) …" vai "13.pants. Teikums."
                        if rest and (PART.match(rest) or rest[-1] in ".:;" or re.match(r"^\((Izslēgt|izslēgt)", rest)):
                            ps.append(Para(t[:m.end()].strip(), "Heading 2"))
                            ps.extend(classify(rest, [], tp, notes))
                        elif rest and DEL_TAIL.search(rest):  # "3.¹ pants. Nosaukums (Izslēgts ar …)"
                            md = DEL_TAIL.search(rest)
                            ps.append(Para((t[:m.end()].strip() + " " + rest[:md.start()].strip()).strip(), "Heading 2"))
                            ps.extend(classify(md.group(1), [], tp, notes))
                        else:
                            ps.append(Para(t, "Heading 2"))
                        continue
                    m = TPNUM.match(re.sub(r"^(\d+)\.([" + SUPC + r"]+)\s*", r"\1\2. ", t))  # "48.¹ …" = 48¹.
                    if tp and m:
                        ps.append(Para(m.group(3), num=(m.group(1) + m.group(2), 0))); continue
                    ps.append(Para(t)); continue
                if eu:
                    m = POINT.match(t)
                    ps.append(Para(m.group(3), num=(m.group(1) + m.group(2), 0)) if m else Para(t)); continue
                ps.extend(classify(t, xc, tp, notes))
        elif c in ("TV215", "TV216", "TV217"):
            for line in ptext(el).split("\n"):
                if norm(line):
                    ps.append(Para(norm(line)))
        elif c == "TV403":
            t = norm(ptext(el)).replace("direktīvām(", "direktīvām (")
            if t.startswith("Informatīva atsauce uz Eiropas Savienības"):
                ps.append(Para(t))
            else:  # piem. izslēgta atsauce uz regulu — vienā likumā drīkst būt tikai viena EuReference
                notes.append(f"Izlaists atsauces bloks (nav ES direktīvu atsauce): {t[:90]}")
        elif c == "TV430":  # redakcijas "PIEZĪMES" pēc paraksta — nav likuma teksts
            notes.append(f"Izlaists piezīmju bloks pēc paraksta: {norm(ptext(el))[:90]}")
        else:
            t = norm(ptext(el))
            if t:
                notes.append(f"Nezināms bloks {c}: {t[:60]}")
                ps.append(Para(t))
    ex = Explicit()
    ex.annexes = annexes
    return ps, ex, raw, f"likumi.lv/ta/id/{likumi_id} (HTML)", notes


if __name__ == "__main__":
    if len(sys.argv) == 4 and sys.argv[1] == "fetch":
        fetch(sys.argv[2], sys.argv[3])
    else:
        print(__doc__)
        sys.exit(2)