#!/usr/bin/env python3
"""Cek fakta #3313 tahap 4 - teks pasal dari salinan HTML (datahukum.com UU 30/1999, catatanhukum Perma 1/2016)."""
import re, urllib.request, urllib.error

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36"}


def get(u, n=3000000):
    try:
        r = urllib.request.Request(u, headers=UA)
        with urllib.request.urlopen(r, timeout=90) as x:
            return x.status, x.read(n)
    except urllib.error.HTTPError as e:
        return e.code, b""
    except Exception as e:
        return type(e).__name__, b""


def teks(b):
    t = b.decode("utf-8", "replace")
    t = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", t, flags=re.S | re.I)
    t = re.sub(r"</(p|div|li|h[1-6]|br)>", "\n", t, flags=re.I)
    t = re.sub(r"<br\s*/?>", "\n", t, flags=re.I)
    t = re.sub(r"<[^>]+>", " ", t)
    t = re.sub(r"&nbsp;?", " ", t)
    t = re.sub(r"&amp;", "&", t)
    return re.sub(r"[ \t]+", " ", t)


def lihat(t, pat, ket, sblm=40, sdhn=560):
    ms = list(re.finditer(pat, t, re.I))
    print(f"\n-- {ket} [{pat}] ({len(ms)} temuan)")
    for m in ms[:2]:
        seg = re.sub(r"\s+", " ", t[max(0, m.start() - sblm):m.start() + sdhn])
        print("  >", seg)


st, b = get("https://datahukum.com/peraturan/UU_30_Tahun_1999.html")
print(f"=== datahukum UU 30/1999 -> HTTP {st} {len(b)}B")
t = teks(b)
print(f"    teks {len(t)}")
for pat, lab in [
    (r"Pasal 1\s*\n?\s*(Dalam Undang-Undang ini|Dalam Undang)", "Pasal 1 definisi"),
    (r"Perjanjian arbitrase adalah", "definisi perjanjian arbitrase"),
    (r"Pasal 11", "Pasal 11"),
    (r"Pasal 60", "Pasal 60"),
    (r"Pasal 70", "Pasal 70"),
    (r"Pasal 59", "Pasal 59"),
]:
    lihat(t, pat, lab)

st, b = get("https://www.catatanhukum.com/dbs-perma/Perma_No_1_Tahun_2016/index.html")
print(f"\n=== catatanhukum Perma 1/2016 -> HTTP {st} {len(b)}B")
tp = teks(b)
print(f"    teks {len(tp)}")
for pat, lab in [
    (r"Pasal 4\b", "Pasal 4 mediasi wajib"),
    (r"Pasal 7\b", "Pasal 7"),
    (r"Pasal 17\b", "Pasal 17"),
    (r"Pasal 18\b", "Pasal 18"),
    (r"Pasal 24\b", "Pasal 24"),
]:
    lihat(tp, pat, lab)