#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TS Law — Harvest BERITA CNN Indonesia (12 Sep 2026, instruksi Bos).
Simpan berita CNN (nasional + ekonomi) yang relevan hukum ke arsip lokal,
supaya bisa dipelajari & dipakai jadi konten artikel website tslaw.id.

Output:
  - tslaw_cnn_arsip.json     : arsip lengkap (judul, url, kanal, tgl, isi paragraf,
                               niche hukum, angle, keyword) — anti-duplikat by URL
  - tslaw_cnn_digest_<tgl>.md : ringkasan harian yang gampang dibaca

Pakai: python3 tslaw_cnn_harvest.py [max_artikel]   (default 30)
"""
import sys, os, re, json, time, html, hashlib, urllib.request, urllib.error
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime

WIB = timezone(timedelta(hours=7))
ROOT = "/home/mantapman/.hermes/profiles/atun/workspace"
ARSIP = os.path.join(ROOT, "tslaw_cnn_arsip.json")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"

FEEDS = {
    "nasional": "https://www.cnnindonesia.com/nasional/rss",
    "ekonomi": "https://www.cnnindonesia.com/ekonomi/rss",
}

# ── Filter relevansi hukum (judul dulu, isi kalau perlu) ──
# KUAT = sinyal proses/isu hukum yang layak dibedah jadi artikel
KW_JUDUL = [
    "hukum", "pidana", "korupsi", "kpk", "kejagung", "kejaksaan", "pengadilan", "vonis",
    "tersangka", "terdakwa", "ditahan", "ditangkap", "polisi", "polres", "polda", "bareskrim",
    "kasus", "sengketa", "gugatan", "gugat", "perdata", "kontrak", "wanprestasi", "pailit",
    "arbitrase", "monopoli", "persaingan usaha", "kppu", "konsumen", "sanksi", "denda",
    "aturan", "uu ", "undang-undang", "regulasi", "peraturan", "legalitas", "izin",
    "pajak", "bea cukai", "cerai", "hak asuh", "warisan", "hutang", "utang", "pinjaman",
    "perampasan aset", "pencucian uang", "tipikor", "suap", "gratifikasi", "penipuan",
]
# KUAT: begitu muncul di judul → hampir pasti layak jadi artikel bedah hukum (Tier A)
KW_KUAT = [
    "hukum", "pidana", "korupsi", "kpk", "tipikor", "kejagung", "kejaksaan", "pengadilan",
    "vonis", "putusan", "tersangka", "terdakwa", "gugatan", "sengketa", "perdata",
    "kontrak", "wanprestasi", "pailit", "arbitrase", "monopoli", "kppu", "konsumen",
    "undang-undang", "uu ", "ruu", "regulasi", "peraturan", "pajak", "bea cukai",
    "perampasan aset", "pencucian uang", "suap", "gratifikasi", "phk", "buruh",
    "cerai", "hak asuh", "warisan", "pengacara", "mk ", "mahkamah", "kasasi",
]
# Kategori niche hukum (untuk angle artikel)
NICHE_RULES = [
    ("Hukum Pidana / Tipikor", ["korupsi", "kpk", "tipikor", "suap", "gratifikasi", "kejagung", "pencucian uang", "perampasan aset", "kerugian negara"]),
    ("Hukum Pidana", ["pidana", "tersangka", "terdakwa", "ditahan", "ditangkap", "polisi", "penganiayaan", "pembunuhan", "penipuan", "kelalaian", "horeg", "vonis", "pasal"]),
    ("Hukum Bisnis", ["monopoli", "persaingan usaha", "kppu", "kontrak", "investasi", "izin usaha", "pajak", "bea cukai", "devsa", "ekspor", "impor", "startup", "umkm", "pt ", "perseroan", "dagang"]),
    ("Hukum Perdata / Konsumen", ["konsumen", "gugatan", "perdata", "sengketa", "wanprestasi", "utang", "hutang", "pinjam", "jdih", "sewa", "warisan"]),
    ("Hukum Ketenagakerjaan", ["buruh", "pekerja", "upah", "phk", "ketenagakerjaan", "tenaga kerja"]),
    ("Hukum Keluarga", ["cerai", "hak asuh", "nikah", "waris", "keluarga", "kekerasan dalam rumah tangga", "kdrt"]),
    ("Hukum Tata Negara / Kebijakan", ["aturan", "uu ", "undang-undang", "regulasi", "peraturan", "dpr", "pemerintah", "menteri", "kebijakan"]),
]
ANGLE_HINTS = {
    "Hukum Pidana / Tipikor": "alur penyidikan-penuntutan, alat bukti tipikor, pemulihan kerugian negara",
    "Hukum Pidana": "unsur pasal, pembuktian, hak tersangka/korban, proses peradilan",
    "Hukum Bisnis": "kepatuhan usaha, risiko kontrak, sanksi administrasi & UU terkait",
    "Hukum Perdata / Konsumen": "hubungan perjanjian, wanprestasi vs PMH, hak konsumen & ganti rugi",
    "Hukum Ketenagakerjaan": "hubungan kerja, hak pekerja, prosedur PHK & pesangon",
    "Hukum Keluarga": "prosedur hukum keluarga di pengadilan agama/negeri, hak anak & nafkah",
    "Hukum Tata Negara / Kebijakan": "dasar hukum kebijakan & implikasi regulasi bagi publik/pelaku usaha",
}


def fetch(url, timeout=30):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "id-ID,id;q=0.9"})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode("utf-8", "replace")


def clean(s):
    s = re.sub(r"<[^>]+>", "", s)
    s = html.unescape(s)
    return re.sub(r"\s+", " ", s).strip()


def parse_rss(xml):
    out = []
    for it in re.findall(r"<item>(.*?)</item>", xml, re.S):
        def g(tag):
            m = re.search(rf"<{tag}>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</{tag}>", it, re.S)
            return m.group(1).strip() if m else ""
        link = g("link")
        if not link:
            continue
        try:
            dt = parsedate_to_datetime(g("pubDate")).astimezone(WIB)
        except Exception:
            dt = None
        out.append({
            "judul": clean(g("title")),
            "url": link,
            "tgl": dt.strftime("%Y-%m-%d %H:%M") if dt else "",
            "deskripsi": clean(g("description")),
        })
    return out


def extract_artikel(url):
    """Ambil isi artikel CNN dari div id=article-content-body."""
    h = fetch(url)
    judul = ""
    m = re.search(r"<h1[^>]*>(.*?)</h1>", h, re.S)
    if m:
        judul = clean(m.group(1))
    i = h.find('id="article-content-body"')
    body = ""
    if i != -1:
        seg = h[i:]
        end = seg.find("TRANSLATE_END")
        if end == -1:
            end = seg.find("</article>")
        seg = seg[:end] if end != -1 else seg[:20000]
        paras = [clean(p) for p in re.findall(r"<p[^>]*>(.*?)</p>", seg, re.S)]
        paras = [p for p in paras if len(p) > 40]
        body = "\n\n".join(paras)
    return judul, body


STOPW = {"yang","dengan","untuk","dari","dalam","pada","akan","tidak","ini","itu","dan","di","ke",
         "kasus","polisi","sebut","diduga","usai","soal","jadi","bisa","tiga","dua","baru","kata"}


def tier(judul, isi):
    """Tier A = layak dibedah jadi artikel; B = berita pendek/soft."""
    t = judul.lower()
    kuat = sum(1 for k in KW_KUAT if k in t)
    if kuat == 0:
        return "B"
    if kuat >= 2:
        return "A"
    return "A" if sum(1 for k in KW_JUDUL if k in isi.lower()) >= 4 else "B"


def klasifikasi(judul, isi):
    for teks in (judul, isi[:900]):          # judul dulu (paling akurat), fallback isi
        tl = teks.lower()
        for niche, kws in NICHE_RULES:
            if any(k in tl for k in kws):
                return niche, ANGLE_HINTS[niche]
    return "Hukum Umum", "aspek hukum umum & edukasi publik"


def tokens(judul):
    return {w for w in re.findall(r"[a-z]{5,}", judul.lower()) if w not in STOPW}


def klaster(berita):
    """Kelompokkan berita satu topik (mis. 6 artikel 'Pengeroyok Bambang Setiawan')."""
    for b in berita:
        b["topik_id"] = ""
        b["peran"] = "utama"
    groups = []
    for b in berita:
        tb = tokens(b["judul"])
        for g in groups:
            tg = g["tokens"]
            irisan = len(tb & tg)
            # overlap coefficient: tahan variasi judul berita satu isu yang sama
            if tb and tg and irisan / min(len(tb), len(tg)) >= 0.5:
                g["anggota"].append(b)
                g["tokens"] |= tb
                break
        else:
            groups.append({"tokens": set(tb), "anggota": [b]})
    for gi, g in enumerate(groups, 1):
        anggota = g["anggota"]
        utama = max(anggota, key=lambda x: x.get("jml_kata", 0))
        for b in anggota:
            b["topik_id"] = f"T{gi:02d}" if len(anggota) > 1 else ""
            b["peran"] = "utama" if b is utama else "varian"
    return groups


def load_arsip():
    if os.path.exists(ARSIP):
        with open(ARSIP, encoding="utf-8") as f:
            return json.load(f)
    return {"dibuat": datetime.now(WIB).strftime("%Y-%m-%d %H:%M WIB"), "berita": []}


def save_arsip(a):
    with open(ARSIP, "w", encoding="utf-8") as f:
        json.dump(a, f, indent=1, ensure_ascii=False)


def tulis_digest(arsip, baru, hari_ini):
    digest = os.path.join(ROOT, f"tslaw_cnn_digest_{hari_ini}.md")
    A = [b for b in arsip["berita"] if b.get("tier") == "A"]
    lines = [f"# Digest Berita CNN — {hari_ini} (WIB)", "",
             f"Arsip total: {len(arsip['berita'])} berita | Tier A (layak artikel): {len(A)} | baru hari ini: {len(baru)}", ""]
    by_niche = {}
    for b in (baru or arsip["berita"]):
        by_niche.setdefault(b["niche"], []).append(b)
    for niche, items in sorted(by_niche.items(), key=lambda x: -len(x[1])):
        lines.append(f"## {niche} ({len(items)})")
        for b in items:
            tag = "🅰️" if b.get("tier") == "A" else "🅱️"
            topik = f" [{b['topik_id']}/{b['peran']}]" if b.get("topik_id") else ""
            lines.append(f"- {tag}{topik} **{b['judul']}** [{b['tgl_berita'] or '-'}] — {b['jml_kata']} kata")
            if b.get("angle_hukum"):
                lines.append(f"  - angle: {b['angle_hukum']}")
            lines.append(f"  - {b['url']}")
        lines.append("")
    with open(digest, "w", encoding="utf-8") as f:
        f.write("\n".join(lines))
    return digest


def main():
    args = [a for a in sys.argv[1:]]
    ulang = "--ulang" in args
    args = [a for a in args if not a.startswith("--")]
    max_art = int(args[0]) if args else 30
    arsip = load_arsip()
    ada = {b["url"] for b in arsip["berita"]}
    hari_ini = datetime.now(WIB).strftime("%Y-%m-%d")
    baru = []

    if not ulang:
        kandidat = []
        for kanal, feed in FEEDS.items():
            try:
                items = parse_rss(fetch(feed))
            except Exception as e:
                print(f"❌ RSS {kanal}: {e}")
                continue
            print(f"📡 RSS {kanal}: {len(items)} item")
            for it in items:
                if it["url"] in ada:
                    continue
                teks = (it["judul"] + " " + it["deskripsi"]).lower()
                if any(k in teks for k in KW_JUDUL):
                    it["kanal"] = kanal
                    kandidat.append(it)

        print(f"🎯 Kandidat relevan hukum (belum ada di arsip): {len(kandidat)}")
        for it in kandidat[:max_art]:
            try:
                judul_web, isi = extract_artikel(it["url"])
                if not isi:
                    print(f"  ⚠️  isi kosong: {it['judul'][:60]}")
                    continue
                judul = it["judul"] or judul_web
                niche, angle = klasifikasi(judul, isi)
                entri = {
                    "id": hashlib.md5(it["url"].encode()).hexdigest()[:12],
                    "disimpan": datetime.now(WIB).strftime("%Y-%m-%d %H:%M WIB"),
                    "tgl_berita": it["tgl"],
                    "kanal": it["kanal"],
                    "judul": judul,
                    "url": it["url"],
                    "niche": niche,
                    "angle_hukum": angle,
                    "tier": tier(judul, isi),
                    "ringkasan": it["deskripsi"],
                    "isi": isi,
                    "jml_kata": len(isi.split()),
                    "status": "kandidat",
                    "artikel_website": "",
                }
                arsip["berita"].append(entri)
                baru.append(entri)
                print(f"  ✅ [{entri['tier']}] [{it['tgl']}] {niche} | {judul[:58]}")
            except Exception as e:
                print(f"  ❌ {it['url'][:70]}: {e}")
            time.sleep(1)
            arsip["diperbarui"] = datetime.now(WIB).strftime("%Y-%m-%d %H:%M WIB")
            save_arsip(arsip)

    # Re-kalkulasi klasifikasi/tier/klaster untuk SEMUA entri
    for b in arsip["berita"]:
        b["niche"], b["angle_hukum"] = klasifikasi(b["judul"], b.get("isi", ""))
        b["tier"] = tier(b["judul"], b.get("isi", ""))
        b["jml_kata"] = b.get("jml_kata") or len(b.get("isi", "").split())
    grup = klaster(arsip["berita"])
    arsip["diperbarui"] = datetime.now(WIB).strftime("%Y-%m-%d %H:%M WIB")
    save_arsip(arsip)

    digest = tulis_digest(arsip, baru, hari_ini)
    A = [b for b in arsip["berita"] if b["tier"] == "A"]
    print(f"\n📊 Arsip: {len(arsip['berita'])} berita total | +{len(baru)} baru | Tier A: {len(A)}")
    print(f"🧩 Klaster topik: {sum(1 for g in grup if len(g['anggota']) > 1)} grup (>=2 berita sejenis)")
    print(f"💾 {ARSIP}")
    print(f"📄 {digest}")


if __name__ == "__main__":
    main()