#!/usr/bin/env python3
"""Daftar lengkap penyebutan 'Klinik DNI' di situs TSC: di konten (REST) dan di halaman tampil (HTML)."""
import base64, importlib.util, json, re, sys, time, urllib.error, urllib.request
from collections import Counter
from concurrent.futures import ThreadPoolExecutor

WS = "/home/mantapman/.hermes/profiles/atun/workspace"
spec = importlib.util.spec_from_file_location("cfg", f"{WS}/cfg_tsc.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
CFG = m.CFG
DOM = CFG["domain"]
AUTH = base64.b64encode(f"{CFG['user']}:{CFG['pass']}".encode()).decode()
UA = {"User-Agent": "Mozilla/5.0"}
POLA = re.compile(r"Klinik\s*DNI(?:\s*Krian|\s*Sidoarjo)?", re.I)


def rest(path):
    h = {"Authorization": f"Basic {AUTH}", **UA}
    r = urllib.request.Request(f"https://{DOM}{path}", headers=h)
    for _ in range(3):
        try:
            with urllib.request.urlopen(r, timeout=180) as x:
                return json.loads(x.read())
        except urllib.error.HTTPError as e:
            return {"err": e.code}
        except Exception:
            time.sleep(4)
    return {}


print("=== A. di konten WordPress (yang bisa disunting) ===")
for jenis in ("posts", "pages"):
    page = 1
    while True:
        d = rest(f"/wp-json/wp/v2/{jenis}?per_page=100&page={page}&status=publish&context=edit"
                 f"&_fields=id,slug,link,title,content,excerpt")
        if not isinstance(d, list) or not d:
            break
        for x in d:
            bagian = {"judul": x["title"]["raw"], "slug": x["slug"], "excerpt": x.get("excerpt", {}).get("raw", ""),
                      "isi": x["content"]["raw"]}
            hit = {k: POLA.findall(v or "") for k, v in bagian.items()}
            hit = {k: v for k, v in hit.items() if v}
            if not hit:
                continue
            print(f"\n  #{x['id']} · {jenis} · {x['link']}")
            for k, v in hit.items():
                print(f"     {k}: {len(v)}× {sorted(set(v))}")
            for mc in list(POLA.finditer(bagian["isi"]))[:6]:
                awal = max(0, mc.start() - 130)
                potong = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", bagian["isi"][awal:mc.end() + 90])).strip()
                print(f"        ... {potong}")
        if len(d) < 100:
            break
        page += 1

print("\n=== B. di halaman tampil (783 URL dari sitemap) ===")


def ambil(u):
    try:
        t = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=25).read().decode("utf-8", "replace")
        return u, len(POLA.findall(t)), None
    except Exception as e:
        return u, 0, str(e)[:30]


sm = urllib.request.urlopen(urllib.request.Request(f"https://www.{DOM}/sitemap_index.xml", headers=UA),
                            timeout=60).read().decode("utf-8", "replace")
sub = re.findall(r"<loc>([^<]+)</loc>", sm)
urls = []
for s in sub:
    try:
        x = urllib.request.urlopen(urllib.request.Request(s, headers=UA), timeout=60).read().decode("utf-8", "replace")
        urls += re.findall(r"<loc>([^<]+)</loc>", x)
    except Exception:
        continue
urls = sorted(set(u for u in urls if u.startswith("http")))
hasil, gagal = [], 0
with ThreadPoolExecutor(max_workers=10) as ex:
    for i, (u, n, err) in enumerate(ex.map(ambil, urls), 1):
        if err:
            gagal += 1
        elif n:
            hasil.append((u, n))
        if i % 200 == 0:
            print(f"   ...{i}/{len(urls)} (temuan {len(hasil)})", flush=True)

print(f"\n   halaman yang menampilkan 'Klinik DNI': {len(hasil)} dari {len(urls)} (gagal {gagal})")
for u, n in sorted(hasil, key=lambda x: -x[1]):
    print(f"   {n:>3}× {u}")