#!/usr/bin/env python3
"""Cari placeholder nomor dan bentuk nomor lain di konten TSC (termasuk tanda hubung non-ASCII)."""
import base64, importlib.util, json, re, time, urllib.error, urllib.request

WS = "/home/mantapman/.hermes/profiles/atun/workspace"
spec = importlib.util.spec_from_file_location("cfg", f"{WS}/cfg_tsc.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
CFG = m.CFG
AUTH = base64.b64encode(f"{CFG['user']}:{CFG['pass']}".encode()).decode()
UA = {"User-Agent": "Mozilla/5.0"}

CARI = [r"XXXX", r"0\d{2}[\s\u2010-\u2015\-]?\d{3,4}[\s\u2010-\u2015\-]?\d{3,5}",
        r"\+62[\s\u2010-\u2015\-]?\d{2,4}[\s\u2010-\u2015\-]?\d{3,4}[\s\u2010-\u2015\-]?\d{3,5}",
        r"telepon|telp|phone|nomor\s+wa|wa\s*:",
        r"contact|kontak"]


def rest(path):
    h = {"Authorization": f"Basic {AUTH}", **UA}
    r = urllib.request.Request(f"https://{CFG['domain']}{path}", headers=h)
    for _ in range(4):
        try:
            with urllib.request.urlopen(r, timeout=90) as x:
                return json.loads(x.read())
        except Exception:
            time.sleep(4)
    return {}


hasil = {}
for jenis in ("posts", "pages"):
    page = 1
    while True:
        d = rest(f"/wp-json/wp/v2/{jenis}?per_page=100&page={page}&status=publish&context=edit&_fields=id,link,content")
        if not isinstance(d, list) or not d:
            print(f"  (berhenti di {jenis} halaman {page})")
            break
        for x in d:
            teks = re.sub(r"<[^>]+>", " ", x["content"]["raw"]).replace("&nbsp;", " ")
            for p in CARI:
                for m2 in re.finditer(p, teks, re.I):
                    kunci = f"{p[:22]} → «{m2.group(0).strip()[:40]}»"
                    hasil.setdefault(kunci, {"contoh": re.sub(r"\s+", " ", teks[max(0, m2.start() - 70):m2.end() + 50]),
                                             "halaman": []})
                    if x["link"] not in hasil[kunci]["halaman"]:
                        hasil[kunci]["halaman"].append(x["link"])
        if len(d) < 100:
            break
        page += 1
        time.sleep(0.4)

print(f"\nbentuk ditemukan: {len(hasil)}\n")
for k, v in sorted(hasil.items(), key=lambda y: -len(y[1]["halaman"])):
    print(f"■ {k} · {len(v['halaman'])} halaman")
    print(f"   …{v['contoh'][:170]}…")
    for u in v["halaman"][:2]:
        print(f"   {u[:86]}")
json.dump(hasil, open(f"{WS}/drafts/tsc_nomor_scan2.json", "w"), ensure_ascii=False, indent=1)