#!/usr/bin/env python3
"""Cari SEMUA nomor WhatsApp (dan telepon) di seluruh halaman TSC, lalu normalkan dan kelompokkan."""
import re, sys
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
import urllib.request

UA = {"User-Agent": "Mozilla/5.0 (compatible; AtunBot/1.0)"}
DOM = "theskincodeclinic.com"

POLA = [
    ("wa.me", re.compile(r"wa\.me/([0-9%\s+]{7,20})")),
    ("api.whatsapp", re.compile(r"phone=([0-9%\s+]{7,20})")),
    ("tel", re.compile(r"tel:([0-9+\-\s()]{7,20})")),
]


def normal(n):
    n = n.replace("%20", "").replace("+", "").strip()
    d = re.sub(r"\D", "", n)
    if not d:
        return ""
    if d.startswith("0"):
        d = "62" + d[1:]
    elif d.startswith("8"):
        d = "62" + d
    if not d.startswith("62") and len(d) > 6:
        d = "62" + d.lstrip("0")
    return d


def ambil(u):
    try:
        t = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=25).read().decode("utf-8", "replace")
    except Exception as e:
        return u, {}, str(e)[:40]
    hasil = {}
    for jenis, pola in POLA:
        for m in pola.finditer(t):
            nm = normal(m.group(1))
            if len(nm) >= 10:
                hasil.setdefault(nm, set()).add(jenis)
    return u, hasil, None


sm = urllib.request.urlopen(urllib.request.Request(f"https://www.{DOM}/sitemap_index.xml", headers=UA),
                            timeout=60).read().decode("utf-8", "replace")
sub = re.findall(r"<loc>([^<]+)</loc>", sm)
urls = []
for s in sub:
    try:
        x = urllib.request.urlopen(urllib.request.Request(s, headers=UA), timeout=60).read().decode("utf-8", "replace")
        urls += re.findall(r"<loc>([^<]+)</loc>", x)
    except Exception:
        continue
urls = sorted(set(u for u in urls if u.startswith("http")))
print(f"{len(urls)} URL diperiksa", flush=True)

per_nomor = defaultdict(lambda: {"halaman": [], "jenis": set()})
gagal = 0
with ThreadPoolExecutor(max_workers=10) as ex:
    for i, (u, hasil, err) in enumerate(ex.map(ambil, urls), 1):
        if err:
            gagal += 1
        for nm, jenis in (hasil or {}).items():
            per_nomor[nm]["halaman"].append(u)
            per_nomor[nm]["jenis"] |= jenis
        if i % 150 == 0:
            print(f"  ...{i}/{len(urls)}", flush=True)

print(f"\n=== nomor ditemukan: {len(per_nomor)} · halaman gagal: {gagal} ===\n")
for nm in sorted(per_nomor, key=lambda x: -len(per_nomor[x]["halaman"])):
    d = per_nomor[nm]
    print(f"■ {nm} · {len(d['halaman'])} halaman · jenis {sorted(d['jenis'])}")
    for u in d["halaman"][:5]:
        print(f"    {u}")
    if len(d["halaman"]) > 5:
        print(f"    ... dan {len(d['halaman']) - 5} halaman lain")