#!/usr/bin/env python3
"""Ambil 100 artikel terbaru tiap domain klien (WP REST, cache-buster) lalu diff
vs SEMUA URL yang sudah dipakai di jadwal Repliz (dump terbaru).

Normalisasi: host tanpa www + path tanpa slash akhir, lowercase.
Output: /tmp/kandidat_29.json
"""
import json, re, subprocess, sys, time
from urllib.parse import urlparse

DUMP = sys.argv[1] if len(sys.argv) > 1 else "/tmp/repliz_all_20260929b.json"
SITES = {
    "tslaw.id": "https://www.tslaw.id",
    "theskincodeclinic.com": "https://www.theskincodeclinic.com",
    "dasindonesia.com": "https://dasindonesia.com",
}


def norm(u):
    u = u.strip().rstrip('\").,;\\"\'')
    p = urlparse(u)
    host = p.netloc.lower().replace("www.", "")
    path = p.path.rstrip("/").lower()
    if not host:
        return ""
    return "%s%s" % (host, path)


docs = json.load(open(DUMP))
if isinstance(docs, dict):
    docs = docs.get("docs", docs.get("data", []))
used = set()
for d in docs:
    for m in re.findall(r'https?://[^\s"\'\\]+', json.dumps(d, ensure_ascii=False)):
        n = norm(m)
        if n:
            used.add(n)
print("URL unik terpakai:", len(used))

ts = int(time.time())
out = {}
for domain, base in SITES.items():
    api = "%s/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date&cb=%d" % (base, ts)
    raw = subprocess.run(["curl", "-4", "-s", "--http1.1", "--max-time", "60", api],
                         capture_output=True, text=True, timeout=90).stdout
    try:
        posts = json.loads(raw)
    except Exception as e:
        print("  %s: WP REST gagal (%s) raw=%s" % (domain, e, raw[:120]))
        continue
    fresh = []
    for p in posts:
        link = p["link"]
        if norm(link) not in used:
            fresh.append({"url": link,
                          "title": re.sub(r"&#0?39;|’", "'", p["title"]["rendered"]),
                          "date": p["date"][:10]})
    out[domain] = fresh
    print("\n## %s: %d artikel -> FRESH %d" % (domain, len(posts), len(fresh)))
    for a in fresh[:30]:
        print("   %s  %s\n      %s" % (a["date"], a["title"][:80], a["url"]))

json.dump(out, open("/tmp/kandidat_29.json", "w"), ensure_ascii=False, indent=1)
print("\n-> /tmp/kandidat_29.json")