#!/usr/bin/env python3
"""Anti-redundant v2: diff artikel WP REST vs dump TERBARU (argumen), dengan retry.
Usage: python3 cek_fresh_new.py /tmp/repliz_all_20260930.json
Output: /tmp/artikel_fresh2.json + cetak FRESH per domain.
"""
import json, re, subprocess, sys, time
from urllib.parse import urlparse

DUMP = sys.argv[1] if len(sys.argv) > 1 else "/tmp/repliz_all_20260930.json"
SITES = {
    "theskincodeclinic.com": "https://www.theskincodeclinic.com/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "dasindonesia.com": "https://dasindonesia.com/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "tslaw.id": "https://www.tslaw.id/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "studiosoft.id": "https://studiosoft.id/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
}


def norm(u):
    u = u.strip().rstrip(").,;\"'")
    p = urlparse(u)
    return f"{p.netloc.lower().replace('www.', '')}{p.path.rstrip('/').lower()}"


docs = json.load(open(DUMP))
used = {}
for d in docs:
    for m in re.findall(r'https?://[^\s"\'\\]+', json.dumps(d, ensure_ascii=False)):
        n = norm(m)
        if not n or "/" not in n:
            continue
        used.setdefault(n.split("/")[0], set()).add(n)
print(f"dump {DUMP}: {len(docs)} jadwal; domain terpakai: " +
      ", ".join(f"{k}={len(v)}" for k, v in sorted(used.items())))

out = {}
for domain, api in SITES.items():
    posts = None
    for i in range(8):
        raw = subprocess.run(["curl", "-s", "--http1.1", "--max-time", "60", api],
                             capture_output=True, text=True, timeout=90).stdout
        try:
            posts = json.loads(raw)
            break
        except Exception:
            time.sleep(3)
    if not posts:
        print(f"  {domain}: WP REST GAGAL", file=sys.stderr)
        continue
    us = used.get(domain, set())
    fresh = []
    for p in posts:
        link = p["link"]
        if norm(link) not in us:
            fresh.append({"url": link, "title": re.sub(r"&#0?39;|’", "'", p["title"]["rendered"]),
                          "date": p["date"][:10]})
    out[domain] = {"fresh": fresh, "total": len(posts), "used": len(posts) - len(fresh)}

with open("/tmp/artikel_fresh2.json", "w") as f:
    json.dump(out, f, indent=2, ensure_ascii=False)

for domain, v in sorted(out.items()):
    print(f"\n## {domain}  fresh {len(v['fresh'])}/{v['total']}")
    for a in v["fresh"][:12]:
        print(f"   {a['date']}  {a['title'][:60]}  {a['url']}")