#!/usr/bin/env python3
"""Anti-redundant: cari artikel website yang BELUM PERNAH dipakai di jadwal Repliz mana pun.

Dump semua jadwal (sudah ada) → set URL terpakai per domain (normalize) → diff dengan WP REST.
Output: /tmp/artikel_fresh.json
"""
import json, re, subprocess, sys
from urllib.parse import urlparse

# PITFALL (30 Sep 2026): path dump dulu HARDCODE ke repliz_all_20260922.json → artikel yang
# dipakai sejak 22 Sep dianggap "fresh" (false positive; contoh: panduan-memilih-dokter-kulit-krian
# sudah tayang 22 Sep tapi tetap muncul fresh). Sekarang: pakai argumen, atau dump TERBARU di /tmp.
if len(sys.argv) > 1:
    DUMP = sys.argv[1]
else:
    import glob
    cands = sorted(glob.glob("/tmp/repliz_all_*.json"))
    if not cands:
        print("❌ Tidak ada dump /tmp/repliz_all_*.json — jalankan dump_semua_jadwal.py dulu", file=sys.stderr)
        sys.exit(1)
    DUMP = cands[-1]
print(f"dump: {DUMP}", file=sys.stderr)
SITES = {
    "theskincodeclinic.com": "https://www.theskincodeclinic.com/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "dasindonesia.com": "https://dasindonesia.com/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "tslaw.id": "https://www.tslaw.id/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
    "studiosoft.id": "https://studiosoft.id/wp-json/wp/v2/posts?per_page=100&_fields=link,title,date",
}

def norm(u):
    u = u.strip().rstrip(").,;\"'")
    p = urlparse(u)
    host = p.netloc.lower().replace("www.", "")
    path = p.path.rstrip("/").lower()
    return f"{host}{path}"

docs = json.load(open(DUMP))
used = {}
for d in docs:
    for m in re.findall(r'https?://[^\s"\'\\]+', json.dumps(d, ensure_ascii=False)):
        n = norm(m)
        if not n or "/" not in n:
            continue
        used.setdefault(n.split("/")[0], set()).add(n)

out = {}
for domain, api in SITES.items():
    try:
        raw = subprocess.run(["curl", "-s", "--http1.1", "--max-time", "120", api],
                             capture_output=True, text=True, timeout=150).stdout
        posts = json.loads(raw)
    except Exception as e:
        print(f"  {domain}: WP REST gagal ({e})", file=sys.stderr)
        continue
    us = used.get(domain, set())
    fresh, dup = [], []
    for p in posts:
        link = p["link"]
        (dup if norm(link) in us else fresh).append(
            {"url": link, "title": re.sub(r"&#0?39;|’", "'", p["title"]["rendered"]), "date": p["date"][:10]})
    out[domain] = {"fresh": fresh, "used_count": len(posts) - len(fresh), "total": len(posts)}
    print(f"  {domain}: {len(posts)} artikel → FRESH {len(fresh)} | sudah dipakai {len(dup)}")

with open("/tmp/artikel_fresh.json", "w") as f:
    json.dump(out, f, indent=2, ensure_ascii=False)
print("\n=== FRESH (belum pernah dipakai) ===")
for domain, v in out.items():
    print(f"\n## {domain}")
    for a in v["fresh"][:25]:
        print(f"  {a['date']}  {a['title'][:65]}  {a['url']}")