#!/usr/bin/env python3
"""Diff kandidat artikel vs URL yang SUDAH dipakai di SEMUA jadwal Repliz.

Normalisasi: host tanpa www + path tanpa slash akhir, lowercase.
Sumber kandidat: /tmp/artikel_fresh.json (80an artikel terbaru per domain, 21 Sep 2026)
plus daftar URL artikel segar dari WP REST kalau tersedia.
"""
import json, re, sys
from urllib.parse import urlparse

FRESH = sys.argv[1] if len(sys.argv) > 1 else "/tmp/artikel_fresh.json"
DUMP = sys.argv[2] if len(sys.argv) > 2 else "/tmp/repliz_all_20260928.json"
OUT = "/tmp/kandidat_artikel_hari_ini.json"
URL_RE = re.compile(r'https?://[^\s"\'\\]+')


def norm(u):
    u = u.strip().rstrip('").,;"\'')
    p = urlparse(u)
    host = p.netloc.lower().replace("www.", "")
    path = p.path.rstrip("/").lower()
    if not host:
        return ""
    return "%s%s" % (host, path)


docs = json.load(open(DUMP))
used = set()
for d in docs:
    for m in URL_RE.findall(json.dumps(d, ensure_ascii=False)):
        n = norm(m)
        if n:
            used.add(n)
print("URL unik terpakai (normalisasi):", len(used))

fresh = json.load(open(FRESH))
out = {}
for domain, v in fresh.items():
    keep = [a for a in v["fresh"] if norm(a["url"]) not in used]
    out[domain] = keep
    print("\n## %s: kandidat fresh %d (dari list %d)" % (domain, len(keep), len(v["fresh"])))
    for a in keep[:25]:
        print("   %s  %s" % (a["date"], a["url"]))

json.dump(out, open(OUT, "w"), ensure_ascii=False, indent=1)
print("\n-> %s" % OUT)