#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Bangun kandidat artikel FRESH per domain utk Threads 4 Okt 2026.

Sumber kandidat: /tmp/threads_fresh_oct3.json (fetch WP REST 2 Okt) + /tmp/wp40_*.json (30 Sep).
Filter: buang URL yang sudah muncul di dump jadwal TERBARU (/tmp/repliz_all_20261004.json)
dan di log_threads_harian.json (threads_used_links).
Output: /tmp/kandidat_20261004.json
"""
import glob, json, re
from urllib.parse import urlparse

def norm(u):
    u = u.strip().rstrip(").,;\"'")
    p = urlparse(u)
    return f"{p.netloc.lower().replace('www.','')}{p.path.rstrip('/').lower()}"

# ---- used set dari dump terbaru + semua dump lama + log
used = set()
dumps = sorted(glob.glob("/tmp/repliz_all_*.json"))
DUMP = dumps[-1]
for fn in [DUMP]:
    docs = json.load(open(fn))
    if isinstance(docs, dict):
        docs = docs.get("docs") or docs.get("data") or []
    for d in docs:
        for m in re.findall(r'https?://[^\s"\'\\]+', json.dumps(d, ensure_ascii=False)):
            used.add(norm(m))
print(f"dump: {fn} docs={len(docs)} used={len(used)}")

log = json.load(open("/home/mantapman/.hermes/profiles/atun/workspace/log_threads_harian.json"))
for day, accs in (log.get("threads_used_links") or {}).items():
    for acc, items in accs.items():
        for it in items:
            used.add(norm(it["url"]))
print(f"used + log = {len(used)}")

# ---- kandidat
pool = {}
def add(dom, url, title, date):
    n = norm(url)
    if n in used:
        return
    pool.setdefault(dom, {})
    if n not in pool[dom]:
        pool[dom][n] = {"url": url, "title": re.sub("&#0?39;|’", "'", title), "date": date}

tf = json.load(open("/tmp/threads_fresh_oct3.json"))
DOMOF = {"tslaw": "tslaw.id", "tsc": "theskincodeclinic.com", "das": "dasindonesia.com",
         "uky": "studiosoft.id", "studiosoft": "studiosoft.id"}
for acc, items in tf.items():
    dom = DOMOF.get(acc)
    if not dom:
        continue
    for p in items:
        t = p.get("title")
        t = t.get("rendered") if isinstance(t, dict) else t
        add(dom, p["link"], str(t or ""), p.get("date", "")[:10])

for fn, dom in [("/tmp/wp40_studiosoft.id.json", "studiosoft.id"),
                ("/tmp/wp40_www.tslaw.id.json", "tslaw.id"),
                ("/tmp/wp40_www.theskincodeclinic.com.json", "theskincodeclinic.com"),
                ("/tmp/wp40_dasindonesia.com.json", "dasindonesia.com")]:
    try:
        arr = json.load(open(fn))
    except Exception as e:
        print(f"  {fn}: {e}")
        continue
    for p in arr:
        t = p.get("title")
        t = t.get("rendered") if isinstance(t, dict) else t
        add(dom, p["link"], str(t or ""), p.get("date", "")[:10])

out = {}
for dom, d in pool.items():
    lst = sorted(d.values(), key=lambda x: x["date"], reverse=True)
    out[dom] = lst
    print(f"\n## {dom}: {len(lst)} fresh")
    for a in lst[:12]:
        print(f"   {a['date']} | len(url)={len(a['url'])} | {a['title'][:72]}")
        print(f"        {a['url']}")

json.dump(out, open("/tmp/kandidat_20261004.json", "w"), indent=1, ensure_ascii=False)