#!/usr/bin/env python3
"""Cari artikel FRESH (belum pernah muncul di jadwal mana pun + log) per domain, 12 Okt 2026."""
import json, re, subprocess

DUMP = "/tmp/repliz_all_20261011.json"
LOG = "/home/mantapman/.hermes/profiles/atun/workspace/log_threads_harian.json"
URL_RE = re.compile(r"https?://[^\s\)\]\"]+")


def norm(u):
    u = u.rstrip("/.,").lower()
    for suf in (".htm", ".html"):
        if u.endswith(suf):
            u = u[: -len(suf)]
    if u.startswith("https://www."):
        u = "https://" + u[12:]
    return u


docs = json.load(open(DUMP))
seen = set()
for d in docs:
    txt = " ".join(str(d.get(k) or "") for k in ("description", "title", "topic", "medias"))
    for u in URL_RE.findall(txt):
        seen.add(norm(u))

log = json.load(open(LOG))
for _d, per_acc in (log.get("threads_used_links") or {}).items():
    for _a, items in per_acc.items():
        for it in items:
            if it.get("url"):
                seen.add(norm(it["url"]))
print(f"seen URL: {len(seen)} (dari {len(docs)} jadwal + log)")


def wp(domain):
    out = []
    for p in range(1, 6):
        url = (f"https://{domain}/wp-json/wp/v2/posts?per_page=100&page={p}"
               f"&_fields=link,title,date&orderby=date&order=desc")
        r = subprocess.run(["curl", "-4", "-s", "-m", "35", "-A", "Mozilla/5.0", url],
                           capture_output=True, text=True)
        try:
            data = json.loads(r.stdout)
        except Exception:
            print(f"  !! {domain} page {p} gagal: {r.stdout[:120]!r}")
            break
        if isinstance(data, dict):
            print(f"  !! {domain} page {p}: {str(data)[:150]}")
            break
        if not data:
            break
        out.extend(data)
        if len(data) < 100:
            break
    return out


for dom, label in [("studiosoft.id", "StudioSoft"), ("www.tslaw.id", "TSLaw"),
                   ("www.theskincodeclinic.com", "TSC"), ("dasindonesia.com", "DAS")]:
    posts = wp(dom)
    fresh = [p for p in posts if norm(p["link"]) not in seen]
    print(f"\n===== {label}: {len(posts)} artikel, {len(fresh)} FRESH =====")
    for p in fresh[:30]:
        t = p["title"]["rendered"] if isinstance(p["title"], dict) else str(p["title"])
        t = re.sub(r"<[^>]+>", "", t)
        print(f"  {p['date'][:10]}  {p['link']}")
        print(f"      {t}")