#!/usr/bin/env python3
"""Ambil kandidat artikel fresh dari WP REST tiap situs + diff vs link terpakai."""
import json, re, sys
sys.path.insert(0, "/home/mantapman/.hermes/profiles/atun/workspace")
from openssl_fetch import fetch

SITES = {
    "studiosoft": "https://studiosoft.id/wp-json/wp/v2/posts?per_page=30&orderby=date&order=desc&_fields=link,title,date",
    "tslaw": "https://www.tslaw.id/wp-json/wp/v2/posts?per_page=30&orderby=date&order=desc&_fields=link,title,date",
    "tsc": "https://www.theskincodeclinic.com/wp-json/wp/v2/posts?per_page=30&orderby=date&order=desc&_fields=link,title,date",
    "das": "https://dasindonesia.com/wp-json/wp/v2/posts?per_page=30&orderby=date&order=desc&_fields=link,title,date",
}

used = json.load(open("/home/mantapman/.hermes/profiles/atun/workspace/_recon_used_links.json"))
usedall = set(u.rstrip("/").lower() for u in used["used_api"]) | set(u.rstrip("/").lower() for u in used["used_log"])

def dechunk(body: bytes) -> bytes:
    out = b""
    i = 0
    while True:
        j = body.find(b"\r\n", i)
        if j < 0:
            break
        try:
            n = int(body[i:j].split(b";")[0].strip(), 16)
        except Exception:
            break
        if n == 0:
            break
        out += body[j + 2:j + 2 + n]
        i = j + 2 + n + 2
    return out


out = {}
for k, url in SITES.items():
    print(f"\n########## {k} ##########")
    try:
        st, head, body = fetch(url)
    except Exception as e:
        print("FETCH ERROR", e); continue
    print("status", st, "bytes", len(body))
    if st != 200:
        print(head[:500]); continue
    try:
        data = json.loads(dechunk(body).decode("utf-8", "replace"))
    except Exception as e:
        print("JSON ERROR", e, body[:300]); continue
    rows = []
    for p in data:
        link = p.get("link", "")
        title = re.sub(r"<[^>]+>", "", (p.get("title") or {}).get("rendered", "") or p.get("title") if isinstance(p.get("title"), dict) else str(p.get("title")))
        fresh = link.rstrip("/").lower() not in usedall
        rows.append({"link": link, "title": title, "date": p.get("date", ""), "fresh": fresh})
    out[k] = rows
    for r in rows:
        print(("  FRESH  " if r["fresh"] else "  used   ") + r["date"][:10] + " | " + r["link"] + " | " + r["title"][:80])

json.dump(out, open("/home/mantapman/.hermes/profiles/atun/workspace/_kandidat_15okt.json", "w"), indent=1, ensure_ascii=False)
print("\nFRESH COUNTS:", {k: sum(1 for r in v if r["fresh"]) for k, v in out.items()})