#!/usr/bin/env python3
"""Cari artikel studiosoft.id yang BELUM pernah dipakai di Threads (scan 4 halaman)."""
import json, re, sys
sys.path.insert(0, "/home/mantapman/.hermes/profiles/atun/workspace")
from openssl_fetch import fetch


def dechunk(body: bytes) -> bytes:
    out = b""
    i = 0
    while True:
        j = body.find(b"\r\n", i)
        if j < 0:
            break
        try:
            n = int(body[i:j].split(b";")[0].strip(), 16)
        except Exception:
            break
        if n == 0:
            break
        out += body[j + 2:j + 2 + n]
        i = j + 2 + n + 2
    return out

used = json.load(open("/home/mantapman/.hermes/profiles/atun/workspace/_recon_used_links.json"))
usedall = set(u.rstrip("/").lower() for u in used["used_api"]) | set(u.rstrip("/").lower() for u in used["used_log"])

rows = []
for pg in range(1, 5):
    url = f"https://studiosoft.id/wp-json/wp/v2/posts?per_page=100&page={pg}&orderby=date&order=desc&_fields=link,title,date"
    try:
        st, head, body = fetch(url)
    except Exception as e:
        print("ERR", pg, e); continue
    if st != 200:
        print("status", st, "page", pg, head[:200]); break
    try:
        data = json.loads(dechunk(body).decode("utf-8", "replace"))
    except Exception as e:
        print("JSONERR", pg, e); continue
    print(f"page {pg}: {len(data)} posts")
    rows.extend(data)
    if len(data) < 100:
        break

fresh = []
for p in rows:
    link = p.get("link", "")
    t = p.get("title")
    title = re.sub(r"<[^>]+>", "", t.get("rendered", "")) if isinstance(t, dict) else str(t)
    if link.rstrip("/").lower() not in usedall:
        fresh.append((p.get("date", "")[:10], link, title))

print(f"\nTOTAL {len(rows)} artikel, FRESH {len(fresh)}")
for d, l, t in fresh:
    print("  ", d, "|", l, "|", t[:90])
json.dump(fresh, open("/home/mantapman/.hermes/profiles/atun/workspace/_fresh_ss_15okt.json", "w"), indent=1, ensure_ascii=False)