#!/usr/bin/env python3
"""Ambil artikel BARU dari WP REST API klien lewat openssl_fetch (curl di-DPI-blok).

Output: /tmp/fresh_articles.json  {akun: [ {link,title,date}, ... ]}  (hanya yang BELUM pernah dipakai)
"""
import json, re, sys, time

sys.path.insert(0, "/home/mantapman/.hermes/profiles/atun/workspace")
from openssl_fetch import fetch

SITES = {
    "studiosoft": "https://studiosoft.id",
    "tslaw": "https://www.tslaw.id",
    "tsc": "https://www.theskincodeclinic.com",
    "das": "https://dasindonesia.com",
}
USED = json.load(open("/tmp/threads_used_all.json"))


def dechunk(body: bytes) -> bytes:
    if not re.match(rb"^[0-9a-fA-F]+\r\n", body):
        return body
    out, rest = b"", body
    while rest:
        m = re.match(rb"([0-9a-fA-F]+)\r\n", rest)
        if not m:
            break
        n = int(m.group(1), 16)
        if n == 0:
            break
        rest = rest[m.end():]
        out += rest[:n]
        rest = rest[n + 2:]
    return out


def get_json(url, tries=3):
    last = None
    for i in range(tries):
        try:
            st, hd, body = fetch(url, timeout=90)
            body = dechunk(body)
            if "gzip" in hd.lower() or body[:2] == b"\x1f\x8b":
                import gzip
                body = gzip.decompress(body)
            if st == 200:
                return json.loads(body.decode("utf-8", "replace"))
            last = f"HTTP {st} len={len(body)}"
        except Exception as e:
            last = str(e)[:100]
        time.sleep(3)
    print(f"   gagal {url}: {last}")
    return None


out = {}
for acc, base in SITES.items():
    posts = None
    # NB: parameter _fields DITOLAK (404) oleh WAF situs klien → minta objek penuh.
    for extra in ("?per_page=20&orderby=date&order=desc", "?per_page=20"):
        posts = get_json(f"{base}/wp-json/wp/v2/posts{extra}")
        if posts:
            break
    if not posts:
        out[acc] = []
        continue
    used = set(u.rstrip("/").lower() for u in USED.get(acc, []))
    fresh = []
    for p in posts:
        link = p.get("link")
        title = (p.get("title") or {}).get("rendered") if isinstance(p.get("title"), dict) else p.get("title")
        if link and link.rstrip("/").lower() not in used:
            fresh.append({"link": link, "title": re.sub(r"&#0?39;|’", "'", title or ""), "date": (p.get("date") or "")[:10]})
    out[acc] = fresh
    print(f"\n=== {acc}: {len(posts)} artikel terbaru, {len(fresh)} FRESH")
    for f in fresh[:15]:
        print(f"   {f['date']} {f['link']}\n        {f['title']}")

json.dump(out, open("/tmp/fresh_articles.json", "w"), indent=1, ensure_ascii=False)
print("\ndisimpan /tmp/fresh_articles.json")