#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Cari artikel studiosoft.id yang BELUM dipakai (paginate 10 halaman x 100)."""
import json, re, urllib.request, ssl, time

DUMP = "/tmp/repliz_full_20261004.json"
OUT = "/tmp/ss_pool_20261004.json"
ctx = ssl.create_default_context()

def raw(url, timeout=40):
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
        return r.read().decode("utf-8", "replace")

def norm(u):
    u = u.strip().rstrip("/").lower()
    u = re.sub(r"^https?://", "", u)
    u = re.sub(r"^www\.", "", u)
    return u.split("?")[0]

def clean_title(t):
    for a, b in [("–", "-"), ("&", "&"), ("&", "&"), ("’", "'"),
                 ("“", '"'), ("”", '"'), ("—", "-")]:
        t = t.replace(a, b)
    return re.sub(r"&#\d+;", "", t).strip()

docs = json.load(open(DUMP))
used = set()
URLRE = re.compile(r'https?://[^\s"\'\\]+')
for d in docs:
    for u in URLRE.findall(json.dumps(d, ensure_ascii=False)):
        used.add(norm(u))
print("URL terpakai:", len(used))

fresh = []
for page in range(1, 11):
    url = (f"https://studiosoft.id/wp-json/wp/v2/posts?per_page=100&page={page}"
           f"&_fields=id,date,slug,link,title&cb={int(time.time())}")
    try:
        data = json.loads(raw(url))
    except Exception as e:
        print("page", page, "gagal:", e)
        break
    if not isinstance(data, list) or not data:
        break
    baru = 0
    for a in data:
        if norm(a["link"]) in used:
            continue
        fresh.append({
            "id": a["id"], "date": a["date"][:10], "slug": a["slug"],
            "link": "https://studiosoft.id/" + a["slug"].strip("/") + "/",
            "title": clean_title(a["title"]["rendered"]),
            "len_link": len("https://studiosoft.id/" + a["slug"].strip("/") + "/"),
        })
        baru += 1
    print(f"page {page}: {len(data)} artikel, fresh +{baru} (total {len(fresh)})", flush=True)
    if len(fresh) > 150:
        break
    time.sleep(0.4)

json.dump(fresh, open(OUT, "w"), ensure_ascii=False, indent=1)
print("TOTAL FRESH:", len(fresh), "->", OUT)
fresh.sort(key=lambda x: x["date"], reverse=True)
for f in fresh[:45]:
    print(f"  {f['len_link']:3}  {f['date']}  {f['title'][:66]}")