#!/usr/bin/env python3
"""Scan artikel dasindonesia.com yang BELUM pernah dipakai di akun DAS FB (Repliz semua status + history)."""
import json, re, subprocess, sys, urllib.request

UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120 Safari/537.36"


def fetch(url):
    for attempt in range(4):
        r = subprocess.run(["curl", "-s", "--max-time", "90", "-A", UA, url],
                           capture_output=True, text=True)
        try:
            return json.loads(r.stdout)
        except Exception:
            print(f"   retry {attempt+1}: {r.stdout[:80]!r}", file=sys.stderr)
    return []


live = []
page = 1
while True:
    url = (f"https://dasindonesia.com/wp-json/wp/v2/posts?per_page=100&page={page}"
           f"&_fields=link,title,date&status=publish")
    data = fetch(url)
    if not data:
        break
    for p in data:
        live.append((p['date'][:10], p['link'].rstrip('/'),
                     re.sub(r'&#0?39;|’|&', "'", p['title']['rendered'])))
    if len(data) < 100:
        break
    page += 1
print("total artikel live dasindonesia.com:", len(live))

from repliz_client import HEADERS, BASE, ACC

used = set()
page = 1
while True:
    req_url = f'{BASE}/public/schedule?page={page}&limit=100'
    req = urllib.request.Request(req_url, headers=HEADERS, method='GET')
    data = json.loads(urllib.request.urlopen(req, timeout=60).read())
    for d in data.get('docs', []):
        if d.get('accountId') != ACC.DAS_FB:
            continue
        for l in re.findall(r'https?://dasindonesia\.com[^\s]*', (d.get('description') or '')):
            used.add(l.rstrip('/'))
    if page >= data.get('totalPages', 1):
        break
    page += 1

hist = json.load(open('post_das_fb_history.json'))
for p in hist['posts']:
    used.add((p.get('article_url') or '').rstrip('/'))
print("total link pernah dipakai (Repliz+history):", len(used))

blocked = ('biaya-', 'harga-')
unused = [x for x in live if x[1] not in used]
clean = [x for x in unused if not any(b in x[1] for b in blocked)]
print(f"\n=== BELUM DIPAKAI: {len(unused)} | tanpa slug biaya-/harga-: {len(clean)} ===")
for d, l, t in sorted(clean, reverse=True)[:40]:
    print(f"{d}  {l}\n          {t}")
json.dump([{'date': d, 'url': l, 'title': t} for d, l, t in sorted(clean, reverse=True)],
          open('/tmp/das_unused_clean.json', 'w'), indent=1, ensure_ascii=False)