#!/usr/bin/env python3
"""Kumpulkan data artikel LEIT untuk penulisan meta: judul, keyphrase, meta desc, kutipan isi."""
import base64, json, os, re, html, urllib.request

SITE = "https://www.leitsurabaya.com"
AUTH = base64.b64encode(b"bambangbot:zibd dLIV xY8x onP9 39qL hiNr").decode()
WS = os.path.dirname(os.path.abspath(__file__))


def get(path):
    r = urllib.request.Request(SITE + path, headers={"Authorization": f"Basic {AUTH}", "User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(r, timeout=180) as x:
        return json.loads(x.read())


posts = []
page = 1
while True:
    d = get(f"/wp-json/wp/v2/posts?per_page=100&page={page}&context=edit&_fields=id,link,slug,title,date,content,featured_media,yoast_head_json")
    posts += d
    if len(d) < 100:
        break
    page += 1

out = []
for p in sorted(posts, key=lambda x: x["date"], reverse=True):
    raw = p["content"]["raw"]
    text = re.sub(r"<[^>]+>", " ", raw)
    text = html.unescape(re.sub(r"\s+", " ", text)).strip()
    y = p["yoast_head_json"]
    out.append({
        "id": p["id"],
        "date": p["date"][:10],
        "slug": p["slug"],
        "title": html.unescape(p["title"]["rendered"]),
        "link": p["link"],
        "focuskw": "",
        "metadesc": y.get("description", ""),
        "words": len(text.split()),
        "h2": [html.unescape(re.sub(r"<[^>]+>", "", h)) for h in
               [m for m in re.findall(r"<h2[^>]*>(.*?)</h2>", raw, re.S)][:8]],
        "excerpt": text[:600],
    })

json.dump(out, open(os.path.join(WS, "leit_artikel_data.json"), "w"), indent=1, ensure_ascii=False)
print("artikel:", len(out))
tgt = [r for r in out if not r["metadesc"]]
print("tanpa meta:", len(tgt), "| ids:", [r["id"] for r in tgt])