#!/usr/bin/env python3
"""Ambil teks artikel (strip tag) lewat openssl_fetch untuk bahan caption."""
import re, sys, time, html

sys.path.insert(0, "/home/mantapman/.hermes/profiles/atun/workspace")
from openssl_fetch import fetch

URLS = sys.argv[1:]


def dechunk(body):
    if not re.match(rb"^[0-9a-fA-F]+\r\n", body):
        return body
    out, rest = b"", body
    while rest:
        m = re.match(rb"([0-9a-fA-F]+)\r\n", rest)
        if not m:
            break
        n = int(m.group(1), 16)
        if n == 0:
            break
        rest = rest[m.end():]
        out += rest[:n]
        rest = rest[n + 2:]
    return out


for u in URLS:
    raw = None
    for i in range(4):
        try:
            st, hd, body = fetch(u, timeout=45)
            if st == 200:
                raw = dechunk(body)
                break
        except Exception as e:
            pass
        time.sleep(2)
    print("=" * 100)
    print(u)
    if not raw:
        print("GAGAL ambil")
        continue
    txt = raw.decode("utf-8", "replace")
    # ambil area konten utama kalau ada
    m = re.search(r"<article[^>]*>(.*?)</article>", txt, re.S) or re.search(r"<main[^>]*>(.*?)</main>", txt, re.S)
    if m:
        txt = m.group(1)
    txt = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", txt, flags=re.S)
    txt = re.sub(r"<h([1-6])[^>]*>", r"\n## ", txt)
    txt = re.sub(r"<[^>]+>", " ", txt)
    txt = html.unescape(txt)
    txt = re.sub(r"[ \t]+", " ", txt)
    txt = re.sub(r"\n\s*\n+", "\n", txt)
    print(txt.strip()[:2600])