#!/usr/bin/env python3
"""Ambil isi artikel (teks bersih) untuk bahan caption."""
import re, sys
sys.path.insert(0, "/home/mantapman/.hermes/profiles/atun/workspace")
from openssl_fetch import fetch

URLS_OLD = [
    "https://studiosoft.id/digital-agency-vs-freelancer-vs-in-house/",
    "https://www.tslaw.id/pkwt-vs-pkwtt-panduan-lengkap-kontrak-kerja-karyawan/",
    "https://www.theskincodeclinic.com/klinik-hapus-tato-sidoarjo/",
    "https://dasindonesia.com/servis-rutin-mobil-alphard-jadwal-lengkap-komponen-yang-dicek-dan-tips-dari-teknisi/",
]
URLS = ["https://www.theskincodeclinic.com/klinik-hapus-tato-sidoarjo/"]


def teks(html: str) -> str:
    html = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", html)
    html = re.sub(r"(?is)<h([1-3])[^>]*>", "\n\n## ", html)
    html = re.sub(r"(?is)<li[^>]*>", "\n- ", html)
    html = re.sub(r"(?is)<br\s*/?>", "\n", html)
    html = re.sub(r"(?is)</p>", "\n\n", html)
    t = re.sub(r"(?s)<[^>]+>", " ", html)
    t = re.sub(r"&nbsp;", " ", t)
    t = re.sub(r"&amp;", "&", t)
    t = re.sub(r"&#8217;|&#039;|&rsquo;", "'", t)
    t = re.sub(r"&quot;", '"', t)
    t = re.sub(r"[ \t]+", " ", t)
    t = re.sub(r"\n{3,}", "\n\n", t)
    return t.strip()


for u in URLS:
    st, head, body = fetch(u)
    t = teks(body.decode("utf-8", "replace"))
    i = t.find("## ")
    print("=" * 78)
    print(u, "status", st)
    print(t[i:i + 2600] if i > 0 else t[:2600])