#!/usr/bin/env python3
"""Dump full plain text of a page (title, headings, body) for source reading."""
import re, sys, urllib.request, json

UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
      "Chrome/124.0.0.0 Safari/537.36")


def text_of(html: str) -> str:
    html = re.sub(r"(?is)<(script|style|noscript|svg).*?</\1>", " ", html)
    html = re.sub(r"(?is)<br\s*/?>", "\n", html)
    html = re.sub(r"(?is)</(p|div|li|h1|h2|h3|h4|tr|table|section)>", "\n", html)
    html = re.sub(r"(?s)<[^>]+>", " ", html)
    import html as H
    html = H.unescape(html)
    html = re.sub(r"[ \t\xa0]+", " ", html)
    html = re.sub(r"\n\s*\n\s*\n+", "\n\n", html)
    return html.strip()


def get(url: str) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA,
                                              "Accept-Language": "id-ID,id;q=0.9,en;q=0.8"})
    with urllib.request.urlopen(req, timeout=45) as r:
        raw = r.read()
    try:
        return raw.decode("utf-8")
    except UnicodeDecodeError:
        return raw.decode("latin-1")


if __name__ == "__main__":
    for url in sys.argv[1:]:
        print("=" * 20, url)
        try:
            t = text_of(get(url))
        except Exception as e:  # noqa
            print("ERROR", e)
            continue
        t = re.sub(r"\n{3,}", "\n\n", t)
        print(t)