#!/usr/bin/env python3
"""Ekstrak teks artikel (strip tag) dari JSON WP atau HTML langsung."""
import json, re, sys, html, urllib.request

def strip(h):
    h = re.sub(r'(?is)<(script|style|nav|footer|header)[^>]*>.*?</\1>', ' ', h)
    h = re.sub(r'(?i)<br\s*/?>', '\n', h)
    h = re.sub(r'(?i)</(p|h[1-6]|li|div)>', '\n', h)
    h = re.sub(r'<[^>]+>', ' ', h)
    h = html.unescape(h)
    h = re.sub(r'[ \t\xa0]+', ' ', h)
    h = re.sub(r'\n\s*\n+', '\n', h)
    return h.strip()

for f in sys.argv[1:]:
    if f.endswith('.json'):
        p = json.load(open(f))
        txt = strip(p['content']['rendered']) if isinstance(p.get('content'), dict) else strip(p.get('content', ''))
        print("\n" + "=" * 78)
        print("FILE:", f)
        print("TITLE:", html.unescape(re.sub('<[^>]+>', '', p['title']['rendered'] if isinstance(p['title'], dict) else p['title'])))
        print("=" * 78)
        print(txt[:6000])
    else:
        req = urllib.request.Request(f, headers={"User-Agent": "Mozilla/5.0"})
        h = urllib.request.urlopen(req, timeout=120).read().decode('utf-8', 'ignore')
        body = re.search(r'(?is)<article.*?</article>', h) or re.search(r'(?is)<main.*?</main>', h)
        print("\n" + "=" * 78)
        print("URL:", f)
        print("=" * 78)
        print(strip(body.group(0) if body else h)[:6000])