#!/usr/bin/env python3
"""Ambil teks bersih halaman dokumentasi Google (untuk verifikasi klaim), simpan ke /tmp."""
import html, re, sys, urllib.request

URLS = {
    "local": "https://developers.google.com/search/docs/appearance/structured-data/local-business",
    "review": "https://developers.google.com/search/docs/appearance/structured-data/review-snippet",
    "article": "https://developers.google.com/search/docs/appearance/structured-data/article",
    "breadcrumb": "https://developers.google.com/search/docs/appearance/structured-data/breadcrumb",
}

for k, u in URLS.items():
    try:
        r = urllib.request.Request(u, headers={"User-Agent": "Mozilla/5.0"})
        with urllib.request.urlopen(r, timeout=60) as x:
            t = x.read().decode("utf-8", "ignore")
    except Exception as e:
        print(k, "GAGAL", e)
        continue
    t = re.sub(r"<script.*?</script>", " ", t, flags=re.S)
    t = re.sub(r"<style.*?</style>", " ", t, flags=re.S)
    t = re.sub(r"<[^>]+>", " ", t)
    t = html.unescape(t)
    t = re.sub(r"[ \t]+", " ", t)
    t = re.sub(r"\n\s*\n+", "\n", t)
    open(f"/tmp/gdoc_{k}.txt", "w").write(t)
    print(k, len(t), "chars ->", f"/tmp/gdoc_{k}.txt")