#!/usr/bin/env python3
"""Pustaka referensi: tarik judul + sub-bagian dari dokumentasi resmi Google, Meta, dan alat lain."""
import html, json, re, time, urllib.request, urllib.error

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
      "Accept-Language": "id-ID,id;q=0.9,en;q=0.8"}

SUMBER = {
    "google-search-central": [
        ("SEO Starter Guide", "https://developers.google.com/search/docs/fundamentals/seo-starter-guide"),
        ("Search Essentials", "https://developers.google.com/search/docs/essentials"),
        ("Crawling & indexing", "https://developers.google.com/search/docs/crawling-indexing"),
        ("Structured data (ringkasan)", "https://developers.google.com/search/docs/appearance/structured-data/search-gallery"),
        ("Pengalaman halaman (Core Web Vitals)", "https://developers.google.com/search/docs/appearance/page-experience"),
        ("Bantuan untuk hasil lokal / GBP", "https://developers.google.com/search/docs/appearance/local-business"),
        ("Panduan kualitas (spam)", "https://developers.google.com/search/docs/essentials/spam-policies"),
    ],
    "google-analytics": [
        ("GA4: mulai", "https://support.google.com/analytics/answer/9304153"),
        ("GA4: event & parameter", "https://support.google.com/analytics/answer/9322688"),
        ("GA4: konversi (key events)", "https://support.google.com/analytics/answer/9267568"),
        ("GA4: audiens", "https://support.google.com/analytics/answer/9267572"),
        ("GA4: Data API", "https://developers.google.com/analytics/devguides/reporting/data/v1"),
    ],
    "google-tag-manager": [
        ("GTM: gambaran", "https://support.google.com/tagmanager/answer/6102821"),
        ("GTM: tag & trigger", "https://support.google.com/tagmanager/answer/6106880"),
        ("GTM: variabel", "https://support.google.com/tagmanager/answer/6106899"),
        ("GTM API", "https://developers.google.com/tag-platform/tag-manager/api/v2"),
    ],
    "google-ads": [
        ("Google Ads: dasar", "https://support.google.com/google-ads/answer/6146252"),
        ("Kata kunci & pencocokan", "https://support.google.com/google-ads/answer/7478529"),
        ("Perencana kata kunci", "https://support.google.com/google-ads/answer/7337243"),
        ("Konversi", "https://support.google.com/google-ads/answer/1722054"),
        ("API Google Ads", "https://developers.google.com/google-ads/api/docs/start"),
    ],
    "google-business-profile": [
        ("GBP: gambaran", "https://support.google.com/business/answer/3038177"),
        ("GBP: verifikasi", "https://support.google.com/business/answer/7107242"),
        ("GBP: ulasan", "https://support.google.com/business/answer/3474122"),
        ("GBP: API", "https://developers.google.com/my-business"),
    ],
    "meta": [
        ("Meta Business Help Center", "https://www.facebook.com/business/help"),
        ("Meta Ads: dasar", "https://www.facebook.com/business/help/3171952609748790"),
        ("Meta Pixel", "https://www.facebook.com/business/help/952192354843755"),
        ("Conversions API", "https://developers.facebook.com/docs/marketing-api/conversions-api/"),
        ("Marketing API", "https://developers.facebook.com/docs/marketing-apis/"),
        ("Instagram untuk bisnis", "https://www.facebook.com/business/help/347420087974694"),
        ("WhatsApp Business API", "https://developers.facebook.com/docs/whatsapp"),
    ],
    "alat-lain": [
        ("Bing Webmaster Tools", "https://www.bing.com/webmasters/help/getting-started-checklist-66a806de"),
        ("Cloudflare: cache", "https://developers.cloudflare.com/cache/"),
        ("Cloudflare: cache rules", "https://developers.cloudflare.com/cache/how-to/cache-rules/"),
        ("WordPress: SEO", "https://wordpress.org/documentation/article/search-engine-optimization/"),
        ("Yoast SEO: fitur", "https://yoast.com/features/"),
        ("Schema.org", "https://schema.org/"),
        ("PageSpeed Insights", "https://developers.google.com/speed/docs/insights/v5/get-started"),
        ("web.dev: Core Web Vitals", "https://web.dev/articles/vitals"),
    ],
}


def ambil(u, timeout=60):
    try:
        with urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=timeout) as x:
            return x.status, x.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, ""
    except Exception as e:
        return 0, str(e)[:80]


def ringkas(isi):
    judul = re.search(r"<title[^>]*>(.*?)</title>", isi, re.S)
    desc = re.search(r'<meta name="description" content="([^"]*)"', isi, re.I)
    h1 = re.findall(r"<h1[^>]*>(.*?)</h1>", isi, re.S)
    h2 = re.findall(r"<h2[^>]*>(.*?)</h2>", isi, re.S)
    bersih = lambda t: html.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", t))).strip()
    return {"judul": bersih(judul.group(1))[:140] if judul else "",
            "deskripsi": bersih(desc.group(1))[:220] if desc else "",
            "h1": [bersih(x)[:90] for x in h1[:2]],
            "h2": [bersih(x)[:90] for x in h2[:14]]}


hasil = {}
for kelompok, daftar in SUMBER.items():
    hasil[kelompok] = []
    print(f"\n=== {kelompok} ===")
    for nama, url in daftar:
        st, isi = ambil(url)
        r = ringkas(isi) if isi else {}
        hasil[kelompok].append({"nama": nama, "url": url, "status": st, **r})
        print(f"   {st} | {nama}")
        if r.get("judul"):
            print(f"        {r['judul'][:110]}")
        time.sleep(1)

json.dump(hasil, open("/home/mantapman/.hermes/profiles/atun/workspace/drafts/literatur_sumber.json", "w"),
          indent=1, ensure_ascii=False)
print("\n→ drafts/literatur_sumber.json")