#!/usr/bin/env python3
"""TSLaw: periksa robots.txt yang berlaku sekarang + sitemap + halaman yang perlu dikecualikan."""
import base64, json, re, time, urllib.request, urllib.error

DOM = "tslaw.id"
AUTH = base64.b64encode(b"bambangbot:EFKm EYGx VQCP Y9go ncPr uM2Y").decode()
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36"}

print("=== 1. robots.txt sekarang ===")
for u in (f"https://{DOM}/robots.txt", f"https://www.{DOM}/robots.txt"):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60)
        isi = r.read().decode("utf-8", "replace")
        print(f"  {u} → {r.status} | {len(isi)} char | cf={r.headers.get('cf-cache-status')} | tipe={r.headers.get('content-type')}")
        print("  --- isi ---")
        for baris in isi.splitlines():
            print("   ", baris)
    except Exception as e:
        print(f"  {u} → gagal {str(e)[:60]}")

print("\n=== 2. sitemap yang tersedia ===")
for u in (f"https://www.{DOM}/sitemap_index.xml", f"https://www.{DOM}/wp-sitemap.xml"):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60)
        isi = r.read().decode("utf-8", "replace")
        n = len(re.findall(r"<loc>", isi))
        print(f"  {u} → {r.status} | {n} entri")
        for m in re.findall(r"<loc>([^<]+)</loc>", isi)[:8]:
            print(f"     {m}")
    except urllib.error.HTTPError as e:
        print(f"  {u} → HTTP {e.code}")
    except Exception as e:
        print(f"  {u} → gagal {str(e)[:60]}")

print("\n=== 3. apa yang sebaiknya dikecualikan (uji keberadaannya) ===")
for u in ("/wp-admin/", "/wp-login.php", "/?s=hukum", "/author/mantapman/", "/feed/", "/sample-page/",
          "/hello-world/", "/wp-json/wp/v2/posts", "/xmlrpc.php", "/tag/advokat/", "/category/hukum-bisnis/"):
    try:
        r = urllib.request.urlopen(urllib.request.Request(f"https://www.{DOM}{u}", headers=UA), timeout=45)
        panjang = len(r.read())
        print(f"  {r.status} | {u:<32} {panjang//1024} KB")
    except urllib.error.HTTPError as e:
        print(f"  {e.code} | {u}")
    except Exception as e:
        print(f"  ERR | {u} ({str(e)[:40]})")

print("\n=== 4. pengaturan Yoast yang berkaitan ===")
try:
    r = urllib.request.urlopen(urllib.request.Request(f"https://{DOM}/wp-json/yoast/v1/get_head?url=https://www.tslaw.id/",
                                                      headers=UA), timeout=90)
    j = json.loads(r.read())
    print("   robots di head beranda:", (j.get("json") or {}).get("robots"))
    print("   ada schema:", bool((j.get("schema") or {}).get("@graph")))
except Exception as e:
    print("   gagal:", str(e)[:80])