#!/usr/bin/env python3
"""TSLaw: verifikasi robots.txt yang baru dipasang + pastikan tidak memblokir hal penting."""
import re, urllib.request, urllib.error

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36"}
LOKAL = "/home/mantapman/.hermes/profiles/atun/workspace/drafts/tslaw_robots.txt"


def ambil(u):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60)
        return r.status, r.read().decode("utf-8", "replace"), dict(r.headers)
    except urllib.error.HTTPError as e:
        return e.code, "", {}
    except Exception as e:
        return 0, str(e)[:80], {}


print("=== 1. isi robots.txt yang berlaku ===")
for u in ("https://www.tslaw.id/robots.txt", "https://tslaw.id/robots.txt"):
    st, isi, h = ambil(u)
    print(f"\n  {u} → HTTP {st} | {len(isi)} char | cf={h.get('cf-cache-status')} | age={h.get('age')}")
    for b in isi.splitlines():
        if b.strip():
            print("     ", b)

print("\n=== 2. bandingkan dengan berkas yang saya susun ===")
st, isi, _ = ambil("https://www.tslaw.id/robots.txt")
lokal = open(LOKAL).read()
# bandingkan baris aturan saja (abaikan komentar)
def aturan(t):
    return [b.strip() for b in t.splitlines() if b.strip() and not b.strip().startswith("#")]
a_live, a_lokal = aturan(isi), aturan(lokal)
print(f"   baris aturan live : {len(a_live)}")
print(f"   baris aturan lokal: {len(a_lokal)}")
hilang = [x for x in a_lokal if x not in a_live]
tambahan = [x for x in a_live if x not in a_lokal]
print(f"   aturan yang belum masuk : {hilang if hilang else 'tidak ada'}")
print(f"   aturan tambahan di live : {tambahan if tambahan else 'tidak ada'}")
print(f"   → {'SESUAI' if not hilang else 'BELUM LENGKAP (mungkin masih versi cache)'}")

print("\n=== 3. pastikan hal penting tidak ikut diblokir ===")
PENTING = ["/", "/tentang/", "/practice-areas/", "/hubungi/", "/artikel/",
           "/pengacara-gugat-cerai-surabaya/", "/wp-content/uploads/2026/05/tim.webp",
           "/sitemap_index.xml", "/post-sitemap.xml"]
for u in PENTING:
    st, isi2, _ = ambil(f"https://www.tslaw.id{u}")
    kode = st if st else "ERR"
    print(f"   {kode} | {u}")

print("\n=== 4. aturan mana yang menyasar halaman ini (uji pola) ===")
aturan_live = [b.split(":", 1)[1].strip() for b in a_live if b.lower().startswith("disallow:")]
print("   daftar Disallow:", aturan_live)