#!/usr/bin/env python3
"""TSLaw: cek robots.txt setelah perbaikan izin berkas."""
import time, urllib.request, urllib.error

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36",
      "Accept": "text/plain,*/*"}
LOKAL = "/home/mantapman/.hermes/profiles/atun/workspace/drafts/tslaw_robots.txt"


def ambil(u):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60)
        return r.status, r.read().decode("utf-8", "replace"), dict(r.headers)
    except urllib.error.HTTPError as e:
        isi = e.read().decode("utf-8", "replace")
        return e.code, isi, dict(e.headers)
    except Exception as e:
        return 0, str(e)[:80], {}


print("=== robots.txt sekarang ===")
for u in ("https://www.tslaw.id/robots.txt", "https://tslaw.id/robots.txt"):
    st, isi, h = ambil(u)
    print(f"\n  {u} → HTTP {st} | {len(isi)} char | cf={h.get('cf-cache-status')} | server={h.get('server')}")
    if st == 200:
        for b in isi.splitlines():
            if b.strip():
                print("     ", b)
    else:
        print("     ", isi.strip().replace("\n", " ")[:200])

print("\n=== bandingkan dengan berkas yang saya susun ===")
st, isi, _ = ambil("https://www.tslaw.id/robots.txt")
lokal = open(LOKAL).read()


def aturan(t):
    return [b.strip() for b in t.splitlines() if b.strip() and not b.strip().startswith("#")]


a_live, a_lokal = aturan(isi), aturan(lokal)
hilang = [x for x in a_lokal if x not in a_live]
print(f"   aturan live : {len(a_live)} baris")
print(f"   aturan lokal: {len(a_lokal)} baris")
print(f"   belum masuk : {hilang if hilang else 'tidak ada'}")
print(f"   → {'SESUAI' if not hilang and st == 200 else 'BELUM SESUAI'}")

print("\n=== aturan yang harus benar-benar berlaku ===")
for u, harus in (("https://www.tslaw.id/?s=hukum", "diblokir untuk perayap"),
                 ("https://www.tslaw.id/author/mantapman/", "diblokir untuk perayap"),
                 ("https://www.tslaw.id/feed/", "diblokir untuk perayap"),
                 ("https://www.tslaw.id/wp-admin/", "diblokir untuk perayap"),
                 ("https://www.tslaw.id/", "tetap harus dirayapi"),
                 ("https://www.tslaw.id/pengacara-gugat-cerai-surabaya/", "tetap harus dirayapi")):
    st2, _, _ = ambil(u)
    print(f"   {u.replace('https://www.tslaw.id', '') or '/':<40} HTTP {st2}  ({harus})")

print("\n=== sitemap masih sehat? ===")
st, isi, _ = ambil("https://www.tslaw.id/sitemap_index.xml")
print(f"   sitemap_index.xml → HTTP {st} | {len(isi)} char")