#!/usr/bin/env python3
"""TSLaw: kenapa /robots.txt balas 403? Cek header, beberapa varian permintaan, dan bandingkan URL lain."""
import urllib.request, urllib.error, time

UA_BROWSER = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36",
              "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
              "Accept-Language": "id-ID,id;q=0.9,en;q=0.8"}
UA_GOOGLE = {"User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}


def cek(u, ua, nama):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=ua), timeout=45)
        isi = r.read()
        return r.status, len(isi), dict(r.headers), isi.decode("utf-8", "replace")[:400]
    except urllib.error.HTTPError as e:
        return e.code, 0, dict(e.headers), ""
    except Exception as e:
        return 0, 0, {}, str(e)[:60]


print("=== robots.txt dengan berbagai permintaan ===")
for u in ("https://www.tslaw.id/robots.txt", "https://tslaw.id/robots.txt",
          "https://www.tslaw.id/robots.txt?x=1", "https://www.tslaw.id/ROBOTS.TXT"):
    for nm, ua in (("browser", UA_BROWSER), ("googlebot", UA_GOOGLE)):
        st, n, h, isi = cek(u, ua, nm)
        print(f"  {u.replace('https://', ''):<34} [{nm:<9}] HTTP {st} | {n} byte | "
              f"server={h.get('server')} | cf-ray={'ada' if h.get('cf-ray') else '-'} | {h.get('cf-cache-status') or ''}")
        if st == 200 and isi:
            for b in isi.splitlines()[:6]:
                print("        ", b)

print("\n=== bandingkan dengan URL lain (masih normal?) ===")
for u in ("https://www.tslaw.id/", "https://www.tslaw.id/layanan/", "https://www.tslaw.id/sitemap_index.xml",
          "https://www.tslaw.id/wp-content/", "https://www.tslaw.id/wp-login.php"):
    st, n, h, _ = cek(u, UA_BROWSER, "browser")
    print(f"  {u.replace('https://www.tslaw.id', '') or '/':<24} HTTP {st} | {n} byte | cf={h.get('cf-cache-status') or '-'}")

print("\n=== header lengkap robots.txt (kalau 403, lihat siapa yang menolak) ===")
st, n, h, _ = cek("https://www.tslaw.id/robots.txt", UA_BROWSER, "browser")
print(f"  status {st}")
for k, v in sorted(h.items()):
    print(f"    {k}: {str(v)[:90]}")