#!/usr/bin/env python3
"""TSLaw: cek robots.txt dari origin (lewati cache Cloudflare) + pastikan /layanan/ masuk sitemap."""
import re, urllib.request, urllib.error, time

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36"}


def ambil(u):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=60)
        return r.status, r.read().decode("utf-8", "replace"), dict(r.headers)
    except urllib.error.HTTPError as e:
        return e.code, "", {}
    except Exception as e:
        return 0, str(e)[:70], {}


print("=== robots.txt: cache vs asli ===")
for u in ("https://www.tslaw.id/robots.txt",
          f"https://www.tslaw.id/robots.txt?cek={int(time.time())}"):
    st, isi, h = ambil(u)
    ada = "wp-admin" in isi
    print(f"  {u.split('/robots.txt')[1] or '(polos)':<16} HTTP {st} | {len(isi)} char | cf={h.get('cf-cache-status')} | "
          f"versi kustom: {'YA' if ada else 'BELUM'}")
    if ada:
        for b in isi.splitlines():
            if b.strip() and not b.strip().startswith("#"):
                print("      ", b)

print("\n=== sitemap halaman (apakah /layanan/ sudah masuk) ===")
st, isi, _ = ambil("https://www.tslaw.id/page-sitemap.xml")
for m in re.findall(r"<loc>([^<]+)</loc>", isi):
    print("   ", m)

print("\n=== cek tautan lama masih muncul di sitemap/beranda? ===")
for u in ("https://www.tslaw.id/", "https://www.tslaw.id/post-sitemap.xml"):
    st, isi, _ = ambil(u)
    print(f"   {u.replace('https://www.tslaw.id','') or '/'}: 'practice-areas' → {isi.count('practice-areas')}x | "
          f"'/layanan/' → {isi.count('/layanan/')}x")

print("\n=== dua artikel baru: tautan keluar ===")
for u in ("/pengacara-gugat-cerai-surabaya/", "/pengacara-surat-wasiat/"):
    st, isi, _ = ambil(f"https://www.tslaw.id{u}")
    ke_beranda = 1 if re.search(r'href="https://www[.]tslaw[.]id/"', isi) else 0
    print(f"   {u}")
    print(f"      ke /layanan/: {isi.count('/layanan/')} | ke practice-areas: {isi.count('practice-areas')} | "
          f"ke beranda: {ke_beranda}")