#!/usr/bin/env python3
"""Kenapa 5 artikel LEIT 'unknown to Google'? Cek robots meta, canonical, sitemap, HTTP."""
import re, urllib.request

UA = {"User-Agent": "Mozilla/5.0", "Cache-Control": "no-cache"}
URLS = [
    "https://www.leitsurabaya.com/kursus-bahasa-inggris-surabaya-barat/",
    "https://www.leitsurabaya.com/kursus-bahasa-inggris-surabaya-timur/",
    "https://www.leitsurabaya.com/les-bahasa-inggris-terdekat-surabaya/",
    "https://www.leitsurabaya.com/kursus-bahasa-inggris-paling-terjangkau-surabaya/",
    "https://www.leitsurabaya.com/bahasa-inggris-untuk-anak-metode-seru/",
]


def fetch(u):
    r = urllib.request.Request(u, headers=UA)
    resp = urllib.request.urlopen(r, timeout=90)
    return resp.status, resp.read().decode("utf-8", "replace")


for u in URLS:
    try:
        st, html = fetch(u)
    except Exception as e:
        print(u, "ERROR", str(e)[:100]); continue
    robots = re.findall(r'<meta name=["\']robots["\'][^>]*content=["\']([^"\']+)', html)
    canon = re.findall(r'rel=["\']canonical["\'] href=["\']([^"\']+)', html)
    title = re.findall(r"<title>(.*?)</title>", html, re.S)
    print(f"\n{u}\n  HTTP {st}\n  title  : {title[0][:90] if title else '-'}\n  robots : {robots}\n  canon  : {canon}")

print("\n=== sitemap ===")
st, sm = fetch("https://www.leitsurabaya.com/post-sitemap.xml")
locs = re.findall(r"<loc>(.*?)</loc>", sm)
print("post-sitemap.xml:", st, "| jumlah URL:", len(locs))
for slug in ["kursus-bahasa-inggris-surabaya-barat", "kursus-bahasa-inggris-surabaya-timur",
             "les-bahasa-inggris-terdekat-surabaya", "alasan-les-bahasa-inggris-terdekat-rumah",
             "7-tempat-les-bahasa-inggris-terdekat-surabaya"]:
    hit = [l for l in locs if slug in l]
    print(f"  {slug:<45} di sitemap: {'YA' if hit else 'TIDAK'}")
print("contoh 5 URL pertama sitemap:", locs[:5])

st, rob = fetch("https://www.leitsurabaya.com/robots.txt")
print("\nrobots.txt:\n", rob[:400])