#!/usr/bin/env python3
"""Ukur tingkat persaingan SERP (via Bing) untuk kandidat keyword StudioSoft, lalu hitung nilai gabungan."""
import base64, json, re, time, urllib.parse, urllib.request, urllib.error
from collections import Counter

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36",
      "Accept-Language": "id-ID,id;q=0.9"}

# domain besar/aggregator: kalau mendominasi halaman 1, keyword itu berat
BESAR = ("linkedin", "facebook", "instagram", "youtube", "tiktok", "twitter", "x.com", "wikipedia", "quora",
         "reddit", "medium.com", "pinterest", "tokopedia", "shopee", "bukalapak", "lazada", "blibli",
         "glints", "jobstreet", "indeed", "upwork", "fiverr", "freelancer", "kalibrr", "dealls", "jobsdb",
         "detik", "kompas", "tribun", "liputan6", "kumparan", "tempo", "cnnindonesia", "suara.com", "okezone",
         "sindonews", "merdeka.com", "antaranews", "bisnis.com", "katadata", "kontan", "money.kompas",
         "blogspot", "wordpress.com", "weebly", "wixsite", "google.com", "bing.com", "yelp", "trustpilot",
         "clutch.co", "goodfirms", "sortlist", "designrush", "semrush", "ahrefs", "moz.com", "similarweb",
         "shopify", "ecwid", "niagahoster", "hostinger", "domainesia", "rumahweb", "dewabiz", "jagoanhosting")
# direktori/listicle lokal
DIREKTORI = ("yellowpages", "cybo", "foursquare", "hotfrog", "indonesiayp", "alamat", "info", "direktori",
             "panduan", "katalog", "tokopedia.com/discovery")


def serp(q, hal=1):
    url = f"https://www.bing.com/search?q={urllib.parse.quote_plus(q)}&setlang=id&count=20&first={1 + (hal - 1) * 10}"
    r = urllib.request.Request(url, headers=UA)
    for _ in range(2):
        try:
            with urllib.request.urlopen(r, timeout=60) as x:
                return x.read().decode("utf-8", "replace")
        except Exception:
            time.sleep(4)
    return ""


def domain_dari(html):
    """Bing membungkus URL asli: bing.com/ck/a?...&u=a1<base64url>. Ini yang dibongkar."""
    hasil = []
    for blok in re.findall(r'<li class="b_algo".*?</li>', html, re.S):
        m = re.search(r'<h2[^>]*>.*?href="([^"]+)"', blok, re.S)
        if not m:
            continue
        u = m.group(1).replace("&amp;", "&")
        tgt = None
        mm = re.search(r"[?&]u=a1([A-Za-z0-9_\-]+)", u)
        if mm:
            s = mm.group(1)
            s += "=" * (-len(s) % 4)
            try:
                tgt = base64.urlsafe_b64decode(s).decode("utf-8", "replace")
            except Exception:
                tgt = None
        if not tgt and u.startswith("http") and "bing.com" not in u:
            tgt = u
        if not tgt:
            continue
        d = urllib.parse.urlparse(tgt).netloc.lower().replace("www.", "")
        if d and d not in hasil:
            hasil.append(d)
    return hasil


data = json.load(open("drafts/ss_keyword_nilai.json"))
kand = [h for h in data if h["imp"] >= 15 and h["pos"] <= 30 and h["nilai"] >= 20]
kand.sort(key=lambda x: x["nilai"], reverse=True)
kand = kand[:16]

print(f"memeriksa {len(kand)} keyword di SERP\n")
hasil = []
for i, h in enumerate(kand, 1):
    q = h["query"]
    html = serp(q)
    dom = domain_dari(html)
    n_besar = sum(1 for d in dom if any(b in d for b in BESAR))
    n_dir = sum(1 for d in dom if any(b in d for b in DIREKTORI))
    n_total = max(len(dom), 1)
    # persaingan: makin banyak penantang kuat, makin berat
    berat = (n_besar * 1.6 + n_dir * 0.5) / n_total
    ringan = max(0.0, 1 - berat)
    # nilai akhir: permintaan x kemudahan x kedekatan posisi
    dekat = 1.0 if h["pos"] <= 10 else (0.9 if h["pos"] <= 20 else 0.75 if h["pos"] <= 30 else 0.5)
    nilai_akhir = round(h["nilai"] * 0.55 + ringan * 100 * 0.35 + h["nilai"] * 0.10 * dekat)
    hasil.append({**h, "pesaing_besar": n_besar, "direktori": n_dir, "domain_top": dom[:6],
                  "ringan": round(ringan * 100), "nilai_final": nilai_akhir})
    print(f"  [{i:>2}/{len(kand)}] {q[:44]:<46} kuat {n_besar}/{n_total} | ringan {round(ringan*100):>3} | "
          f"nilai {h['nilai']}→{nilai_akhir}")
    time.sleep(2)

hasil.sort(key=lambda x: x["nilai_final"], reverse=True)
print("\n=== PERINGKAT AKHIR ===")
for h in hasil:
    print(f"  {h['nilai_final']:>3} | ringan {h['ringan']:>3} | imp {h['imp']:>4} | pos {h['pos']:>5} | {h['query'][:44]:<46} | {' '.join(h['domain_top'][:3])}")
json.dump(hasil, open("drafts/ss_keyword_serp.json", "w"), indent=1, ensure_ascii=False)
print("\n→ drafts/ss_keyword_serp.json")