#!/usr/bin/env python3
"""TSC — peta halaman tumpang tindih: kelompokkan per niat kueri, tentukan kanonik dan kandidat gabung."""
import base64, importlib.util, json, os, re, time, urllib.error, urllib.request
from collections import defaultdict
from datetime import date, timedelta

from google.oauth2 import service_account
from googleapiclient.discovery import build

WS = "/home/mantapman/.hermes/profiles/atun/workspace"
spec = importlib.util.spec_from_file_location("cfg", f"{WS}/cfg_tsc.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
CFG = m.CFG
DOM = CFG["domain"]
SITE = f"https://www.{DOM}/"
AUTH = base64.b64encode(f"{CFG['user']}:{CFG['pass']}".encode()).decode()
UA = {"User-Agent": "Mozilla/5.0"}

creds = service_account.Credentials.from_service_account_file(
    os.path.expanduser("~/.hermes/gsc_service_account.json"),
    scopes=["https://www.googleapis.com/auth/webmasters.readonly"])
gsc = build("webmasters", "v3", credentials=creds, cache_discovery=False)
AKHIR = date.today() - timedelta(days=2)
AWAL = AKHIR - timedelta(days=89)


def rest(path):
    h = {"Authorization": f"Basic {AUTH}", **UA}
    r = urllib.request.Request(f"https://{DOM}{path}", headers=h)
    for i in range(4):
        try:
            with urllib.request.urlopen(r, timeout=120) as x:
                return json.loads(x.read())
        except urllib.error.HTTPError as e:
            if e.code in (429, 404):
                time.sleep(4)
                continue
            return {"err": e.code}
        except Exception:
            time.sleep(4)
    return {}


# 1. konten
konten = {}
for jenis in ("posts", "pages"):
    page = 1
    while True:
        d = rest(f"/wp-json/wp/v2/{jenis}?per_page=100&page={page}&status=publish&context=edit"
                 f"&_fields=id,slug,link,title,content,date")
        if not isinstance(d, list) or not d:
            break
        for x in d:
            kata = len(re.sub(r"<[^>]+>", " ", x["content"]["raw"]).split())
            konten[x["link"].rstrip("/")] = {"jenis": jenis, "id": x["id"], "slug": x["slug"],
                                             "judul": re.sub(r"<[^>]+>", "", str(x["title"]["raw"]))[:80],
                                             "kata": kata, "tgl": x["date"][:10]}
        if len(d) < 100:
            break
        page += 1
print(f"konten: {len(konten)}")

# 2. GSC per halaman dan per kueri
per_hal = {}
for r in gsc.searchanalytics().query(siteUrl=SITE, body={
        "startDate": AWAL.isoformat(), "endDate": AKHIR.isoformat(), "dimensions": ["page"],
        "rowLimit": 3000}).execute().get("rows", []):
    per_hal[r["keys"][0].rstrip("/")] = {"tayang": int(r["impressions"]), "klik": int(r["clicks"]),
                                         "pos": round(r["position"], 1)}
kueri = {}
for r in gsc.searchanalytics().query(siteUrl=SITE, body={
        "startDate": AWAL.isoformat(), "endDate": AKHIR.isoformat(), "dimensions": ["page", "query"],
        "rowLimit": 5000}).execute().get("rows", []):
    kueri.setdefault(r["keys"][0].rstrip("/"), []).append((int(r["impressions"]), r["keys"][1]))
for u in kueri:
    kueri[u].sort(reverse=True)

STOP = set("dan atau yang untuk di ke dari pada dengan adalah ini itu cara tips klinik kulit dokter "
           "terdekat dekat the skin code clinic krian sidoarjo surabaya".split())


def inti(q):
    return " ".join(w for w in re.findall(r"[a-z]+", q.lower()) if w not in STOP)[:40]


klaster = defaultdict(list)
for u, d in konten.items():
    g = per_hal.get(u, {"tayang": 0, "klik": 0, "pos": None})
    k = kueri.get(u, [])
    utama = inti(k[0][1]) if k else ""
    klaster[utama].append({"url": u, **d, **g, "kueri_utama": k[0][1] if k else "",
                           "kueri": [x[1] for x in k[:4]]})

print("\n=== klaster dengan lebih dari satu halaman ===")
rencana = []
for nama, isi in sorted(klaster.items(), key=lambda x: -sum(y["tayang"] for y in x[1])):
    if not nama or len(isi) < 2:
        continue
    isi.sort(key=lambda y: (-y["klik"], -y["tayang"]))
    print(f"\n■ klaster «{nama}» ({len(isi)} halaman)")
    for i, y in enumerate(isi):
        tanda = "KANONIK" if i == 0 else "gabung "
        print(f"   {tanda} {y['tayang']:>5} tayang | {y['klik']:>2} klik | {y['kata']:>5} kata | /{y['slug'][:56]}")
    rencana.append({"klaster": nama, "kanonik": isi[0], "lain": isi[1:]})

json.dump(rencana, open(f"{WS}/drafts/tsc_klaster.json", "w"), ensure_ascii=False, indent=1)
print(f"\nklaster dengan duplikasi: {len(rencana)}")
print(f"total halaman di dalamnya: {sum(len(r['lain']) + 1 for r in rencana)}")