#!/usr/bin/env python3
"""Riset Threads tahap 2: lengkapi sumber yang gagal + periksa kemampuan alat kita sendiri."""
import json, os, re, sys, time, urllib.request, urllib.error
import html as H

WS = "/home/mantapman/.hermes/profiles/atun/workspace"
DIR = f"{WS}/drafts/riset_threads"
os.makedirs(DIR, exist_ok=True)
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
      "Accept-Language": "en-US,en;q=0.9"}

KANDIDAT = [
    ("meta_ranking_umum", "https://transparency.meta.com/features/explaining-ranking/"),
    ("hootsuite_marketing", "https://blog.hootsuite.com/threads-marketing/"),
    ("buffer_threads2", "https://buffer.com/resources/threads/"),
    ("sprout_threads2", "https://sproutsocial.com/insights/threads/"),
    ("socialpilot", "https://www.socialpilot.co/blog/threads-algorithm"),
    ("influencerhub", "https://influencermarketinghub.com/threads-algorithm/"),
    ("shopify_threads", "https://www.shopify.com/blog/threads-marketing"),
    ("sendible", "https://www.sendible.com/insights/threads-algorithm"),
    ("datareportal", "https://datareportal.com/reports/digital-2026-global-overview-report"),
]


def ambil(u, timeout=70):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=timeout)
        return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, ""
    except Exception as e:
        return 0, str(e)[:80]


def bersihkan(h):
    h = re.sub(r"<script.*?</script>|<style.*?</style>|<nav.*?</nav>|<header.*?</header>|<footer.*?</footer>|<aside.*?</aside>",
               " ", h, flags=re.S | re.I)
    bagian = re.findall(r"<(?:h1|h2|h3|p|li)[^>]*>(.*?)</(?:h1|h2|h3|p|li)>", h, re.S | re.I)
    return [H.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", b))).strip()
            for b in bagian if len(re.sub(r"<[^>]+>", "", b).strip()) > 55]


print("=== A. sumber tambahan ===")
dapat = {}
for nama, u in KANDIDAT:
    st, h = ambil(u)
    if st != 200 or len(h) < 2000:
        print(f"   ❌ {nama:<22} {st}")
        continue
    baris = bersihkan(h)
    open(f"{DIR}/{nama}.txt", "w").write("\n".join(baris))
    dapat[nama] = {"url": u, "baris": len(baris)}
    j = re.search(r"<title[^>]*>(.*?)</title>", h, re.S)
    print(f"   ✅ {nama:<22} {len(baris):>4} baris | {(H.unescape(re.sub(r'<[^>]+>','',j.group(1)))[:70] if j else '')}")
    time.sleep(1)

print("\n=== B. kemampuan alat kita sendiri ===")
sys.path.insert(0, WS)
from repliz_client import get_client, ACC                              # noqa: E402
client = get_client()

print("   a. metode yang tersedia di repliz_client:")
import inspect
import repliz_client
for n, f in inspect.getmembers(repliz_client.ReplizClient, predicate=inspect.isfunction):
    if not n.startswith("_"):
        print(f"      - {n}")

print("\n   b. endpoint API Repliz yang bisa dipanggil (uji beberapa):")
for jalur in ("/public/account", "/public/schedule", "/public/account/insights",
              "/public/analytics", "/public/engagement", "/public/post"):
    try:
        d = client.get(jalur + ("?page=1&limit=1" if "schedule" in jalur or "account" in jalur and "insights" not in jalur else ""))
        cuplik = json.dumps(d)[:120] if isinstance(d, (dict, list)) else str(d)[:120]
        print(f"      ✅ {jalur:<32} → {cuplik}")
    except Exception as e:
        print(f"      ❌ {jalur:<32} → {str(e)[:80]}")

print("\n   c. skrip engagement yang sudah ada di workspace:")
import glob
for f in sorted(glob.glob(f"{WS}/*threads*")) + sorted(glob.glob(f"{WS}/*engagement*")):
    print("      " + os.path.basename(f))

json.dump(dapat, open(f"{DIR}/_daftar_tambahan.json", "w"), indent=1, ensure_ascii=False)