#!/usr/bin/env python3
"""Riset Threads tahap 1: tarik sumber resmi Meta + beberapa artikel industri, simpan teksnya."""
import html as H, json, os, re, time, urllib.request, urllib.error

WS = "/home/mantapman/.hermes/profiles/atun/workspace"
DIR = f"{WS}/drafts/riset_threads"
os.makedirs(DIR, exist_ok=True)
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
      "Accept-Language": "en-US,en;q=0.9", "Accept": "text/html,application/xhtml+xml"}

SUMBER = [
    # dokumen resmi
    ("meta_transparansi_ranking", "https://transparency.meta.com/features/explaining-ranking/threads/"),
    ("meta_newsroom_threads", "https://about.fb.com/news/2023/07/introducing-threads/"),
    ("meta_api_threads", "https://developers.facebook.com/docs/threads"),
    ("meta_api_insights", "https://developers.facebook.com/docs/threads/insights"),
    # industri
    ("hootsuite_algoritma", "https://blog.hootsuite.com/threads-algorithm/"),
    ("metricool_algoritma", "https://metricool.com/threads-algorithm/"),
    ("socialbee_algoritma", "https://socialbee.com/blog/threads-algorithm/"),
    ("buffer_threads", "https://buffer.com/resources/threads-marketing/"),
    ("later_threads", "https://later.com/blog/threads-for-business/"),
    ("sprout_threads", "https://sproutsocial.com/insights/threads-marketing/"),
    ("sej_threads", "https://www.searchenginejournal.com/what-is-threads/"),
    ("smt_threads", "https://www.socialmediatoday.com/topics/threads/"),
]


def ambil(u, timeout=70):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=timeout)
        return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, ""
    except Exception as e:
        return 0, str(e)[:90]


def bersihkan(h):
    h = re.sub(r"<script.*?</script>|<style.*?</style>|<nav.*?</nav>|<header.*?</header>|<footer.*?</footer>|<aside.*?</aside>",
               " ", h, flags=re.S | re.I)
    bagian = re.findall(r"<(?:h1|h2|h3|p|li)[^>]*>(.*?)</(?:h1|h2|h3|p|li)>", h, re.S | re.I)
    baris = []
    for b in bagian:
        t = H.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", b))).strip()
        if len(t) > 55:
            baris.append(t)
    return baris


ringkas = {}
for nama, u in SUMBER:
    st, h = ambil(u)
    if st != 200 or len(h) < 2000:
        print(f"   ❌ {nama:<26} status {st}")
        ringkas[nama] = {"url": u, "status": st}
        continue
    judul = re.search(r"<title[^>]*>(.*?)</title>", h, re.S)
    baris = bersihkan(h)
    berkas = f"{DIR}/{nama}.txt"
    open(berkas, "w").write((H.unescape(re.sub(r"\s+", " ", judul.group(1))) if judul else "") + "\n\n" + "\n".join(baris))
    ringkas[nama] = {"url": u, "status": st, "judul": (H.unescape(re.sub(r"\s+", " ", judul.group(1)))[:120] if judul else ""),
                     "baris": len(baris), "berkas": berkas}
    print(f"   ✅ {nama:<26} {len(baris):>4} baris | {(ringkas[nama]['judul'] or '')[:70]}")
    time.sleep(1)

json.dump(ringkas, open(f"{DIR}/_daftar_sumber.json", "w"), indent=1, ensure_ascii=False)
print(f"\ntersimpan di {DIR}/")