#!/usr/bin/env python3
"""Ambil artikel industri soal algoritma & pertumbuhan Threads lewat curl, lalu ekstrak intinya."""
import html as H, json, re, urllib.request, urllib.error

UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
      "Accept-Language": "en-US,en;q=0.9", "Accept": "text/html,application/xhtml+xml"}

KANDIDAT = [
    ("Hootsuite — algoritma Threads", "https://blog.hootsuite.com/threads-algorithm/"),
    ("Sprout Social — algoritma Threads", "https://sproutsocial.com/insights/threads-algorithm/"),
    ("Later — algoritma Threads", "https://later.com/blog/threads-algorithm/"),
    ("Buffer — tumbuh di Threads", "https://buffer.com/resources/threads-growth/"),
    ("Metricool — Threads", "https://metricool.com/threads-algorithm/"),
    ("SocialBee — Threads", "https://socialbee.com/blog/threads-algorithm/"),
]


def ambil(u, timeout=60):
    try:
        r = urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=timeout)
        return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, ""
    except Exception as e:
        return 0, str(e)[:90]


def teks_artikel(h):
    h = re.sub(r"<script.*?</script>|<style.*?</style>|<nav.*?</nav>|<header.*?</header>|<footer.*?</footer>",
               " ", h, flags=re.S | re.I)
    bagian = re.findall(r"<(?:h2|h3|p|li)[^>]*>(.*?)</(?:h2|h3|p|li)>", h, re.S | re.I)
    baris = []
    for b in bagian:
        t = H.unescape(re.sub(r"\s+", " ", re.sub(r"<[^>]+>", "", b))).strip()
        if len(t) > 60:
            baris.append(t)
    return baris


hasil = {}
for nama, u in KANDIDAT:
    st, h = ambil(u)
    print(f"\n=== {nama} | status {st} | {len(h):,} byte ===")
    if st != 200 or not h:
        continue
    judul = re.search(r"<title[^>]*>(.*?)</title>", h, re.S)
    print("   judul:", H.unescape(re.sub(r"\s+", " ", judul.group(1)))[:110] if judul else "-")
    baris = teks_artikel(h)
    hasil[nama] = {"url": u, "baris": baris[:60]}
    # tampilkan baris yang memuat kata kunci algoritma/engagement/reply
    kunci = re.compile(r"algorithm|engage|reply|conversation|reach|follower|comment", re.I)
    penting = [b for b in baris if kunci.search(b)][:12]
    for b in penting:
        print("   • " + b[:210])

json.dump(hasil, open("/home/mantapman/.hermes/profiles/atun/workspace/drafts/riset_threads_industri.json", "w"), indent=1, ensure_ascii=False)
print("\n→ drafts/riset_threads_industri.json")