#!/usr/bin/env python3
"""Download all Instagram posts from leitenglishinstitute"""
import json, os, sys, time, urllib.request, urllib.error, concurrent.futures

USERNAME = "leitenglishinstitute"
OUTPUT_DIR = "/home/mantapman/.hermes/profiles/atun/workspace/leit_ig_download/posts"
os.makedirs(OUTPUT_DIR, exist_ok=True)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.9",
    "X-IG-App-ID": "936619743392459",
}

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
]
import random

def req(url, retries=3):
    ua = random.choice(USER_AGENTS)
    h = {**HEADERS, "User-Agent": ua}
    r = urllib.request.Request(url, headers=h)
    for attempt in range(retries):
        try:
            time.sleep(random.uniform(1.5, 3.5))
            with urllib.request.urlopen(r, timeout=20) as resp:
                return resp.read().decode()
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = (attempt + 1) * 10
                print(f"  429! Waiting {wait}s (attempt {attempt+1}/{retries})")
                time.sleep(wait)
                continue
            return f"ERROR:{e}"
        except Exception as e:
            return f"ERROR:{e}"
    return "ERROR:Max retries (429)"

# Step 1: Get user info
print("Getting user info...")
info_url = f"https://www.instagram.com/api/v1/users/web_profile_info/?username={USERNAME}"
info_raw = req(info_url)
if info_raw.startswith("ERROR:"):
    print(f"Failed: {info_raw}")
    sys.exit(1)

info = json.loads(info_raw)
user = info["data"]["user"]
user_id = user["id"]
total_posts = user["edge_owner_to_timeline_media"]["count"]
print(f"User ID: {user_id}, Total posts: {total_posts}")

# Step 2: Scrape all posts pagination
QUERY_HASH = "e769aa130647d2354c40ea6a439bfc08"
all_posts = []
end_cursor = None
page = 0

while page < 50:  # safety limit
    page += 1
    if end_cursor:
        variables = json.dumps({"id": user_id, "first": 50, "after": end_cursor})
    else:
        variables = json.dumps({"id": user_id, "first": 50})
    
    enc_vars = urllib.parse.quote(variables)
    url = f"https://www.instagram.com/graphql/query/?query_hash={QUERY_HASH}&variables={enc_vars}"
    
    raw = req(url)
    if raw.startswith("ERROR:"):
        print(f"  Page {page} error: {raw}")
        break
    
    try:
        data = json.loads(raw)["data"]["user"]["edge_owner_to_timeline_media"]
    except (KeyError, json.JSONDecodeError) as e:
        print(f"  Page {page} parse error: {e}")
        print(f"  Raw[:200]: {raw[:200]}")
        break
    
    edges = data["edges"]
    for e in edges:
        n = e["node"]
        all_posts.append({
            "shortcode": n["shortcode"],
            "typename": n.get("__typename", ""),
            "display_url": n.get("display_url", ""),
            "is_video": n.get("is_video", False),
            "video_url": n.get("video_url", ""),
            "taken_at": n.get("taken_at_timestamp", 0),
            "caption": n.get("edge_media_to_caption", {}).get("edges", [{}])[0].get("node", {}).get("text", "")[:200]
            if n.get("edge_media_to_caption", {}).get("edges") else "",
        })
    
    page_info = data["page_info"]
    has_next = page_info["has_next_page"]
    end_cursor = page_info.get("end_cursor")
    
    print(f"  Page {page}: {len(edges)} posts, total={len(all_posts)}, more={has_next}")
    
    if not has_next:
        break
    time.sleep(0.5)

print(f"\nTotal posts collected: {len(all_posts)}")

# Step 3: Download media concurrently
def download_post(post):
    shortcode = post["shortcode"]
    is_video = post["is_video"]
    
    if is_video:
        url = post.get("video_url", "")
        ext = ".mp4"
    else:
        url = post.get("display_url", "")
        ext = ".jpg"
    
    if not url:
        return f"{shortcode}: no url"
    
    out_path = os.path.join(OUTPUT_DIR, f"{shortcode}{ext}")
    if os.path.exists(out_path) and os.path.getsize(out_path) > 1000:
        return f"{shortcode}: exists"
    
    try:
        r = urllib.request.Request(url, headers={"User-Agent": HEADERS["User-Agent"]})
        with urllib.request.urlopen(r, timeout=30) as resp:
            data = resp.read()
        with open(out_path, "wb") as f:
            f.write(data)
        return f"{shortcode}: OK ({len(data)} bytes)"
    except Exception as e:
        return f"{shortcode}: ERROR {e}"

print("\nDownloading media...")
downloaded = 0
errors = 0

with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
    futures = {executor.submit(download_post, p): p for p in all_posts}
    for future in concurrent.futures.as_completed(futures):
        result = future.result()
        if "OK" in result:
            downloaded += 1
        elif "exists" in result:
            downloaded += 1
        else:
            errors += 1
        if downloaded % 50 == 0:
            print(f"  Progress: {downloaded} downloaded, {errors} errors")

print(f"\n=== DONE ===")
print(f"Total posts: {len(all_posts)}")
print(f"Downloaded: {downloaded}")
print(f"Errors: {errors}")
print(f"Location: {OUTPUT_DIR}")