#!/usr/bin/env python3
"""post-type.py — media vs text-only, and forwarded-in vs original, compared
inside each channel. Writes post-type-stats.json.

Reads posts.csv (made by analyse.py) and, for the photo / video split only, the
wave-one preview pages in pages/w1/ that posts.csv was built from. Never
touches the network. Deterministic: the one random step (the label-shuffle
check) uses a fixed seed.

Why a separate script: stats.json already carries the headline within-channel
comparison (`media_vs_text_within_channel`). The article on media vs text needs
a few more cuts of the same rows — the comparison restricted to posts old
enough to have settled, the split by channel size, photo vs video, forwards,
and a check of how much of the spread chance alone produces — and adding them
here keeps every figure in stats.json exactly as published.

    python3 post-type.py
"""

import csv
import glob
import gzip
import json
import os
import random
import re
import statistics

HERE = os.path.dirname(os.path.abspath(__file__))
MIN_EACH = 3            # same rule as stats.json: at least 3 posts of each kind
MATURE_HOURS = 48.0     # same line as analyse.py's "settled" cut
SHUFFLES = 200
SEED = 20261008

# Same message split and id pattern as analyse.py.
RE_MSG_SPLIT = re.compile(r'(?=<div class="tgme_widget_message[ "])')
RE_DATA_POST = re.compile(r'data-post="([A-Za-z0-9_]+)/(\d+)"')
VIDEO = ("tgme_widget_message_video", "tgme_widget_message_roundvideo")
PHOTO = ("tgme_widget_message_photo",)


def pct(values, q):
    v = sorted(values)
    if not v:
        return None
    if len(v) == 1:
        return v[0]
    i = (len(v) - 1) * q
    lo, hi = int(i), min(int(i) + 1, len(v) - 1)
    return v[lo] + (v[hi] - v[lo]) * (i - lo)


def summary(values):
    v = list(values)
    if not v:
        return None
    return {
        "n": len(v),
        "p10": round(pct(v, 0.10), 4),
        "p25": round(pct(v, 0.25), 4),
        "median": round(pct(v, 0.50), 4),
        "p75": round(pct(v, 0.75), 4),
        "p90": round(pct(v, 0.90), 4),
    }


def ranks(v):
    order = sorted(range(len(v)), key=lambda i: v[i])
    r = [0.0] * len(v)
    i = 0
    while i < len(order):
        j = i
        while j + 1 < len(order) and v[order[j + 1]] == v[order[i]]:
            j += 1
        for k in range(i, j + 1):
            r[order[k]] = (i + j) / 2.0 + 1
        i = j + 1
    return r


def spearman(x, y):
    rx, ry = ranks(x), ranks(y)
    mx, my = statistics.fmean(rx), statistics.fmean(ry)
    num = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
    den = (sum((a - mx) ** 2 for a in rx) * sum((b - my) ** 2 for b in ry)) ** 0.5
    return round(num / den, 4) if den else None


# The public preview renders some attachments only as a placeholder ("this
# media is not supported in your browser"). analyse.py's media markers do not
# include it, so has_media is 0 on those posts. Recorded here so the text
# column can be checked with and without them.
UNSUPPORTED = "message_media_not_supported"
UNSUPPORTED_POSTS = set()


def media_kind(block):
    if any(k in block for k in VIDEO):
        return "video"
    if any(k in block for k in PHOTO):
        return "photo"
    return "other"


def load_kinds():
    """(channel_lower, post_id) -> photo | video | other, from wave-one pages."""
    kinds = {}
    for path in sorted(glob.glob(os.path.join(HERE, "pages", "w1", "*.posts.html.gz"))):
        name = os.path.basename(path)[:-len(".posts.html.gz")]
        with gzip.open(path, "rt", encoding="utf-8", errors="replace") as fh:
            html = fh.read()
        for block in RE_MSG_SPLIT.split(html):
            m = RE_DATA_POST.search(block)
            if not m or m.group(1).lower() != name.lower():
                continue
            kinds[(name.lower(), int(m.group(2)))] = media_kind(block)
            if UNSUPPORTED in block:
                UNSUPPORTED_POSTS.add((name.lower(), int(m.group(2))))
    return kinds


def load():
    kinds = load_kinds()
    chans = {}
    with open(os.path.join(HERE, "posts.csv"), encoding="utf-8") as fh:
        for r in csv.DictReader(fh):
            p = {
                "views": int(r["views"]),
                "ratio": float(r["view_to_subscriber_ratio"]),
                "age": float(r["age_hours_at_capture"]),
                "fwd": r["is_forward"] == "1",
                "media": r["has_media"] == "1",
            }
            p["unsupported"] = (r["channel"].lower(), int(r["post_id"])) in UNSUPPORTED_POSTS
            p["kind"] = kinds.get((r["channel"].lower(), int(r["post_id"])), "other") \
                if p["media"] else "text"
            c = chans.setdefault(r["channel"], {
                "subscribers": int(r["subscribers"]),
                "band": r["size_band"], "posts": []})
            c["posts"].append(p)
    return chans


def within(chans, is_a, is_b, keep=lambda p: True):
    """Per channel: median views of group A over median views of group B."""
    out = []
    for name, c in chans.items():
        posts = [p for p in c["posts"] if keep(p)]
        a = [p["views"] for p in posts if is_a(p)]
        b = [p["views"] for p in posts if is_b(p)]
        if len(a) >= MIN_EACH and len(b) >= MIN_EACH:
            mb = statistics.median(b)
            if mb:
                out.append({"channel": name, "band": c["band"],
                            "ratio": statistics.median(a) / mb,
                            "a": a, "b": b})
    return out


def describe(rows):
    v = [r["ratio"] for r in rows]
    if not v:
        return {"channels_compared": 0}
    return {
        "channels_compared": len(v),
        "summary": summary(v),
        "share_where_first_group_did_better": round(sum(1 for x in v if x > 1) / len(v), 4),
        "share_within_25pct_either_way": round(
            sum(1 for x in v if 0.8 <= x <= 1.25) / len(v), 4),
        "share_first_group_at_least_1_5x": round(sum(1 for x in v if x >= 1.5) / len(v), 4),
        "share_first_group_at_most_two_thirds": round(
            sum(1 for x in v if x <= 2 / 3) / len(v), 4),
    }


def shuffle_check(rows):
    """Shuffle the labels inside each channel, keeping its counts, and see what
    spread of ratios chance alone produces. Also: in how many channels is the
    observed ratio outside the middle 90% of its own shuffles?"""
    rng = random.Random(SEED)
    per_run = []
    outside = 0
    for r in rows:
        r["_null"] = []
    for _ in range(SHUFFLES):
        run = []
        for r in rows:
            pool = r["a"] + r["b"]
            rng.shuffle(pool)
            a, b = pool[:len(r["a"])], pool[len(r["a"]):]
            mb = statistics.median(b)
            x = statistics.median(a) / mb if mb else None
            if x is not None:
                run.append(x)
                r["_null"].append(x)
        per_run.append(run)
    for r in rows:
        lo, hi = pct(r["_null"], 0.05), pct(r["_null"], 0.95)
        if r["ratio"] < lo or r["ratio"] > hi:
            outside += 1
        del r["_null"]
    return {
        "what_it_is": "labels shuffled inside each channel %d times (seed %d), "
                      "keeping each channel's count of each kind. The figures are "
                      "the median, across shuffles, of each summary statistic." % (SHUFFLES, SEED),
        "median_of_shuffled_p25": round(statistics.median(pct(x, 0.25) for x in per_run), 4),
        "median_of_shuffled_median": round(statistics.median(pct(x, 0.5) for x in per_run), 4),
        "median_of_shuffled_p75": round(statistics.median(pct(x, 0.75) for x in per_run), 4),
        "median_of_shuffled_share_over_1": round(statistics.median(
            sum(1 for v in x if v > 1) / len(x) for x in per_run), 4),
        "channels_outside_their_own_middle_90pct": outside,
        "channels_tested": len(rows),
        "share_outside": round(outside / len(rows), 4) if rows else None,
        "share_expected_by_chance": 0.10,
    }


def main():
    chans = load()
    posts = [p for c in chans.values() for p in c["posts"]]
    media = lambda p: p["media"]
    text = lambda p: not p["media"]
    mature = lambda p: p["age"] >= MATURE_HOURS

    st = {"source": "posts.csv and pages/w1/*.posts.html.gz; wave one, 2026-09-09",
          "rule": "a channel enters a comparison only with at least %d posts of each "
                  "kind; ratio = median views of the first kind / median views of "
                  "the second, inside the same channel" % MIN_EACH,
          "posts": len(posts), "channels": len(chans)}

    all_ages = within(chans, media, text)
    st["media_vs_text_all_ages"] = describe(all_ages)
    st["media_vs_text_all_ages"]["note"] = (
        "reproduces stats.json media_vs_text_within_channel (146 channels, median 1.0339)")

    unsup = [p for p in posts if not p["media"] and p["unsupported"]]
    unsup_chans = sum(1 for c in chans.values()
                      if any(not p["media"] and p["unsupported"] for p in c["posts"]))
    dropped = within(chans, media, text, lambda p: p["media"] or not p["unsupported"])
    moved = within(chans, lambda p: p["media"] or p["unsupported"],
                   lambda p: not p["media"] and not p["unsupported"])
    st["text_posts_with_unsupported_attachment"] = {
        "what_it_is": "has_media=0 posts whose stored preview HTML contains %s; "
                      "the preview shows an attachment placeholder, not text alone" % UNSUPPORTED,
        "posts": len(unsup),
        "of_text_posts": sum(1 for p in posts if not p["media"]),
        "channels_with_any": unsup_chans,
        "media_vs_text_with_them_dropped": describe(dropped),
        "media_vs_text_with_them_counted_as_media": describe(moved),
    }

    mat = within(chans, media, text, mature)
    st["media_vs_text_posts_48h_or_older"] = describe(mat)

    # Age is the largest single driver of a post's count. Do media and text posts
    # in the same channel differ in age? Median age of media posts over median
    # age of text posts, per channel.
    age_ratio = []
    for r in all_ages:
        ps = chans[r["channel"]]["posts"]
        am = statistics.median(p["age"] for p in ps if p["media"])
        at = statistics.median(p["age"] for p in ps if not p["media"])
        if at:
            age_ratio.append(am / at)
    st["age_of_media_over_age_of_text_within_channel"] = summary(age_ratio)

    st["media_vs_text_by_size_band"] = {}
    for band in sorted({c["band"] for c in chans.values()}):
        keep_band = [r for r in all_ages if r["band"] == band]
        keep_mat = [r for r in mat if r["band"] == band]
        st["media_vs_text_by_size_band"][band] = {
            "all_ages": describe(keep_band),
            "posts_48h_or_older": describe(keep_mat),
        }

    kinds = {}
    for kind in ("photo", "video", "other"):
        is_k = (lambda k: (lambda p: p["kind"] == k))(kind)
        kinds[kind] = {
            "posts": sum(1 for p in posts if p["kind"] == kind),
            "vs_text_all_ages": describe(within(chans, is_k, text)),
            "vs_text_posts_48h_or_older": describe(within(chans, is_k, text, mature)),
        }
    kinds["note"] = ("photo = a photo marker and no video marker; video = video or "
                     "round video; other = document, voice, audio, sticker, poll, "
                     "location. An album with any video counts as video.")
    kinds["text_posts"] = sum(1 for p in posts if p["kind"] == "text")
    st["media_kind_vs_text"] = kinds

    fwd = lambda p: p["fwd"]
    own = lambda p: not p["fwd"]
    st["forwarded_in_vs_original"] = {
        "note": "is_forward = forwarded INTO this channel from elsewhere. The source "
                "cannot see how often a channel's own posts were forwarded outward.",
        "channels_with_any_forward": sum(1 for c in chans.values()
                                         if any(p["fwd"] for p in c["posts"])),
        "all_ages": describe(within(chans, fwd, own)),
        "posts_48h_or_older": describe(within(chans, fwd, own, mature)),
    }

    st["shuffle_check_media_vs_text_all_ages"] = shuffle_check(all_ages)

    # Why the pooled figure in stats.json post_type points the other way.
    bands = {}
    for band in sorted({c["band"] for c in chans.values()}):
        bp = [p for c in chans.values() if c["band"] == band for p in c["posts"]]
        bands[band] = {
            "posts": len(bp),
            "share_with_media": round(sum(p["media"] for p in bp) / len(bp), 4),
            "pooled_median_ratio_media": round(pct([p["ratio"] for p in bp if p["media"]], 0.5), 4),
            "pooled_median_ratio_text": round(pct([p["ratio"] for p in bp if not p["media"]], 0.5), 4),
        }
    text_posts = [p for c in chans.values() for p in c["posts"] if not p["media"]]
    media_posts = [p for c in chans.values() for p in c["posts"] if p["media"]]
    small = "small (<1k)"
    sub_media = [c["subscribers"] for c in chans.values() for p in c["posts"] if p["media"]]
    sub_text = [c["subscribers"] for c in chans.values() for p in c["posts"] if not p["media"]]
    shares, ratios = [], []
    for c in chans.values():
        shares.append(sum(p["media"] for p in c["posts"]) / len(c["posts"]))
        ratios.append(statistics.median(p["ratio"] for p in c["posts"]))
    st["why_pooled_differs"] = {
        "by_size_band": bands,
        "share_of_text_posts_from_small_channels": round(
            sum(1 for c in chans.values() if c["band"] == small for p in c["posts"] if not p["media"])
            / len(text_posts), 4),
        "share_of_media_posts_from_small_channels": round(
            sum(1 for c in chans.values() if c["band"] == small for p in c["posts"] if p["media"])
            / len(media_posts), 4),
        "median_subscribers_behind_a_media_post": int(statistics.median(sub_media)),
        "median_subscribers_behind_a_text_post": int(statistics.median(sub_text)),
        "spearman_channel_media_share_vs_channel_median_ratio": spearman(shares, ratios),
        "channels_all_media": sum(1 for s in shares if s == 1.0),
        "channels_no_media": sum(1 for s in shares if s == 0.0),
    }

    out = os.path.join(HERE, "post-type-stats.json")
    with open(out, "w", encoding="utf-8") as fh:
        json.dump(st, fh, indent=1, ensure_ascii=False)
        fh.write("\n")
    print("wrote", out)


if __name__ == "__main__":
    main()