# -*- coding: utf-8 -*-
"""What moved between the two captures of the same fifteen catalogues.

The first capture (2 September 2026, 00:16Z) was a single instant. The page
said so, and said that prices on one night are not prices in general. This
script is the second half of that sentence: the same fifteen /services pages,
read again inside another twelve-second window on 13 September 2026 at 20:28Z,
and joined listing by listing.

A listing is the same listing in both captures when the panel and the service
id match. That is the panel's own key - it is what an API order is placed
against - so a renamed service is still the same service, and a service that
disappears and comes back under a new id is counted as one removal and one
addition, which is what it is from the buyer's side. peakerr.com publishes no
ids at all and is joined on its eight headline names instead; it is reported
but kept out of every price figure, as in the first capture.

Inputs:  services-all.json.gz, services-all-w2.json.gz,
         services-basket.csv, services-basket-w2.csv
Outputs: changes.json          every aggregate the article quotes
         price-changes.csv     one row per listing whose price changed (the 13 panels the figures cover)
         renamed-listings.csv  every listing that kept its id and changed its name

    python3 compare.py

No network. Run parse_panels.py and analyse.py for both captures first, or use
the files as published.
"""
import csv, gzip, json, os, statistics, collections

HERE = os.path.dirname(os.path.abspath(__file__))

DOMAIN = {"smmlaunch":"smmlaunch.com","adderpanel":"adderpanel.com","crescitaly":"crescitaly.com",
 "smmcost":"smmcost.com","smmkings":"smmkings.com","growfastsmm":"growfastsmm.com","1xpanel":"1xpanel.com",
 "morethanpanel":"morethanpanel.com","mysmm":"mysmm.co","smmfolgen":"smmfolgen.com","easytopromo":"easytopromo.com",
 "smmpanelone":"smmpanel.one","globalsmm":"global-smm.com","autosmo":"autosmo.com","peakerr":"peakerr.com"}
PUBLISHER = {"smmlaunch", "adderpanel"}
# Same two exclusions as analyse.py, for the same reasons: growfastsmm prices in
# rupees, and peakerr's rows are headline prices with no id or order limits.
NOT_USD = {"growfastsmm"}
NO_IDS = {"peakerr"}

def load(name):
    return json.load(gzip.open(os.path.join(HERE, name), "rt", encoding="utf-8"))

def tidy(v):
    return " ".join((v or "").replace("|", " ").split())

def key(r):
    if r["panel"] in NO_IDS:
        return tidy(r["name"]).lower()
    # tidy(), because two table templates leave the id cell as "| 7450 |";
    # first token only, because smmfolgen.com's id cell also carries the text
    # of its "details" link, which would break the join if the wording changed.
    parts = tidy(r["sid"]).split()
    return parts[0] if parts else ""

def pct(a, b):
    return (b - a) / a * 100.0

def r2(x):
    return None if x is None else round(x, 2)

w1, w2 = load("services-all.json.gz"), load("services-all-w2.json.gz")

by = {1: collections.defaultdict(dict), 2: collections.defaultdict(dict)}
dupes = collections.Counter()
for wave, rows in ((1, w1), (2, w2)):
    for r in rows:
        k = key(r)
        if not k:
            dupes[(wave, r["panel"], "no-key")] += 1
            continue
        if k in by[wave][r["panel"]]:
            # A panel that prints the same id twice (it happens when a service
            # is filed under two categories). Keep the first and count it, so
            # the join never silently pairs the wrong row.
            dupes[(wave, r["panel"], "duplicate-id")] += 1
            continue
        by[wave][r["panel"]][k] = r

panels = list(DOMAIN)
per_panel = {}
changes = []
renames = []
all_pcts = []
tot = collections.Counter()

for p in panels:
    a, b = by[1][p], by[2][p]
    kept = a.keys() & b.keys()
    removed = a.keys() - b.keys()
    added = b.keys() - a.keys()
    repriced, up, down, renamed = 0, 0, 0, 0
    pcts = []
    # sorted: a set's order changes with the hash seed, and so would every
    # output file, on a survey that says it reproduces byte for byte.
    for k in sorted(kept):
        ra, rb = a[k], b[k]
        if tidy(ra["name"]) != tidy(rb["name"]):
            renamed += 1
            if p not in NOT_USD and p not in NO_IDS:
                renames.append(dict(panel=DOMAIN[p], service_id=k, name_2sep=tidy(ra["name"]),
                                    name_13sep=tidy(rb["name"]), price_2sep=ra["rate"], price_13sep=rb["rate"],
                                    repriced="yes" if ra["rate"] != rb["rate"] else "no"))
        if ra["rate"] is None or rb["rate"] is None or ra["rate"] <= 0:
            continue
        if abs(ra["rate"] - rb["rate"]) > 1e-12:
            repriced += 1
            c = pct(ra["rate"], rb["rate"])
            pcts.append(c)
            up += c > 0
            down += c < 0
            changes.append(dict(panel=DOMAIN[p], service_id=k, name_2sep=tidy(ra["name"]),
                                name_13sep=tidy(rb["name"]), price_2sep=ra["rate"], price_13sep=rb["rate"],
                                currency=rb["currency"], change_pct=round(c, 2),
                                publisher_owned="yes" if p in PUBLISHER else "no"))
    n1, n2 = len(a), len(b)
    withdrawn_cats = collections.Counter(tidy(a[k].get("category")) or "(no category heading)" for k in removed)
    per_panel[DOMAIN[p]] = dict(
        doubled_or_more=sum(1 for c in pcts if c >= 100), halved_or_more=sum(1 for c in pcts if c <= -50),
        withdrawn_top_categories=[[c, n] for c, n in sorted(withdrawn_cats.items(), key=lambda x: (-x[1], x[0]))[:3]],
        publisher_owned=p in PUBLISHER, currency_usd=p not in NOT_USD, has_service_ids=p not in NO_IDS,
        listed_2sep=n1, listed_13sep=n2, kept=len(kept), removed=len(removed), added=len(added),
        removed_pct=r2(len(removed) / n1 * 100 if n1 else None),
        added_pct=r2(len(added) / n2 * 100 if n2 else None),
        repriced=repriced, repriced_pct_of_kept=r2(repriced / len(kept) * 100 if kept else None),
        up=up, down=down, renamed=renamed,
        median_change_pct=r2(statistics.median(pcts) if pcts else None),
    )
    if p not in NOT_USD and p not in NO_IDS:
        all_pcts += pcts
        tot.update(dict(listed_2sep=n1, listed_13sep=n2, kept=len(kept), removed=len(removed),
                        added=len(added), repriced=repriced, up=up, down=down, renamed=renamed))

# Sizes of the moves, over the thirteen USD panels with service ids.
ups = [c for c in all_pcts if c > 0]
downs = [c for c in all_pcts if c < 0]
bands = collections.OrderedDict([
    ("under 5%", lambda c: abs(c) < 5),
    ("5% to under 25%", lambda c: 5 <= abs(c) < 25),
    ("25% to under 100%", lambda c: 25 <= abs(c) < 100),
    ("100% or more", lambda c: abs(c) >= 100),
])
band_counts = {name: dict(up=sum(1 for c in ups if f(c)), down=sum(1 for c in downs if f(c)))
               for name, f in bands.items()}
repricing_panels = sorted(d for d, v in per_panel.items()
                          if v["currency_usd"] and v["has_service_ids"] and v["repriced"] > 0)
concentration = sorted(((v["repriced"], d) for d, v in per_panel.items()
                        if v["currency_usd"] and v["has_service_ids"]), reverse=True)

# Most repricing is not a decision about a listing. When hundreds of listings on
# one panel move by the same factor, that is one rule applied to a catalogue - a
# supplier's price passed through, or a margin changed - and counting it as
# hundreds of separate price moves overstates how much anyone is pricing.
# A "block" is ten or more listings on one panel whose new/old price ratio is
# the same to three decimals. Ten is arbitrary and printed so it can be changed.
BLOCK_MIN = 10
blocks = []
for p in panels:
    if p in NOT_USD or p in NO_IDS:
        continue
    groups = collections.defaultdict(list)
    for r in changes:
        if r["panel"] == DOMAIN[p]:
            groups[round(r["price_13sep"] / r["price_2sep"], 3)].append(r)
    for ratio, rs in groups.items():
        if len(rs) >= BLOCK_MIN and ratio != 1.0:
            # How many different starting prices the block spans. A block of
            # 150 listings that all went from $0.715 to $0.832 is one product
            # line repriced; 500 listings from 159 different prices moving by
            # the same factor is a proportional rule across a catalogue.
            blocks.append(dict(panel=DOMAIN[p], ratio=ratio, change_pct=round((ratio - 1) * 100, 1),
                               listings=len(rs), distinct_old_prices=len({r["price_2sep"] for r in rs})))
blocks.sort(key=lambda b: (-b["listings"], b["panel"], b["ratio"]))
in_blocks = sum(b["listings"] for b in blocks)
# Rounding to three decimals splits one rule across neighbouring ratios (a 2%
# cut on prices quoted to four decimals lands on 0.980, 0.981 and 0.982), and
# the pieces under ten are not counted, so this share is a lower bound.

# The words a listing uses about its own price, checked against what its price did.
usd_ids = {DOMAIN[p] for p in panels if p not in NOT_USD and p not in NO_IDS}
stable_kept = sum(1 for p in panels if DOMAIN[p] in usd_ids
                  for k in by[1][p].keys() & by[2][p].keys()
                  if "stable price" in tidy(by[1][p][k]["name"]).lower())
stable_moved = [r for r in changes if r["panel"] in usd_ids and "stable price" in r["name_2sep"].lower()]
stable_by_panel = collections.Counter(DOMAIN[p] for p in panels if DOMAIN[p] in usd_ids
                                      for k in by[1][p].keys() & by[2][p].keys()
                                      if "stable price" in tidy(by[1][p][k]["name"]).lower())
stable_moved_by_panel = collections.Counter(r["panel"] for r in stable_moved)

# One panel repriced more listings than the other twelve together, and a single
# 2% rule on it sets the median cut. The medians without it are printed beside
# the headline ones so nobody has to take "the typical cut was small" on trust.
largest = concentration[0][1]
rest = [r["change_pct"] for r in changes if r["panel"] in usd_ids and r["panel"] != largest]
rest_up = [c for c in rest if c > 0]
rest_down = [c for c in rest if c < 0]
without_largest = dict(panel=largest, repriced=len(rest),
                       median_rise_pct=r2(statistics.median(rest_up) if rest_up else None),
                       median_cut_pct=r2(statistics.median(rest_down) if rest_down else None),
                       cuts=len(rest_down), rises=len(rest_up))

corpus = dict(tot)
corpus.update(
    renamed_and_repriced=sum(1 for r in renames if r["repriced"] == "yes"),
    unchanged_pct_of_kept=r2((tot["kept"] - tot["repriced"]) / tot["kept"] * 100),
    # Percentages are lopsided (a cut cannot pass -100%), so compare moves by
    # factor: at least doubled against at least halved.
    doubled_or_more=sum(1 for c in all_pcts if c >= 100),
    halved_or_more=sum(1 for c in all_pcts if c <= -50),
    without_largest_repricer=without_largest,
    same_factor_blocks=dict(min_listings=BLOCK_MIN, blocks=blocks, listings_in_blocks=in_blocks,
                            listings_in_blocks_spanning_5_or_more_prices=sum(b["listings"] for b in blocks if b["distinct_old_prices"] >= 5),
                            share_of_repriced_pct=r2(in_blocks / tot["repriced"] * 100 if tot["repriced"] else None)),
    labelled_stable_price=dict(kept_listings_with_label=stable_kept, of_them_repriced=len(stable_moved),
                               repriced_pct=r2(len(stable_moved) / stable_kept * 100 if stable_kept else None),
                               kept_by_panel=dict(stable_by_panel), repriced_by_panel=dict(stable_moved_by_panel),
                               repriced_pct_by_panel={d: r2(stable_moved_by_panel[d] / n * 100) for d, n in stable_by_panel.items()},
                               kept_listings_by_panel={d: per_panel[d]["kept"] for d in stable_by_panel},
                               largest_rise_pct=max((r["change_pct"] for r in stable_moved), default=None)),
    panels=13,
    removed_pct_of_2sep=r2(tot["removed"] / tot["listed_2sep"] * 100),
    added_pct_of_13sep=r2(tot["added"] / tot["listed_13sep"] * 100),
    repriced_pct_of_kept=r2(tot["repriced"] / tot["kept"] * 100),
    up_pct_of_repriced=r2(tot["up"] / tot["repriced"] * 100 if tot["repriced"] else None),
    median_change_pct=r2(statistics.median(all_pcts) if all_pcts else None),
    median_rise_pct=r2(statistics.median(ups) if ups else None),
    median_cut_pct=r2(statistics.median(downs) if downs else None),
    size_bands=band_counts,
    panels_that_repriced_anything=len(repricing_panels),
    top_repricers=[dict(panel=d, repriced=n) for n, d in concentration[:5]],
)

# ---------- the basket table: why each cheapest price did or did not move ----------
def basket(name):
    out = {}
    with open(os.path.join(HERE, name), encoding="utf-8") as fh:
        for r in csv.DictReader(fh):
            if r["service_name_verbatim"] == "not listed" or r["note"]:
                continue
            out[(r["panel"], r["basket_item"])] = r
    return out

b1, b2 = basket("services-basket.csv"), basket("services-basket-w2.csv")
rev = {v: k for k, v in DOMAIN.items()}
cells = []
cause_count = collections.Counter()
for cell in sorted(b1.keys() | b2.keys(), key=lambda c: (c[1], c[0])):
    x, y = b1.get(cell), b2.get(cell)
    panel = rev[cell[0]]
    if x and not y:
        cause = "no qualifying listing on 13 Sep"
    elif y and not x:
        cause = "no qualifying listing on 2 Sep"
    else:
        s1, s2 = (tidy(x["service_id"]).split() or [""])[0], (tidy(y["service_id"]).split() or [""])[0]
        p1, p2 = float(x["price_per_1000"].lstrip("$").replace(",", "")), float(y["price_per_1000"].lstrip("$").replace(",", ""))
        if s1 == s2:
            # An id is the panel's key, but a panel can put a different product
            # behind the same id. When the name changed too, say so rather than
            # calling it the same listing.
            renamed_here = tidy(by[1][panel][s1]["name"]) != tidy(by[2][panel][s1]["name"])
            if abs(p1 - p2) < 1e-12:
                cause = "unchanged price, renamed" if renamed_here else "unchanged"
            else:
                cause = "same id, renamed and repriced" if renamed_here else "same listing repriced"
        elif s1 not in by[2][panel]:
            cause = "cheapest listing withdrawn"
        elif s2 not in by[1][panel]:
            cause = "new cheaper listing added"
        else:
            # Both listings exist on both dates, so one of two opposite things
            # happened, and they need different words: the old cheapest went up
            # past another listing, or another listing came down past it.
            old_then, old_now = by[1][panel][s1]["rate"], by[2][panel][s1]["rate"]
            new_then, new_now = by[1][panel][s2]["rate"], by[2][panel][s2]["rate"]
            if old_now > old_then:
                cause = "old cheapest repriced up, another listing now cheapest"
            elif new_now < new_then:
                cause = "another listing cut below the old cheapest"
            else:
                cause = "eligibility or name change moved the cheapest"
    cause_count[cause] += 1
    cells.append(dict(panel=cell[0], basket_item=cell[1],
                      price_2sep=x["price_per_1000"] if x else None, service_id_2sep=x["service_id"] if x else None,
                      price_13sep=y["price_per_1000"] if y else None, service_id_13sep=y["service_id"] if y else None,
                      cause=cause))

def spread(stats_name):
    s = json.load(open(os.path.join(HERE, stats_name), encoding="utf-8"))["spread"]
    return {k: dict(panels=v[0], cheapest=v[1], median=v[2], dearest=v[3], ratio=round(v[4], 1)) for k, v in s.items()}

result = dict(
    captures=dict(first="2026-09-02T00:16:39Z/2026-09-02T00:16:51Z", second="2026-09-13T20:27:59Z/2026-09-13T20:28:11Z"),
    join="panel + service id (peakerr.com: name, reported but not priced)",
    duplicate_or_missing_keys={f"w{w}:{DOMAIN[p]}:{kind}": n for (w, p, kind), n in dupes.items()},
    corpus_13_usd_panels_with_ids=corpus,
    per_panel=per_panel,
    basket_spread=dict(first=spread("stats.json"), second=spread("stats-w2.json")),
    basket_cells=cells,
    basket_cell_causes=dict(cause_count),
)
with open(os.path.join(HERE, "changes.json"), "w", encoding="utf-8") as fh:
    json.dump(result, fh, indent=1, ensure_ascii=False)
    fh.write("\n")

cols = ["panel", "publisher_owned", "service_id", "name_2sep", "name_13sep", "price_2sep", "price_13sep", "currency", "change_pct"]
with open(os.path.join(HERE, "price-changes.csv"), "w", newline="", encoding="utf-8") as fh:
    w = csv.DictWriter(fh, fieldnames=cols)
    w.writeheader()
    # Only the thirteen panels every figure is computed over: growfastsmm.com's
    # rupee rows would otherwise make this file disagree with the 2,811 quoted.
    for r in sorted((r for r in changes if r["panel"] in usd_ids),
                    key=lambda r: (r["panel"], -abs(r["change_pct"]), r["service_id"])):
        w.writerow({c: r[c] for c in cols})

# Every listing that kept its id and changed its name - the rows a reader needs
# in order to judge whether "same listing" is fair for any given id.
cols = ["panel", "service_id", "name_2sep", "name_13sep", "price_2sep", "price_13sep", "repriced"]
with open(os.path.join(HERE, "renamed-listings.csv"), "w", newline="", encoding="utf-8") as fh:
    w = csv.DictWriter(fh, fieldnames=cols)
    w.writeheader()
    for r in sorted(renames, key=lambda r: (r["panel"], r["service_id"])):
        w.writerow(r)

print(json.dumps({k: result[k] for k in ("duplicate_or_missing_keys", "corpus_13_usd_panels_with_ids", "basket_cell_causes")}, indent=1))
print(f"{'panel':20s} {'2sep':>6s} {'13sep':>6s} {'kept':>6s} {'gone':>6s} {'new':>6s} {'repr':>6s} {'up':>5s} {'down':>5s} {'renam':>6s} {'med%':>7s}")
for d, v in per_panel.items():
    print(f"{d:20s} {v['listed_2sep']:6d} {v['listed_13sep']:6d} {v['kept']:6d} {v['removed']:6d} {v['added']:6d} {v['repriced']:6d} {v['up']:5d} {v['down']:5d} {v['renamed']:6d} {str(v['median_change_pct']):>7s}")
for c in cells:
    print(f"  {c['basket_item']:26s} {c['panel']:18s} {str(c['price_2sep']):>10s} -> {str(c['price_13sep']):>10s}  {c['cause']}")
print("price-changes.csv rows:", len(changes))