# -*- coding: utf-8 -*- """Does a refill promise show up in price, and is a promised listing's price more stable? Offline, deterministic, read-only on the survey directory. Writes refill-stats.json next to itself, section by section, so partial work survives. Definitions are copied verbatim from the published scripts: - refill classification: analyse.py (tidy, blob = name + " || " + desc, REFILL_PERIOD, REFILL_NONE, REFILL_ANY, eligible()). Five Perfect-Panel-style panels hide descriptions, so only their names count. - join across captures: compare.py (tidy, key = panel + first token of sid, index() keeps the first row for a duplicate id, rows with no key dropped; followed = key on all three captures with rate > 0 on all three; 13 USD panels with ids). python3 refill.py """ import gzip, json, os, re, statistics, collections, sys, unicodedata SRC = os.path.dirname(os.path.abspath(__file__)) OUTDIR = SRC STATS = os.path.join(OUTDIR, "refill-stats.json") CAPS = [("w1", "services-all.json.gz", "2 Sep"), ("w2", "services-all-w2.json.gz", "13 Sep"), ("w3", "services-all-w3.json.gz", "2 Oct")] DOMAIN = {"smmlaunch":"smmlaunch.com","adderpanel":"adderpanel.com","crescitaly":"crescitaly.com", "smmcost":"smmcost.com","smmkings":"smmkings.com","growfastsmm":"growfastsmm.com","1xpanel":"1xpanel.com", "morethanpanel":"morethanpanel.com","mysmm":"mysmm.co","smmfolgen":"smmfolgen.com","easytopromo":"easytopromo.com", "smmpanelone":"smmpanel.one","globalsmm":"global-smm.com","autosmo":"autosmo.com","peakerr":"peakerr.com"} PANELS = list(DOMAIN) PUBLISHER = {"smmlaunch", "adderpanel"} HIDDEN_DESC = {"smmlaunch", "adderpanel", "crescitaly", "smmcost", "smmkings"} # method.md s.5 NOT_USD = {"growfastsmm"} NO_IDS = {"peakerr"} USD = [p for p in PANELS if p not in NOT_USD] COMPARABLE = [p for p in PANELS if p not in NOT_USD and p not in NO_IDS] BIG = "1xpanel" def label(p): return DOMAIN[p] + (" (publisher-owned)" if p in PUBLISHER else "") result = {} def save(): with open(STATS, "w", encoding="utf-8") as fh: json.dump(result, fh, indent=1, ensure_ascii=False, sort_keys=False) fh.write("\n") def r2(x): return None if x is None else round(x, 2) def r4(x): return None if x is None else round(x, 4) def pct(a, b): return r2(a / b * 100) if b else None # ---------------- verbatim from analyse.py ---------------- def tidy_a(v): if not isinstance(v, str): return v v = re.sub(r"^\s*\|\s*", "", v) v = re.sub(r"\s*\|\s*$", "", v) return re.sub(r"\s+", " ", v).strip() def fold(s): s = unicodedata.normalize("NFKC", s) s = "".join(ch for ch in s if not unicodedata.category(ch).startswith("So")) s = s.lower() s = re.sub(r"[^a-z0-9]+", " ", s) return re.sub(r"\s+", " ", s).strip() DEAD_CAT = re.compile(r"do\s*n.?t\s*use|do\s+not\s+use|deprecated|disabled|closed|test\s*only", re.I) def order_max(r): d = re.sub(r"[^\d]", "", r.get("max") or "") return int(d) if d else None def eligible(r): if r["rate"] is None or r["rate"] <= 0: return False m = order_max(r) if m is None or m < 1000: return False if DEAD_CAT.search(r.get("category") or ""): return False if re.search(r"ask me|make ticket|contact us|do not order", r["name"], re.I): return False if re.search(r"^\s*(?:id\s+)?test\b", r["name"], re.I): return False return True REFILL_PERIOD = re.compile(r"\b\d+\s*[- ]?\s*(?:day|days|hour|hours|month|months|year|years|d|h)\s*(?:refill|guarantee|guaranteed|gaurantee|warranty)|refill\s*[:\-]?\s*\d+\s*(?:day|days|month|months|year|years)|\b(?:lifetime|life\s*time|permanent)\s*[- ]?\s*(?:refill|guarantee|guaranteed|warranty)", re.I) REFILL_NONE = re.compile(r"\b(?:no|non|zero|without)\s*[- ]?\s*(?:refill|guarantee|guaranteed|warranty)\b", re.I) REFILL_ANY = re.compile(r"refill|guarantee|guaranteed|gaurantee|warranty", re.I) # ---------------- verbatim from compare.py ---------------- def tidy_c(v): return " ".join((v or "").replace("|", " ").split()) def key(r): if r["panel"] in NO_IDS: return tidy_c(r["name"]).lower() parts = tidy_c(r["sid"]).split() return parts[0] if parts else "" def index(rows, wave, dupes): out = collections.defaultdict(dict) for r in rows: k = key(r) if not k: dupes[(wave, r["panel"], "no-key")] += 1 continue if k in out[r["panel"]]: dupes[(wave, r["panel"], "duplicate-id")] += 1 continue out[r["panel"]][k] = r return out # ---------------- classification ---------------- CLASSES = ["period", "none", "both", "word_only", "silent"] def classify(r): # analyse.py tidies name/desc/category/... before building the blob; done on copies here so # compare.py's join reads the rows exactly as it does. name, desc = tidy_a(r["name"]), tidy_a(r.get("desc") or "") blob = name + " || " + (desc or "") per, non, word = bool(REFILL_PERIOD.search(blob)), bool(REFILL_NONE.search(blob)), bool(REFILL_ANY.search(blob)) r["_per"], r["_non"], r["_word"] = per, non, word r["_cls"] = "both" if per and non else "period" if per else "none" if non else "word_only" if word else "silent" tr = dict(r) for k in ("name", "rate_raw", "min", "max", "avg", "category", "desc", "quality"): if k in tr: tr[k] = tidy_a(tr[k]) r["_elig"] = eligible(tr) r["_cat"] = tr.get("category") or "" r["_hay"] = fold((tr["name"] or "") + " " + (tr.get("category") or "")) r["_hay_name"], r["_hay_cat"] = fold(tr["name"] or ""), fold(tr.get("category") or "") def load(fn): return json.load(gzip.open(os.path.join(SRC, fn), "rt", encoding="utf-8")) caps = {} for w, fn, _ in CAPS: rows = load(fn) for r in rows: classify(r) caps[w] = rows # ---------------- 0. sanity: reproduce published numbers ---------------- def corpus_counts(rows): u = [r for r in rows if r["panel"] in USD] return dict(usd_rows=len(u), period_flag=sum(r["_per"] for r in u), none_flag=sum(r["_non"] for r in u), silent=sum(not r["_word"] for r in u), word_only=sum(1 for r in u if r["_word"] and not r["_per"] and not r["_non"]), overlap_both=sum(1 for r in u if r["_per"] and r["_non"])) sanity = {w: corpus_counts(caps[w]) for w, _, _ in CAPS} EXPECT_W1 = dict(usd_rows=43812, period_flag=9662, none_flag=7932, silent=24062, word_only=2204, overlap_both=48) ok = all(sanity["w1"][k] == v for k, v in EXPECT_W1.items()) for k in ("period_flag", "none_flag", "silent", "word_only"): sanity["w1"][k + "_pct"] = pct(sanity["w1"][k], sanity["w1"]["usd_rows"]) dupes = collections.Counter() idx = {w: index(caps[w], i, dupes) for i, (w, _, _) in enumerate(CAPS)} followed = [] # (panel, key, r0, r1, r2) for p in PANELS: if p in NOT_USD or p in NO_IDS: continue x0, x1, x2 = idx["w1"][p], idx["w2"][p], idx["w3"][p] for k in sorted(x0.keys() & x1.keys() & x2.keys()): a, b, c = x0[k], x1[k], x2[k] ra, rb, rc = a["rate"], b["rate"], c["rate"] if not ra or not rb or not rc or ra <= 0 or rb <= 0 or rc <= 0: continue followed.append((p, k, a, b, c)) tc = json.load(open(os.path.join(SRC, "changes-w3.json"), encoding="utf-8"))["three_captures"] c1n = sum(1 for p, k, a, b, c in followed if abs(b["rate"] - a["rate"]) > 1e-12) c2n = sum(1 for p, k, a, b, c in followed if abs(c["rate"] - b["rate"]) > 1e-12) bothn = sum(1 for p, k, a, b, c in followed if abs(b["rate"] - a["rate"]) > 1e-12 and abs(c["rate"] - b["rate"]) > 1e-12) ok3 = (len(followed) == tc["listings_on_all_three"] == 40870 and c1n == tc["changed_in_first_interval_only"] + tc["changed_in_both"] and c2n == tc["changed_in_second_interval_only"] + tc["changed_in_both"] and bothn == tc["changed_in_both"]) sanity["three_captures"] = dict(listings_on_all_three=len(followed), changed_first=c1n, changed_second=c2n, changed_both=bothn, matches_changes_w3_json=ok3, duplicate_or_missing_keys={f"{w}:{DOMAIN[p]}:{kind}": n for (w, p, kind), n in sorted(dupes.items())}) sanity["w1_matches_published"] = ok result["sanity"] = sanity result["definitions"] = dict( classes={"period": "REFILL_PERIOD matches and REFILL_NONE does not", "none": "REFILL_NONE matches and REFILL_PERIOD does not", "both": "both match (overlap rows; analyse.py counts them in both lines; excluded from period-vs-none comparisons)", "word_only": "REFILL_ANY matches, neither of the above", "silent": "REFILL_ANY does not match"}, blob="analyse.py: tidy(name) + ' || ' + tidy(desc); smmlaunch, adderpanel, crescitaly, smmcost, smmkings print no desc in HTML, so only the name counts there", join="compare.py: panel + first token of sid; first row kept on duplicate id; rows without key dropped; followed = on all three captures with rate > 0 each time; 13 USD panels with ids", ) save() if not (ok and ok3): print(json.dumps(sanity, indent=1)); sys.exit("SANITY FAILED - published numbers do not reproduce; stopping") print("sanity ok:", sanity["w1"], sanity["three_captures"]["listings_on_all_three"]) # ---------------- 1. classes per capture ---------------- def class_table(rows): out = {} for p in USD: pr = [r for r in rows if r["panel"] == p] c = collections.Counter(r["_cls"] for r in pr) out[label(p)] = dict(rows=len(pr), **{k: c[k] for k in CLASSES}, desc_hidden=p in HIDDEN_DESC) u = [r for r in rows if r["panel"] in USD] c = collections.Counter(r["_cls"] for r in u) tot = dict(rows=len(u), **{k: c[k] for k in CLASSES}) tot.update({k + "_pct": pct(c[k], len(u)) for k in CLASSES}) return dict(usd_corpus=tot, per_panel=out) result["classes"] = {w: class_table(caps[w]) for w, _, _ in CAPS} save() # ---------------- 2. price premium ---------------- PLATFORMS = [("telegram", r"telegram|\btg\b"), ("instagram", r"instagram|\big\b|\binsta\b"), ("youtube", r"youtube|\byt\b"), ("tiktok", r"tiktok|tik tok"), ("facebook", r"facebook|\bfb\b"), ("twitter", r"twitter|tweet"), ("spotify", r"spotify"), ("threads", r"\bthreads\b"), ("linkedin", r"linkedin"), ("discord", r"discord"), ("twitch", r"twitch"), ("soundcloud", r"soundcloud"), ("pinterest", r"pinterest"), ("reddit", r"reddit"), ("snapchat", r"snapchat"), ("kick", r"\bkick\b"), ("whatsapp", r"whatsapp"), ("website", r"website|traffic|google")] # watchtime: a bare "hours" was dropped on review (2026-10-08). In names it is mostly a speed # ("10K/Hours") or a start time ("0-6 Hours"), which filed likes and comments as watch time. TYPES = [("watchtime", r"watch ?time|watch ?hours?"), ("live", r"\blive\b|\bstream"), ("comments", r"\bcomments?\b"), ("reactions", r"\breactions?\b|\breact\b"), ("votes", r"\bvotes?\b|\bpoll\b"), ("shares", r"\bshares?\b|retweet|repost"), ("saves", r"\bsaves?\b"), ("impressions", r"impressions?|\breach\b"), ("likes", r"\blikes?\b|\bhearts?\b"), ("views", r"\bviews?\b|\bplays?\b"), ("audience", r"\bfollowers?\b|\bmembers?\b|\bsubscribers?\b|\bsubs\b")] def first_match(h, table): for name, rx in table: if re.search(rx, h): return name return "other" def ptype(r): # Name first, category heading only as a fallback (fixed on review, 2026-10-08). Reading the # two as one string let a heading such as "TikTok Brazil Services [Followers & Likes]" file # every follower listing under it as likes, because "likes" comes first in TYPES. def pick(table): v = first_match(r["_hay_name"], table) return v if v != "other" else first_match(r["_hay_cat"], table) return pick(PLATFORMS) + "/" + pick(TYPES) def premium(rows, fine, require_eligible=True, panels=COMPARABLE): groups = collections.defaultdict(lambda: {"period": [], "none": []}) for r in rows: if r["panel"] not in panels or r["_cls"] not in ("period", "none"): continue if require_eligible: if not r["_elig"]: continue elif not r["rate"] or r["rate"] <= 0: continue g = (r["panel"], r["_cat"]) + ((ptype(r),) if fine else ()) groups[g][r["_cls"]].append(r["rate"]) per_group = [] for g in sorted(groups): v = groups[g] if v["period"] and v["none"]: per_group.append((g[0], statistics.median(v["period"]) / statistics.median(v["none"]), len(v["period"]), len(v["none"]))) def summ(sel): if not sel: return dict(groups=0) rs = [x[1] for x in sel] return dict(groups=len(sel), listings_period=sum(x[2] for x in sel), listings_none=sum(x[3] for x in sel), median_ratio=r4(statistics.median(rs)), groups_period_dearer=sum(1 for x in rs if x > 1 + 1e-12), groups_period_cheaper=sum(1 for x in rs if x < 1 - 1e-12), groups_equal=sum(1 for x in rs if abs(x - 1) <= 1e-12), p25=r4(statistics.quantiles(rs, n=4)[0]) if len(rs) >= 2 else None, p75=r4(statistics.quantiles(rs, n=4)[2]) if len(rs) >= 2 else None) out = dict(pooled=summ(per_group), pooled_without_1xpanel=summ([x for x in per_group if x[0] != BIG]), pooled_without_publisher_panels=summ([x for x in per_group if x[0] not in PUBLISHER]), pooled_desc_visible_panels_only=summ([x for x in per_group if x[0] not in HIDDEN_DESC]), per_panel={label(p): summ([x for x in per_group if x[0] == p]) for p in panels}) # eligible period/none listings available at all, for the denominator out["period_none_listings_considered"] = dict( period=sum(len(v["period"]) for v in groups.values()), none=sum(len(v["none"]) for v in groups.values())) return out result["premium"] = dict( method=("Group = panel + category string (analyse.py tidy, exact match). 'fine' adds platform/type parsed " "from fold(name), falling back to fold(category) when the name matches nothing: first matching platform " "keyword and first matching service-type keyword in the fixed orders PLATFORMS and TYPES in refill.py; " "unmatched -> 'other'. Rows: analyse.py " "eligible() (rate>0, published max>=1000, no dead category, no instruction/test names), 13 USD panels " "with ids. Per group: median(rate period)/median(rate none); group kept if >=1 of each. Pooled = " "statistics.median of group ratios."), w1=dict(category=premium(caps["w1"], False), category_and_type=premium(caps["w1"], True), category_and_type_all_positive_rates=premium(caps["w1"], True, require_eligible=False)), w3=dict(category=premium(caps["w3"], False), category_and_type=premium(caps["w3"], True)), ) save() print("premium w1:", result["premium"]["w1"]["category_and_type"]["pooled"]) # ---------------- 3. stability ---------------- def chg(x, y): return abs(y["rate"] - x["rate"]) > 1e-12 def stab(sel): n = len(sel) f = sum(1 for p, k, a, b, c in sel if chg(a, b)); s = sum(1 for p, k, a, b, c in sel if chg(b, c)) e = sum(1 for p, k, a, b, c in sel if chg(a, b) or chg(b, c)) return dict(listings=n, changed_first=f, changed_second=s, changed_either=e, changed_first_pct=pct(f, n), changed_second_pct=pct(s, n), changed_either_pct=pct(e, n)) def by_class(sel): return {cl: stab([t for t in sel if t[2]["_cls"] == cl]) for cl in CLASSES} stability = dict(class_from="w1 row kept by compare.py's index", all=by_class(followed), without_1xpanel=by_class([t for t in followed if t[0] != BIG]), only_1xpanel=by_class([t for t in followed if t[0] == BIG]), without_publisher_panels=by_class([t for t in followed if t[0] not in PUBLISHER]), desc_visible_panels_only=by_class([t for t in followed if t[0] not in HIDDEN_DESC]), per_panel={label(p): by_class([t for t in followed if t[0] == p]) for p in COMPARABLE}) # within panel+category strata that contain both period and none listings: compares like with like strata = collections.defaultdict(list) for t in followed: if t[2]["_cls"] in ("period", "none"): strata[(t[0], tidy_a(t[2].get("category") or ""))].append(t) mixed = [s for s in strata.values() if {t[2]["_cls"] for t in s} == {"period", "none"}] flat = [t for s in mixed for t in s] stability["within_mixed_panel_category_strata"] = dict( strata=len(mixed), period=stab([t for t in flat if t[2]["_cls"] == "period"]), none=stab([t for t in flat if t[2]["_cls"] == "none"]), without_1xpanel=dict(strata=sum(1 for s in mixed if s[0][0] != BIG), period=stab([t for t in flat if t[2]["_cls"] == "period" and t[0] != BIG]), none=stab([t for t in flat if t[2]["_cls"] == "none" and t[0] != BIG]))) # panels where period listings changed less often (either interval) than none listings cmp_ = {} for p in COMPARABLE: pp = stability["per_panel"][label(p)] if pp["period"]["listings"] and pp["none"]["listings"]: cmp_[label(p)] = dict(period_either_pct=pp["period"]["changed_either_pct"], none_either_pct=pp["none"]["changed_either_pct"], period_n=pp["period"]["listings"], none_n=pp["none"]["listings"]) stability["panels_with_both_classes"] = cmp_ stability["panels_period_more_stable"] = sum(1 for v in cmp_.values() if v["period_either_pct"] < v["none_either_pct"]) stability["panels_period_less_stable"] = sum(1 for v in cmp_.values() if v["period_either_pct"] > v["none_either_pct"]) result["stability"] = stability save() print("stability all:", {k: v["changed_either_pct"] for k, v in stability["all"].items()}) # ---------------- 4. wording drift ---------------- def drift(i, j, label_): trans = collections.Counter(); price_if = collections.Counter(); per_panel = collections.defaultdict(collections.Counter) for t in followed: a, b = t[2 + i], t[2 + j] moved = a["_cls"] != b["_cls"] pc = chg(a, b) price_if[("class_changed" if moved else "class_same", pc)] += 1 if moved: trans[a["_cls"] + "->" + b["_cls"]] += 1 per_panel[label(t[0])]["class_changed"] += 1 per_panel[label(t[0])]["and_repriced"] += pc nm = tidy_c(a["name"]) != tidy_c(b["name"]) per_panel[label(t[0])]["name_changed"] += nm cc = price_if[("class_changed", True)] + price_if[("class_changed", False)] cs = price_if[("class_same", True)] + price_if[("class_same", False)] return dict(interval=label_, class_changed=cc, class_changed_and_repriced=price_if[("class_changed", True)], repriced_pct_if_class_changed=pct(price_if[("class_changed", True)], cc), class_same=cs, repriced_pct_if_class_same=pct(price_if[("class_same", True)], cs), transitions=dict(sorted(trans.items(), key=lambda x: (-x[1], x[0]))), per_panel={k: dict(v) for k, v in sorted(per_panel.items())}) d1, d2 = drift(0, 1, "2 Sep -> 13 Sep"), drift(1, 2, "13 Sep -> 2 Oct") ever = [t for t in followed if len({t[2]["_cls"], t[3]["_cls"], t[4]["_cls"]}) > 1] lost_period = [t for t in followed if t[2]["_cls"] == "period" and t[4]["_cls"] != "period"] gained_period = [t for t in followed if t[2]["_cls"] != "period" and t[4]["_cls"] == "period"] def ex(ts, n=6): return [dict(panel=label(t[0]), id=t[1], classes=[t[2]["_cls"], t[3]["_cls"], t[4]["_cls"]], names=[tidy_c(t[2]["name"]), tidy_c(t[3]["name"]), tidy_c(t[4]["name"])], rates=[t[2]["rate"], t[3]["rate"], t[4]["rate"]]) for t in ts[:n]] result["wording_drift"] = dict( first_interval=d1, second_interval=d2, changed_class_at_least_once=len(ever), changed_class_at_least_once_and_any_price_change=sum(1 for t in ever if chg(t[2], t[3]) or chg(t[3], t[4])), period_on_2sep_not_period_on_2oct=len(lost_period), period_lost_and_any_price_change=sum(1 for t in lost_period if chg(t[2], t[3]) or chg(t[3], t[4])), period_lost_by_panel=dict(collections.Counter(label(t[0]) for t in lost_period)), not_period_on_2sep_period_on_2oct=len(gained_period), period_gained_and_any_price_change=sum(1 for t in gained_period if chg(t[2], t[3]) or chg(t[3], t[4])), period_gained_by_panel=dict(collections.Counter(label(t[0]) for t in gained_period)), examples_period_lost=ex(lost_period), examples_period_gained=ex(gained_period), all_class_changes=ex(ever, n=len(ever)), ) save() print("drift:", d1["class_changed"], d2["class_changed"], len(ever)) print("done ->", STATS) # ---------------- 5. sensitivity: wording the published patterns miss ---------------- # Added 2026-10-08 on review. The analyse.py patterns above are the definition of record, so # sections 1-4 use them unchanged. Reading the rows they leave out shows three spellings they # cannot see: a period written with the recycling symbol and no word ("30 Days ♻️", # "Lifetime ♻️", mostly smmcost.com and adderpanel.com), the shorthand "R30"/"R365", and the # negative written as a field ("Refill: No"). The bare ♻️ is NOT taken as a period: 1xpanel.com # prints it after "Stable price" on 1,370 rows where it means nothing about refill. A period # stated in hours ("24h Guarantee") is a Twitter trend-topic duration on the rows checked, not a # refill; it is counted here, not removed. PERIOD_EXT = re.compile(r"\b\d+\s*[- ]?\s*(?:day|days|d)\s*♻|\b(?:lifetime|life\s*time)\s*♻" r"|(?" + r["_cls"] for r in caps[w] if r["panel"] in USD and r["_cls_pub"] != r["_cls"]) moved_panel = collections.Counter(label(r["panel"]) for r in caps[w] if r["panel"] in USD and r["_cls_pub"] != r["_cls"]) sens[w] = dict(reclassified=dict(moved.most_common()), reclassified_by_panel=dict(moved_panel.most_common()), category_and_type=premium(caps[w], True)) for r in caps[w]: r["_cls"] = r["_cls_pub"] sens["w1_1xpanel_rows_with_stable_price_recycle_symbol"] = sum( 1 for r in caps["w1"] if r["panel"] == BIG and re.search(r"stable price\s*\u267b", tidy_a(r["name"]) + " || " + tidy_a(r.get("desc") or ""), re.I)) result["sensitivity_extended_patterns"] = sens save() print("sensitivity w1:", sens["w1"]["category_and_type"]["pooled"]) print("sensitivity w3:", sens["w3"]["category_and_type"]["pooled"]) # ---------------- 6. how the promise is worded (2 Sep, USD panels, published patterns) ---------------- def term_of(m): t = m.lower() if re.search(r"life\s*time|lifetime|permanent", t): return "lifetime" n = re.search(r"\d+", t) unit = re.search(r"month|year|hour|\d\s*h\b|day|\d\s*d\b", t) u = unit.group(0) if unit else "day" if "month" in u: return f"{n.group(0)} month" if "year" in u: return f"{n.group(0)} year" if "hour" in u or u.endswith("h"): return f"{n.group(0)} hour" return f"{n.group(0)} day" usd1 = [r for r in caps["w1"] if r["panel"] in USD] ph_p, ph_n, terms = collections.Counter(), collections.Counter(), collections.Counter() for r in usd1: blob = tidy_a(r["name"]) + " || " + tidy_a(r.get("desc") or "") if r["_cls"] == "period": m = REFILL_PERIOD.search(blob).group(0) ph_p[re.sub(r"\s+", " ", m.lower())] += 1 terms[term_of(m)] += 1 elif r["_cls"] == "none": ph_n[re.sub(r"\s+", " ", REFILL_NONE.search(blob).group(0).lower())] += 1 result["wording"] = dict( note="2 Sep, 14 USD panels, rows classed 'period' or 'none' by analyse.py's patterns; first match in name || desc", period_listings=sum(terms.values()), period_phrasings_top=dict(ph_p.most_common(15)), none_listings=sum(ph_n.values()), none_phrasings=dict(ph_n.most_common()), stated_terms_top=dict(terms.most_common(12))) # ---------------- 7. one worked group per panel (2 Sep, category + type) ---------------- groups = collections.defaultdict(lambda: {"period": [], "none": []}) for r in caps["w1"]: if r["panel"] in COMPARABLE and r["_cls"] in ("period", "none") and r["_elig"]: groups[(r["panel"], r["_cat"], ptype(r))][r["_cls"]].append(r) ex_out = {} for p in COMPARABLE: gs = [(g, v) for g, v in sorted(groups.items()) if g[0] == p and v["period"] and v["none"]] if not gs: continue med = result["premium"]["w1"]["category_and_type"]["per_panel"][label(p)]["median_ratio"] def ratio(v): return statistics.median(x["rate"] for x in v["period"]) / statistics.median(x["rate"] for x in v["none"]) g, v = min(gs, key=lambda gv: (abs(ratio(gv[1]) - med), -len(gv[1]["period"]) - len(gv[1]["none"]), gv[0])) ex_out[label(p)] = dict(category=g[1], type=g[2], ratio=r4(ratio(v)), panel_median=med, period=[dict(id=tidy_c(x["sid"]).split()[0] if tidy_c(x["sid"]) else "", name=tidy_c(x["name"]), rate=x["rate"]) for x in v["period"]], none=[dict(id=tidy_c(x["sid"]).split()[0] if tidy_c(x["sid"]) else "", name=tidy_c(x["name"]), rate=x["rate"]) for x in v["none"]]) result["worked_groups_w1"] = ex_out save() print("wording:", result["wording"]["stated_terms_top"])