# -*- coding: utf-8 -*-
"""Parse public SMM panel service catalogues into a normalised service list."""
import gzip, re, json, html, os, sys

# Reads the stored captures next to this script and writes the parsed list back
# beside them. Until 13 September 2026 this line named a temporary directory on
# the laptop that parsed the first capture, and the parser wrote `_parsed.json`
# into it - so the published script could not run on the published pages, on a
# survey whose method note says it reproduces offline. Run against `pages/` it
# rebuilds the published `services-all.json.gz` row for row (checked: 44,064 of
# 44,064 identical).
#
#   python3 parse_panels.py        # pages/*.html.gz     -> services-all.json.gz     (2 Sep capture)
#   python3 parse_panels.py w2     # pages/w2/*.html.gz  -> services-all-w2.json.gz  (13 Sep capture)
HERE = os.path.dirname(os.path.abspath(__file__))
WAVE = sys.argv[1] if len(sys.argv) > 1 else ""
if WAVE and not re.fullmatch(r"w[0-9]+", WAVE):
    sys.exit("usage: parse_panels.py [w2|w3|...]")
R = os.path.join(HERE, "pages", WAVE) if WAVE else os.path.join(HERE, "pages")
OUT = os.path.join(HERE, "services-all%s.json.gz" % ("-" + WAVE if WAVE else ""))

COMMENT = re.compile(r"<!--.*?-->", re.S)
def load(n):
    h = gzip.open(os.path.join(R, n + ".html.gz"), "rt", encoding="utf-8", errors="replace").read()
    return COMMENT.sub("", h)   # strip HTML comments (they split prices/names in some templates)

TAG = re.compile(r"<[^>]+>")
def txt(s):
    s = re.sub(r"<br\s*/?>", " \n ", s, flags=re.I)
    s = TAG.sub("", s)
    s = html.unescape(s)
    s = s.replace(" ", " ").replace(" ", " ").replace(" ", " ")
    s = re.sub(r"[ \t]+", " ", s)
    s = re.sub(r"\s*\n\s*", " | ", s)
    return s.strip()

def num(s):
    if s is None: return None
    s = s.replace(" ", "").replace(",", "").replace(" ", "")
    m = re.search(r"\d+(?:\.\d+)?", s)
    return float(m.group()) if m else None

def cur(s):
    for sym in ["$", "₹", "€", "£", "₺", "₽", "R$"]:
        if sym in s: return sym
    return ""

# ---------------- generic <table> parser -----------------
def parse_table(h, panel):
    """Parse every <table> whose header row names a per-1000 rate column."""
    out = []
    for tm in re.finditer(r"<table\b[^>]*>(.*?)</table>", h, re.S | re.I):
        tbl = tm.group(1)
        hm = re.search(r"<thead[^>]*>(.*?)</thead>", tbl, re.S | re.I)
        if not hm: continue
        heads = [txt(c) for c in re.findall(r"<th[^>]*>(.*?)</th>", hm.group(1), re.S | re.I)]
        joined = " ".join(heads).lower()
        if not any(k in joined for k in ("per 1000", "per 1,000", "rate per", "price per", "rate / 1000", "/ 1000")):
            continue
        body = tbl[hm.end():]
        bm = re.search(r"<tbody[^>]*>(.*)", body, re.S | re.I)
        if bm: body = bm.group(1)

        idx = {}
        for i, hd in enumerate(heads):
            l = hd.lower()
            if l.startswith("id") or l == "#": idx["id"] = i
            elif "service" in l: idx["name"] = i
            elif any(k in l for k in ("per 1000", "per 1,000", "rate per", "price per", "rate / 1000", "/ 1000")): idx["rate"] = i
            elif "min" in l and "max" in l: idx["minmax"] = i
            elif "min" in l: idx["min"] = i
            elif "max" in l: idx["max"] = i
            elif "average" in l or "delivery" in l or "time" in l: idx["avg"] = i
            elif "descr" in l: idx["desc"] = i
            elif "quality" in l: idx["quality"] = i
            elif "platform" in l: idx["platform"] = i

        cat = ""
        for rm in re.finditer(r"<tr\b([^>]*)>(.*?)</tr>", body, re.S | re.I):
            inner = rm.group(2)
            cells = re.findall(r"<t[dh]\b([^>]*)>(.*?)</t[dh]>", inner, re.S | re.I)
            if not cells: continue
            if len(cells) == 1 and ("colspan" in cells[0][0].lower()):
                c = txt(cells[0][1])
                if c: cat = c
                continue
            vals = [txt(c[1]) for c in cells]
            if len(vals) < 3: continue
            def g(k):
                i = idx.get(k)
                return vals[i] if i is not None and i < len(vals) else None
            name, rate = g("name"), g("rate")
            if not name or rate is None or num(rate) is None: continue
            mn, mx = g("min"), g("max")
            if mn is None and g("minmax"):
                parts = re.split(r"[-\u2013]", g("minmax"))
                if len(parts) == 2: mn, mx = parts[0], parts[1]
            out.append(dict(panel=panel, sid=(g("id") or "").strip(), category=cat, name=name,
                            rate_raw=rate, rate=num(rate), currency=cur(rate),
                            min=mn or "", max=mx or "", avg=g("avg") or "", desc="",
                            quality=g("quality") or ""))
    return out, heads if out else []

# ---------------- 1xpanel: div cards + inline modal ------------
def parse_1xpanel(h, panel="1xpanel"):
    catmap = dict(re.findall(r'<option value="(\d+)">([^<]*)</option>', h))
    out = []
    blocks = re.split(r'(?=<div class="service-item")', h)
    for b in blocks[1:]:
        n = re.search(r'data-services="([^"]*)"', b)
        p = re.search(r'data-price="([^"]*)"', b)
        i = re.search(r'data-service-id="([^"]*)"', b)
        c = re.search(r'data-category="([^"]*)"', b)
        if not (n and p): continue
        mn = re.search(r'<span class="min">.*?<strong>(.*?)</strong>', b, re.S)
        mx = re.search(r'<span class="max">.*?<strong>(.*?)</strong>', b, re.S)
        av = re.search(r'<span class="avg[^"]*"[^>]*>.*?<strong>(.*?)</strong>', b, re.S)
        d = re.search(r'id="serviceModalText"[^>]*>(.*?)</div>', b, re.S)
        out.append(dict(panel=panel, sid=i.group(1) if i else "",
                        category=catmap.get(c.group(1), "") if c else "",
                        name=html.unescape(n.group(1)).strip(),
                        rate_raw=p.group(1), rate=num(p.group(1)), currency=cur(p.group(1)),
                        min=txt(mn.group(1)) if mn else "", max=txt(mx.group(1)) if mx else "",
                        avg=txt(av.group(1)) if av else "",
                        desc=txt(d.group(1)) if d else "", quality=""))
    return out, ["div-card"]

# ---------------- morethanpanel: div rows + modal --------------
def parse_mtp(h, panel="morethanpanel"):
    out = []
    blocks = re.split(r'(?=<div class="service-item")', h)
    for b in blocks[1:]:
        i = re.search(r'data-service-id="([^"]*)"', b)
        n = re.search(r'class="sp-serv-title"[^>]*>(.*?)</span>', b, re.S)
        p = re.search(r'data-title="Rate per 1000"[^>]*>\s*<span[^>]*>(.*?)</span>', b, re.S)
        mn = re.search(r'<span class="min">(.*?)</span>', b, re.S)
        mx = re.search(r'<span class="max">(.*?)</span>', b, re.S)
        if not (n and p): continue
        # description = modal body text
        d = re.search(r'class="modal-body"[^>]*>(.*?)</div>\s*</div>', b, re.S)
        out.append(dict(panel=panel, sid=i.group(1) if i else "", category="",
                        name=txt(n.group(1)), rate_raw=txt(p.group(1)),
                        rate=num(txt(p.group(1))), currency=cur(txt(p.group(1))),
                        min=txt(mn.group(1)) if mn else "", max=txt(mx.group(1)) if mx else "",
                        avg="", desc=txt(d.group(1))[:1200] if d else "", quality=""))
    return out, ["div-row"]

PARSERS = {"1xpanel": parse_1xpanel, "morethanpanel": parse_mtp}

PANELS = ["smmlaunch","adderpanel","crescitaly","smmcost","smmkings","growfastsmm",
          "1xpanel","morethanpanel","mysmm","smmfolgen","easytopromo","smmpanelone",
          "globalsmm","autosmo","peakerr"]

if __name__ == "__main__":
    allrows = []
    for p in PANELS:
        h = load(p)
        fn = PARSERS.get(p, parse_table)
        rows, heads = (fn(h, p) if p in PARSERS else fn(h, p))
        curs = {}
        for r in rows: curs[r["currency"]] = curs.get(r["currency"], 0) + 1
        print(f"{p:15s} rows={len(rows):6d} headers={heads} currencies={curs}")
        if rows: print("      e.g.", rows[len(rows)//2]["name"][:80], "|", rows[len(rows)//2]["rate_raw"], "|", rows[len(rows)//2]["min"], "-", rows[len(rows)//2]["max"], "| avg:", rows[len(rows)//2]["avg"][:30])
        allrows += rows
    with gzip.open(OUT, "wt", encoding="utf-8") as fh:
        json.dump(allrows, fh, ensure_ascii=False)
    print("TOTAL", len(allrows), "->", os.path.basename(OUT))