[{'Text': 'Edited /Users/peterdonaghey/Projects/point-topic-mcp/_explore_gbs.py:\n\n```diff\n@@ -1,167 +1,220 @@\n-"""Explore GBS with aggregation pipelines — single efficient run."""\n-import json, os, subprocess, sys\n+"""Final tool design — inline test of minimal GBS tools."""\n+import json, os, subprocess\n from collections import defaultdict\n-from datetime import datetime\n \n URI = os.environ["PT_RESEARCH_DATABASE_URI"]\n-CACHE = {}  # namespace -> result\n \n-def agg(pipeline, coll="Operator", timeout=30):\n-    """Run an aggregation pipeline, return parsed JSON."""\n-    js = f"EJSON.stringify(db.{coll}.aggregate({json.dumps(pipeline)}).toArray())"\n-    uri = URI\n+def _uri():\n+    u = URI\n     if os.environ.get("PT_RESEARCH_TLS_SKIP_VERIFY", "").strip() in ("1", "true", "yes"):\n-        sep = "&" if "?" in uri else "?"\n-        uri = f"{uri}{sep}tlsAllowInvalidCertificates=true"\n-    r = subprocess.run(["mongosh", uri, "--eval", js, "--quiet"],\n+        sep = "&" if "?" in u else "?"\n+        u = f"{u}{sep}tlsAllowInvalidCertificates=true"\n+    return u\n+\n+def run(js, timeout=30):\n+    r = subprocess.run(["mongosh", _uri(), "--eval", js, "--quiet"],\n                        capture_output=True, text=True, timeout=timeout)\n-    if r.returncode != 0:\n-        raise RuntimeError(r.stderr or r.stdout)\n+    if r.returncode != 0: raise RuntimeError(r.stderr or r.stdout)\n     out = r.stdout.strip()\n     return json.loads(out) if out else []\n \n-def find_one(coll, query, timeout=15):\n-    js = f"EJSON.stringify(db.{coll}.findOne({json.dumps(query)}))"\n-    uri = URI\n-    if os.environ.get("PT_RESEARCH_TLS_SKIP_VERIFY", "").strip() in ("1", "true", "yes"):\n-        sep = "&" if "?" in uri else "?"\n-        uri = f"{uri}{sep}tlsAllowInvalidCertificates=true"\n-    r = subprocess.run(["mongosh", uri, "--eval", js, "--quiet"],\n-                       capture_output=True, text=True, timeout=timeout)\n-    if r.returncode != 0:\n-        raise RuntimeError(r.stderr or r.stdout)\n-    out = r.stdout.strip()\n-    return json.loads(out) if out else None\n+def find(js, timeout=30):\n+    """Run JS that returns a single EJSON value."""\n+    return run(js, timeout)\n \n-def uid(doc):\n-    """Extract ObjectId string from a doc."""\n-    v = doc.get("_id", {})\n-    if isinstance(v, dict): return v.get("$oid", "")\n-    return str(v)\n+# ─── PROTOTYPE TOOLS ─────────────────────────────────────────────\n \n-# ── 1. Global State ──────────────────────────────────────────────\n-print("── 1. GLOBALS ──")\n-gv = find_one("GlobalVariables", {"isCurrent": True})\n-current = gv.get("currentStatsPeriod", "")\n-print(f"Current stats period: {current}")\n-print(f"Current tariffs period: {gv.get(\'currentTariffsPeriod\', \'\')}")\n-print(f"Updated by: {gv.get(\'updatedBy\', \'\')} @ {gv.get(\'updatedAt\', {}).get(\'$date\', \'\')}")\n+def get_current_period():\n+    """Get the current stats period from GlobalVariables."""\n+    docs = run("EJSON.stringify(db.GlobalVariables.find({isCurrent: true}).toArray())")\n+    if not docs:\n+        return None\n+    gv = docs[0]\n+    period = gv.get("currentStatsPeriod")\n+    tariffs_period = gv.get("currentTariffsPeriod")\n+    updated_by = gv.get("updatedBy", "")\n+    gv_id = gv.get("_id", {}).get("$oid", "")\n+    return {\n+        "stats_period": period,\n+        "tariffs_period": tariffs_period,\n+        "updated_by": updated_by,\n+        "gv_id": gv_id,\n+        "full": gv,\n+    }\n \n-# ── 2. All statistics periods — one aggregation ──────────────────\n-print("\\n── 2. PERIOD COVERAGE (ALL STATS) ──")\n-period_data = agg([\n-    {"$group": {"_id": "$period", "count": {"$sum": 1}, "operators": {"$addToSet": "$operator"}}}\n-], "Statistics")\n-periods = sorted(period_data, key=lambda x: x["_id"])\n-for p in periods:\n-    label = " ← CURRENT" if p["_id"] == current else (" ← FUTURE?" if p["_id"] > current else "")\n-    print(f"  {p[\'_id\']}: {p[\'count\']:>5} records, {len(p[\'operators\']):>4} operators{label}")\n+def get_operator_status(country, operator_name=None, period=None):\n+    """\n+    Hierarchical GBS status.\n+    \n+    - No period → uses current\n+    - No operator → all operators in country\n+    - Both → specific operator\n+    """\n+    current = get_current_period()\n+    period = period or current["stats_period"]\n+    \n+    # Build aggregation pipeline\n+    match = {"country": country}\n+    if operator_name:\n+        match["name"] = operator_name\n+    match["isArchived"] = {"$ne": True}\n+    \n+    pipeline = [\n+        {"$match": match},\n+        {"$lookup": {\n+            "from": "Statistics",\n+            "let": {"oid": "$_id"},\n+            "pipeline": [\n+                {"$match": {"$expr": {\n+                    "$and": [\n+                        {"$eq": ["$operatorId", "$$oid"]},\n+                        {"$eq": ["$period", period]}\n+                    ]\n+                }}}\n+            ],\n+            "as": "stats"\n+        }},\n+        {"$addFields": {"statCount": {"$size": "$stats"}}},\n+        {"$project": {"name": 1, "technologies": 1, "statCount": 1, "stats": 1}},\n+        {"$sort": {"name": 1}},\n+    ]\n+    \n+    operators = run(f"EJSON.stringify(db.Operator.aggregate({json.dumps(pipeline)}).toArray())", timeout=60)\n+    \n+    result = {\n+        "current_period": period,\n+        "country": country,\n+        "total_operators": len(operators),\n+    }\n+    \n+    if operator_name:\n+        if not operators:\n+            return {**result, "error": f"Operator \'{operator_name}\' not found in {country}"}\n+        op = operators[0]\n+        result["operator"] = _analyze_operator(op, current["stats_period"])\n+        return result\n+    \n+    # Country roll-up\n+    with_data = [o for o in operators if o["statCount"] > 0]\n+    without = [o for o in operators if o["statCount"] == 0]\n+    result["with_data"] = len(with_data)\n+    result["without_data"] = len(without)\n+    result["coverage_pct"] = round(100 * len(with_data) / max(len(operators), 1), 1)\n+    result["operators_with_data"] = [{\n+        "name": o["name"], "records": o["statCount"],\n+        "techs": len(o.get("technologies", []))\n+    } for o in sorted(with_data, key=lambda x: -x["statCount"])[:50]]\n+    result["operators_without_data"] = [o["name"] for o in without[:30]]\n+    return result\n \n-# ── 3. UK overview — one aggregation with $lookup ────────────────\n-print(f"\\n── 3. UK OPERATOR COVERAGE ({current}) ──")\n-uk_overview = agg([\n-    {"$match": {"country": "United Kingdom", "isArchived": {"$ne": True}}},\n-    {"$lookup": {\n-        "from": "Statistics",\n-        "let": {"oid": "$_id"},\n-        "pipeline": [\n-            {"$match": {"$expr": {\n-                "$and": [\n-                    {"$eq": ["$operatorId", "$$oid"]},\n-                    {"$eq": ["$period", current]}\n-                ]\n-            }}}\n-        ],\n-        "as": "stats"\n-    }},\n-    {"$addFields": {\n-        "statCount": {"$size": "$stats"}\n-    }},\n-    {"$project": {\n-        "name": 1, "technologies": 1, "statCount": 1, "stats": 1\n-    }},\n-    {"$sort": {"name": 1}}\n-])\n-\n-total_uk = len(uk_overview)\n-with_data = [o for o in uk_overview if o["statCount"] > 0]\n-without_data = [o for o in uk_overview if o["statCount"] == 0]\n-print(f"Total UK operators: {total_uk}")\n-print(f"With data: {len(with_data)}, Without data: {len(without_data)}")\n-\n-# Coverage rate\n-print(f"\\nCoverage: {len(with_data)}/{total_uk} ({100*len(with_data)/total_uk:.1f}%)")\n-\n-# Top operators by record count\n-print(f"\\nTop operators by stat count in {current}:")\n-for op in sorted(with_data, key=lambda x: -x["statCount"])[:10]:\n-    print(f"  {op[\'name\'][:30]:30s} {op[\'statCount\']:>3} records  techs: {op.get(\'technologies\', [])}")\n-\n-# ── 4. GAP ANALYSIS: what combos are missing for UK operators ────\n-print(f"\\n── 4. GAP ANALYSIS (UK, {current}) ──")\n-# For a sample of operators (top 3 with data, bottom 3 without)\n-VALID_TYPES = ["broadband", "mobile", "iptv"]\n-VALID_CHANNELS = ["Infrastructure", "Retail"]\n-VALID_DOMAINS = ["Residential", "Business", "Total"]\n-\n-def show_gaps(op):\n+def _analyze_operator(op, current_period):\n+    """Analyze a single operator: what exists, gaps, state distribution."""\n     name = op["name"]\n     techs = op.get("technologies", [])\n-    stats = op["stats"]\n-    # Build set of what exists\n-    exist = set()\n-    for s in stats:\n-        exist.add((s.get("type"), s.get("tech"), s.get("channel"), s.get("domain"), s.get("state")))\n+    stats = op.get("stats", [])\n     \n-    # Group by type\n-    gap_count = 0\n-    gap_detail = defaultdict(list)\n-    filled = 0\n+    VALID_TYPES = ["broadband", "mobile", "iptv"]\n+    VALID_CHANNELS = ["Infrastructure", "Retail"]\n+    VALID_DOMAINS = ["Residential", "Business", "Total"]\n+    \n+    # What exists\n+    by_state = defaultdict(int)\n+    by_type = defaultdict(int)\n+    by_tech = defaultdict(int)\n+    combos = set()\n+    \n+    for s in stats:\n+        by_state[s.get("state")] += 1\n+        by_type[s.get("type", "?")] += 1\n+        by_tech[s.get("tech", "?")] += 1\n+        combos.add((s.get("type"), s.get("tech"), s.get("channel"), s.get("domain")))\n+    \n+    state_names = {1: "pending", 2: "approved", 3: "published"}\n+    \n+    # Fill count: how many {type, tech, channel, domain} combos are populated\n+    filled = len(combos)\n+    \n+    # Expected combos: only for types that exist in the operator\'s data or all types if no data\n+    techs_in_use = sorted(set(s.get("tech") for s in stats)) or techs[:3]\n+    expected = len(VALID_TYPES) * len(techs_in_use) * len(VALID_CHANNELS) * len(VALID_DOMAINS)\n+    \n+    # Gaps by type/tech\n+    gaps_by_type = defaultdict(list)\n     for t in VALID_TYPES:\n-        for tech in techs:\n+        for tech in techs_in_use:\n             for ch in VALID_CHANNELS:\n                 for dom in VALID_DOMAINS:\n-                    combo = (t, tech, ch, dom)\n-                    found = [s for s in stats if s.get("type")==t and s.get("tech")==tech \n-                             and s.get("channel")==ch and s.get("domain")==dom]\n-                    if found:\n-                        filled += 1\n-                    else:\n-                        gap_count += 1\n-                        gap_detail[t].append(f"{tech}/{ch}/{dom}")\n+                    if (t, tech, ch, dom) not in combos:\n+                        gaps_by_type[t].append(f"{tech}/{ch}/{dom}")\n     \n-    state_dist = defaultdict(int)\n-    for s in stats:\n-        state_dist[s.get("state")] += 1\n-    state_names = {1: "pending", 2: "approved", 3: "published"}\n-    state_str = ", ".join(f"{state_names.get(k,k)}: {v}" for k,v in sorted(state_dist.items())) if state_dist else "none"\n-    \n-    print(f"\\n  {name} ({len(stats)} records, {filled} filled/{gap_count} gaps)")\n-    print(f"    States: {state_str}")\n-    for t, glist in gap_detail.items():\n-        if glist:\n-            print(f"    {t} gaps ({len(glist)}): {\', \'.join(glist[:5])}{\'...\' if len(glist)>5 else \'\'}")\n+    return {\n+        "name": name,\n+        "operator_id": op.get("_id", {}).get("$oid", ""),\n+        "technologies": techs,\n+        "total_records": len(stats),\n+        "filled_combos": filled,\n+        "gap_combos": expected - filled,\n+        "states": {state_names.get(k, str(k)): v for k, v in sorted(by_state.items())},\n+        "by_type": dict(by_type),\n+        "by_tech": dict(by_tech),\n+        "gaps": {t: glist for t, glist in gaps_by_type.items() if glist},\n+    }\n \n-print("Operators WITH data (sample top 3):")\n-for op in with_data[:3]:\n-    show_gaps(op)\n \n-print("\\nOperators WITHOUT data (sample 3):")\n-for op in without_data[:3]:\n-    show_gaps(op)\n+# ─── INLINE TESTS ────────────────────────────────────────────────\n \n-# ── 5. WHAT DOES A COMPLETE OPERATOR LOOK LIKE? ──────────────────\n-print(f"\\n── 5. MOST COMPLETE UK OPERATOR IN {current} ──")\n-most_complete = max(with_data, key=lambda x: x["statCount"])\n-show_gaps(most_complete)\n+print("═══ TEST 1: get_current_period() ═══")\n+cp = get_current_period()\n+print(f"Stats period: {cp[\'stats_period\']}")\n+print(f"Tariffs period: {cp[\'tariffs_period\']}")\n+print(f"Updated by: {cp[\'updated_by\']}")\n \n-# ── 6. ADMIN LINK ────────────────────────────────────────────────\n-print(f"\\n── 6. ADMIN CONTEXT ──")\n-admin_url = "https://research-app.pt-research.net/admin"\n-print(f"Admin URL: {admin_url}")\n-# MongoDB IDs for quick reference\n-print(f"\\nGlobalVariables ID: {uid(gv)}")\n-print(f"Current period: {current}")\n-print(f"Previous complete period: 2025Q2 (unless shown differently above)")\n+print("\\n═══ TEST 2: Country status (UK, no operator) ═══")\n+uk = get_operator_status("United Kingdom")\n+print(f"Current period: {uk[\'current_period\']}")\n+print(f"Total UK operators: {uk[\'total_operators\']}")\n+print(f"With data: {uk[\'with_data\']}, Without: {uk[\'without_data\']} ({uk[\'coverage_pct\']}%)")\n+print(f"\\nTop operators with data:")\n+for op in uk["operators_with_data"][:5]:\n+    print(f"  {op[\'name\'][:30]:30s} {op[\'records\']} records")\n+print(f"\\nOperators without data (first 5): {uk[\'operators_without_data\'][:5]}")\n \n-print("\\n── DONE ──")\n+print("\\n═══ TEST 3: Operator status (BT Group, 2025Q3) ═══")\n+bt = get_operator_status("United Kingdom", "BT Group", "2025Q3")\n+op = bt["operator"]\n+print(f"Operator: {op[\'name\']} ({op[\'operator_id\']})")\n+print(f"Records: {op[\'total_records\']}, Filled combos: {op[\'filled_combos\']}, Gaps: {op[\'gap_combos\']}")\n+print(f"States: {op[\'states\']}")\n+print(f"By type: {op[\'by_type\']}")\n+print(f"By tech: {op[\'by_tech\']}")\n+# Show first few gaps per type\n+for t, glist in op[\'gaps\'].items():\n+    print(f"  {t} gaps (showing first 3): {glist[:3]}")\n+\n+print("\\n═══ TEST 4: Operator status (unknown operator) ═══")\n+unknown = get_operator_status("United Kingdom", "NoSuchOperator")\n+print(f"Error expected: {unknown.get(\'error\', \'none\')}")\n+\n+print("\\n═══ TEST 5: Operator with NO data (first from list) ═══")\n+# Find first operator without data\n+empty = get_operator_status("United Kingdom")\n+first_empty = empty["operators_without_data"][0] if empty["operators_without_data"] else None\n+if first_empty:\n+    print(f"Checking: {first_empty}")\n+    empty_op = get_operator_status("United Kingdom", first_empty)\n+    print(f"Records: {empty_op[\'operator\'][\'total_records\']}, Gaps: {empty_op[\'operator\'][\'gap_combos\']}")\n+    print(f"Technologies: {empty_op[\'operator\'][\'technologies\']}")\n+\n+print("\\n═══ TEST 6: Performance ═══")\n+import time\n+t0 = time.time()\n+_ = get_operator_status("United Kingdom")\n+t1 = time.time()\n+print(f"Country-level query: {t1-t0:.2f}s")\n+\n+t0 = time.time()\n+_ = get_operator_status("United Kingdom", "BT Group", "2025Q3")\n+t1 = time.time()\n+print(f"Operator-level query: {t1-t0:.2f}s")\n+\n+print("\\n═══ DONE ═══")\n\n```'}]