=== BIP_v10_16_9.ipynb ===

# ---
# @title 1. Configuration & Setup { display-mode: "form" }

# @markdown ---
# @markdown ### Version
BIP_VERSION = "10.16.5"  # @param {type:"string"}
# @markdown Central version number - change to update all references
# @markdown ## Data Source Configuration

DATA_MODE = "Update missing"  # @param ["Refresh all", "Update missing", "Cache only"]
# @markdown - **Refresh all**: Re-download everything from source (slow, ~2hrs)
# @markdown - **Update missing**: Use cache, download only what's missin

# ---
# @title 2. Load Corpora (v10.12 - Self-Contained) { display-mode: "form" }
# @markdown Downloads from verified external sources - fully self-contained, no external imports
# @markdown
# @markdown **Sources (9 categories):**
# @markdown - Sanskrit: Itihasa (93K shlokas)
# @markdown - Pali: SuttaCentral API (Full Canon)
# @markdown - Arabic: Tanzil.net (Quran)
# @markdown - Hebrew/Aramaic: Sefaria GitHub

INCLUDE_RESPONSA = False  # @param {type:"boolean"}
# @markdown - **INCLUDE_RESPONSA**: Incl

# ---
# @title 6. Model Architecture { display-mode: "form" }
# @markdown BIP v10.9 model with configurable backbone and adversarial heads
# @markdown - Updated: 8 languages, 26 periods

import json

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from tqdm.auto import tqdm
from transformers import AutoModel, AutoTokenizer

print("=" * 60)
print("MODEL ARCHITECTURE")
print("=" * 60)
print(f"Backbone: {BACKBONE} ({MODEL_NAME})")
print(

# ---
# @title 7. Training Loop { display-mode: "form" }
# @markdown Training with tuned adversarial weights and hardware-optimized parameters
# @markdown v10.16.9: Encoder unfreezing + stronger adversarial training

# ===== SUPPRESS DATALOADER MULTIPROCESSING WARNINGS =====
# These occur during garbage collection and bypass normal exception handling
import io
import logging
import os
import random
import sys
import warnings

# Method 1: Filter warnings
warnings.filterwarnings("ignore", message=".*can

# ---
# @title 8. Geometric Analysis & Linear Probe { display-mode: "form" }
# @markdown v10.9: New geometric analysis module + linear probe test
# @markdown Tests latent space structure (axis discovery, role swap analysis)
# @markdown Tests if z_bond encodes language/period (should be low = invariant)


import numpy as np
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler


# ===== v10.9: GEOMETRIC ANALYZER CLASS 

# ---
# @title 9. Fuzz Testing v10.12: Structural vs Surface Perturbations { display-mode: "form" }
# @markdown Tests whether structural perturbations move embeddings more than surface perturbations.
# @markdown **Run immediately after Cell 6/7 training completes - uses model in memory.**
# @markdown
# @markdown v10.12 enhancements:
# @markdown - **30+ samples per category** for 6-sigma statistical confidence
# @markdown - **Runtime-adaptive thresholds** based on GPU type (L4/A100/T4)
# @markdown - **

