# Snakefile
# Modern workflow orchestration with consolidated experiment structure
#
# REPORTING ARCHITECTURE:
#   1. Streamlit Dashboard (interactive) - experiments/streamlit/
#      Launch: sigx dashboard
#      Purpose: Daily exploration, parameter tuning, interactive analysis
#
#   2. Quarto Reports (static, future) - experiments/quarto/
#      Generate: snakemake quarto_reports
#      Purpose: Publication-quality reports, archival, presentations

import sys

configfile: "experiments/conf/config.yaml"

# Resolve the Python interpreter that launched Snakemake so shell blocks use
# the correct conda environment even when spawned as subprocesses.
PYTHON = sys.executable

# Pre-initialize MLflow SQLite database before any parallel rules execute.
# Without this, concurrent Snakemake jobs all try to run Alembic migrations
# simultaneously, causing "table _alembic_tmp_* already exists" race conditions.
# Once the DB schema exists, parallel processes skip migration and only do DML
# (INSERT/UPDATE), which SQLite handles with its built-in busy timeout.
onstart:
    import subprocess, sys
    subprocess.run([
        sys.executable, "-c",
        "import mlflow, os; "
        "os.makedirs('artifacts', exist_ok=True); "
        "mlflow.set_tracking_uri('sqlite:///artifacts/mlruns.db'); "
        "mlflow.set_experiment('_init')"
    ], check=True)

# Default target: benchmark data generation only
# For reporting:
#   - Interactive: sigx dashboard (Streamlit)
#   - Static: snakemake quarto_reports (Quarto, future)
rule all:
    input:
        # Core experiments (new consolidated structure)
        "artifacts/data/baseline_100k.done",
        "artifacts/data/baseline_48k.done",
        "artifacts/data/low_nfft_scaling.done",
        "artifacts/data/full_parameter_grid_100k.done",
        "artifacts/data/full_parameter_grid_48k.done",
        "artifacts/data/ionosphere_specialized.done",

        # Kept experiments (updated with metadata)
        "artifacts/data/ionosphere_streaming.done",
        "artifacts/data/ionosphere_batch_throughput.done",
        "artifacts/data/ionosphere_streaming_latency.done",
        "artifacts/data/accuracy_validation.done",
        "artifacts/data/execution_mode_comparison.done",

        # NEW: Streaming mode gap-filling experiments (Phase 1)
        "artifacts/data/ionosphere_streaming_throughput.done",
        "artifacts/data/ionosphere_streaming_hires.done",
        "artifacts/data/baseline_batch_100k_latency.done",
        "artifacts/data/baseline_streaming_100k_realtime.done",

        # NEW: Phase 2 - BATCH Baseline Experiments
        "artifacts/data/baseline_batch_100k_throughput.done",
        "artifacts/data/baseline_batch_48k_throughput.done",
        "artifacts/data/baseline_batch_48k_latency.done",
        "artifacts/data/baseline_batch_high_nfft_throughput.done",

        # NEW: Phase 3 - STREAMING Baseline Experiments
        "artifacts/data/baseline_streaming_100k_throughput.done",
        "artifacts/data/baseline_streaming_100k_latency.done",
        "artifacts/data/baseline_streaming_48k_latency.done",
        "artifacts/data/baseline_streaming_48k_realtime.done",

        # Static spectrograms
        "artifacts/figures/spectrograms/general_spectrogram.png",
        "artifacts/figures/spectrograms/general_spectrogram.json",
        "artifacts/figures/spectrograms/accuracy_engine.png",
        "artifacts/figures/spectrograms/accuracy_numpy.png",
        "artifacts/figures/spectrograms/accuracy_difference.png",
        "artifacts/figures/spectrograms/accuracy_metrics.json"

# ============================================================================
# Core Experiments (New Consolidated Structure)
# ============================================================================

# Baseline 100kHz - Academic general-purpose benchmarking
rule run_baseline_100k:
    output:
        touch("artifacts/data/baseline_100k.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_100k \
            +benchmark=throughput
        """

# Baseline 48kHz - Ionosphere-specific characterization
rule run_baseline_48k:
    output:
        touch("artifacts/data/baseline_48k.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_48k \
            +benchmark=throughput
        """

# Low-NFFT Scaling - Fills 256-2048 NFFT gap, extreme channel counts
rule run_low_nfft_scaling:
    output:
        touch("artifacts/data/low_nfft_scaling.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=low_nfft_scaling \
            +benchmark=throughput
        """

# Full Parameter Grid 100kHz - Complete parameter space exploration
rule run_full_parameter_grid_100k:
    output:
        touch("artifacts/data/full_parameter_grid_100k.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=full_parameter_grid_100k \
            +benchmark=throughput
        """

# Full Parameter Grid 48kHz - Ionosphere parameter space
rule run_full_parameter_grid_48k:
    output:
        touch("artifacts/data/full_parameter_grid_48k.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=full_parameter_grid_48k \
            +benchmark=throughput
        """

# Ionosphere Specialized - VLF/ULF phenomena (lightning, Schumann, whistlers)
rule run_ionosphere_specialized:
    output:
        touch("artifacts/data/ionosphere_specialized.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=ionosphere_specialized \
            +benchmark=throughput
        """

# ============================================================================
# Kept Experiments (Updated with Metadata)
# ============================================================================

# Run ionosphere streaming sweep (dual-channel real-time performance)
rule run_ionosphere_streaming:
    output:
        touch("artifacts/data/ionosphere_streaming.done")
    shell:
        """
        {PYTHON} benchmarks/run_realtime.py --multirun \
            experiment=ionosphere_streaming \
            +benchmark=realtime
        """

# Run ionosphere batch throughput (dual-channel high-NFFT)
rule run_ionosphere_batch_throughput:
    output:
        touch("artifacts/data/ionosphere_batch_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=ionosphere_batch_throughput \
            +benchmark=throughput
        """

# Run ionosphere streaming latency (dual-channel 4096/8192)
rule run_ionosphere_streaming_latency:
    output:
        touch("artifacts/data/ionosphere_streaming_latency.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py --multirun \
            experiment=ionosphere_streaming_latency \
            +benchmark=latency
        """

# Run accuracy validation (single-channel, zero-overlap, both modes)
rule run_accuracy_validation:
    output:
        touch("artifacts/data/accuracy_validation.done")
    shell:
        """
        {PYTHON} benchmarks/run_accuracy.py --multirun \
            experiment=accuracy_validation \
            +benchmark=accuracy
        """

# Run execution mode comparison (BATCH vs STREAMING overhead analysis)
rule run_execution_mode_comparison:
    output:
        touch("artifacts/data/execution_mode_comparison.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py \
            experiment=execution_mode_comparison \
            +benchmark=latency
        """

# ============================================================================
# NEW: Streaming Mode Gap-Filling Experiments (Phase 1)
# ============================================================================

# Run ionosphere streaming throughput (sustained FPS in STREAMING mode)
rule run_ionosphere_streaming_throughput:
    output:
        touch("artifacts/data/ionosphere_streaming_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=ionosphere_streaming_throughput \
            +benchmark=throughput
        """

# Run ionosphere streaming high-resolution (validate high-NFFT streaming)
rule run_ionosphere_streaming_hires:
    output:
        touch("artifacts/data/ionosphere_streaming_hires.done")
    shell:
        """
        {PYTHON} benchmarks/run_realtime.py --multirun \
            experiment=ionosphere_streaming_hires \
            +benchmark=realtime
        """

# Run baseline BATCH latency at 100kHz (BATCH mode for methods paper)
rule run_baseline_batch_100k_latency:
    output:
        touch("artifacts/data/baseline_batch_100k_latency.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py --multirun \
            experiment=baseline_batch_100k_latency \
            +benchmark=latency
        """

# Run baseline STREAMING realtime at 100kHz (STREAMING mode for methods paper)
rule run_baseline_streaming_100k_realtime:
    output:
        touch("artifacts/data/baseline_streaming_100k_realtime.done")
    shell:
        """
        {PYTHON} benchmarks/run_realtime.py --multirun \
            experiment=baseline_streaming_100k_realtime \
            +benchmark=realtime
        """

# ============================================================================
# NEW: Phase 2 - BATCH Baseline Experiments
# ============================================================================

# Run baseline BATCH throughput at 100kHz
rule run_baseline_batch_100k_throughput:
    output:
        touch("artifacts/data/baseline_batch_100k_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_batch_100k_throughput \
            +benchmark=throughput
        """

# Run baseline BATCH throughput at 48kHz
rule run_baseline_batch_48k_throughput:
    output:
        touch("artifacts/data/baseline_batch_48k_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_batch_48k_throughput \
            +benchmark=throughput
        """

# Run high-NFFT BATCH throughput baseline (fill coverage gaps)
rule run_baseline_batch_high_nfft_throughput:
    output:
        touch("artifacts/data/baseline_batch_high_nfft_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_batch_high_nfft_throughput \
            +benchmark=throughput
        """

# Run baseline BATCH latency at 48kHz
rule run_baseline_batch_48k_latency:
    output:
        touch("artifacts/data/baseline_batch_48k_latency.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py --multirun \
            experiment=baseline_batch_48k_latency \
            +benchmark=latency
        """

# ============================================================================
# NEW: Phase 3 - STREAMING Baseline Experiments
# ============================================================================

# Run baseline STREAMING throughput at 100kHz
rule run_baseline_streaming_100k_throughput:
    output:
        touch("artifacts/data/baseline_streaming_100k_throughput.done")
    shell:
        """
        {PYTHON} benchmarks/run_throughput.py --multirun \
            experiment=baseline_streaming_100k_throughput \
            +benchmark=throughput
        """

# Run baseline STREAMING latency at 100kHz
rule run_baseline_streaming_100k_latency:
    output:
        touch("artifacts/data/baseline_streaming_100k_latency.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py --multirun \
            experiment=baseline_streaming_100k_latency \
            +benchmark=latency
        """

# Run baseline STREAMING latency at 48kHz
rule run_baseline_streaming_48k_latency:
    output:
        touch("artifacts/data/baseline_streaming_48k_latency.done")
    shell:
        """
        {PYTHON} benchmarks/run_latency.py --multirun \
            experiment=baseline_streaming_48k_latency \
            +benchmark=latency
        """

# Run baseline STREAMING realtime at 48kHz
rule run_baseline_streaming_48k_realtime:
    output:
        touch("artifacts/data/baseline_streaming_48k_realtime.done")
    shell:
        """
        {PYTHON} benchmarks/run_realtime.py --multirun \
            experiment=baseline_streaming_48k_realtime \
            +benchmark=realtime
        """

# ============================================================================
# Static spectrogram artifacts for Streamlit dashboard
# ============================================================================
rule generate_general_spectrogram:
    input:
        "artifacts/data/baseline_48k.done"  # Changed from ionosphere_resolution.done
    output:
        png="artifacts/figures/spectrograms/general_spectrogram.png",
        metadata="artifacts/figures/spectrograms/general_spectrogram.json"
    shell:
        """
        {PYTHON} experiments/analysis/generate_static_spectrograms.py \
            --targets general \
            --output-dir artifacts/figures/spectrograms
        """

rule generate_accuracy_spectrograms:
    input:
        "artifacts/data/accuracy_validation.done"
    output:
        engine="artifacts/figures/spectrograms/accuracy_engine.png",
        numpy="artifacts/figures/spectrograms/accuracy_numpy.png",
        delta="artifacts/figures/spectrograms/accuracy_difference.png",
        metadata="artifacts/figures/spectrograms/accuracy_metrics.json"
    shell:
        """
        {PYTHON} experiments/analysis/generate_static_spectrograms.py \
            --targets accuracy \
            --output-dir artifacts/figures/spectrograms
        """

# ============================================================================
# REPORTING
# ============================================================================
#
# SigTekX uses TWO reporting solutions:
#
# 1. Streamlit Dashboard (PRIMARY) - Interactive exploration
#    Launch: sigx dashboard
#    Location: experiments/streamlit/
#
# 2. Quarto Reports (FUTURE) - Publication-quality static reports
#    Generate: snakemake quarto_reports
#    Location: experiments/quarto/
#
# Both solutions share core analysis modules from experiments/analysis/
#
# To add Quarto report generation (future):
#
# rule general_performance_report:
#     input:
#         data="artifacts/data/baseline_48k.done",
#         template="experiments/quarto/templates/general_performance.qmd"
#     output:
#         "artifacts/reports/general_performance.pdf"
#     shell:
#         "quarto render {input.template} --output {output}"
#
# rule quarto_reports:
#     input:
#         "artifacts/reports/general_performance.pdf",
#         "artifacts/reports/ionosphere_research.pdf"
#
# ============================================================================

# Clean all artifacts
rule clean:
    shell:
        """
        {PYTHON} -c "
import shutil, os, glob
dirs_to_clean = ['artifacts/data', 'artifacts/figures', 'artifacts/reports', 'outputs', 'multirun']
for d in dirs_to_clean:
    if os.path.exists(d):
        for item in glob.glob(os.path.join(d, '*')):
            if os.path.isdir(item):
                shutil.rmtree(item)
            else:
                os.remove(item)
        print(f'Cleaned {d}')
print('All artifacts cleaned')
"
        """

# Minimal end-to-end smoke test — pipeline sanity check, not a workload.
rule smoke:
    shell:
        """
        {PYTHON} benchmarks/run_latency.py experiment=smoke_test +benchmark=latency
        echo "Smoke test complete."
        """

# Quick realistic ionosphere test run (streaming + batch variants)
rule test:
    shell:
        """
        {PYTHON} benchmarks/run_latency.py experiment=ionosphere_test +benchmark=latency
        {PYTHON} benchmarks/run_throughput.py experiment=ionosphere_test_batch +benchmark=throughput
        echo "Test benchmarks complete. View results with: sigx dashboard"
        """
