{% extends "base.html" %}

{% block extra_head %}
<style>
    .cluster-precheck-results {
        margin-top: 1.5rem;
        padding-top: 1rem;
        border-top: 1px solid rgba(255, 255, 255, 0.08);
    }

    .cluster-precheck-header {
        margin-bottom: 1rem;
    }

    .cluster-precheck-chart {
        position: relative;
        height: 320px;
        margin-top: 1rem;
    }

    .cluster-precheck-summary th,
    .cluster-precheck-summary td {
        white-space: nowrap;
    }

    .cluster-precheck-detail {
        white-space: normal;
        min-width: 16rem;
    }

    .cluster-region-section + .cluster-region-section {
        margin-top: 2rem;
    }

    .cluster-region-header {
        margin-bottom: 1rem;
    }

    .cluster-job-card {
        border-style: dashed;
    }

    .cluster-diagnostics {
        margin-top: 1rem;
        padding-top: 1rem;
        border-top: 1px solid rgba(255, 255, 255, 0.08);
    }

    .cluster-diagnostics summary {
        cursor: pointer;
    }

    .cluster-diagnostics-actions {
        margin-top: 0.75rem;
    }

    .cluster-probe-result {
        margin-top: 0.75rem;
    }

    .cluster-probe-card {
        border: 1px solid rgba(255, 255, 255, 0.08);
        border-radius: 6px;
        padding: 0.85rem;
        background: rgba(255, 255, 255, 0.02);
    }

    .cluster-probe-pre {
        margin: 0.4rem 0 0;
        max-height: 18rem;
        overflow: auto;
        white-space: pre-wrap;
        word-break: break-word;
        border-radius: 6px;
        border: 1px solid rgba(255, 255, 255, 0.08);
        background: rgba(0, 0, 0, 0.24);
        padding: 0.75rem;
        font-size: 0.82rem;
        line-height: 1.45;
    }

    .cluster-probe-block + .cluster-probe-block {
        margin-top: 0.75rem;
    }
</style>
{% endblock %}

{% block content %}
<div class="container">
    <div class="page-header d-flex justify-between align-center flex-wrap gap-lg">
        <div>
            <h1 class="page-title">ParallelCluster Instances</h1>
            <p class="page-subtitle">Running AWS HPC clusters across configured regions with the PR19 cluster manager restored.</p>
        </div>
        <div class="page-actions">
            {% if is_admin %}
            <button class="btn btn-primary" type="button" onclick="toggleCreateClusterPanel()" id="create-cluster-btn">
                <i class="fas fa-plus"></i> Create Cluster
            </button>
            {% endif %}
            <button class="btn btn-outline" type="button" onclick="refreshClusters()" id="refresh-btn">
                <i class="fas fa-sync-alt"></i> Refresh
            </button>
        </div>
    </div>

    {% if is_admin %}
    <div class="card mb-xl">
        <div class="card-header d-flex justify-between align-center flex-wrap gap-md">
            <div>
                <h3 class="card-title" style="margin-bottom: 0.25rem;">
                    <i class="fas fa-globe"></i> Cluster Scan Regions
                </h3>
                <div class="text-sm text-muted">Controls which AWS regions Ursa scans for live ParallelCluster inventory.</div>
            </div>
            <span class="badge badge-outline">Profile {{ aws_profile_label }}</span>
        </div>
        <div class="card-body">
            <div class="grid grid-2 gap-lg">
                <div>
                    <label class="form-label">Scanned Regions CSV</label>
                    <input
                        class="form-input"
                        id="cluster-scan-regions-csv"
                        type="text"
                        value="{{ scan_regions_csv }}"
                        placeholder="us-west-2,us-east-1,eu-central-1"
                        autocomplete="off"
                        spellcheck="false"
                    >
                    <div class="text-sm text-muted mt-xs">Existing region-specific SSH settings are preserved for unchanged regions.</div>
                </div>
                <div>
                    <label class="form-label">Current Regions</label>
                    <div class="text-sm">{{ regions | join(', ') if regions else '—' }}</div>
                    <div class="text-sm text-muted mt-xs">Refresh the page after saving to rescan with the new region set.</div>
                </div>
            </div>
            <div class="d-flex gap-md mt-md flex-wrap">
                <button class="btn btn-outline" type="button" onclick="saveClusterScanRegions()" id="cluster-scan-regions-save-btn">
                    <i class="fas fa-floppy-disk"></i> Save Scan Regions
                </button>
            </div>
        </div>
    </div>

    <details class="card mb-xl" id="create-cluster-panel" {% if create_mode %}open{% endif %}>
        <summary class="card-header cursor-pointer d-flex justify-between align-center">
            <div>
                <h3 class="card-title" style="margin-bottom: 0.25rem;">
                    <i class="fas fa-plus-circle"></i> Create Cluster
                </h3>
                <div class="text-sm text-muted">Queues a current Ursa cluster-create job and tracks it in the Recent Create Jobs table.</div>
            </div>
            <span class="badge badge-outline">Admin only</span>
        </summary>
        <div class="card-body">
            <div class="grid grid-2 gap-lg">
                <div>
                    <label class="form-label">Region</label>
                    <input
                        class="form-input"
                        id="cc-region"
                        type="text"
                        list="cc-region-options"
                        value="{{ prefill_region }}"
                        placeholder="us-west-2"
                        autocomplete="off"
                        spellcheck="false"
                        onchange="handleClusterRegionChange()"
                    >
                    <datalist id="cc-region-options">
                        {% for r in create_regions %}
                        <option value="{{ r }}"></option>
                        {% endfor %}
                    </datalist>
                    <div class="text-sm text-muted mt-xs">Choose a suggested region or type another AWS region.</div>
                </div>
                <div>
                    <label class="form-label">Availability Zone</label>
                    <select class="form-select" id="cc-region-az" onchange="handleClusterRegionAzChange()">
                        <option value="">Select availability zone</option>
                    </select>
                </div>
                <div>
                    <label class="form-label">Cluster Name</label>
                    <input class="form-input" id="cc-cluster-name" type="text" placeholder="daylily-1700000000">
                </div>
                <div>
                    <label class="form-label">AWS Profile</label>
                    <input class="form-input" id="cc-aws-profile" type="text" value="{{ aws_profile_label if aws_profile_label != 'default' else '' }}" placeholder="lsmc">
                </div>
                <div>
                    <label class="form-label">Owner User ID (Optional)</label>
                    <input class="form-input" id="cc-owner-user-id" type="text" placeholder="owner-user-id">
                </div>
                <div>
                    <label class="form-label">Contact Email</label>
                    <input class="form-input" id="cc-contact-email" type="email" placeholder="ops@example.com">
                </div>
                <div>
                    <label class="form-label">SSH Keypair</label>
                    <select class="form-select" id="cc-ssh-key-select" onchange="syncManualField('cc-ssh-key-select', 'cc-ssh-key')">
                        <option value="">Select keypair (or type below)</option>
                    </select>
                    <input class="form-input mt-xs" id="cc-ssh-key" type="text" placeholder="omics-keypair">
                </div>
                <div>
                    <label class="form-label">S3 Bucket Name</label>
                    <select class="form-select" id="cc-s3-bucket-select" onchange="syncManualField('cc-s3-bucket-select', 'cc-s3-bucket')">
                        <option value="">Select bucket (or type below)</option>
                    </select>
                    <input class="form-input mt-xs" id="cc-s3-bucket" type="text" placeholder="omics-analysis-bucket">
                </div>
                <div>
                    <label class="form-label">Config Path (Optional)</label>
                    <input class="form-input" id="cc-config-path" type="text" placeholder="config/daylily_ephemeral_cluster_template.yaml">
                    <div class="text-sm text-muted mt-xs">Leave blank to generate a daylily-ec config from the fields below.</div>
                </div>
                <div>
                    <label class="form-label">Repository Overrides</label>
                    <textarea class="form-input" id="cc-repo-overrides" rows="2" placeholder="daylily-omics-analysis:main"></textarea>
                    <div class="text-sm text-muted mt-xs">One <code>repo-key:git-ref</code> per line.</div>
                </div>
            </div>

            <details class="mt-lg" open>
                <summary class="cursor-pointer">
                    <strong>daylily-ec create prompts</strong>
                    <span class="badge badge-outline ml-sm">non-interactive</span>
                </summary>
                <div class="grid grid-2 gap-lg mt-md">
                    <div>
                        <label class="form-label">Public Subnet ID</label>
                        <input class="form-input" id="cc-public-subnet-id" type="text" placeholder="subnet-...">
                    </div>
                    <div>
                        <label class="form-label">Private Subnet ID</label>
                        <input class="form-input" id="cc-private-subnet-id" type="text" placeholder="subnet-...">
                    </div>
                    <div>
                        <label class="form-label">IAM Policy ARN</label>
                        <input class="form-input" id="cc-iam-policy-arn" type="text" placeholder="arn:aws:iam::...:policy/pclusterTagsAndBudget">
                    </div>
                    <div>
                        <label class="form-label">Cluster Template YAML</label>
                        <input class="form-input" id="cc-cluster-template-yaml" type="text" value="config/day_cluster/prod_cluster.yaml">
                    </div>
                    <div>
                        <label class="form-label">Headnode Instance Type</label>
                        <input class="form-input" id="cc-headnode-instance-type" type="text" value="r7i.2xlarge">
                    </div>
                    <div>
                        <label class="form-label">FSx Size (GiB)</label>
                        <input class="form-input" id="cc-fsx-fs-size" type="number" min="1200" step="1200" value="4800">
                    </div>
                    <div>
                        <label class="form-label">Budget Email</label>
                        <input class="form-input" id="cc-budget-email" type="email" placeholder="ops@example.com">
                    </div>
                    <div>
                        <label class="form-label">Heartbeat Email</label>
                        <input class="form-input" id="cc-heartbeat-email" type="email" placeholder="ops@example.com">
                    </div>
                    <div>
                        <label class="form-label">Allowed Budget Users</label>
                        <input class="form-input" id="cc-allowed-budget-users" type="text" placeholder="user1,user2">
                    </div>
                    <div>
                        <label class="form-label">Global Allowed Budget Users</label>
                        <input class="form-input" id="cc-global-allowed-budget-users" type="text" placeholder="user1,user2">
                    </div>
                    <div>
                        <label class="form-label">Cluster Budget Amount</label>
                        <input class="form-input" id="cc-budget-amount" type="number" min="1" step="1" value="200">
                    </div>
                    <div>
                        <label class="form-label">Global Budget Amount</label>
                        <input class="form-input" id="cc-global-budget-amount" type="number" min="1" step="1" value="1000">
                    </div>
                    <div>
                        <label class="form-label">Heartbeat Schedule</label>
                        <input class="form-input" id="cc-heartbeat-schedule" type="text" value="rate(60 minutes)">
                    </div>
                    <div>
                        <label class="form-label">Heartbeat Scheduler Role ARN</label>
                        <input class="form-input" id="cc-heartbeat-scheduler-role-arn" type="text" placeholder="arn:aws:iam::...:role/...">
                    </div>
                    <div>
                        <label class="form-label">Spot Allocation Strategy</label>
                        <select class="form-select" id="cc-spot-instance-allocation-strategy">
                            <option value="price-capacity-optimized" selected>price-capacity-optimized</option>
                            <option value="capacity-optimized">capacity-optimized</option>
                            <option value="lowest-price">lowest-price</option>
                        </select>
                    </div>
                    <div>
                        <label class="form-label">Enforce Budget</label>
                        <select class="form-select" id="cc-enforce-budget">
                            <option value="skip" selected>skip</option>
                            <option value="true">true</option>
                            <option value="false">false</option>
                        </select>
                    </div>
                    <div>
                        <label class="form-label">Detailed Monitoring</label>
                        <select class="form-select" id="cc-enable-detailed-monitoring">
                            <option value="false" selected>false</option>
                            <option value="true">true</option>
                        </select>
                    </div>
                    <div>
                        <label class="form-label">Delete Local Root</label>
                        <select class="form-select" id="cc-delete-local-root">
                            <option value="true" selected>true</option>
                            <option value="false">false</option>
                        </select>
                    </div>
                    <div>
                        <label class="form-label">Auto Delete FSx</label>
                        <select class="form-select" id="cc-auto-delete-fsx">
                            <option value="Delete" selected>Delete</option>
                            <option value="Retain">Retain</option>
                        </select>
                    </div>
                    <div>
                        <label class="form-label">Max 8xlarge Count</label>
                        <input class="form-input" id="cc-max-count-8i" type="number" min="0" step="1" value="1">
                    </div>
                    <div>
                        <label class="form-label">Max 128xlarge Count</label>
                        <input class="form-input" id="cc-max-count-128i" type="number" min="0" step="1" value="1">
                    </div>
                    <div>
                        <label class="form-label">Max 192xlarge Count</label>
                        <input class="form-input" id="cc-max-count-192i" type="number" min="0" step="1" value="1">
                    </div>
                </div>
            </details>

            <div class="d-flex gap-lg align-center mt-md flex-wrap">
                <label class="d-flex align-center gap-sm text-sm">
                    <input type="checkbox" id="cc-pass-on-warn">
                    <span>Pass on warning</span>
                </label>
                <label class="d-flex align-center gap-sm text-sm">
                    <input type="checkbox" id="cc-debug">
                    <span>Debug</span>
                </label>
            </div>
            <div class="d-flex gap-md mt-md flex-wrap">
                <button class="btn btn-outline" type="button" onclick="loadCreateOptions()" id="cc-load-options-btn">
                    <i class="fas fa-cloud-download-alt"></i> Load Region Options
                </button>
                <button class="btn btn-outline" type="button" onclick="verifyPartitionInstances()" id="cc-verify-btn">
                    <i class="fas fa-shield-halved"></i> Verify Partition Instances
                </button>
                <button class="btn btn-outline" type="button" onclick="calculatePartitionPricing()" id="cc-pricing-btn">
                    <i class="fas fa-chart-column"></i> Calculate Spot Pricing Per Partition
                </button>
                <button class="btn btn-outline" type="button" onclick="runAwsCheckAll()" id="cc-aws-check-btn">
                    <i class="fas fa-clipboard-check"></i> AWS Check All
                </button>
                <button class="btn btn-primary" type="button" onclick="createCluster()" id="cc-submit-btn">
                    <i class="fas fa-play"></i> Dry Run + Start Create Job
                </button>
            </div>
            <section class="cluster-precheck-results d-none" id="cc-verification-results"></section>
            <section class="cluster-precheck-results d-none" id="cc-pricing-results"></section>
            <section class="cluster-precheck-results d-none" id="cc-aws-check-results"></section>
        </div>
    </details>
    {% endif %}

    <div class="card mb-xl" id="cluster-create-jobs-card">
        <div class="card-header d-flex justify-between align-center flex-wrap gap-md">
            <div>
                <h3 class="card-title" style="margin-bottom: 0.25rem;">
                    <i class="fas fa-list"></i> Recent Cluster Create Jobs
                </h3>
                <div class="text-sm text-muted">Current Ursa cluster job records.</div>
            </div>
            <button class="btn btn-outline btn-sm" type="button" onclick="refreshCreateJobs()">
                <i class="fas fa-sync-alt"></i> Refresh Jobs
            </button>
        </div>
        <div class="table-wrapper">
            <table class="table" id="cluster-create-jobs-table">
                <thead>
                    <tr>
                        <th>Job ID</th>
                        <th>Cluster</th>
                        <th>Region/AZ</th>
                        <th>Status</th>
                        <th>Return</th>
                        <th>Created</th>
                        <th>Actions</th>
                    </tr>
                </thead>
                <tbody id="cluster-create-jobs-tbody">
                    {% for job in jobs %}
                    <tr>
                        <td><a href="/clusters/jobs/{{ job.job_euid }}" class="mono">{{ job.job_euid }}</a></td>
                        <td>{{ job.cluster_name }}</td>
                        <td>{{ job.region_az }}</td>
                        <td><span class="badge badge-{{ job.state | lower | replace('_', '-') }}">{{ job.state }}</span></td>
                        <td>{{ job.return_code if job.return_code is not none else '—' }}</td>
                        <td>{{ job.created_at[:19] }}</td>
                        <td><a href="/clusters/jobs/{{ job.job_euid }}" class="btn btn-outline btn-sm">View</a></td>
                    </tr>
                    {% else %}
                    <tr><td colspan="7" class="text-muted">No create jobs yet.</td></tr>
                    {% endfor %}
                </tbody>
            </table>
        </div>
    </div>

    <div class="grid grid-4 mb-xl">
        <div class="stat-card">
            <div class="card-icon primary"><i class="fas fa-server"></i></div>
            <div>
                <div class="stat-value" id="total-clusters">{{ clusters | length }}</div>
                <div class="stat-label">Total Clusters</div>
            </div>
        </div>
        <div class="stat-card">
            <div class="card-icon success"><i class="fas fa-check-circle"></i></div>
            <div>
                <div class="stat-value" id="running-clusters">{{ clusters | selectattr('compute_fleet_status', 'equalto', 'RUNNING') | list | length }}</div>
                <div class="stat-label">Fleet Running</div>
            </div>
        </div>
        <div class="stat-card">
            <div class="card-icon info"><i class="fas fa-globe"></i></div>
            <div>
                <div class="stat-value" id="total-regions">{{ regions | length }}</div>
                <div class="stat-label">Regions Scanned</div>
            </div>
        </div>
        <div class="stat-card">
            <div class="card-icon warning"><i class="fas fa-microchip"></i></div>
            <div>
                <div class="stat-value" id="active-create-jobs">{{ active_create_jobs_count }}</div>
                <div class="stat-label">Create Jobs In Flight</div>
            </div>
        </div>
    </div>

    <div class="mb-lg text-muted text-sm">
        <i class="fas fa-info-circle"></i>
        Scanning regions: <strong>{{ regions | join(', ') }}</strong>
    </div>

    {% if cluster_regions %}
        {% for region_group in cluster_regions %}
        <section class="cluster-region-section">
            <div class="cluster-region-header d-flex justify-between align-center flex-wrap gap-md">
                <div>
                    <h3 class="card-title" style="margin-bottom: 0.25rem;">
                        <i class="fas fa-globe"></i> {{ region_group.region }}
                    </h3>
                    <div class="text-sm text-muted">
                        <span data-region-live-count="{{ region_group.region }}">{{ region_group.live_count }} live cluster{{ '' if region_group.live_count == 1 else 's' }}</span>,
                        {{ region_group.pending_count }} create job{{ '' if region_group.pending_count == 1 else 's' }} in flight.
                    </div>
                </div>
                <div class="d-flex gap-sm flex-wrap">
                    <span class="badge badge-outline">{{ region_group.region }}</span>
                    {% if region_group.pending_count %}
                    <span class="badge badge-warning">{{ region_group.pending_count }} creating</span>
                    {% endif %}
                </div>
            </div>

            <div class="grid grid-2 mb-xl" data-cluster-region-grid data-region="{{ region_group.region }}">
                {% for job in region_group.pending_jobs %}
                <div class="card cluster-card cluster-job-card" data-region="{{ job.region }}" data-job-state="{{ job.state }}">
                    <div class="card-header d-flex justify-between align-center">
                        <div>
                            <h3 class="card-title" style="margin-bottom: 0.25rem;">
                                <i class="fas fa-spinner fa-spin"></i> {{ job.cluster_name }}
                            </h3>
                            <span class="badge badge-outline">{{ job.region }}</span>
                        </div>
                        <div class="d-flex gap-sm">
                            <a class="btn btn-sm btn-outline" href="/clusters/jobs/{{ job.job_euid }}">
                                <i class="fas fa-eye"></i>
                            </a>
                            <span class="badge badge-warning">{{ job.state }}</span>
                        </div>
                    </div>
                    <div class="card-body">
                        <div class="grid grid-2 gap-md">
                            <div>
                                <span class="text-muted text-xs">Create Job</span>
                                <div><a class="mono" href="/clusters/jobs/{{ job.job_euid }}">{{ job.job_euid }}</a></div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Region/AZ</span>
                                <div>{{ job.region_az or job.region }}</div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">State</span>
                                <div><span class="badge badge-warning">{{ job.state }}</span></div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Queued</span>
                                <div>{{ job.created_at[:19] if job.created_at else '—' }}</div>
                            </div>
                        </div>
                    </div>
                </div>
                {% endfor %}

                {% for cluster in region_group.clusters %}
                <div class="card cluster-card" data-status="{{ cluster.cluster_status }}" data-region="{{ cluster.region }}">
                    <div class="card-header d-flex justify-between align-center">
                        <div>
                            <h3 class="card-title" style="margin-bottom: 0.25rem;">
                                <i class="fas fa-server"></i> {{ cluster.cluster_name }}
                            </h3>
                            <span class="badge badge-outline">{{ cluster.region }}</span>
                        </div>
                        <div class="d-flex gap-sm">
                            <button class="btn btn-sm btn-outline" type="button" onclick="window.location.assign('/clusters/{{ cluster.cluster_name }}?region={{ cluster.region }}')">
                                <i class="fas fa-eye"></i>
                            </button>
                            <button class="btn btn-sm btn-danger" type="button" onclick="deleteCluster('{{ cluster.cluster_name }}', '{{ cluster.region }}')">
                                <i class="fas fa-trash"></i>
                            </button>
                            <span class="badge badge-{{ 'success' if cluster.cluster_status == 'CREATE_COMPLETE' else 'warning' if 'IN_PROGRESS' in cluster.cluster_status else 'error' if 'FAILED' in cluster.cluster_status else 'secondary' }}">
                                {{ cluster.cluster_status }}
                            </span>
                        </div>
                    </div>
                    <div class="card-body">
                        {% if cluster.head_node %}
                        <div class="mb-md">
                            <h4 class="text-sm text-muted mb-sm">Head Node</h4>
                            <div class="d-flex flex-wrap gap-md">
                                <div>
                                    <span class="text-muted text-xs">Instance Type</span>
                                    <div class="text-sm"><code>{{ cluster.head_node.instance_type }}</code></div>
                                </div>
                                <div>
                                    <span class="text-muted text-xs">State</span>
                                    <div class="text-sm"><span class="badge badge-{{ 'success' if cluster.head_node.state == 'running' else 'warning' }}">{{ cluster.head_node.state }}</span></div>
                                </div>
                                {% if cluster.head_node.public_ip %}
                                <div>
                                    <span class="text-muted text-xs">Public IP</span>
                                    <div class="text-sm"><code>{{ cluster.head_node.public_ip }}</code></div>
                                </div>
                                {% endif %}
                            </div>
                        </div>
                        {% endif %}

                        <div class="grid grid-2 gap-md">
                            <div>
                                <span class="text-muted text-xs">daylily-ephemeral-cluster Pin</span>
                                <div><code>{{ cluster.daylily_ec_pinned_version or '—' }}</code></div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Headnode EC2</span>
                                <div>
                                    {% if cluster.aws_console_url %}
                                    <a href="{{ cluster.aws_console_url }}" target="_blank" rel="noopener noreferrer">
                                        AWS Console <i class="fas fa-external-link-alt"></i>
                                    </a>
                                    {% else %}
                                    <span class="text-muted">No instance id</span>
                                    {% endif %}
                                </div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Compute Fleet</span>
                                <div><span class="badge badge-outline">{{ cluster.compute_fleet_status }}</span></div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Scheduler</span>
                                <div>{{ cluster.scheduler_type or 'slurm' }}</div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Created</span>
                                <div>{{ cluster.creation_time[:19] if cluster.creation_time else '—' }}</div>
                            </div>
                            <div>
                                <span class="text-muted text-xs">Updated</span>
                                <div>{{ cluster.last_updated_time[:19] if cluster.last_updated_time else '—' }}</div>
                            </div>
                        </div>

                        <details
                            class="cluster-diagnostics"
                            data-cluster-diagnostics
                            data-cluster-name="{{ cluster.cluster_name }}"
                            data-region="{{ cluster.region }}"
                        >
                            <summary class="d-flex justify-between align-center gap-md">
                                <span><i class="fas fa-terminal"></i> Headnode Diagnostics</span>
                                <span class="badge badge-outline">Admin read-only</span>
                            </summary>
                            <div class="cluster-diagnostics-actions d-flex gap-sm flex-wrap">
                                <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'static', false)">
                                    <i class="fas fa-screwdriver-wrench"></i> Probe tools
                                </button>
                                <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'scheduler', true)">
                                    <i class="fas fa-list-check"></i> Refresh Slurm
                                </button>
                                <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'fsx', true)">
                                    <i class="fas fa-hard-drive"></i> Check FSx
                                </button>
                            </div>

                            {% set static_probe = cluster.headnode_probes.static if cluster.headnode_probes else none %}
                            {% set scheduler_probe = cluster.headnode_probes.scheduler if cluster.headnode_probes else none %}
                            {% set fsx_probe = cluster.headnode_probes.fsx if cluster.headnode_probes else none %}

                            <section class="cluster-probe-result" data-probe-result="static">
                                {% if static_probe %}
                                <div class="cluster-probe-card">
                                    <div class="d-flex justify-between align-center flex-wrap gap-sm">
                                        <strong>Tools Probe</strong>
                                        <span class="badge badge-outline">{{ 'cache' if static_probe.cached else 'fresh' }} {{ static_probe.captured_at[:19] }}</span>
                                    </div>
                                    {% if static_probe.error %}
                                    <div class="alert alert-danger mt-sm">{{ static_probe.error }}</div>
                                    {% endif %}
                                    <div class="grid grid-2 gap-sm mt-sm">
                                        <div><span class="text-muted text-xs">Pinned</span><div><code>{{ static_probe.data.daylily_ec_pinned_version or cluster.daylily_ec_pinned_version or '—' }}</code></div></div>
                                        <div><span class="text-muted text-xs">Remote</span><div><code>{{ static_probe.data.remote_daylily_ec_version or 'unavailable' }}</code></div></div>
                                        {% if static_probe.data.remote_git_hash %}
                                        <div><span class="text-muted text-xs">Remote Git Hash</span><div><code>{{ static_probe.data.remote_git_hash }}</code></div></div>
                                        {% endif %}
                                        <div><span class="text-muted text-xs">day-clone</span><div>{{ 'available' if static_probe.data.day_clone_available else 'unavailable' }}</div></div>
                                    </div>
                                    <div class="cluster-probe-block">
                                        <div class="text-muted text-xs">day-clone --help</div>
                                        <pre class="cluster-probe-pre">{{ static_probe.data.day_clone_help or '—' }}</pre>
                                    </div>
                                </div>
                                {% else %}
                                <div class="text-muted text-sm">Tools probe has not run for this cluster.</div>
                                {% endif %}
                            </section>

                            <section class="cluster-probe-result" data-probe-result="scheduler">
                                {% if scheduler_probe %}
                                <div class="cluster-probe-card">
                                    <div class="d-flex justify-between align-center flex-wrap gap-sm">
                                        <strong>Slurm Probe</strong>
                                        <span class="badge badge-outline">{{ 'cache' if scheduler_probe.cached else 'fresh' }} {{ scheduler_probe.captured_at[:19] }}</span>
                                    </div>
                                    {% if scheduler_probe.error %}
                                    <div class="alert alert-danger mt-sm">{{ scheduler_probe.error }}</div>
                                    {% endif %}
                                    <div class="cluster-probe-block">
                                        <div class="text-muted text-xs">squeue</div>
                                        <pre class="cluster-probe-pre">{{ scheduler_probe.data.squeue_output or '—' }}</pre>
                                    </div>
                                    <div class="cluster-probe-block">
                                        <div class="text-muted text-xs">sinfo</div>
                                        <pre class="cluster-probe-pre">{{ scheduler_probe.data.sinfo_output or '—' }}</pre>
                                    </div>
                                </div>
                                {% else %}
                                <div class="text-muted text-sm">Slurm probe has not run for this cluster.</div>
                                {% endif %}
                            </section>

                            <section class="cluster-probe-result" data-probe-result="fsx">
                                {% if fsx_probe %}
                                <div class="cluster-probe-card">
                                    <div class="d-flex justify-between align-center flex-wrap gap-sm">
                                        <strong>FSx Probe</strong>
                                        <span class="badge badge-outline">{{ 'cache' if fsx_probe.cached else 'fresh' }} {{ fsx_probe.captured_at[:19] }}</span>
                                    </div>
                                    {% if fsx_probe.error %}
                                    <div class="alert alert-danger mt-sm">{{ fsx_probe.error }}</div>
                                    {% endif %}
                                    <div class="cluster-probe-block">
                                        <div class="text-muted text-xs">df -h /fsx</div>
                                        <pre class="cluster-probe-pre">{{ fsx_probe.data.df_output or '—' }}</pre>
                                    </div>
                                </div>
                                {% else %}
                                <div class="text-muted text-sm">FSx probe has not run for this cluster.</div>
                                {% endif %}
                            </section>
                        </details>
                    </div>
                </div>
                {% endfor %}
                <div class="card cluster-card" data-cluster-region-loading>
                    <div class="card-body text-muted text-sm">
                        <i class="fas fa-spinner fa-spin"></i> Loading live clusters in {{ region_group.region }}...
                    </div>
                </div>
            </div>
        </section>
        {% endfor %}
    {% else %}
    <div class="empty-state">
        <div class="empty-state-icon"><i class="fas fa-server"></i></div>
        <h4 class="empty-state-title">No clusters found</h4>
        <p class="empty-state-text">Refresh the cluster manager or create a new cluster.</p>
    </div>
    {% endif %}
</div>
{% endblock %}

{% block extra_js %}
<script src="https://cdn.jsdelivr.net/npm/@sgratzl/chartjs-chart-boxplot@4.4.4/build/index.umd.min.js"></script>
<script>
let clusterVerificationState = null;
let clusterPricingChart = null;
let latestAwsGapAnalysis = null;
let clusterInventoryState = {
    totalExpected: 0,
    running: 0,
    failed: 0,
};

const CLUSTER_DETAIL_CONCURRENCY = 3;

const CLUSTER_CONFIG_INPUTS = {
    public_subnet_id: 'cc-public-subnet-id',
    private_subnet_id: 'cc-private-subnet-id',
    iam_policy_arn: 'cc-iam-policy-arn',
    budget_email: 'cc-budget-email',
    allowed_budget_users: 'cc-allowed-budget-users',
    budget_amount: 'cc-budget-amount',
    global_allowed_budget_users: 'cc-global-allowed-budget-users',
    global_budget_amount: 'cc-global-budget-amount',
    enforce_budget: 'cc-enforce-budget',
    cluster_template_yaml: 'cc-cluster-template-yaml',
    headnode_instance_type: 'cc-headnode-instance-type',
    fsx_fs_size: 'cc-fsx-fs-size',
    enable_detailed_monitoring: 'cc-enable-detailed-monitoring',
    delete_local_root: 'cc-delete-local-root',
    auto_delete_fsx: 'cc-auto-delete-fsx',
    heartbeat_email: 'cc-heartbeat-email',
    heartbeat_schedule: 'cc-heartbeat-schedule',
    heartbeat_scheduler_role_arn: 'cc-heartbeat-scheduler-role-arn',
    spot_instance_allocation_strategy: 'cc-spot-instance-allocation-strategy',
    max_count_8I: 'cc-max-count-8i',
    max_count_128I: 'cc-max-count-128i',
    max_count_192I: 'cc-max-count-192i',
};

function clusterActionErrorMessage(error, defaultMessage) {
    const message = String(error?.message || '').trim();
    return message || defaultMessage;
}

async function withClusterButtonBusy(buttonId, loadingMessage, action) {
    const button = document.getElementById(buttonId);
    const originalHtml = button?.innerHTML || '';
    const originalDisabled = button?.disabled || false;
    if (button) {
        button.disabled = true;
        button.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Working...';
    }
    showLoading(loadingMessage);
    try {
        return await action();
    } finally {
        hideLoading();
        if (button) {
            button.disabled = originalDisabled;
            button.innerHTML = originalHtml;
        }
    }
}

function toggleCreateClusterPanel() {
    const panel = document.getElementById('create-cluster-panel');
    if (!panel) return;
    panel.open = !panel.open;
}

function syncManualField(selectId, inputId) {
    const selectEl = document.getElementById(selectId);
    const inputEl = document.getElementById(inputId);
    if (selectEl && inputEl && selectEl.value) {
        inputEl.value = selectEl.value;
    }
}

function selectedClusterRegion() {
    return document.getElementById('cc-region')?.value.trim() || '';
}

function selectedClusterRegionAz() {
    return document.getElementById('cc-region-az')?.value || '';
}

function clusterInputValue(id) {
    return document.getElementById(id)?.value.trim() || '';
}

function collectClusterConfigValues() {
    const values = {};
    Object.entries(CLUSTER_CONFIG_INPUTS).forEach(([field, elementId]) => {
        const value = clusterInputValue(elementId);
        if (value) {
            values[field] = value;
        }
    });
    const clusterName = clusterInputValue('cc-cluster-name');
    const s3BucketName = clusterInputValue('cc-s3-bucket');
    if (clusterName) values.cluster_name = clusterName;
    if (s3BucketName) values.s3_bucket_name = s3BucketName;
    return values;
}

function collectRepoOverrides() {
    return clusterInputValue('cc-repo-overrides')
        .split(/\r?\n/)
        .map((line) => line.trim())
        .filter(Boolean);
}

function collectClusterCreatePayload() {
    return {
        cluster_name: clusterInputValue('cc-cluster-name'),
        region: selectedClusterRegion(),
        region_az: selectedClusterRegionAz(),
        ssh_key_name: clusterInputValue('cc-ssh-key'),
        s3_bucket_name: clusterInputValue('cc-s3-bucket'),
        owner_user_id: clusterInputValue('cc-owner-user-id') || null,
        aws_profile: clusterInputValue('cc-aws-profile') || null,
        config_path: clusterInputValue('cc-config-path') || null,
        contact_email: clusterInputValue('cc-contact-email') || null,
        pass_on_warn: document.getElementById('cc-pass-on-warn')?.checked || false,
        debug: document.getElementById('cc-debug')?.checked || false,
        cluster_config_values: collectClusterConfigValues(),
        repo_overrides: collectRepoOverrides(),
    };
}

function setSelectOptions(selectId, placeholder, values, selectedValue = '') {
    const selectEl = document.getElementById(selectId);
    if (!selectEl) return;
    const currentValue = selectedValue || selectEl.value || '';
    selectEl.innerHTML = `<option value="">${escapeHtml(placeholder)}</option>` + (values || []).map((value) => `
        <option value="${escapeHtml(value)}">${escapeHtml(value)}</option>
    `).join('');
    if (currentValue && (values || []).includes(currentValue)) {
        selectEl.value = currentValue;
    }
}

function destroyClusterPricingChart() {
    if (clusterPricingChart) {
        clusterPricingChart.destroy();
        clusterPricingChart = null;
    }
}

function updateCreateButtonState() {
    const submitButton = document.getElementById('cc-submit-btn');
    if (!submitButton) return;
    const appliesToCurrentSelection = Boolean(
        clusterVerificationState
        && clusterVerificationState.region === selectedClusterRegion()
        && clusterVerificationState.region_az === selectedClusterRegionAz()
    );
    submitButton.disabled = Boolean(appliesToCurrentSelection && clusterVerificationState.has_failures);
}

function clearClusterVerificationResults() {
    clusterVerificationState = null;
    const container = document.getElementById('cc-verification-results');
    if (container) {
        container.classList.add('d-none');
        container.innerHTML = '';
    }
    updateCreateButtonState();
}

function clearClusterPricingResults() {
    destroyClusterPricingChart();
    const container = document.getElementById('cc-pricing-results');
    if (container) {
        container.classList.add('d-none');
        container.innerHTML = '';
    }
}

function clearAwsCheckResults() {
    latestAwsGapAnalysis = null;
    const container = document.getElementById('cc-aws-check-results');
    if (container) {
        container.classList.add('d-none');
        container.innerHTML = '';
    }
}

function clearClusterPrechecks() {
    clearClusterVerificationResults();
    clearClusterPricingResults();
    clearAwsCheckResults();
}

function handleClusterRegionChange() {
    clearClusterPrechecks();
    setSelectOptions('cc-region-az', 'Select availability zone', []);
    const region = selectedClusterRegion();
    if (!region) {
        return;
    }
    loadCreateOptions(false).catch(() => {});
}

function handleClusterRegionAzChange() {
    clearClusterPrechecks();
}

function clusterProbeOutputBlock(title, value) {
    return `
        <div class="cluster-probe-block">
            <div class="text-muted text-xs">${escapeHtml(title)}</div>
            <pre class="cluster-probe-pre">${escapeHtml(value || '—')}</pre>
        </div>
    `;
}

function clusterProbeTitle(probeType) {
    if (probeType === 'static') return 'Tools Probe';
    if (probeType === 'scheduler') return 'Slurm Probe';
    if (probeType === 'fsx') return 'FSx Probe';
    return 'Headnode Probe';
}

function headnodeProbeHtml(payload) {
    const data = payload?.data || {};
    const probeType = payload?.probe_type || '';
    const capturedAt = String(payload?.captured_at || '').slice(0, 19);
    const cacheLabel = payload?.cached ? 'cache' : 'fresh';
    const errorBlock = payload?.error
        ? `<div class="alert alert-danger mt-sm">${escapeHtml(payload.error)}</div>`
        : '';
    const instanceBlock = payload?.instance_id
        ? `<span class="badge badge-outline">${escapeHtml(payload.instance_id)}</span>`
        : '';
    let body = '';

    if (probeType === 'static') {
        const gitHashBlock = data.remote_git_hash
            ? `<div><span class="text-muted text-xs">Remote Git Hash</span><div><code>${escapeHtml(data.remote_git_hash)}</code></div></div>`
            : '';
        body = `
            <div class="grid grid-2 gap-sm mt-sm">
                <div><span class="text-muted text-xs">Pinned</span><div><code>${escapeHtml(data.daylily_ec_pinned_version || '')}</code></div></div>
                <div><span class="text-muted text-xs">Remote</span><div><code>${escapeHtml(data.remote_daylily_ec_version || 'unavailable')}</code></div></div>
                ${gitHashBlock}
                <div><span class="text-muted text-xs">day-clone</span><div>${data.day_clone_available ? 'available' : 'unavailable'}</div></div>
            </div>
            ${clusterProbeOutputBlock('day-clone --help', data.day_clone_help)}
        `;
    } else if (probeType === 'scheduler') {
        body = `
            ${clusterProbeOutputBlock('squeue', data.squeue_output)}
            ${clusterProbeOutputBlock('sinfo', data.sinfo_output)}
        `;
    } else if (probeType === 'fsx') {
        body = clusterProbeOutputBlock('df -h /fsx', data.df_output);
    } else {
        body = clusterProbeOutputBlock('stdout', data.stdout);
    }

    return `
        <div class="cluster-probe-card">
            <div class="d-flex justify-between align-center flex-wrap gap-sm">
                <strong>${escapeHtml(clusterProbeTitle(probeType))}</strong>
                <div class="d-flex gap-sm flex-wrap">
                    ${instanceBlock}
                    <span class="badge badge-outline">${escapeHtml(`${cacheLabel} ${capturedAt}`.trim())}</span>
                </div>
            </div>
            ${errorBlock}
            ${body}
        </div>
    `;
}

function renderHeadnodeProbe(container, payload) {
    container.innerHTML = headnodeProbeHtml(payload);
}

function clusterStatusBadgeClass(status) {
    const value = String(status || '').toUpperCase();
    if (value === 'CREATE_COMPLETE') return 'success';
    if (value.includes('IN_PROGRESS')) return 'warning';
    if (value.includes('FAILED')) return 'error';
    return 'secondary';
}

function resetClusterInventoryStats() {
    clusterInventoryState = { totalExpected: 0, running: 0, failed: 0 };
    updateClusterInventoryStats();
}

function updateClusterInventoryStats() {
    const totalEl = document.getElementById('total-clusters');
    const runningEl = document.getElementById('running-clusters');
    if (totalEl) totalEl.textContent = String(clusterInventoryState.totalExpected);
    if (runningEl) runningEl.textContent = String(clusterInventoryState.running);
}

function setRegionLiveCount(region, count, failed = false) {
    const label = document.querySelector(`[data-region-live-count="${CSS.escape(region)}"]`);
    if (!label) return;
    if (failed) {
        label.textContent = 'cluster scan failed';
        return;
    }
    label.textContent = `${count} live cluster${count === 1 ? '' : 's'}`;
}

function clusterProbeSectionHtml(probeType, payload) {
    if (payload) {
        return `<section class="cluster-probe-result" data-probe-result="${escapeHtml(probeType)}">${headnodeProbeHtml(payload)}</section>`;
    }
    const title = probeType === 'static' ? 'Tools' : probeType === 'scheduler' ? 'Slurm' : 'FSx';
    return `
        <section class="cluster-probe-result" data-probe-result="${escapeHtml(probeType)}">
            <div class="text-muted text-sm">${title} probe has not run for this cluster.</div>
        </section>
    `;
}

function headNodeHtml(cluster) {
    const headNode = cluster?.head_node || null;
    if (!headNode) return '';
    const publicIp = headNode.public_ip
        ? `
            <div>
                <span class="text-muted text-xs">Public IP</span>
                <div class="text-sm"><code>${escapeHtml(headNode.public_ip)}</code></div>
            </div>
        `
        : '';
    return `
        <div class="mb-md">
            <h4 class="text-sm text-muted mb-sm">Head Node</h4>
            <div class="d-flex flex-wrap gap-md">
                <div>
                    <span class="text-muted text-xs">Instance Type</span>
                    <div class="text-sm"><code>${escapeHtml(headNode.instance_type || '—')}</code></div>
                </div>
                <div>
                    <span class="text-muted text-xs">State</span>
                    <div class="text-sm"><span class="badge badge-${headNode.state === 'running' ? 'success' : 'warning'}">${escapeHtml(headNode.state || 'unknown')}</span></div>
                </div>
                ${publicIp}
            </div>
        </div>
    `;
}

function renderClusterCard(cluster) {
    const clusterName = String(cluster?.cluster_name || '').trim();
    const region = String(cluster?.region || '').trim();
    const status = String(cluster?.cluster_status || 'UNKNOWN').trim();
    const probes = cluster?.headnode_probes || {};
    const detailUrl = `/clusters/${encodeURIComponent(clusterName)}?region=${encodeURIComponent(region)}`;
    const consoleLink = cluster?.aws_console_url
        ? `<a href="${escapeHtml(cluster.aws_console_url)}" target="_blank" rel="noopener noreferrer">AWS Console <i class="fas fa-external-link-alt"></i></a>`
        : '<span class="text-muted">No instance id</span>';
    return `
        <div class="card cluster-card" data-live-cluster-card data-status="${escapeHtml(status)}" data-region="${escapeHtml(region)}">
            <div class="card-header d-flex justify-between align-center">
                <div>
                    <h3 class="card-title" style="margin-bottom: 0.25rem;">
                        <i class="fas fa-server"></i> ${escapeHtml(clusterName)}
                    </h3>
                    <span class="badge badge-outline">${escapeHtml(region)}</span>
                </div>
                <div class="d-flex gap-sm">
                    <a class="btn btn-sm btn-outline" href="${escapeHtml(detailUrl)}">
                        <i class="fas fa-eye"></i>
                    </a>
                    <button class="btn btn-sm btn-danger" type="button" onclick="deleteCluster('${escapeHtml(clusterName)}', '${escapeHtml(region)}')">
                        <i class="fas fa-trash"></i>
                    </button>
                    <span class="badge badge-${clusterStatusBadgeClass(status)}">${escapeHtml(status)}</span>
                </div>
            </div>
            <div class="card-body">
                ${headNodeHtml(cluster)}
                <div class="grid grid-2 gap-md">
                    <div>
                        <span class="text-muted text-xs">daylily-ephemeral-cluster Pin</span>
                        <div><code>${escapeHtml(cluster?.daylily_ec_pinned_version || '—')}</code></div>
                    </div>
                    <div>
                        <span class="text-muted text-xs">Headnode EC2</span>
                        <div>${consoleLink}</div>
                    </div>
                    <div>
                        <span class="text-muted text-xs">Compute Fleet</span>
                        <div><span class="badge badge-outline">${escapeHtml(cluster?.compute_fleet_status || 'UNKNOWN')}</span></div>
                    </div>
                    <div>
                        <span class="text-muted text-xs">Scheduler</span>
                        <div>${escapeHtml(cluster?.scheduler_type || 'slurm')}</div>
                    </div>
                    <div>
                        <span class="text-muted text-xs">Created</span>
                        <div>${escapeHtml(String(cluster?.creation_time || '—').slice(0, 19) || '—')}</div>
                    </div>
                    <div>
                        <span class="text-muted text-xs">Updated</span>
                        <div>${escapeHtml(String(cluster?.last_updated_time || '—').slice(0, 19) || '—')}</div>
                    </div>
                </div>

                <details
                    class="cluster-diagnostics"
                    data-cluster-diagnostics
                    data-cluster-name="${escapeHtml(clusterName)}"
                    data-region="${escapeHtml(region)}"
                >
                    <summary class="d-flex justify-between align-center gap-md">
                        <span><i class="fas fa-terminal"></i> Headnode Diagnostics</span>
                        <span class="badge badge-outline">Admin read-only</span>
                    </summary>
                    <div class="cluster-diagnostics-actions d-flex gap-sm flex-wrap">
                        <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'static', false)">
                            <i class="fas fa-screwdriver-wrench"></i> Probe tools
                        </button>
                        <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'scheduler', true)">
                            <i class="fas fa-list-check"></i> Refresh Slurm
                        </button>
                        <button class="btn btn-outline btn-sm" type="button" onclick="runHeadnodeProbe(this, 'fsx', true)">
                            <i class="fas fa-hard-drive"></i> Check FSx
                        </button>
                    </div>
                    ${clusterProbeSectionHtml('static', probes.static)}
                    ${clusterProbeSectionHtml('scheduler', probes.scheduler)}
                    ${clusterProbeSectionHtml('fsx', probes.fsx)}
                </details>
            </div>
        </div>
    `;
}

function clusterLoadingCard(clusterName, region) {
    return `
        <div class="card cluster-card" data-live-cluster-card data-region="${escapeHtml(region)}">
            <div class="card-body text-muted text-sm">
                <i class="fas fa-spinner fa-spin"></i> Loading ${escapeHtml(clusterName)}...
            </div>
        </div>
    `;
}

function clusterErrorCard(clusterName, region, error) {
    return `
        <div class="card cluster-card" data-live-cluster-card data-region="${escapeHtml(region)}">
            <div class="card-header d-flex justify-between align-center">
                <h3 class="card-title" style="margin-bottom: 0.25rem;">
                    <i class="fas fa-server"></i> ${escapeHtml(clusterName)}
                </h3>
                <span class="badge badge-error">load failed</span>
            </div>
            <div class="card-body">
                <div class="alert alert-danger">${escapeHtml(clusterActionErrorMessage(error, 'Cluster detail failed'))}</div>
            </div>
        </div>
    `;
}

function clearRegionLiveCards(grid) {
    grid.querySelectorAll('[data-live-cluster-card], [data-cluster-region-loading], [data-cluster-empty], [data-cluster-region-error]').forEach((node) => node.remove());
}

async function loadClusterRegion(region, refresh = false) {
    const grid = document.querySelector(`[data-cluster-region-grid][data-region="${CSS.escape(region)}"]`);
    if (!grid) return;
    clearRegionLiveCards(grid);
    grid.insertAdjacentHTML('beforeend', `
        <div class="card cluster-card" data-cluster-region-loading>
            <div class="card-body text-muted text-sm">
                <i class="fas fa-spinner fa-spin"></i> Scanning live clusters in ${escapeHtml(region)}...
            </div>
        </div>
    `);

    let refs = [];
    try {
        const payload = await UrsaPortal.apiRequest(`/api/v1/clusters/regions/${encodeURIComponent(region)}/names?refresh=${refresh ? 'true' : 'false'}`);
        refs = Array.isArray(payload.items) ? payload.items : [];
    } catch (error) {
        clearRegionLiveCards(grid);
        setRegionLiveCount(region, 0, true);
        grid.insertAdjacentHTML('beforeend', `
            <div class="card cluster-card" data-cluster-region-error>
                <div class="card-body">
                    <div class="alert alert-danger">${escapeHtml(clusterActionErrorMessage(error, `Failed to scan ${region}`))}</div>
                </div>
            </div>
        `);
        return;
    }

    clearRegionLiveCards(grid);
    setRegionLiveCount(region, refs.length);
    clusterInventoryState.totalExpected += refs.length;
    updateClusterInventoryStats();

    if (!refs.length) {
        grid.insertAdjacentHTML('beforeend', `
            <div class="card cluster-card" data-cluster-empty>
                <div class="card-body text-muted text-sm">No live clusters in ${escapeHtml(region)}.</div>
            </div>
        `);
        return;
    }

    let cursor = 0;
    async function detailWorker() {
        while (cursor < refs.length) {
            const ref = refs[cursor];
            cursor += 1;
            const clusterName = String(ref.cluster_name || '').trim();
            const clusterRegion = String(ref.region || region).trim();
            if (!clusterName || !clusterRegion) continue;
            const placeholder = document.createElement('div');
            placeholder.innerHTML = clusterLoadingCard(clusterName, clusterRegion).trim();
            const card = placeholder.firstElementChild;
            grid.appendChild(card);
            try {
                const detail = await UrsaPortal.apiRequest(`/api/v1/clusters/${encodeURIComponent(clusterName)}?region=${encodeURIComponent(clusterRegion)}`);
                if (String(detail.compute_fleet_status || '').toUpperCase() === 'RUNNING') {
                    clusterInventoryState.running += 1;
                }
                card.outerHTML = renderClusterCard(detail);
            } catch (error) {
                clusterInventoryState.failed += 1;
                card.outerHTML = clusterErrorCard(clusterName, clusterRegion, error);
            } finally {
                updateClusterInventoryStats();
            }
        }
    }

    const workerCount = Math.min(CLUSTER_DETAIL_CONCURRENCY, refs.length);
    await Promise.all(Array.from({ length: workerCount }, () => detailWorker()));
}

async function loadClusterInventory(refresh = false) {
    resetClusterInventoryStats();
    const regions = Array.from(document.querySelectorAll('[data-cluster-region-grid]'))
        .map((node) => node.dataset.region || '')
        .filter(Boolean);
    await Promise.all(regions.map((region) => loadClusterRegion(region, refresh)));
}

async function runHeadnodeProbe(button, probeType, refresh = false) {
    const allowedProbeTypes = new Set(['static', 'scheduler', 'fsx']);
    if (!allowedProbeTypes.has(probeType)) {
        showToast('error', 'Unsupported headnode probe type');
        return;
    }
    const panel = button.closest('[data-cluster-diagnostics]');
    const clusterName = panel?.dataset.clusterName || '';
    const region = panel?.dataset.region || '';
    const container = panel?.querySelector(`[data-probe-result="${probeType}"]`);
    if (!clusterName || !region || !container) {
        showToast('error', 'Cluster diagnostics target is incomplete');
        return;
    }

    const originalHtml = button.innerHTML;
    const originalDisabled = button.disabled;
    button.disabled = true;
    button.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Running...';
    container.innerHTML = '<div class="text-muted text-sm">Running headnode probe...</div>';
    try {
        const result = await UrsaPortal.apiRequest(
            `/api/v1/clusters/${encodeURIComponent(clusterName)}/headnode/${encodeURIComponent(probeType)}?region=${encodeURIComponent(region)}&refresh=${refresh ? 'true' : 'false'}`,
            { method: 'POST' },
        );
        renderHeadnodeProbe(container, result);
        if (result.error) {
            showToast('error', `${clusterProbeTitle(probeType)} returned an error`);
        } else {
            showToast('success', `${clusterProbeTitle(probeType)} completed`);
        }
    } catch (error) {
        container.innerHTML = `<div class="alert alert-danger">${escapeHtml(clusterActionErrorMessage(error, 'Headnode probe failed'))}</div>`;
        showToast('error', clusterActionErrorMessage(error, 'Headnode probe failed'));
    } finally {
        button.disabled = originalDisabled;
        button.innerHTML = originalHtml;
    }
}

async function refreshClusters() {
    await loadClusterInventory(true);
}

async function saveClusterScanRegions() {
    const input = document.getElementById('cluster-scan-regions-csv');
    const regionsCsv = input?.value.trim() || '';
    await withClusterButtonBusy('cluster-scan-regions-save-btn', 'Saving cluster scan regions...', async () => {
        try {
            const result = await UrsaPortal.apiRequest('/api/v1/clusters/scan-regions', {
                method: 'POST',
                body: {
                    regions_csv: regionsCsv,
                },
            });
            showToast('success', `Updated cluster scan regions to ${result.regions.join(', ')}`);
            window.location.reload();
        } catch (error) {
            showToast('error', clusterActionErrorMessage(error, 'Failed to update cluster scan regions'));
        }
    });
}

async function loadCreateOptions(showSuccess = true, showErrors = true) {
    const region = selectedClusterRegion();
    if (!region) {
        showToast('error', 'Select a region first');
        return;
    }
    const run = async () => {
        try {
            const payload = await UrsaPortal.apiRequest(`/api/v1/clusters/create-options?region=${encodeURIComponent(region)}`);
            setSelectOptions('cc-ssh-key-select', 'Select keypair (or type below)', payload.keypairs || []);
            setSelectOptions('cc-s3-bucket-select', 'Select bucket (or type below)', payload.buckets || []);
            setSelectOptions(
                'cc-region-az',
                'Select availability zone',
                payload.availability_zones || [],
                selectedClusterRegionAz(),
            );
            clearClusterPrechecks();
            if (showSuccess) {
                showToast('success', 'Region options loaded');
            }
        } catch (error) {
            if (showErrors) {
                showToast('error', clusterActionErrorMessage(error, 'Failed to load region options'));
            }
        }
    };
    if (showErrors) {
        await withClusterButtonBusy('cc-load-options-btn', 'Loading region options...', run);
        return;
    }
    await run();
}

function formatSpotStat(value) {
    if (value === null || value === undefined) {
        return '—';
    }
    return Number(value).toFixed(4);
}

const CLUSTER_PRICING_PALETTE = [
    { backgroundColor: 'rgba(0, 217, 166, 0.2)', borderColor: '#00d9a6', outlierColor: '#e94560' },
    { backgroundColor: 'rgba(255, 184, 0, 0.2)', borderColor: '#ffb800', outlierColor: '#e94560' },
    { backgroundColor: 'rgba(67, 33, 202, 0.2)', borderColor: '#4321ca', outlierColor: '#e94560' },
    { backgroundColor: 'rgba(233, 69, 96, 0.18)', borderColor: '#e94560', outlierColor: '#00d9a6' },
    { backgroundColor: 'rgba(46, 196, 182, 0.18)', borderColor: '#2ec4b6', outlierColor: '#4321ca' },
];

function clusterPricingPalette(index) {
    return CLUSTER_PRICING_PALETTE[index % CLUSTER_PRICING_PALETTE.length];
}

function requireRegion() {
    const region = selectedClusterRegion();
    if (!region) {
        showToast('error', 'Select a region first');
        return null;
    }
    return region;
}

function requireRegionAndAz() {
    const region = selectedClusterRegion();
    const regionAz = selectedClusterRegionAz();
    if (!region || !regionAz) {
        showToast('error', 'Select a region and availability zone first');
        return null;
    }
    return { region, regionAz };
}

function renderPartitionVerification(result) {
    const container = document.getElementById('cc-verification-results');
    if (!container) return;

    const failCount = (result.partitions || []).filter((item) => item.status === 'FAIL').length;
    const warnCount = (result.partitions || []).filter((item) => item.status === 'WARN').length;
    const passCount = (result.partitions || []).filter((item) => item.status === 'PASS').length;

    container.innerHTML = `
        <div class="cluster-precheck-header d-flex justify-between align-center flex-wrap gap-md">
            <div>
                <h4 class="card-title" style="margin-bottom: 0.25rem;">Partition Instance Verification</h4>
                <div class="text-sm text-muted">Current Spot availability for ${escapeHtml(result.region_az)} using ${escapeHtml(result.cluster_config_path)}.</div>
            </div>
            <div class="d-flex gap-sm flex-wrap">
                <span class="badge badge-success">${passCount} PASS</span>
                <span class="badge badge-warning">${warnCount} WARN</span>
                <span class="badge badge-error">${failCount} FAIL</span>
            </div>
        </div>
        <div class="table-wrapper">
            <table class="table cluster-precheck-summary">
                <thead>
                    <tr>
                        <th>Partition</th>
                        <th>Status</th>
                        <th>Available</th>
                        <th>Missing</th>
                        <th class="cluster-precheck-detail">Summary</th>
                    </tr>
                </thead>
                <tbody>
                    ${(result.partitions || []).map((item) => `
                        <tr>
                            <td><strong>${escapeHtml(item.partition)}</strong></td>
                            <td><span class="badge badge-${item.status === 'PASS' ? 'success' : item.status === 'WARN' ? 'warning' : 'error'}">${escapeHtml(item.status)}</span></td>
                            <td>${escapeHtml((item.spot_available_instance_types || []).join(', ') || '—')}</td>
                            <td>${escapeHtml((item.missing_instance_types || []).join(', ') || '—')}</td>
                            <td class="cluster-precheck-detail">${escapeHtml(item.summary)}</td>
                        </tr>
                    `).join('')}
                </tbody>
            </table>
        </div>
    `;
    container.classList.remove('d-none');
}

async function verifyPartitionInstances() {
    const selection = requireRegionAndAz();
    if (!selection) return;
    await withClusterButtonBusy('cc-verify-btn', 'Verifying partition instances...', async () => {
        try {
            const result = await runPartitionVerification(selection, true);
            if (result.has_failures) {
                return;
            }
        } catch (error) {
            showToast('error', clusterActionErrorMessage(error, 'Partition verification failed'));
        }
    });
}

async function runPartitionVerification(selection, showSuccessToast = false) {
    const result = await UrsaPortal.apiRequest('/api/v1/clusters/verify-partitions', {
        method: 'POST',
        body: {
            region: selection.region,
            region_az: selection.regionAz,
        },
    });
    clusterVerificationState = result;
    renderPartitionVerification(result);
    updateCreateButtonState();
    if (result.has_failures) {
        showToast('error', 'Verification found blocking partition failures');
    } else if (showSuccessToast) {
        showToast('success', 'Partition verification completed');
    }
    return result;
}

function findPartitionPricingZone(partition, availabilityZone) {
    return (partition.availability_zones || []).find((item) => item.availability_zone === availabilityZone) || null;
}

function flattenPartitionPricingRows(result) {
    const rows = [];
    (result.partitions || []).forEach((partition) => {
        (partition.availability_zones || []).forEach((availabilityZone) => {
            rows.push({
                partition: partition.partition,
                availability_zone: availabilityZone.availability_zone,
                count: availabilityZone.count ?? 0,
                min: availabilityZone.min,
                q1: availabilityZone.q1,
                median: availabilityZone.median,
                mean: availabilityZone.mean,
                q3: availabilityZone.q3,
                max: availabilityZone.max,
                points: availabilityZone.points || [],
            });
        });
    });
    return rows;
}

function renderPartitionPricingChart(partitions, availabilityZones) {
    destroyClusterPricingChart();
    const canvas = document.getElementById('cc-pricing-chart');
    if (!canvas || typeof Chart === 'undefined' || !(availabilityZones || []).length) {
        return;
    }
    clusterPricingChart = new Chart(canvas, {
        type: 'boxplot',
        data: {
            labels: partitions.map((item) => item.partition),
            datasets: availabilityZones.map((availabilityZone, index) => {
                const palette = clusterPricingPalette(index);
                return {
                    type: 'boxplot',
                    label: availabilityZone,
                    data: partitions.map((partition) => {
                        const zone = findPartitionPricingZone(partition, availabilityZone);
                        return (zone?.points || []).map((point) => point.hourly_spot_price);
                    }),
                    backgroundColor: palette.backgroundColor,
                    borderColor: palette.borderColor,
                    borderWidth: 1.5,
                    outlierColor: palette.outlierColor,
                };
            }),
        },
        options: {
            responsive: true,
            maintainAspectRatio: false,
            interaction: {
                mode: 'index',
                intersect: false,
            },
            scales: {
                y: {
                    title: {
                        display: true,
                        text: 'USD / hour',
                    },
                },
            },
            plugins: {
                legend: {
                    position: 'bottom',
                },
                tooltip: {
                    callbacks: {
                        afterBody(items) {
                            const item = items && items[0];
                            if (!item) return [];
                            const partition = partitions[item.dataIndex];
                            const availabilityZone = availabilityZones[item.datasetIndex];
                            const zone = findPartitionPricingZone(partition, availabilityZone);
                            if (!zone) return [];
                            return [
                                `Count: ${zone.count ?? 0}`,
                                `Mean: ${formatSpotStat(zone.mean)}`,
                                `Median: ${formatSpotStat(zone.median)}`,
                                `Min: ${formatSpotStat(zone.min)}`,
                                `Max: ${formatSpotStat(zone.max)}`,
                            ];
                        },
                    },
                },
            },
        },
    });
}

function renderPartitionPricing(result) {
    const container = document.getElementById('cc-pricing-results');
    if (!container) return;
    const pricingRows = flattenPartitionPricingRows(result);
    const pricingSummary = result.availability_zones && result.availability_zones.length
        ? `Hourly Spot price distribution for ${escapeHtml(result.region)} across ${escapeHtml(String(result.availability_zones.length))} availability zone${result.availability_zones.length === 1 ? '' : 's'} using ${escapeHtml(result.cluster_config_path)}.`
        : `No Spot price data was returned for ${escapeHtml(result.region)} using ${escapeHtml(result.cluster_config_path)}.`;
    container.innerHTML = `
        <div class="cluster-precheck-header d-flex justify-between align-center flex-wrap gap-md">
            <div>
                <h4 class="card-title" style="margin-bottom: 0.25rem;">Partition Spot Pricing</h4>
                <div class="text-sm text-muted">${pricingSummary}</div>
            </div>
            <span class="badge badge-outline">${escapeHtml(result.captured_at || '')}</span>
        </div>
        <div class="table-wrapper">
            <table class="table cluster-precheck-summary">
                <thead>
                    <tr>
                        <th>Partition</th>
                        <th>AZ</th>
                        <th>Count</th>
                        <th>Mean</th>
                        <th>Median</th>
                        <th>Min</th>
                        <th>Max</th>
                    </tr>
                </thead>
                <tbody>
                    ${pricingRows.length ? pricingRows.map((item) => `
                        <tr>
                            <td><strong>${escapeHtml(item.partition)}</strong></td>
                            <td>${escapeHtml(item.availability_zone)}</td>
                            <td>${escapeHtml(String(item.count ?? 0))}</td>
                            <td>${escapeHtml(formatSpotStat(item.mean))}</td>
                            <td>${escapeHtml(formatSpotStat(item.median))}</td>
                            <td>${escapeHtml(formatSpotStat(item.min))}</td>
                            <td>${escapeHtml(formatSpotStat(item.max))}</td>
                        </tr>
                    `).join('') : `
                        <tr>
                            <td colspan="7" class="text-muted">No Spot price data available for the selected region.</td>
                        </tr>
                    `}
                </tbody>
            </table>
        </div>
        ${pricingRows.length ? `
            <div class="cluster-precheck-chart">
                <canvas id="cc-pricing-chart"></canvas>
            </div>
        ` : ''}
    `;
    container.classList.remove('d-none');
    renderPartitionPricingChart(result.partitions || [], result.availability_zones || []);
}

async function calculatePartitionPricing() {
    const region = requireRegion();
    if (!region) return;
    await withClusterButtonBusy('cc-pricing-btn', 'Calculating partition spot pricing...', async () => {
        try {
            const result = await UrsaPortal.apiRequest('/api/v1/clusters/partition-pricing', {
                method: 'POST',
                body: {
                    region,
                },
            });
            renderPartitionPricing(result);
            showToast('success', 'Partition spot pricing calculated');
        } catch (error) {
            showToast('error', clusterActionErrorMessage(error, 'Partition pricing calculation failed'));
        }
    });
}

function awsCheckStatusCounts(result) {
    const summary = result?.report?.summary || {};
    if (Object.keys(summary).length) {
        return {
            pass: Number(summary.PASS || summary.pass || 0),
            warn: Number(summary.WARN || summary.warn || 0),
            fail: Number(summary.FAIL || summary.fail || 0),
        };
    }
    const checks = result?.report?.checks || [];
    return {
        pass: checks.filter((item) => String(item.status || '').toUpperCase() === 'PASS').length,
        warn: checks.filter((item) => String(item.status || '').toUpperCase() === 'WARN').length,
        fail: checks.filter((item) => String(item.status || '').toUpperCase() === 'FAIL').length,
    };
}

function renderAwsCheckAll(result) {
    const container = document.getElementById('cc-aws-check-results');
    if (!container) return;
    latestAwsGapAnalysis = {
        filename: result.gap_analysis_filename || 'daylily-ec-gap-analysis.md',
        content: result.gap_analysis || '',
    };
    const counts = awsCheckStatusCounts(result);
    const statusBadge = result.return_code === 0
        ? '<span class="badge badge-success">PASS</span>'
        : '<span class="badge badge-error">CHECKS NEED ATTENTION</span>';
    const checks = result?.report?.checks || [];
    const interestingChecks = checks
        .filter((item) => String(item.status || '').toUpperCase() !== 'PASS')
        .slice(0, 20);

    container.innerHTML = `
        <div class="cluster-precheck-header d-flex justify-between align-center flex-wrap gap-md">
            <div>
                <h4 class="card-title" style="margin-bottom: 0.25rem;">daylily-ec AWS Check All</h4>
                <div class="text-sm text-muted">Runs <code>daylily-ec --json aws validate all --profile --region-az --config --gap-analysis</code> with the current create inputs.</div>
            </div>
            <div class="d-flex gap-sm flex-wrap">
                ${statusBadge}
                <span class="badge badge-success">${counts.pass} PASS</span>
                <span class="badge badge-warning">${counts.warn} WARN</span>
                <span class="badge badge-error">${counts.fail} FAIL</span>
            </div>
        </div>
        ${interestingChecks.length ? `
            <div class="table-wrapper mb-md">
                <table class="table cluster-precheck-summary">
                    <thead>
                        <tr>
                            <th>Status</th>
                            <th>Check</th>
                            <th class="cluster-precheck-detail">Remediation</th>
                        </tr>
                    </thead>
                    <tbody>
                        ${interestingChecks.map((item) => `
                            <tr>
                                <td><span class="badge badge-${String(item.status || '').toUpperCase() === 'FAIL' ? 'error' : 'warning'}">${escapeHtml(item.status || '')}</span></td>
                                <td><code>${escapeHtml(item.id || item.check_id || '')}</code></td>
                                <td class="cluster-precheck-detail">${escapeHtml(item.remediation || item.message || '')}</td>
                            </tr>
                        `).join('')}
                    </tbody>
                </table>
            </div>
        ` : ''}
        <div class="d-flex gap-md flex-wrap mb-md">
            <button class="btn btn-outline btn-sm" type="button" onclick="downloadAwsGapAnalysis()" ${latestAwsGapAnalysis.content ? '' : 'disabled'}>
                <i class="fas fa-download"></i> Download gap_analysis
            </button>
            <span class="text-sm text-muted">Exit code ${escapeHtml(String(result.return_code))}</span>
        </div>
        <div class="cluster-probe-block">
            <div class="text-muted text-xs">stdout</div>
            <pre class="cluster-probe-pre">${escapeHtml(result.stdout || '—')}</pre>
        </div>
        <div class="cluster-probe-block">
            <div class="text-muted text-xs">stderr</div>
            <pre class="cluster-probe-pre">${escapeHtml(result.stderr || '—')}</pre>
        </div>
        <div class="cluster-probe-block">
            <div class="text-muted text-xs">gap_analysis</div>
            <pre class="cluster-probe-pre">${escapeHtml(result.gap_analysis || '—')}</pre>
        </div>
    `;
    container.classList.remove('d-none');
}

function downloadAwsGapAnalysis() {
    if (!latestAwsGapAnalysis?.content) {
        showToast('error', 'No gap_analysis content is available');
        return;
    }
    const blob = new Blob([latestAwsGapAnalysis.content], { type: 'text/markdown' });
    const link = document.createElement('a');
    link.href = URL.createObjectURL(blob);
    link.download = latestAwsGapAnalysis.filename;
    document.body.appendChild(link);
    link.click();
    URL.revokeObjectURL(link.href);
    link.remove();
}

async function runAwsCheckAll() {
    const payload = collectClusterCreatePayload();
    if (!payload.region || !payload.region_az) {
        showToast('error', 'Select a region and availability zone first');
        return;
    }
    if (!payload.aws_profile) {
        showToast('error', 'AWS profile is required for AWS Check All');
        return;
    }
    await withClusterButtonBusy('cc-aws-check-btn', 'Running daylily-ec AWS Check All...', async () => {
        try {
            const result = await UrsaPortal.apiRequest('/api/v1/clusters/aws/check-all', {
                method: 'POST',
                body: payload,
            });
            renderAwsCheckAll(result);
            if (result.return_code === 0) {
                showToast('success', 'AWS Check All passed');
            } else {
                showToast('error', 'AWS Check All found permission or quota gaps');
            }
        } catch (error) {
            showToast('error', clusterActionErrorMessage(error, 'AWS Check All failed'));
        }
    });
}

async function createCluster() {
    await withClusterButtonBusy('cc-submit-btn', 'Starting cluster create job...', async () => {
        try {
            const payload = collectClusterCreatePayload();
            if (!payload.region || !payload.region_az || !payload.cluster_name || !payload.ssh_key_name || !payload.s3_bucket_name) {
                showToast('error', 'Region, availability zone, cluster name, SSH key, and S3 bucket are required');
                return;
            }
            const verification = await runPartitionVerification({ region: payload.region, regionAz: payload.region_az }, false);
            if (verification.has_failures) {
                showToast('error', 'Create blocked until partition verification passes for the selected region and availability zone');
                return;
            }
            const created = await UrsaPortal.apiRequest('/api/v1/clusters', {
                method: 'POST',
                body: payload,
            });
            window.location.assign(`/clusters/jobs/${encodeURIComponent(created.job_euid)}`);
        } catch (error) {
            showToast('error', clusterActionErrorMessage(error, 'Failed to start cluster creation'));
        }
    });
}

async function refreshCreateJobs() {
    try {
        const jobs = await UrsaPortal.apiRequest('/api/v1/clusters/jobs');
        const tbody = document.getElementById('cluster-create-jobs-tbody');
        tbody.innerHTML = jobs.length ? jobs.map((job) => `
            <tr>
                <td><a href="/clusters/jobs/${encodeURIComponent(job.job_euid)}" class="mono">${escapeHtml(job.job_euid)}</a></td>
                <td>${escapeHtml(job.cluster_name)}</td>
                <td>${escapeHtml(job.region_az)}</td>
                <td><span class="badge badge-${escapeHtml(String(job.state || '').toLowerCase().replace(/_/g, '-'))}">${escapeHtml(job.state)}</span></td>
                <td>${job.return_code ?? '—'}</td>
                <td>${escapeHtml(String(job.created_at || '').slice(0, 19))}</td>
                <td><a href="/clusters/jobs/${encodeURIComponent(job.job_euid)}" class="btn btn-outline btn-sm">View</a></td>
            </tr>
        `).join('') : '<tr><td colspan="7" class="text-muted">No create jobs yet.</td></tr>';
    } catch (error) {
        showToast('error', clusterActionErrorMessage(error, 'Failed to refresh cluster jobs'));
    }
}

async function deleteCluster(clusterName, region) {
    try {
        const plan = await UrsaPortal.apiRequest(`/api/v1/clusters/${encodeURIComponent(clusterName)}/delete-plan?region=${encodeURIComponent(region)}`, {
            method: 'POST',
        });
        const token = String(plan.confirmation_token || '').trim();
        if (!token) {
            throw new Error('Cluster delete plan did not return a confirmation token');
        }
        const dryRunOutput = [plan.dry_run_stdout, plan.dry_run_stderr]
            .map((value) => String(value || '').trim())
            .filter(Boolean)
            .join('\n')
            .slice(0, 1800);
        const prompt = dryRunOutput
            ? `Delete cluster ${clusterName} in ${region}?\n\nDry run output:\n${dryRunOutput}`
            : `Delete cluster ${clusterName} in ${region}?`;
        if (!confirm(prompt)) return;
        const query = new URLSearchParams({
            region,
            confirmation_token: token,
            confirm_cluster_name: clusterName,
        });
        await UrsaPortal.apiRequest(`/api/v1/clusters/${encodeURIComponent(clusterName)}?${query.toString()}`, {
            method: 'DELETE',
        });
        showToast('success', `Cluster delete submitted for ${clusterName}`);
        window.location.reload();
    } catch (error) {
        showToast('error', clusterActionErrorMessage(error, `Failed to delete cluster ${clusterName}`));
    }
}

document.addEventListener('DOMContentLoaded', () => {
    updateCreateButtonState();
    loadClusterInventory(false).catch((error) => {
        showToast('error', clusterActionErrorMessage(error, 'Failed to load cluster inventory'));
    });
    if (selectedClusterRegion()) {
        loadCreateOptions(false, false).catch(() => {});
    }
});
</script>
{% endblock %}