{% extends "base.html" %}
{% from "components/help_macros.html" import tooltip, help_panel, help_step, help_tip %}

{% set active_page = 'metrics' %}

{% block title %}Token Usage & Context Analytics - Deep Research System{% endblock %}

{% block extra_head %}
<link rel="stylesheet" href="/static/css/context_overflow.css">
{% endblock %}

{% block content %}
<div class="ldr-page active" id="context-overflow">
    <div class="ldr-page-header" style="display: flex; align-items: center; justify-content: space-between;">
        <div style="display: flex; align-items: center; gap: 1rem;">
            <h1>Token Usage & Context Analytics</h1>
            <a href="/metrics" style="background: var(--primary-color); color: white; padding: 8px 16px; border-radius: 6px; text-decoration: none; font-size: 0.875rem;">
                ← Back to Metrics
            </a>
        </div>
        <div class="ldr-time-range-selector">
            <span style="color: var(--text-secondary); margin-right: 0.5rem; font-size: 0.875rem;">Time Range:</span>
            <button class="ldr-time-range-btn" data-period="7d">7D</button>
            <button class="ldr-time-range-btn active" data-period="30d">30D</button>
            <button class="ldr-time-range-btn" data-period="3m">3M</button>
            <button class="ldr-time-range-btn" data-period="1y">1Y</button>
            <button class="ldr-time-range-btn" data-period="all">All</button>
        </div>
    </div>

    <!-- Understanding Context Limits Help Panel -->
    {% call help_panel('context-how', 'Understanding Context Limits', icon='exclamation-triangle', collapsed=true, dismissible=true) %}
    <div class="ldr-help-steps">
        {{ help_step(1, "Context Window", "Every AI model has a limit on how much text it can process at once (measured in tokens). This is the 'context window'.") }}
        {{ help_step(2, "Truncation", "When your input exceeds the limit, text is cut off (truncated). High truncation means lost information.") }}
        {{ help_step(3, "Prevention", "Use larger-context models, reduce query size, or be more specific in your research questions.") }}
    </div>
    {{ help_tip("<strong>Color guide:</strong> Green (<10% truncation) = Good. Orange (10-20%) = Monitor. Red (>20%) = Action needed.") }}
    {% endcall %}

    <div id="loading" class="ldr-loading-spinner" style="text-align: center; padding: 3rem;" role="status">
        <i aria-hidden="true" class="fas fa-spinner fa-spin fa-2x"></i>
        <p>Loading token usage data...</p>
    </div>

    <div id="content" style="display: none;">
        <!-- Warning Banner if high truncation rate -->
        <div id="warning-banner" class="ldr-warning-banner" style="display: none;" role="alert">
            <i aria-hidden="true" class="fas fa-exclamation-triangle"></i>
            <div>
                <strong>High Context Truncation Detected!</strong>
                <p style="margin: 0.25rem 0 0 0; font-size: 0.875rem;">
                    <span id="warning-rate"></span>% of your requests are hitting context limits.
                    Consider adjusting your context window settings or using shorter prompts.
                </p>
            </div>
        </div>

        <!-- Token Summary Overview Cards (always populated) -->
        <div class="ldr-card">
            <div class="card-header">
                <h2><i aria-hidden="true" class="fas fa-chart-line"></i> Token Usage Overview</h2>
            </div>
            <div class="ldr-card-content">
                <div class="ldr-overflow-grid">
                    <div class="ldr-overflow-card">
                        <div class="ldr-metric-icon">
                            <i aria-hidden="true" class="fas fa-exchange-alt"></i>
                        </div>
                        <div class="ldr-metric-label">Total Requests</div>
                        <div class="ldr-metric-value" id="summary-total-requests">0</div>
                        <div style="font-size: 0.875rem; color: var(--text-secondary);">
                            LLM calls tracked
                        </div>
                    </div>

                    <div class="ldr-overflow-card">
                        <div class="ldr-metric-icon">
                            <i aria-hidden="true" class="fas fa-coins"></i>
                        </div>
                        <div class="ldr-metric-label">Total Tokens</div>
                        <div class="ldr-metric-value" id="summary-total-tokens">0</div>
                        <div style="font-size: 0.875rem; color: var(--text-secondary);">
                            prompt + completion
                        </div>
                    </div>

                    <div class="ldr-overflow-card">
                        <div class="ldr-metric-icon">
                            <i aria-hidden="true" class="fas fa-arrow-right"></i>
                        </div>
                        <div class="ldr-metric-label">Avg Prompt Tokens</div>
                        <div class="ldr-metric-value" id="summary-avg-prompt">0</div>
                        <div style="font-size: 0.875rem; color: var(--text-secondary);">
                            per request
                        </div>
                    </div>

                    <div class="ldr-overflow-card">
                        <div class="ldr-metric-icon">
                            <i aria-hidden="true" class="fas fa-arrow-up"></i>
                        </div>
                        <div class="ldr-metric-label">Max Prompt Tokens</div>
                        <div class="ldr-metric-value" id="summary-max-prompt">0</div>
                        <div style="font-size: 0.875rem; color: var(--text-secondary);">
                            largest single request
                        </div>
                    </div>
                </div>
            </div>
        </div>

        <!-- Token Usage Over Time Chart (always has data) -->
        <div class="ldr-card" style="margin-top: 2rem;">
            <div class="card-header">
                <h2><i aria-hidden="true" class="fas fa-chart-bar"></i> Token Usage Over Time</h2>
            </div>
            <div class="ldr-card-content">
                <div class="ldr-chart-container">
                    <canvas id="token-usage-chart"></canvas>
                </div>
            </div>
        </div>

        <!-- Model Usage Breakdown (always has data) -->
        <div class="ldr-card" style="margin-top: 2rem;">
            <div class="card-header">
                <h2><i aria-hidden="true" class="fas fa-robot"></i> Model Usage Breakdown</h2>
            </div>
            <div class="ldr-card-content">
                <div id="model-token-stats">
                    <!-- Populated dynamically -->
                </div>
            </div>
        </div>

        <!-- Phase Breakdown Chart (always has data) -->
        <div class="ldr-card" style="margin-top: 2rem;">
            <div class="card-header">
                <h2><i aria-hidden="true" class="fas fa-tasks"></i> Token Usage by Phase</h2>
            </div>
            <div class="ldr-card-content">
                <div id="phase-breakdown-container">
                    <div class="ldr-chart-container" style="height: 300px;">
                        <canvas id="phase-chart"></canvas>
                    </div>
                </div>
            </div>
        </div>

        <!-- Context Overflow Section (conditional) -->
        <div id="context-overflow-section">
            <!-- Populated dynamically: either context overflow details or info message -->
        </div>

        <!-- All Requests Detailed Table -->
        <div class="ldr-card" style="margin-top: 2rem;">
            <div class="card-header" style="display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 0.5rem;">
                <h2><i aria-hidden="true" class="fas fa-table"></i> All Requests Details</h2>
                <input type="text" id="requests-search" class="ldr-table-search" placeholder="Search by model, query, phase...">
            </div>
            <div class="ldr-card-content">
                <div style="overflow-x: auto;">
                    <table id="requests-table" class="ldr-data-table" style="width: 100%; font-size: 0.9rem;">
                        <thead>
                            <tr>
                                <th scope="col" class="ldr-sortable" data-sort="timestamp">Timestamp</th>
                                <th scope="col" class="ldr-sortable" data-sort="research_id">Research ID</th>
                                <th scope="col" class="ldr-sortable" data-sort="model">Model</th>
                                <th scope="col" class="ldr-sortable" data-sort="provider">Provider</th>
                                <th scope="col" class="ldr-sortable" data-sort="research_phase">Phase</th>
                                <th scope="col" class="ldr-sortable" data-sort="prompt_tokens">Prompt Tokens</th>
                                <th scope="col" class="ldr-sortable" data-sort="completion_tokens">Completion Tokens</th>
                                <th scope="col" class="ldr-sortable" data-sort="total_tokens">Total Tokens</th>
                                <th scope="col" class="ldr-sortable" data-sort="context_limit">Context Limit</th>
                                <th scope="col" class="ldr-sortable" data-sort="context_truncated">Truncated</th>
                                <th scope="col" class="ldr-sortable" data-sort="tokens_truncated">Tokens Lost</th>
                                <th scope="col">Query</th>
                            </tr>
                        </thead>
                        <tbody id="requests-tbody">
                            <!-- Populated dynamically -->
                        </tbody>
                    </table>
                </div>
                <div class="ldr-pagination" id="requests-pagination">
                    <button class="ldr-pagination-btn" id="pagination-prev" disabled>← Previous</button>
                    <span class="ldr-pagination-info" id="pagination-info">Page 1 of 1</span>
                    <button class="ldr-pagination-btn" id="pagination-next" disabled>Next →</button>
                </div>
            </div>
        </div>
    </div>
</div>
{% endblock %}

{% block component_scripts %}
<script>
(function() {
    let tokenUsageChart = null;
    let phaseChart = null;
    let contextChart = null;
    let distributionChart = null;
    let currentPeriod = '30d';
    let currentPage = 1;
    let totalPages = 1;
    let currentPageData = [];  // Current page data for client-side sort/filter
    let sortColumn = null;
    let sortDirection = 'desc';
    let searchTerm = '';

    // Format number with commas
    function formatNumber(num) {
        if (num == null) return '0';
        return Math.round(num).toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
    }

    // Escape HTML to prevent XSS when rendering user-controlled data
    function escapeHtml(text) {
        if (text == null) return '';
        const div = document.createElement('div');
        div.textContent = String(text);
        return div.innerHTML;
    }

    // Helper function to format model names for better display
    function formatModelName(model) {
        if (!model) return 'N/A';
        // Shorten long model names for display
        if (model.includes('/')) {
            const parts = model.split('/');
            if (parts[0].toLowerCase() === 'google' && parts[1].includes('gemini')) {
                return `Gemini ${parts[1].replace('gemini-', '').replace('-001', '')}`;
            } else if (parts[0].toLowerCase() === 'anthropic' && parts[1].includes('claude')) {
                return `Claude ${parts[1].replace('claude-', '')}`;
            } else if (parts[0].toLowerCase() === 'openai') {
                return parts[1].toUpperCase();
            } else if (parts[0].toLowerCase() === 'ai21' && parts[1].includes('jamba')) {
                return `Jamba ${parts[1].replace('jamba-', '')}`;
            }
            return `${parts[1].substring(0, 20)}${parts[1].length > 20 ? '...' : ''}`;
        }
        return model.length > 25 ? model.substring(0, 25) + '...' : model;
    }

    // Helper function to format provider names
    function formatProviderName(provider) {
        if (!provider) return 'N/A';
        const providerMap = {
            'openai_endpoint': 'OpenAI Endpoint',
            'ollama': 'Ollama',
            'openai': 'OpenAI',
            'anthropic': 'Anthropic',
            'google': 'Google',
            'openrouter': 'OpenRouter',
            'lmstudio': 'LM Studio',
            'llamacpp': 'Llama.cpp',
            'xai': 'xAI',
            'ionos': 'IONOS'
        };
        return providerMap[provider.toLowerCase()] || provider;
    }

    // Load context overflow data
    async function loadContextData(period = currentPeriod, page = 1) {
        try {
            document.getElementById('loading').style.display = 'block';
            document.getElementById('content').style.display = 'none';

            const response = await fetch(`/metrics/api/context-overflow?period=${period}&page=${page}&per_page=50`);
            if (!response.ok) {
                throw new Error('Failed to load data');
            }

            const data = await response.json();

            if (data.status === 'success') {
                document.getElementById('loading').style.display = 'none';
                document.getElementById('content').style.display = 'block';
                displayContextData(data);
            } else {
                showError();
            }
        } catch (error) {
            console.error('Error loading context data:', error);
            showError();
        }
    }

    // Display context overflow data
    function displayContextData(data) {
        const { overview, token_summary, model_token_stats, phase_breakdown,
                model_stats, recent_truncated, chart_data, context_limits,
                all_requests, pagination } = data;

        // Update token summary overview cards
        document.getElementById('summary-total-requests').textContent = formatNumber(token_summary.total_requests);
        document.getElementById('summary-total-tokens').textContent = formatNumber(token_summary.total_tokens);
        document.getElementById('summary-avg-prompt').textContent = formatNumber(token_summary.avg_prompt_tokens);
        document.getElementById('summary-max-prompt').textContent = formatNumber(token_summary.max_prompt_tokens);

        // Show warning if high truncation rate
        if (overview.truncation_rate > 20) {
            document.getElementById('warning-banner').style.display = 'flex';
            document.getElementById('warning-rate').textContent = overview.truncation_rate.toFixed(1);
        } else {
            document.getElementById('warning-banner').style.display = 'none';
        }

        // Create token usage over time chart
        createTokenUsageChart(chart_data);

        // Display model token stats table
        displayModelTokenStats(model_token_stats);

        // Create phase breakdown chart
        createPhaseChart(phase_breakdown);

        // Display context overflow section (conditional)
        displayContextOverflowSection(overview, model_stats, recent_truncated, chart_data, context_limits);

        // Populate all requests table with pagination
        currentPageData = all_requests || [];
        if (pagination) {
            currentPage = pagination.page;
            totalPages = pagination.total_pages;
        }
        renderRequestsTable();
        updatePaginationControls();
    }

    // Create token usage over time stacked bar chart
    function createTokenUsageChart(chartData) {
        const ctx = document.getElementById('token-usage-chart').getContext('2d');

        if (tokenUsageChart) {
            tokenUsageChart.destroy();
        }

        if (!chartData || chartData.length === 0) {
            tokenUsageChart = new Chart(ctx, {
                type: 'bar',
                data: { labels: [], datasets: [] },
                options: {
                    responsive: true,
                    maintainAspectRatio: false,
                    plugins: {
                        title: { display: true, text: 'No token data available yet' }
                    }
                }
            });
            return;
        }

        // Aggregate by day
        const dailyData = {};
        chartData.forEach(d => {
            const day = d.timestamp.substring(0, 10); // YYYY-MM-DD
            if (!dailyData[day]) {
                dailyData[day] = { prompt: 0, completion: 0 };
            }
            dailyData[day].prompt += d.prompt_tokens || 0;
            dailyData[day].completion += d.completion_tokens || 0;
        });

        const sortedDays = Object.keys(dailyData).sort();
        const promptTokens = sortedDays.map(d => dailyData[d].prompt);
        const completionTokens = sortedDays.map(d => dailyData[d].completion);

        tokenUsageChart = new Chart(ctx, {
            type: 'bar',
            data: {
                labels: sortedDays,
                datasets: [{
                    label: 'Prompt Tokens',
                    data: promptTokens,
                    backgroundColor: 'rgba(54, 162, 235, 0.8)',
                    borderColor: 'rgba(54, 162, 235, 1)',
                    borderWidth: 1
                }, {
                    label: 'Completion Tokens',
                    data: completionTokens,
                    backgroundColor: 'rgba(255, 99, 132, 0.8)',
                    borderColor: 'rgba(255, 99, 132, 1)',
                    borderWidth: 1
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
                scales: {
                    x: {
                        stacked: true,
                    },
                    y: {
                        stacked: true,
                        beginAtZero: true
                    }
                },
                plugins: {
                    legend: {
                        position: 'bottom'
                    },
                    tooltip: {
                        callbacks: {
                            label: function(context) {
                                let label = context.dataset.label || '';
                                if (label) {
                                    label += ': ';
                                }
                                label += formatNumber(context.parsed.y);
                                return label;
                            }
                        }
                    }
                }
            }
        });
    }

    // Display model token stats as a table
    function displayModelTokenStats(stats) {
        const container = document.getElementById('model-token-stats');

        if (!stats || stats.length === 0) {
            container.innerHTML = `
                <div class="ldr-no-data-message">
                    <i aria-hidden="true" class="fas fa-robot"></i>
                    <p>No model data available yet</p>
                </div>
            `;
            return;
        }

        let html = `
            <div style="overflow-x: auto;">
                <table class="ldr-data-table">
                    <thead>
                        <tr>
                            <th>Model</th>
                            <th>Provider</th>
                            <th style="text-align: right;">Requests</th>
                            <th style="text-align: right;">Total Tokens</th>
                            <th style="text-align: right;">Avg Prompt</th>
                            <th style="text-align: right;">Max Prompt</th>
                            <th style="text-align: right;">Avg Response Time</th>
                        </tr>
                    </thead>
                    <tbody>
        `;

        stats.forEach(s => {
            const responseTime = s.avg_response_time_ms > 0
                ? `${formatNumber(s.avg_response_time_ms)} ms`
                : '<span class="ldr-text-muted">N/A</span>';

            html += `
                <tr>
                    <td title="${escapeHtml(s.model)}">${escapeHtml(formatModelName(s.model))}</td>
                    <td>${escapeHtml(formatProviderName(s.provider))}</td>
                    <td style="text-align: right;">${formatNumber(s.total_requests)}</td>
                    <td style="text-align: right; font-weight: bold;">${formatNumber(s.total_tokens)}</td>
                    <td style="text-align: right;">${formatNumber(s.avg_prompt)}</td>
                    <td style="text-align: right;">${formatNumber(s.max_prompt)}</td>
                    <td style="text-align: right;">${responseTime}</td>
                </tr>
            `;
        });

        html += '</tbody></table></div>';
        container.innerHTML = html;
    }

    // Create phase breakdown horizontal bar chart
    function createPhaseChart(phaseData) {
        const container = document.getElementById('phase-breakdown-container');
        const ctx = document.getElementById('phase-chart').getContext('2d');

        if (phaseChart) {
            phaseChart.destroy();
        }

        if (!phaseData || phaseData.length === 0) {
            container.innerHTML = `
                <div class="ldr-no-data-message">
                    <i aria-hidden="true" class="fas fa-tasks"></i>
                    <p>No phase data available yet</p>
                </div>
            `;
            return;
        }

        // Sort by total tokens descending
        const sorted = [...phaseData].sort((a, b) => b.total_tokens - a.total_tokens);
        const labels = sorted.map(p => p.phase);
        const tokens = sorted.map(p => p.total_tokens);
        const counts = sorted.map(p => p.count);

        const colors = [
            'rgba(54, 162, 235, 0.8)',
            'rgba(255, 99, 132, 0.8)',
            'rgba(75, 192, 192, 0.8)',
            'rgba(255, 205, 86, 0.8)',
            'rgba(153, 102, 255, 0.8)',
            'rgba(255, 159, 64, 0.8)',
            'rgba(201, 203, 207, 0.8)'
        ];

        phaseChart = new Chart(ctx, {
            type: 'bar',
            data: {
                labels: labels,
                datasets: [{
                    label: 'Total Tokens',
                    data: tokens,
                    backgroundColor: labels.map((_, i) => colors[i % colors.length]),
                    borderWidth: 1
                }]
            },
            options: {
                indexAxis: 'y',
                responsive: true,
                maintainAspectRatio: false,
                scales: {
                    x: {
                        beginAtZero: true
                    }
                },
                plugins: {
                    legend: {
                        display: false
                    },
                    tooltip: {
                        callbacks: {
                            label: function(context) {
                                const idx = context.dataIndex;
                                return `${formatNumber(context.parsed.x)} tokens (${counts[idx]} requests)`;
                            }
                        }
                    }
                }
            }
        });
    }

    // Display context overflow section conditionally
    function displayContextOverflowSection(overview, modelStats, recentTruncated, chartData, contextLimits) {
        const section = document.getElementById('context-overflow-section');

        if (overview.requests_with_context_data > 0) {
            // Show context overflow analytics
            section.innerHTML = `
                <!-- Context Overflow Overview -->
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-percentage"></i> Context Overflow Overview</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div class="ldr-overflow-grid">
                            <div class="ldr-overflow-card">
                                <div class="ldr-metric-icon" style="background: var(--warning-color);">
                                    <i aria-hidden="true" class="fas fa-percentage"></i>
                                </div>
                                <div class="ldr-metric-label">Truncation Rate</div>
                                <div class="ldr-metric-value" id="truncation-rate">0%</div>
                                <div style="font-size: 0.875rem; color: var(--text-secondary);">
                                    <span id="truncated-count">0</span> of <span id="total-count">0</span> requests
                                </div>
                            </div>
                            <div class="ldr-overflow-card">
                                <div class="ldr-metric-icon" style="background: var(--error-color);">
                                    <i aria-hidden="true" class="fas fa-cut"></i>
                                </div>
                                <div class="ldr-metric-label">Avg Tokens Lost</div>
                                <div class="ldr-metric-value" id="avg-tokens-lost">0</div>
                                <div style="font-size: 0.875rem; color: var(--text-secondary);">
                                    per truncated request
                                </div>
                            </div>
                            <div class="ldr-overflow-card">
                                <div class="ldr-metric-icon">
                                    <i aria-hidden="true" class="fas fa-microchip"></i>
                                </div>
                                <div class="ldr-metric-label">Models Tracked</div>
                                <div class="ldr-metric-value" id="models-tracked">0</div>
                                <div style="font-size: 0.875rem; color: var(--text-secondary);">
                                    with context data
                                </div>
                            </div>
                            <div class="ldr-overflow-card">
                                <div class="ldr-metric-icon">
                                    <i aria-hidden="true" class="fas fa-database"></i>
                                </div>
                                <div class="ldr-metric-label">Data Coverage</div>
                                <div class="ldr-metric-value" id="data-coverage">0%</div>
                                <div style="font-size: 0.875rem; color: var(--text-secondary);">
                                    requests with context info
                                </div>
                            </div>
                        </div>
                    </div>
                </div>

                <!-- Scatter Chart: Request Size vs Context Limit -->
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-chart-area"></i> Request Size vs Context Limits</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div class="ldr-chart-container">
                            <canvas id="context-chart"></canvas>
                        </div>
                        <div style="margin-top: 1rem; padding: 1rem; background: var(--bg-tertiary); border-radius: 0.375rem;">
                            <p style="margin: 0; color: var(--text-secondary); font-size: 0.875rem;">
                                <i aria-hidden="true" class="fas fa-info-circle" style="color: var(--primary-color); margin-right: 0.5rem;"></i>
                                <strong>Chart Guide:</strong> Points above the red line indicate truncated requests.
                                The horizontal lines show different context limits configured for your models.
                            </p>
                        </div>
                    </div>
                </div>

                <!-- Model Context Usage -->
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-robot"></i> Model Context Usage</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div id="model-stats"></div>
                    </div>
                </div>

                <!-- Recent Truncated Requests -->
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-exclamation-triangle"></i> Recent Truncated Requests</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div id="truncated-list"></div>
                    </div>
                </div>

                <!-- Context Limit Distribution -->
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-chart-pie"></i> Context Limit Distribution</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div style="display: grid; grid-template-columns: 1fr 2fr; gap: 2rem;">
                            <div>
                                <h3>Configured Limits</h3>
                                <div id="context-limits-list"></div>
                            </div>
                            <div>
                                <canvas id="distribution-chart" style="max-height: 300px;"></canvas>
                            </div>
                        </div>
                    </div>
                </div>
            `;

            // Now populate the dynamically created elements
            document.getElementById('truncation-rate').textContent = `${overview.truncation_rate}%`;
            document.getElementById('truncated-count').textContent = formatNumber(overview.truncated_requests);
            document.getElementById('total-count').textContent = formatNumber(overview.requests_with_context_data);
            document.getElementById('avg-tokens-lost').textContent = formatNumber(Math.round(overview.avg_tokens_truncated));
            document.getElementById('models-tracked').textContent = modelStats.length;

            const coverage = overview.total_requests > 0
                ? Math.round((overview.requests_with_context_data / overview.total_requests) * 100)
                : 0;
            document.getElementById('data-coverage').textContent = `${coverage}%`;

            displayModelStats(modelStats);
            displayTruncatedRequests(recentTruncated);
            displayContextLimits(contextLimits);
            createContextChart(chartData);
            createDistributionChart(contextLimits);
        } else {
            // Show informative message
            section.innerHTML = `
                <div class="ldr-card" style="margin-top: 2rem;">
                    <div class="card-header">
                        <h2><i aria-hidden="true" class="fas fa-info-circle"></i> Context Overflow Analytics</h2>
                    </div>
                    <div class="ldr-card-content">
                        <div class="ldr-info-message">
                            <i aria-hidden="true" class="fas fa-info-circle"></i>
                            <p>Context limits will be tracked automatically for new research sessions.</p>
                            <p style="font-size: 0.875rem; margin-top: 0.5rem;">
                                Your local model context window is configured in <a href="/settings" style="color: var(--primary-color);">Settings</a> > LLM.
                            </p>
                        </div>
                    </div>
                </div>
            `;
        }
    }

    // Render the requests table from currentPageData (with client-side sort and filter)
    function renderRequestsTable() {
        const tbody = document.getElementById('requests-tbody');
        let data = [...currentPageData];

        // Client-side search filter
        if (searchTerm) {
            data = data.filter(req => {
                const haystack = [
                    req.model, req.provider, req.research_query,
                    req.research_phase, req.research_id
                ].filter(Boolean).join(' ').toLowerCase();
                return haystack.includes(searchTerm);
            });
        }

        // Client-side sort (on current page data)
        if (sortColumn) {
            data.sort((a, b) => {
                let valA = a[sortColumn];
                let valB = b[sortColumn];

                // Handle nulls
                if (valA == null) valA = '';
                if (valB == null) valB = '';

                // Numeric columns
                if (typeof valA === 'number' || typeof valB === 'number') {
                    valA = Number(valA) || 0;
                    valB = Number(valB) || 0;
                    return sortDirection === 'asc' ? valA - valB : valB - valA;
                }

                // Boolean
                if (typeof valA === 'boolean') {
                    return sortDirection === 'asc' ? (valA ? 1 : 0) - (valB ? 1 : 0) : (valB ? 1 : 0) - (valA ? 1 : 0);
                }

                // String comparison
                valA = String(valA).toLowerCase();
                valB = String(valB).toLowerCase();
                const cmp = valA.localeCompare(valB);
                return sortDirection === 'asc' ? cmp : -cmp;
            });
        }

        if (data.length === 0) {
            tbody.innerHTML = `
                <tr>
                    <td colspan="12" style="text-align: center; padding: 2rem;">
                        ${searchTerm ? 'No matching requests found' : 'No request data available'}
                    </td>
                </tr>
            `;
            return;
        }

        let tableRows = '';
        data.forEach(req => {
            const timestamp = new Date(req.timestamp).toLocaleString();
            const truncatedBadge = req.context_truncated
                ? '<span class="ldr-text-error">Yes</span>'
                : '<span class="ldr-text-success">No</span>';
            const contextLimit = req.context_limit
                ? formatNumber(req.context_limit)
                : '<span class="ldr-text-muted">Unknown</span>';
            const tokensLost = req.tokens_truncated || 0;
            const truncationClass = tokensLost > 0 ? 'ldr-text-error' : 'ldr-text-secondary';
            const researchLink = req.research_id
                ? `<a href="/details/${encodeURIComponent(req.research_id)}" style="color: var(--primary-color); font-family: monospace; font-size: 0.8rem;">${escapeHtml(req.research_id.substring(0, 8))}</a>`
                : 'N/A';

            tableRows += `
                <tr>
                    <td style="white-space: nowrap;">${timestamp}</td>
                    <td>${researchLink}</td>
                    <td title="${escapeHtml(req.model)}">${escapeHtml(formatModelName(req.model))}</td>
                    <td>${escapeHtml(formatProviderName(req.provider))}</td>
                    <td>${escapeHtml(req.research_phase || 'N/A')}</td>
                    <td style="text-align: right;">${formatNumber(req.prompt_tokens)}</td>
                    <td style="text-align: right;">${formatNumber(req.completion_tokens)}</td>
                    <td style="text-align: right; font-weight: bold;">${formatNumber(req.total_tokens)}</td>
                    <td style="text-align: right;">${contextLimit}</td>
                    <td style="text-align: center;">${truncatedBadge}</td>
                    <td style="text-align: right;" class="${truncationClass}">${formatNumber(tokensLost)}</td>
                    <td style="max-width: 200px; overflow: hidden; text-overflow: ellipsis;" title="${escapeHtml(req.research_query)}">${escapeHtml(req.research_query || 'N/A')}</td>
                </tr>
            `;
        });

        tbody.innerHTML = tableRows;
    }

    // Update pagination controls
    function updatePaginationControls() {
        const prevBtn = document.getElementById('pagination-prev');
        const nextBtn = document.getElementById('pagination-next');
        const info = document.getElementById('pagination-info');

        prevBtn.disabled = currentPage <= 1;
        nextBtn.disabled = currentPage >= totalPages;
        info.textContent = `Page ${currentPage} of ${totalPages}`;
    }

    // Display model-specific stats (context overflow section)
    function displayModelStats(modelStats) {
        const container = document.getElementById('model-stats');

        if (modelStats.length === 0) {
            container.innerHTML = `
                <div class="ldr-no-data-message">
                    <i aria-hidden="true" class="fas fa-robot"></i>
                    <p>No model data available</p>
                </div>
            `;
            return;
        }

        container.innerHTML = '';
        modelStats.forEach(stat => {
            const truncationPercent = stat.truncation_rate;
            const card = document.createElement('div');
            card.className = 'ldr-model-card';

            const truncationRateClass = truncationPercent > 20 ? 'ldr-truncation-high' : truncationPercent > 10 ? 'ldr-truncation-medium' : 'ldr-truncation-low';
            card.innerHTML = `
                <div class="ldr-model-header">
                    <div>
                        <div class="ldr-model-name" title="${escapeHtml(stat.model)}">${escapeHtml(formatModelName(stat.model))}</div>
                        <div class="ldr-model-provider">${escapeHtml(formatProviderName(stat.provider))}</div>
                    </div>
                    <div style="text-align: right;">
                        <div style="font-size: 1.25rem; font-weight: 600;" class="${truncationRateClass}">
                            ${truncationPercent}%
                        </div>
                        <div style="font-size: 0.75rem;" class="ldr-text-secondary">
                            truncation rate
                        </div>
                    </div>
                </div>
                <div style="display: grid; grid-template-columns: repeat(3, 1fr); gap: 1rem; margin-top: 1rem;">
                    <div>
                        <div style="font-size: 0.75rem;" class="ldr-text-secondary">Total Requests</div>
                        <div style="font-weight: 600;">${formatNumber(stat.total_requests)}</div>
                    </div>
                    <div>
                        <div style="font-size: 0.75rem;" class="ldr-text-secondary">Truncated</div>
                        <div style="font-weight: 600;" class="ldr-text-error">${formatNumber(stat.truncated_count)}</div>
                    </div>
                    <div>
                        <div style="font-size: 0.75rem;" class="ldr-text-secondary">Context Limit</div>
                        <div style="font-weight: 600;">${stat.avg_context_limit ? formatNumber(stat.avg_context_limit) : '<span class="ldr-text-muted" style="font-size: 0.85rem;">Unknown</span>'}</div>
                    </div>
                </div>
                <div class="ldr-progress-bar">
                    <div class="ldr-progress-fill" style="width: ${Math.min(truncationPercent, 100)}%;"></div>
                </div>
            `;

            container.appendChild(card);
        });
    }

    // Display recent truncated requests
    function displayTruncatedRequests(requests) {
        const container = document.getElementById('truncated-list');

        if (requests.length === 0) {
            container.innerHTML = `
                <div class="ldr-no-data-message">
                    <i aria-hidden="true" class="fas fa-check-circle ldr-icon-success"></i>
                    <p>No truncated requests found</p>
                    <p style="font-size: 0.875rem;">All your requests are within context limits!</p>
                </div>
            `;
            return;
        }

        container.innerHTML = '';
        const table = document.createElement('table');
        table.className = 'ldr-data-table';

        table.innerHTML = `
            <thead>
                <tr>
                    <th>Time</th>
                    <th>Query</th>
                    <th>Model</th>
                    <th>Prompt Tokens</th>
                    <th>Context Limit</th>
                    <th>Tokens Lost</th>
                    <th>Action</th>
                </tr>
            </thead>
            <tbody>
                ${requests.map(req => `
                    <tr>
                        <td>${new Date(req.timestamp).toLocaleString()}</td>
                        <td style="max-width: 300px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">
                            ${escapeHtml(req.research_query || 'N/A')}
                        </td>
                        <td>${escapeHtml(formatModelName(req.model))}</td>
                        <td>${formatNumber(req.actual_tokens || req.prompt_tokens)}</td>
                        <td><span class="ldr-context-limit-badge">${formatNumber(req.context_limit)}</span></td>
                        <td class="ldr-text-error" style="font-weight: 600;">${formatNumber(req.tokens_truncated || 0)}</td>
                        <td>
                            <a href="/details/${encodeURIComponent(req.research_id)}" style="color: var(--primary-color);">
                                View Details
                            </a>
                        </td>
                    </tr>
                `).join('')}
            </tbody>
        `;

        const tableContainer = document.createElement('div');
        tableContainer.className = 'ldr-table-container';
        tableContainer.appendChild(table);
        container.appendChild(tableContainer);
    }

    // Display context limits distribution
    function displayContextLimits(limits) {
        const container = document.getElementById('context-limits-list');

        if (limits.length === 0) {
            container.innerHTML = '<p style="color: var(--text-secondary);">No context limit data</p>';
            return;
        }

        container.innerHTML = '';
        limits.forEach(limit => {
            const item = document.createElement('div');
            item.style.cssText = 'display: flex; justify-content: space-between; padding: 0.5rem 0; border-bottom: 1px solid var(--border-color);';
            item.innerHTML = `
                <span>${formatNumber(limit.limit)} tokens</span>
                <span style="font-weight: 600;">${limit.count} requests</span>
            `;
            container.appendChild(item);
        });
    }

    // Create main context scatter chart
    function createContextChart(chartData) {
        const ctx = document.getElementById('context-chart').getContext('2d');

        if (contextChart) {
            contextChart.destroy();
        }

        if (!chartData || chartData.length === 0) {
            contextChart = new Chart(ctx, {
                type: 'scatter',
                data: { datasets: [{ label: 'No data', data: [] }] },
                options: {
                    responsive: true,
                    maintainAspectRatio: false,
                    plugins: { title: { display: true, text: 'No context data available yet' } }
                }
            });
            return;
        }

        // Get unique context limits for horizontal lines
        const contextLimits = [...new Set(chartData.map(d => d.context_limit).filter(l => l))];

        const normalData = chartData.filter(d => !d.truncated).map(d => ({
            x: new Date(d.timestamp),
            y: d.actual_prompt_tokens || d.prompt_tokens || 0,
            research_id: d.research_id,
            model: d.model
        }));

        const truncatedData = chartData.filter(d => d.truncated).map(d => ({
            x: new Date(d.timestamp),
            y: d.actual_prompt_tokens || d.prompt_tokens || 0,
            research_id: d.research_id,
            model: d.model,
            tokens_truncated: d.tokens_truncated
        }));

        const datasets = [
            {
                label: 'Normal Requests',
                data: normalData,
                backgroundColor: 'rgba(76, 175, 80, 0.6)',
                borderColor: 'rgba(76, 175, 80, 1)',
                pointRadius: 4,
                pointHoverRadius: 6
            },
            {
                label: 'Truncated Requests',
                data: truncatedData,
                backgroundColor: 'rgba(244, 67, 54, 0.6)',
                borderColor: 'rgba(244, 67, 54, 1)',
                pointRadius: 6,
                pointHoverRadius: 8
            }
        ];

        // Add horizontal lines for context limits
        contextLimits.forEach((limit, index) => {
            const color = `hsl(${index * 60}, 70%, 50%)`;
            datasets.push({
                label: `Context Limit: ${formatNumber(limit)}`,
                data: [
                    { x: new Date(Math.min(...chartData.map(d => new Date(d.timestamp)))), y: limit },
                    { x: new Date(Math.max(...chartData.map(d => new Date(d.timestamp)))), y: limit }
                ],
                type: 'line',
                borderColor: color,
                borderWidth: 2,
                borderDash: [10, 5],
                fill: false,
                pointRadius: 0,
                pointHoverRadius: 0,
                tension: 0
            });
        });

        contextChart = new Chart(ctx, {
            type: 'scatter',
            data: { datasets },
            options: {
                responsive: true,
                maintainAspectRatio: false,
                scales: {
                    x: {
                        type: 'time',
                        title: { display: true, text: 'Time' }
                    },
                    y: {
                        title: { display: true, text: 'Token Count' },
                        beginAtZero: true
                    }
                },
                plugins: {
                    tooltip: {
                        callbacks: {
                            label: function(context) {
                                const point = context.raw;
                                let label = `${context.dataset.label}: ${formatNumber(point.y)} tokens`;
                                if (point.model) {
                                    label += ` (${point.model})`;
                                }
                                if (point.tokens_truncated) {
                                    label += ` - Lost: ${formatNumber(point.tokens_truncated)} tokens`;
                                }
                                return label;
                            }
                        }
                    },
                    legend: { position: 'bottom' }
                },
                onClick: (event, elements) => {
                    if (elements.length > 0) {
                        const dataPoint = elements[0];
                        const dataset = contextChart.data.datasets[dataPoint.datasetIndex];
                        const point = dataset.data[dataPoint.index];
                        if (point.research_id) {
                            window.location.href = `/details/${encodeURIComponent(point.research_id)}`;
                        }
                    }
                }
            }
        });
    }

    // Create distribution chart
    function createDistributionChart(limits) {
        const ctx = document.getElementById('distribution-chart').getContext('2d');

        if (distributionChart) {
            distributionChart.destroy();
        }

        if (!limits || limits.length === 0) {
            return;
        }

        distributionChart = new Chart(ctx, {
            type: 'doughnut',
            data: {
                labels: limits.map(l => `${formatNumber(l.limit)} tokens`),
                datasets: [{
                    data: limits.map(l => l.count),
                    backgroundColor: [
                        'rgba(255, 99, 132, 0.8)',
                        'rgba(54, 162, 235, 0.8)',
                        'rgba(255, 205, 86, 0.8)',
                        'rgba(75, 192, 192, 0.8)',
                        'rgba(153, 102, 255, 0.8)'
                    ]
                }]
            },
            options: {
                responsive: true,
                maintainAspectRatio: false,
                plugins: {
                    legend: { position: 'right' },
                    tooltip: {
                        callbacks: {
                            label: function(context) {
                                const total = context.dataset.data.reduce((a, b) => a + b, 0);
                                const percentage = ((context.parsed / total) * 100).toFixed(1);
                                return `${context.label}: ${context.parsed} requests (${percentage}%)`;
                            }
                        }
                    }
                }
            }
        });
    }

    // Show error state
    function showError() {
        document.getElementById('loading').style.display = 'none';
        document.getElementById('content').innerHTML = `
            <div class="ldr-card">
                <div class="ldr-card-content">
                    <div class="ldr-no-data-message">
                        <i aria-hidden="true" class="fas fa-exclamation-circle ldr-icon-error"></i>
                        <p>Error loading token usage data</p>
                        <p style="font-size: 0.875rem;">Please try refreshing the page</p>
                    </div>
                </div>
            </div>
        `;
        document.getElementById('content').style.display = 'block';
    }

    // Handle time range changes
    function handleTimeRangeChange(period) {
        currentPeriod = period;
        currentPage = 1;

        document.querySelectorAll('.ldr-time-range-btn').forEach(btn => {
            btn.classList.remove('active');
        });
        document.querySelector(`[data-period="${period}"]`).classList.add('active');

        loadContextData(period, 1);
    }

    // Initialize when DOM is loaded
    function initialize() {
        // Set up time range buttons
        document.querySelectorAll('.ldr-time-range-btn').forEach(btn => {
            btn.addEventListener('click', function() {
                handleTimeRangeChange(this.getAttribute('data-period'));
            });
        });

        // Set up pagination buttons
        document.getElementById('pagination-prev').addEventListener('click', function() {
            if (currentPage > 1) {
                currentPage--;
                loadContextData(currentPeriod, currentPage);
            }
        });

        document.getElementById('pagination-next').addEventListener('click', function() {
            if (currentPage < totalPages) {
                currentPage++;
                loadContextData(currentPeriod, currentPage);
            }
        });

        // Set up search input
        document.getElementById('requests-search').addEventListener('input', function() {
            searchTerm = this.value.trim().toLowerCase();
            renderRequestsTable();
        });

        // Set up sortable column headers
        document.querySelectorAll('.ldr-sortable').forEach(th => {
            th.addEventListener('click', function() {
                const col = this.getAttribute('data-sort');

                // Toggle direction or set new column
                if (sortColumn === col) {
                    sortDirection = sortDirection === 'asc' ? 'desc' : 'asc';
                } else {
                    sortColumn = col;
                    sortDirection = 'desc';
                }

                // Update header classes
                document.querySelectorAll('.ldr-sortable').forEach(h => {
                    h.classList.remove('ldr-sort-asc', 'ldr-sort-desc');
                });
                this.classList.add(sortDirection === 'asc' ? 'ldr-sort-asc' : 'ldr-sort-desc');

                renderRequestsTable();
            });
        });

        // Load initial data
        loadContextData();
    }

    // Check if DOM is already loaded
    if (document.readyState === 'loading') {
        document.addEventListener('DOMContentLoaded', initialize);
    } else {
        initialize();
    }
})();
</script>
{% endblock %}