{% extends "core/base.html" %}

{% load static %}
{% load i18n %}

{% block breadcrumbs %}
    <div class="breadcrumbs">
        <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
        {% if title %} &rsaquo; {{ title }}{% endif %}
    </div>
{% endblock %}

{% block extra_head %}
    <link rel="stylesheet" href="{% static 'add.css' %}" />
{% endblock %}

{% block body %}
    <div style="max-width: 1440px; margin: auto; float: none">
        <br/><br/>
        {% if stdout %}
            <h1>Add new URLs to your archive: results</h1>
            <pre id="stdout">
                {{ stdout | safe }}
                <br/><br/>
            </pre>
            <br/>
            <center>
                <a href="/add" id="submit">&nbsp; Add more URLs ➕</a>
            </center>
        {% else %}
            <div id="in-progress" style="display: none;">
                <center><h3>Creating crawl and queueing snapshots...</h3>
                    <p>Your crawl is being created. The background runner will process URLs and create snapshots.</p>
                    <br/>
                    <div class="loader"></div>
                    <br/>
                    Check the server log or the <a href="/admin/core/archiveresult/?o=-1">Log</a> page for detailed progress...
                </center>
            </div>
            <form id="add-form" method="POST" class="p-form">{% csrf_token %}
                <center>
                    <h1>Create a new Crawl</h1>
                </center>
                <div class="crawl-explanation">
                    <p>
                        A <strong>Crawl</strong> is a job that processes URLs and creates <strong>Snapshots</strong> (archived copies) for each URL discovered.
                        The settings below apply to the entire crawl and all snapshots it creates.
                    </p>
                </div>

                <!-- Basic fields -->
                <div class="form-section">
                    <div class="form-field">
                        <div class="url-workbench">
                            <div class="url-editor-column">
                                <div class="field-header">
                                    {{ form.url.label_tag }}
                                    <div id="url-counter" class="url-counter">0 URLs detected</div>
                                </div>
                                <div class="url-editor-shell">
                                    <pre id="url-highlight-layer" class="url-highlight-layer" aria-hidden="true"></pre>
                                    {{ form.url }}
                                </div>
                            </div>
                            <aside class="detected-urls-panel" aria-live="polite">
                                <div class="detected-urls-header">
                                    <strong>Filter URLs</strong>
                                    <span id="detected-urls-summary" class="detected-urls-summary">No URLs yet</span>
                                </div>
                                <div id="detected-urls-list" class="detected-urls-list">
                                    <div class="detected-urls-empty">Paste URLs, markdown, JSON, or prose to preview matches here.</div>
                                </div>
                            </aside>
                        </div>
                        {% if form.url.errors %}
                            <div class="error">{{ form.url.errors }}</div>
                        {% endif %}
                        <div class="help-text">
                            Enter URLs to archive, as one per line, CSV, JSON, or embedded in text (e.g. markdown, HTML, etc.). Examples:<br/>
                            <code>https://example.com</code><br/>
                            <code>https://news.ycombinator.com,https://news.google.com</code><br/>
                            <code>[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)</code>
                        </div>
                    </div>

                    <div class="form-field">
                        {{ form.tag.label_tag }}
                        {{ form.tag }}
                        {% if form.tag.errors %}
                            <div class="error">{{ form.tag.errors }}</div>
                        {% endif %}
                        <div class="help-text">Tags will be applied to all snapshots created by this crawl.</div>
                    </div>

                    <div class="settings-row">
                        <div class="form-field">
                            {{ form.depth.label_tag }}
                            {{ form.depth }}
                            {% if form.depth.errors %}
                                <div class="error">{{ form.depth.errors }}</div>
                            {% endif %}
                            <div class="help-text">Controls how many links deep the crawl will follow from the starting URLs.</div>

                            <div class="crawl-limit-grid">
                                <div class="crawl-limit-field">
                                    {{ form.max_urls.label_tag }}
                                    {{ form.max_urls }}
                                    {% if form.max_urls.errors %}
                                        <div class="error">{{ form.max_urls.errors }}</div>
                                    {% endif %}
                                    <div class="help-text">0 means unlimited. When set, only the first N filtered URLs will be snapshotted.</div>
                                </div>

                                <div class="crawl-limit-field">
                                    {{ form.crawl_max_size.label_tag }}
                                    {{ form.crawl_max_size }}
                                    {% if form.crawl_max_size.errors %}
                                        <div class="error">{{ form.crawl_max_size.errors }}</div>
                                    {% endif %}
                                    <div class="help-text">0 means unlimited across the whole crawl. Accepts bytes or units like <code>45mb</code> and <code>1gb</code>.</div>
                                </div>

                                <div class="crawl-limit-field">
                                    {{ form.snapshot_max_size.label_tag }}
                                    {{ form.snapshot_max_size }}
                                    {% if form.snapshot_max_size.errors %}
                                        <div class="error">{{ form.snapshot_max_size.errors }}</div>
                                    {% endif %}
                                    <div class="help-text">0 means unlimited per snapshot. Accepts bytes or units like <code>45mb</code> and <code>1gb</code>.</div>
                                </div>

                                <div class="crawl-limit-field">
                                    {{ form.delete_after.label_tag }}
                                    {{ form.delete_after }}
                                    {% if form.delete_after.errors %}
                                        <div class="error">{{ form.delete_after.errors }}</div>
                                    {% endif %}
                                    <div class="help-text">0 keeps rows forever. Use <code>1h</code>, <code>7d</code>, <code>4w</code>, <code>6mo</code>, or <code>1y</code>.</div>
                                </div>

                                <div class="crawl-limit-field">
                                    {{ form.crawl_max_concurrent_snapshots.label_tag }}
                                    {{ form.crawl_max_concurrent_snapshots }}
                                    {% if form.crawl_max_concurrent_snapshots.errors %}
                                        <div class="error">{{ form.crawl_max_concurrent_snapshots.errors }}</div>
                                    {% endif %}
                                    <div class="help-text">Caps how many snapshots from this crawl archive at the same time.</div>
                                </div>
                            </div>
                        </div>

                        <div class="form-field url-filters-field">
                            {{ form.url_filters }}
                            {% if form.url_filters.errors %}
                                <div class="error">{{ form.url_filters.errors }}</div>
                            {% endif %}
                        </div>
                    </div>

                    <div class="form-field">
                        {{ form.notes.label_tag }}
                        {{ form.notes }}
                        {% if form.notes.errors %}
                            <div class="error">{{ form.notes.errors }}</div>
                        {% endif %}
                        <div class="help-text">Optional description for this crawl (visible in the admin interface).</div>
                    </div>

                    <div class="form-field">
                        {{ form.persona.label_tag }}
                        {{ form.persona }}
                        {% if form.persona.errors %}
                            <div class="error">{{ form.persona.errors }}</div>
                        {% endif %}
                        <div class="help-text">
                            Authentication profile (Chrome profile, cookies, etc.) to use when accessing URLs.
                            <a href="/admin/personas/persona/add/" target="_blank">Create new persona / import from Chrome →</a>
                        </div>
                    </div>
                </div>

                <!-- Plugins section -->
                <div class="form-section">
                    <h3>Crawl Plugins</h3>
                    <p class="section-description">
                        Select which archiving methods to run for all snapshots in this crawl. If none selected, all available plugins will be used.
                        <a href="/admin/environment/plugins/" target="_blank">View plugin details →</a>
                    </p>

                    <div class="plugin-presets">
                        <span class="preset-label">Quick Select:</span>
                        <button type="button" class="preset-btn" data-preset="text-only">📄 Text Only</button>
                        <button type="button" class="preset-btn" data-preset="select-all">✓ Select All</button>
                        <button type="button" class="preset-btn" data-preset="clear-all">✗ Clear All</button>
                    </div>

                    {% include "core/plugin_config_grid.html" with plugin_groups=form.plugin_groups %}
                </div>

                <!-- Advanced options (collapsible) -->
                <div class="form-section">
                    <details class="advanced-section">
                        <summary><h3>Advanced Crawl Options</h3></summary>
                        <p class="section-description">Additional settings that control how this crawl processes URLs and creates snapshots.</p>

                        <div class="form-field">
                            {{ form.schedule.label_tag }}
                            {{ form.schedule }}
                            {% if form.schedule.errors %}
                                <div class="error">{{ form.schedule.errors }}</div>
                            {% endif %}
                            <div class="help-text">
                                Optional: Schedule this crawl to repeat automatically. Examples:<br/>
                                <code>daily</code> - Run once per day<br/>
                                <code>weekly</code> - Run once per week<br/>
                                <code>0 */6 * * *</code> - Every 6 hours (cron format)<br/>
                                <code>0 0 * * 0</code> - Every Sunday at midnight (cron format)
                            </div>
                        </div>

                        <div class="form-field checkbox-field">
                            {{ form.index_only }}
                            {{ form.index_only.label_tag }}
                            {% if form.index_only.errors %}
                                <div class="error">{{ form.index_only.errors }}</div>
                            {% endif %}
                            <div class="help-text">Create the crawl and queue snapshots without running archive plugins yet.</div>
                        </div>

                        <div class="form-field">
                            {{ form.config.label_tag }}
                            {{ form.config }}
                            {% if form.config.errors %}
                                <div class="error">{{ form.config.errors }}</div>
                            {% endif %}
                            <div class="help-text">
                                Override any config option for this crawl (e.g., TIMEOUT, USER_AGENT, CHROME_BINARY, etc.). <code>URL_ALLOWLIST</code>, <code>URL_DENYLIST</code>, and <code>ENABLED_PLUGINS</code> are updated automatically from the fields above.
                            </div>
                        </div>
                    </details>
                </div>

                <center>
                    <button role="submit" id="submit">&nbsp; Create Crawl and Start Archiving ➕</button>
                </center>
            </form>
            <br/><br/><br/>
            <center id="delay-warning" style="display: none">
                <small>(you will be redirected to your new Crawl page momentarily, it's safe to close this page at any time)</small>
            </center>
            {% if absolute_add_path %}
            <!-- <center id="bookmarklet">
              <p>Bookmark this link to quickly add to your archive:
                <a href="javascript:void(window.open('{{ absolute_add_path }}?url='+encodeURIComponent(document.location.href)));">Add to ArchiveBox</a></p>
            </center> -->
            {% endif %}
            <script>
                // URL preview / counter
                const urlTextarea = document.querySelector('textarea[name="url"]');
                const urlCounter = document.getElementById('url-counter');
                const urlHighlightLayer = document.getElementById('url-highlight-layer');
                const detectedURLsPanel = document.querySelector('.detected-urls-panel');
                const detectedURLsSummary = document.getElementById('detected-urls-summary');
                const detectedURLsList = document.getElementById('detected-urls-list');
                const maxURLsInput = document.querySelector('input[name="max_urls"]');
                const sharedURLPattern = urlTextarea.dataset.urlRegex || '(?=(http[s]?://(?:[a-zA-Z]|[0-9]|[-_$@.&+!*\\(\\),]|[^\\u0000-\\u007F])+[^\\]\\[<>"\\\'\\s]+))';
                const previewURLPattern = (
                    sharedURLPattern && sharedURLPattern.startsWith('(?=(') && sharedURLPattern.endsWith('))')
                ) ? sharedURLPattern.slice(4, -2) : sharedURLPattern;
                const urlMatchRegex = previewURLPattern ? new RegExp(`^${previewURLPattern}`, 'i') : null;
                const fallbackURLMatchRegex = /^https?:\/\/[^\s<>"']+/i;
                const urlValidationRegex = previewURLPattern ? new RegExp(`^${previewURLPattern}$`, 'i') : fallbackURLMatchRegex;
                const urlPalette = [
                    { bg: 'rgba(0, 115, 230, 0.16)', border: 'rgba(0, 115, 230, 0.5)' },
                    { bg: 'rgba(214, 90, 49, 0.18)', border: 'rgba(214, 90, 49, 0.55)' },
                    { bg: 'rgba(5, 132, 95, 0.17)', border: 'rgba(5, 132, 95, 0.52)' },
                    { bg: 'rgba(142, 68, 173, 0.17)', border: 'rgba(142, 68, 173, 0.52)' },
                    { bg: 'rgba(191, 90, 242, 0.16)', border: 'rgba(191, 90, 242, 0.5)' },
                    { bg: 'rgba(210, 105, 30, 0.16)', border: 'rgba(210, 105, 30, 0.5)' },
                ];
                const requiredSearchPlugin = '{{ required_search_plugin|default:""|escapejs }}';
                const pluginDependencyMap = JSON.parse('{{ plugin_dependency_map_json|default:"{}"|escapejs }}');
                const personaConfigMap = JSON.parse('{{ persona_config_map_json|default:"{}"|escapejs }}');
                const personaSelect = document.querySelector('select[name="persona"]');

                function dispatchChange(el) {
                    el.dispatchEvent(new Event('input', { bubbles: true }));
                    el.dispatchEvent(new Event('change', { bubbles: true }));
                }

                function getSavedFormState() {
                    try {
                        return JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}');
                    } catch (e) {
                        return {};
                    }
                }

                function getStoredPluginPreference(pluginName) {
                    if (!pluginName) return false;
                    const state = getSavedFormState();
                    const key = `search_plugins:${pluginName}`;
                    if (!Object.prototype.hasOwnProperty.call(state, key)) {
                        return null;
                    }
                    return Boolean(state[key]);
                }

                function getRequiredSearchCheckbox() {
                    if (!requiredSearchPlugin) return null;
                    return document.querySelector(`.plugin-section-toggle[value="${requiredSearchPlugin}"]`);
                }

                function getPluginCheckbox(pluginName) {
                    if (!pluginName) return null;
                    return document.querySelector(`.plugin-section-toggle[value="${pluginName}"]`);
                }

                function getRequiredPlugins(pluginName) {
                    const requiredPlugins = pluginDependencyMap[pluginName];
                    return Array.isArray(requiredPlugins) ? requiredPlugins : [];
                }

                function getDependentPlugins(pluginName) {
                    return Object.entries(pluginDependencyMap)
                        .filter(([, requiredPlugins]) => Array.isArray(requiredPlugins) && requiredPlugins.includes(pluginName))
                        .map(([dependentPlugin]) => dependentPlugin);
                }

                function getConfigEditorRows() {
                    return document.getElementById('id_config_rows');
                }

                function getConfigUpdater() {
                    return window.updateHiddenField_id_config || null;
                }

                function findConfigRow(key) {
                    const rows = getConfigEditorRows();
                    if (!rows) return null;
                    const matches = Array.from(rows.querySelectorAll('.key-value-row')).filter(row => {
                        const keyInput = row.querySelector('.kv-key');
                        return keyInput && keyInput.value.trim() === key;
                    });
                    return matches.length ? matches[0] : null;
                }

                function addConfigRow() {
                    if (typeof window.addKeyValueRow_id_config !== 'function') return null;
                    window.addKeyValueRow_id_config();
                    const rows = getConfigEditorRows();
                    return rows ? rows.lastElementChild : null;
                }

                function setConfigRow(key, value) {
                    const rows = getConfigEditorRows();
                    const updater = getConfigUpdater();
                    if (!rows || !updater) return;

                    let row = findConfigRow(key);
                    if (value === undefined || value === null || value === '') {
                        if (row) {
                            row.remove();
                            updater();
                        }
                        return;
                    }

                    if (!row) {
                        row = addConfigRow();
                    }
                    if (!row) return;

                    const keyInput = row.querySelector('.kv-key');
                    const valueInput = row.querySelector('.kv-value');
                    if (!keyInput || !valueInput) return;

                    keyInput.value = key;
                    valueInput.value = typeof value === 'string' ? value : JSON.stringify(value);
                    keyInput.dispatchEvent(new Event('input', { bubbles: true }));
                    valueInput.dispatchEvent(new Event('input', { bubbles: true }));
                    updater();
                }

                function syncEnabledPluginsConfig() {
                    const selectedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
                        .map(cb => cb.value)
                        .filter(Boolean)
                        .sort((left, right) => left.localeCompare(right));
                    setConfigRow('ENABLED_PLUGINS', selectedPlugins.join(','));
                }

                function replaceConfigRows(config) {
                    const rows = getConfigEditorRows();
                    const updater = getConfigUpdater();
                    if (!rows || !updater || !config || typeof config !== 'object') return;
                    Array.from(rows.querySelectorAll('.key-value-row')).forEach(row => row.remove());
                    Object.entries(config).forEach(([key, value]) => setConfigRow(key, value));
                    if (!rows.querySelector('.key-value-row')) addConfigRow();
                    updater();
                }

                function applyPersonaConfig(personaName) {
                    const personaData = personaConfigMap[personaName];
                    if (!personaData) return;
                    const concurrencyInput = document.querySelector('input[name="crawl_max_concurrent_snapshots"]');
                    const concurrencyValue = personaData.effective_config?.CRAWL_MAX_CONCURRENT_SNAPSHOTS;
                    if (concurrencyInput && concurrencyValue) {
                        concurrencyInput.value = concurrencyValue;
                    }
                    const deleteAfterInput = document.querySelector('input[name="delete_after"]');
                    const deleteAfterValue = personaData.effective_config?.DELETE_AFTER;
                    if (deleteAfterInput && deleteAfterValue !== undefined && deleteAfterValue !== null) {
                        deleteAfterInput.value = deleteAfterValue || '0';
                    }
                    if (typeof window.archiveboxSetPluginConfigValues === 'function') {
                        window.archiveboxSetPluginConfigValues(personaData.effective_config || {}, personaData.binary_urls || {});
                    }
                    replaceConfigRows(personaData.config || {});
                    updateURLPreview();
                }

                function ensureRequiredPluginsChecked(pluginName, visited = new Set()) {
                    if (!pluginName || visited.has(pluginName)) {
                        return;
                    }
                    visited.add(pluginName);

                    getRequiredPlugins(pluginName).forEach(requiredPlugin => {
                        const requiredCheckbox = getPluginCheckbox(requiredPlugin);
                        if (!requiredCheckbox) {
                            return;
                        }
                        requiredCheckbox.checked = true;
                        dispatchChange(requiredCheckbox);
                        ensureRequiredPluginsChecked(requiredPlugin, visited);
                    });
                }

                function uncheckDependentPlugins(pluginName, visited = new Set()) {
                    if (!pluginName || visited.has(pluginName)) {
                        return;
                    }
                    visited.add(pluginName);

                    getDependentPlugins(pluginName).forEach(dependentPlugin => {
                        const dependentCheckbox = getPluginCheckbox(dependentPlugin);
                        if (!dependentCheckbox) {
                            return;
                        }
                        dependentCheckbox.checked = false;
                        dispatchChange(dependentCheckbox);
                        uncheckDependentPlugins(dependentPlugin, visited);
                    });
                }

                function normalizePluginSelections() {
                    const checkedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
                        .map(cb => cb.value)
                        .filter(Boolean);
                    checkedPlugins.forEach(pluginName => ensureRequiredPluginsChecked(pluginName));
                }

                function applyRequiredSearchPlugin(preferredValue = undefined) {
                    const requiredCheckbox = getRequiredSearchCheckbox();
                    if (!requiredCheckbox) return;
                    const storedPreference = preferredValue === undefined
                        ? getStoredPluginPreference(requiredSearchPlugin)
                        : preferredValue;
                    if (storedPreference === false) return;
                    if (storedPreference === true || !requiredCheckbox.checked) {
                        requiredCheckbox.checked = true;
                        dispatchChange(requiredCheckbox);
                    }
                }

                function parensAreMatched(string, openChar = '(', closeChar = ')') {
                    let count = 0;
                    for (const char of string) {
                        if (char === openChar) count += 1;
                        else if (char === closeChar) count -= 1;
                        if (count < 0) return false;
                    }
                    return count === 0;
                }

                function fixURLFromMarkdown(url) {
                    let trimmedURL = url;
                    while (trimmedURL && !parensAreMatched(trimmedURL)) {
                        trimmedURL = trimmedURL.slice(0, -1);
                    }

                    urlValidationRegex.lastIndex = 0;
                    return urlValidationRegex.test(trimmedURL) ? trimmedURL : url;
                }

                function nextEmbeddedURLIndex(url) {
                    const httpIndex = url.indexOf('http://', 1);
                    const httpsIndex = url.indexOf('https://', 1);
                    if (httpIndex === -1) return httpsIndex;
                    if (httpsIndex === -1) return httpIndex;
                    return Math.min(httpIndex, httpsIndex);
                }

                function splitCombinedURLMatch(url) {
                    const parts = [];
                    let offset = 0;

                    while (true) {
                        const nextIndex = nextEmbeddedURLIndex(url);
                        if (nextIndex === -1 || url[nextIndex - 1] !== ',') {
                            parts.push({ offset, url });
                            return parts;
                        }

                        parts.push({ offset, url: url.slice(0, nextIndex - 1) });
                        offset += nextIndex;
                        url = url.slice(nextIndex);
                    }
                }

                function escapeHTML(text) {
                    const div = document.createElement('div');
                    div.textContent = text;
                    return div.innerHTML;
                }

                function splitFilterPatterns(value) {
                    const seen = new Set();
                    return String(value || '')
                        .split(/\r?\n+/)
                        .map(pattern => pattern.trim())
                        .filter(pattern => {
                            if (!pattern || seen.has(pattern)) {
                                return false;
                            }
                            seen.add(pattern);
                            return true;
                        });
                }

                function escapeRegex(text) {
                    return String(text || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
                }

                function buildExactURLRegex(url) {
                    return `^${escapeRegex(url)}$`;
                }

                function buildDomainPattern(url) {
                    const normalized = normalizeDomain(url);
                    return normalized || '';
                }

                function getURLExtension(url) {
                    try {
                        const parsed = new URL(url);
                        const pathname = parsed.pathname || '';
                        const lastSegment = pathname.split('/').pop() || '';
                        const match = lastSegment.match(/(\.[A-Za-z0-9]{1,16})$/);
                        return match ? match[1].toLowerCase() : '';
                    } catch (error) {
                        return '';
                    }
                }

                function buildExtensionPattern(url) {
                    const extension = getURLExtension(url);
                    if (!extension) {
                        return '';
                    }
                    return `^https?://.+${escapeRegex(extension)}(?:[?#].*)?$`;
                }

                function normalizeDomain(value) {
                    let candidate = String(value || '').trim().toLowerCase();
                    if (!candidate) {
                        return '';
                    }
                    if (!candidate.includes('://') && !candidate.includes('/')) {
                        candidate = 'https://' + candidate.replace(/^\.+/, '');
                    }
                    try {
                        const parsed = new URL(candidate);
                        if (!parsed.hostname) {
                            return '';
                        }
                        return parsed.port ? `${parsed.hostname}_${parsed.port}` : parsed.hostname;
                    } catch (error) {
                        return '';
                    }
                }

                function isSimpleFilterPattern(pattern) {
                    return /^[\w.*:-]+$/.test(pattern);
                }

                function patternMatchesURL(url, pattern) {
                    const normalizedPattern = String(pattern || '').trim();
                    if (!normalizedPattern) {
                        return false;
                    }

                    if (isSimpleFilterPattern(normalizedPattern)) {
                        const wildcardOnlySubdomains = normalizedPattern.startsWith('*.');
                        const normalizedDomain = normalizeDomain(
                            wildcardOnlySubdomains ? normalizedPattern.slice(2) : normalizedPattern
                        );
                        const normalizedURLDomain = normalizeDomain(url);
                        if (!normalizedDomain || !normalizedURLDomain) {
                            return false;
                        }

                        const patternHost = normalizedDomain.split('_', 1)[0];
                        const urlHost = normalizedURLDomain.split('_', 1)[0];

                        if (wildcardOnlySubdomains) {
                            return urlHost.endsWith(`.${patternHost}`);
                        }
                        if (normalizedURLDomain === normalizedDomain) {
                            return true;
                        }
                        return urlHost === patternHost || urlHost.endsWith(`.${patternHost}`);
                    }

                    try {
                        return new RegExp(normalizedPattern).test(url);
                    } catch (error) {
                        return false;
                    }
                }

                function getConfigOverrideValue(key) {
                    const rows = document.querySelectorAll('#id_config_rows .key-value-row');
                    let value = '';
                    rows.forEach(row => {
                        const keyInput = row.querySelector('.kv-key');
                        const valueInput = row.querySelector('.kv-value');
                        if (keyInput && valueInput && keyInput.value.trim() === key) {
                            value = valueInput.value.trim();
                        }
                    });
                    return value;
                }

                function getEffectiveFilterText(key, fallbackSelector) {
                    const configValue = getConfigOverrideValue(key);
                    if (configValue) {
                        return configValue;
                    }
                    const fallbackField = document.querySelector(fallbackSelector);
                    return fallbackField ? fallbackField.value.trim() : '';
                }

                function getEffectiveFilterPatterns(key, fallbackSelector) {
                    const fallbackField = document.querySelector(fallbackSelector);
                    const fallbackPatterns = splitFilterPatterns(fallbackField ? fallbackField.value : '');
                    const configPatterns = splitFilterPatterns(getConfigOverrideValue(key));
                    return Array.from(new Set([...fallbackPatterns, ...configPatterns]));
                }

                function getFilterState(url) {
                    const allowPatterns = getEffectiveFilterPatterns('URL_ALLOWLIST', 'textarea[name="url_filters_allowlist"]');
                    const denyPatterns = getEffectiveFilterPatterns('URL_DENYLIST', 'textarea[name="url_filters_denylist"]');

                    const matchedDenyPattern = denyPatterns.find(pattern => patternMatchesURL(url, pattern));
                    if (matchedDenyPattern) {
                        return {
                            status: 'denied',
                            message: `Excluded by URL_DENYLIST: ${matchedDenyPattern}`,
                        };
                    }

                    if (allowPatterns.length) {
                        const matchedAllowPattern = allowPatterns.find(pattern => patternMatchesURL(url, pattern));
                        if (matchedAllowPattern) {
                            return {
                                status: 'allowlisted',
                                message: `Included by URL_ALLOWLIST: ${matchedAllowPattern}`,
                            };
                        }
                        return {
                            status: 'filtered',
                            message: 'Not matched by URL_ALLOWLIST',
                        };
                    }

                    return {
                        status: 'neutral',
                        message: 'No URL filters applied',
                    };
                }

                function getCurrentDenyPatterns() {
                    return getEffectiveFilterPatterns('URL_DENYLIST', 'textarea[name="url_filters_denylist"]');
                }

                function getMaxURLsLimit() {
                    if (!maxURLsInput) {
                        return 0;
                    }
                    const rawValue = String(maxURLsInput.value || '').trim();
                    if (!rawValue) {
                        return 0;
                    }
                    const parsed = parseInt(rawValue, 10);
                    return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
                }

                function applyMaxURLsLimit(matches) {
                    const limit = getMaxURLsLimit();
                    if (!limit) {
                        return matches;
                    }

                    let allowedCount = 0;
                    return matches.map(match => {
                        if (match.filter.status === 'denied' || match.filter.status === 'filtered') {
                            return match;
                        }
                        if (allowedCount < limit) {
                            allowedCount += 1;
                            return match;
                        }
                        return {
                            ...match,
                            filter: {
                                status: 'limited',
                                message: `Excluded by max_urls limit (${limit})`,
                            },
                        };
                    });
                }

                function hasDenyPattern(pattern) {
                    return !!pattern && getCurrentDenyPatterns().includes(pattern);
                }

                function findURLStarts(text) {
                    const starts = [];
                    let index = text.indexOf('http');
                    while (index !== -1) {
                        if (text.startsWith('http://', index) || text.startsWith('https://', index)) {
                            starts.push(index);
                        }
                        index = text.indexOf('http', index + 4);
                    }
                    return starts;
                }

                function extractURLs(text) {
                    const urls = [];
                    const skippedStarts = new Set();

                    for (const start of findURLStarts(text)) {
                        if (skippedStarts.has(start)) {
                            continue;
                        }

                        const candidate = text.slice(start);
                        const match = (urlMatchRegex && candidate.match(urlMatchRegex)) || candidate.match(fallbackURLMatchRegex);
                        const rawURL = fixURLFromMarkdown((match && match[0]) || '');
                        if (!rawURL) {
                            continue;
                        }

                        for (const part of splitCombinedURLMatch(rawURL)) {
                            if (!part.url) {
                                continue;
                            }
                            const partStart = start + part.offset;
                            const end = partStart + part.url.length;
                            if (part.offset) {
                                skippedStarts.add(partStart);
                            }
                            urls.push({ url: part.url, start: partStart, end });
                        }
                    }

                    return urls;
                }

                function buildSegmentStyle(activeMatches) {
                    if (!activeMatches.length) {
                        return '';
                    }

                    const isDenied = activeMatches.some(match => match.filter.status === 'denied');

                    if (activeMatches.length === 1) {
                        const color = urlPalette[activeMatches[0].colorIndex];
                        const backgrounds = [];
                        if (isDenied) {
                            backgrounds.push('linear-gradient(180deg, transparent 0 44%, rgba(180, 35, 24, 0.72) 44% 56%, transparent 56% 100%)');
                        }
                        backgrounds.push(color.bg);
                        return `background:${backgrounds.join(', ')}; box-shadow: inset 0 -1px 0 ${color.border};`;
                    }

                    const stripeSize = 100 / activeMatches.length;
                    const stripes = activeMatches.map((match, index) => {
                        const color = urlPalette[match.colorIndex];
                        const start = (index * stripeSize).toFixed(2);
                        const end = ((index + 1) * stripeSize).toFixed(2);
                        return `${color.bg} ${start}% ${end}%`;
                    }).join(', ');
                    const borderShadows = activeMatches.map((match, index) => {
                        const color = urlPalette[match.colorIndex];
                        return `inset 0 ${index + 1}px 0 ${color.border}`;
                    }).join(', ');
                    const backgrounds = [];
                    if (isDenied) {
                        backgrounds.push('linear-gradient(180deg, transparent 0 44%, rgba(180, 35, 24, 0.72) 44% 56%, transparent 56% 100%)');
                    }
                    backgrounds.push(`linear-gradient(180deg, ${stripes})`);
                    return `background:${backgrounds.join(', ')}; box-shadow: ${borderShadows};`;
                }

                function renderHighlightLayer(text, matches) {
                    if (!text) {
                        urlHighlightLayer.innerHTML = '&nbsp;';
                        return;
                    }

                    const boundaries = new Set([0, text.length]);
                    matches.forEach(match => {
                        boundaries.add(match.start);
                        boundaries.add(match.end);
                    });

                    const sortedBoundaries = Array.from(boundaries).sort((left, right) => left - right);
                    const html = [];

                    for (let index = 0; index < sortedBoundaries.length - 1; index += 1) {
                        const start = sortedBoundaries[index];
                        const end = sortedBoundaries[index + 1];
                        const segment = text.slice(start, end);
                        const activeMatches = matches.filter(match => match.start < end && match.end > start);

                        if (!activeMatches.length) {
                            html.push(escapeHTML(segment));
                            continue;
                        }

                        html.push(
                            `<span class="url-highlight-segment" style="${buildSegmentStyle(activeMatches)}">${escapeHTML(segment)}</span>`
                        );
                    }

                    if (text.endsWith('\n')) {
                        html.push('\n');
                    }

                    urlHighlightLayer.innerHTML = html.join('');
                }

                function renderDetectedURLs(matches) {
                    if (!matches.length) {
                        detectedURLsSummary.textContent = 'No URLs yet';
                        detectedURLsList.innerHTML = '<div class="detected-urls-empty">Paste URLs, markdown, JSON, or prose to preview matches here.</div>';
                        return;
                    }

                    const summary = {
                        allowlisted: matches.filter(match => match.filter.status === 'allowlisted').length,
                        denied: matches.filter(match => match.filter.status === 'denied').length,
                        filtered: matches.filter(match => match.filter.status === 'filtered').length,
                        limited: matches.filter(match => match.filter.status === 'limited').length,
                    };
                    summary.total = matches.length - summary.denied - summary.filtered - summary.limited;
                    const summaryParts = [];
                    if (summary.allowlisted) summaryParts.push(`${summary.allowlisted} allowed`);
                    if (summary.denied) summaryParts.push(`${summary.denied} denied`);
                    if (summary.limited) summaryParts.push(`${summary.limited} over limit`);
                    summaryParts.push(`${summary.total} total`);
                    detectedURLsSummary.textContent = summaryParts.join(' • ');

                    detectedURLsList.innerHTML = matches.map(match => {
                        const color = urlPalette[match.colorIndex];
                        const messageHTML = match.filter.status === 'neutral'
                            ? ''
                            : `<div class="detected-url-message">${escapeHTML(match.filter.message)}</div>`;
                        const exactPattern = buildExactURLRegex(match.url);
                        const domainPattern = buildDomainPattern(match.url);
                        const extensionPattern = buildExtensionPattern(match.url);
                        const controls = [
                            {
                                pattern: exactPattern,
                                active: hasDenyPattern(exactPattern),
                                label: 'URL',
                                titleAdd: 'Add exact-match regex to URL_DENYLIST',
                                titleRemove: 'Remove exact-match regex from URL_DENYLIST',
                            },
                            {
                                pattern: domainPattern,
                                active: hasDenyPattern(domainPattern),
                                label: 'Domain',
                                titleAdd: 'Add domain pattern to URL_DENYLIST',
                                titleRemove: 'Remove domain pattern from URL_DENYLIST',
                            },
                            {
                                pattern: extensionPattern,
                                active: hasDenyPattern(extensionPattern),
                                label: getURLExtension(match.url) || 'EXT',
                                titleAdd: 'Add extension regex to URL_DENYLIST',
                                titleRemove: 'Remove extension regex from URL_DENYLIST',
                            },
                        ].filter(control => control.pattern);
                        const controlsHTML = controls.map(control => {
                            const prefix = control.active ? '🚫' : '⚪';
                            const title = control.active ? control.titleRemove : control.titleAdd;
                            return `
                                <button
                                    type="button"
                                    class="detected-url-toggle-btn ${control.active ? 'detected-url-toggle-btn-active' : 'detected-url-toggle-btn-inactive'}"
                                    data-pattern="${escapeHTML(control.pattern)}"
                                    data-active="${control.active ? '1' : '0'}"
                                    title="${title}"
                                >${prefix} ${escapeHTML(control.label)}</button>
                            `;
                        }).join('');
                        return `
                            <div class="detected-url-item detected-url-${match.filter.status}" style="--detected-url-bg:${color.bg}; --detected-url-border:${color.border};">
                                <div class="detected-url-topline">
                                    <div class="detected-url-number">${match.number}</div>
                                    <div class="detected-url-controls">${controlsHTML}</div>
                                </div>
                                <div class="detected-url-body">
                                    <code class="detected-url-value">${escapeHTML(match.url)}</code>
                                    ${messageHTML}
                                </div>
                            </div>
                        `;
                    }).join('');
                }

                function dedupeMatchesForFilterView(matches) {
                    const seen = new Set();
                    return matches.filter(match => {
                        if (seen.has(match.url)) {
                            return false;
                        }
                        seen.add(match.url);
                        return true;
                    }).map((match, index) => ({
                        ...match,
                        number: index + 1,
                    }));
                }

                function setDenylistPatternEntry(pattern, shouldDeny) {
                    if (!pattern) {
                        return;
                    }
                    const denylistField = document.querySelector('textarea[name="url_filters_denylist"]');
                    const existingPatterns = splitFilterPatterns(
                        denylistField ? denylistField.value : getEffectiveFilterText('URL_DENYLIST', '')
                    );
                    const nextPatterns = shouldDeny
                        ? (existingPatterns.includes(pattern) ? existingPatterns : [...existingPatterns, pattern])
                        : existingPatterns.filter(existingPattern => existingPattern !== pattern);
                    const nextValue = nextPatterns.join('\n');
                    if (denylistField) {
                        denylistField.value = nextValue;
                        dispatchChange(denylistField);
                        return;
                    }

                    setConfigRow('URL_DENYLIST', nextValue);
                    updateURLPreview();
                }

                function syncHighlightScroll() {
                    urlHighlightLayer.scrollTop = urlTextarea.scrollTop;
                    urlHighlightLayer.scrollLeft = urlTextarea.scrollLeft;
                }

                function syncHighlightMetrics() {
                    const style = window.getComputedStyle(urlTextarea);
                    urlHighlightLayer.style.font = style.font;
                    urlHighlightLayer.style.fontFamily = style.fontFamily;
                    urlHighlightLayer.style.fontSize = style.fontSize;
                    urlHighlightLayer.style.fontWeight = style.fontWeight;
                    urlHighlightLayer.style.fontStyle = style.fontStyle;
                    urlHighlightLayer.style.lineHeight = style.lineHeight;
                    urlHighlightLayer.style.letterSpacing = style.letterSpacing;
                    urlHighlightLayer.style.wordSpacing = style.wordSpacing;
                    urlHighlightLayer.style.textIndent = style.textIndent;
                    urlHighlightLayer.style.textTransform = style.textTransform;
                    urlHighlightLayer.style.tabSize = style.tabSize;
                    urlHighlightLayer.style.paddingTop = style.paddingTop;
                    urlHighlightLayer.style.paddingRight = style.paddingRight;
                    urlHighlightLayer.style.paddingBottom = style.paddingBottom;
                    urlHighlightLayer.style.paddingLeft = style.paddingLeft;
                }

                function syncPreviewPanelHeight() {
                    detectedURLsPanel.style.height = `${urlTextarea.offsetHeight}px`;
                }

                function updateURLPreview() {
                    const matches = extractURLs(urlTextarea.value).map((match, index) => ({
                        ...match,
                        number: index + 1,
                        colorIndex: index % urlPalette.length,
                        filter: getFilterState(match.url),
                    }));
                    const uniqueMatches = dedupeMatchesForFilterView(matches);
                    const limitedMatches = applyMaxURLsLimit(uniqueMatches);
                    const count = matches.length;
                    urlCounter.textContent = `${count} URL${count !== 1 ? 's' : ''} detected`;
                    urlCounter.className = count > 0 ? 'url-counter url-counter-positive' : 'url-counter';
                    renderHighlightLayer(urlTextarea.value, matches);
                    renderDetectedURLs(limitedMatches);
                    syncHighlightScroll();
                }

                urlTextarea.addEventListener('input', updateURLPreview);
                urlTextarea.addEventListener('scroll', syncHighlightScroll);
                window.addEventListener('resize', syncHighlightMetrics);
                window.addEventListener('resize', syncPreviewPanelHeight);
                if (window.ResizeObserver) {
                    new ResizeObserver(function() {
                        syncHighlightMetrics();
                        syncPreviewPanelHeight();
                        syncHighlightScroll();
                    }).observe(urlTextarea);
                }

                document.getElementById('add-form').addEventListener('input', function(event) {
                    if (
                        event.target === urlTextarea ||
                        event.target === maxURLsInput ||
                        event.target.matches('textarea[name="url_filters_allowlist"]') ||
                        event.target.matches('textarea[name="url_filters_denylist"]') ||
                        event.target.matches('#id_config_rows .kv-key') ||
                        event.target.matches('#id_config_rows .kv-value')
                    ) {
                        updateURLPreview();
                    }
                });

                document.getElementById('add-form').addEventListener('change', function(event) {
                    if (
                        event.target.matches('textarea[name="url_filters_allowlist"]') ||
                        event.target.matches('textarea[name="url_filters_denylist"]') ||
                        event.target.matches('input[name="max_urls"]') ||
                        event.target.matches('input[name="url_filters_same_domain_only"]') ||
                        event.target.matches('#id_config_rows .kv-key') ||
                        event.target.matches('#id_config_rows .kv-value')
                    ) {
                        updateURLPreview();
                    }
                });

                detectedURLsList.addEventListener('click', function(event) {
                    const button = event.target.closest('.detected-url-toggle-btn');
                    if (!button || button.disabled) {
                        return;
                    }
                    setDenylistPatternEntry(button.dataset.pattern || '', button.dataset.active !== '1');
                });

                syncHighlightMetrics();
                syncPreviewPanelHeight();
                updateURLPreview(); // Initial count + preview

                // Plugin Presets
                const presetConfigs = {
                    'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon']
                };

                document.querySelectorAll('.preset-btn').forEach(btn => {
                    btn.addEventListener('click', function() {
                        const preset = this.dataset.preset;
                        const allCheckboxes = document.querySelectorAll('.plugin-section-toggle');
                        const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);

                        if (preset === 'select-all') {
                            allCheckboxes.forEach(cb => {
                                cb.checked = true;
                                dispatchChange(cb);
                            });
                        } else if (preset === 'clear-all') {
                            allCheckboxes.forEach(cb => {
                                cb.checked = false;
                                dispatchChange(cb);
                            });
                        } else if (presetConfigs[preset]) {
                            const pluginsToSelect = presetConfigs[preset];
                            allCheckboxes.forEach(cb => {
                                cb.checked = pluginsToSelect.includes(cb.value);
                                dispatchChange(cb);
                            });
                        }

                        normalizePluginSelections();
                        applyRequiredSearchPlugin(requiredSearchPreference);
                        syncEnabledPluginsConfig();
                        updateChromeToggleButton();
                        saveFormState();
                    });
                });

                function updateChromeToggleButton() {
                    document.querySelectorAll('.select-all-btn').forEach(btn => {
                        const group = btn.dataset.group;
                        const container = document.getElementById(group + '-plugins');
                        if (!container) return;
                        const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
                        const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
                        btn.textContent = allChecked ? 'Deselect All Chrome' : 'Select All Chrome';
                    });
                }

                document.querySelectorAll('.select-all-btn').forEach(btn => {
                    btn.addEventListener('click', function() {
                        const group = this.dataset.group;
                        const container = document.getElementById(group + '-plugins');
                        if (!container) return;

                        const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
                        const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
                        const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);

                        checkboxes.forEach(cb => {
                            cb.checked = !allChecked;
                            dispatchChange(cb);
                        });

                        normalizePluginSelections();
                        applyRequiredSearchPlugin(requiredSearchPreference);
                        syncEnabledPluginsConfig();
                        updateChromeToggleButton();
                        saveFormState();
                    });
                });

                document.querySelectorAll('.plugin-section-toggle').forEach(checkbox => {
                    checkbox.addEventListener('change', function() {
                        if (this.checked) {
                            ensureRequiredPluginsChecked(this.value);
                        } else {
                            uncheckDependentPlugins(this.value);
                        }
                        syncEnabledPluginsConfig();
                        updateChromeToggleButton();
                        saveFormState();
                    });
                });

                // LocalStorage: Save/Load form state (all fields including URLs for repeat crawls)
                const STORAGE_KEY = 'archivebox_add_form_state';

                function saveFormState() {
                    const state = {};
                    document.querySelectorAll('#add-form input, #add-form textarea, #add-form select').forEach(el => {
                        if (el.name === 'csrfmiddlewaretoken') return;
                        if (el.name === 'config' || el.name.startsWith('plugin_config__') || el.closest('#id_config_container')) return;
                        if (el.type === 'checkbox' || el.type === 'radio') {
                            state[el.name + ':' + el.value] = el.checked;
                        } else {
                            state[el.name] = el.value;
                        }
                    });
                    localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
                }

                function loadFormState() {
                    try {
                        const state = JSON.parse(localStorage.getItem(STORAGE_KEY) || '{}');
                        for (const [key, value] of Object.entries(state)) {
                            if (key.includes(':')) {
                                const [name, val] = key.split(':');
                                if (name === 'config' || name.startsWith('plugin_config__')) continue;
                                const el = document.querySelector(`[name="${name}"][value="${val}"]`);
                                if (el) el.checked = Boolean(value);
                            } else {
                                if (key === 'config' || key.startsWith('plugin_config__')) continue;
                                const el = document.querySelector(`[name="${key}"]`);
                                if (el && el.type !== 'checkbox' && el.type !== 'radio') {
                                    if (el.tagName === 'SELECT') {
                                        const stringValue = String(value || '').trim();
                                        if (!stringValue || !Array.from(el.options).some(option => option.value === stringValue)) {
                                            continue;
                                        }
                                    }
                                    el.value = value;
                                    if (el.type === 'hidden') {
                                        el.dispatchEvent(new Event('archivebox:sync-tags', { bubbles: true }));
                                        el.dispatchEvent(new Event('change', { bubbles: true }));
                                    }
                                }
                            }
                        }
                        updateURLPreview(); // Update preview after loading URLs
                        if (personaSelect) {
                            applyPersonaConfig(personaSelect.value);
                        }
                        normalizePluginSelections();
                        applyRequiredSearchPlugin();
                        syncEnabledPluginsConfig();
                        updateChromeToggleButton();
                        saveFormState();
                    } catch (e) {}
                }

                // Auto-save on changes
                document.querySelectorAll('#add-form input, #add-form textarea, #add-form select').forEach(el => {
                    el.addEventListener('change', saveFormState);
                });

                if (personaSelect) {
                    personaSelect.addEventListener('change', function() {
                        applyPersonaConfig(personaSelect.value);
                        saveFormState();
                    });
                }

                loadFormState();

                // Form submission handler
                document.getElementById('add-form').addEventListener('submit', function(event) {
                    document.getElementById('in-progress').style.display = 'block'
                    document.getElementById('add-form').style.display = 'none'
                    document.getElementById('delay-warning').style.display = 'block'
                    setTimeout(function() {
                        window.location = '/'
                    }, 2000)
                    return true
                })
            </script>
        {% endif %}
    </div>
{% endblock %}

{% block footer %}{% endblock %}

{% block sidebar %}{% endblock %}