#!/usr/bin/env python3
"""
ERSEC - Security Behavior Verification Platform with Runtime Shield and Offline Remediation Advisor
============================================================================

A defensive security tool: it finds evidence-backed vulnerability signals
on a target you own or are authorized to test, then applies deterministic
reasoning and an offline remediation advisor to explain findings and suggest
reviewable remediation and verification steps. Optional local model support
is available for explanation/triage, but the deterministic evidence layer
remains the source of truth.

This tool deliberately does NOT contain or generate destructive exploitation
workflows, persistence, credential theft, or data-extraction automation.

Usage:
    python3 ersec -t example.com -m -o report.json --html report.html
    python3 ersec --shield --shield-upstream http://127.0.0.1:8000 --shield-port 8080

Legal: Only scan systems you own or have explicit written authorization to
test. Unauthorized scanning of third-party systems may be illegal in your
jurisdiction (e.g., CFAA in the US, Computer Misuse Act in the UK).
"""

from __future__ import annotations

import argparse
import ipaddress
import dataclasses
import datetime
import hashlib
import html
import json
import os
import re
import sys
import shlex
import shutil
import threading
import time
import urllib.parse
import socket
import ssl
from collections import defaultdict, deque
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field, asdict
from enum import Enum
from typing import Any, Dict, List, Optional, Set, Tuple

import warnings
warnings.filterwarnings("ignore")
import importlib.metadata as importlib_metadata

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

import requests
from bs4 import BeautifulSoup

from ersec_behavior import (
    SecurityBehaviorModel, SecurityBehaviorGraphV2, SecurityBehaviorVerifier,
    SecurityBehaviorStateStore, SecurityInvariantEngine, CounterexamplePathEngine,
    SecurityBehaviorGraphV3, SecurityAuthorizationMatrix,
)

try:
    from rich.console import Console
    from rich.table import Table
    from rich.panel import Panel
    RICH = True
except ImportError:
    RICH = False


ERSEC_VERSION = "28.1.0"
ERSEC_PROOF_SCHEMA = "ersec-proof/2"
ERSEC_CONTRACT_SCHEMA = "ersec-contract/2"
ERSEC_AUTH_MANIFEST_SCHEMA = "ersec-authorization-manifest/1"

# =============================================================================
# Errors
# =============================================================================

class ERSECError(Exception):
    pass


class ScopeError(ERSECError):
    """Raised when an operation would step outside the authorized scope."""


class TargetValidationError(ERSECError):
    pass


# =============================================================================
# Scan configuration & scope
# =============================================================================

class ScanProfile(Enum):
    PASSIVE = "passive"      # headers, TLS, cookies, banners - zero payloads sent
    BASELINE = "baseline"    # + safe boundary probes for common web vuln classes
    DEEP = "deep"            # + broader crawl, more parameters, more checks


@dataclass
class ScopeConfig:
    allowed_hosts: List[str] = field(default_factory=list)
    allowed_ports: List[int] = field(default_factory=lambda: [80, 443])
    allowed_paths: List[str] = field(default_factory=list)   # empty = whole site
    allowed_methods: List[str] = field(default_factory=lambda: ["GET", "POST", "HEAD", "OPTIONS", "TRACE"])
    max_requests: int = 2000
    rate_limit_seconds: float = 0.15         # min delay between requests
    concurrency: int = 8
    timeout_seconds: float = 8.0
    max_crawl_depth: int = 3
    max_crawl_pages: int = 150
    stop_on_scope_violation: bool = True
    adaptive_discovery: bool = True
    max_context_probes: int = 120
    max_api_paths: int = 80
    coverage_mode: str = "maximum"  # maximum | balanced | conservative


@dataclass
class ScanConfig:
    profile: ScanProfile = ScanProfile.BASELINE
    target: str = ""
    scope: ScopeConfig = field(default_factory=ScopeConfig)
    cookies: Dict[str, str] = field(default_factory=dict)
    bearer_token: str = ""
    second_bearer_token: str = ""  # optional second authorized identity for cross-identity authorization testing
    extra_headers: Dict[str, str] = field(default_factory=dict)
    verify_tls: bool = True
    user_agent: str = "ERSEC/7.0 (Authorized-Security-Assessment; +contact-owner)"
    verbose: bool = False
    ai_enabled: bool = True
    local_model_path: Optional[str] = None  # optional GGUF path for LocalLLMPatchGenerator; see AdvisorEngine
    crown_jewels: List[str] = field(default_factory=list)  # path substrings marking high-value endpoints for blast-radius scoring
    browser_discovery: bool = False
    api_intelligence: bool = True
    behavioral_twin: bool = True
    journey_intelligence: bool = True
    invariant_learning: bool = True
    counterfactual_checks: bool = True
    evidence_capsules: bool = True
    continuous_memory_path: Optional[str] = None
    local_llm_triage: bool = True
    context_aware_fuzzing: bool = True
    evidence_triage: bool = True
    triage_model_path: Optional[str] = None
    patch_output_dir: Optional[str] = None
    policy_output_dir: Optional[str] = None
    ide_output_dir: Optional[str] = None
    cloud_native: bool = True
    commerce_intelligence: bool = True
    security_genome: bool = True
    temporal_reasoning: bool = True
    contract_drift: bool = True
    role_differential: bool = True
    mutation_safety: bool = True
    explainable_prioritization: bool = True
    genome_memory_path: Optional[str] = None
    role_profile_files: List[str] = field(default_factory=list)
    max_sensitive_response_bytes: int = 12000
    component_failure_policy: str = "record"  # record | fail
    semantic_metamorphic: bool = True
    workflow_replay: bool = True
    identity_tokens: Dict[str, str] = field(default_factory=dict)
    max_identity_workflow_urls: int = 120
    max_identity_workflows: int = 20
    identity_role_order: Dict[str, int] = field(default_factory=lambda: {"anonymous":0, "user":1, "tenant":1, "admin":2})
    causal_impact: bool = True
    autonomous_agent: bool = True
    federated_mesh: bool = True
    metamorphic_lam: bool = True
    remediation_twin: bool = True
    contract_drift_sentry: bool = True
    federation_store_path: Optional[str] = None
    remediation_twin_dir: Optional[str] = None
    drift_baseline_path: Optional[str] = None
    autonomous_max_actions: int = 24
    control_plane: bool = True
    control_plane_memory_path: Optional[str] = None
    security_slo_path: Optional[str] = None
    hypothesis_budget: int = 40
    shield_enabled: bool = False
    shield_upstream: str = ""
    shield_bind: str = "127.0.0.1"
    shield_port: int = 8080
    shield_mode: str = "block"  # monitor | block | learn
    shield_policy_path: Optional[str] = None
    shield_from_report: Optional[str] = None
    shield_log_path: Optional[str] = None
    shield_max_body_bytes: int = 2 * 1024 * 1024
    shield_max_url_length: int = 8192
    shield_rate_per_minute: int = 120
    shield_burst: int = 30
    shield_backend_timeout: float = 15.0
    shield_learning_path: Optional[str] = None
    shield_tls_cert: Optional[str] = None
    shield_tls_key: Optional[str] = None
    # Security governance / assurance layer
    authorization_manifest_path: Optional[str] = None
    authorization_manifest: Optional[Dict[str, Any]] = None
    stateful_tests_enabled: bool = False
    risk_budget: int = 40
    contract_output_dir: Optional[str] = None
    security_graph_path: Optional[str] = None
    benchmark_quality_path: Optional[str] = None
    security_model_path: Optional[str] = None
    behavior_verification_path: Optional[str] = None
    behavior_state_path: Optional[str] = None
    behavior_graph_v3_path: Optional[str] = None
    behavior_assurance_path: Optional[str] = None


def _utc_now_iso() -> str:
    """Timezone-aware UTC timestamp as ISO 8601 with a 'Z' suffix - replaces
    the deprecated datetime.utcnow() pattern used throughout this file."""
    return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")


def validate_target(target: str) -> str:
    if not target:
        raise TargetValidationError("Empty target")
    t = target.strip()
    if re.match(r"^https?://https?://", t, re.I):
        raise TargetValidationError(
            f"'{target}' looks like it has a doubled scheme (e.g. 'https://https://...'). "
            "Remove the extra 'http(s)://' prefix - this is a common copy-paste mistake."
        )
    if t.startswith(("http://", "https://")):
        parsed = urllib.parse.urlparse(t)
        if not parsed.hostname:
            raise TargetValidationError("Invalid URL: missing hostname")
        host = parsed.hostname
    else:
        host = t.split("/")[0].split(":")[0]
    if host.lower() in ("http", "https"):
        raise TargetValidationError(
            f"'{target}' resolved to the hostname '{host}', which is almost certainly a doubled or "
            "malformed scheme, not a real target. Check the URL for a typo."
        )
    if re.match(r"^\d{1,3}(\.\d{1,3}){3}$", host):
        if any(int(p) > 255 for p in host.split(".")):
            raise TargetValidationError(f"Invalid IP: {host}")
    elif not re.match(
        r"^[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)*$", host
    ):
        raise TargetValidationError(f"Invalid hostname: {host}")
    return host


def is_in_scope(url: str, config: ScanConfig) -> bool:
    parsed = urllib.parse.urlparse(url)
    host = parsed.hostname
    if not host:
        return False
    allowed_hosts = config.scope.allowed_hosts or [config.target]
    if host not in allowed_hosts:
        return False
    port = parsed.port or (443 if parsed.scheme == "https" else 80)
    if config.scope.allowed_ports and port not in config.scope.allowed_ports:
        return False
    if config.scope.allowed_paths:
        if not any(parsed.path.startswith(p) for p in config.scope.allowed_paths):
            return False
    return True


# =============================================================================
# Evidence & Findings
# =============================================================================

@dataclass
class RequestEvidence:
    method: str = ""
    url: str = ""
    status_code: int = 0
    response_time_ms: int = 0
    response_headers: Dict[str, str] = field(default_factory=dict)
    response_excerpt: str = ""   # truncated, sanitized
    request_headers_sent: Dict[str, str] = field(default_factory=dict)  # only headers that differ from the
                                                                          # session baseline - i.e. the actual
                                                                          # probe payload, for --reverify replay

    def redacted(self) -> Dict[str, Any]:
        d = asdict(self)
        for sensitive in ("set-cookie", "authorization"):
            for k in list(d["response_headers"].keys()):
                if k.lower() == sensitive:
                    d["response_headers"][k] = "REDACTED"
            for k in list(d["request_headers_sent"].keys()):
                if k.lower() in (sensitive, "cookie"):
                    d["request_headers_sent"][k] = "REDACTED"
        d["response_excerpt"] = d["response_excerpt"][:300]
        return d


class Severity(Enum):
    CRITICAL = 4
    HIGH = 3
    MEDIUM = 2
    LOW = 1
    INFO = 0


@dataclass(frozen=True)
class SecurityCategoryMetadata:
    canonical: str
    test_family: str
    shield_compatible: bool = False
    minimum_confidence: str = "Likely"
    remediation_key: str = "general"
    description: str = ""


class SecurityCategoryRegistry:
    """Single source of truth for detector semantics, Shield compatibility and proof metadata.

    This deliberately favors a small canonical vocabulary over a large marketing taxonomy.
    Unknown detector categories remain valid but default to non-enforceable semantics.
    """
    _ALIASES = {
        "sql_injection": "sql_injection_signal",
        "sqli": "sql_injection_signal",
        "xss": "xss_signal",
        "path_traversal": "path_traversal_signal",
        "idor": "idor_heuristic",
        "ssrf": "ssrf_signal",
        "template_injection": "ssti_signal",
        "prototype_pollution": "prototype_pollution_signal",
    }
    _META = {
        "sql_injection_signal": SecurityCategoryMetadata("sql_injection_signal", "injection", True, "Confirmed", "sql"),
        "nosql_injection_signal": SecurityCategoryMetadata("nosql_injection_signal", "injection", True, "Confirmed", "nosql"),
        "ldap_injection_signal": SecurityCategoryMetadata("ldap_injection_signal", "injection", True, "Confirmed", "ldap"),
        "xss_signal": SecurityCategoryMetadata("xss_signal", "browser-input", True, "Likely", "xss"),
        "ssti_signal": SecurityCategoryMetadata("ssti_signal", "template-input", True, "Confirmed", "ssti"),
        "path_traversal_signal": SecurityCategoryMetadata("path_traversal_signal", "path-normalization", True, "Likely", "path-traversal"),
        "crlf_injection": SecurityCategoryMetadata("crlf_injection", "header-injection", True, "Likely", "crlf"),
        "prototype_pollution_signal": SecurityCategoryMetadata("prototype_pollution_signal", "structured-input", True, "Likely", "prototype-pollution"),
        "command_injection_timing": SecurityCategoryMetadata("command_injection_timing", "command-injection", True, "Confirmed", "command-injection"),
        "xxe_signal": SecurityCategoryMetadata("xxe_signal", "xml-input", True, "Confirmed", "xxe"),
        "idor_heuristic": SecurityCategoryMetadata("idor_heuristic", "authorization", False, "Likely", "idor"),
        "mass_assignment": SecurityCategoryMetadata("mass_assignment", "authorization", False, "Likely", "mass-assignment"),
        "admin_exposure": SecurityCategoryMetadata("admin_exposure", "access-control", False, "Confirmed", "access-control"),
        "csrf": SecurityCategoryMetadata("csrf", "state-change", False, "Likely", "csrf"),
        "jwt": SecurityCategoryMetadata("jwt", "authentication", False, "Likely", "jwt"),
        "weak_session_token": SecurityCategoryMetadata("weak_session_token", "authentication", False, "Likely", "session"),
    }

    @classmethod
    def normalize(cls, category: str) -> str:
        key = str(category or "unknown").strip().lower()
        return cls._ALIASES.get(key, key)

    @classmethod
    def metadata(cls, category: str) -> SecurityCategoryMetadata:
        canonical = cls.normalize(category)
        return cls._META.get(canonical, SecurityCategoryMetadata(canonical, "unclassified", False, "Likely", "general"))

    @classmethod
    def manifest(cls) -> Dict[str, Any]:
        return {k: asdict(v) for k, v in sorted(cls._META.items())}


@dataclass
class Finding:
    finding_id: str
    category: str
    title: str
    severity: Severity
    confidence: str            # "Confirmed" | "Likely" | "Possible"
    owasp: str
    cwe: str
    url: str
    parameter: Optional[str]
    description: str
    evidence: RequestEvidence
    remediation_summary: str = ""          # short built-in fallback remediation
    ai_explanation: str = ""               # filled in by AdvisorEngine
    ai_remediation_steps: str = ""         # filled in by AdvisorEngine
    ai_verification_steps: str = ""        # filled in by AdvisorEngine
    affected_urls: List[str] = field(default_factory=list)  # for site-wide findings consolidated across pages
    evidence_score: float = 0.0
    attack_surface: str = "web"
    state_context: str = ""
    tags: List[str] = field(default_factory=list)
                                                              # (see CONSOLIDATABLE_CATEGORIES) - empty for
                                                              # genuinely page-specific findings

    def diff_key(self) -> str:
        """Stable identity for baseline comparison across scans - independent of
        finding_id (which is just a per-run counter) or full evidence."""
        return f"{self.category}|{self.url.split('?')[0]}|{self.parameter or ''}"

    def to_dict(self) -> Dict[str, Any]:
        redacted_evidence = self.evidence.redacted()
        d = {
            "finding_id": self.finding_id,
            "category": self.category,
            "title": self.title,
            "severity": self.severity.name,
            "confidence": self.confidence,
            "owasp": self.owasp,
            "cwe": self.cwe,
            "compliance": _COMPLIANCE_MAP.get(self.category, []),
            "url": self.url,
            "parameter": self.parameter,
            "description": self.description,
            "evidence": redacted_evidence,
            "remediation_summary": self.remediation_summary,
            "ai_explanation": self.ai_explanation,
            "ai_remediation_steps": self.ai_remediation_steps,
            "ai_verification_steps": self.ai_verification_steps,
            "affected_urls": self.affected_urls,
            "evidence_score": self.evidence_score,
            "attack_surface": self.attack_surface,
            "state_context": self.state_context,
            "tags": self.tags,
            "category_metadata": asdict(SecurityCategoryRegistry.metadata(self.category)),
            "diff_key": self.diff_key(),
            # Minimal spec needed to re-run this exact probe later via --reverify.
            # Session-level auth (cookies/bearer) is deliberately excluded here -
            # it's supplied fresh at replay time via CLI flags, never persisted.
            "replay": {
                "method": self.evidence.method,
                "url": self.evidence.url,
                "extra_headers": redacted_evidence.get("request_headers_sent", {}),
                "category": self.category,
            },
        }
        return d


# Informational mapping only - not a certified compliance assessment. Clause
# numbers can shift between framework versions; verify against your actual
# compliance program before citing these in an audit.
_COMPLIANCE_MAP: Dict[str, List[str]] = {
    "sql_injection":            ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "nosql_injection":          ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "ssrf":                     ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "xss":                      ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "ssti":                     ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "security_headers":         ["ISO 27001 A.8.9", "SOC 2 CC6.6"],
    "tls":                      ["PCI-DSS 4.2.1", "ISO 27001 A.8.24", "SOC 2 CC6.7"],
    "server_banner":            ["ISO 27001 A.8.9"],
    "http_methods":             ["ISO 27001 A.8.9"],
    "cors":                     ["ISO 27001 A.8.9", "SOC 2 CC6.1"],
    "exposed_files":            ["PCI-DSS 3.4.1", "PCI-DSS 8.3.1", "ISO 27001 A.8.9", "SOC 2 CC6.1"],
    "open_redirect":            ["ISO 27001 A.8.28"],
    "csrf":                     ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "jwt":                      ["PCI-DSS 8.3.1", "ISO 27001 A.8.24", "SOC 2 CC6.1"],
    "directory_listing":        ["ISO 27001 A.8.9"],
    "verbose_errors":           ["ISO 27001 A.8.9", "SOC 2 CC7.1"],
    "mixed_content":            ["PCI-DSS 4.2.1", "ISO 27001 A.8.24"],
    "sri":                      ["ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "insecure_form":            ["PCI-DSS 4.2.1", "PCI-DSS 8.3.1"],
    "graphql_introspection":    ["ISO 27001 A.8.9"],
    "crlf_injection":           ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "host_header":              ["ISO 27001 A.8.28"],
    "vulnerable_library":       ["PCI-DSS 6.3.2", "ISO 27001 A.8.8", "SOC 2 CC7.1"],
    "info_disclosure_comments": ["ISO 27001 A.8.9"],
    "cache_control":            ["PCI-DSS 3.4.1", "ISO 27001 A.8.9"],
    "ldap_injection":           ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "header_injection":         ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "admin_exposure":           ["PCI-DSS 7.2.1", "PCI-DSS 8.3.1", "ISO 27001 A.8.3", "SOC 2 CC6.1"],
    "weak_session_token":       ["PCI-DSS 8.3.1", "ISO 27001 A.8.24", "SOC 2 CC6.1"],
    "prototype_pollution":      ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "xxe":                      ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "xpath_injection":          ["PCI-DSS 6.2.4", "ISO 27001 A.8.28"],
    "command_injection":        ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "idor_heuristic":           ["PCI-DSS 7.2.1", "ISO 27001 A.8.3", "SOC 2 CC6.1"],
    "rate_limiting":            ["PCI-DSS 8.3.4", "ISO 27001 A.8.5"],
    "websocket":                ["ISO 27001 A.8.24", "SOC 2 CC6.7"],
    "email_security":           ["ISO 27001 A.8.9"],
    "security_txt":             ["ISO 27001 A.8.9"],
    "mass_assignment":          ["PCI-DSS 6.2.4", "ISO 27001 A.8.28", "SOC 2 CC6.1"],
    "user_enumeration":         ["PCI-DSS 8.3.4", "ISO 27001 A.8.5"],
    "subdomain_takeover":       ["ISO 27001 A.8.9", "SOC 2 CC6.1"],
    "api_method_leakage":       ["ISO 27001 A.8.9"],
    "source_map_exposure":      ["ISO 27001 A.8.9"],
    "service_worker_exposure":  ["ISO 27001 A.8.24"],
    "oauth_state":              ["ISO 27001 A.8.28"],
    "password_recovery_leakage":["ISO 27001 A.8.28"],
    "cookie_scope":             ["ISO 27001 A.8.24"],
    "path_canonicalization":    ["ISO 27001 A.8.28"],
    "forwarded_host_trust":     ["ISO 27001 A.8.28"],
    "content_negotiation_auth": ["ISO 27001 A.8.28"],
    "file_upload_surface":      ["ISO 27001 A.8.28"],
    "cors_preflight_consistency":["ISO 27001 A.8.9"],
    "security_boundary_exposure":["ISO 27001 A.5.15", "SOC 2 CC6.1"],
    "authorization_differential":["ISO 27001 A.5.15", "SOC 2 CC6.1"],
}


# =============================================================================
# HTTP client - scope-enforced, rate-limited
# =============================================================================

class ScanProgress:
    """Thread-safe live progress state. In an interactive terminal, pressing
    Enter prints a snapshot without interrupting the scan."""

    def __init__(self):
        self._lock = threading.RLock()
        self.started_at = time.time()
        self.stage = "initializing"
        self.detail = ""
        self.requests = 0
        self.pages = 0
        self.findings = 0
        self.detectors_started = 0
        self.detectors_finished = 0
        self.detectors_failed = 0
        self.current_detector = ""
        self.last_error = ""
        self.finished = False
        self.interrupted = False

    def set_stage(self, stage: str, detail: str = "") -> None:
        with self._lock:
            self.stage, self.detail = stage, detail

    def request(self):
        with self._lock:
            self.requests += 1

    def page(self):
        with self._lock:
            self.pages += 1

    def detector_start(self, name: str):
        with self._lock:
            self.detectors_started += 1
            self.current_detector = name

    def detector_finish(self, findings: int = 0):
        with self._lock:
            self.detectors_finished += 1
            self.findings += findings

    def detector_fail(self, error: str):
        with self._lock:
            self.detectors_finished += 1
            self.detectors_failed += 1
            self.last_error = error

    def mark_finished(self, interrupted: bool = False):
        with self._lock:
            self.finished = True
            self.interrupted = interrupted
            self.stage = "interrupted" if interrupted else "complete"

    def snapshot(self) -> Dict[str, Any]:
        with self._lock:
            elapsed = max(0.001, time.time() - self.started_at)
            return {
                "stage": self.stage,
                "detail": self.detail,
                "elapsed_seconds": round(elapsed, 1),
                "requests": self.requests,
                "pages": self.pages,
                "findings": self.findings,
                "detectors": {
                    "started": self.detectors_started,
                    "finished": self.detectors_finished,
                    "failed": self.detectors_failed,
                    "current": self.current_detector,
                },
                "request_rate_per_second": round(self.requests / elapsed, 2),
                "last_error": self.last_error,
                "finished": self.finished,
                "interrupted": self.interrupted,
            }

    def format(self) -> str:
        d = self.snapshot()
        det = d["detectors"]
        text = (
            f"[ERSEC progress] stage={d['stage']} | elapsed={d['elapsed_seconds']}s | "
            f"requests={d['requests']} | pages={d['pages']} | findings={d['findings']} | "
            f"detectors={det['finished']}/{det['started']}"
        )
        if det["current"]:
            text += f" | current={det['current']}"
        if det["failed"]:
            text += f" | detector_failures={det['failed']}"
        if d["last_error"]:
            text += f" | last_error={d['last_error'][:160]}"
        return text


def start_enter_progress_monitor(progress: ScanProgress) -> threading.Event:
    """Watch stdin on a daemon thread. Blank Enter prints a progress snapshot.
    Non-TTY runs are left untouched so CI/stdin pipelines never block."""
    stop = threading.Event()
    if not (hasattr(sys.stdin, "isatty") and sys.stdin.isatty()):
        return stop

    def watch():
        while not stop.is_set():
            try:
                line = sys.stdin.readline()
                if line == "":
                    return
                if not line.strip():
                    print("\n" + progress.format(), flush=True)
            except (EOFError, OSError):
                return

    threading.Thread(target=watch, name="ersec-progress-input", daemon=True).start()
    return stop


class SafeHttpClient:
    def __init__(self, config: ScanConfig):
        self.config = config
        self.session = requests.Session()
        self.session.verify = config.verify_tls
        self.session.headers.update({"User-Agent": config.user_agent, **config.extra_headers})
        if config.cookies:
            self.session.cookies.update(config.cookies)
        if config.bearer_token:
            self.session.headers["Authorization"] = f"Bearer {config.bearer_token}"
        self._lock = threading.Lock()
        self._last_request_time = 0.0
        self.request_count = 0
        self._consecutive_429s = 0
        self._backoff_until = 0.0
        self.progress: Optional[ScanProgress] = getattr(config, "_progress", None)
        self._thread_local = threading.local()

    def _session_for_thread(self) -> requests.Session:
        session = getattr(self._thread_local, "session", None)
        if session is None:
            session = requests.Session()
            session.verify = self.config.verify_tls
            session.headers.update(dict(self.session.headers))
            session.cookies.update(self.session.cookies)
            self._thread_local.session = session
        return session

    def _throttle(self):
        with self._lock:
            now = time.time()
            if now < self._backoff_until:
                time.sleep(self._backoff_until - now)
            elapsed = time.time() - self._last_request_time
            wait = self.config.scope.rate_limit_seconds - elapsed
            if wait > 0:
                time.sleep(wait)
            self._last_request_time = time.time()

    def _handle_response_backoff(self, resp: "requests.Response") -> None:
        """Adaptive courtesy backoff: if the target starts returning 429/503,
        slow down automatically rather than hammering a server that's asking
        us to back off. This is politeness, not evasion - it makes the tool a
        better guest on shared/rate-limited infrastructure and reduces the
        chance of an authorized scan looking like a DoS attempt."""
        with self._lock:
            if resp.status_code in (429, 503):
                self._consecutive_429s += 1
                retry_after = resp.headers.get("Retry-After")
                delay = None
                if retry_after:
                    try:
                        delay = float(retry_after)
                    except ValueError:
                        delay = None
                if delay is None:
                    delay = min(2.0 * (2 ** min(self._consecutive_429s, 5)), 30.0)
                self._backoff_until = time.time() + delay
            else:
                self._consecutive_429s = 0

    def request(self, method: str, url: str, **kwargs) -> requests.Response:
        with self._lock:
            if self.request_count >= self.config.scope.max_requests:
                raise ERSECError("Request budget exhausted for this scan")
            self.request_count += 1
        if self.progress:
            self.progress.request()
        if not is_in_scope(url, self.config):
            raise ScopeError(f"URL out of scope: {url}")
        if method.upper() not in self.config.scope.allowed_methods:
            raise ScopeError(f"Method not allowed by scope: {method}")
        manifest = getattr(self.config, "authorization_manifest", None)
        if manifest:
            ok, reason = AuthorizationManifest(manifest).allows_request(method, url)
            if not ok:
                raise ScopeError(f"Authorization manifest denied request ({reason}): {method} {url}")
        self._throttle()
        try:
            # A scalar requests timeout is not a wall-clock timeout. Explicitly
            # bound connect and read waits so a TLS peer that stops responding
            # cannot make ERSEC appear frozen indefinitely.
            timeout = max(0.5, float(self.config.scope.timeout_seconds))
            kwargs.setdefault("timeout", (min(timeout, 5.0), timeout))
            resp = self._session_for_thread().request(
                method, url, allow_redirects=False, **kwargs
            )
            self._handle_response_backoff(resp)
            return resp
        except requests.exceptions.SSLError as e:
            raise ERSECError(f"TLS error: {e}") from e
        except requests.exceptions.ConnectionError as e:
            raise ERSECError(f"Connection failed: {e}") from e
        except requests.exceptions.Timeout as e:
            raise ERSECError(f"Timed out: {e}") from e
        except requests.exceptions.RequestException as e:
            raise ERSECError(f"HTTP error: {e}") from e


def _make_evidence(resp: requests.Response, elapsed_ms: int, baseline_headers: Optional[Dict[str, str]] = None) -> RequestEvidence:
    extra_headers = {}
    if baseline_headers is not None:
        sent = dict(resp.request.headers)
        for k, v in sent.items():
            if k not in baseline_headers or baseline_headers.get(k) != v:
                if k.lower() not in ("content-length",):  # not a meaningful "probe" header
                    extra_headers[k] = v
    return RequestEvidence(
        method=resp.request.method,
        url=resp.url,
        status_code=resp.status_code,
        response_time_ms=elapsed_ms,
        response_headers=dict(resp.headers),
        response_excerpt=resp.text[:800] if resp.text else "",
        request_headers_sent=extra_headers,
    )


# =============================================================================
# Crawler
# =============================================================================

@dataclass
class FormInfo:
    page_url: str
    action: str
    method: str
    inputs: List[Dict[str, str]]


_API_LIKE_PATH_RE = re.compile(r"^/(?:api|rest|v[0-9]+|graphql)(?:/|$)", re.I)


class WebCrawler:
    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client
        self.visited: Set[str] = set()
        self.successful_fetches = 0  # incremented only on an actual successful response - unlike
                                      # `visited` (marked pre-attempt), this is the real signal for
                                      # "did the scan ever actually reach the target"
        self.endpoints: List[str] = []
        self.forms: List[FormInfo] = []
        self.skipped_out_of_scope: List[str] = []
        self.third_party_origins: Set[str] = set()
        self.link_graph: Dict[str, Set[str]] = {}  # page URL -> set of pages it links to (for blast-radius BFS)
        self.api_like_endpoints: Set[str] = set()  # bare (no-query) API-shaped paths - see _API_LIKE_PATH_RE
        self.response_hashes: Dict[str, List[str]] = {}  # normalized-body hash -> list of URLs (soft-404 detection)
        self.numeric_id_endpoints: Dict[str, Set[str]] = {}  # base-path -> set of numeric IDs seen (IDOR heuristic)

    def _seed_from_robots_and_sitemap(self, start_url: str) -> List[str]:
        """Passive reconnaissance: robots.txt Disallow entries and sitemap.xml URLs
        often reveal pages the normal link-crawl would never find. This is read-only
        (a GET on public metadata files), the same technique ZAP's spider uses."""
        seeds: List[str] = []
        robots_url = urllib.parse.urljoin(start_url, "/robots.txt")
        try:
            resp = self.client.request("GET", robots_url)
            if resp.status_code == 200:
                for line in resp.text.splitlines():
                    line = line.strip()
                    if line.lower().startswith(("disallow:", "allow:")):
                        path = line.split(":", 1)[1].strip()
                        if path and path != "/":
                            full = urllib.parse.urljoin(start_url, path)
                            if is_in_scope(full, self.config):
                                seeds.append(full)
                    elif line.lower().startswith("sitemap:"):
                        sitemap_url = line.split(":", 1)[1].strip()
                        if is_in_scope(sitemap_url, self.config):
                            seeds.extend(self._parse_sitemap(sitemap_url))
        except (ScopeError, ERSECError):
            pass
        sitemap_default = urllib.parse.urljoin(start_url, "/sitemap.xml")
        try:
            seeds.extend(self._parse_sitemap(sitemap_default))
        except (ScopeError, ERSECError):
            pass
        return seeds

    def _parse_sitemap(self, sitemap_url: str) -> List[str]:
        found = []
        try:
            resp = self.client.request("GET", sitemap_url)
            if resp.status_code == 200 and "<urlset" in resp.text[:2000]:
                for loc in re.findall(r"<loc>(.*?)</loc>", resp.text):
                    if is_in_scope(loc, self.config):
                        found.append(loc)
        except (ScopeError, ERSECError):
            pass
        return found[: self.config.scope.max_crawl_pages]

    def crawl(self, start_url: str) -> Tuple[List[str], List[FormInfo]]:
        queue: List[Tuple[str, int]] = [(start_url, 0)]
        for seed in self._seed_from_robots_and_sitemap(start_url):
            queue.append((seed, 1))
        while queue and len(self.visited) < self.config.scope.max_crawl_pages:
            url, depth = queue.pop(0)
            norm = url.split("#")[0]
            if norm in self.visited or depth > self.config.scope.max_crawl_depth:
                continue
            if not is_in_scope(norm, self.config):
                self.skipped_out_of_scope.append(norm)
                continue
            self.visited.add(norm)
            try:
                t0 = time.time()
                resp = self.client.request("GET", norm)
                elapsed_ms = int((time.time() - t0) * 1000)
            except (ScopeError, ERSECError) as e:
                if self.config.verbose:
                    print(f"  [!] crawl skip {norm}: {e}")
                continue
            self.successful_fetches += 1
            self.endpoints.append(norm)
            path_only = urllib.parse.urlparse(norm).path
            if not urllib.parse.urlparse(norm).query and _API_LIKE_PATH_RE.match(path_only):
                self.api_like_endpoints.add(norm)
            self._record_numeric_id(norm)
            self._record_response_hash(norm, resp)
            ctype = resp.headers.get("Content-Type", "")
            if "text/html" not in ctype:
                continue
            try:
                soup = BeautifulSoup(resp.text, "html.parser")
            except Exception:
                continue
            for link in soup.find_all("a", href=True):
                full_url = urllib.parse.urljoin(norm, link["href"])
                target = full_url.split("#")[0]
                self.link_graph.setdefault(norm, set()).add(target)
                if target not in self.visited:
                    queue.append((full_url, depth + 1))
            for form in soup.find_all("form"):
                action = urllib.parse.urljoin(norm, form.get("action", "") or norm)
                method = (form.get("method") or "get").upper()
                inputs = []
                for inp in form.find_all(["input", "textarea", "select"]):
                    name = inp.get("name")
                    if name:
                        inputs.append({
                            "name": name,
                            "type": inp.get("type", "text"),
                            "value": inp.get("value", ""),
                        })
                self.forms.append(FormInfo(page_url=norm, action=action, method=method, inputs=inputs))

            own_host = urllib.parse.urlparse(norm).hostname
            for tag, attr in (("script", "src"), ("iframe", "src"), ("link", "href")):
                for el in soup.find_all(tag):
                    src = el.get(attr, "")
                    if not src or src.startswith(("data:", "javascript:", "#")):
                        continue
                    full_src = urllib.parse.urljoin(norm, src)
                    src_host = urllib.parse.urlparse(full_src).hostname
                    if src_host and src_host != own_host:
                        self.third_party_origins.add(src_host)
                    elif tag == "script" and src_host == own_host and full_src.split("?")[0].endswith(".js"):
                        # Same-origin JS file: passively parse it for API endpoint strings.
                        # This is read-only recon (LinkFinder's technique) - it finds routes
                        # a plain HTML link-crawl would never see, it doesn't call them.
                        for discovered in self._extract_endpoints_from_js(full_src, norm):
                            if discovered not in self.visited:
                                queue.append((discovered, depth + 1))
        return self.endpoints, self.forms

    _JS_PATH_RE = re.compile(r"""["'](/(?:api|rest|v[0-9]+|graphql)[a-zA-Z0-9_\-/]*)["']""")
    _NUMERIC_ID_RE = re.compile(r"/(\d{1,12})(?:/|$|\?)")

    def _record_numeric_id(self, url: str) -> None:
        """Track sequential-looking numeric path segments per base path, e.g.
        /orders/1042 -> base '/orders/{id}' -> {'1042'}. Purely observational
        during the crawl; IDORHeuristicModule later decides whether to probe
        neighboring IDs, and only for values it actually observed in-app."""
        parsed = urllib.parse.urlparse(url)
        m = self._NUMERIC_ID_RE.search(parsed.path)
        if not m:
            return
        base = parsed.path[: m.start(1)] + "{id}" + parsed.path[m.end(1):]
        self.numeric_id_endpoints.setdefault(base, set()).add(m.group(1))

    def _record_response_hash(self, url: str, resp: requests.Response) -> None:
        """Cheap soft-404 fingerprint: hash of status code + a normalized body
        (whitespace collapsed, numbers stripped) so a custom 'not found' page
        that always returns HTTP 200 can still be recognized as a repeated
        pattern rather than treated as N distinct real pages."""
        body = resp.text or ""
        normalized = re.sub(r"\d+", "#", re.sub(r"\s+", " ", body[:2000])).strip()
        h = hashlib.sha256(f"{resp.status_code}|{normalized}".encode("utf-8", "ignore")).hexdigest()[:16]
        self.response_hashes.setdefault(h, []).append(url)

    def _extract_endpoints_from_js(self, js_url: str, referring_page: str) -> List[str]:
        found = []
        if len(self.visited) >= self.config.scope.max_crawl_pages:
            return found
        try:
            resp = self.client.request("GET", js_url)
        except (ScopeError, ERSECError):
            return found
        if resp.status_code != 200:
            return found
        for match in set(self._JS_PATH_RE.findall(resp.text or "")):
            full = urllib.parse.urljoin(referring_page, match)
            if is_in_scope(full, self.config):
                found.append(full)
        return found[:20]  # cap per file so one large bundle can't blow the request budget


# =============================================================================
# Detection modules
#
# Design principle: every "active" check sends at most one or two harmless
# boundary-condition requests (e.g. a single quote, a nonexistent-but-safe
# path segment, a differing Origin header) and looks for a *behavioral
# signal* (error text, timing, header reflection) that indicates a class of
# bug exists. None of these attempt to actually extract data, gain shell
# access, or chain into exploitation. That's the ZAP/Nikto model, not the
# sqlmap/metasploit model.
# =============================================================================

_FINDING_COUNTER = itertools_count = 0
_FINDING_COUNTER_LOCK = threading.Lock()
def _next_id() -> str:
    global _FINDING_COUNTER
    with _FINDING_COUNTER_LOCK:
        _FINDING_COUNTER += 1
        return f"F-{_FINDING_COUNTER:04d}"


class BaseModule:
    category = "generic"
    title = "Generic Check"
    owasp = ""
    cwe = ""
    min_profile = ScanProfile.PASSIVE

    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client

    def applies(self) -> bool:
        order = [ScanProfile.PASSIVE, ScanProfile.BASELINE, ScanProfile.DEEP]
        return order.index(self.config.profile) >= order.index(self.min_profile)

    def _finding(self, url, evidence, description, remediation, severity, confidence, parameter=None):
        return Finding(
            finding_id=_next_id(), category=self.category, title=self.title,
            severity=severity, confidence=confidence, owasp=self.owasp, cwe=self.cwe,
            url=url, parameter=parameter, description=description, evidence=evidence,
            remediation_summary=remediation,
        )

    def _get(self, url, **kwargs) -> Tuple[requests.Response, RequestEvidence]:
        t0 = time.time()
        resp = self.client.request("GET", url, **kwargs)
        return resp, _make_evidence(resp, int((time.time() - t0) * 1000), dict(self.client.session.headers))

    def _post(self, url, **kwargs) -> Tuple[requests.Response, RequestEvidence]:
        t0 = time.time()
        resp = self.client.request("POST", url, **kwargs)
        return resp, _make_evidence(resp, int((time.time() - t0) * 1000), dict(self.client.session.headers))

    def run_url(self, url: str) -> List[Finding]:
        return []

    def run_param(self, url: str, param: str, method: str = "GET") -> List[Finding]:
        return []


def _with_param(url: str, param: str, value: str) -> str:
    parsed = urllib.parse.urlparse(url)
    q = dict(urllib.parse.parse_qsl(parsed.query))
    q[param] = value
    return urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(q)))


class SecurityHeadersModule(BaseModule):
    category = "security_headers"; title = "Security Header Misconfiguration"
    owasp = "A05:2021"; cwe = "CWE-693"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        h = resp.headers

        hsts = h.get("Strict-Transport-Security")
        if not hsts:
            findings.append(self._finding(url, ev, "No Strict-Transport-Security header - connections can be downgraded to plain HTTP.",
                "Add 'Strict-Transport-Security: max-age=31536000; includeSubDomains; preload'.",
                Severity.MEDIUM, "Confirmed"))
        elif "max-age=0" in hsts.replace(" ", ""):
            findings.append(self._finding(url, ev, "HSTS is present but max-age=0, which disables the protection.",
                "Set max-age to at least 31536000 seconds (1 year).", Severity.MEDIUM, "Confirmed"))

        csp = h.get("Content-Security-Policy")
        if not csp:
            findings.append(self._finding(url, ev, "No Content-Security-Policy header - reduces defense-in-depth against XSS.",
                "Implement a CSP starting with default-src 'self' and tighten per-resource.",
                Severity.MEDIUM, "Confirmed"))
        elif "unsafe-inline" in csp or "unsafe-eval" in csp:
            findings.append(self._finding(url, ev, "CSP allows 'unsafe-inline' or 'unsafe-eval', which significantly weakens XSS protection.",
                "Remove unsafe-inline/unsafe-eval; use nonces or hashes for required inline scripts.",
                Severity.LOW, "Confirmed"))

        xfo = h.get("X-Frame-Options")
        if not xfo and (not csp or "frame-ancestors" not in csp):
            findings.append(self._finding(url, ev, "No clickjacking protection (missing X-Frame-Options and CSP frame-ancestors).",
                "Add 'X-Frame-Options: DENY' or a CSP 'frame-ancestors' directive.",
                Severity.MEDIUM, "Confirmed"))

        xcto = h.get("X-Content-Type-Options")
        if not xcto or xcto.lower() != "nosniff":
            findings.append(self._finding(url, ev, "Missing or incorrect X-Content-Type-Options header - allows MIME-sniffing attacks.",
                "Add 'X-Content-Type-Options: nosniff'.", Severity.LOW, "Confirmed"))

        if not h.get("Referrer-Policy"):
            findings.append(self._finding(url, ev, "No Referrer-Policy set - full URLs (possibly with sensitive query params) may leak to third parties via the Referer header.",
                "Add 'Referrer-Policy: strict-origin-when-cross-origin' or stricter.",
                Severity.LOW, "Confirmed"))

        if not h.get("Permissions-Policy"):
            findings.append(self._finding(url, ev, "No Permissions-Policy header - the page does not explicitly restrict access to powerful "
                "browser features (camera, microphone, geolocation, etc.) for itself or any framed/embedded content.",
                "Add a Permissions-Policy header that disables features the page does not use, e.g. "
                "'Permissions-Policy: geolocation=(), camera=(), microphone=()'.",
                Severity.LOW, "Confirmed"))

        if not h.get("Cross-Origin-Opener-Policy"):
            findings.append(self._finding(url, ev, "No Cross-Origin-Opener-Policy header - the page's window can be referenced by a "
                "cross-origin popup it opens, enabling certain cross-origin leak and Spectre-class side-channel "
                "attacks that COOP is designed to close off.",
                "Add 'Cross-Origin-Opener-Policy: same-origin' (verify it doesn't break intentional cross-origin popups/OAuth flows first).",
                Severity.LOW, "Confirmed"))

        corp = h.get("Cross-Origin-Resource-Policy")
        if not corp:
            findings.append(self._finding(url, ev, "No Cross-Origin-Resource-Policy header - this resource can be loaded cross-origin "
                "(e.g. embedded into another site's page) without an explicit opt-out.",
                "Add 'Cross-Origin-Resource-Policy: same-origin' (or 'same-site') for responses that are not meant to be embedded elsewhere.",
                Severity.INFO, "Confirmed"))

        xxp = h.get("X-XSS-Protection")
        if xxp and "0" not in xxp:
            findings.append(self._finding(url, ev, "X-XSS-Protection is set to a non-zero value. This legacy header's filter has known "
                "bypasses and, in old browsers, could itself be abused to selectively suppress page content (an "
                "information-disclosure gadget) - modern guidance is to disable it explicitly and rely on CSP instead.",
                "Set 'X-XSS-Protection: 0' and rely on a strong Content-Security-Policy instead.",
                Severity.INFO, "Confirmed"))

        server_timing = h.get("Server-Timing")
        if server_timing and re.search(r"(db|sql|query|cache|internal)", server_timing, re.I):
            findings.append(self._finding(url, ev, "Server-Timing header exposes internal metric names (e.g. database/cache timing labels) "
                "to any page that can read response headers, which can aid infrastructure fingerprinting.",
                "Strip or generalize Server-Timing metric names in production, or remove the header entirely if not actively used for RUM.",
                Severity.INFO, "Possible"))

        for cookie in resp.cookies:
            issues = []
            if not cookie.secure:
                issues.append("missing Secure flag")
            if not cookie.has_nonstandard_attr("HttpOnly"):
                issues.append("missing HttpOnly flag")
            if not cookie.has_nonstandard_attr("SameSite"):
                issues.append("missing SameSite attribute")
            else:
                samesite_val = (cookie.get_nonstandard_attr("SameSite") or "").lower()
                if samesite_val == "none" and not cookie.secure:
                    issues.append("SameSite=None without Secure (browsers will reject or, worse, some legacy clients may still send it insecurely)")
            if cookie.name.lower() in ("sessionid", "session", "phpsessid", "jsessionid", "connect.sid",
                                        "auth_token", "session_token", "sid") and not cookie.name.startswith(("__Host-", "__Secure-")):
                issues.append("not using an __Host-/__Secure- cookie name prefix, which would give the browser extra enforcement of Secure/Path/no-Domain")
            if issues:
                findings.append(self._finding(url, ev, f"Cookie '{cookie.name}' is missing security attributes: {', '.join(issues)}.",
                    "Set Secure, HttpOnly, and SameSite=Lax/Strict on all session/auth cookies; consider the __Host- prefix for session cookies.",
                    Severity.MEDIUM, "Confirmed"))
        return findings


class TLSConfigModule(BaseModule):
    category = "tls"; title = "TLS/SSL Configuration"; owasp = "A02:2021"; cwe = "CWE-326"
    min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        parsed = urllib.parse.urlparse(url)
        if parsed.scheme != "https":
            resp, ev = self._get(url)
            findings.append(self._finding(url, ev, "Site is served over plain HTTP, not HTTPS - all traffic is unencrypted and interceptable.",
                "Deploy TLS (e.g. via Let's Encrypt) and redirect all HTTP to HTTPS.",
                Severity.HIGH, "Confirmed"))
            return findings
        import ssl, socket as _socket
        host = parsed.hostname
        port = parsed.port or 443
        try:
            ctx = ssl.create_default_context()
            with _socket.create_connection((host, port), timeout=self.config.scope.timeout_seconds) as sock:
                with ctx.wrap_socket(sock, server_hostname=host) as ssock:
                    cert = ssock.getpeercert()
                    not_after = cert.get("notAfter")
                    not_before = cert.get("notBefore")
                    version = ssock.version()
                    cipher = ssock.cipher()  # (name, protocol, secret_bits)
                    san_entries = [v for k, v in cert.get("subjectAltName", ()) if k == "DNS"]
            ev = RequestEvidence(method="TLS-HANDSHAKE", url=url, status_code=0, response_time_ms=0,
                                  response_headers={}, response_excerpt=f"protocol={version}, notAfter={not_after}, cipher={cipher}")
            if version in ("TLSv1", "TLSv1.1", "SSLv3", "SSLv2"):
                findings.append(self._finding(url, ev, f"Server negotiated outdated protocol {version}, which has known weaknesses.",
                    "Disable TLS 1.0/1.1 and SSLv3; require TLS 1.2 minimum (prefer 1.3).",
                    Severity.HIGH, "Confirmed"))
            if cipher:
                cipher_name = (cipher[0] or "").upper()
                weak_markers = ("RC4", "3DES", "DES", "NULL", "EXPORT", "MD5", "PSK-", "ANON")
                if any(m in cipher_name for m in weak_markers):
                    findings.append(self._finding(url, ev, f"Server negotiated a weak/legacy cipher suite ({cipher_name}) with a "
                        f"{cipher[2]}-bit effective key.",
                        "Restrict the server's cipher suite list to modern AEAD ciphers (AES-GCM, ChaCha20-Poly1305); disable RC4/3DES/export/NULL/anon suites.",
                        Severity.HIGH, "Confirmed"))
            if not_after:
                try:
                    expiry = datetime.datetime.strptime(not_after, "%b %d %H:%M:%S %Y %Z")
                    days_left = (expiry - datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)).days
                    if days_left < 14:
                        findings.append(self._finding(url, ev, f"TLS certificate expires in {days_left} days.",
                            "Renew the certificate; automate renewal (e.g. certbot).",
                            Severity.HIGH if days_left < 3 else Severity.MEDIUM, "Confirmed"))
                    elif days_left > 398:
                        # CA/Browser Forum baseline requirements cap public cert lifetime at 398 days;
                        # a longer-lived cert is either a private/internal CA (fine) or a config anomaly worth a look.
                        findings.append(self._finding(url, ev, f"Certificate validity period is unusually long ({days_left} days remaining), "
                            "exceeding the 398-day maximum public CAs are allowed to issue - likely fine for an internal CA, "
                            "but worth confirming this wasn't issued by a misconfigured or legacy process.",
                            "Confirm this certificate's issuing CA and lifetime are intentional; public-trust certs should be re-issued at least annually.",
                            Severity.INFO, "Possible"))
                except ValueError:
                    pass
            if not_before:
                try:
                    issued = datetime.datetime.strptime(not_before, "%b %d %H:%M:%S %Y %Z")
                    if issued > datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None):
                        findings.append(self._finding(url, ev, "TLS certificate's 'not valid before' date is in the future - "
                            "clients with correct clocks will currently reject this certificate.",
                            "Verify server/certificate issuance clock settings and reissue if needed.",
                            Severity.HIGH, "Confirmed"))
                except ValueError:
                    pass
            if host and not any(self._hostname_matches(host, san) for san in san_entries):
                findings.append(self._finding(url, ev, f"Certificate Subject Alternative Names ({', '.join(san_entries) or 'none'}) "
                    f"do not appear to cover the connected hostname '{host}' - this may indicate a misconfigured "
                    "default/fallback certificate being served.",
                    "Ensure the correct SNI-matched certificate is served for this hostname.",
                    Severity.MEDIUM, "Possible"))
        except Exception as e:
            if self.config.verbose:
                print(f"  [!] TLS check failed for {host}:{port}: {e}")
        return findings

    @staticmethod
    def _hostname_matches(host: str, san: str) -> bool:
        host = host.lower()
        san = san.lower()
        if san.startswith("*."):
            return host.endswith(san[1:]) and host.count(".") == san.count(".")
        return host == san


class ServerBannerModule(BaseModule):
    category = "server_banner"; title = "Server/Framework Version Disclosure"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        for header_name in ("Server", "X-Powered-By", "X-AspNet-Version", "X-Generator", "X-Runtime", "X-Drupal-Cache"):
            val = resp.headers.get(header_name)
            if val and re.search(r"\d+\.\d+", val):
                findings.append(self._finding(url, ev, f"'{header_name}: {val}' discloses specific software version, aiding attackers in matching known CVEs.",
                    f"Remove or generalize the {header_name} header at the reverse proxy/server config level.",
                    Severity.LOW, "Confirmed"))
        return findings


class SecurityTxtModule(BaseModule):
    """Checks for RFC 9116 security.txt - not a vulnerability by itself, but
    its absence is a real, actionable gap: it means there's no documented,
    discoverable channel for a good-faith researcher to report a future
    finding responsibly. Purely informational/hygiene, kept at LOW/INFO."""
    category = "security_txt"; title = "Missing security.txt (RFC 9116)"
    owasp = "A05:2021"; cwe = "CWE-1059"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        for path in ("/.well-known/security.txt", "/security.txt"):
            test_url = urllib.parse.urljoin(url, path)
            try:
                resp, ev = self._get(test_url)
            except (ScopeError, ERSECError):
                continue
            if resp.status_code == 200 and ("contact:" in (resp.text or "").lower()):
                if "expires:" not in resp.text.lower():
                    findings.append(self._finding(test_url, ev,
                        "security.txt is published but has no 'Expires:' field, which RFC 9116 requires - "
                        "without it, tooling can't tell whether the document is stale.",
                        "Add an 'Expires:' field (ISO 8601 date) to security.txt and keep it refreshed periodically.",
                        Severity.INFO, "Confirmed"))
                return findings  # found and valid enough - no missing-file finding
        # Only fetch the homepage once, not per-crawled-page, to avoid noise; caller already
        # dedupes host-level categories, but this check should only ever fire once regardless.
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        findings.append(self._finding(url, ev,
            "No security.txt found at /.well-known/security.txt - there's no standardized, "
            "discoverable channel for security researchers to report vulnerabilities responsibly.",
            "Publish a security.txt per RFC 9116 with a Contact field and an Expires date at /.well-known/security.txt.",
            Severity.INFO, "Possible"))
        return findings


class ExposedFilesModule(BaseModule):
    category = "exposed_files"; title = "Sensitive File / Path Exposure"
    owasp = "A01:2021"; cwe = "CWE-200"; min_profile = ScanProfile.BASELINE

    SENSITIVE_PATHS = [
        "/.git/config", "/.git/HEAD", "/.env", "/.env.local", "/.env.production",
        "/config.php.bak", "/wp-config.php.bak", "/wp-config.php.save", "/.svn/entries",
        "/.hg/hgrc", "/server-status", "/server-info", "/.htaccess", "/.htpasswd",
        "/backup.sql", "/backup.zip", "/dump.sql", "/database.sql", "/db_backup.sql",
        "/.DS_Store", "/web.config", "/composer.json", "/composer.lock",
        "/package.json.bak", "/.aws/credentials", "/id_rsa", "/id_rsa.pub",
        "/phpinfo.php", "/info.php", "/test.php", "/swagger.json", "/swagger-ui.html",
        "/openapi.json", "/api-docs", "/v2/api-docs", "/.well-known/security.txt",
        "/actuator/env", "/actuator/health", "/actuator/heapdump", "/debug/pprof/",
        "/.idea/workspace.xml", "/.vscode/sftp.json", "/docker-compose.yml",
        "/Dockerfile", "/.npmrc", "/.pypirc", "/credentials.json", "/secrets.yml",
        "/config/database.yml", "/storage/logs/laravel.log", "/error_log",
        "/.git/logs/HEAD", "/.gitlab-ci.yml", "/.github/workflows/deploy.yml",
        "/terraform.tfstate", "/terraform.tfvars", "/.terraform/terraform.tfstate",
        "/appsettings.json", "/appsettings.Production.json", "/local.settings.json",
        "/wp-content/debug.log", "/storage/oauth-private.key", "/.ssh/id_rsa",
        "/config.yaml.bak", "/settings.py.bak", "/.circleci/config.yml",
        "/kubeconfig", "/.kube/config", "/serviceaccount/token",
        "/.well-known/openid-configuration", "/graphql/schema.json",
    ]

    def run_url(self, url):
        findings = []
        # Soft-404 baseline: fetch one deliberately-nonexistent, obviously-fake path
        # first and remember its shape. Many real sites (especially SPAs with a
        # catch-all route, or misconfigured servers) return HTTP 200 with the SAME
        # generic page for literally any path - without this baseline, every
        # sensitive-path probe below would look "successful" on such a site and
        # this module would report dozens of false positives. A signal is only
        # trusted if the response meaningfully differs from this baseline shape.
        baseline_body = ""
        baseline_status = None
        try:
            baseline_url = urllib.parse.urljoin(url, f"/ersec-soft-404-baseline-{os.urandom(4).hex()}.probe")
            baseline_resp, _ = self._get(baseline_url)
            baseline_body = baseline_resp.text or ""
            baseline_status = baseline_resp.status_code
        except (ScopeError, ERSECError):
            pass

        for path in self.SENSITIVE_PATHS:
            test_url = urllib.parse.urljoin(url, path)
            try:
                resp, ev = self._get(test_url)
            except (ScopeError, ERSECError):
                continue
            if resp.status_code != 200 or not resp.text:
                continue
            body = resp.text
            looks_like_soft_404 = (
                baseline_status == resp.status_code
                and baseline_body
                and abs(len(body) - len(baseline_body)) < max(50, 0.1 * len(baseline_body))
                and body[:300] == baseline_body[:300]
            )
            if looks_like_soft_404:
                continue
            signals = {
                "/.git/config": "[core]" in body,
                "/.git/HEAD": body.strip().startswith("ref:"),
                "/.git/logs/HEAD": bool(re.search(r"^[0-9a-f]{40}\s", body.strip())),
                "/.env": bool(re.search(r"(DB_PASSWORD|SECRET_KEY|API_KEY)\s*=", body)),
                "/.env.local": bool(re.search(r"(DB_PASSWORD|SECRET_KEY|API_KEY)\s*=", body)),
                "/.env.production": bool(re.search(r"(DB_PASSWORD|SECRET_KEY|API_KEY)\s*=", body)),
                "/server-status": "Apache Server Status" in body,
                "/server-info": "Apache Server Information" in body,
                "/.aws/credentials": "aws_secret_access_key" in body.lower(),
                "/id_rsa": "PRIVATE KEY" in body,
                "/.ssh/id_rsa": "PRIVATE KEY" in body,
                "/id_rsa.pub": body.strip().startswith("ssh-rsa"),
                "/phpinfo.php": "phpinfo()" in body or "PHP Version" in body,
                "/info.php": "phpinfo()" in body or "PHP Version" in body,
                "/swagger.json": '"swagger"' in body or '"openapi"' in body,
                "/openapi.json": '"openapi"' in body,
                "/api-docs": '"swagger"' in body or '"openapi"' in body,
                "/v2/api-docs": '"swagger"' in body or '"openapi"' in body,
                "/graphql/schema.json": '"__schema"' in body or '"types"' in body,
                "/actuator/env": '"activeProfiles"' in body or '"propertySources"' in body,
                "/actuator/heapdump": resp.headers.get("Content-Type", "").startswith("application/octet-stream"),
                "/docker-compose.yml": "version:" in body and "services:" in body,
                "/credentials.json": '"private_key"' in body or '"client_secret"' in body,
                "/secrets.yml": bool(re.search(r"(secret_key_base|password):", body)),
                "/config/database.yml": "adapter:" in body and "database:" in body,
                "/storage/logs/laravel.log": "local.ERROR" in body or "Stack trace:" in body,
                "/terraform.tfstate": '"terraform_version"' in body or '"resources"' in body,
                "/terraform.tfvars": (bool(re.search(r"^\s*\w+\s*=\s*\"", body, re.M)) and "resource" not in body[:200]
                                      and "<html" not in body.lower()[:200] and "<!doctype" not in body.lower()[:200]),
                "/appsettings.json": '"ConnectionStrings"' in body,
                "/appsettings.Production.json": '"ConnectionStrings"' in body,
                "/local.settings.json": '"IsEncrypted"' in body or '"Values"' in body,
                "/wp-content/debug.log": "PHP Fatal error" in body or "PHP Warning" in body,
                "/storage/oauth-private.key": "PRIVATE KEY" in body,
                "/kubeconfig": "apiVersion:" in body and "clusters:" in body,
                "/.kube/config": "apiVersion:" in body and "clusters:" in body,
                "/serviceaccount/token": bool(re.match(r"^[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+$", body.strip())),
                "/.gitlab-ci.yml": "stages:" in body or "script:" in body,
                "/.github/workflows/deploy.yml": "jobs:" in body and "runs-on:" in body,
            }
            confirmed = signals.get(path)
            looks_real = confirmed if confirmed is not None else (
                len(body) > 20 and "<html" not in body.lower()[:200]
            )
            if looks_real:
                critical_paths = ("/.env", "/.env.local", "/.env.production", "/.aws/credentials",
                                   "/id_rsa", "/.ssh/id_rsa", "/credentials.json", "/secrets.yml",
                                   "/actuator/heapdump", "/terraform.tfstate", "/kubeconfig", "/.kube/config",
                                   "/serviceaccount/token", "/storage/oauth-private.key",
                                   "/appsettings.Production.json")
                findings.append(self._finding(test_url, ev, f"Potentially sensitive file/path is publicly accessible: {path}",
                    "Remove the file from the web root, block it at the web-server config level, and rotate any exposed credentials immediately.",
                    Severity.CRITICAL if path in critical_paths else Severity.HIGH,
                    "Likely"))
        return findings


class CORSModule(BaseModule):
    category = "cors"; title = "CORS Misconfiguration"; owasp = "A01:2021"; cwe = "CWE-942"
    min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings = []
        probe_origin = "https://ersec-probe.invalid"
        resp, ev = self._get(url, headers={"Origin": probe_origin})
        acao = resp.headers.get("Access-Control-Allow-Origin")
        acac = resp.headers.get("Access-Control-Allow-Credentials", "").lower() == "true"
        if acao == probe_origin:
            sev = Severity.CRITICAL if acac else Severity.HIGH
            findings.append(self._finding(url, ev,
                f"Server reflects an arbitrary Origin ('{probe_origin}') in Access-Control-Allow-Origin"
                + (" with Allow-Credentials: true, allowing any site to make authenticated requests on a victim's behalf." if acac else "."),
                "Validate Origin against an explicit allow-list server-side; never reflect arbitrary Origins, especially with credentials enabled.",
                sev, "Confirmed"))
        elif acao == "*" and acac:
            findings.append(self._finding(url, ev, "Access-Control-Allow-Origin is '*' combined with Allow-Credentials: true (invalid/dangerous combination some clients still honor).",
                "Never combine wildcard origin with credentialed requests.", Severity.HIGH, "Confirmed"))

        # Null-origin probe: sandboxed iframes, some redirects, and local file:// contexts send
        # 'Origin: null'. A server that reflects/allows it opens a path for a sandboxed-iframe
        # attacker page to make credentialed cross-origin requests too.
        try:
            resp_null, ev_null = self._get(url, headers={"Origin": "null"})
        except (ScopeError, ERSECError):
            return findings
        acao_null = resp_null.headers.get("Access-Control-Allow-Origin")
        if acao_null == "null":
            findings.append(self._finding(url, ev_null,
                "Server explicitly allows Access-Control-Allow-Origin: null, which a sandboxed iframe "
                "(easy for an attacker to serve from any site) can trigger, potentially bypassing an "
                "otherwise-reasonable origin allow-list.",
                "Never explicitly allow the literal 'null' origin; treat it as untrusted like any other unrecognized origin.",
                Severity.HIGH, "Confirmed"))
        return findings


class HTTPMethodsModule(BaseModule):
    category = "http_methods"; title = "Dangerous HTTP Methods Enabled"
    owasp = "A05:2021"; cwe = "CWE-16"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings = []
        try:
            t0 = time.time()
            resp = self.client.request("TRACE", url)
            ev = _make_evidence(resp, int((time.time() - t0) * 1000))
            if resp.status_code == 200 and "TRACE" in (resp.text or ""):
                findings.append(self._finding(url, ev, "HTTP TRACE method is enabled, which can enable Cross-Site Tracing (XST) attacks against cookie protections.",
                    "Disable the TRACE method at the web server/reverse-proxy level.",
                    Severity.MEDIUM, "Confirmed"))
        except (ScopeError, ERSECError):
            pass
        try:
            t0 = time.time()
            resp = self.client.request("OPTIONS", url)
            ev = _make_evidence(resp, int((time.time() - t0) * 1000))
            allow = resp.headers.get("Allow", "")
            risky = [m for m in ("PUT", "DELETE", "TRACE", "CONNECT") if m in allow.upper()]
            if risky:
                findings.append(self._finding(url, ev, f"OPTIONS response advertises potentially risky methods: {', '.join(risky)}.",
                    "Disable unused HTTP methods; ensure PUT/DELETE require authentication and are intentional.",
                    Severity.LOW, "Confirmed"))
        except (ScopeError, ERSECError):
            pass
        # Method-override / verb-tampering signal: some frameworks let a client override the
        # effective HTTP verb via a header, sometimes bypassing verb-scoped auth middleware
        # (e.g. an ACL that only checks for GET vs POST but trusts X-HTTP-Method-Override blindly).
        try:
            baseline_resp, baseline_ev = self._get(url)
            t0 = time.time()
            override_resp = self.client.request("GET", url, headers={"X-HTTP-Method-Override": "DELETE"})
            ev2 = _make_evidence(override_resp, int((time.time() - t0) * 1000))
            if override_resp.status_code != baseline_resp.status_code and override_resp.status_code not in (400, 404, 405, 501):
                findings.append(self._finding(url, ev2,
                    f"Sending X-HTTP-Method-Override: DELETE changed the response status ({baseline_resp.status_code} -> "
                    f"{override_resp.status_code}) versus a plain GET, suggesting the server honors verb-override "
                    "headers - if authorization middleware checks the original verb rather than the effective one, "
                    "this can bypass verb-scoped access controls.",
                    "If method override is required, apply authorization checks AFTER resolving the effective "
                    "method, not before; otherwise disable X-HTTP-Method-Override / _method support entirely.",
                    Severity.MEDIUM, "Possible"))
        except (ScopeError, ERSECError):
            pass
        return findings


class SQLInjectionSignalModule(BaseModule):
    category = "sql_injection"; title = "Possible SQL Injection"
    owasp = "A03:2021"; cwe = "CWE-89"; min_profile = ScanProfile.BASELINE

    ERROR_PATTERNS = [
        "sql syntax", "mysql_fetch", "you have an error in your sql",
        "ora-01756", "ora-00933", "ora-00921", "ora-01722",
        "sqlite3.operationalerror", "sqlite3::sqlexception", "sqlitesyntaxerror",
        "postgresql query failed", "unclosed quotation mark", "unterminated quoted string",
        "microsoft odbc", "microsoft sql server", "unclosed quotation mark after the character string",
        "pg::syntaxerror", "warning: pg_", "npgsql.postgresexception",
        "syntax error at or near", "sqlstate[", "com.microsoft.sqlserver.jdbc",
        "system.data.sqlclient", "oledbexception", "jet database engine",
        "supplied argument is not a valid mysql", "mysqli_fetch", "mysqld: ",
        "conversion failed when converting", "incorrect syntax near",
    ]

    # Cross-database timing probe payloads - matched with response-time confirmation,
    # not just presence in the URL, so this stays a real signal rather than a guess.
    TIME_PROBES = ["1' AND SLEEP(5)--", "1' AND pg_sleep(5)--", "1; WAITFOR DELAY '0:0:5'--"]

    def run_param(self, url, param, method="GET"):
        findings = []
        baseline_resp, baseline_ev = (self._get(url) if method == "GET" else self._post(url))
        probe = "'"
        if method.upper() == "GET":
            test_url = _with_param(url, param, probe)
            resp, ev = self._get(test_url)
            bodies_to_check = [(resp.text or "", ev)]
        else:
            # Try both form-encoding and a JSON body: modern API backends
            # frequently accept only application/json and never see a
            # form-encoded probe at all, which silently missed this class of
            # target entirely before this JSON attempt was added.
            resp_form, ev_form = self._post(url, data={param: probe})
            bodies_to_check = [(resp_form.text or "", ev_form)]
            try:
                resp_json, ev_json = self._post(url, json={param: probe})
                bodies_to_check.append((resp_json.text or "", ev_json))
            except (ScopeError, ERSECError):
                pass

        for body_text, evidence in bodies_to_check:
            lower = body_text.lower()
            if any(p in lower for p in self.ERROR_PATTERNS):
                findings.append(self._finding(evidence.url, evidence,
                    f"Parameter '{param}' triggered a database error signature when sent a single-quote character - "
                    "a classic indicator that user input reaches a SQL query without proper parameterization.",
                    "Use parameterized queries / prepared statements exclusively; never concatenate user input into SQL.",
                    Severity.CRITICAL, "Likely", parameter=param))
                return findings

        # Boolean-based differential: compare TRUE vs FALSE tautology response sizes/status
        if method.upper() == "GET":
            true_url = _with_param(url, param, "1' OR '1'='1")
            false_url = _with_param(url, param, "1' AND '1'='2")
            try:
                true_resp, true_ev = self._get(true_url)
                false_resp, _ = self._get(false_url)
                if (true_resp.status_code == 200 and false_resp.status_code == 200 and
                        abs(len(true_resp.text) - len(baseline_resp.text)) < 5 and
                        abs(len(false_resp.text) - len(baseline_resp.text)) > 50):
                    findings.append(self._finding(true_ev.url, true_ev,
                        f"Parameter '{param}' shows a content-length differential between a true and false SQL "
                        "boolean condition, consistent with boolean-based blind SQL injection.",
                        "Use parameterized queries / prepared statements exclusively.",
                        Severity.CRITICAL, "Possible", parameter=param))
                    return findings
            except (ScopeError, ERSECError):
                pass

        # Time-based blind: try each DB dialect's sleep syntax, confirmed only when the
        # delayed response actually takes meaningfully longer than an unmodified baseline
        # (avoids false positives from generally slow endpoints), and cross-checked with a
        # second, cheap non-delaying probe on the same parameter to rule out an endpoint
        # that's simply always this slow regardless of payload content.
        if method.upper() == "GET":
            for time_probe in self.TIME_PROBES:
                try:
                    t0 = time.time()
                    delayed_resp, delayed_ev = self._get(_with_param(url, param, time_probe))
                    elapsed = time.time() - t0
                except (ScopeError, ERSECError):
                    continue
                if elapsed > 4.0 and baseline_ev.response_time_ms < 2000:
                    # Confirmation pass: same-length non-sleep payload should NOT reproduce the delay.
                    try:
                        control_probe = "1' AND 'a'='a"
                        t1 = time.time()
                        self._get(_with_param(url, param, control_probe))
                        control_elapsed = time.time() - t1
                    except (ScopeError, ERSECError):
                        control_elapsed = 0.0
                    if control_elapsed < 2.0:
                        findings.append(self._finding(delayed_ev.url, delayed_ev,
                            f"Parameter '{param}' caused a ~{elapsed:.1f}s response delay when sent a "
                            f"database-specific time-delay payload ({time_probe.split(chr(39))[0]}...), versus a "
                            f"normal baseline of {baseline_ev.response_time_ms}ms and a same-shape control probe "
                            f"that returned in {control_elapsed:.1f}s - consistent with time-based blind SQL injection.",
                            "Use parameterized queries / prepared statements exclusively.",
                            Severity.CRITICAL, "Likely", parameter=param))
                        break
        return findings


class ReflectedXSSSignalModule(BaseModule):
    category = "xss"; title = "Possible Reflected Cross-Site Scripting"
    owasp = "A03:2021"; cwe = "CWE-79"; min_profile = ScanProfile.BASELINE

    # Three distinct injection contexts, because real-world apps often escape
    # one context (bare HTML tags) while missing another (breaking out of an
    # attribute value, or out of a JS string literal) - a tag-only marker
    # check misses both of the latter, which are extremely common in practice
    # (e.g. a template that HTML-encodes < and > but not quote characters).
    _MARKER = "ersec_xss_probe_9f2c"

    def _probes(self):
        m = self._MARKER
        return [
            ("tag", f"<{m}>", f"<{m}>",
             f"a harmless marker tag '<{m}>' round-tripped verbatim into the HTML body"),
            ("attribute_breakout", f'{m}" onmouseover="1', f'{m}" onmouseover="1',
             f"a quote character in the probe ('{m}\" onmouseover=\"1') was not encoded, breaking out of an "
             "HTML attribute value and adding a new event-handler attribute"),
            ("js_string_breakout", f"{m}';alert(1);'", f"{m}';alert(1);'",
             f"a quote character in the probe ('{m}';alert(1);'') was not encoded, breaking out of a "
             "JavaScript string literal inside a <script> block"),
            ("url_attribute_breakout", f"javascript:{m}(1)", f"javascript:{m}(1)",
             f"a javascript: URI probe was reflected verbatim into an href/src-style attribute without "
             "scheme filtering, which browsers will execute on click/load in that attribute context"),
        ]

    def run_param(self, url, param, method="GET"):
        findings = []
        for context_name, probe, expected_verbatim, evidence_desc in self._probes():
            if method.upper() == "GET":
                test_url = _with_param(url, param, probe)
                resp, ev = self._get(test_url)
            else:
                resp, ev = self._post(url, data={param: probe})
            body = resp.text or ""
            if expected_verbatim in body:
                findings.append(self._finding(ev.url, ev,
                    f"Parameter '{param}' is reflected back into the response without adequate output encoding "
                    f"for its context ({context_name.replace('_', ' ')}): {evidence_desc}, indicating missing "
                    "or context-incomplete output encoding.",
                    "Apply context-aware output encoding: HTML-entity-encode for tag content, "
                    "attribute-encode (including quotes) for attribute values, and JS-string-encode for content "
                    "placed inside <script> blocks. Add a strict Content-Security-Policy as defense-in-depth.",
                    Severity.HIGH, "Confirmed", parameter=param))
                break  # one confirmed context is enough evidence for this parameter
        return findings


class DOMXSSSinkSignalModule(BaseModule):
    """Static, read-only pattern check for classic DOM-XSS sinks fed by
    obviously-attacker-influenceable sources in the page's own inline/linked
    JavaScript. This is NOT execution or fuzzing - it's a source-fed-to-sink
    string match, the same technique DOMPurify's own testing guidance and
    tools like DOMinator use for triage. Always reported at "Possible"
    confidence since a static match doesn't prove reachability or that the
    value is unsanitized in between."""
    category = "xss"; title = "Possible DOM-based XSS Sink"
    owasp = "A03:2021"; cwe = "CWE-79"; min_profile = ScanProfile.DEEP

    _SOURCES = ("location.hash", "location.search", "location.href", "document.referrer",
                "window.name", "document.URL")
    _SINKS = ("innerHTML", "outerHTML", "document.write", "document.writeln",
              "insertAdjacentHTML", "eval(", "setTimeout(", "setInterval(", "Function(")
    # A source assigned/read within roughly one statement of a sink call, in either order.
    _PATTERN = re.compile(
        r"(?:(" + "|".join(re.escape(s) for s in _SOURCES) + r")[^;{}]{0,80}(" +
        "|".join(re.escape(k) for k in _SINKS) + r")"
        r"|(" + "|".join(re.escape(k) for k in _SINKS) + r")[^;{}]{0,80}(" +
        "|".join(re.escape(s) for s in _SOURCES) + r"))",
        re.I,
    )

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        m = self._PATTERN.search(resp.text or "")
        if m:
            findings.append(self._finding(url, ev,
                f"Inline script contains an attacker-influenceable source (e.g. location.hash/search/href, "
                f"document.referrer, or window.name) used near a DOM-XSS sink (innerHTML, document.write, eval, "
                f"etc.) - matched pattern: '{m.group(0)[:120]}'. This is a static source-to-sink co-occurrence "
                "check, not confirmation the value actually flows unsanitized between them.",
                "Trace the actual data flow in browser devtools or a JS-aware SAST tool; if confirmed, replace "
                "the sink with a safe equivalent (textContent instead of innerHTML, DOMPurify.sanitize() before "
                "any HTML sink) and never pass unsanitized location/referrer/name data into eval-family functions.",
                Severity.MEDIUM, "Possible"))
        return findings


class OpenRedirectModule(BaseModule):
    category = "open_redirect"; title = "Open Redirect"; owasp = "A01:2021"; cwe = "CWE-601"
    min_profile = ScanProfile.BASELINE
    LIKELY_PARAMS = {"redirect", "url", "next", "return", "returnurl", "dest", "destination", "continue"}

    def run_param(self, url, param, method="GET"):
        findings = []
        if param.lower() not in self.LIKELY_PARAMS:
            return findings
        probe = "https://ersec-probe.invalid/"
        test_url = _with_param(url, param, probe) if method.upper() == "GET" else url
        try:
            if method.upper() == "GET":
                resp, ev = self._get(test_url)
            else:
                resp, ev = self._post(url, data={param: probe})
        except (ScopeError, ERSECError):
            return findings
        location = resp.headers.get("Location", "")
        if resp.status_code in (301, 302, 303, 307, 308) and "ersec-probe.invalid" in location:
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' controls an unvalidated redirect destination, enabling phishing via a trusted-looking link.",
                "Validate redirect targets against an allow-list of internal paths, or map to indirect reference tokens instead of raw URLs.",
                Severity.MEDIUM, "Confirmed", parameter=param))
            return findings
        # Protocol-relative and backslash bypass variants: some allow-list checks only test for
        # 'http(s)://' literally and miss '//host' (browsers treat as protocol-relative) or
        # backslash forms that certain URL parsers normalize to slashes.
        for bypass_probe in ("//ersec-probe.invalid/", "/\\ersec-probe.invalid/", "https:/ersec-probe.invalid/"):
            try:
                if method.upper() == "GET":
                    resp2, ev2 = self._get(_with_param(url, param, bypass_probe))
                else:
                    resp2, ev2 = self._post(url, data={param: bypass_probe})
            except (ScopeError, ERSECError):
                continue
            loc2 = resp2.headers.get("Location", "")
            if resp2.status_code in (301, 302, 303, 307, 308) and "ersec-probe.invalid" in loc2:
                findings.append(self._finding(ev2.url, ev2,
                    f"Parameter '{param}' accepts a protocol-relative/malformed-scheme redirect bypass "
                    f"('{bypass_probe}') even though a plain 'https://' probe may have been rejected - the "
                    "redirect validator likely does a naive string-prefix check instead of proper URL parsing.",
                    "Parse the redirect target with a proper URL library and check the resulting scheme/host "
                    "against an allow-list, rather than checking for a literal 'http://' or 'https://' prefix string.",
                    Severity.MEDIUM, "Confirmed", parameter=param))
                break
        return findings


class SSRFSignalModule(BaseModule):
    """Server-Side Request Forgery detection. Probes URL-accepting parameters
    with a benign self-referencing pattern that reveals whether the server
    actually fetches the given URL server-side, without pointing the probe at
    real cloud metadata endpoints or internal infrastructure (which would
    edge toward reconnaissance against real internal targets even in a
    detection context) - a safe canary domain and a timing-based signal are
    used instead, matching this tool's non-destructive detection tier."""
    category = "ssrf"; title = "Possible Server-Side Request Forgery"
    owasp = "A10:2021"; cwe = "CWE-918"; min_profile = ScanProfile.BASELINE

    LIKELY_PARAMS = {"url", "uri", "path", "dest", "redirect", "callback", "webhook", "feed",
                      "image", "img", "src", "proxy", "fetch", "target", "endpoint", "avatar", "file"}

    def run_param(self, url, param, method="GET"):
        findings = []
        if param.lower() not in self.LIKELY_PARAMS:
            return findings

        # Signal 1: point at a non-existent local port. If the server fetches
        # this server-side, a connection-refused/timeout pattern differs
        # measurably from a normal response, and some frameworks leak a
        # connection-error message mentioning the probe host into the response.
        probe = "http://127.0.0.1:1/ersec-ssrf-probe"
        try:
            if method.upper() == "GET":
                resp, ev = self._get(_with_param(url, param, probe))
            else:
                resp, ev = self._post(url, data={param: probe})
        except (ScopeError, ERSECError):
            return findings

        lower = (resp.text or "").lower()
        connection_error_signals = [
            "connection refused", "econnrefused", "couldn't connect", "failed to connect",
            "connect() failed", "no route to host", "connection timed out",
        ]
        if any(sig in lower for sig in connection_error_signals):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' triggered a connection-error message when pointed at a local, "
                "non-existent service (127.0.0.1:1), indicating the server fetches this URL itself rather "
                "than treating it as opaque client data - the precondition for SSRF.",
                "Validate and allow-list destination hosts/schemes server-side before fetching any "
                "user-supplied URL; block requests to private/link-local IP ranges (127.0.0.0/8, "
                "169.254.0.0/16, 10.0.0.0/8, 172.17.0.0/12, 192.168.0.0/16) at the network layer as well.",
                Severity.HIGH, "Likely", parameter=param))
            return findings

        # Signal 2: response reflects or resembles a fetched-resource pattern
        # (e.g. an image proxy returning binary/empty differently for a valid
        # vs. invalid internal target) - a softer, lower-confidence signal.
        try:
            baseline_resp, baseline_ev = (self._get(url) if method.upper() == "GET" else self._post(url))
        except (ScopeError, ERSECError):
            return findings
        if resp.status_code != baseline_resp.status_code and resp.status_code in (200, 500, 502, 504):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' produced a different status code ({resp.status_code} vs baseline "
                f"{baseline_resp.status_code}) when pointed at a local address, consistent with the server "
                "attempting to fetch the supplied URL server-side.",
                "Validate and allow-list destination hosts/schemes server-side before fetching any "
                "user-supplied URL; block requests to private/link-local IP ranges at the network layer as well.",
                Severity.MEDIUM, "Possible", parameter=param))
            return findings

        # Signal 3: alternate encodings of localhost that a naive string-based
        # deny-list (checking literally for "127.0.0.1" or "localhost") would miss -
        # decimal IP form and an IPv6 loopback literal. Still fully benign/non-routable.
        for alt_probe, alt_desc in (
            ("http://2130706433:1/ersec-ssrf-probe", "decimal-encoded 127.0.0.1"),
            ("http://[::1]:1/ersec-ssrf-probe", "IPv6 loopback literal"),
        ):
            try:
                if method.upper() == "GET":
                    resp3, ev3 = self._get(_with_param(url, param, alt_probe))
                else:
                    resp3, ev3 = self._post(url, data={param: alt_probe})
            except (ScopeError, ERSECError):
                continue
            lower3 = (resp3.text or "").lower()
            if any(sig in lower3 for sig in connection_error_signals):
                findings.append(self._finding(ev3.url, ev3,
                    f"Parameter '{param}' triggered a connection-error signature when pointed at {alt_desc} "
                    "even though a plain-text loopback probe may not have - suggesting any deny-list in place "
                    "checks for literal strings like '127.0.0.1' rather than resolving/normalizing the address.",
                    "Resolve and normalize the destination host before checking it against a deny/allow-list "
                    "(reject after DNS resolution, not on the raw input string), and re-validate after any redirect.",
                    Severity.HIGH, "Likely", parameter=param))
                break
        return findings


class NoSQLInjectionSignalModule(BaseModule):
    category = "nosql_injection"; title = "Possible NoSQL Injection"
    owasp = "A03:2021"; cwe = "CWE-943"; min_profile = ScanProfile.BASELINE
    ERROR_PATTERNS = ["mongoerror", "bsonerror", "couchdb error", "unterminated string",
                       "objectid failed", "castError".lower(), "$where"]

    def run_param(self, url, param, method="GET"):
        findings = []
        # Operator-injection probe: harmless structural payload, no data access attempted.
        probe = '[$ne]=1'
        test_url = f"{url}{'&' if '?' in url else '?'}{param}{probe}"
        try:
            if method.upper() == "GET":
                resp, ev = self._get(test_url)
            else:
                resp, ev = self._post(url, data={f"{param}[$ne]": "1"})
        except (ScopeError, ERSECError):
            return findings
        lower = (resp.text or "").lower()
        if any(p in lower for p in self.ERROR_PATTERNS):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' triggered a NoSQL database error signature when sent a MongoDB-style "
                "operator payload, suggesting user input reaches a query object without sanitization.",
                "Reject non-scalar input for fields that should be strings/numbers; use a schema validator "
                "(e.g. Joi, Mongoose schemas) before query construction; avoid building queries from raw request bodies.",
                Severity.HIGH, "Likely", parameter=param))
            return findings

        # JSON-body operator injection: the GET/form probe above never reaches an endpoint that only
        # accepts application/json bodies with nested objects (very common for Mongo-backed APIs) -
        # this sends the operator as a genuine nested JSON value instead of a flat form-encoded key.
        if method.upper() == "POST":
            try:
                resp_json, ev_json = self._post(url, json={param: {"$ne": None}})
            except (ScopeError, ERSECError):
                return findings
            lower_json = (resp_json.text or "").lower()
            if any(p in lower_json for p in self.ERROR_PATTERNS):
                findings.append(self._finding(ev_json.url, ev_json,
                    f"Parameter '{param}' triggered a NoSQL database error signature when sent as a nested "
                    "JSON operator object ({{\"$ne\": null}}), suggesting the JSON body is passed into a query "
                    "object without schema validation.",
                    "Validate JSON request bodies against a strict schema (Joi/Zod/Mongoose) before using any "
                    "field in a database query; reject fields containing operator-shaped objects where a scalar is expected.",
                    Severity.HIGH, "Likely", parameter=param))
        return findings


class SSTISignalModule(BaseModule):
    category = "ssti"; title = "Possible Server-Side Template Injection"
    owasp = "A03:2021"; cwe = "CWE-1336"; min_profile = ScanProfile.DEEP

    def run_param(self, url, param, method="GET"):
        findings = []
        # Arithmetic probe that only evaluates inside a template engine's expression syntax -
        # if the engine evaluates it, "49" appears; if not, the literal string is reflected/absent.
        probes = {"{{7*7}}": "49", "${7*7}": "49", "<%= 7*7 %>": "49", "#{7*7}": "49", "{{=7*7}}": "49"}
        for probe, expected in probes.items():
            test_url = _with_param(url, param, probe) if method.upper() == "GET" else url
            try:
                if method.upper() == "GET":
                    resp, ev = self._get(test_url)
                else:
                    resp, ev = self._post(url, data={param: probe})
            except (ScopeError, ERSECError):
                continue
            body = resp.text or ""
            if expected in body and probe not in body:
                findings.append(self._finding(ev.url, ev,
                    f"Parameter '{param}' appears to be evaluated by a server-side template engine "
                    f"(sent '{probe}', got back the computed result '{expected}' instead of the literal text).",
                    "Never pass user input directly into template-rendering functions; use a sandboxed/logic-less "
                    "template mode, or treat all user input strictly as data, never as template source.",
                    Severity.CRITICAL, "Likely", parameter=param))
                break
        return findings


class XXESignalModule(BaseModule):
    """XML External Entity injection detection. The probe defines a benign,
    non-resolving internal entity (referencing a harmless string, not a file
    path or network URL) - this only tests whether the parser expands
    *internal* DTD entities at all, which is the precondition for XXE. It
    deliberately never references file:// or http:// in the entity, so even
    a successful hit here has no data-exfiltration side effect of its own -
    it's a capability signal, not an extraction attempt."""
    category = "xxe"; title = "Possible XML External Entity (XXE) Processing Enabled"
    owasp = "A05:2021"; cwe = "CWE-611"; min_profile = ScanProfile.DEEP

    _MARKER = "ersec_xxe_probe_4d1a"
    _PAYLOAD = (
        '<?xml version="1.0"?>\n'
        '<!DOCTYPE root [<!ENTITY ersec_marker "' + "ersec_xxe_probe_4d1a" + '">]>\n'
        '<root>&ersec_marker;</root>'
    )

    def run_url(self, url):
        findings = []
        # Only worth trying on endpoints that look like they might accept XML at all -
        # guessing this against every HTML page would be pure noise and wasted budget.
        ctype_probe_headers = {"Content-Type": "application/xml"}
        try:
            resp, ev = self._post(url, data=self._PAYLOAD, headers=ctype_probe_headers)
        except (ScopeError, ERSECError):
            return findings
        body = resp.text or ""
        if self._MARKER in body and resp.status_code < 500:
            findings.append(self._finding(ev.url, ev,
                "Posting a minimal XML document with an internal DTD entity declaration to this endpoint "
                "resulted in the entity being expanded and reflected back, indicating the XML parser resolves "
                "DOCTYPE/entity declarations rather than rejecting or ignoring them - the precondition for XXE "
                "(reading local files or reaching internal network resources via crafted entities). No file or "
                "network-referencing entity was sent by this check.",
                "Disable DTD processing and external entity resolution in the XML parser configuration "
                "(e.g. for Java: setFeature('http://apache.org/xml/features/disallow-doctype-decl', true); "
                "for Python's lxml: resolve_entities=False, no_network=True; for PHP: "
                "libxml_disable_entity_loader(true) on older versions, or use a hardened parser preset).",
                Severity.CRITICAL, "Likely"))
        return findings


class IDORHeuristicModule(BaseModule):
    """Insecure Direct Object Reference heuristic. Uses ONLY numeric IDs the
    crawler actually observed the application itself use in real links -
    never a guessed/brute-forced range. For an observed ID N, it requests
    N-1 and N+1 (values effectively adjacent to ones the app already
    exposed, not a scan of arbitrary ID space) and checks whether the
    response looks like a materially different, successfully-loaded record
    rather than a 403/404/redirect-to-login. This alone doesn't prove
    unauthorized access to *another user's* data (that requires knowing
    which ID belongs to whom, out of scope for an unauthenticated/single-
    session scanner) - it's flagged as "Possible" and framed as "worth a
    manual authorization check", not as confirmed IDOR."""
    category = "idor_heuristic"; title = "Possible Insecure Direct Object Reference"
    owasp = "A01:2021"; cwe = "CWE-639"; min_profile = ScanProfile.DEEP

    _NOT_FOUND_SIGNALS = re.compile(r"(not found|404|no longer available|does not exist|access denied|forbidden|"
                                     r"sign in|log in|unauthorized)", re.I)

    def run_url(self, url):
        # This module works off crawler-observed IDs rather than a single URL,
        # so the real logic lives in run_for_crawler(); run_url is a no-op.
        return []

    def run_for_crawler(self, numeric_id_endpoints: Dict[str, Set[str]]) -> List[Finding]:
        findings = []
        checked_bases = 0
        for base, ids in numeric_id_endpoints.items():
            if checked_bases >= 6:  # cap: this is a heuristic pass, not the whole scan budget
                break
            if not ids:
                continue
            try:
                sample_id = int(sorted(ids)[0])
            except ValueError:
                continue
            if sample_id <= 1:
                continue  # nothing meaningfully "adjacent" below it
            checked_bases += 1
            original_url = base.replace("{id}", str(sample_id))
            neighbor_url = base.replace("{id}", str(sample_id - 1))
            try:
                orig_resp, orig_ev = self._get(original_url)
                neighbor_resp, neighbor_ev = self._get(neighbor_url)
            except (ScopeError, ERSECError):
                continue
            orig_body = orig_resp.text or ""
            neighbor_body = neighbor_resp.text or ""
            orig_gated = bool(self._NOT_FOUND_SIGNALS.search(orig_body[:1500]))
            neighbor_gated = bool(self._NOT_FOUND_SIGNALS.search(neighbor_body[:1500]))
            same_shape = (
                neighbor_resp.status_code == 200 and not neighbor_gated
                and abs(len(neighbor_body) - len(orig_body)) < max(200, 0.25 * len(orig_body))
            )
            if orig_resp.status_code == 200 and not orig_gated and same_shape:
                findings.append(self._finding(neighbor_ev.url, neighbor_ev,
                    f"The adjacent numeric ID {sample_id - 1} at this same URL pattern ({base}) returned a "
                    f"200 response of similar shape/size to the originally-crawled ID {sample_id}, with no "
                    "authorization-gate signal (no login redirect, 403, or 'not found' text). This only shows "
                    "that sequential IDs resolve to *some* record without a visible gate - it does NOT confirm "
                    "the record belongs to a different user, which requires manual verification with two "
                    "distinct authenticated accounts.",
                    "Manually verify with two separate user sessions whether adjacent IDs expose another "
                    "user's data. If so, add an object-level authorization check (verify the requesting user "
                    "owns/may access the specific record) rather than relying on the ID being hard to guess, "
                    "and consider switching to non-sequential identifiers (UUIDs) for user-facing object references.",
                    Severity.MEDIUM, "Possible"))
        return findings


class ClickjackingRenderModule(BaseModule):
    """Complements SecurityHeadersModule's static header check with an actual
    framing test: builds a local (never uploaded, never served) HTML document
    that iframes the target and inspects only the HTTP response used for the
    check - this module doesn't execute JS or render a browser, it re-derives
    the effective framing policy the same way a browser's header parser
    would, including a case the static check simplifies: multiple
    X-Frame-Options values from a misconfigured proxy stacking headers,
    which browsers treat inconsistently (some pick the first, some refuse to
    render at all, some pick the most permissive) - see RFC vs. real-world
    UA divergence in MDN's docs."""
    category = "security_headers"; title = "Inconsistent Clickjacking Header Configuration"
    owasp = "A05:2021"; cwe = "CWE-1021"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        xfo_values = resp.raw.headers.get_all("X-Frame-Options") if hasattr(resp.raw, "headers") and hasattr(resp.raw.headers, "get_all") else None
        if xfo_values and len(xfo_values) > 1:
            findings.append(self._finding(url, ev,
                f"Multiple X-Frame-Options headers were sent in the same response ({xfo_values}) - browsers "
                "disagree on how to handle duplicate framing headers (some honor only the first, some refuse "
                "to render the page at all, some take the most permissive), so the effective protection is "
                "unpredictable across your userbase's browser mix.",
                "Emit exactly one X-Frame-Options header (or better, rely solely on CSP frame-ancestors, which "
                "has clearer multi-value semantics); check for a reverse proxy and origin server both adding the header.",
                Severity.LOW, "Confirmed"))
        return findings


class WebSocketExposureModule(BaseModule):
    """Passive check: does the page reference a ws:// (unencrypted) WebSocket
    endpoint, or a wss:// endpoint whose handshake doesn't validate Origin?
    Only the read-only signal (URL scheme in markup/JS, and a real handshake
    attempt with a foreign Origin to see if the server accepts it) is used -
    no messages are sent over an established socket."""
    category = "websocket"; title = "Insecure WebSocket Configuration"
    owasp = "A05:2021"; cwe = "CWE-319"; min_profile = ScanProfile.BASELINE
    _WS_URL_RE = re.compile(r"""["'](wss?://[^"'\s]+)["']""")

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        for match in set(self._WS_URL_RE.findall(resp.text or "")):
            if match.startswith("ws://"):
                findings.append(self._finding(url, ev,
                    f"Page references an unencrypted WebSocket endpoint ({match}), so any data exchanged over "
                    "it can be read or modified in transit exactly like plain HTTP.",
                    "Serve WebSocket connections over wss:// (TLS) exclusively, matching the page's own HTTPS posture.",
                    Severity.MEDIUM, "Confirmed"))
            ws_host = urllib.parse.urlparse(match).hostname
            if ws_host and is_in_scope(match.replace("ws://", "http://").replace("wss://", "https://"), self.config):
                handshake_url = match.replace("wss://", "https://").replace("ws://", "http://")
                try:
                    hs_resp, hs_ev = self._get(handshake_url, headers={
                        "Origin": "https://ersec-probe.invalid",
                        "Connection": "Upgrade", "Upgrade": "websocket",
                        "Sec-WebSocket-Version": "13", "Sec-WebSocket-Key": "ersecx1RaKf/8y1EE3nD/A==",
                    })
                except (ScopeError, ERSECError):
                    continue
                if hs_resp.status_code == 101:
                    findings.append(self._finding(handshake_url, hs_ev,
                        "The WebSocket handshake succeeded (HTTP 101) when sent from an arbitrary, untrusted "
                        "Origin header, suggesting the server does not validate Origin during the upgrade "
                        "handshake - this is the WebSocket equivalent of a missing CORS check and can allow "
                        "cross-site WebSocket hijacking against authenticated users.",
                        "Validate the Origin header server-side during the WebSocket upgrade handshake against "
                        "an explicit allow-list, exactly as you would for CORS.",
                        Severity.HIGH, "Likely"))
        return findings


class DNSEmailSecurityModule(BaseModule):
    """Passive DNS TXT-record lookups for SPF/DMARC (and a DKIM presence
    hint). Pure read-only DNS resolution against public records - the same
    class of check as robots.txt fetching, no different trust boundary than
    resolving the target's own hostname, which every module here already
    does. Uses only the stdlib resolver via socket, no new dependency."""
    category = "email_security"; title = "Missing or Weak Email Anti-Spoofing DNS Records"
    owasp = "A05:2021"; cwe = "CWE-290"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        host = urllib.parse.urlparse(url).hostname
        if not host:
            return findings
        base_domain = self._registrable_guess(host)
        ev = RequestEvidence(method="DNS-TXT", url=f"dns://{base_domain}", status_code=0, response_time_ms=0,
                              response_headers={}, response_excerpt="")
        spf_records = self._txt_lookup(base_domain, prefix=None, contains="v=spf1")
        dmarc_records = self._txt_lookup("_dmarc." + base_domain, prefix=None, contains="v=dmarc1")

        if not spf_records:
            findings.append(self._finding(url, ev,
                f"No SPF TXT record found for {base_domain} - mail servers aren't told which hosts are "
                "authorized to send email as this domain, making it easier to spoof sender addresses in phishing.",
                f"Publish an SPF record (e.g. 'v=spf1 include:_spf.<provider>.com -all') for {base_domain}.",
                Severity.LOW, "Confirmed"))
        elif any("+all" in r or (r.strip().endswith(" all") and "-all" not in r and "~all" not in r and "?all" not in r) for r in spf_records):
            findings.append(self._finding(url, ev,
                f"SPF record for {base_domain} ends in a permissive/neutral 'all' mechanism rather than '-all' "
                "(hard fail), which weakens its anti-spoofing effect.",
                "Change the SPF record's final mechanism to '-all' once all legitimate senders are confirmed covered.",
                Severity.LOW, "Confirmed"))

        if not dmarc_records:
            findings.append(self._finding(url, ev,
                f"No DMARC TXT record found for _dmarc.{base_domain} - even with SPF/DKIM in place, there's no "
                "policy telling receiving mail servers what to do with messages that fail those checks, and no "
                "reporting to notice abuse of the domain.",
                f"Publish a DMARC record at _dmarc.{base_domain} starting at 'v=DMARC1; p=none; rua=mailto:...' "
                "for visibility, then move to p=quarantine/p=reject once legitimate traffic is confirmed covered.",
                Severity.MEDIUM, "Confirmed"))
        else:
            for r in dmarc_records:
                if "p=none" in r.lower():
                    findings.append(self._finding(url, ev,
                        f"DMARC policy for {base_domain} is 'p=none' - reporting-only, with no actual enforcement "
                        "against spoofed mail claiming to be from this domain.",
                        "Move the DMARC policy to 'p=quarantine' or 'p=reject' once SPF/DKIM alignment is confirmed clean via reports.",
                        Severity.LOW, "Confirmed"))
        return findings

    @staticmethod
    def _registrable_guess(host: str) -> str:
        """Best-effort eTLD+1 guess without a public-suffix-list dependency:
        good enough for the common case (example.com, sub.example.co.uk is
        the one shape this under-handles, acceptable for an informational
        DNS hygiene check)."""
        parts = host.split(".")
        if len(parts) <= 2:
            return host
        return ".".join(parts[-2:])

    @staticmethod
    def _txt_lookup(name: str, prefix: Optional[str], contains: str) -> List[str]:
        import socket
        try:
            # dns.resolver isn't a guaranteed dependency; fall back to a raw
            # DNS-over-nothing approach is out of scope, so this uses
            # socket's getaddrinfo indirectly is not applicable for TXT -
            # attempt dnspython if present, else skip gracefully.
            import dns.resolver  # type: ignore
            answers = dns.resolver.resolve(name, "TXT", lifetime=4.0)
            results = []
            for rdata in answers:
                txt = b"".join(rdata.strings).decode("utf-8", "ignore") if hasattr(rdata, "strings") else str(rdata)
                if contains.lower() in txt.lower():
                    results.append(txt)
            return results
        except Exception:
            return []


class JWTInspectionModule(BaseModule):
    category = "jwt"; title = "JWT Configuration Weakness"; owasp = "A02:2021"; cwe = "CWE-347"
    min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        candidates = []
        auth = self.client.session.headers.get("Authorization", "")
        if auth.startswith("Bearer "):
            candidates.append(auth[7:])
        for c in resp.cookies:
            if c.value and c.value.count(".") == 2:
                candidates.append(c.value)
        import base64
        for token in candidates:
            try:
                header_b64 = token.split(".")[0]
                header_b64 += "=" * (-len(header_b64) % 4)
                header = json.loads(base64.urlsafe_b64decode(header_b64))
            except Exception:
                continue
            alg = str(header.get("alg", "")).lower()
            if alg == "none":
                findings.append(self._finding(url, ev,
                    "A JWT was observed with alg=none, meaning signature verification can potentially be bypassed entirely.",
                    "Reject tokens with alg=none server-side; explicitly allow-list expected algorithms when verifying.",
                    Severity.CRITICAL, "Confirmed"))
            elif alg in ("hs256", "hs384", "hs512"):
                findings.append(self._finding(url, ev,
                    f"A JWT uses symmetric algorithm {alg.upper()}. This is fine only if the signing secret is "
                    "strong and never exposed client-side; if the app also accepts asymmetric keys, this can enable "
                    "algorithm-confusion attacks.",
                    "Prefer RS256/ES256 for tokens validated by multiple services; if HS256 is used, ensure the "
                    "secret is high-entropy (32+ bytes) and never shipped to clients.",
                    Severity.LOW, "Confirmed"))
            try:
                payload_b64 = token.split(".")[1]
                payload_b64 += "=" * (-len(payload_b64) % 4)
                payload = json.loads(base64.urlsafe_b64decode(payload_b64))
            except Exception:
                payload = {}
            if payload:
                exp = payload.get("exp")
                iat = payload.get("iat")
                if exp is None:
                    findings.append(self._finding(url, ev,
                        "A JWT was observed with no 'exp' (expiration) claim, meaning the token is valid forever "
                        "once issued and can never be invalidated by waiting it out - only by revoking it "
                        "server-side, if such a mechanism even exists.",
                        "Always set a reasonably short 'exp' claim and use refresh tokens for longer sessions; "
                        "implement server-side revocation (e.g. a denylist or token version field) for early invalidation.",
                        Severity.MEDIUM, "Confirmed"))
                elif isinstance(exp, (int, float)) and isinstance(iat, (int, float)) and (exp - iat) > 60 * 60 * 24 * 30:
                    findings.append(self._finding(url, ev,
                        f"A JWT's lifetime (exp - iat) is unusually long (~{int((exp - iat) / 86400)} days), "
                        "meaning a stolen token remains usable for an extended period.",
                        "Shorten access-token lifetime substantially (minutes to hours) and use a separate, "
                        "revocable refresh-token flow for long-lived sessions.",
                        Severity.LOW, "Confirmed"))
                if isinstance(payload, dict) and any(
                    k.lower() in ("password", "ssn", "credit_card", "secret", "api_key") for k in payload.keys()
                ):
                    findings.append(self._finding(url, ev,
                        "A JWT payload contains a claim key name suggestive of sensitive data (password/SSN/credit "
                        "card/secret-like field). JWT payloads are only base64-encoded, not encrypted, and are "
                        "fully readable by anyone holding the token (including the browser and any proxy in between).",
                        "Never place sensitive data in a JWT payload; store a reference/ID instead and look up "
                        "sensitive fields server-side, or use a JWE (encrypted JWT) if the data must travel in-token.",
                        Severity.HIGH, "Possible"))
        return findings


class DirectoryListingModule(BaseModule):
    category = "directory_listing"; title = "Directory Listing Enabled"
    owasp = "A05:2021"; cwe = "CWE-548"; min_profile = ScanProfile.BASELINE
    PROBE_DIRS = ["/images/", "/uploads/", "/assets/", "/backup/", "/files/", "/static/", "/media/",
                  "/tmp/", "/logs/", "/old/", "/temp/", "/data/", "/export/", "/downloads/"]

    def run_url(self, url):
        findings = []
        for d in self.PROBE_DIRS:
            test_url = urllib.parse.urljoin(url, d)
            try:
                resp, ev = self._get(test_url)
            except (ScopeError, ERSECError):
                continue
            body = resp.text or ""
            if resp.status_code == 200 and ("Index of /" in body or "<title>Directory listing for" in body):
                findings.append(self._finding(test_url, ev,
                    f"Directory listing is enabled at {d}, exposing the full file structure to visitors.",
                    "Disable autoindex/directory browsing at the web server config level (e.g. 'Options -Indexes' in Apache, 'autoindex off;' in nginx).",
                    Severity.MEDIUM, "Confirmed"))
        return findings


class VerboseErrorModule(BaseModule):
    category = "verbose_errors"; title = "Verbose Error / Stack Trace Disclosure"
    owasp = "A05:2021"; cwe = "CWE-209"; min_profile = ScanProfile.BASELINE
    STACK_PATTERNS = [
        "traceback (most recent call last)", "at System.", "stack trace:", "django.core.exceptions",
        "org.springframework", "in <module>", "fatal error:", "warning: include(",
        "unhandled exception", "debug=true", "whoops! there was an error",
    ]

    def run_param(self, url, param, method="GET"):
        findings = []
        probe = "\x00%s" % ("A" * 5000)  # null byte + oversized value: safe, commonly trips debug handlers
        try:
            if method.upper() == "GET":
                resp, ev = self._get(_with_param(url, param, probe))
            else:
                resp, ev = self._post(url, data={param: probe})
        except (ScopeError, ERSECError):
            return findings
        lower = (resp.text or "").lower()
        if resp.status_code >= 500 and any(p in lower for p in self.STACK_PATTERNS):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' triggered a server error that returned a detailed stack trace or debug page, "
                "which can leak file paths, framework versions, and internal logic to attackers.",
                "Disable debug/verbose error modes in production (e.g. DEBUG=False in Django, "
                "NODE_ENV=production, custom error pages); log details server-side only.",
                Severity.MEDIUM, "Confirmed", parameter=param))
        return findings


class MixedContentModule(BaseModule):
    category = "mixed_content"; title = "Mixed Content"; owasp = "A02:2021"; cwe = "CWE-319"
    min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        if not url.startswith("https://"):
            return findings
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        try:
            soup = BeautifulSoup(resp.text, "html.parser")
        except Exception:
            return findings
        insecure = []
        for tag, attr in (("script", "src"), ("img", "src"), ("link", "href"), ("iframe", "src")):
            for el in soup.find_all(tag):
                src = el.get(attr, "")
                if src.startswith("http://"):
                    insecure.append(src)
        if insecure:
            findings.append(self._finding(url, ev,
                f"Page loads {len(insecure)} resource(s) over plain HTTP on an HTTPS page (e.g. {insecure[0]}), "
                "which browsers may block and which can be tampered with in transit.",
                "Change all resource URLs to HTTPS or protocol-relative; use a CSP 'upgrade-insecure-requests' directive as a backstop.",
                Severity.LOW, "Confirmed"))
        return findings


class SubresourceIntegrityModule(BaseModule):
    category = "sri"; title = "Missing Subresource Integrity on Third-Party Scripts"
    owasp = "A08:2021"; cwe = "CWE-829"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        try:
            soup = BeautifulSoup(resp.text, "html.parser")
        except Exception:
            return findings
        host = urllib.parse.urlparse(url).hostname
        missing = []
        for tag in soup.find_all("script", src=True):
            src = tag["src"]
            src_host = urllib.parse.urlparse(urllib.parse.urljoin(url, src)).hostname
            if src_host and src_host != host and not tag.get("integrity"):
                missing.append(src)
        missing_style = []
        for tag in soup.find_all("link", rel="stylesheet", href=True):
            href = tag["href"]
            src_host = urllib.parse.urlparse(urllib.parse.urljoin(url, href)).hostname
            if src_host and src_host != host and not tag.get("integrity"):
                missing_style.append(href)
        if missing:
            findings.append(self._finding(url, ev,
                f"{len(missing)} third-party script(s) are loaded without a Subresource Integrity (SRI) hash "
                f"(e.g. {missing[0]}), so a compromised CDN could inject arbitrary code into this page.",
                "Add integrity=\"sha384-...\" and crossorigin=\"anonymous\" attributes to third-party <script> tags, "
                "or self-host critical dependencies.",
                Severity.LOW, "Confirmed"))
        if missing_style:
            findings.append(self._finding(url, ev,
                f"{len(missing_style)} third-party stylesheet(s) are loaded without SRI (e.g. {missing_style[0]}). "
                "CSS can be used for data-exfiltration attacks (attribute selectors leaking form values) and "
                "UI redressing, so a compromised CDN serving modified CSS is a real, if lower-severity, risk.",
                "Add integrity/crossorigin attributes to third-party <link rel=\"stylesheet\"> tags, or self-host.",
                Severity.INFO, "Confirmed"))
        return findings


class InsecureFormModule(BaseModule):
    category = "insecure_form"; title = "Sensitive Form Submitted Over HTTP / Insecure Autocomplete"
    owasp = "A02:2021"; cwe = "CWE-319"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        try:
            soup = BeautifulSoup(resp.text, "html.parser")
        except Exception:
            return findings
        for form in soup.find_all("form"):
            pw_inputs = form.find_all("input", {"type": "password"})
            if not pw_inputs:
                continue
            action = urllib.parse.urljoin(url, form.get("action", "") or url)
            if action.startswith("http://"):
                findings.append(self._finding(action, ev,
                    "A password field submits to an http:// (non-TLS) form action, exposing credentials in transit.",
                    "Serve the entire site over HTTPS and ensure form actions use https:// or relative URLs.",
                    Severity.CRITICAL, "Confirmed"))
            for pw in pw_inputs:
                autocomplete = (pw.get("autocomplete") or "").lower()
                if autocomplete not in ("off", "new-password", "current-password"):
                    findings.append(self._finding(url, ev,
                        "A password field does not set autocomplete='new-password'/'current-password' or 'off', "
                        "allowing browsers to store/autofill credentials on shared devices.",
                        "Set an explicit autocomplete attribute appropriate to the field's purpose on all password inputs.",
                        Severity.LOW, "Confirmed"))
                    break
        return findings


class GraphQLIntrospectionModule(BaseModule):
    category = "graphql_introspection"; title = "GraphQL Introspection Enabled"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.BASELINE
    ENDPOINTS = ["/graphql", "/api/graphql", "/v1/graphql"]

    def run_url(self, url):
        findings = []
        query = {"query": "{__schema{types{name}}}"}
        for ep in self.ENDPOINTS:
            test_url = urllib.parse.urljoin(url, ep)
            try:
                resp, ev = self._post(test_url, json=query, headers={"Content-Type": "application/json"})
            except (ScopeError, ERSECError):
                continue
            if resp.status_code == 200 and '"__schema"' in (resp.text or ""):
                findings.append(self._finding(test_url, ev,
                    f"GraphQL introspection is enabled at {ep}, allowing anyone to enumerate the entire API schema "
                    "including fields that may not be intended for public use.",
                    "Disable introspection in production (e.g. NoSchemaIntrospectionCustomRule in graphql-js, "
                    "or the framework's production-mode flag), or gate it behind authentication.",
                    Severity.MEDIUM, "Confirmed"))
                # Query-depth / batching DoS signal: a deeply-nested but still harmless
                # introspection-shaped query (no data mutation, no large result set) that
                # only costs the server extra parse/validation work if it's accepted at all.
                deep_query = {"query": "{__type(name:\"Query\"){fields{type{fields{type{fields{type{name}}}}}}}}"}
                try:
                    deep_resp, deep_ev = self._post(test_url, json=deep_query, headers={"Content-Type": "application/json"})
                    if deep_resp.status_code == 200 and "errors" not in (deep_resp.text or "").lower()[:200]:
                        findings.append(self._finding(test_url, deep_ev,
                            f"A deeply-nested introspection query (6 levels) at {ep} was accepted without a "
                            "query-depth or complexity error, suggesting no query-depth limiting is configured - "
                            "a real attacker query nested much deeper (not sent by this check) could cause "
                            "denial-of-service through exponential resolver execution.",
                            "Add query depth limiting and/or query cost analysis (e.g. graphql-depth-limit, "
                            "graphql-cost-analysis, or the framework's built-in complexity limiter).",
                            Severity.LOW, "Possible"))
                except (ScopeError, ERSECError):
                    pass
        return findings


class CRLFInjectionSignalModule(BaseModule):
    category = "crlf_injection"; title = "Possible CRLF / HTTP Response Splitting"
    owasp = "A03:2021"; cwe = "CWE-93"; min_profile = ScanProfile.BASELINE

    def run_param(self, url, param, method="GET"):
        findings = []
        marker = "ersec-crlf-check"
        probes = [
            f"test%0d%0aX-Ersec-Injected:{marker}",
            f"test%0d%0a%0d%0a<html>{marker}</html>",  # response-splitting into a fresh body
            f"test%0aX-Ersec-Injected:{marker}",         # bare-LF variant some parsers still honor
        ]
        if method.upper() != "GET":
            return findings
        for probe in probes:
            test_url = _with_param(url, param, probe)
            try:
                resp, ev = self._get(test_url)
            except (ScopeError, ERSECError):
                continue
            if resp.headers.get("X-Ersec-Injected") == marker or marker in (resp.text or ""):
                findings.append(self._finding(ev.url, ev,
                    f"Parameter '{param}' allows injection of raw CRLF/LF sequences that create an arbitrary "
                    "new response header or split the response body, indicating unsanitized input is written "
                    "into raw HTTP responses.",
                    "URL-encode or strip CR/LF characters from any user input used in header values or redirects; "
                    "use your framework's header-setting API rather than raw string concatenation.",
                    Severity.HIGH, "Confirmed", parameter=param))
                break
        return findings


class HostHeaderInjectionModule(BaseModule):
    category = "host_header"; title = "Host Header Injection"; owasp = "A05:2021"; cwe = "CWE-644"
    min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings = []
        probe_host = "ersec-host-probe.invalid"
        try:
            resp, ev = self._get(url, headers={"Host": probe_host})
        except (ScopeError, ERSECError):
            return findings
        location = resp.headers.get("Location", "")
        if probe_host in (resp.text or "") or probe_host in location:
            findings.append(self._finding(url, ev,
                "A forged Host header is reflected into the page body or a redirect Location, which can enable "
                "password-reset-link poisoning or cache poisoning.",
                "Validate the Host header against an explicit allow-list of expected domains server-side; do not "
                "trust it when building absolute URLs (password reset links, canonical URLs, etc.).",
                Severity.HIGH, "Likely"))

        # X-Forwarded-Host variant: many apps trust a proxy-set forwarded-host header for building
        # absolute URLs even when they correctly ignore the raw Host header itself.
        try:
            resp2, ev2 = self._get(url, headers={"X-Forwarded-Host": probe_host})
        except (ScopeError, ERSECError):
            return findings
        location2 = resp2.headers.get("Location", "")
        if probe_host in (resp2.text or "") or probe_host in location2:
            findings.append(self._finding(url, ev2,
                "A forged X-Forwarded-Host header is reflected into the page body or a redirect Location. Apps "
                "behind a reverse proxy often trust this header for building absolute URLs - if the proxy "
                "doesn't strip/overwrite client-supplied values for it, this is exploitable exactly like Host "
                "header injection.",
                "Only trust X-Forwarded-Host when it's set by your own reverse proxy (strip any client-supplied "
                "value at the edge); validate it against an allow-list before using it to build absolute URLs.",
                Severity.HIGH, "Likely"))
        return findings


class VulnerableLibraryModule(BaseModule):
    category = "vulnerable_library"; title = "Vulnerable Client-Side Library"
    owasp = "A06:2021"; cwe = "CWE-1104"; min_profile = ScanProfile.BASELINE

    # Small curated table of well-known vulnerable version ranges (major.minor floor
    # below which known CVEs exist). Not exhaustive - a real dependency-scanning tool
    # (npm audit, Snyk, Retire.js) should still be run in CI for full coverage.
    _KNOWN_VULNERABLE = {
        "jquery": (3, 5, 0),        # pre-3.5.0: XSS via htmlPrefilter (CVE-2020-11022/23)
        "angular": (1, 8, 0),       # pre-1.8.0: various sanitizer bypass issues
        "bootstrap": (3, 4, 1),     # pre-3.4.1: XSS in tooltip/affix/scrollspy
        "lodash": (4, 17, 12),      # pre-4.17.12: prototype pollution (CVE-2019-10744)
        "moment": (2, 29, 4),       # pre-2.29.4: ReDoS (CVE-2022-31129)
        "handlebars": (4, 7, 7),    # pre-4.7.7: prototype pollution
        "underscore": (1, 12, 1),   # pre-1.12.1: template() sandbox escape (CVE-2021-23358)
        "axios": (0, 21, 2),        # pre-0.21.2: SSRF via redirect handling (CVE-2021-3749 family)
        "yui": (2, 9, 0),           # legacy YUI2 - multiple unpatched XSS issues, deprecated entirely
        "prototype": (1, 7, 3),     # pre-1.7.3: known XSS gadget chains
    }
    _VERSION_RE = re.compile(r"(jquery|angular|bootstrap|lodash|moment|handlebars|underscore|axios|yui|prototype)[.\-]?(\d+)\.(\d+)\.(\d+)", re.I)

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        try:
            soup = BeautifulSoup(resp.text, "html.parser")
        except Exception:
            return findings
        for tag in soup.find_all("script", src=True):
            src = tag["src"]
            m = self._VERSION_RE.search(src)
            if not m:
                continue
            lib = m.group(1).lower()
            version = (int(m.group(2)), int(m.group(3)), int(m.group(4)))
            floor = self._KNOWN_VULNERABLE.get(lib)
            if floor and version < floor:
                findings.append(self._finding(url, ev,
                    f"Page loads {lib} version {'.'.join(map(str,version))} (from {src}), which is below "
                    f"the {'.'.join(map(str,floor))} floor where known public CVEs were fixed for this library.",
                    f"Upgrade {lib} to the latest stable release and re-check via 'npm audit' or Snyk for the exact CVE list.",
                    Severity.MEDIUM, "Likely"))
        return findings


class InfoDisclosureCommentsModule(BaseModule):
    category = "info_disclosure_comments"; title = "Sensitive Information in HTML Comments"
    owasp = "A01:2021"; cwe = "CWE-615"; min_profile = ScanProfile.BASELINE
    SUSPICIOUS = re.compile(r"(password|passwd|api[_-]?key|secret|todo.{0,20}(fix|remove|security)|"
                             r"internal use only|do not deploy|admin panel|backdoor)", re.I)

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        comments = re.findall(r"<!--(.*?)-->", resp.text or "", re.DOTALL)
        for c in comments:
            if self._SUSPICIOUS_MATCH(c):
                findings.append(self._finding(url, ev,
                    "An HTML comment in the page source contains text matching a sensitive-information pattern "
                    "(credentials, internal notes, or a security-relevant TODO).",
                    "Strip HTML comments from production builds; move sensitive notes out of source templates entirely.",
                    Severity.LOW, "Possible"))
                break

        # Same idea applied to inline <script> blocks: hardcoded-secret-shaped string
        # literals (API key/token patterns) are a distinct and often higher-severity
        # leak versus a stray comment - genuine credentials, not just notes.
        secret_re = re.compile(
            r"(?:api[_-]?key|secret[_-]?key|access[_-]?token|client[_-]?secret)\s*[:=]\s*['\"]([A-Za-z0-9_\-]{16,})['\"]",
            re.I,
        )
        for script in re.findall(r"<script(?:[^>]*)>(.*?)</script>", resp.text or "", re.DOTALL | re.I):
            m = secret_re.search(script)
            if m:
                findings.append(self._finding(url, ev,
                    "An inline <script> block contains what looks like a hardcoded API key/secret/token string "
                    "literal assigned to a credential-shaped variable name.",
                    "Never embed secrets in client-side JavaScript - any value shipped to the browser is public. "
                    "Move the credential server-side and expose only a scoped, short-lived token if the client "
                    "genuinely needs to call the API directly; rotate the leaked value immediately.",
                    Severity.HIGH, "Possible"))
                break
        return findings

    def _SUSPICIOUS_MATCH(self, text):
        return bool(self.SUSPICIOUS.search(text))


class CacheControlModule(BaseModule):
    category = "cache_control"; title = "Missing Cache-Control on Authenticated Content"
    owasp = "A01:2021"; cwe = "CWE-525"; min_profile = ScanProfile.BASELINE
    _AUTH_COOKIE_HINTS = {"sessionid", "session", "phpsessid", "jsessionid", "connect.sid",
                           "auth_token", "access_token", "laravel_session"}

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        has_auth_cookie = any(c.name.lower() in self._AUTH_COOKIE_HINTS for c in self.client.session.cookies) or \
                           any(c.name.lower() in self._AUTH_COOKIE_HINTS for c in resp.cookies)
        if not has_auth_cookie:
            return findings
        cache_control = resp.headers.get("Cache-Control", "").lower()
        if "no-store" not in cache_control:
            findings.append(self._finding(url, ev,
                "A response associated with an authenticated session does not set Cache-Control: no-store, "
                "which can allow shared caches or browser back/forward navigation to expose it to a different user.",
                "Set Cache-Control: no-store (and Pragma: no-cache) on all authenticated/sensitive responses.",
                Severity.LOW, "Possible"))
        return findings


class HeaderInjectionModule(BaseModule):
    """Many apps log or reflect request headers (User-Agent in an analytics
    dashboard, Referer in a 'referred from' widget, X-Forwarded-For in
    error pages/logs) without the same output-encoding discipline applied
    to query parameters - this is a commonly-missed injection surface that
    a query-parameter-only scanner never touches."""
    category = "header_injection"; title = "Injectable Request Header (XSS/SQLi Signal)"
    owasp = "A03:2021"; cwe = "CWE-79"; min_profile = ScanProfile.BASELINE

    HEADERS_TO_TEST = ["User-Agent", "Referer", "X-Forwarded-For", "X-Forwarded-Host"]
    XSS_MARKER = "ersec_hdr_xss_7f3a"

    def run_url(self, url):
        findings = []
        sqli_patterns = SQLInjectionSignalModule.ERROR_PATTERNS
        for header_name in self.HEADERS_TO_TEST:
            xss_probe = f"<{self.XSS_MARKER}>"
            try:
                resp, ev = self._get(url, headers={header_name: xss_probe})
            except (ScopeError, ERSECError):
                continue
            if xss_probe in (resp.text or ""):
                findings.append(self._finding(url, ev,
                    f"The {header_name} request header is reflected back into the HTML response completely "
                    "unescaped, indicating this header is logged or displayed (e.g. an analytics/admin view, "
                    "an error page, or a 'referred from' widget) without output encoding.",
                    f"Apply the same output encoding to {header_name} (and any other logged/displayed request "
                    "header) that you apply to query parameters - headers are just as attacker-controlled.",
                    Severity.HIGH, "Confirmed"))
                continue  # one finding per header is enough; move to next header

            sqli_probe = "'"
            try:
                resp2, ev2 = self._get(url, headers={header_name: sqli_probe})
            except (ScopeError, ERSECError):
                continue
            lower = (resp2.text or "").lower()
            if any(p in lower for p in sqli_patterns):
                findings.append(self._finding(url, ev2,
                    f"The {header_name} request header triggered a database error signature when it contained "
                    "a single quote, suggesting this header reaches a SQL query (commonly via request logging "
                    "to a database) without parameterization.",
                    f"Parameterize any query that incorporates {header_name} or other request headers, "
                    "exactly as you would for query parameters or form fields.",
                    Severity.CRITICAL, "Likely"))
        return findings


class MassAssignmentSignalModule(BaseModule):
    """Mass assignment / excessive data binding. Sends a POST/JSON body that
    adds a handful of privilege-shaped extra fields (isAdmin, role, etc.)
    ON TOP OF the form's own legitimate fields, and checks only whether the
    API echoes those extra fields back accepted/unchanged in a JSON
    response - never whether the privilege actually took effect (that would
    require an authenticated follow-up request this tool doesn't make). A
    positive is a precondition signal ("the field was accepted"), not proof
    of privilege escalation - always reported at "Possible"."""
    category = "mass_assignment"; title = "Possible Mass Assignment / Excessive Data Binding"
    owasp = "A08:2021"; cwe = "CWE-915"; min_profile = ScanProfile.DEEP

    _PRIVILEGE_FIELDS = {
        "isAdmin": True, "is_admin": True, "admin": True, "role": "admin",
        "isVerified": True, "is_verified": True, "accountBalance": 999999,
        "credits": 999999, "permissions": ["admin"],
    }

    def run_url(self, url):
        # Handled per-form in run_for_forms(); run_url is intentionally a no-op
        # since this needs the form's own field set to build a realistic body.
        return []

    def run_for_forms(self, forms: List["FormInfo"]) -> List[Finding]:
        findings = []
        checked = 0
        for form in forms:
            if form.method.upper() != "POST" or checked >= 10:
                continue
            checked += 1
            base_body = {inp["name"]: (inp.get("value") or "ersec_test") for inp in form.inputs
                         if inp.get("type") not in ("submit", "button")}
            if not base_body:
                continue
            probe_body = dict(base_body)
            probe_body.update(self._PRIVILEGE_FIELDS)
            target_url = form.action or form.page_url
            try:
                resp, ev = self._post(target_url, json=probe_body, headers={"Content-Type": "application/json"})
            except (ScopeError, ERSECError):
                continue
            ctype = resp.headers.get("Content-Type", "")
            if "application/json" not in ctype:
                continue
            body_lower = (resp.text or "").lower()
            echoed = [k for k in self._PRIVILEGE_FIELDS if k.lower() in body_lower]
            if echoed and resp.status_code < 400:
                findings.append(self._finding(ev.url, ev,
                    f"Submitting extra, privilege-shaped JSON fields ({', '.join(echoed)}) alongside this "
                    f"form's normal fields resulted in a {resp.status_code} response that echoes those fields "
                    "back, suggesting the backend binds the entire request body to a model/entity rather than "
                    "an explicit allow-list of expected fields. This shows the fields were ACCEPTED, not that "
                    "privilege escalation actually occurred - confirm by checking, with a second authenticated "
                    "request, whether the account's actual role/balance/permissions changed.",
                    "Use an explicit allow-list (DTO / serializer with defined fields) for what a request body "
                    "may set, rather than binding the full request body directly onto a database model. "
                    "Never let client-supplied input set privilege, role, or balance fields directly.",
                    Severity.MEDIUM, "Possible"))
        return findings


class UserEnumerationSignalModule(BaseModule):
    """Login/registration/password-reset forms often respond differently for
    a known-format-but-nonexistent identifier versus a subtly different one,
    letting an attacker enumerate valid usernames/emails before ever
    attempting a password. This sends exactly two harmless, clearly-fake
    probe values (never real-looking credentials, never a password field
    populated with anything but a fixed dummy string) and diffs the
    responses - it does not attempt to log in as anyone or guess passwords."""
    category = "user_enumeration"; title = "Possible Username/Email Enumeration"
    owasp = "A07:2021"; cwe = "CWE-204"; min_profile = ScanProfile.DEEP
    _RELEVANT_FIELD_NAMES = re.compile(r"(email|username|user|login|identifier)", re.I)
    _RELEVANT_PATH = re.compile(r"(login|signin|sign-in|register|signup|sign-up|forgot|reset|password)", re.I)

    def run_for_forms(self, forms: List["FormInfo"]) -> List[Finding]:
        findings = []
        checked = 0
        for form in forms:
            if form.method.upper() != "POST" or checked >= 8:
                continue
            if not self._RELEVANT_PATH.search(form.page_url) and not self._RELEVANT_PATH.search(form.action or ""):
                continue
            id_field = next((i["name"] for i in form.inputs if self._RELEVANT_FIELD_NAMES.search(i["name"])), None)
            if not id_field:
                continue
            checked += 1
            base_body = {inp["name"]: "ersec_probe_value" for inp in form.inputs if inp.get("type") not in ("submit", "button")}
            pw_field = next((i["name"] for i in form.inputs if i.get("type") == "password"), None)
            if pw_field:
                base_body[pw_field] = "ErsecProbe!2024x"  # fixed, obviously-fake dummy - never a real/guessable password
            target_url = form.action or form.page_url

            body_a = dict(base_body)
            body_a[id_field] = "ersec-nonexistent-probe-account-9f31@ersec-probe.invalid"
            body_b = dict(base_body)
            body_b[id_field] = "ersec-nonexistent-probe-account-4c02@ersec-probe.invalid"
            try:
                resp_a, ev_a = self._post(target_url, data=body_a)
                resp_b, ev_b = self._post(target_url, data=body_b)
            except (ScopeError, ERSECError):
                continue
            # Both probe identifiers are equally fake/nonexistent, so a well-behaved
            # endpoint should respond IDENTICALLY to both. This check only looks for
            # differences between two equally-fake inputs - it never compares a real
            # account against a fake one, so it cannot itself reveal which of two
            # values corresponds to a real account.
            if (resp_a.status_code != resp_b.status_code or
                    abs(len(resp_a.text or "") - len(resp_b.text or "")) > 30):
                findings.append(self._finding(ev_a.url, ev_a,
                    f"Two equally-nonexistent probe identifiers submitted to this form produced measurably "
                    f"different responses (status {resp_a.status_code} vs {resp_b.status_code}, body length "
                    f"{len(resp_a.text or '')} vs {len(resp_b.text or '')}), which is unexpected if the "
                    "endpoint treats all unknown identifiers uniformly - this pattern is how username/email "
                    "enumeration usually surfaces, though a real enumeration test needs one probe value that "
                    "corresponds to an actual account, which this check deliberately never sends.",
                    "Return an identical, generic response (same status code, same message, same approximate "
                    "timing) regardless of whether the submitted identifier corresponds to a real account, for "
                    "login, registration, and password-reset flows alike.",
                    Severity.LOW, "Possible"))
        return findings


class SubdomainTakeoverSignalModule(BaseModule):
    """Read-only DNS CNAME check: if the target's hostname (or a same-site
    subdomain the crawler discovered via third-party origins) has a CNAME
    pointing at a known SaaS platform's default domain, and requesting that
    platform's default hostname pattern returns a 'not claimed here' style
    error page, the CNAME target is available for anyone to claim - a
    classic subdomain-takeover setup. This performs only a DNS CNAME lookup
    plus a single benign GET to the CNAME target itself (never registers,
    claims, or modifies anything)."""
    category = "subdomain_takeover"; title = "Possible Dangling CNAME / Subdomain Takeover"
    owasp = "A05:2021"; cwe = "CWE-350"; min_profile = ScanProfile.BASELINE

    # (CNAME suffix substring, response-body substring indicating "unclaimed")
    _FINGERPRINTS = [
        ("github.io", "there isn't a github pages site here"),
        ("herokudns.com", "no such app"),
        ("herokuapp.com", "no such app"),
        ("azurewebsites.net", "404 web site not found"),
        ("cloudapp.net", "404"),
        ("s3.amazonaws.com", "nosuchbucket"),
        ("s3-website", "nosuchbucket"),
        ("readme.io", "project doesnt exist"),
        ("shopify.com", "sorry, this shop is currently unavailable"),
        ("unbouncepages.com", "the page you were looking for doesn't exist"),
        ("wordpress.com", "do you want to register"),
        ("fastly.net", "fastly error: unknown domain"),
        ("pantheonsite.io", "404 error: unknown site"),
        ("surge.sh", "project not found"),
        ("zendesk.com", "help center closed"),
    ]

    def run_url(self, url):
        findings = []
        host = urllib.parse.urlparse(url).hostname
        if not host:
            return findings
        cname = self._resolve_cname(host)
        if not cname:
            return findings
        for suffix, error_marker in self._FINGERPRINTS:
            if suffix not in cname.lower():
                continue
            probe_url = f"https://{host}/"
            try:
                resp, ev = self._get(probe_url)
            except (ScopeError, ERSECError):
                return findings
            if error_marker in (resp.text or "").lower():
                findings.append(self._finding(probe_url, ev,
                    f"'{host}' has a CNAME record pointing to '{cname}' (a {suffix} service), and requesting "
                    f"the site returns content matching that platform's 'unclaimed/not found' error page "
                    f"('{error_marker}'). If the account/site that used to own this DNS target was deleted or "
                    "renamed, anyone can now register that same name on the platform and serve content under "
                    "this organization's subdomain.",
                    f"Either remove the dangling CNAME record for '{host}' if it's no longer in use, or "
                    f"re-claim/re-register the corresponding resource on {suffix} under this organization's account.",
                    Severity.HIGH, "Likely"))
            break
        return findings

    @staticmethod
    def _resolve_cname(host: str) -> Optional[str]:
        try:
            import dns.resolver  # type: ignore
            answers = dns.resolver.resolve(host, "CNAME", lifetime=4.0)
            for rdata in answers:
                return str(rdata.target).rstrip(".")
        except Exception:
            return None
        return None


class JWTKeyConfusionSignalModule(BaseModule):
    """Passive, decode-only checks for two well-known JWT signing-key
    confusion setups: (1) a 'jku' or 'x5u' header claiming an externally-
    fetchable key URL, which - if the verifier doesn't restrict it to a
    trusted host allow-list - lets an attacker host their own signing key
    and point the token at it; (2) a 'kid' (key ID) header value that looks
    like a filesystem path or contains SQL/path-traversal metacharacters,
    suggesting 'kid' is used to build a file path or database lookup
    without sanitization. Purely decodes tokens already seen on the wire -
    never forges, re-signs, or attempts to construct a working forged token."""
    category = "jwt"; title = "JWT Key-Confusion / Untrusted Key Source Signal"
    owasp = "A02:2021"; cwe = "CWE-347"; min_profile = ScanProfile.PASSIVE

    _SUSPICIOUS_KID = re.compile(r"(\.\./|/etc/|['\";]|union\s+select|\$\{)", re.I)

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        candidates = []
        auth = self.client.session.headers.get("Authorization", "")
        if auth.startswith("Bearer "):
            candidates.append(auth[7:])
        for c in resp.cookies:
            if c.value and c.value.count(".") == 2:
                candidates.append(c.value)
        import base64
        for token in candidates:
            try:
                header_b64 = token.split(".")[0]
                header_b64 += "=" * (-len(header_b64) % 4)
                header = json.loads(base64.urlsafe_b64decode(header_b64))
            except Exception:
                continue
            if "jku" in header or "x5u" in header:
                key_url = header.get("jku") or header.get("x5u")
                findings.append(self._finding(url, ev,
                    f"A JWT header includes a '{'jku' if 'jku' in header else 'x5u'}' claim pointing at "
                    f"'{key_url}' - if the verifying server fetches and trusts whatever key this URL serves "
                    "without checking it against an allow-list of trusted key-hosting domains, an attacker "
                    "can self-sign a token, host their own key at a URL they control, and have it accepted.",
                    "Never resolve jku/x5u dynamically from token content; hard-code trusted key hosts server-side "
                    "and reject or ignore jku/x5u claims that don't match, or disable support for them entirely "
                    "if all valid keys are already known/embedded server-side.",
                    Severity.HIGH, "Possible"))
            kid = header.get("kid")
            if kid and self._SUSPICIOUS_KID.search(str(kid)):
                findings.append(self._finding(url, ev,
                    f"A JWT header's 'kid' (key ID) value ('{kid}') contains path-traversal or SQL "
                    "metacharacters, suggesting the server may use this attacker-controlled field directly to "
                    "build a file path or database query when looking up the verification key - a known JWT "
                    "attack class ('kid' injection).",
                    "Treat 'kid' as an index into a small, fixed, server-controlled set of known key IDs "
                    "(reject anything not in that set) rather than using it to construct a file path or query.",
                    Severity.HIGH, "Possible"))
        return findings


class StoredXSSHeuristicModule(BaseModule):
    """Lightweight second-order/stored-XSS heuristic: submits the same
    unescaped-HTML marker used by ReflectedXSSSignalModule into a POST
    form's text-shaped fields, then re-fetches the ORIGINAL page the form
    lived on (a plain GET, no state assumed) to check whether the marker
    now appears unescaped there - the classic 'comment box' / 'profile
    bio' stored-XSS shape. This is a heuristic, not a guarantee: many
    forms show the submitted value on a different page (a redirect target,
    a moderation queue, a different user's view) this single-page revisit
    can't see, so absence of a hit here does NOT mean the form is safe -
    it only means this specific low-cost check didn't catch it."""
    category = "xss"; title = "Possible Stored/Second-Order Cross-Site Scripting"
    owasp = "A03:2021"; cwe = "CWE-79"; min_profile = ScanProfile.DEEP
    _MARKER = "ersec_stored_xss_probe_6b2e"

    def run_for_forms(self, forms: List["FormInfo"]) -> List[Finding]:
        findings = []
        checked = 0
        for form in forms:
            if form.method.upper() != "POST" or checked >= 8:
                continue
            text_fields = [i["name"] for i in form.inputs
                           if i.get("type") in ("text", "textarea", "") and i["name"]]
            if not text_fields:
                continue
            checked += 1
            probe = f"<{self._MARKER}>"
            body = {inp["name"]: (inp.get("value") or "ersec") for inp in form.inputs
                    if inp.get("type") not in ("submit", "button")}
            for field in text_fields:
                body[field] = probe
            target_url = form.action or form.page_url
            try:
                self._post(target_url, data=body)
                revisit_resp, revisit_ev = self._get(form.page_url)
            except (ScopeError, ERSECError):
                continue
            if probe in (revisit_resp.text or ""):
                findings.append(self._finding(form.page_url, revisit_ev,
                    f"After submitting an unescaped-HTML marker into this form's text field(s), revisiting "
                    f"the original page ({form.page_url}) shows the marker reflected back verbatim (not "
                    "HTML-encoded) - consistent with stored/second-order XSS, where the payload is saved "
                    "server-side and rendered to later visitors rather than reflected immediately in the "
                    "same response.",
                    "Apply the same context-aware output encoding to stored/persisted data as to any other "
                    "user input, at the point it is rendered - encoding at write-time is not sufficient if a "
                    "different rendering path (export, admin view, API) skips it. Sanitize on output, every time.",
                    Severity.HIGH, "Likely"))
        return findings


class LDAPInjectionSignalModule(BaseModule):
    category = "ldap_injection"; title = "Possible LDAP Injection"
    owasp = "A03:2021"; cwe = "CWE-90"; min_profile = ScanProfile.BASELINE
    ERROR_PATTERNS = ["invalid dn syntax", "ldapexception", "javax.naming", "ldap: error code",
                       "bad search filter"]

    def run_param(self, url, param, method="GET"):
        findings = []
        probe = "*)(uid=*))(|(uid=*"
        try:
            if method.upper() == "GET":
                resp, ev = self._get(_with_param(url, param, probe))
            else:
                resp, ev = self._post(url, data={param: probe})
        except (ScopeError, ERSECError):
            return findings
        lower = (resp.text or "").lower()
        if any(p in lower for p in self.ERROR_PATTERNS):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' triggered an LDAP-related error signature when sent LDAP filter "
                "metacharacters, suggesting input reaches a directory query without escaping.",
                "Escape LDAP special characters per RFC 4515, or use your LDAP library's parameterized filter API.",
                Severity.HIGH, "Likely", parameter=param))
        return findings


class XPathInjectionSignalModule(BaseModule):
    """Same detection shape as the LDAP/SQL modules, applied to XPath -
    apps that query XML documents (older SOAP-era or embedded-XML-config
    backends) with unescaped user input into an XPath expression."""
    category = "xpath_injection"; title = "Possible XPath Injection"
    owasp = "A03:2021"; cwe = "CWE-643"; min_profile = ScanProfile.BASELINE
    ERROR_PATTERNS = ["xpathexception", "invalid xpath", "xpath syntax error", "unterminated string literal",
                       "unclosed string literal in xpath", "org.w3c.dom.domexception"]

    def run_param(self, url, param, method="GET"):
        findings = []
        probe = "' or '1'='1"
        try:
            if method.upper() == "GET":
                resp, ev = self._get(_with_param(url, param, probe))
            else:
                resp, ev = self._post(url, data={param: probe})
        except (ScopeError, ERSECError):
            return findings
        lower = (resp.text or "").lower()
        if any(p in lower for p in self.ERROR_PATTERNS):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' triggered an XPath-related error signature when sent an XPath "
                "metacharacter payload, suggesting user input reaches an XPath query without escaping.",
                "Use parameterized XPath APIs (e.g. XPath variable bindings) instead of string-concatenating "
                "user input into an XPath expression; validate input against an expected format first.",
                Severity.HIGH, "Likely", parameter=param))
        return findings


class CommandInjectionTimingSignalModule(BaseModule):
    """OS command injection - the highest-impact injection class this tool
    checks for - detected ONLY via a timing side-channel from a harmless
    'sleep' shell command chained after the expected input, never via any
    payload that reads, writes, or deletes anything. Cross-platform (both
    POSIX `sleep` and Windows `timeout`/`ping` idioms) since the target OS
    is unknown ahead of time. Confirmed with a same-shape non-delaying
    control probe on the same parameter, exactly like the SQLi time-based
    check, to rule out endpoints that are simply naturally slow."""
    category = "command_injection"; title = "Possible OS Command Injection"
    owasp = "A03:2021"; cwe = "CWE-78"; min_profile = ScanProfile.DEEP

    _DELAY_PROBES = [
        "; sleep 5", "| sleep 5", "`sleep 5`", "$(sleep 5)",
        "& ping -n 6 127.0.0.1 & ", "| timeout 5",
    ]
    _CONTROL_PROBE = "; true"

    def run_param(self, url, param, method="GET"):
        findings = []
        try:
            baseline_resp, baseline_ev = (self._get(url) if method.upper() == "GET" else self._post(url))
        except (ScopeError, ERSECError):
            return findings
        if baseline_ev.response_time_ms >= 2000:
            return findings  # endpoint's baseline is already slow; timing signal would be unreliable
        for probe in self._DELAY_PROBES:
            try:
                t0 = time.time()
                if method.upper() == "GET":
                    self._get(_with_param(url, param, probe))
                else:
                    self._post(url, data={param: probe})
                elapsed = time.time() - t0
            except (ScopeError, ERSECError):
                continue
            if elapsed > 4.0:
                try:
                    t1 = time.time()
                    if method.upper() == "GET":
                        control_resp, control_ev = self._get(_with_param(url, param, self._CONTROL_PROBE))
                    else:
                        control_resp, control_ev = self._post(url, data={param: self._CONTROL_PROBE})
                    control_elapsed = time.time() - t1
                except (ScopeError, ERSECError):
                    control_elapsed = 0.0
                if control_elapsed < 2.0:
                    ev = RequestEvidence(method=method.upper(), url=_with_param(url, param, probe) if method.upper() == "GET" else url,
                                          status_code=0, response_time_ms=int(elapsed * 1000),
                                          response_headers={}, response_excerpt="")
                    findings.append(self._finding(ev.url, ev,
                        f"Parameter '{param}' caused a ~{elapsed:.1f}s response delay when sent a shell "
                        f"command-injection timing probe ('{probe}'), versus a same-shape control probe that "
                        f"returned in {control_elapsed:.1f}s and a normal baseline of {baseline_ev.response_time_ms}ms - "
                        "consistent with the input reaching a shell/OS command execution context.",
                        "Never pass user input to a shell (os.system, subprocess with shell=True, exec() family, "
                        "backticks). Use language-level APIs that take an argument list without shell "
                        "interpretation (e.g. subprocess.run([...], shell=False)), and validate/allow-list input "
                        "strictly if a shell call is truly unavoidable.",
                        Severity.CRITICAL, "Likely", parameter=param))
                    return findings
        return findings


class AdminPanelExposureModule(BaseModule):
    category = "admin_exposure"; title = "Unauthenticated Admin/Management Interface"
    owasp = "A01:2021"; cwe = "CWE-284"; min_profile = ScanProfile.BASELINE
    CANDIDATE_PATHS = [
        "/admin", "/administrator", "/admin/dashboard", "/administration",
        "/management", "/manage", "/wp-admin/", "/cpanel", "/phpmyadmin",
        "/adminer.php", "/console", "/actuator", "/_admin", "/backend",
        "/rest/admin/application-configuration",  # common in Node/Express-style apps (incl. Juice Shop's pattern)
        "/grafana", "/kibana", "/jenkins", "/rabbitmq", "/solr/admin",
        "/elasticsearch/_cat", "/_cat/indices", "/metrics", "/debug/vars",
        "/api/admin", "/api/v1/admin", "/django-admin", "/portainer",
    ]
    # If the response body strongly suggests a real login gate, this is expected
    # behavior, not a finding - only flag when it looks like live admin content.
    LOGIN_SIGNALS = re.compile(r"(sign in|log in|login|password|unauthorized|forbidden|401|403|access denied)", re.I)
    ADMIN_CONTENT_SIGNALS = re.compile(r"(dashboard|user management|admin panel|manage users|site settings|"
                                        r"configuration|analytics|total users|delete user)", re.I)

    def run_url(self, url):
        findings = []
        baseline_body = ""
        try:
            baseline_url = urllib.parse.urljoin(url, f"/ersec-soft-404-baseline-{os.urandom(4).hex()}.probe")
            baseline_resp, _ = self._get(baseline_url)
            baseline_body = baseline_resp.text or ""
        except (ScopeError, ERSECError):
            pass
        for path in self.CANDIDATE_PATHS:
            test_url = urllib.parse.urljoin(url, path)
            try:
                resp, ev = self._get(test_url)
            except (ScopeError, ERSECError):
                continue
            if resp.status_code != 200 or not resp.text:
                continue
            body = resp.text
            if baseline_body and body[:300] == baseline_body[:300] and abs(len(body) - len(baseline_body)) < 50:
                continue  # same catch-all/soft-404 page every unmatched route serves - not a real admin panel
            looks_gated = bool(self.LOGIN_SIGNALS.search(body[:3000]))
            looks_live = bool(self.ADMIN_CONTENT_SIGNALS.search(body))
            if looks_live and not looks_gated:
                findings.append(self._finding(test_url, ev,
                    f"An admin/management interface at {path} returned content that looks like live "
                    "administrative functionality without any visible authentication gate.",
                    "Require authentication (and ideally IP allow-listing or a VPN) in front of every "
                    "admin/management route; verify this at the routing/middleware layer, not just in the UI.",
                    Severity.CRITICAL, "Possible"))
        return findings


class WeakSessionTokenModule(BaseModule):
    category = "weak_session_token"; title = "Weak or Predictable Session Token"
    owasp = "A07:2021"; cwe = "CWE-330"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        session_cookie_names = {"sessionid", "session", "phpsessid", "jsessionid", "connect.sid",
                                 "auth_token", "session_token", "sid"}
        for c in resp.cookies:
            if c.name.lower() not in session_cookie_names or not c.value:
                continue
            value = c.value
            issues = []
            if len(value) < 16:
                issues.append(f"only {len(value)} characters long")
            if value.isdigit():
                issues.append("appears to be a plain sequential/numeric identifier, not a random token")
            distinct_chars = len(set(value))
            if distinct_chars <= 4 and len(value) > 8:
                issues.append(f"very low character diversity ({distinct_chars} distinct characters) for its length")
            if issues:
                findings.append(self._finding(url, ev,
                    f"Session cookie '{c.name}' looks weak: {'; '.join(issues)}. A predictable session "
                    "identifier can potentially be guessed or brute-forced instead of stolen.",
                    "Generate session identifiers with a cryptographically secure random source, at least "
                    "128 bits of entropy, and avoid any sequential or user-derived component.",
                    Severity.HIGH, "Possible"))
        return findings


class PrototypePollutionSignalModule(BaseModule):
    category = "prototype_pollution"; title = "Possible Client/Server Prototype Pollution"
    owasp = "A03:2021"; cwe = "CWE-1321"; min_profile = ScanProfile.DEEP

    def run_param(self, url, param, method="GET"):
        findings = []
        # Structural probe only: asks whether a nested-object nototation for this
        # parameter is accepted and echoed back in a JSON response, which is the
        # precondition for prototype pollution (not proof of it - flagged as
        # low-confidence "Possible" precisely because this is just a precondition check).
        probe_key = f"{param}[__proto__][ersecMarker]"
        test_url = f"{url}{'&' if '?' in url else '?'}{probe_key}=1"
        try:
            resp, ev = self._get(test_url)
        except (ScopeError, ERSECError):
            return findings
        ctype = resp.headers.get("Content-Type", "")
        if "application/json" in ctype and "ersecMarker" in (resp.text or ""):
            findings.append(self._finding(ev.url, ev,
                f"Parameter '{param}' accepts a __proto__-nested key and echoes it back in a JSON response, "
                "which is the precondition for prototype pollution if the backend later merges this input "
                "into an object without guarding against __proto__/constructor keys.",
                "Use a JSON schema validator, or explicitly strip __proto__/constructor/prototype keys before "
                "any recursive merge/assign of user-controlled objects (e.g. avoid lodash merge()/_.merge on raw request bodies).",
                Severity.MEDIUM, "Possible", parameter=param))
        return findings


class RateLimitingSignalModule(BaseModule):
    """Passive-leaning check for missing rate limiting on sensitive-looking
    endpoints (login/password-reset/OTP paths). Sends a small, fixed burst
    (default 6 requests, well under any reasonable threshold) and checks
    whether the server ever responds with a 429/throttling signal. This
    never attempts to actually brute-force credentials - no login data is
    guessed, varied, or submitted meaningfully; the same fixed harmless
    payload is sent every time purely to observe throttling behavior."""
    category = "rate_limiting"; title = "Missing Rate Limiting on Sensitive Endpoint"
    owasp = "A07:2021"; cwe = "CWE-307"; min_profile = ScanProfile.DEEP
    SENSITIVE_PATH_HINTS = re.compile(r"(login|signin|sign-in|auth|password|reset|otp|verify|2fa|mfa)", re.I)
    _BURST = 6

    def run_url(self, url):
        findings = []
        if not self.SENSITIVE_PATH_HINTS.search(urllib.parse.urlparse(url).path):
            return findings
        statuses = []
        last_ev = None
        for _ in range(self._BURST):
            try:
                resp, ev = self._get(url)
            except (ScopeError, ERSECError):
                break
            statuses.append(resp.status_code)
            last_ev = ev
        if last_ev and statuses and all(s not in (429, 503) for s in statuses) and len(set(statuses)) <= 1:
            findings.append(self._finding(url, last_ev,
                f"Sent {len(statuses)} identical requests to a sensitive-looking endpoint (path suggests "
                f"login/password-reset/OTP) and received the same status code ({statuses[0]}) every time with "
                "no throttling (429) or other rate-limiting signal - this is a small, fixed-size probe and does "
                "NOT confirm the absence of rate limiting at higher volumes, but the total absence of any "
                "throttling signal even at this scale is worth checking.",
                "Implement rate limiting (per-IP and per-account) on authentication, password-reset, and "
                "OTP/MFA-verification endpoints; return 429 with a Retry-After header once a threshold is exceeded, "
                "and consider progressive delays or CAPTCHA after repeated failures.",
                Severity.LOW, "Possible"))
        return findings


class CSRFModule(BaseModule):
    category = "csrf"; title = "Missing CSRF Protection"; owasp = "A01:2021"; cwe = "CWE-352"
    min_profile = ScanProfile.BASELINE
    TOKEN_NAMES = {"csrf", "csrf_token", "_csrf", "authenticity_token", "csrfmiddlewaretoken", "__requestverificationtoken"}

    def check_form(self, form: FormInfo) -> List[Finding]:
        if form.method.upper() != "POST":
            return []
        names = {i["name"].lower() for i in form.inputs}
        if names & self.TOKEN_NAMES:
            return []
        ev = RequestEvidence(method="POST", url=form.action or form.page_url, status_code=0,
                              response_time_ms=0, response_headers={}, response_excerpt="")
        return [self._finding(form.action or form.page_url, ev,
            "A state-changing POST form was found with no recognizable CSRF token field.",
            "Implement per-session (or per-request) CSRF tokens validated server-side, and set SameSite=Lax/Strict on session cookies.",
            Severity.MEDIUM, "Possible")]


# =============================================================================
# AI Advisor Engine - fully local, built in, no network calls, no API keys
#
# This is a rule-based expert system, not a neural network: a structured
# knowledge base of remediation guidance, indexed by vulnerability category,
# with the specific text selected and filled in based on:
#   - the finding's own details (URL, parameter, severity)
#   - the fingerprinted tech stack (so a Django site gets Django-specific
#     code, an Express site gets Express-specific code, etc.)
#   - whether the finding participates in a correlated risk chain
#     (computed separately by correlate_findings - deterministic, not this
#     engine's job to invent chains, only to explain ones it's given)
#
# Being upfront about what this is: an expert system (a real, long-standing
# branch of AI) built from a curated knowledge base, not a language model.
# It runs instantly and works completely offline.
# =============================================================================

@dataclass
class KBEntry:
    explain: str                                  # 2-4 sentence plain-English explanation
    fix_generic: List[str]                        # numbered remediation steps, generic stack
    fix_by_stack: Dict[str, List[str]]             # stack name -> stack-specific steps (overrides generic)
    verify: str                                    # how to confirm the fix
    why_it_matters: str = ""                       # business-risk framing: what actually happens if this is exploited
    references: List[str] = field(default_factory=list)  # real OWASP/vendor references, not fabricated links
    false_positive_note: str = ""                  # how to tell a real hit from a coincidental match


_KNOWLEDGE_BASE: Dict[str, KBEntry] = {

    "sql_injection": KBEntry(
        explain=(
            "SQL injection happens when user-controlled input is inserted directly into a database query "
            "instead of being treated as pure data. An attacker can manipulate the query's logic to read, "
            "modify, or delete data far outside what the application intended to expose."
        ),
        fix_generic=[
            "1. Stop building SQL strings by concatenating or formatting user input into the query text.",
            "2. Use parameterized queries / prepared statements for every query that includes user input.",
            "3. If an ORM is available, prefer it over raw SQL for standard CRUD operations.",
            "4. Apply least-privilege database credentials for the app account (no DROP/ALTER rights it doesn't need).",
            "5. Add input validation (type/length/format) as defense-in-depth, not as the primary control.",
        ],
        fix_by_stack={
            "Django (Python)": [
                "1. Replace any raw SQL (cursor.execute(f\"...{value}...\")) with the Django ORM, which "
                "parameterizes automatically: Model.objects.filter(field=value).",
                "2. If raw SQL is unavoidable, use cursor.execute(\"...%s...\", [value]) - never f-strings or % formatting.",
                "3. Run python manage.py check --deploy and review for SQL-related warnings.",
            ],
            "Laravel (PHP)": [
                "1. Use Eloquent or the query builder (DB::table(...)->where('field', $value)), which binds parameters automatically.",
                "2. If using raw queries, use DB::select('... where field = ?', [$value]) with bindings - never string interpolation.",
            ],
            "Express (Node.js)": [
                "1. Use a parameterized query API (pg with $1 placeholders, mysql2 with ? placeholders, or an ORM like Prisma/Sequelize).",
                "2. Never build queries with template literals containing request data.",
            ],
            "Ruby on Rails": [
                "1. Use ActiveRecord query methods (Model.where(field: value)) which parameterize automatically.",
                "2. If using find_by_sql or raw SQL fragments, use ? placeholders with an array of bindings, never string interpolation.",
            ],
            "Spring Boot (Java)": [
                "1. Use Spring Data JPA repository methods or @Query with named/positional parameters (@Param), never string-concatenated JPQL/native SQL.",
                "2. If using JdbcTemplate, use the parameterized overloads (jdbcTemplate.query(sql, args)) - never String.format() into the SQL text.",
            ],
            "Flask (Python)": [
                "1. Use SQLAlchemy's query API (session.query(Model).filter_by(field=value)), which parameterizes automatically.",
                "2. If using raw connections (sqlite3, psycopg2), use ? or %s placeholders with a parameter tuple - never f-strings.",
            ],
        },
        why_it_matters=(
            "This is consistently one of the highest-impact web vulnerability classes because it bypasses the "
            "application layer entirely - an attacker isn't limited to what the app's UI lets them do, they're "
            "talking to the database directly. In practice this means full data exfiltration (customer records, "
            "password hashes, payment data if stored), and depending on database privileges, sometimes writing "
            "files or executing OS commands via database-specific features (e.g. MySQL's INTO OUTFILE, "
            "PostgreSQL's COPY, or SQL Server's xp_cmdshell)."
        ),
        references=[
            "https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html",
            "https://owasp.org/Top10/A03_2021-Injection/",
        ],
        false_positive_note=(
            "A single-quote error match is high-confidence and rarely a false positive. The boolean/time-based "
            "signals are more prone to false positives on endpoints with naturally variable response "
            "size/timing (e.g. pagination, caching) - if in doubt, manually replay the exact request twice and "
            "confirm the timing/size difference is consistent, not one-off noise."
        ),
        verify="Send the same single-quote probe again after the fix; the response should no longer show a database "
               "error, and the boolean-condition test (' OR '1'='1 vs ' AND '1'='2) should return identical results.",
    ),

    "command_injection": KBEntry(
        explain=(
            "OS command injection occurs when user input reaches a shell/command-execution context, letting "
            "an attacker chain additional shell commands onto what the application intended to run. A "
            "response-time delay from a harmless 'sleep' probe is the signal used here, confirmed against a "
            "same-shape non-delaying control to rule out a naturally slow endpoint."
        ),
        fix_generic=[
            "1. Never pass user input to a shell interpreter (os.system, exec/eval, backticks, subprocess with shell=True).",
            "2. Use language APIs that execute a program directly with an argument list, bypassing shell parsing entirely.",
            "3. If a shell is truly unavoidable, strictly allow-list acceptable input (exact match against a known set), never a deny-list of 'dangerous characters'.",
            "4. Run the executing process with the least privilege necessary (dedicated low-privilege service account, containerized/sandboxed).",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Replace child_process.exec()/execSync() (which invokes a shell) with execFile()/spawn() passing arguments as an array, not a concatenated string."],
            "Django (Python)": ["1. Replace os.system()/os.popen() with subprocess.run([...], shell=False), passing each argument as a separate list element."],
            "Flask (Python)": ["1. Same as Django: subprocess.run([...], shell=False) with an argument list, never shell=True with a formatted string."],
        },
        why_it_matters=(
            "Command injection is typically the single most severe finding a scanner can report: successful "
            "exploitation usually means arbitrary code execution on the server itself, not just data access - "
            "full compromise of the host, lateral movement into internal networks, and persistence, not just a "
            "single application's data."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/OS_Command_Injection_Defense_Cheat_Sheet.html"],
        false_positive_note=(
            "Timing signals can be noisy on endpoints with variable load. This check already cross-validates "
            "with a same-shape control probe, but on a genuinely inconsistent/slow target, manually replay the "
            "exact delay probe 2-3 times and confirm the delay is consistent before treating this as confirmed."
        ),
        verify="Re-send the same delay probe; the response should return at normal baseline speed, not after the injected delay.",
    ),

    "xxe": KBEntry(
        explain=(
            "The XML parser behind this endpoint resolved an internal DTD entity declaration sent in a test "
            "document, meaning it processes DOCTYPE/entity definitions rather than rejecting them outright - "
            "the precondition for XML External Entity (XXE) attacks, which can read local files or reach "
            "internal network resources via crafted external entities."
        ),
        fix_generic=[
            "1. Disable DTD processing entirely wherever possible - most applications never need arbitrary DOCTYPE declarations.",
            "2. If DTDs must be supported, disable external entity and external DTD resolution specifically.",
            "3. Prefer a data format that has no entity/DTD concept at all (JSON) where the schema allows it.",
        ],
        fix_by_stack={
            "Spring Boot (Java)": ["1. Configure the XML parser (DocumentBuilderFactory/SAXParserFactory/XMLInputFactory) to set FEATURE_SECURE_PROCESSING and disallow-doctype-decl, or use a library preset like OWASP's XXE-safe defaults."],
            "Express (Node.js)": ["1. If using libxmljs or fast-xml-parser, ensure entity/DOCTYPE expansion is disabled (these are often safe by default, but confirm the specific parser and version in use)."],
            "Django (Python)": ["1. Use defusedxml instead of the standard library's xml.etree/xml.dom modules, which are documented as unsafe against several XML attack classes including XXE by default."],
        },
        why_it_matters=(
            "A successful XXE exploit (beyond what this non-destructive check performs) can read arbitrary "
            "files from the server's filesystem, including configuration files and credentials, and can be "
            "used as an SSRF vector to reach internal-only network services via crafted external entity URLs."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html"],
        false_positive_note="This check only sends a benign internal entity with no file:// or network reference, so a positive result is high-confidence for entity expansion being enabled - it does not by itself confirm file/network read access is achievable.",
        verify="Re-send the same minimal internal-entity XML document; the entity should no longer be expanded (parser should reject the DOCTYPE or ignore the entity).",
    ),

    "xpath_injection": KBEntry(
        explain=(
            "A parameter reflects an XPath-related error when sent XPath metacharacters, suggesting user "
            "input reaches an XPath query against an XML document store without escaping - functionally "
            "similar to SQL injection but against XML data rather than a relational database."
        ),
        fix_generic=[
            "1. Use parameterized/variable-bound XPath APIs instead of building XPath expression strings via concatenation.",
            "2. Validate input against an expected format (e.g. numeric ID, known enum) before using it in a query at all.",
        ],
        fix_by_stack={},
        verify="Re-send the same probe; the response should no longer show an XPath error and should treat the input as a literal string value.",
    ),

    "idor_heuristic": KBEntry(
        explain=(
            "An ID adjacent to one the application itself linked to during the crawl resolved to a full, "
            "ungated 200 response of similar shape - this shows sequential IDs aren't gated by *some* check, "
            "but does not confirm the record belongs to a different user, which requires manual verification "
            "with two separate authenticated accounts."
        ),
        fix_generic=[
            "1. Add an explicit object-level authorization check on every request: verify the authenticated user "
            "actually owns or has been granted access to the specific record ID requested, not just that they're logged in.",
            "2. Do this server-side on every read/write/delete endpoint that takes an ID, not only in the UI.",
            "3. Consider non-sequential identifiers (UUIDv4) for user-facing object references so IDs can't be enumerated even if authorization has a gap.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "Broken Object Level Authorization (IDOR's formal OWASP API Top 10 name) is consistently one of "
            "the most commonly exploited API vulnerability classes in practice, precisely because it's easy to "
            "overlook: authentication ('are you logged in') is often implemented correctly while authorization "
            "('are you allowed to see THIS specific record') is missing entirely."
        ),
        references=["https://owasp.org/API-Security/editions/2023/en/0xa1-broken-object-level-authorization/"],
        false_positive_note="This heuristic can false-positive on genuinely public catalogs (e.g. product listings) where sequential IDs are intentionally all publicly viewable - confirm the endpoint is meant to be per-user-private before treating this as a real gap.",
        verify="With two separate authenticated test accounts, confirm whether Account A can access a record ID that was created by / belongs to Account B.",
    ),

    "rate_limiting": KBEntry(
        explain=(
            "A small, fixed-size burst of identical requests to a login/password-reset/OTP-shaped endpoint "
            "returned the same status code every time with no throttling signal (429) - at the scale this "
            "check probes, there's no visible rate limiting, though this does not confirm behavior at higher volumes."
        ),
        fix_generic=[
            "1. Implement rate limiting per-IP and per-account on authentication, password-reset, and OTP/MFA endpoints.",
            "2. Return HTTP 429 with a Retry-After header once a threshold is exceeded.",
            "3. Consider progressive delays or a CAPTCHA challenge after a small number of consecutive failures.",
            "4. Alert on/log abnormal request volume against these endpoints for detection even if prevention has gaps.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Add express-rate-limit (or a Redis-backed equivalent for multi-instance deployments) scoped specifically to auth routes."],
            "Django (Python)": ["1. Use django-ratelimit or django-axes for login-specific throttling and account lockout after repeated failures."],
        },
        why_it_matters=(
            "Missing rate limiting on authentication endpoints is what turns a weak or leaked-elsewhere "
            "password into a successful credential-stuffing or brute-force compromise - the control doesn't "
            "prevent any single guess, but it makes guessing at scale impractical."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/Authentication_Cheat_Sheet.html#account-lockout"],
        verify="Re-send the same burst; the server should return 429 (or otherwise visibly throttle) after a small number of requests.",
    ),

    "websocket": KBEntry(
        explain=(
            "The page uses a WebSocket connection that is either unencrypted (ws://) or whose server accepted "
            "the upgrade handshake from an arbitrary, untrusted Origin header - the WebSocket equivalent of a "
            "missing CORS check, which can enable cross-site WebSocket hijacking against authenticated users."
        ),
        fix_generic=[
            "1. Serve all WebSocket connections over wss:// (TLS) exclusively.",
            "2. Validate the Origin header server-side during the WebSocket upgrade handshake against an explicit allow-list.",
            "3. Require an additional authentication token in the WebSocket handshake/first message rather than relying on ambient cookies alone.",
        ],
        fix_by_stack={},
        verify="Re-attempt the handshake with an untrusted Origin header; the server should reject the upgrade (not return HTTP 101).",
    ),

    "email_security": KBEntry(
        explain=(
            "The domain is missing or has weak SPF/DMARC DNS records, which tell receiving mail servers which "
            "hosts may send email as this domain and what to do with messages that fail that check - without "
            "them, spoofed phishing email claiming to be from this domain is easier to deliver successfully."
        ),
        fix_generic=[
            "1. Publish an SPF record listing all legitimate sending sources, ending in '-all' (hard fail) once confirmed complete.",
            "2. Publish a DMARC record starting at 'p=none' with aggregate reporting enabled, then move to 'p=quarantine'/'p=reject' once legitimate traffic is confirmed covered.",
            "3. Ensure DKIM signing is configured for all legitimate sending sources and referenced correctly by the DMARC alignment.",
        ],
        fix_by_stack={},
        verify="Re-query the domain's TXT records (dig TXT example.com; dig TXT _dmarc.example.com) and confirm the updated policy is published and propagated.",
    ),

    "nosql_injection": KBEntry(
        explain=(
            "NoSQL injection occurs when user input is passed into a document-database query as a structured "
            "object (e.g. a MongoDB operator like $ne or $gt) instead of a plain scalar value, letting an "
            "attacker override the query's intended conditions."
        ),
        fix_generic=[
            "1. Reject non-scalar input for any field that should be a string or number before it reaches the query.",
            "2. Use a schema validation library (Joi, Zod, Mongoose schemas, class-validator) on all request bodies.",
            "3. Avoid $where and raw JavaScript query fragments entirely.",
            "4. Cast/sanitize identifiers explicitly rather than passing req.body or req.query straight into a query object.",
        ],
        fix_by_stack={},
        verify="Re-send a request with field[$ne]=1 as the value; the API should reject it with a validation error "
               "rather than executing a query.",
    ),

    "ssrf": KBEntry(
        explain=(
            "The server appears to fetch a user-supplied URL itself rather than treating it as opaque data - "
            "confirmed by a connection-error signature when pointed at a non-existent local service. An "
            "attacker can use this to make the server issue requests to internal-only systems it can reach "
            "but the attacker cannot reach directly."
        ),
        fix_generic=[
            "1. Validate and allow-list destination hosts/schemes server-side before fetching any user-supplied URL.",
            "2. Block requests to private/link-local IP ranges (127.0.0.0/8, 169.254.0.0/16, 10.0.0.0/8, "
            "172.17.0.0/12, 192.168.0.0/16) at the network layer, not just in application code.",
            "3. If the feature only needs to fetch specific known resource types (e.g. images), proxy through "
            "a dedicated fetching service with its own strict egress rules rather than the main application.",
            "4. Disable HTTP redirects on the server-side fetch, or re-validate the destination after each redirect hop.",
            "5. Resolve and normalize the destination host (including decimal/hex/IPv6 loopback encodings) "
            "before checking it against a deny/allow-list - a raw string comparison against 'localhost'/'127.0.0.1' "
            "misses alternate encodings of the same address.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Use a library like `ssrf-req-filter` or implement DNS-rebinding-safe IP validation before any outbound fetch based on user input."],
            "Django (Python)": ["1. Never pass user-supplied URLs directly to `requests.get()`; validate against an allow-list first, and consider a network policy (e.g. egress firewall rules) as a second layer."],
        },
        why_it_matters=(
            "SSRF is frequently the pivot point that turns a 'minor' feature (an image proxy, a webhook "
            "validator, a PDF generator that fetches a URL) into access to cloud metadata services, internal "
            "admin panels, or other systems that were never meant to be internet-reachable - it's consistently "
            "one of the highest-impact bug classes in modern cloud-hosted applications."
        ),
        references=["https://owasp.org/Top10/A10_2021-Server-Side_Request_Forgery_%28SSRF%29/",
                    "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html"],
        false_positive_note=(
            "The connection-error signal is high-confidence. The status-code-differential signal (used when no "
            "explicit connection error is returned) is weaker - some apps legitimately return different status "
            "codes for any unreachable/invalid URL regardless of SSRF; manually confirm by trying a probe "
            "against a service you control and checking for the request in its access log."
        ),
        verify="Point the same parameter at a URL/port you control and confirm the request actually arrives there server-side.",
    ),

    "xss": KBEntry(
        explain=(
            "Reflected cross-site scripting happens when user input is echoed back into an HTML page without "
            "proper encoding, letting an attacker craft a link that runs arbitrary JavaScript in another user's "
            "browser session - commonly used to steal session cookies or perform actions as that user."
        ),
        fix_generic=[
            "1. HTML-encode all user-controlled data at the point it's rendered into HTML (not just at input time).",
            "2. Use your framework's auto-escaping template engine rather than raw string concatenation into HTML.",
            "3. Add a strict Content-Security-Policy (e.g. default-src 'self') as defense-in-depth against any encoding gaps.",
            "4. Set cookies with HttpOnly so injected scripts cannot read them even if XSS occurs.",
        ],
        fix_by_stack={
            "Django (Python)": ["1. Django templates auto-escape by default - if this value uses {{ value|safe }} or mark_safe(), remove it unless the content is fully sanitized."],
            "Express (Node.js)": ["1. If using a template engine, ensure auto-escaping is on (e.g. EJS <%= %> not <%- %>); if building HTML manually, use a library like 'he' to encode."],
            "Laravel (PHP)": ["1. Use Blade's {{ $value }} (auto-escaped), not {!! $value !!}, unless the content is explicitly sanitized."],
            "Ruby on Rails": ["1. Rails escapes by default in ERB (<%= %>) - if this uses raw() or html_safe, remove it unless the content is sanitized."],
            "Spring Boot (Java)": ["1. Thymeleaf auto-escapes with th:text by default - if this uses th:utext (unescaped) on user input, switch to th:text."],
            "Flask (Python)": ["1. Jinja2 (Flask's default template engine) auto-escapes by default - if this uses the |safe filter or Markup() on user input, remove it unless the content is fully sanitized."],
        },
        why_it_matters=(
            "XSS is the most common way real-world session hijacking happens against logged-in users, because "
            "unlike stealing credentials directly, it works even against users with strong passwords and MFA - "
            "the attacker rides on an already-authenticated session. Beyond cookie theft, an injected script "
            "can also silently modify the page (fake login forms, redirect payment flows) or act as a keylogger "
            "for anything typed on the page afterward. For the DOM-based variant, the entire chain can happen "
            "client-side with no server round-trip at all, which means server-side output encoding alone won't "
            "catch it - the client-side sink also needs to be fixed."
        ),
        references=[
            "https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html",
            "https://owasp.org/Top10/A03_2021-Injection/",
            "https://cheatsheetseries.owasp.org/cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.html",
        ],
        false_positive_note=(
            "A harmless marker tag round-tripping unescaped in the raw HTML is a strong signal, but confirm it "
            "lands in an HTML execution context (not inside a JSON API response, a code comment, or an "
            "attribute value that's separately escaped) before treating it as exploitable. For the DOM-sink "
            "check specifically: a static source-to-sink text match does NOT confirm the value is actually "
            "unsanitized in between - always trace the real data flow before treating it as confirmed."
        ),
        verify="Re-submit the same harmless marker value; it should appear in the page HTML-encoded "
               "(e.g. &lt;tag&gt;) rather than as a live tag.",
    ),

    "ssti": KBEntry(
        explain=(
            "Server-side template injection occurs when user input is passed into a template-rendering "
            "function as template source rather than as data, letting the template engine evaluate "
            "attacker-controlled expressions - a computed result appeared where a literal string was expected."
        ),
        fix_generic=[
            "1. Never pass user input into a template-render call as the template string itself.",
            "2. Pass user input only as template variables/context, never as the template source.",
            "3. If user-authored templates are a required feature, use a sandboxed/logic-less engine (e.g. Jinja2 SandboxedEnvironment, Mustache/Handlebars in logic-less mode).",
        ],
        fix_by_stack={},
        why_it_matters=(
            "SSTI frequently escalates all the way to remote code execution, not just data disclosure - many "
            "template engines expose enough of the underlying language (Python, Java, Ruby) through their "
            "expression syntax that an attacker who can inject template code can often run arbitrary server-side code."
        ),
        references=["https://portswigger.net/web-security/server-side-template-injection"],
        verify="Re-send the same arithmetic probe (e.g. {{7*7}}); the response should contain the literal text, not the computed number.",
    ),

    "security_headers": KBEntry(
        explain=(
            "Missing or misconfigured security response headers remove browser-enforced protections (HTTPS "
            "enforcement, clickjacking prevention, MIME-sniffing prevention, cross-origin isolation) that cost "
            "nothing to enable and meaningfully reduce the impact of other bugs."
        ),
        fix_generic=[
            "1. Add Strict-Transport-Security: max-age=31536000; includeSubDomains (only once HTTPS is fully working).",
            "2. Add a Content-Security-Policy starting with default-src 'self' and loosen only where needed.",
            "3. Add X-Frame-Options: DENY (or CSP frame-ancestors) to prevent clickjacking - and emit it exactly once, not stacked from both an origin server and a proxy.",
            "4. Add X-Content-Type-Options: nosniff.",
            "5. Add Referrer-Policy: strict-origin-when-cross-origin.",
            "6. Add Permissions-Policy disabling unused browser features, and Cross-Origin-Opener-Policy: same-origin.",
            "7. Set Secure, HttpOnly, and SameSite=Lax (or Strict) on every session/auth cookie; consider the __Host- prefix for session cookies.",
            "8. Publish a security.txt (RFC 9116) at /.well-known/security.txt with a Contact and Expires field.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Add the helmet middleware (app.use(helmet())) - it sets most of these headers correctly by default, including COOP/CORP/Permissions-Policy in recent versions."],
            "Django (Python)": ["1. Set SECURE_HSTS_SECONDS, SECURE_CONTENT_TYPE_NOSNIFF, X_FRAME_OPTIONS, SESSION_COOKIE_SECURE, and SESSION_COOKIE_HTTPONLY in settings.py."],
            "Laravel (PHP)": ["1. Add these headers via middleware, or use a package like spatie/laravel-csp for CSP management."],
            "Flask (Python)": ["1. Use the flask-talisman extension to set HSTS, CSP, and frame-options with sensible defaults in a few lines."],
        },
        why_it_matters=(
            "None of these headers fix a bug on their own - they're browser-enforced backstops that limit how "
            "much damage OTHER bugs can do. A missing CSP means an XSS bug becomes full script execution "
            "instead of being contained; a missing X-Frame-Options means a real UI can be framed and tricked "
            "into clickjacked actions; missing HSTS means a single unencrypted request can be intercepted and "
            "used to strip TLS on every subsequent one; a missing COOP opens Spectre-class cross-origin leaks "
            "via popup window references."
        ),
        references=["https://owasp.org/www-project-secure-headers/"],
        verify="Re-fetch the page and inspect response headers (e.g. curl -I); each header listed above should now be present with the recommended value, and only once each (check for duplicates from a proxy layer too).",
    ),

    "tls": KBEntry(
        explain=(
            "TLS misconfiguration - an outdated protocol version, a weak cipher suite, an expiring or "
            "hostname-mismatched certificate, or no TLS at all - means traffic between users and the server "
            "can potentially be intercepted, read, or tampered with."
        ),
        fix_generic=[
            "1. Serve the entire site over HTTPS only; redirect all HTTP requests to HTTPS.",
            "2. Disable TLS 1.0 and 1.1 (and SSLv3/v2 if still enabled); require TLS 1.2 minimum, prefer 1.3.",
            "3. Restrict the cipher suite list to modern AEAD ciphers; remove RC4/3DES/export/NULL/anonymous suites.",
            "4. Automate certificate renewal (e.g. certbot with a cron/systemd timer) so it never gets close to expiring.",
            "5. Confirm the certificate's SAN list actually covers every hostname it's served for.",
        ],
        fix_by_stack={},
        references=["https://cheatsheetseries.owasp.org/cheatsheets/Transport_Layer_Security_Cheat_Sheet.html"],
        verify="Run openssl s_client -connect host:443 -tls1_1 and confirm the handshake fails; check certificate expiry with openssl s_client -connect host:443 2>/dev/null | openssl x509 -noout -enddate.",
    ),

    "server_banner": KBEntry(
        explain=(
            "The server discloses a specific software version in a response header. This doesn't grant access "
            "by itself, but it lets an attacker quickly check for known CVEs affecting that exact version "
            "instead of having to guess."
        ),
        fix_generic=[
            "1. Remove or generalize the Server/X-Powered-By/X-Runtime header at the reverse proxy or web server config level.",
            "2. Keep the underlying software patched regardless of whether the header is hidden - this is a minor hardening step, not a substitute for patching.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. app.disable('x-powered-by') removes Express's default header."],
        },
        verify="Re-fetch the page and confirm the Server/X-Powered-By header is absent or generic.",
    ),

    "http_methods": KBEntry(
        explain=(
            "Dangerous or unnecessary HTTP methods are enabled on the server, or the server honors a client-"
            "supplied method-override header. TRACE can enable cross-site tracing attacks against cookie "
            "protections; unused methods like PUT/DELETE widen the attack surface; and honoring "
            "X-HTTP-Method-Override can bypass verb-scoped authorization checks if they run before the "
            "override is resolved."
        ),
        fix_generic=[
            "1. Disable the TRACE method at the web server or reverse-proxy level.",
            "2. Disable any HTTP method the application doesn't intentionally use.",
            "3. Where PUT/DELETE are intentional (REST APIs), ensure they require authentication and are scoped per-resource.",
            "4. If method override support is required, run authorization checks after the effective method is resolved, not before; otherwise disable it entirely.",
        ],
        fix_by_stack={},
        verify="Send an OPTIONS request and confirm the Allow header no longer lists TRACE or other disabled methods; re-test with X-HTTP-Method-Override and confirm it no longer changes behavior.",
    ),

    "cors": KBEntry(
        explain=(
            "The server reflects an arbitrary request Origin back in Access-Control-Allow-Origin (including "
            "the literal 'null' origin, reachable from a sandboxed iframe), sometimes combined with "
            "Allow-Credentials: true. This lets any website make authenticated cross-origin requests on "
            "behalf of a visitor and read the response."
        ),
        fix_generic=[
            "1. Replace origin reflection with an explicit allow-list of trusted origins, checked server-side.",
            "2. Never combine Access-Control-Allow-Origin: * with Access-Control-Allow-Credentials: true.",
            "3. Never explicitly allow the literal 'null' origin.",
            "4. Return CORS headers only for origins on the allow-list; omit them entirely for everyone else.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Use the cors package with an origin function that checks against an allow-list, not origin: '*' or blind reflection."],
            "Spring Boot (Java)": ["1. Configure CorsConfigurationSource with an explicit allowedOrigins list - never allowedOrigins(\"*\") combined with allowCredentials(true)."],
        },
        why_it_matters=(
            "This turns any website on the internet into a potential proxy for authenticated requests against "
            "your API - a victim just has to visit a malicious page while logged into your app, and their "
            "browser will happily attach cookies/credentials to the cross-origin request the malicious page makes."
        ),
        references=["https://portswigger.net/web-security/cors"],
        verify="Re-send a request with an Origin header not on your allow-list; Access-Control-Allow-Origin should be absent from the response.",
    ),

    "exposed_files": KBEntry(
        explain=(
            "A sensitive file or path (credentials, environment config, version control internals, cloud/IaC "
            "state, or backups) is directly downloadable from the public web root, which can hand an attacker "
            "database credentials, API keys, cloud service-account tokens, or source code outright."
        ),
        fix_generic=[
            "1. Remove the file from the web-servable directory entirely, or move it outside the web root.",
            "2. Block access to dotfiles and common sensitive paths at the web server config level as a backstop.",
            "3. Rotate any credentials or secrets that were exposed, immediately - assume they are compromised.",
            "4. Add these paths to your deployment checklist / CI pipeline so they're checked before every release.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "This is one of the few findings where 'fix later' isn't an option - if a scanner found it, so can "
            "anyone else, and credentials should be treated as already compromised the moment this is "
            "discovered, not after confirmed misuse. A .git/config exposure specifically can let an attacker "
            "reconstruct your entire source history, and a terraform.tfstate or kubeconfig exposure can hand "
            "over the keys to your entire cloud/cluster infrastructure, not just this one application."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html"],
        verify="Re-request the exact path; it should return 403/404, and any rotated credentials should show as changed in your secrets manager.",
    ),

    "open_redirect": KBEntry(
        explain=(
            "A parameter controls where the site redirects the user, and the destination isn't validated "
            "(including protocol-relative and malformed-scheme bypass variants). Attackers can craft a link "
            "that appears to point at your trusted domain but redirects victims to a phishing site."
        ),
        fix_generic=[
            "1. Parse the redirect target with a proper URL library and validate the resulting scheme/host against an allow-list - never a naive string-prefix check.",
            "2. Prefer indirect references (e.g. a lookup key mapped server-side to a URL) over passing raw URLs in parameters.",
            "3. If external redirects are genuinely required, show an interstitial warning page before redirecting.",
        ],
        fix_by_stack={},
        verify="Re-send the same parameter with an external URL (including a protocol-relative '//host' form); the app should reject it or redirect only to an allow-listed destination.",
    ),

    "csrf": KBEntry(
        explain=(
            "A state-changing form has no CSRF token, meaning a malicious page on another site can trigger "
            "this form's submission using the victim's existing session, without their knowledge or consent."
        ),
        fix_generic=[
            "1. Generate a per-session (or per-request) CSRF token and include it as a hidden form field.",
            "2. Validate the token server-side on every state-changing request; reject requests where it's missing or wrong.",
            "3. Set SameSite=Lax (or Strict) on session cookies as defense-in-depth.",
        ],
        fix_by_stack={
            "Django (Python)": ["1. Use {% csrf_token %} in the template - Django's CsrfViewMiddleware handles validation automatically once it's included."],
            "Laravel (PHP)": ["1. Use @csrf in the Blade form - Laravel's VerifyCsrfToken middleware validates it automatically."],
            "Ruby on Rails": ["1. Rails includes protect_from_forgery by default; ensure the form helper (form_with) is used so the token is auto-inserted."],
            "Express (Node.js)": ["1. Add the csurf middleware (or an equivalent) and render its token into the form."],
            "Spring Boot (Java)": ["1. Spring Security enables CSRF protection by default - if this is disabled via .csrf().disable(), re-enable it and ensure the token is included in forms/AJAX headers."],
            "Flask (Python)": ["1. Use Flask-WTF's CSRFProtect(app) and {{ form.csrf_token }} in templates."],
        },
        why_it_matters=(
            "The realistic impact depends entirely on what the form does - a CSRF gap on a comment form is "
            "low-stakes; the same gap on a password-change, funds-transfer, or account-email-change form means "
            "an attacker can take over or drain an account just by getting a logged-in victim to visit a page, "
            "with no phishing of credentials required."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html"],
        false_positive_note=(
            "Some frameworks validate CSRF tokens via a custom header or double-submit cookie rather than a "
            "visible form field - if the app uses that pattern, this may be a false positive. Check the "
            "network tab for an X-CSRF-Token header or similar before treating this as confirmed."
        ),
        verify="Submit the form without the token (or with a wrong one); the server should reject the request.",
    ),

    "jwt": KBEntry(
        explain=(
            "A JSON Web Token was observed with a configuration weakness - alg=none, no expiration claim, an "
            "unusually long lifetime, sensitive data embedded in the (unencrypted, only base64-encoded) "
            "payload, or a symmetric algorithm whose safety depends entirely on a high-entropy, well-guarded secret."
        ),
        fix_generic=[
            "1. Explicitly allow-list the accepted algorithm(s) when verifying tokens server-side; never accept alg=none.",
            "2. Always set a reasonably short 'exp' claim; use refresh tokens and server-side revocation for longer sessions.",
            "3. Never place sensitive data directly in a JWT payload - store a reference/ID and look it up server-side.",
            "4. If using HS256, ensure the signing secret is at least 32 bytes of random data and is never exposed client-side.",
            "5. If tokens are validated by multiple services, prefer RS256/ES256 (asymmetric) so only one service holds the private key.",
        ],
        fix_by_stack={},
        verify="Attempt to verify a token with alg=none or a tampered signature; the verification library should reject it. Decode a fresh token's payload and confirm no sensitive fields and a reasonable exp value.",
    ),

    "directory_listing": KBEntry(
        explain=(
            "Directory browsing is enabled on a folder, exposing the full list of files inside it - "
            "which can reveal backups, source files, or other content never meant to be linked publicly."
        ),
        fix_generic=[
            "1. Disable directory listing at the web server config level (Options -Indexes in Apache, autoindex off; in nginx).",
            "2. Add an index file to the directory as a backup safeguard.",
            "3. Review what's actually in the directory and remove anything that shouldn't be publicly served.",
        ],
        fix_by_stack={},
        verify="Re-request the directory path; it should return 403/404 or your site's normal 404 page instead of a file listing.",
    ),

    "verbose_errors": KBEntry(
        explain=(
            "An oversized or malformed input triggered a server error that returned a full stack trace or "
            "debug page, leaking file paths, framework/library versions, and internal logic to anyone who sends the same input."
        ),
        fix_generic=[
            "1. Disable debug/verbose error modes in production.",
            "2. Return a generic error page/message to clients; log full details server-side only.",
            "3. Add centralized error handling so unhandled exceptions never reach the response body.",
        ],
        fix_by_stack={
            "Django (Python)": ["1. Set DEBUG = False in production settings.py, and configure ALLOWED_HOSTS properly."],
            "Express (Node.js)": ["1. Set NODE_ENV=production and add a final error-handling middleware that returns a generic message."],
            "Laravel (PHP)": ["1. Set APP_DEBUG=false in .env for production."],
        },
        verify="Re-send the same malformed input; the response should be a generic error page with no stack trace.",
    ),

    "mixed_content": KBEntry(
        explain=(
            "An HTTPS page loads one or more resources over plain HTTP, which browsers may block and which "
            "can be intercepted or modified in transit even though the page itself is encrypted."
        ),
        fix_generic=[
            "1. Change all resource URLs (scripts, images, stylesheets, iframes) to HTTPS or protocol-relative.",
            "2. Add a Content-Security-Policy with upgrade-insecure-requests as a backstop for anything missed.",
        ],
        fix_by_stack={},
        verify="Reload the page and check the browser console/network tab for mixed-content warnings; there should be none.",
    ),

    "sri": KBEntry(
        explain=(
            "A third-party script or stylesheet is loaded without a Subresource Integrity hash. If that CDN "
            "or third party is ever compromised, it could serve modified code/CSS that runs or renders "
            "unchecked on this page."
        ),
        fix_generic=[
            "1. Add integrity=\"sha384-...\" and crossorigin=\"anonymous\" to third-party script/link tags (most CDNs publish the correct hash alongside the URL).",
            "2. Where feasible, self-host critical dependencies instead of relying on a third-party CDN.",
        ],
        fix_by_stack={},
        verify="Re-fetch the page source and confirm the integrity attribute is present on third-party script and stylesheet tags.",
    ),

    "insecure_form": KBEntry(
        explain=(
            "A form containing a password field either submits over plain HTTP or doesn't set an appropriate "
            "autocomplete attribute, risking credential exposure in transit or unwanted browser autofill/storage on shared devices."
        ),
        fix_generic=[
            "1. Serve the entire site over HTTPS and ensure form actions use https:// or relative URLs.",
            "2. Set autocomplete=\"current-password\" or \"new-password\" explicitly on password fields (or \"off\" where autofill is genuinely undesirable).",
        ],
        fix_by_stack={},
        verify="Re-inspect the form's action URL and the password field's autocomplete attribute in the page source.",
    ),

    "graphql_introspection": KBEntry(
        explain=(
            "GraphQL introspection is enabled, letting anyone query the full API schema - including fields, "
            "types, and mutations that may not be intended for public documentation - and a deeply-nested "
            "query was accepted without any query-depth/complexity error, suggesting no protection against "
            "resource-exhaustion via nested queries."
        ),
        fix_generic=[
            "1. Disable introspection in production (most GraphQL server libraries have a production-mode flag or a validation rule for this).",
            "2. If introspection is needed for internal tooling, gate it behind authentication or restrict it by IP/environment.",
            "3. Add query depth limiting and/or query cost analysis to prevent resource-exhaustion via deeply nested queries.",
        ],
        fix_by_stack={},
        verify="Re-send the same introspection query ({__schema{types{name}}}); it should return an error or be rejected, not the schema. Re-send a deeply nested query and confirm it's rejected with a depth/complexity error.",
    ),

    "crlf_injection": KBEntry(
        explain=(
            "A parameter allows injection of raw CR/LF characters that create an arbitrary new HTTP response "
            "header or split the response body, indicating unsanitized input is written directly into a raw "
            "HTTP response - a foothold for response-splitting or header-injection attacks."
        ),
        fix_generic=[
            "1. Strip or encode CR/LF characters from any user input used in header values or redirect targets.",
            "2. Use your framework's header-setting API (which typically encodes automatically) instead of building raw response strings.",
        ],
        fix_by_stack={},
        verify="Re-send the same CRLF probe; the injected header/body content should no longer appear in the response.",
    ),

    "host_header": KBEntry(
        explain=(
            "A forged Host header (or X-Forwarded-Host, if the app trusts it for building absolute URLs behind "
            "a reverse proxy) is reflected into the page or a redirect Location, which can be used to poison "
            "password-reset links or cached responses into pointing at an attacker-controlled domain."
        ),
        fix_generic=[
            "1. Validate the Host header against an explicit allow-list of expected domains server-side.",
            "2. Only trust X-Forwarded-Host when set by your own reverse proxy - strip any client-supplied value for it at the edge.",
            "3. Never trust either header when constructing absolute URLs (password reset links, canonical URLs) - use a configured base URL instead.",
        ],
        fix_by_stack={},
        verify="Re-send a request with a forged Host and X-Forwarded-Host header; neither should be reflected anywhere in the response.",
    ),

    "vulnerable_library": KBEntry(
        explain=(
            "The page loads a client-side JavaScript library version with known, publicly disclosed "
            "vulnerabilities. Depending on the library, this can enable XSS, prototype pollution, SSRF (via a "
            "vulnerable HTTP client library), or other client-side attacks that don't require finding a bug in your own code."
        ),
        fix_generic=[
            "1. Upgrade the flagged library to the latest stable version.",
            "2. Add a dependency-scanning step to CI (e.g. npm audit, Snyk, Dependabot) so this is caught automatically going forward.",
        ],
        fix_by_stack={},
        verify="Re-check the script's version string or file hash after upgrading; confirm it matches a patched release.",
    ),

    "info_disclosure_comments": KBEntry(
        explain=(
            "An HTML comment or inline script block in production-served markup contains what looks like "
            "internal notes, a security-relevant TODO, or a hardcoded API key/secret string literal that "
            "shouldn't be visible to site visitors."
        ),
        fix_generic=[
            "1. Strip HTML comments from production builds (most templating/build systems can do this automatically).",
            "2. Move genuinely sensitive notes (credentials, internal URLs) out of source templates entirely - use a secrets manager or internal wiki instead.",
            "3. Never embed API keys/secrets in client-side JavaScript; any value shipped to the browser is effectively public. Rotate any that were found this way.",
        ],
        fix_by_stack={},
        verify="View the page source in production and confirm the comment/hardcoded secret is gone.",
    ),

    "cache_control": KBEntry(
        explain=(
            "A page that appears to require an authenticated session doesn't set Cache-Control: no-store, "
            "meaning shared or browser caches (or the back button on a shared/public computer) could expose "
            "that page's content to a different user."
        ),
        fix_generic=[
            "1. Set Cache-Control: no-store (and Pragma: no-cache for older clients) on all authenticated/sensitive responses.",
            "2. Apply this via middleware for the whole authenticated section of the site rather than per-route.",
        ],
        fix_by_stack={},
        verify="Re-fetch the authenticated page and confirm the Cache-Control header is present with no-store.",
    ),

    "header_injection": KBEntry(
        explain=(
            "A request header (User-Agent, Referer, X-Forwarded-For, or similar) is reflected unescaped into "
            "the HTML response or reaches a database query, indicating this header is logged/displayed without "
            "the same output-encoding or parameterization discipline applied to query parameters and form fields."
        ),
        fix_generic=[
            "1. Apply the same output encoding to every logged/displayed request header that you apply to query parameters.",
            "2. If headers reach a database query (e.g. via request logging), parameterize that query exactly as you would for user input.",
            "3. Audit any analytics/admin dashboards that display raw request metadata (User-Agent, Referer) for the same gap.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "This is a commonly-missed injection surface precisely because request headers don't look like "
            "'user input' the way a form field does - but they're just as attacker-controlled. A vulnerability "
            "here often survives code review specifically because reviewers scrutinize form/query handling and "
            "skip over header handling in logging or analytics code."
        ),
        references=["https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html"],
        verify="Re-send the same request with the header restored to a normal value; re-send the probe again to confirm the marker no longer appears unescaped.",
    ),

    "ldap_injection": KBEntry(
        explain=(
            "A parameter reflects an LDAP-related error when sent characters that have special meaning in "
            "LDAP filter syntax, suggesting user input reaches a directory-service query without escaping."
        ),
        fix_generic=[
            "1. Escape LDAP special characters (* ( ) \\ NUL) in any user input used to build a filter, per RFC 4515.",
            "2. Use your LDAP library's parameterized/escaped filter-building API instead of string concatenation.",
        ],
        fix_by_stack={},
        verify="Re-send the same probe; the response should no longer show an LDAP error and should behave as if the input were treated as a literal string.",
    ),

    "admin_exposure": KBEntry(
        explain=(
            "An admin or management interface is reachable without any visible authentication requirement, "
            "and its content looks like live administrative functionality rather than a login screen - "
            "meaning anyone who finds the URL can potentially act as an administrator."
        ),
        fix_generic=[
            "1. Require authentication in front of every admin/management route, enforced at the routing or middleware layer (not just hidden from navigation).",
            "2. Add a second layer where feasible: IP allow-listing, VPN-only access, or mutual TLS for admin interfaces.",
            "3. Audit for any other management endpoints that might share the same gap (API-only admin routes, monitoring dashboards like Grafana/Kibana/actuator/metrics endpoints are easy to miss).",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Add an authentication-checking middleware (`router.use('/admin', requireAuth)`) before any admin route handlers are registered."],
            "Django (Python)": ["1. Ensure admin views use `@login_required` / `@staff_member_required`, and confirm `ADMIN_URL` isn't left at a guessable default without additional protection."],
        },
        why_it_matters=(
            "This is as close to a worst-case finding as this scanner produces: no injection, no bypass, no "
            "cleverness required to exploit it - the administrative functionality is simply sitting there. "
            "Whatever an authenticated admin can do (manage users, change settings, view all data) an "
            "unauthenticated visitor can do too."
        ),
        references=["https://owasp.org/Top10/A01_2021-Broken_Access_Control/"],
        false_positive_note=(
            "This detector deliberately only flags pages that both return 200 AND contain content that looks "
            "like live admin functionality (not a login form) - but heuristics on page content can still be "
            "wrong on unusual UIs. Manually load the URL in a fresh, cookie-less browser session to confirm "
            "before treating this as certain."
        ),
        verify="Re-request the admin path anonymously (no cookies/tokens); it should redirect to a login page or return 401/403, not admin content.",
    ),

    "weak_session_token": KBEntry(
        explain=(
            "The session cookie's value shows characteristics of weak randomness - too short, sequential, "
            "or low character diversity - meaning it may be guessable or brute-forceable rather than requiring theft."
        ),
        fix_generic=[
            "1. Generate session identifiers using a cryptographically secure random source (e.g. `secrets.token_urlsafe(32)` in Python, `crypto.randomBytes(32)` in Node).",
            "2. Ensure at least 128 bits of entropy in the token.",
            "3. Never derive session IDs from predictable data (user ID, timestamp, counter).",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Use `express-session` with its default secure ID generator - don't override `genid` with custom logic unless it's equally random."],
            "Django (Python)": ["1. Django's built-in session framework already generates cryptographically secure IDs - this finding likely means a custom session mechanism is in use; consider migrating to the built-in one."],
        },
        verify="Inspect several freshly-issued session cookies; values should look like high-entropy random strings with no visible pattern between them.",
    ),

    "prototype_pollution": KBEntry(
        explain=(
            "A parameter accepts a __proto__-nested key and it's echoed back in a JSON response - the "
            "precondition for prototype pollution if the backend later merges this input into an object "
            "without guarding against __proto__/constructor/prototype keys."
        ),
        fix_generic=[
            "1. Validate incoming JSON against a strict schema (reject unexpected keys, especially __proto__/constructor/prototype).",
            "2. Avoid recursive merge/assign functions (e.g. lodash `merge()`, `_.assign()`) on raw, unvalidated request bodies.",
            "3. Use `Object.create(null)` or `Map` for objects built from user input where prototype inheritance isn't needed.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Upgrade lodash to >=4.17.21 and audit any use of `_.merge`/`_.mergeWith`/`_.defaultsDeep` on request bodies.", "2. Consider a body-parsing guard that strips `__proto__` keys before they reach application logic."],
        },
        verify="Re-send the probe and inspect whether Object.prototype has actually been polluted server-side (e.g. via a follow-up request whose behavior would only change if pollution succeeded) - this detector only confirms the precondition, not exploitation.",
    ),
    "mass_assignment": KBEntry(
        explain=(
            "Extra, privilege-shaped fields (isAdmin, role, credits, etc.) sent alongside a form's normal "
            "fields were accepted and echoed back by the API, suggesting the backend binds the whole request "
            "body onto a data model rather than an explicit allow-list of expected fields - the precondition "
            "for mass-assignment privilege escalation."
        ),
        fix_generic=[
            "1. Use an explicit allow-list (DTO/serializer with a fixed field set) for what a request body may set.",
            "2. Never bind a raw request body directly onto a database model/entity.",
            "3. Explicitly exclude privilege, role, balance, and verification-status fields from any client-writable path; set them only through dedicated, separately-authorized endpoints.",
        ],
        fix_by_stack={
            "Express (Node.js)": ["1. Avoid Model.create(req.body) / Object.assign(model, req.body) patterns; explicitly destructure only the allowed fields."],
            "Ruby on Rails": ["1. Use strong parameters (params.require(:user).permit(:name, :email)) - never permit! or a blanket permit(params[:user].keys)."],
            "Django (Python)": ["1. Use a DRF serializer with an explicit `fields` list (never `fields = '__all__'` for user-writable input) and mark privileged fields `read_only=True`."],
        },
        why_it_matters=(
            "This is precisely how several real-world privilege-escalation incidents happened: a signup or "
            "profile-update endpoint bound the entire request body onto the user model, and a client that "
            "simply added \"role\": \"admin\" to their own registration request became an administrator."
        ),
        references=["https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/04-Authentication_Testing/04-Testing_for_Mass_Assignment"],
        false_positive_note="Field acceptance/echo does not confirm the privilege actually changed - always confirm with a second authenticated request checking the account's real role/balance/permissions before treating this as a live escalation.",
        verify="Re-send the same probe body and confirm the extra fields are now rejected (400/422) or silently dropped rather than accepted.",
    ),

    "user_enumeration": KBEntry(
        explain=(
            "Two equally-nonexistent probe identifiers sent to a login/registration/password-reset form "
            "produced measurably different responses, which is the classic shape of username/email "
            "enumeration - though confirming it fully requires one probe that corresponds to a real account, "
            "which this check deliberately never sends."
        ),
        fix_generic=[
            "1. Return an identical, generic response (status code, message text, and roughly equal timing) regardless of whether the submitted identifier corresponds to a real account.",
            "2. Apply this consistently across login, registration, and password-reset flows - a gap in any one of them re-opens enumeration.",
            "3. Rate-limit these endpoints regardless, since enumeration and credential-stuffing share the same entry point.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "User enumeration is rarely the end goal - it's reconnaissance that makes every subsequent attack "
            "(credential stuffing, targeted phishing, password reset abuse) more efficient by first confirming "
            "which identifiers are worth attacking at all."
        ),
        references=["https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/04-Authentication_Testing/03-Testing_for_Weak_Lock_Out_Mechanism"],
        verify="Re-send both probe identifiers and confirm the responses (status, body, and approximate timing) are now indistinguishable.",
    ),

    "security_txt": KBEntry(
        explain=(
            "There is no security.txt (RFC 9116) published at /.well-known/security.txt, so there is no "
            "standardized, discoverable channel for a security researcher who finds a real vulnerability to "
            "report it responsibly - or, where one exists, it's missing the required Expires field."
        ),
        fix_generic=[
            "1. Publish a security.txt file at /.well-known/security.txt with at minimum a Contact field (email or URL) and an Expires field (ISO 8601 date).",
            "2. Keep the Expires date refreshed periodically - stale/expired security.txt files are treated as untrustworthy by tooling that consumes them.",
            "3. Optionally add a Policy field linking to a vulnerability-disclosure or bug-bounty policy page.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "This is a low-severity hygiene item, not a vulnerability - but its absence has a real cost: a "
            "researcher who finds something serious with no obvious reporting channel may give up, post it "
            "publicly without warning, or never report it at all."
        ),
        references=["https://www.rfc-editor.org/rfc/rfc9116"],
        verify="Re-fetch /.well-known/security.txt and confirm it exists with a Contact and current Expires field.",
    ),

    "subdomain_takeover": KBEntry(
        explain=(
            "A DNS CNAME record points at a third-party SaaS platform (GitHub Pages, Heroku, S3, Azure, etc.), "
            "and requesting the hostname returns that platform's 'not claimed/not found' error page - meaning "
            "the DNS record still points at the platform, but whatever resource used to live there is gone, "
            "so anyone can register that same name on the platform and start serving content under this domain."
        ),
        fix_generic=[
            "1. Remove the dangling CNAME record if the corresponding resource is no longer in use.",
            "2. If the resource is still needed, re-register/re-claim it on the third-party platform under this organization's account.",
            "3. Periodically audit DNS records against actually-provisioned resources, especially after decommissioning any cloud service.",
        ],
        fix_by_stack={},
        why_it_matters=(
            "A successful takeover lets an attacker serve arbitrary content - including phishing pages, "
            "malware, or content that steals cookies set on the parent domain - from a subdomain that looks "
            "completely legitimate to users and often to automated trust checks (some SSO/CORS allow-lists "
            "trust entire subdomains)."
        ),
        references=["https://github.com/EdOverflow/can-i-take-over-xyz"],
        false_positive_note="Confirm the CNAME target and the 'unclaimed' response are both still true at the moment of remediation - DNS and third-party service state can both change between the scan and the fix.",
        verify="Re-resolve the CNAME and re-request the hostname; either the CNAME should be gone, or the resource should now be properly claimed and serving expected content.",
    ),

}

_GENERIC_KB_ENTRY = KBEntry(
    explain="A potential weakness was identified in this area based on the scan's detection heuristics.",
    fix_generic=["1. Review the finding details and evidence above.",
                 "2. Apply the OWASP recommendation for this vulnerability class (search cheatsheetseries.owasp.org for the category name).",
                 "3. Re-test after the fix."],
    fix_by_stack={},
    verify="Re-run the scan after applying a fix to confirm the finding no longer appears.",
)


_CHAIN_GUIDANCE: Dict[str, str] = {
    "Session hijack chain": (
        "Fix the missing cookie/CSP protections first if the underlying XSS can't be patched immediately - "
        "HttpOnly cookies and a strict CSP limit how much an attacker can do with an injected script even "
        "before the injection point itself is fixed. Then eliminate the XSS at its source with output encoding."
    ),
    "Credential exposure chain": (
        "Rotate any exposed credentials immediately - this is time-sensitive regardless of which fix lands "
        "first. Then remove the exposed file/path, and separately fix the verbose error output so future "
        "mistakes don't hand out the same level of detail."
    ),
    "CSRF + auth chain": (
        "Fix the CORS policy first - a correct origin allow-list closes off the cross-origin read/write path "
        "immediately. Add CSRF tokens as a second, independent layer so the app isn't relying on CORS alone."
    ),
    "Open redirect + host trust chain": (
        "Fix Host header validation first since it affects more than just this redirect (password resets, "
        "canonical URLs). Then add an allow-list to the redirect parameter itself."
    ),
    "Injection + verbose error chain": (
        "Fix the verbose error output first - it's usually a one-line config change and immediately removes "
        "the attacker's fastest feedback loop, buying time while the injection vulnerability itself is patched properly."
    ),
    "Supply-chain XSS chain": (
        "Add a strict CSP now - it's a config-only change that contains the blast radius of the vulnerable "
        "library regardless of when the upgrade lands. Then schedule the library upgrade through your normal "
        "dependency-update process."
    ),
    "Backup exposure chain": (
        "Disable directory listing first - it's the smaller, faster fix and immediately stops the exposed "
        "file from acting as a map to everything else in that directory. Then remove/relocate the exposed "
        "file itself and rotate any credentials it contained."
    ),
    "Unauthenticated admin discovery chain": (
        "Gate the admin interface behind authentication immediately - this is the higher-impact fix by far. "
        "Hiding the server version header is good hygiene but doesn't meaningfully help until the admin panel itself is locked down."
    ),
    "Predictable session chain": (
        "Fix session token generation first (switch to a cryptographically secure random generator with "
        "sufficient length) since that's the root cause; the missing cookie flags make exploitation of "
        "already-known tokens easier but don't create the weakness on their own."
    ),
    "GraphQL over-exposure chain": (
        "Fix the CORS policy first to stop cross-origin browsers from riding along with a victim's "
        "credentials. Disabling introspection separately reduces reconnaissance value but doesn't address "
        "the more serious cross-origin request/read issue."
    ),
    "Remote code execution chain": (
        "Treat this as an active-incident-severity finding, not a normal remediation-backlog item: an "
        "unauthenticated path to command execution combined with confirmed injection means the fix for the "
        "injection itself is the entire priority - patch the vulnerable input handling first, then work "
        "backward through anything the injection point could have already reached (credentials, other hosts on the same network)."
    ),
    "No object authorization at scale chain": (
        "Fix object-level authorization first (verify record ownership on every request) - the missing rate "
        "limit on top of it means an attacker could enumerate the entire ID space quickly once the "
        "authorization gap exists, so closing the authorization gap removes the incentive to even try."
    ),
    "XXE to internal reach chain": (
        "Disable external entity/DTD processing in the XML parser first - this closes the XXE vector "
        "entirely and is usually a single config flag. The SSRF-adjacent internal-network-reach concern only "
        "matters if the XXE vector remains open, so there's no reason to sequence it any other way."
    ),
    "Spoofable identity chain": (
        "Fix email anti-spoofing (SPF/DMARC) and the underlying auth weakness in parallel where possible - "
        "they compound (a spoofed 'reset your password' email is far more convincing when combined with a "
        "believable-looking, non-validated host in the reset link) but neither fix depends on the other."
    ),
}


class LocalLLMPatchGenerator:
    """Optional, fully-offline patch generation via a local GGUF model through
    llama-cpp-python. This is NOT enabled by default and NOT required for the
    tool to work - the knowledge-base engine above is complete on its own.

    Important honesty note: this integration has not been tested against
    actual model inference in the environment this tool was built in (no
    network access to download model weights, and llama-cpp-python is not
    guaranteed to be installed). The code degrades gracefully at every step:
    if the package isn't installed, the model file doesn't exist, or
    generation raises any exception, it silently returns None and the
    knowledge-base text is used instead. Treat this path as experimental
    until you've verified it yourself with your own model file.

    Even when active, the prompt only ever asks for a configuration/code
    patch for an already-identified, already-explained defensive fix - the
    same restriction as the rest of the advisor, just running locally
    instead of over an API."""

    _SYSTEM_PROMPT = (
        "You are a defensive security patch generator. Given a vulnerability category, "
        "the detected tech stack, and a generic remediation summary, output ONLY the exact "
        "configuration block or code snippet that fixes it for that stack. No explanation, "
        "no attack content, no markdown fences - just the patch."
    )

    def __init__(self, model_path: Optional[str], verbose: bool = False):
        self.model_path = model_path
        self.verbose = verbose
        self._llm = None
        self._load_attempted = False

    def _ensure_loaded(self) -> bool:
        if self._llm is not None:
            return True
        if self._load_attempted:
            return False
        self._load_attempted = True
        if not self.model_path or not os.path.exists(self.model_path):
            return False
        try:
            from llama_cpp import Llama  # optional dependency, not required
            self._llm = Llama(model_path=self.model_path, n_ctx=2048, verbose=False)
            return True
        except Exception as e:
            if self.verbose:
                print(f"  [!] Local LLM unavailable, falling back to built-in knowledge base: {e}")
            return False

    def generate_patch(self, category: str, stack: Optional[str], remediation_summary: str) -> Optional[str]:
        if not self._ensure_loaded():
            return None
        prompt = (
            f"Vulnerability category: {category}\n"
            f"Detected stack: {stack or 'unknown/generic'}\n"
            f"Remediation summary: {remediation_summary}\n\n"
            "Output the exact patch:"
        )
        try:
            result = self._llm.create_chat_completion(
                messages=[{"role": "system", "content": self._SYSTEM_PROMPT},
                          {"role": "user", "content": prompt}],
                max_tokens=400,
                temperature=0.2,
            )
            text = result["choices"][0]["message"]["content"].strip()
            return text or None
        except Exception as e:
            if self.verbose:
                print(f"  [!] Local LLM generation failed for {category}: {e}")
            return None


# Rough remediation-effort estimate per category, in developer-hours, for the
# generic (non-stack-specific) fix path. Deliberately coarse buckets (not a
# false-precision single number) - useful for sprint planning without
# pretending to know a specific codebase's complexity. Purely advisory.
_EFFORT_ESTIMATE_HOURS: Dict[str, Tuple[float, float]] = {
    "security_headers": (0.5, 2.0), "tls": (0.5, 3.0), "server_banner": (0.25, 1.0),
    "http_methods": (0.5, 2.0), "cors": (1.0, 4.0), "exposed_files": (0.25, 2.0),
    "sql_injection": (2.0, 16.0), "nosql_injection": (2.0, 12.0), "ssrf": (2.0, 8.0),
    "xss": (1.0, 8.0), "ssti": (2.0, 8.0), "open_redirect": (0.5, 2.0),
    "csrf": (1.0, 4.0), "jwt": (1.0, 6.0), "directory_listing": (0.25, 1.0),
    "verbose_errors": (0.25, 1.0), "mixed_content": (0.5, 2.0), "sri": (0.5, 2.0),
    "insecure_form": (0.25, 1.0), "graphql_introspection": (0.5, 3.0),
    "crlf_injection": (0.5, 2.0), "host_header": (0.5, 2.0), "vulnerable_library": (0.5, 3.0),
    "info_disclosure_comments": (0.25, 1.0), "cache_control": (0.25, 1.0),
    "ldap_injection": (1.0, 4.0), "header_injection": (1.0, 4.0), "admin_exposure": (1.0, 6.0),
    "weak_session_token": (0.5, 3.0), "prototype_pollution": (1.0, 4.0),
    "xxe": (0.5, 3.0), "xpath_injection": (1.0, 4.0), "command_injection": (2.0, 12.0),
    "idor_heuristic": (2.0, 12.0), "rate_limiting": (1.0, 6.0), "websocket": (1.0, 4.0),
    "email_security": (0.5, 2.0),
}


def estimate_remediation_effort(category: str) -> Dict[str, Any]:
    lo, hi = _EFFORT_ESTIMATE_HOURS.get(category, (0.5, 4.0))
    return {"low_hours": lo, "high_hours": hi,
            "note": "Rough order-of-magnitude estimate for the generic fix path on a typical codebase; "
                    "actual effort depends heavily on how the affected code is structured. Not a commitment."}


class AdvisorEngine:
    """Local, built-in remediation advisor. No network calls, no API key,
    no external dependency - a rule-based expert system over a curated
    knowledge base, with output tailored by stack fingerprint and by
    risk-chain membership (both computed deterministically elsewhere).
    Optionally augmented by a local LLM (see LocalLLMPatchGenerator) if the
    user has supplied a GGUF model path; falls back to pure KB text otherwise."""

    def __init__(self, config: ScanConfig):
        self.config = config
        self.enabled = config.ai_enabled
        self.local_llm = LocalLLMPatchGenerator(
            getattr(config, "local_model_path", None), verbose=config.verbose
        )

    def _pick_stack_name(self, stack_fingerprint: Dict[str, str]) -> Optional[str]:
        return stack_fingerprint.get("framework_guess")

    def annotate(self, finding: Finding, stack_fingerprint: Optional[Dict[str, str]] = None) -> None:
        stack_fingerprint = stack_fingerprint or {}
        if not self.enabled:
            finding.ai_explanation = finding.description
            finding.ai_remediation_steps = finding.remediation_summary
            finding.ai_verification_steps = "Re-run the scan after applying the fix to confirm the finding no longer appears."
            return

        kb = _KNOWLEDGE_BASE.get(finding.category, _GENERIC_KB_ENTRY)
        stack = self._pick_stack_name(stack_fingerprint)
        steps = list(kb.fix_generic)
        if stack and stack in kb.fix_by_stack:
            steps = steps + ["--- " + stack + "-specific ---"] + kb.fix_by_stack[stack]

        llm_patch = self.local_llm.generate_patch(finding.category, stack, kb.explain)
        if llm_patch:
            steps = steps + ["--- Local-LLM-generated patch (experimental, verify before applying) ---", llm_patch]

        finding.ai_explanation = kb.explain
        finding.ai_remediation_steps = "\n".join(steps)
        finding.ai_verification_steps = kb.verify

    def annotate_all(self, findings: List[Finding], stack_fingerprint: Optional[Dict[str, str]] = None) -> None:
        for f in findings:
            self.annotate(f, stack_fingerprint)

    def explain_chain(self, chain_name: str, categories: List[str], rationale: str) -> str:
        if not self.enabled:
            return rationale
        return _CHAIN_GUIDANCE.get(chain_name, rationale)

    def business_impact_summary(self, findings: List[Finding], posture_grade: str) -> str:
        """A short, deterministic (template-based, not generative) paragraph
        aimed at a non-technical stakeholder - separate from the per-finding
        technical explanations, for the executive-view mode of the dashboard."""
        if not findings:
            return (f"This scan found no issues at the profile/depth used. Posture grade: {posture_grade}. "
                     "A clean scan reduces risk but does not guarantee the absence of vulnerabilities outside "
                     "this tool's detection scope (e.g. business-logic flaws, issues requiring authenticated "
                     "multi-step workflows, or areas outside the crawled surface).")
        crit = sum(1 for f in findings if f.severity == Severity.CRITICAL)
        high = sum(1 for f in findings if f.severity == Severity.HIGH)
        worst_titles = [f.title for f in findings if f.severity == Severity.CRITICAL][:3]
        parts = [f"This scan identified {len(findings)} finding(s), posture grade {posture_grade}."]
        if crit:
            parts.append(f"{crit} finding(s) are rated Critical - the kind of issue that, in a real incident, "
                          "typically leads directly to data exposure or full system compromise rather than a "
                          "contained, minor bug.")
            if worst_titles:
                parts.append("Most urgent: " + "; ".join(worst_titles) + ".")
        if high:
            parts.append(f"{high} additional finding(s) are rated High and should be scheduled promptly after "
                          "the Critical items.")
        parts.append("Recommend prioritizing fixes in the order shown in the findings ledger, which already "
                      "accounts for severity, confidence, and whether a finding participates in a compound risk chain.")
        return " ".join(parts)


# =============================================================================
# Orchestrator
# =============================================================================

# -----------------------------------------------------------------------------
# Stack fingerprinting - deterministic, not AI. Informs which checks matter
# most and lets the AI advisor give stack-specific remediation instead of
# generic advice.
# -----------------------------------------------------------------------------

_COOKIE_STACK_HINTS = {
    "phpsessid": "PHP", "jsessionid": "Java", "asp.net_sessionid": "ASP.NET",
    "laravel_session": "Laravel (PHP)", "connect.sid": "Express (Node.js)",
    "csrftoken": "Django (Python)", "sessionid": "Django (Python)",
    "_rails_session": "Ruby on Rails", "wordpress_logged_in": "WordPress",
}
_HEADER_STACK_HINTS = {
    "x-powered-by": lambda v: v,
    "server": lambda v: v,
}


def fingerprint_stack(resp: "requests.Response") -> Dict[str, str]:
    fp: Dict[str, str] = {}
    for header, extractor in _HEADER_STACK_HINTS.items():
        val = resp.headers.get(header)
        if val:
            fp[header] = extractor(val)
    for cookie_name in resp.cookies.keys():
        hint = _COOKIE_STACK_HINTS.get(cookie_name.lower())
        if hint:
            fp["framework_guess"] = hint
    # Body-level tells, used only when headers gave nothing - a last-resort signal,
    # not a primary one, since these markers can appear in vendored/bundled code
    # unrelated to the actual backend framework.
    if "framework_guess" not in fp:
        body_hint_map = {
            "csrfmiddlewaretoken": "Django (Python)",
            "__requestverificationtoken": "ASP.NET",
            "authenticity_token": "Ruby on Rails",
            "__viewstate": "ASP.NET",
        }
        body_lower = (getattr(resp, "text", "") or "")[:20000].lower()
        for marker, guess in body_hint_map.items():
            if marker in body_lower:
                fp["framework_guess"] = guess
                fp["framework_guess_basis"] = "body marker (lower confidence than header/cookie signal)"
                break
    return fp


# -----------------------------------------------------------------------------
# Deterministic risk-correlation engine. This is plain rule-based logic (not
# an LLM call) that looks at the *set* of findings together and flags when
# individually-moderate issues combine into a materially worse attack chain,
# and computes a priority order for fixes. The LLM is then used downstream
# only to explain/phrase these already-derived conclusions.
# -----------------------------------------------------------------------------

@dataclass
class RiskChain:
    name: str
    member_categories: List[str]
    combined_severity: Severity
    rationale: str


_CHAIN_RULES: List[Tuple[str, Set[str], Severity, str]] = [
    (
        "Session hijack chain",
        {"xss", "security_headers"},
        Severity.CRITICAL,
        "Reflected/stored XSS combined with missing cookie security flags or CSP means an attacker's "
        "injected script can directly read and exfiltrate session cookies, turning an XSS bug into full "
        "account takeover rather than a cosmetic issue.",
    ),
    (
        "Credential exposure chain",
        {"exposed_files", "verbose_errors"},
        Severity.CRITICAL,
        "An exposed config/secrets file combined with verbose error pages gives an attacker both the "
        "credentials and the internal file paths/framework details needed to use them effectively.",
    ),
    (
        "CSRF + auth chain",
        {"csrf", "cors"},
        Severity.HIGH,
        "Missing CSRF protection combined with a permissive CORS policy means cross-origin pages can both "
        "read authenticated responses and forge state-changing requests on a victim's behalf.",
    ),
    (
        "Open redirect + host trust chain",
        {"open_redirect", "host_header"},
        Severity.HIGH,
        "An open redirect combined with Host header trust issues makes phishing links and password-reset "
        "poisoning significantly more convincing, since both can be used to build a URL that looks native to the domain.",
    ),
    (
        "Injection + verbose error chain",
        {"sql_injection", "verbose_errors"},
        Severity.CRITICAL,
        "SQL injection combined with verbose error/debug output means an attacker gets detailed database "
        "error feedback, dramatically speeding up exploitation even without a dedicated tool.",
    ),
    (
        "Supply-chain XSS chain",
        {"vulnerable_library", "security_headers"},
        Severity.HIGH,
        "A known-vulnerable client-side library combined with a missing/weak Content-Security-Policy means "
        "there's no browser-level backstop if that library's disclosed vulnerability is a DOM-XSS or "
        "sanitizer-bypass bug - the CSP would normally contain the blast radius even without patching immediately.",
    ),
    (
        "Backup exposure chain",
        {"directory_listing", "exposed_files"},
        Severity.CRITICAL,
        "Directory listing being enabled turns a single exposed backup/config file into a much bigger problem: "
        "an attacker can browse the whole directory to find every other backup, dump, or config file sitting alongside it.",
    ),
    (
        "Unauthenticated admin discovery chain",
        {"admin_exposure", "server_banner"},
        Severity.CRITICAL,
        "An unauthenticated admin/management interface combined with a disclosed server/framework version "
        "lets an attacker immediately cross-reference the admin panel's software against known CVEs for that "
        "exact version - reconnaissance and target selection happen in a single request.",
    ),
    (
        "Predictable session chain",
        {"weak_session_token", "security_headers"},
        Severity.CRITICAL,
        "A weakly-generated, guessable session identifier combined with missing cookie security flags means "
        "an attacker doesn't even need to steal a session token via XSS - they may be able to guess or "
        "brute-force valid ones directly, and nothing stops the resulting cookie from being used cross-context.",
    ),
    (
        "GraphQL over-exposure chain",
        {"graphql_introspection", "cors"},
        Severity.HIGH,
        "GraphQL introspection combined with a permissive CORS policy means any third-party site can both "
        "enumerate the full API schema and issue authenticated queries/mutations against it from a victim's browser.",
    ),
    (
        "Remote code execution chain",
        {"command_injection", "sql_injection"},
        Severity.CRITICAL,
        "Confirmed command injection combined with SQL injection findings elsewhere on the same target "
        "suggests a systemic pattern of unsanitized input reaching execution/interpretation contexts, not an "
        "isolated mistake - review input-handling practices across the whole codebase, not just these two endpoints.",
    ),
    (
        "No object authorization at scale chain",
        {"idor_heuristic", "rate_limiting"},
        Severity.HIGH,
        "A possible object-authorization gap combined with no rate limiting on top of it means that even if "
        "each individual guess is 'possible, not confirmed', an attacker has no practical obstacle to testing "
        "the entire ID space quickly to find out.",
    ),
    (
        "XXE to internal reach chain",
        {"xxe", "ssrf"},
        Severity.CRITICAL,
        "XXE processing being enabled alongside a separate SSRF finding on the same target suggests the "
        "application's server-side request/parsing surface broadly lacks destination validation - the XXE "
        "vector specifically can be escalated to reach the same internal resources the SSRF finding already "
        "demonstrates are reachable from this server.",
    ),
    (
        "Spoofable identity chain",
        {"email_security", "open_redirect"},
        Severity.MEDIUM,
        "Weak email anti-spoofing (SPF/DMARC) combined with an open redirect on the same domain is a "
        "particularly convincing phishing combination: a spoofed email that appears to come from this domain, "
        "linking to a real URL on this domain that then redirects the victim elsewhere.",
    ),
    (
        "Privilege escalation via unvalidated input chain",
        {"mass_assignment", "idor_heuristic"},
        Severity.CRITICAL,
        "A mass-assignment gap (extra fields silently accepted) combined with a possible object-authorization "
        "gap is a materially worse combination than either alone: even if a direct role='admin' field write "
        "doesn't stick, weak object-level authorization may let an attacker who mass-assigns themselves "
        "elevated-looking fields on one record then read or modify records belonging to other users to check "
        "whether the elevation actually took hold anywhere in the data model.",
    ),
    (
        "Credential stuffing setup chain",
        {"user_enumeration", "rate_limiting"},
        Severity.HIGH,
        "Username/email enumeration combined with no rate limiting removes both obstacles a credential-"
        "stuffing attack needs cleared in sequence: first knowing which identifiers are real accounts, then "
        "being able to try passwords against them without being throttled.",
    ),
    (
        "Full authentication bypass chain",
        {"jwt", "cors"},
        Severity.CRITICAL,
        "A JWT configuration weakness (alg=none, missing expiration, or a key-confusion-prone jku/kid header) "
        "combined with a permissive CORS policy means a forged or leaked token isn't just usable directly - it "
        "can also be exfiltrated and replayed cross-origin from any site a victim visits, widening how a "
        "single token weakness can be discovered and abused.",
    ),
    (
        "Brand impersonation infrastructure chain",
        {"subdomain_takeover", "email_security"},
        Severity.CRITICAL,
        "A dangling CNAME (subdomain takeover) combined with weak email anti-spoofing on the same root domain "
        "gives an attacker both a legitimately-hosted-looking subdomain to serve phishing content from AND a "
        "way to send email that appears to originate from the same trusted domain - a substantially more "
        "convincing and durable phishing/BEC (business email compromise) setup than either weakness alone.",
    ),
    (
        "Stored payload amplification chain",
        {"xss", "admin_exposure"},
        Severity.CRITICAL,
        "A stored/second-order XSS finding combined with an exposed, unauthenticated admin interface raises "
        "the realistic worst case substantially: if the admin panel ever renders user-submitted content "
        "(support tickets, user profiles, uploaded content) without escaping, a stored payload could execute "
        "in an administrator's browser session specifically, not just an ordinary visitor's.",
    ),
]


def correlate_findings(findings: List[Finding]) -> List[RiskChain]:
    present_categories = {f.category for f in findings}
    chains = []
    for name, required, severity, rationale in _CHAIN_RULES:
        if required.issubset(present_categories):
            chains.append(RiskChain(name=name, member_categories=sorted(required),
                                     combined_severity=severity, rationale=rationale))
    return chains


def prioritize(findings: List[Finding], chains: List[RiskChain]) -> List[Finding]:
    chained_categories: Set[str] = set()
    for c in chains:
        chained_categories.update(c.member_categories)

    def score(f: Finding) -> Tuple[int, int]:
        boost = 1 if f.category in chained_categories else 0
        return (-f.severity.value - boost, -{"Confirmed": 2, "Likely": 1, "Possible": 0}.get(f.confidence, 0))

    return sorted(findings, key=score)


# Categories where the same underlying root cause (a web-server/framework-wide
# config setting) manifests identically on every crawled page. Reporting one
# finding per page for these is noise, not signal - a 50-page site with one
# missing-CSP misconfiguration should report ONE finding affecting 50 pages,
# not 50 near-duplicate findings. Categories left out of this set (mixed
# content, SRI, verbose errors, etc.) are deliberately kept per-page because
# their evidence genuinely differs page to page.
_CONSOLIDATABLE_CATEGORIES = {"security_headers", "cache_control", "security_txt", "email_security"}


def consolidate_sitewide_findings(findings: List[Finding]) -> List[Finding]:
    """Merge same-category, same-description findings (the real signature of
    "this is the same root cause") into one representative Finding carrying
    the full list of affected URLs, instead of one Finding object per page."""
    consolidatable = [f for f in findings if f.category in _CONSOLIDATABLE_CATEGORIES]
    other = [f for f in findings if f.category not in _CONSOLIDATABLE_CATEGORIES]

    groups: Dict[Tuple[str, str], List[Finding]] = {}
    for f in consolidatable:
        groups.setdefault((f.category, f.description), []).append(f)

    consolidated: List[Finding] = []
    for (_category, _description), group in groups.items():
        representative = group[0]
        representative.affected_urls = sorted({g.url for g in group})
        consolidated.append(representative)

    return other + consolidated


# -----------------------------------------------------------------------------
# Attack-path fusion: a small forward-chaining rule engine (the same
# rule-based-AI paradigm as AdvisorEngine, applied to composition instead of
# remediation text). Each finding grants one or more "capabilities" an
# attacker would have if it were real; composition rules combine capability
# SETS into higher-order capabilities when their preconditions are all
# present among the scan's findings. Iterating to a fixpoint surfaces
# multi-hop paths - e.g. XSS grants "run_script_in_victim_browser"; a
# missing HttpOnly flag grants "session_cookie_js_readable"; together they
# compose into "full_session_hijack", a materially different and worse
# capability than either finding implies alone.
#
# Scope honesty: this is a small, hand-curated capability model (not a
# general graph-security-analysis system), and it reasons over DETECTED
# findings only - it never executes anything to confirm a composed path is
# truly exploitable, the same non-exploitation boundary as the rest of this
# tool. Treat its output as a prioritization aid, not a proof.
# -----------------------------------------------------------------------------

_CAPABILITY_GRANTS: Dict[str, Set[str]] = {
    "xss":                      {"run_script_in_victim_browser"},
    "header_injection":         {"run_script_in_victim_browser"},
    "ssti":                     {"run_script_in_victim_browser", "server_code_execution"},
    "command_injection":        {"server_code_execution"},
    "xxe":                      {"reach_internal_network", "read_local_files_possible"},
    "sql_injection":            {"read_database", "write_database"},
    "nosql_injection":          {"read_database", "auth_bypass_possible"},
    "ldap_injection":           {"read_directory_data"},
    "xpath_injection":          {"read_xml_data"},
    "prototype_pollution":      {"logic_bypass_possible"},
    "csrf":                     {"forge_authenticated_request"},
    "cors":                     {"cross_origin_credentialed_read"},
    "open_redirect":            {"credible_phishing_link"},
    "host_header":              {"poison_absolute_url"},
    "jwt":                      {"forge_or_bypass_auth_token"},
    "weak_session_token":       {"guess_valid_session"},
    "admin_exposure":           {"admin_control_no_auth"},
    "exposed_files":            {"obtain_credentials_or_source"},
    "directory_listing":        {"enumerate_all_files"},
    "verbose_errors":           {"leak_internal_details"},
    "graphql_introspection":    {"enumerate_api_schema"},
    "vulnerable_library":       {"known_client_cve_present"},
    "server_banner":            {"leak_internal_details"},
    "ssrf":                     {"reach_internal_network"},
    "idor_heuristic":           {"enumerate_other_users_records_possible"},
    "rate_limiting":            {"unlimited_guess_attempts"},
    "weak_session_token_pair":  set(),
    "email_security":           {"credible_spoofed_sender"},
    "websocket":                {"cross_origin_credentialed_read"},
    "mass_assignment":          {"privilege_field_write_possible"},
    "user_enumeration":         {"account_identifiers_enumerable"},
    "subdomain_takeover":       {"attacker_controlled_trusted_subdomain"},
    "security_headers":         set(),  # contributes via absence, handled specially below
}

# Preconditions -> composed capability. Every capability in the frozenset
# must be present among the scan's granted capabilities for the rule to fire.
_COMPOSITION_RULES: List[Tuple[frozenset, str, Severity, str]] = [
    (
        frozenset({"run_script_in_victim_browser", "cookie_not_httponly"}),
        "full_session_hijack",
        Severity.CRITICAL,
        "An attacker's injected script can read the session cookie directly (no HttpOnly protection) and "
        "exfiltrate it, turning script execution into full account takeover rather than page defacement.",
    ),
    (
        frozenset({"forge_authenticated_request", "cross_origin_credentialed_read"}),
        "cross_origin_account_takeover",
        Severity.CRITICAL,
        "A malicious page can both trigger authenticated state-changing requests AND read the JSON responses "
        "back cross-origin - well beyond a classic blind CSRF, this allows full request/response round-trips "
        "as the victim.",
    ),
    (
        frozenset({"obtain_credentials_or_source", "read_database"}),
        "credential_reuse_data_breach",
        Severity.CRITICAL,
        "Exposed credentials plus direct database read access means the credentials found in one finding can "
        "likely be validated and reused via the other, confirming a real breach rather than two separate "
        "theoretical issues.",
    ),
    (
        frozenset({"admin_control_no_auth", "leak_internal_details"}),
        "targeted_admin_compromise",
        Severity.CRITICAL,
        "An unauthenticated path to admin functionality combined with disclosed internal/version details lets "
        "an attacker tailor an approach (known CVEs, internal naming, framework quirks) specifically at the "
        "exposed admin surface.",
    ),
    (
        frozenset({"guess_valid_session", "cookie_not_httponly"}),
        "session_guessing_takeover",
        Severity.CRITICAL,
        "A weakly-generated session token doesn't even require theft - combined with no HttpOnly protection "
        "signaling generally weak cookie hygiene, this suggests session handling as a whole hasn't been hardened.",
    ),
    (
        frozenset({"enumerate_api_schema", "cross_origin_credentialed_read"}),
        "full_api_surface_exposure",
        Severity.HIGH,
        "The complete API schema is enumerable AND readable cross-origin with credentials - an attacker has "
        "both the map and a way to use it from outside the app's own origin.",
    ),
    (
        frozenset({"known_client_cve_present", "run_script_in_victim_browser"}),
        "supply_chain_exploit_path",
        Severity.HIGH,
        "A known-vulnerable library is present on a page that also has a working script-injection point - "
        "the injection point could be used to specifically target that library's disclosed vulnerability.",
    ),
    (
        frozenset({"credible_phishing_link", "poison_absolute_url"}),
        "convincing_phishing_infrastructure",
        Severity.HIGH,
        "Both an open redirect and Host-header trust issues are present - an attacker can construct links and "
        "password-reset flows that both look and behave like they originate from the real domain.",
    ),
    (
        frozenset({"reach_internal_network", "admin_control_no_auth"}),
        "ssrf_to_internal_admin_pivot",
        Severity.CRITICAL,
        "SSRF (or XXE-enabled internal reach) gives an attacker the ability to make the server issue requests "
        "to internal-only network locations, and an unauthenticated admin interface is exposed - if that admin "
        "interface is only reachable from inside the network, this may be the missing piece that makes it "
        "reachable at all.",
    ),
    (
        frozenset({"reach_internal_network", "leak_internal_details"}),
        "ssrf_reconnaissance_path",
        Severity.HIGH,
        "SSRF (or XXE-enabled internal reach) combined with disclosed internal details (framework/server "
        "version, verbose errors) means an attacker can use it to probe internal infrastructure while already "
        "knowing what software to expect there.",
    ),
    (
        frozenset({"server_code_execution", "leak_internal_details"}),
        "full_host_compromise_path",
        Severity.CRITICAL,
        "Confirmed or likely server-side code execution combined with disclosed internal details (paths, "
        "framework/server version, stack traces) gives an attacker both the ability to run arbitrary code and "
        "the specific environment knowledge to do so effectively (expected file locations, installed runtime version).",
    ),
    (
        frozenset({"enumerate_other_users_records_possible", "unlimited_guess_attempts"}),
        "mass_data_scraping_path",
        Severity.HIGH,
        "A possible object-authorization gap combined with no rate limiting removes the practical obstacle to "
        "testing it at scale - even a 'Possible'-confidence IDOR finding becomes a real scraping risk once "
        "nothing throttles how fast an attacker can iterate through IDs.",
    ),
    (
        frozenset({"credible_spoofed_sender", "credible_phishing_link"}),
        "convincing_phishing_campaign_path",
        Severity.HIGH,
        "Weak email anti-spoofing combined with an open redirect on the same domain means a phishing email "
        "can plausibly appear to come from this domain AND link to a URL that visibly starts on this domain "
        "before redirecting - a materially more convincing campaign than either weakness alone would support.",
    ),
    (
        frozenset({"privilege_field_write_possible", "enumerate_other_users_records_possible"}),
        "self_service_privilege_escalation_path",
        Severity.CRITICAL,
        "A mass-assignment field-write signal combined with a possible object-authorization gap means an "
        "attacker's own account could potentially both acquire elevated-looking fields AND read/act on other "
        "users' records to determine whether that elevation is honored anywhere in the system - two "
        "individually 'Possible'-confidence findings that together sketch a plausible full escalation path.",
    ),
    (
        frozenset({"account_identifiers_enumerable", "unlimited_guess_attempts"}),
        "credential_stuffing_ready_path",
        Severity.HIGH,
        "Enumerable account identifiers combined with no rate limiting means both prerequisites for an "
        "automated credential-stuffing campaign against real accounts are already in place.",
    ),
    (
        frozenset({"forge_or_bypass_auth_token", "cross_origin_credentialed_read"}),
        "token_exfiltration_replay_path",
        Severity.CRITICAL,
        "A JWT configuration weakness combined with a permissive CORS policy means a token isn't just "
        "forgeable/bypassable in isolation - it can also potentially be read cross-origin from a victim's "
        "browser and replayed, widening the practical discovery and abuse surface for the same underlying flaw.",
    ),
    (
        frozenset({"attacker_controlled_trusted_subdomain", "credible_spoofed_sender"}),
        "brand_impersonation_infrastructure_path",
        Severity.CRITICAL,
        "A dangling CNAME an attacker can claim, combined with weak email anti-spoofing on the same root "
        "domain, gives an attacker both a legitimately-hosted-looking place to serve content from AND a way "
        "to deliver email that appears to originate from the same trusted domain.",
    ),
    (
        frozenset({"run_script_in_victim_browser", "admin_control_no_auth"}),
        "admin_session_targeted_xss_path",
        Severity.CRITICAL,
        "A working script-injection point combined with an exposed, unauthenticated admin interface raises "
        "the realistic worst case: if the admin panel ever renders the same user-submitted content the "
        "injection point reaches, a payload could specifically target an administrator's session rather than "
        "an ordinary visitor's.",
    ),
]


@dataclass
class AttackPath:
    name: str
    severity: Severity
    rationale: str
    contributing_findings: List[str]  # finding_ids that granted the composing capabilities
    hop_count: int


def fuse_attack_paths(findings: List[Finding]) -> List[AttackPath]:
    """Forward-chaining fixpoint: start with base capabilities granted directly
    by findings, repeatedly apply composition rules, stop when nothing new is
    derived (or after a small iteration cap, since the rule set is finite and
    this always converges quickly in practice)."""
    capability_sources: Dict[str, Set[str]] = {}  # capability -> set of finding_ids that granted it

    for f in findings:
        for cap in _CAPABILITY_GRANTS.get(f.category, set()):
            capability_sources.setdefault(cap, set()).add(f.finding_id)
        # Special case: "cookie_not_httponly" is a specific SIGNAL within the
        # generic security_headers category, not implied by the category alone.
        if f.category == "security_headers" and "httponly" in f.description.lower():
            capability_sources.setdefault("cookie_not_httponly", set()).add(f.finding_id)

    derived_paths: List[AttackPath] = []
    seen_names: Set[str] = set()
    present_caps = set(capability_sources.keys())

    for _ in range(3):  # small fixpoint cap; the rule set here can't chain deeper than this
        newly_derived = False
        for preconditions, name, severity, rationale in _COMPOSITION_RULES:
            if name in seen_names:
                continue
            if preconditions.issubset(present_caps):
                contributing = sorted({
                    fid for cap in preconditions for fid in capability_sources.get(cap, set())
                })
                derived_paths.append(AttackPath(
                    name=name, severity=severity, rationale=rationale,
                    contributing_findings=contributing, hop_count=len(preconditions),
                ))
                seen_names.add(name)
                present_caps.add(name)  # composed capabilities can themselves feed further rules
                capability_sources.setdefault(name, set()).update(contributing)
                newly_derived = True
        if not newly_derived:
            break

    derived_paths.sort(key=lambda p: -p.severity.value)
    return derived_paths


# -----------------------------------------------------------------------------
# ERSEC Risk Quotient (ERQ) - a bespoke, documented scoring model, not a CVSS
# wrapper. CVSS scores a vulnerability's intrinsic properties; ERQ instead
# asks "how much does THIS finding, in THIS scan, actually raise attacker
# opportunity" by combining four independently-reasoned factors:
#
#   Impact          - severity's base weight (0-10 scale, ERSEC's own bucket
#                      boundaries, not CVSS's)
#   Confidence       - how sure the detector is (Confirmed/Likely/Possible)
#   Discoverability  - how much attacker effort finding this took. A missing
#                      security header sits in the FIRST response an attacker
#                      ever gets; a CSRF gap needs the attacker to find and
#                      parse a specific form first. Easier-to-find findings
#                      get a HIGHER multiplier, because ease-of-discovery is
#                      itself part of real-world risk, not a side note.
#   Exposure         - a scan-wide multiplier: findings on an app with no
#                      authentication configured (fully anonymous surface)
#                      are more broadly exposed than the same findings on an
#                      app being scanned behind a valid session cookie.
#
# Compound risk chains (see correlate_findings) add an extra penalty on top,
# since a chain represents attacker capability greater than the sum of its
# parts - the same reasoning the chain rules themselves encode, reflected
# back into the score.
#
# This produces a 0-100 score and an SSL-Labs-style letter grade. It is
# deliberately punitive: a single Critical, trivially-discoverable, confirmed
# finding on an anonymous surface can swing the grade to F on its own,
# because that combination is close to worst-case for a defender.
# -----------------------------------------------------------------------------

_ERQ_IMPACT = {"CRITICAL": 10.0, "HIGH": 7.0, "MEDIUM": 4.0, "LOW": 2.0, "INFO": 0.0}
_ERQ_CONFIDENCE = {"Confirmed": 1.0, "Likely": 0.75, "Possible": 0.5}

# Tier 1: visible in the very first response(s) a scanner/attacker gets, zero
# crawling or interaction required beyond a single GET.
_ERQ_TIER1 = {"security_headers", "tls", "server_banner", "http_methods", "cors",
              "exposed_files", "directory_listing", "mixed_content", "sri",
              "vulnerable_library", "info_disclosure_comments", "cache_control",
              "jwt", "graphql_introspection", "host_header", "admin_exposure",
              "weak_session_token", "email_security", "websocket", "security_txt",
              "subdomain_takeover"}
# Tier 3: needs the attacker to first find a specific form/param and try
# several payload variants before the issue surfaces.
_ERQ_TIER3 = {"csrf", "ssti", "ldap_injection", "nosql_injection", "prototype_pollution", "ssrf",
              "xxe", "xpath_injection", "command_injection", "idor_heuristic", "rate_limiting",
              "mass_assignment", "user_enumeration"}
_ERQ_DISCOVERABILITY = {"tier1": 1.15, "tier2": 1.0, "tier3": 0.85}


def _erq_discoverability(category: str) -> float:
    if category in _ERQ_TIER1:
        return _ERQ_DISCOVERABILITY["tier1"]
    if category in _ERQ_TIER3:
        return _ERQ_DISCOVERABILITY["tier3"]
    return _ERQ_DISCOVERABILITY["tier2"]


def compute_posture_score(findings: List[Finding], chains: List["RiskChain"],
                           authenticated_scan: bool = False,
                           proximity_by_url: Optional[Dict[str, float]] = None) -> Dict[str, Any]:
    exposure_multiplier = 1.0 if authenticated_scan else 1.2
    proximity_by_url = proximity_by_url or {}

    per_finding_erq = []
    for f in findings:
        impact = _ERQ_IMPACT.get(f.severity.name, 0.0)
        confidence = _ERQ_CONFIDENCE.get(f.confidence, 0.5)
        discoverability = _erq_discoverability(f.category)
        proximity = proximity_by_url.get(f.url.split("?")[0], 1.0)
        erq = impact * confidence * discoverability * proximity
        per_finding_erq.append(erq)

    base_deduction = sum(per_finding_erq) * exposure_multiplier
    chain_deduction = sum(_ERQ_IMPACT.get(c.combined_severity.name, 0.0) * 0.5 for c in chains)
    total_deduction = base_deduction + chain_deduction

    score = max(0.0, min(100.0, 100.0 - total_deduction))

    if score >= 97:
        grade = "A+"
    elif score >= 93:
        grade = "A"
    elif score >= 90:
        grade = "A-"
    elif score >= 80:
        grade = "B"
    elif score >= 70:
        grade = "C"
    elif score >= 60:
        grade = "D"
    else:
        grade = "F"

    return {
        "score": round(score, 1),
        "grade": grade,
        "model": "ERSEC Risk Quotient (ERQ) v1 - not CVSS",
        "exposure_multiplier": exposure_multiplier,
        "chain_deduction": round(chain_deduction, 1),
    }


# -----------------------------------------------------------------------------
# Blast-radius-aware scoring: crown-jewel proximity. Instead of a static
# severity table treating every XSS as equally risky wherever it's found,
# this computes each finding's graph distance (via the crawl's actual link
# structure - not just URL string matching) to user-declared "crown jewel"
# endpoints (admin panels, payment flows, PII pages), and feeds that into
# the ERQ score as a proximity multiplier: a medium-severity finding right
# next to a crown jewel can outrank a high-severity finding in a dead
# corner of the site that no path leads to.
# -----------------------------------------------------------------------------

def compute_crown_jewel_distances(link_graph: Dict[str, Set[str]], crown_jewels: List[str],
                                    all_urls: List[str]) -> Dict[str, int]:
    """BFS distance (in link hops) from each crown-jewel URL outward through the
    crawled link graph, taking the minimum over all declared crown jewels.
    Falls back to a path-prefix heuristic for URLs the graph doesn't connect to
    (e.g. found via robots.txt/JS-endpoint discovery rather than an <a> link)."""
    if not crown_jewels:
        return {}

    jewel_urls = set()
    for pattern in crown_jewels:
        for url in all_urls:
            if pattern in url:
                jewel_urls.add(url)

    distances: Dict[str, int] = {}
    if jewel_urls:
        reverse_graph: Dict[str, Set[str]] = {}
        for src, targets in link_graph.items():
            for t in targets:
                reverse_graph.setdefault(t, set()).add(src)
        frontier = {u: 0 for u in jewel_urls}
        queue = list(jewel_urls)
        visited_bfs = set(jewel_urls)
        while queue:
            current = queue.pop(0)
            d = frontier[current]
            neighbors = link_graph.get(current, set()) | reverse_graph.get(current, set())
            for n in neighbors:
                if n not in visited_bfs:
                    visited_bfs.add(n)
                    frontier[n] = d + 1
                    queue.append(n)
        distances = frontier

    for url in all_urls:
        if url not in distances:
            base_path = urllib.parse.urlparse(url).path
            for pattern in crown_jewels:
                first_seg = pattern.strip("/").split("/")[0]
                if first_seg and first_seg in base_path:
                    distances[url] = 2  # "nearby but not graph-confirmed"
                    break
    return distances


def proximity_multiplier(distance: Optional[int]) -> float:
    if distance is None:
        return 1.0
    if distance == 0:
        return 1.5
    if distance == 1:
        return 1.3
    if distance == 2:
        return 1.15
    return 1.05


# -----------------------------------------------------------------------------
# Asset inventory - a recon summary of what the scan actually touched, the
# kind of "sites tree" overview professional tools (Burp, ZAP) show alongside
# findings, not just a flat vulnerability list.
# -----------------------------------------------------------------------------

def build_asset_inventory(crawler: "WebCrawler", client: "SafeHttpClient") -> Dict[str, Any]:
    cookie_names = sorted({c.name for c in client.session.cookies})
    endpoints_sorted = sorted(crawler.endpoints)
    soft_404_groups = [urls for urls in crawler.response_hashes.values() if len(urls) >= 3]
    return {
        "pages_discovered": len(crawler.visited),
        "endpoints_with_parameters": sum(1 for e in crawler.endpoints if "?" in e),
        "forms_discovered": len(crawler.forms),
        "cookies_observed": cookie_names,
        "third_party_origins": sorted(crawler.third_party_origins),
        "sample_endpoints": endpoints_sorted[:25],
        "excluded_out_of_scope_count": len(crawler.skipped_out_of_scope),
        "api_like_endpoints_discovered": len(crawler.api_like_endpoints),
        "likely_soft_404_clusters": len(soft_404_groups),
        "numeric_id_url_patterns": len(crawler.numeric_id_endpoints),
    }


# -----------------------------------------------------------------------------
# Baseline diffing - regression tracking across scans, the way CI security
# gates work: compare this run's findings against a saved snapshot and report
# what's new, what's fixed, and what's still open.
# -----------------------------------------------------------------------------

def diff_against_baseline(current_findings: List[Dict[str, Any]], baseline_path: str) -> Optional[Dict[str, Any]]:
    if not os.path.exists(baseline_path):
        return None
    try:
        with open(baseline_path) as f:
            baseline = json.load(f)
    except (json.JSONDecodeError, OSError):
        return None
    old_keys = {f["diff_key"]: f for f in baseline.get("findings", [])}
    new_keys = {f["diff_key"]: f for f in current_findings}

    new_findings = [new_keys[k] for k in new_keys if k not in old_keys]
    resolved_findings = [old_keys[k] for k in old_keys if k not in new_keys]
    persistent_findings = [new_keys[k] for k in new_keys if k in old_keys]

    return {
        "baseline_scanned_at": baseline.get("scanned_at", "unknown"),
        "new_count": len(new_findings),
        "resolved_count": len(resolved_findings),
        "persistent_count": len(persistent_findings),
        "new_findings": new_findings,
        "resolved_findings": resolved_findings,
    }


def save_baseline(results: Dict[str, Any], baseline_path: str) -> None:
    with open(baseline_path, "w") as f:
        json.dump({"scanned_at": results["scanned_at"], "findings": results["findings"]}, f, indent=2, default=str)


# -----------------------------------------------------------------------------
# Regression fingerprinting across scans - persistent local history (SQLite,
# stdlib only, no new dependency). Where --baseline answers "what changed
# since last time", this answers "what keeps coming back" - the same root
# cause reappearing scan after scan, which a single-scan diff can't show
# because a finding that was fixed and then reintroduced shows as "new"
# again in every individual diff, hiding the pattern.
# -----------------------------------------------------------------------------

def _history_db_connect(db_path: str):
    import sqlite3
    conn = sqlite3.connect(db_path)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS scan_findings (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            target TEXT NOT NULL,
            scanned_at TEXT NOT NULL,
            diff_key TEXT NOT NULL,
            category TEXT NOT NULL,
            severity TEXT NOT NULL,
            url TEXT NOT NULL
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_target_key ON scan_findings(target, diff_key)")
    conn.commit()
    return conn


def record_scan_history(results: Dict[str, Any], db_path: str) -> None:
    conn = _history_db_connect(db_path)
    try:
        rows = [
            (results["target"], results["scanned_at"], f["diff_key"], f["category"], f["severity"], f["url"])
            for f in results["findings"]
        ]
        conn.executemany(
            "INSERT INTO scan_findings (target, scanned_at, diff_key, category, severity, url) VALUES (?,?,?,?,?,?)",
            rows,
        )
        conn.commit()
    finally:
        conn.close()


def build_recurrence_report(target: str, db_path: str, min_occurrences: int = 2) -> List[Dict[str, Any]]:
    """Findings (by diff_key) that have shown up in 2+ distinct historical scans
    for this target, most-recurring first. Requires the user to have run scans
    with the same --history-db path over time - this is only as good as the
    history it's been given."""
    if not os.path.exists(db_path):
        return []
    conn = _history_db_connect(db_path)
    try:
        cur = conn.execute("""
            SELECT diff_key, category, MIN(scanned_at) as first_seen, MAX(scanned_at) as last_seen,
                   COUNT(DISTINCT scanned_at) as occurrences, MAX(severity) as severity, MAX(url) as sample_url
            FROM scan_findings
            WHERE target = ?
            GROUP BY diff_key
            HAVING occurrences >= ?
            ORDER BY occurrences DESC, last_seen DESC
        """, (target, min_occurrences))
        cols = [c[0] for c in cur.description]
        return [dict(zip(cols, row)) for row in cur.fetchall()]
    finally:
        conn.close()


# -----------------------------------------------------------------------------
# 4.1 Proof-carrying exposure-change intelligence
#
# Upgrades the flat --baseline diff into a real historical identity system:
# every finding gets a canonical ID stable across scans (normalized host +
# path + category + parameter + check version, not just a per-run counter),
# tracked with first_seen/last_seen/status/evidence_hash so the report can
# say what's genuinely NEW, what RESOLVED, what's simply PERSISTENT, and -
# a case a one-shot diff can't distinguish - what was fixed and then
# REINTRODUCED. The narrative is generated by a deterministic template, not
# a model: per the doc this builds from, the advisor stays authoritative
# over what's true; language generation doesn't get to invent the facts.
# -----------------------------------------------------------------------------

_CHECK_VERSION = "2"  # bumped from "1": several detection modules changed meaningfully in this
                       # revision (new XSS URI-context probe, SQLi control-probe confirmation,
                       # expanded exposed-files list, etc.) - old and new fingerprints for the
                       # "same" check are deliberately distinguished rather than silently merged


def canonical_finding_id(category: str, url: str, parameter: Optional[str], description: str = "",
                          check_version: str = _CHECK_VERSION) -> str:
    """A stable identity for a finding across scans. Includes a short hash of
    the description text, not just category+path+parameter - several checks
    (SecurityHeadersModule, CacheControlModule, and any future multi-issue
    check) legitimately report MULTIPLE distinct findings for the same
    category on the same page with no parameter (e.g. "missing HSTS" and
    "missing CSP" are both category=security_headers, parameter=None). The
    description text is what actually distinguishes them, so it has to be
    part of the identity - omitting it caused two different findings to
    collide onto the identical ID (a real crash, not hypothetical: see
    test_multiple_distinct_findings_same_category_path_param_do_not_collide).

    This does mean a finding whose description contains changing dynamic
    detail (e.g. a version string) gets treated as resolved+new across scans
    rather than "changed" - an accepted, documented trade-off in favor of
    never colliding, over perfect changed-field granularity for that edge case."""
    parsed = urllib.parse.urlparse(url)
    normalized_host = (parsed.hostname or "").lower()
    normalized_path = parsed.path.rstrip("/") or "/"
    description_signature = hashlib.sha256(description.encode("utf-8")).hexdigest()[:10]
    raw = f"{normalized_host}|{normalized_path}|{category}|{parameter or ''}|{description_signature}|{check_version}"
    return "EXP-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]


def _evidence_fingerprint(finding_dict: Dict[str, Any]) -> Dict[str, str]:
    """The specific fields tracked for change detection - deliberately a small,
    named set (not 'the whole evidence blob changed') so changed_fields can
    say exactly what moved, not just that something did."""
    ev = finding_dict.get("evidence", {})
    return {
        "severity": finding_dict.get("severity", ""),
        "confidence": finding_dict.get("confidence", ""),
        "url": finding_dict.get("url", ""),
        "status_code": str(ev.get("status_code", "")),
        "description_hash": hashlib.sha256(finding_dict.get("description", "").encode("utf-8")).hexdigest()[:12],
    }


def _exposure_db_connect(db_path: str):
    import sqlite3
    conn = sqlite3.connect(db_path)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS exposure_history (
            canonical_id TEXT NOT NULL,
            target TEXT NOT NULL,
            category TEXT NOT NULL,
            url TEXT NOT NULL,
            parameter TEXT,
            check_version TEXT NOT NULL,
            status TEXT NOT NULL,
            first_seen TEXT NOT NULL,
            last_seen TEXT NOT NULL,
            resolved_at TEXT,
            fingerprint_json TEXT NOT NULL,
            verification_of TEXT,
            PRIMARY KEY (canonical_id, target)
        )
    """)
    conn.commit()
    return conn


def record_and_diff_exposure_history(results: Dict[str, Any], db_path: str) -> Dict[str, Any]:
    """The core of 4.1: reconcile this scan's findings against exposure_history
    for this target, updating status per canonical ID, and return a structured
    diff (new/resolved/persistent/reintroduced/changed) plus a narrative."""
    target = results["target"]
    now = results["scanned_at"]
    conn = _exposure_db_connect(db_path)
    changes = {"new": [], "resolved": [], "persistent": [], "reintroduced": [], "changed": []}

    try:
        cur = conn.execute(
            "SELECT canonical_id, status, fingerprint_json FROM exposure_history WHERE target = ?", (target,)
        )
        existing = {row[0]: {"status": row[1], "fingerprint": json.loads(row[2])} for row in cur.fetchall()}
        current_ids = set()

        for f in results["findings"]:
            cid = canonical_finding_id(f["category"], f["url"], f.get("parameter"), f.get("description", ""))
            f["canonical_id"] = cid  # attach for downstream renderers (SARIF, dashboard)
            current_ids.add(cid)
            fingerprint = _evidence_fingerprint(f)
            fingerprint_json = json.dumps(fingerprint, sort_keys=True)
            prior = existing.get(cid)

            if prior is None:
                status = "new"
                # Upsert, not a bare INSERT: even with the canonical-ID fix
                # above, this is the last line of defense against a crash if
                # any future check ever produces a genuine within-scan
                # collision - a report with slightly imperfect history beats
                # a scan that dies partway through and produces nothing.
                conn.execute(
                    "INSERT INTO exposure_history (canonical_id, target, category, url, parameter, check_version, "
                    "status, first_seen, last_seen, resolved_at, fingerprint_json, verification_of) "
                    "VALUES (?,?,?,?,?,?,?,?,?,NULL,?,NULL) "
                    "ON CONFLICT(canonical_id, target) DO UPDATE SET "
                    "last_seen=excluded.last_seen, status=excluded.status, "
                    "fingerprint_json=excluded.fingerprint_json, resolved_at=NULL",
                    (cid, target, f["category"], f["url"], f.get("parameter"), _CHECK_VERSION,
                     status, now, now, fingerprint_json),
                )
                f["exposure_status"] = status
                changes["new"].append({"canonical_id": cid, "category": f["category"], "url": f["url"]})
            else:
                was_resolved = prior["status"] == "resolved"
                status = "reintroduced" if was_resolved else "persistent"
                changed_fields = [k for k in fingerprint if fingerprint.get(k) != prior["fingerprint"].get(k)]
                conn.execute(
                    "UPDATE exposure_history SET status=?, last_seen=?, resolved_at=NULL, fingerprint_json=? "
                    "WHERE canonical_id=? AND target=?",
                    (status, now, fingerprint_json, cid, target),
                )
                f["exposure_status"] = status
                bucket = changes["reintroduced"] if was_resolved else changes["persistent"]
                bucket.append({"canonical_id": cid, "category": f["category"], "url": f["url"]})
                if changed_fields and not was_resolved:
                    changes["changed"].append({
                        "canonical_id": cid, "category": f["category"], "url": f["url"],
                        "changed_fields": changed_fields,
                    })

        # Anything previously open (new/persistent/reintroduced) but absent this scan -> resolved.
        for cid, info in existing.items():
            if cid not in current_ids and info["status"] != "resolved":
                conn.execute(
                    "UPDATE exposure_history SET status='resolved', resolved_at=? WHERE canonical_id=? AND target=?",
                    (now, cid, target),
                )
                changes["resolved"].append({"canonical_id": cid})

        conn.commit()
    finally:
        conn.close()

    changes["narrative"] = generate_exposure_narrative(changes, target)
    return changes


def generate_exposure_narrative(changes: Dict[str, Any], target: str) -> str:
    """Deterministic template, not a model call - this describes exactly what
    the diff computed above, nothing inferred or embellished."""
    parts = []
    n_new, n_resolved, n_persistent, n_reintro, n_changed = (
        len(changes["new"]), len(changes["resolved"]), len(changes["persistent"]),
        len(changes["reintroduced"]), len(changes["changed"]),
    )
    if not any([n_new, n_resolved, n_persistent, n_reintro]):
        return f"No prior exposure history found for {target} - this scan establishes the baseline."

    if n_new:
        cats = sorted({n["category"] for n in changes["new"]})
        parts.append(f"{n_new} new exposure(s) appeared since the last scan (categories: {', '.join(cats)}).")
    if n_reintro:
        cats = sorted({n["category"] for n in changes["reintroduced"]})
        parts.append(f"{n_reintro} previously-resolved exposure(s) REAPPEARED (categories: {', '.join(cats)}) - "
                      "worth checking whether a recent deploy reverted the fix.")
    if n_resolved:
        parts.append(f"{n_resolved} exposure(s) present in the last scan are no longer detected (resolved, or moved out of scope - verify with --reverify).")
    if n_changed:
        parts.append(f"{n_changed} persistent exposure(s) changed shape (severity, confidence, or URL shifted) without disappearing.")
    if n_persistent and not n_changed:
        parts.append(f"{n_persistent} exposure(s) remain unchanged from the last scan.")
    return " ".join(parts)


# -----------------------------------------------------------------------------
# Calibrated trust score - confidence that improves from real usage. Every
# check starts with a static prior (Confirmed/Likely/Possible -> a baseline
# percentage), but as a user marks findings true/false-positive over time,
# the displayed confidence for that (category, stack) pair shifts to reflect
# actual observed precision in their environment - a "CSRF missing" finding
# might be far more reliable on a Django target than a legacy PHP one, and
# this lets that difference show up instead of pretending every check is
# equally trustworthy forever.
# -----------------------------------------------------------------------------

_CONFIDENCE_PRIOR = {"Confirmed": 0.90, "Likely": 0.70, "Possible": 0.50}
_PRIOR_WEIGHT = 4  # how many "virtual" feedback observations the static prior counts as,
                   # before real feedback starts dominating the estimate


def _feedback_db_connect(db_path: str):
    import sqlite3
    conn = sqlite3.connect(db_path)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS finding_feedback (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            category TEXT NOT NULL,
            stack TEXT NOT NULL,
            verdict TEXT NOT NULL CHECK(verdict IN ('tp','fp')),
            diff_key TEXT,
            recorded_at TEXT NOT NULL
        )
    """)
    conn.commit()
    return conn


def record_feedback(db_path: str, category: str, stack: Optional[str], verdict: str, diff_key: Optional[str] = None) -> None:
    if verdict not in ("tp", "fp"):
        raise ValueError("verdict must be 'tp' or 'fp'")
    conn = _feedback_db_connect(db_path)
    try:
        conn.execute(
            "INSERT INTO finding_feedback (category, stack, verdict, diff_key, recorded_at) VALUES (?,?,?,?,?)",
            (category, stack or "unknown", verdict, diff_key, _utc_now_iso()),
        )
        conn.commit()
    finally:
        conn.close()


def calibrated_confidence(db_path: Optional[str], category: str, stack: Optional[str], static_confidence: str) -> Dict[str, Any]:
    """Laplace/Beta(prior)-smoothed precision estimate: starts at the static
    prior for this confidence label, shifts toward observed tp/(tp+fp) as real
    feedback accumulates, weighted so a handful of early data points don't
    wildly swing the number."""
    prior = _CONFIDENCE_PRIOR.get(static_confidence, 0.5)
    if not db_path or not os.path.exists(db_path):
        return {"pct": round(prior * 100, 1), "sample_size": 0, "basis": "static prior (no feedback recorded yet)"}
    conn = _feedback_db_connect(db_path)
    try:
        cur = conn.execute(
            "SELECT verdict, COUNT(*) FROM finding_feedback WHERE category=? AND stack=? GROUP BY verdict",
            (category, stack or "unknown"),
        )
        counts = {"tp": 0, "fp": 0}
        for verdict, n in cur.fetchall():
            counts[verdict] = n
        tp, fp = counts["tp"], counts["fp"]
        total = tp + fp
        if total == 0:
            return {"pct": round(prior * 100, 1), "sample_size": 0, "basis": "static prior (no feedback recorded yet)"}
        # Prior acts as _PRIOR_WEIGHT virtual observations at the prior rate.
        calibrated = (prior * _PRIOR_WEIGHT + tp) / (_PRIOR_WEIGHT + total)
        return {
            "pct": round(calibrated * 100, 1),
            "sample_size": total,
            "basis": f"calibrated from {total} user-marked finding(s) for this check on this stack",
        }
    finally:
        conn.close()


# -----------------------------------------------------------------------------
# Remediation-verification loop: re-run the EXACT probe that produced a
# specific finding (same method, URL, and any extra probe headers - not a
# fresh full scan) and classify the result. Closes the loop a plain "scan
# again and diff" doesn't: it confirms the precise finding is gone, not just
# that a new scan happens not to have re-triggered it.
#
# Each verification run is appended to a local hash-chained log: every
# record includes the SHA-256 of the previous record, so any edit to an
# earlier entry breaks the chain from that point forward and is detectable
# by verify_audit_log_integrity(). This is a TAMPER-EVIDENT LOCAL LOG, not a
# trusted third-party timestamp - it proves the log wasn't silently edited
# after the fact, not that the timestamp itself is independently attested
# (that would need an RFC 3161 timestamp authority over the network, which
# this offline tool doesn't reach out to). Say so plainly if this is ever
# used as compliance evidence: it's tamper-evidence, not notarization.
# -----------------------------------------------------------------------------


def _sha256_hex(data: str) -> str:
    return hashlib.sha256(data.encode("utf-8")).hexdigest()


_VERIFY_MARKER_RE = re.compile(r"<(ersec_[a-z0-9_]+)>")


def _predicate_sql_injection(resp_text: str, resp: requests.Response) -> bool:
    lower = (resp_text or "").lower()
    return any(p in lower for p in SQLInjectionSignalModule.ERROR_PATTERNS)


def _predicate_marker_reflection(resp_text: str, resp: requests.Response) -> bool:
    m = _VERIFY_MARKER_RE.search(resp_text or "")
    return bool(m) and m.group(0) in (resp_text or "")


def _predicate_exposed_files(resp_text: str, resp: requests.Response) -> bool:
    return resp.status_code == 200 and len(resp_text or "") > 0


def _predicate_directory_listing(resp_text: str, resp: requests.Response) -> bool:
    return resp.status_code == 200 and ("Index of /" in (resp_text or "") or "Directory listing for" in (resp_text or ""))


def _predicate_graphql_introspection(resp_text: str, resp: requests.Response) -> bool:
    return resp.status_code == 200 and '"__schema"' in (resp_text or "")


def _predicate_open_redirect(resp_text: str, resp: requests.Response) -> bool:
    location = resp.headers.get("Location", "")
    return resp.status_code in (301, 302, 303, 307, 308) and "ersec-probe.invalid" in location


def _predicate_cors(resp_text: str, resp: requests.Response) -> bool:
    acao = resp.headers.get("Access-Control-Allow-Origin", "")
    return "ersec-probe.invalid" in acao or acao == "*"


def _predicate_admin_exposure(resp_text: str, resp: requests.Response) -> bool:
    if resp.status_code != 200:
        return False
    looks_gated = bool(AdminPanelExposureModule.LOGIN_SIGNALS.search((resp_text or "")[:3000]))
    looks_live = bool(AdminPanelExposureModule.ADMIN_CONTENT_SIGNALS.search(resp_text or ""))
    return looks_live and not looks_gated


def _predicate_xxe(resp_text: str, resp: requests.Response) -> bool:
    return "ersec_xxe_probe_4d1a" in (resp_text or "") and resp.status_code < 500


# Categories with a real, targeted single-request predicate. Everything else
# falls back to a generic "did the response meaningfully change" heuristic -
# labeled as such, not presented with false certainty.
_VERIFY_PREDICATES: Dict[str, Any] = {
    "sql_injection": _predicate_sql_injection,
    "xss": _predicate_marker_reflection,
    "header_injection": _predicate_marker_reflection,
    "exposed_files": _predicate_exposed_files,
    "directory_listing": _predicate_directory_listing,
    "graphql_introspection": _predicate_graphql_introspection,
    "open_redirect": _predicate_open_redirect,
    "cors": _predicate_cors,
    "admin_exposure": _predicate_admin_exposure,
    "xxe": _predicate_xxe,
}

# Categories that are inherently not single-request-replayable (multi-step,
# form-based, or non-HTTP checks) - re-verifying these honestly requires a
# fresh scan, not a probe replay.
_VERIFY_UNSUPPORTED = {"csrf", "tls", "http_methods", "weak_session_token", "email_security",
                       "idor_heuristic", "rate_limiting", "mass_assignment", "user_enumeration"}


def reverify_finding(client: "SafeHttpClient", finding_dict: Dict[str, Any]) -> Dict[str, Any]:
    replay = finding_dict.get("replay", {})
    category = replay.get("category", finding_dict.get("category", ""))
    method = replay.get("method", "GET")
    url = replay.get("url", finding_dict.get("url", ""))
    extra_headers = {k: v for k, v in replay.get("extra_headers", {}).items() if v != "REDACTED"}

    record = {
        "diff_key": finding_dict.get("diff_key"),
        "category": category,
        "url": url,
        "verified_at": _utc_now_iso(),
        "before_status_code": finding_dict.get("evidence", {}).get("status_code"),
        "before_excerpt_hash": _sha256_hex(finding_dict.get("evidence", {}).get("response_excerpt", "")),
    }

    if category in _VERIFY_UNSUPPORTED:
        record.update({
            "verdict": "inconclusive",
            "note": f"'{category}' findings are not single-request-replayable (multi-step or non-HTTP check) - "
                    "run a fresh full scan to confirm this one.",
        })
        return record

    try:
        resp = client.request(method, url, headers=extra_headers or None)
        resp_text = resp.text or ""
    except (ScopeError, ERSECError) as e:
        record.update({"verdict": "inconclusive", "note": f"Replay request failed: {e}"})
        return record

    record["after_status_code"] = resp.status_code
    record["after_excerpt_hash"] = _sha256_hex(resp_text[:800])

    predicate = _VERIFY_PREDICATES.get(category)
    if predicate:
        still_present = predicate(resp_text, resp)
        record["verdict"] = "still_present" if still_present else "resolved"
    else:
        # Generic fallback: compare status code and a rough content fingerprint.
        same_status = record["before_status_code"] == record["after_status_code"]
        same_hash = record["before_excerpt_hash"] == record["after_excerpt_hash"]
        if same_status and same_hash:
            record["verdict"] = "likely_still_present"
            record["note"] = f"No targeted predicate for '{category}' - response is byte-identical to the " \
                              "original evidence excerpt, suggesting the finding is unchanged. Manually confirm."
        else:
            record["verdict"] = "changed_uncertain"
            record["note"] = f"No targeted predicate for '{category}' - response differs from the original " \
                              "evidence, which is consistent with either a fix or an unrelated site change. Manually confirm."
    return record


def reverify_report(report_path: str, config: ScanConfig) -> List[Dict[str, Any]]:
    with open(report_path) as f:
        report = json.load(f)
    client = SafeHttpClient(config)
    return [reverify_finding(client, f) for f in report.get("findings", [])]


def append_to_audit_log(records: List[Dict[str, Any]], audit_log_path: str) -> None:
    chain = []
    prev_hash = "0" * 64
    if os.path.exists(audit_log_path):
        with open(audit_log_path) as f:
            try:
                chain = json.load(f)
            except json.JSONDecodeError:
                chain = []
        if chain:
            prev_hash = chain[-1]["record_hash"]
    for r in records:
        r = dict(r)
        r["prev_hash"] = prev_hash
        r["record_hash"] = _sha256_hex(json.dumps(r, sort_keys=True, default=str))
        chain.append(r)
        prev_hash = r["record_hash"]
    with open(audit_log_path, "w") as f:
        json.dump(chain, f, indent=2, default=str)


def verify_audit_log_integrity(audit_log_path: str) -> Dict[str, Any]:
    if not os.path.exists(audit_log_path):
        return {"valid": False, "records": 0, "broken_at": None, "error": "log file does not exist"}
    with open(audit_log_path) as f:
        chain = json.load(f)
    prev_hash = "0" * 64
    for i, r in enumerate(chain):
        expected_prev = r.get("prev_hash")
        if expected_prev != prev_hash:
            return {"valid": False, "records": len(chain), "broken_at": i,
                     "error": f"record {i} references prev_hash {expected_prev} but chain computed {prev_hash}"}
        r_check = {k: v for k, v in r.items() if k != "record_hash"}
        recomputed = _sha256_hex(json.dumps(r_check, sort_keys=True, default=str))
        if recomputed != r.get("record_hash"):
            return {"valid": False, "records": len(chain), "broken_at": i,
                     "error": f"record {i} hash does not match its own content - tampered"}
        prev_hash = r["record_hash"]
    return {"valid": True, "records": len(chain), "broken_at": None, "error": None}


# -----------------------------------------------------------------------------
# Plugin loading - lets a user drop a custom detection module into a
# directory and have it picked up automatically, without editing this file.
# A plugin is any .py file exposing one or more BaseModule subclasses at
# module level. Loaded via importlib from an explicit --plugin-dir path
# only (never from a network location, never implicitly from cwd) so this
# stays an opt-in extension point, not an auto-execution surface.
# -----------------------------------------------------------------------------

def load_plugin_modules(plugin_dir: str, config: "ScanConfig", client: "SafeHttpClient") -> List["BaseModule"]:
    import importlib.util
    instances: List[BaseModule] = []
    if not plugin_dir or not os.path.isdir(plugin_dir):
        return instances
    for fname in sorted(os.listdir(plugin_dir)):
        if not fname.endswith(".py") or fname.startswith("_"):
            continue
        fpath = os.path.join(plugin_dir, fname)
        modname = f"ersec_plugin_{fname[:-3]}"
        try:
            spec = importlib.util.spec_from_file_location(modname, fpath)
            if spec is None or spec.loader is None:
                continue
            mod = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(mod)
        except Exception as e:
            print(f"  [!] Failed to load plugin {fname}: {e}")
            continue
        for attr_name in dir(mod):
            attr = getattr(mod, attr_name)
            if (isinstance(attr, type) and issubclass(attr, BaseModule) and attr is not BaseModule
                    and attr.__module__ == modname):
                try:
                    instances.append(attr(config, client))
                    print(f"  [+] Loaded plugin module: {attr_name} (category={getattr(attr, 'category', '?')}) from {fname}")
                except Exception as e:
                    print(f"  [!] Failed to instantiate plugin module {attr_name}: {e}")
    return instances


# -----------------------------------------------------------------------------
# Scan configuration file support (JSON). Lets a recurring authorized
# assessment (e.g. a weekly CI job against the same staging target) keep its
# full configuration - scope, crown jewels, headers, cookies - in one
# version-controlled file instead of a long CLI invocation. CLI flags always
# override config-file values when both are given, so ad-hoc runs can still
# tweak one thing without editing the file.
# -----------------------------------------------------------------------------

def load_scan_config_file(path: str) -> Dict[str, Any]:
    with open(path) as f:
        data = json.load(f)
    if not isinstance(data, dict):
        raise ERSECError(f"Config file {path} must contain a JSON object at the top level")
    return data


def apply_config_file(config: "ScanConfig", data: Dict[str, Any]) -> None:
    """Mutates config in place with values from a loaded config file. Only
    known, safe fields are applied - unknown keys are reported but ignored,
    rather than silently accepted (typo protection) or causing a hard crash
    (forward-compatibility with newer config files on an older binary)."""
    known_top = {"profile", "cookies", "bearer_token", "extra_headers", "verify_tls",
                 "user_agent", "ai_enabled", "crown_jewels", "scope"}
    for key in data:
        if key not in known_top:
            print(f"  [!] Config file: ignoring unrecognized top-level key '{key}'")
    if "profile" in data:
        try:
            config.profile = ScanProfile(data["profile"])
        except ValueError:
            raise ERSECError(f"Config file: invalid profile '{data['profile']}'")
    if "cookies" in data and isinstance(data["cookies"], dict):
        config.cookies.update(data["cookies"])
    if "bearer_token" in data:
        config.bearer_token = str(data["bearer_token"])
    if "extra_headers" in data and isinstance(data["extra_headers"], dict):
        config.extra_headers.update(data["extra_headers"])
    if "verify_tls" in data:
        config.verify_tls = bool(data["verify_tls"])
    if "user_agent" in data:
        config.user_agent = str(data["user_agent"])
    if "ai_enabled" in data:
        config.ai_enabled = bool(data["ai_enabled"])
    if "crown_jewels" in data and isinstance(data["crown_jewels"], list):
        config.crown_jewels = list(data["crown_jewels"])
    if "scope" in data and isinstance(data["scope"], dict):
        scope_known = {"allowed_hosts", "allowed_ports", "allowed_paths", "allowed_methods",
                       "max_requests", "rate_limit_seconds", "concurrency", "timeout_seconds",
                       "max_crawl_depth", "max_crawl_pages", "stop_on_scope_violation"}
        for key in data["scope"]:
            if key not in scope_known:
                print(f"  [!] Config file: ignoring unrecognized scope key '{key}'")
        for key, value in data["scope"].items():
            if key in scope_known:
                setattr(config.scope, key, value)



# =============================================================================
# ERSEC 6.0 - Behavioral Twin / Journey Intelligence / Invariant Reasoning
# =============================================================================

@dataclass
class JourneyStep:
    index: int
    source: str
    destination: str
    method: str = "GET"
    parameter_names: List[str] = field(default_factory=list)
    state_hints: List[str] = field(default_factory=list)

@dataclass
class BehavioralInvariant:
    invariant_id: str
    name: str
    category: str
    severity: str
    confidence: str
    statement: str
    observed_evidence: List[str] = field(default_factory=list)
    validation: str = ""
    violated: bool = False

class BehaviorMemory:
    """Small local memory store for stable application behavior.

    This deliberately stores hashes/metadata rather than credentials or full response bodies.
    It lets repeated scans ask a better question than "did we see a finding?":
    "did this application's security-relevant behavior drift?".
    """
    def __init__(self, path: Optional[str]):
        self.path = path
        self.data = {"version": 1, "observations": {}, "invariants": {}}
        if path:
            try:
                with open(path, "r", encoding="utf-8") as fh:
                    loaded = json.load(fh)
                    if isinstance(loaded, dict):
                        self.data.update(loaded)
            except (OSError, json.JSONDecodeError):
                pass

    def save(self):
        if not self.path:
            return
        tmp = self.path + ".tmp"
        try:
            with open(tmp, "w", encoding="utf-8") as fh:
                json.dump(self.data, fh, indent=2, sort_keys=True)
            os.replace(tmp, self.path)
        except OSError:
            try:
                os.unlink(tmp)
            except OSError:
                pass

    def remember(self, key: str, value: Dict[str, Any]):
        self.data.setdefault("observations", {})[key] = value

    def get(self, key: str):
        return self.data.get("observations", {}).get(key)

class JourneyCompiler:
    """Compiles crawl/form evidence into short, explainable user journeys.

    The novelty is that the scanner does not treat pages as isolated URLs. It builds
    small behavioral programs from edges the application itself exposed, then reasons
    over transitions and invariants. No credential guessing or exploit sequencing is
    performed.
    """
    ACTION_WORDS = re.compile(r"(?:login|signin|register|signup|logout|checkout|cart|order|pay|upload|download|admin|account|profile|settings|password|token|oauth)", re.I)

    def __init__(self, crawler: WebCrawler):
        self.crawler = crawler

    def compile(self, endpoints: List[str], forms: List[FormInfo]) -> List[JourneyStep]:
        steps: List[JourneyStep] = []
        idx = 0
        edges = self.crawler.link_graph
        for src, dsts in edges.items():
            for dst in sorted(dsts):
                if not is_in_scope(dst, self.crawler.config):
                    continue
                path = urllib.parse.urlparse(dst).path
                hints = [m.group(0).lower() for m in self.ACTION_WORDS.finditer(path)]
                steps.append(JourneyStep(idx, src, dst, "GET", [], sorted(set(hints))))
                idx += 1
                if idx >= 300:
                    return steps
        for form in forms[:150]:
            target = form.action or form.page_url
            names = [i.get("name") for i in form.inputs if i.get("name")]
            hints = [n.lower() for n in names if self.ACTION_WORDS.search(n or "")]
            steps.append(JourneyStep(idx, form.page_url, target, form.method, names, sorted(set(hints))))
            idx += 1
        return steps

class InvariantLearner:
    """Learns security invariants from the observable surface.

    Instead of only learning signatures of known vulnerabilities, it learns properties
    the application appears to promise: authentication gates, stable resource classes,
    monotonic cart totals, consistent response types, and isolation between unrelated
    routes. Violations are evidence candidates, not automatic exploit claims.
    """
    def __init__(self, config: ScanConfig, client: SafeHttpClient, memory: BehaviorMemory):
        self.config = config
        self.client = client
        self.memory = memory

    @staticmethod
    def _sig(resp: requests.Response) -> Dict[str, Any]:
        body = resp.text or ""
        ctype = resp.headers.get("Content-Type", "").split(";", 1)[0].lower()
        redacted = re.sub(r"(?i)(token|secret|password|api[_-]?key)\s*[:=]\s*[^\s,;&]+", r"\1=<redacted>", body[:4000])
        return {
            "status": resp.status_code,
            "ctype": ctype,
            "body_len": len(body),
            "body_hash": hashlib.sha256(redacted.encode("utf-8", "ignore")).hexdigest()[:20],
            "location": resp.headers.get("Location", "")[:300],
        }

    def learn(self, endpoints: List[str], forms: List[FormInfo]) -> List[BehavioralInvariant]:
        inv: List[BehavioralInvariant] = []
        authish = [u for u in endpoints if re.search(r"/(?:admin|account|profile|settings|orders?|checkout|billing|me)(?:/|$)", urllib.parse.urlparse(u).path, re.I)]
        publicish = [u for u in endpoints if u not in authish]
        # Authentication-boundary invariant: protected-looking routes should not all look identical to public pages.
        if authish and publicish:
            protected = authish[:5]
            pub = publicish[:5]
            protected_sigs = []
            public_sigs = []
            for u in protected:
                try:
                    protected_sigs.append((u, self._sig(self.client.request("GET", u))))
                except (ScopeError, ERSECError):
                    pass
            for u in pub:
                try:
                    public_sigs.append((u, self._sig(self.client.request("GET", u))))
                except (ScopeError, ERSECError):
                    pass
            if protected_sigs:
                # A protected route returning the exact public signature is suspicious, but not proof of auth bypass.
                public_hashes = {x[1]["body_hash"] for x in public_sigs}
                same = [u for u, sig in protected_sigs if sig["body_hash"] in public_hashes and sig["status"] == 200]
                inv.append(BehavioralInvariant("INV-AUTH-BOUNDARY", "Protected-route boundary", "authorization", "HIGH", "Possible",
                    "Protected-looking routes should not be observably equivalent to unrelated public routes without an authentication transition.",
                    same, "Compare the protected route with an authenticated session and an unauthenticated session.", bool(same)))
        # Error-shape invariant: a group of resource routes should distinguish malformed/missing resources from generic success pages.
        resource_groups: Dict[str, List[str]] = {}
        for u in endpoints:
            p = urllib.parse.urlparse(u).path
            base = re.sub(r"/[^/]+$", "/{resource}", p) if "/" in p[1:] else p
            resource_groups.setdefault(base, []).append(u)
        for base, urls in list(resource_groups.items())[:25]:
            if len(urls) < 2:
                continue
            sigs=[]
            for u in urls[:4]:
                try: sigs.append(self._sig(self.client.request("GET",u)))
                except (ScopeError,ERSECError): pass
            if len({s["status"] for s in sigs}) == 1 and sigs and sigs[0]["status"] == 200:
                inv.append(BehavioralInvariant("INV-RESOURCE-DISTINCT", "Resource outcome distinction", "api-consistency", "MEDIUM", "Possible",
                    f"Resources in {base} currently share a uniform success status; applications commonly need distinct not-found/denied semantics.",
                    urls[:4], "Verify a known-good resource and a known-missing resource using an authorized test fixture.", False))
        return inv

class CounterfactualEngine:
    """Safe differential tests based on application behavior rather than payload spray.

    The engine asks bounded "what if" questions: what if a route is requested with a
    different representation hint, what if a known query parameter is removed, what
    if a redirecting route is observed without its state token? It never attempts to
    steal credentials or execute commands.
    """
    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config=config; self.client=client

    def evaluate(self, endpoints: List[str], forms: List[FormInfo]) -> Dict[str, Any]:
        checks=[]
        for u in endpoints[:20]:
            parsed=urllib.parse.urlparse(u)
            if parsed.query:
                params=urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
                for key in list(params)[:3]:
                    base_q={k:v for k,v in params.items() if k!=key}
                    alt=urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode([(k,x) for k,vals in base_q.items() for x in vals])))
                    if alt==u: continue
                    try:
                        a=self.client.request("GET",u); b=self.client.request("GET",alt)
                    except (ScopeError,ERSECError):
                        continue
                    checks.append({"type":"parameter-removal-diff","url":u,"parameter":key,
                                   "baseline_status":a.status_code,"counterfactual_status":b.status_code,
                                   "baseline_length":len(a.text or ""),"counterfactual_length":len(b.text or ""),
                                   "behavior_changed":(a.status_code!=b.status_code or len(a.text or "")!=len(b.text or ""))})
        return {"checks":checks[:50],"changed":sum(1 for c in checks if c["behavior_changed"])}

class EvidenceCapsuleBuilder:
    """Build structured, replay-oriented proof records without persisting credentials."""
    def __init__(self, config: ScanConfig): self.config=config
    def build(self, finding: Finding) -> Dict[str, Any]:
        d=finding.to_dict()
        evidence=d.get("evidence",{})
        metadata=SecurityCategoryRegistry.metadata(finding.category)
        observation={
            "method": evidence.get("method"),
            "url": evidence.get("url"),
            "status_code": evidence.get("status_code"),
            "response_time_ms": evidence.get("response_time_ms"),
            "response_headers": evidence.get("response_headers", {}),
            "response_excerpt": evidence.get("response_excerpt", "")[:300],
        }
        replay={
            "method": evidence.get("method") or "GET",
            "url": evidence.get("url") or finding.url,
            "headers": evidence.get("request_headers_sent", {}),
            "credentials_persisted": False,
        }
        capsule={
            "schema": ERSEC_PROOF_SCHEMA,
            "finding_id": finding.finding_id,
            "category": metadata.canonical,
            "category_metadata": asdict(metadata),
            "title": finding.title,
            "severity": finding.severity.name,
            "confidence": finding.confidence,
            "observation": observation,
            "baseline": {
                "available": False,
                "note": "The current detector did not persist a distinct control request in this capsule."
            },
            "detector_reasoning": {
                "detector_category": metadata.canonical,
                "test_family": metadata.test_family,
                "evidence_score": finding.evidence_score,
                "statement": finding.description,
            },
            "impact": {
                "url": finding.url,
                "parameter": finding.parameter,
                "attack_surface": finding.attack_surface,
                "state_context": finding.state_context,
            },
            "limitations": [
                "The capsule records controlled evidence and is not an exploit transcript.",
                "Authentication credentials are intentionally excluded.",
                "Timing differences are supporting evidence only and do not establish exploitability by themselves.",
            ],
            "replay": replay,
            "remediation": {
                "summary": finding.remediation_summary,
                "verification": finding.ai_verification_steps,
            },
            "integrity": {},
            "created_at": _utc_now_iso(),
        }
        evidence_canonical=json.dumps(observation,sort_keys=True,separators=(",",":"),default=str)
        config_material={
            "profile":self.config.profile.value,
            "target":self.config.target,
            "scope":asdict(self.config.scope),
            "authorization_manifest":getattr(self.config,"authorization_manifest",None),
        }
        config_canonical=json.dumps(config_material,sort_keys=True,separators=(",",":"),default=str)
        capsule["integrity"]={
            "algorithm":"sha256",
            "evidence_sha256":hashlib.sha256(evidence_canonical.encode()).hexdigest(),
            "configuration_sha256":hashlib.sha256(config_canonical.encode()).hexdigest(),
        }
        canonical=json.dumps(capsule,sort_keys=True,separators=(",",":"),default=str)
        capsule["integrity"]["canonical_sha256"]=hashlib.sha256(canonical.encode()).hexdigest()
        return capsule

class MaturityOrchestrator:
    """ERSEC 6 reasoning layer: behavior -> invariant -> counterfactual -> proof."""
    def __init__(self, config: ScanConfig, crawler: WebCrawler, client: SafeHttpClient):
        self.config=config
        self.crawler=crawler
        self.client=client
        self.memory=BehaviorMemory(getattr(config,"continuous_memory_path",None))
        self.journeys=JourneyCompiler(crawler)
        self.invariants=InvariantLearner(config,client,self.memory)
        self.counterfactual=CounterfactualEngine(config,client)
        self.capsules=EvidenceCapsuleBuilder(config)

    def analyze(self, endpoints: List[str], forms: List[FormInfo], findings: List[Finding]) -> Dict[str, Any]:
        journeys=self.journeys.compile(endpoints,forms) if getattr(self.config,"journey_intelligence",True) else []
        invariants=self.invariants.learn(endpoints,forms) if getattr(self.config,"invariant_learning",True) else []
        counter={"checks":[],"changed":0}
        if getattr(self.config,"counterfactual_checks",True) and self.config.profile != ScanProfile.PASSIVE:
            counter=self.counterfactual.evaluate(endpoints,forms)
        capsules=[self.capsules.build(f) for f in findings[:200]] if getattr(self.config,"evidence_capsules",True) else []
        risk_signals=[]
        for i in invariants:
            if i.violated:
                risk_signals.append({"invariant_id":i.invariant_id,"severity":i.severity,"statement":i.statement,"evidence":i.observed_evidence})
        for j in journeys[:100]:
            self.memory.remember("journey:"+hashlib.sha256(f"{j.source}|{j.destination}|{j.method}".encode()).hexdigest()[:16], asdict(j))
        self.memory.save()
        return {
            "engine":"behavioral-twin-v1",
            "journeys":[asdict(j) for j in journeys[:100]],
            "invariants":[asdict(i) for i in invariants],
            "counterfactuals":counter,
            "proof_capsules":capsules,
            "risk_signals":risk_signals,
            "novelty":{
                "behavioral_model":bool(journeys),
                "invariant_learning":bool(invariants),
                "counterfactual_reasoning":bool(counter.get("checks")),
                "proof_carrying_findings":bool(capsules),
            }
        }


# =============================================================================
# ERSEC 7.0 - Security Behavior Genome / Temporal Reasoning
# =============================================================================

@dataclass
class GenomeObservation:
    key: str
    kind: str
    source: str
    state: str
    signature: str
    metadata: Dict[str, Any] = field(default_factory=dict)
    observed_at: str = field(default_factory=_utc_now_iso)

@dataclass
class GenomeTransition:
    source_state: str
    destination_state: str
    trigger: str
    method: str
    evidence_key: str
    confidence: float
    risk: float = 0.0

@dataclass
class GenomeInvariant:
    invariant_id: str
    statement: str
    class_name: str
    confidence: float
    status: str
    evidence: List[str] = field(default_factory=list)
    reason: str = ""

class SecurityBehaviorGenome:
    """ERSEC 7 behavior model.

    The genome is intentionally not an exploit engine. It models observable application
    states, transitions, contracts, and invariants. Its differentiator is temporal
    reasoning: a security property can be violated only after a particular transition.
    """
    VERSION = "7.0"

    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client
        self.memory_path = getattr(config, "genome_memory_path", None) or getattr(config, "continuous_memory_path", None)
        self.memory = self._load_memory()

    def _load_memory(self) -> Dict[str, Any]:
        if not self.memory_path:
            return {"version": 1, "states": {}, "contracts": {}, "invariants": {}, "scans": []}
        try:
            with open(self.memory_path, "r", encoding="utf-8") as fh:
                obj = json.load(fh)
                if isinstance(obj, dict):
                    return obj
        except (OSError, json.JSONDecodeError):
            pass
        return {"version": 1, "states": {}, "contracts": {}, "invariants": {}, "scans": []}

    def save(self):
        if not self.memory_path:
            return
        Path = __import__("pathlib").Path
        path = Path(self.memory_path)
        path.parent.mkdir(parents=True, exist_ok=True)
        tmp = Path(str(path) + ".tmp")
        try:
            tmp.write_text(json.dumps(self.memory, indent=2, sort_keys=True), encoding="utf-8")
            tmp.replace(path)
        except OSError:
            try: tmp.unlink(missing_ok=True)
            except Exception: pass

    @staticmethod
    def state_for_url(url: str) -> str:
        path = urllib.parse.urlparse(url).path or "/"
        bits = [b for b in path.split("/") if b]
        if not bits:
            return "public:root"
        first = bits[0].lower()
        if first in {"login", "signin", "auth", "oauth"}: return "auth:entry"
        if first in {"logout", "signout"}: return "auth:exit"
        if first in {"admin", "manage", "internal"}: return "privileged:admin"
        if first in {"account", "profile", "me", "settings"}: return "authenticated:account"
        if first in {"cart", "checkout", "payment", "order", "orders"}: return "transactional:commerce"
        if first in {"api", "rest", "v1", "v2", "v3", "graphql", "grpc"}: return "api:service"
        if first in {"webhook", "webhooks", "ws", "socket", "events"}: return "realtime:eventing"
        return "public:resource"

    @staticmethod
    def response_signature(resp: requests.Response) -> str:
        body = resp.text or ""
        normalized = re.sub(r"(?i)(token|secret|password|authorization|api[_-]?key)\s*[:=]\s*[^\s,;&]+", r"\1=<redacted>", body[:6000])
        normalized = re.sub(r"\b\d{2,}\b", "#", normalized)
        normalized = re.sub(r"\s+", " ", normalized).strip()
        data = f"{resp.status_code}|{resp.headers.get('Content-Type','').split(';')[0].lower()}|{normalized[:1500]}"
        return hashlib.sha256(data.encode("utf-8", "ignore")).hexdigest()[:24]

    def observe(self, endpoints: List[str], forms: List[FormInfo], browser_data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
        observations: List[GenomeObservation] = []
        transitions: List[GenomeTransition] = []
        by_source = {}
        for url in endpoints[:250]:
            try:
                resp = self.client.request("GET", url)
            except (ScopeError, ERSECError):
                continue
            state = self.state_for_url(url)
            key = f"url:{hashlib.sha256(url.split('?')[0].encode()).hexdigest()[:20]}"
            obs = GenomeObservation(key, "http-state", url, state, self.response_signature(resp), {
                "status": resp.status_code,
                "content_type": resp.headers.get("Content-Type", "").split(";",1)[0],
                "method": "GET",
            })
            observations.append(obs)
            by_source[url] = state
        for src, dsts in self._safe_edges():
            if src not in by_source: continue
            for dst in list(dsts)[:50]:
                if dst not in by_source: continue
                src_state, dst_state = by_source[src], by_source[dst]
                transitions.append(GenomeTransition(src_state, dst_state, "observed-link", "GET",
                                                     hashlib.sha256(f"{src}->{dst}".encode()).hexdigest()[:20],
                                                     0.90, self.transition_risk(src_state, dst_state)))
        for f in forms[:150]:
            src = self.state_for_url(f.page_url)
            dst = self.state_for_url(f.action or f.page_url)
            transitions.append(GenomeTransition(src, dst, "observed-form", f.method,
                                                 hashlib.sha256(f"form:{f.page_url}|{f.action}|{f.method}".encode()).hexdigest()[:20],
                                                 0.85, self.transition_risk(src, dst)))
        if browser_data:
            for item in browser_data.get("requests", [])[:250]:
                url = item.get("url") if isinstance(item, dict) else None
                method = item.get("method", "GET") if isinstance(item, dict) else "GET"
                if not url or not is_in_scope(url, self.config): continue
                observations.append(GenomeObservation(
                    f"browser:{hashlib.sha256((method+'|'+url).encode()).hexdigest()[:20]}",
                    "browser-traffic", url, self.state_for_url(url),
                    hashlib.sha256(f"{method}|{url}".encode()).hexdigest()[:24],
                    {"method": method, "source": "headless-browser"}))
        self._remember(observations, transitions)
        return {"version": self.VERSION, "observations": [asdict(x) for x in observations[:500]],
                "transitions": [asdict(x) for x in transitions[:500]],
                "state_count": len({x.state for x in observations}),
                "transition_count": len(transitions)}

    def _safe_edges(self):
        return self._crawler_edges if hasattr(self, "_crawler_edges") else []

    def bind_crawler(self, crawler: WebCrawler):
        self._crawler_edges = list(crawler.link_graph.items())

    @staticmethod
    def transition_risk(src: str, dst: str) -> float:
        pairs = {
            ("public:resource", "privileged:admin"): 0.95,
            ("public:resource", "authenticated:account"): 0.65,
            ("authenticated:account", "transactional:commerce"): 0.70,
            ("public:resource", "transactional:commerce"): 0.80,
            ("api:service", "privileged:admin"): 0.90,
            ("realtime:eventing", "privileged:admin"): 0.90,
        }
        return pairs.get((src, dst), 0.15)

    def _remember(self, observations: List[GenomeObservation], transitions: List[GenomeTransition]):
        states = self.memory.setdefault("states", {})
        contracts = self.memory.setdefault("contracts", {})
        for o in observations:
            prev = states.get(o.key)
            states[o.key] = {"state": o.state, "signature": o.signature, "updated_at": o.observed_at,
                             "changed": bool(prev and prev.get("signature") != o.signature)}
            contracts[o.key] = o.metadata
        self.memory.setdefault("scans", []).append({"at": _utc_now_iso(), "observations": len(observations), "transitions": len(transitions)})
        self.memory["scans"] = self.memory["scans"][-30:]
        self.save()

class TemporalReasoner:
    """Finds security-relevant drift and suspicious state transitions without exploitation."""
    def __init__(self, genome: SecurityBehaviorGenome): self.genome = genome

    def analyze(self, genome_report: Dict[str, Any], findings: List[Finding]) -> Dict[str, Any]:
        transitions = genome_report.get("transitions", [])
        high_risk = [t for t in transitions if float(t.get("risk",0)) >= 0.7]
        category_states = {f.category: SecurityBehaviorGenome.state_for_url(f.url) for f in findings}
        correlated = []
        for f in findings:
            state = category_states.get(f.category)
            if state == "privileged:admin" and f.severity in (Severity.HIGH, Severity.CRITICAL):
                correlated.append({"finding_id": f.finding_id, "reason": "high-severity finding occurs in privileged state", "state": state})
        return {"high_risk_transitions": high_risk[:100], "state_correlations": correlated[:100],
                "novel_temporal_signal": bool(high_risk or correlated)}

class ContractDriftEngine:
    """Compares observed API-like response contracts with prior local memory."""
    def __init__(self, genome: SecurityBehaviorGenome): self.genome=genome
    def analyze(self, endpoints: List[str]) -> Dict[str, Any]:
        current = {}
        drift=[]
        for u in endpoints[:150]:
            p=urllib.parse.urlparse(u)
            if not re.search(r"/(?:api|rest|v\d+|graphql|grpc)(?:/|$)", p.path, re.I): continue
            try: resp=self.genome.client.request("GET",u)
            except (ScopeError,ERSECError): continue
            sig=self.genome.response_signature(resp)
            key=hashlib.sha256(u.split('?')[0].encode()).hexdigest()[:20]
            previous=self.genome.memory.get("contracts",{}).get("url:"+key)
            current[key]={"url":u,"status":resp.status_code,"content_type":resp.headers.get("Content-Type",""),"signature":sig}
            if previous and previous.get("signature") and previous.get("signature") != sig:
                drift.append({"url":u,"previous":previous.get("signature"),"current":sig,"type":"response-contract-drift"})
        self.genome.memory.setdefault("api_contracts", {}).update(current)
        self.genome.save()
        return {"contracts_observed":len(current),"drift":drift[:100],"drift_count":len(drift)}

class ExplainablePriorityEngine:
    """Produces a decision record so every priority score has inspectable reasons."""
    def score(self, finding: Finding, attack_paths: List[AttackPath], temporal: Dict[str, Any]) -> Dict[str, Any]:
        severity=float(finding.severity.value)
        confidence={"Confirmed":1.0,"Likely":0.75,"Possible":0.45}.get(finding.confidence,0.5)
        chain_bonus=0.25 if any(finding.finding_id in p.contributing_findings for p in attack_paths) else 0.0
        temporal_bonus=0.20 if any(x.get("finding_id")==finding.finding_id for x in temporal.get("state_correlations",[])) else 0.0
        score=round(min(100.0, (severity/4.0)*60*confidence + chain_bonus*100 + temporal_bonus*100),2)
        reasons=[]
        reasons.append(f"severity={finding.severity.name}")
        reasons.append(f"confidence={finding.confidence}")
        if chain_bonus: reasons.append("participates in an attack-path fusion")
        if temporal_bonus: reasons.append("occurs in a security-sensitive behavioral state")
        return {"priority_score":score,"decision":"prioritize" if score>=60 else "monitor","reasons":reasons}

class ERSEC7Orchestrator:
    """ERSEC 7 super-layer: genome + temporal reasoning + contract drift + explainable priority."""
    def __init__(self, config: ScanConfig, crawler: WebCrawler, client: SafeHttpClient):
        self.config=config; self.crawler=crawler; self.client=client
        self.genome=SecurityBehaviorGenome(config,client); self.genome.bind_crawler(crawler)
        self.temporal=TemporalReasoner(self.genome); self.contracts=ContractDriftEngine(self.genome)
        self.priority=ExplainablePriorityEngine()

    def analyze(self, endpoints, forms, findings, attack_paths, browser_data=None):
        if not getattr(self.config,"security_genome",True):
            return {"engine":"security-behavior-genome-v7","enabled":False}
        genome=self.genome.observe(endpoints,forms,browser_data)
        contracts=self.contracts.analyze(endpoints) if getattr(self.config,"contract_drift",True) else {"drift":[],"drift_count":0,"contracts_observed":0}
        temporal=self.temporal.analyze(genome,findings) if getattr(self.config,"temporal_reasoning",True) else {"high_risk_transitions":[],"state_correlations":[]}
        decisions={f.finding_id:self.priority.score(f,attack_paths,temporal) for f in findings} if getattr(self.config,"explainable_prioritization",True) else {}
        invariants=[]
        for t in genome.get("transitions",[])[:150]:
            if t.get("risk",0)>=0.9:
                invariants.append(asdict(GenomeInvariant(
                    "GENOME-"+hashlib.sha256(json.dumps(t,sort_keys=True).encode()).hexdigest()[:12],
                    f"Sensitive state transition {t.get('source_state')} -> {t.get('destination_state')} deserves an explicit authorization invariant.",
                    "authorization-boundary", float(t.get("confidence",0.0)), "observed",
                    [t.get("evidence_key","")], "Derived from an observed transition into a privileged state.")))
        return {"engine":"security-behavior-genome-v7","version":SecurityBehaviorGenome.VERSION,
                "genome":genome,"temporal":temporal,"contract_drift":contracts,
                "invariants":invariants[:100],"priority_decisions":decisions,
                "innovation":{
                    "temporal_reasoning":True,"application_security_genome":True,
                    "contract_drift_memory":True,"explainable_priority":True,
                    "state_transition_authorization_model":True,
                }}


# =============================================================================
# ERSEC 7.2 - Detection Intelligence Layer
# =============================================================================

@dataclass
class CapabilityEdge:
    source: str
    relation: str
    destination: str
    confidence: float
    evidence: List[str] = field(default_factory=list)


class JSONSchemaMiner:
    """Recover lightweight response/request schemas from observed HTTP traffic.
    This is intentionally local and heuristic; it creates *candidates* that the
    active scanner can prioritize instead of guessing a tiny fixed parameter set.
    """
    _SENSITIVE = re.compile(r"(?:password|secret|token|api[_-]?key|private[_-]?key|authorization)", re.I)

    @staticmethod
    def _walk(obj: Any, prefix: str = "") -> List[str]:
        out=[]
        if isinstance(obj, dict):
            for k,v in obj.items():
                name=f"{prefix}.{k}" if prefix else str(k)
                out.append(name)
                out.extend(JSONSchemaMiner._walk(v,name))
        elif isinstance(obj, list):
            for i,v in enumerate(obj[:3]):
                out.extend(JSONSchemaMiner._walk(v,f"{prefix}[]"))
        return out

    def mine(self, client: SafeHttpClient, urls: Iterable[str], limit: int = 40) -> Dict[str, Any]:
        models=[]; params=set(); secrets=[]
        for url in list(dict.fromkeys(urls))[:limit]:
            try:
                t0=time.time(); resp=client.request("GET",url); elapsed=int((time.time()-t0)*1000)
            except (ScopeError,ERSECError):
                continue
            ctype=(resp.headers.get("Content-Type") or "").lower()
            if "json" not in ctype and not (resp.text or "").lstrip().startswith(("{","[")):
                continue
            try: data=resp.json()
            except Exception: continue
            fields=self._walk(data)
            top=sorted({f.split(".")[0].replace("[]","") for f in fields if f})
            for f in top:
                if f and not self._SENSITIVE.search(f): params.add(f)
            secrets.extend([f for f in fields if self._SENSITIVE.search(f)])
            models.append({"url":url,"status":resp.status_code,"elapsed_ms":elapsed,
                           "top_level_fields":top[:80],"field_count":len(fields),
                           "response_schema_hash":hashlib.sha256(json.dumps(data,sort_keys=True,default=str).encode()).hexdigest()[:16]})
        return {"models":models,"candidate_parameters":sorted(params)[:200],
                "sensitive_field_paths":sorted(set(secrets))[:200],"model_count":len(models)}


class SecurityBoundaryDiscoveryEngine:
    """Probe high-value *read-only* application boundaries and classify them.
    It never attempts authentication bypass; it only records whether a boundary
    is publicly reachable, redirect-protected, or access-controlled.
    """
    PATHS=(
        "/admin","/administrator","/manage","/management","/actuator","/actuator/health",
        "/metrics","/debug","/internal","/private","/api/admin","/api/internal",
        "/swagger.json","/openapi.json","/api-docs","/graphql","/graphiql","/health","/ready",
    )
    def __init__(self,config:ScanConfig,client:SafeHttpClient): self.config=config; self.client=client
    def scan(self,base_url:str)->Dict[str,Any]:
        observations=[]; findings=[]
        for path in self.PATHS:
            u=urllib.parse.urljoin(base_url,path)
            try:
                t0=time.time(); r=self.client.request("GET",u); ms=int((time.time()-t0)*1000)
            except (ScopeError,ERSECError): continue
            status=r.status_code
            classification="not_present"
            if status in (200,206): classification="publicly_reachable"
            elif status in (401,403): classification="access_controlled"
            elif status in (301,302,303,307,308): classification="redirected"
            elif status not in (404,410): classification="other"
            observations.append({"path":path,"url":u,"status":status,"classification":classification,"response_time_ms":ms})
            if classification=="publicly_reachable" and path in {"/admin","/administrator","/manage","/management","/debug","/internal","/private","/api/admin","/api/internal","/actuator","/metrics"}:
                body=(r.text or "")[:1200]
                looks_management=bool(re.search(r"(?:dashboard|management|actuator|metrics|debug|administrator|admin)",body,re.I))
                if looks_management:
                    ev=_make_evidence(r,ms)
                    findings.append(Finding(_next_id(),"security_boundary_exposure","Publicly reachable security-sensitive application boundary",Severity.HIGH,
                        "Likely","A01:2021","CWE-284",u,None,
                        f"A security-sensitive boundary at {path} returned HTTP {status} without an authentication challenge.",ev,
                        "Require authentication and authorization on management/internal interfaces and verify them separately from obscurity.",
                    ))
        return {"observations":observations,"findings":findings}


class AuthorizationDifferentialEngine:
    """Compare the same safe GET surface with and without supplied session auth.
    A suspiciously identical response is evidence for manual authorization review,
    not proof of an access-control bypass.
    """
    def __init__(self,config:ScanConfig,authenticated_client:SafeHttpClient): self.config=config; self.auth_client=authenticated_client
    @staticmethod
    def _fingerprint(resp: requests.Response)->Tuple[int,str,int]:
        body=re.sub(r"\d+","#",re.sub(r"\s+"," ",(resp.text or "")[:4000])).strip()
        return resp.status_code,hashlib.sha256(body.encode()).hexdigest()[:16],len(body)
    def compare(self,urls:Iterable[str],limit:int=50)->List[Finding]:
        if not (self.config.cookies or self.config.bearer_token): return []
        anon_cfg=dataclasses.replace(self.config,cookies={},bearer_token="",extra_headers=dict(self.config.extra_headers))
        anon_client=SafeHttpClient(anon_cfg)
        out=[]
        for url in list(dict.fromkeys(urls))[:limit]:
            try:
                ar=anon_client.request("GET",url)
                ur=self.auth_client.request("GET",url)
            except (ScopeError,ERSECError): continue
            af=self._fingerprint(ar); uf=self._fingerprint(ur)
            path=urllib.parse.urlparse(url).path.lower()
            sensitive=any(x in path for x in ("/account","/profile","/user","/order","/invoice","/admin","/private","/api/"))
            if sensitive and af[:2]==uf[:2] and ur.status_code==200:
                ev=_make_evidence(ur,0)
                out.append(Finding(_next_id(),"authorization_differential","Authenticated and anonymous responses are materially identical",
                    Severity.MEDIUM,"Possible","A01:2021","CWE-862",url,None,
                    "The authenticated and unauthenticated GET responses share the same status and normalized body fingerprint on a sensitive-looking route. This is a review signal, not a proof of missing authorization.",
                    ev,"Enforce authorization at the resource/service layer and add role- and ownership-specific automated tests."))
        return out


class CapabilityRiskGraph:
    """Evidence-driven risk chaining. Existing rules remain authoritative; this
    engine discovers additional compound paths from capability relationships.
    """
    CAPABILITIES={
        "exposed_files":{"credential_exposure","configuration_disclosure"},
        "verbose_errors":{"reconnaissance","configuration_disclosure"},
        "open_redirect":{"redirect_control","phishing_facilitation"},
        "host_header":{"host_trust_manipulation"},
        "cors":{"cross_origin_access"},
        "csrf":{"cross_origin_state_change"},
        "xss":{"script_execution","session_impact"},
        "security_headers":{"weak_browser_boundaries"},
        "jwt":{"token_forgery_or_misuse"},
        "weak_session_token":{"session_prediction"},
        "idor_heuristic":{"cross_account_data_access"},
        "authorization_differential":{"authorization_uncertainty"},
        "cross_identity_authorization":{"cross_account_data_access","authorization_uncertainty"},
        "sensitive_data_exposure":{"sensitive_data_disclosure","credential_exposure"},
        "authenticated_cache_leak":{"shared_cache_exposure","sensitive_data_disclosure"},
        "security_boundary_exposure":{"privileged_surface"},
        "rate_limiting":{"automation_enablement"},
        "user_enumeration":{"identity_discovery"},
        "mass_assignment":{"unintended_field_control"},
        "ssrf":{"server_side_network_access"},
        "xxe":{"server_side_file_or_network_access"},
        "command_injection":{"server_execution"},
        "sql_injection":{"database_interpretation"},
        "graphql_introspection":{"schema_disclosure"},
        "vulnerable_library":{"client_side_supply_chain_risk"},
        "subdomain_takeover":{"attacker_controlled_origin"},
        "email_security":{"identity_spoofing"},
        "commerce_business_logic":{"transaction_integrity_risk"},
    }
    RELATIONS={
        ("credential_exposure","privileged_surface"):"credentials_can_reach_privileged_surface",
        ("identity_discovery","automation_enablement"):"account_targeting_is_amplified",
        ("authorization_uncertainty","cross_account_data_access"):"access_control_gap_needs_validation",
        ("redirect_control","identity_spoofing"):"trusted-domain_phishing_amplification",
        ("cross_origin_access","token_forgery_or_misuse"):"credentialed_cross_origin_token_risk",
        ("script_execution","session_impact"):"browser_to_session_impact",
        ("server_side_network_access","privileged_surface"):"server_to_internal_boundary",
        ("database_interpretation","configuration_disclosure"):"data-layer-feedback-loop",
        ("unintended_field_control","cross_account_data_access"):"authorization_and_object-control_intersection",
         ("transaction_integrity_risk","authorization_uncertainty"):"business_transaction_boundary_uncertainty",
        ("sensitive_data_disclosure","privileged_surface"):"sensitive_data_to_privileged_surface",
        ("shared_cache_exposure","sensitive_data_disclosure"):"cache_to_sensitive_data_leak",
        ("cross_account_data_access","sensitive_data_disclosure"):"cross_account_data_to_sensitive_data",
    }
    def fuse(self,findings:List[Finding])->List[Dict[str,Any]]:
        bycap= {}
        for f in findings:
            for cap in self.CAPABILITIES.get(f.category,set()): bycap.setdefault(cap,[]).append(f)
        chains=[]
        for (a,b),rel in self.RELATIONS.items():
            if a not in bycap or b not in bycap: continue
            members=[]
            for f in bycap[a]+bycap[b]:
                if f.finding_id not in [m["finding_id"] for m in members]:
                    members.append({"finding_id":f.finding_id,"category":f.category,"confidence":f.confidence})
            conf=min(({"Confirmed":1.0,"Likely":0.72,"Possible":0.45}.get(x["confidence"],0.2) for x in members),default=0.0)
            sev=Severity.CRITICAL if conf>=0.72 else Severity.HIGH
            chains.append({"name":f"Capability chain: {a} → {b}","relationship":rel,"severity":sev.name,
                           "confidence":round(conf,2),"members":members,
                           "rationale":f"Evidence establishes or suggests {a}; separately, evidence establishes or suggests {b}. Together the capabilities create a materially stronger review path: {rel}."})
        # 3-hop bridge: A -> B and B -> C, requiring evidence at every hop.
        return chains[:50]


class CoverageMatrix:
    """Report what ERSEC actually tested and what could not be tested.
    This prevents a clean scan from masquerading as complete coverage.
    """
    FAMILIES=("injection","client_side","authz","authn_session","api","cloud","protocol","config","business_logic","secrets","dependency","browser")
    def build(self,findings,config,browser_data=None):
        cats={f.category for f in findings}
        enabled={
            "browser":bool(getattr(config,"browser_discovery",False)),
            "cloud":bool(getattr(config,"cloud_native",True)),
            "business_logic":bool(getattr(config,"commerce_intelligence",True)),
            "api":bool(getattr(config,"api_intelligence",True)),
        }
        tested={
            "injection":any(c in cats for c in {"sql_injection","nosql_injection","ssti","xxe","ldap_injection","xpath_injection","command_injection","crlf_injection","header_injection"}),
            "client_side":any(c in cats for c in {"xss","stored_xss","dom_xss","open_redirect","cors"}),
            "authz":any(c in cats for c in {"idor_heuristic","mass_assignment","authorization_differential","cross_identity_authorization"}),
            "authn_session":any(c in cats for c in {"jwt","weak_session_token","rate_limiting","user_enumeration","csrf"}),
            "api":enabled["api"],"cloud":enabled["cloud"],"protocol":any(c in cats for c in {"websocket","http_methods"}),
            "config":any(c in cats for c in {"security_headers","tls","cors","cache_control","authenticated_cache_leak"}),
            "business_logic":enabled["business_logic"],"secrets":any(c in cats for c in {"exposed_files","sensitive_data_exposure"}),
            "dependency":any(c in cats for c in {"vulnerable_library"}),"browser":enabled["browser"],
        }
        blind=[]
        if not enabled["browser"]: blind.append("browser_dynamic_execution_disabled")
        if not enabled["api"]: blind.append("api_intelligence_disabled")
        if not (config.cookies or config.bearer_token): blind.append("authenticated_authorization_differential_not_available")
        return {"tested_families":tested,"disabled_or_unavailable":blind,
                "coverage_ratio":round(sum(bool(v) for v in tested.values())/len(tested),2),
                "interpretation":"coverage is a capability indicator, not proof of vulnerability absence"}


def enrich_advanced_detection(findings:List[Finding], config:ScanConfig, crawler:WebCrawler, client:SafeHttpClient, start_url:str)->Dict[str,Any]:
    boundary=SecurityBoundaryDiscoveryEngine(config,client).scan(start_url)
    findings.extend(boundary["findings"])
    schema=JSONSchemaMiner().mine(client,crawler.endpoints)
    authz=AuthorizationDifferentialEngine(config,client).compare(crawler.endpoints)
    findings.extend(authz)
    cross_identity=CrossIdentityAuthorizationEngine(config,client).compare(crawler.endpoints)
    findings.extend(cross_identity)
    contract=APIContractMiner(config,client).mine(start_url)
    planner=AdaptiveCoveragePlanner(config).build(crawler.endpoints,crawler.forms,schema.get("candidate_parameters",[])) if config.adaptive_discovery else {"strategy":"disabled","candidates":[]}
    return {"security_boundaries":boundary["observations"],"json_schema_intelligence":schema,
            "api_contract":contract,"adaptive_plan":planner,
            "authorization_differential_count":len(authz),
            "cross_identity_authorization_count":len(cross_identity),
            "coverage":CoverageMatrix().build(findings,config)}



# =============================================================================
# ERSEC 8.0 - Adaptive Detection & Contextual Risk Intelligence
# =============================================================================

@dataclass(frozen=True)
class ProbeCandidate:
    name: str
    parameter: str
    score: float
    reasons: Tuple[str, ...] = ()


class ParameterSemanticEngine:
    """Classify parameter names and endpoint context without a network model.
    The classifier is intentionally deterministic and transparent; a local LLM
    can later augment the explanation, but it never controls scope or executes
    arbitrary actions.
    """
    TAXONOMY = {
        "redirect": ("url", "uri", "redirect", "return", "next", "continue", "callback", "dest", "target"),
        "identity": ("user_id", "userid", "user", "uid", "account_id", "customer_id", "member_id", "owner_id"),
        "role": ("role", "admin", "is_admin", "permission", "privilege", "scope", "group"),
        "file": ("file", "filename", "path", "filepath", "download", "document", "template"),
        "query": ("q", "query", "search", "term", "keyword", "filter", "where", "sort", "order", "select"),
        "identity_string": ("email", "username", "login", "user_name", "account", "name"),
        "numeric": ("id", "page", "limit", "offset", "count", "quantity", "amount"),
        "money": ("price", "amount", "cost", "total", "discount", "coupon", "tax", "credit", "refund"),
        "callback": ("webhook", "endpoint", "callback_url", "notify_url", "return_url"),
        "template": ("template", "view", "layout", "render", "format", "expression"),
        "xml": ("xml", "soap", "document", "payload"),
    }

    def classify(self, param: str, endpoint: str = "") -> List[str]:
        p = (param or "").lower().replace("-", "_")
        e = (endpoint or "").lower()
        labels=[]
        for label, words in self.TAXONOMY.items():
            if any(w == p or w in p for w in words):
                labels.append(label)
        if any(x in e for x in ("/admin", "/manage", "/internal")): labels.append("privileged_surface")
        if any(x in e for x in ("/checkout", "/cart", "/order", "/payment", "/refund")): labels.append("transactional")
        return sorted(set(labels))

    def rank(self, params: Iterable[str], endpoint: str = "", learned: Iterable[str] = ()) -> List[ProbeCandidate]:
        learned_set={x.lower() for x in learned}
        out=[]
        for p in dict.fromkeys(params):
            labels=self.classify(p,endpoint); reasons=[]; score=0.0
            if labels: score += 2.0; reasons.append("semantic_parameter_match")
            if p.lower() in learned_set: score += 1.5; reasons.append("observed_in_response_schema")
            if "privileged_surface" in labels: score += 1.5; reasons.append("privileged_endpoint_context")
            if "transactional" in labels: score += 1.25; reasons.append("transactional_workflow_context")
            if not labels: score += 0.25; reasons.append("generic_fallback")
            out.append(ProbeCandidate(p,p,score,tuple(reasons)))
        return sorted(out,key=lambda x:(-x.score,x.parameter.lower()))


class ResponseDifferentialAnalyzer:
    """Stable, low-noise comparison of two already-authorized responses."""
    @staticmethod
    def fingerprint(resp: requests.Response) -> Dict[str, Any]:
        body=(resp.text or "")[:6000]
        norm=re.sub(r"\d+","#",re.sub(r"\s+"," ",body)).strip()
        headers={k.lower():v for k,v in resp.headers.items() if k.lower() not in {"date","server-timing","set-cookie"}}
        return {"status":resp.status_code,"body_hash":hashlib.sha256(norm.encode()).hexdigest()[:20],
                "body_len":len(norm),"content_type":headers.get("content-type","")}

    @classmethod
    def compare(cls,a:requests.Response,b:requests.Response)->Dict[str,Any]:
        fa=cls.fingerprint(a); fb=cls.fingerprint(b)
        changed=[]
        for k in fa:
            if fa[k]!=fb[k]: changed.append(k)
        return {"changed":changed,"same":not changed,"before":fa,"after":fb,
                "strength":round(len(changed)/max(1,len(fa)),2)}


class AdaptiveCoveragePlanner:
    """Selects what to test next from discovered evidence and the remaining
    request budget. It never increases the budget; it only allocates it."""
    def __init__(self, config: ScanConfig): self.config=config
    def build(self, endpoints: List[str], forms: List[FormInfo], schema_params: Iterable[str]) -> Dict[str, Any]:
        sem=ParameterSemanticEngine(); candidates=[]
        base={"q","id","email","search","name","page","limit","sort","filter","next","redirect","url","path","file","role","user_id","account_id"}
        observed=set(schema_params)
        for u in endpoints[:self.config.scope.max_crawl_pages]:
            parsed=urllib.parse.urlparse(u)
            q=set(urllib.parse.parse_qsl(parsed.query,keep_blank_values=True) for _ in [0])
            qparams=[]
            for pair in q:
                qparams.extend([k for k,_ in pair])
            pool=set(qparams)|base|observed
            ranked=sem.rank(pool,parsed.path,observed)
            for c in ranked[:24]:
                candidates.append({"url":u,"parameter":c.parameter,"score":c.score,"reasons":c.reasons})
        dedup={}
        for c in candidates:
            k=(c["url"].split("?")[0],c["parameter"])
            dedup[k]=c if k not in dedup else max(dedup[k],c,key=lambda x:x["score"])
        ordered=sorted(dedup.values(),key=lambda x:-x["score"])
        budget=max(0,min(self.config.max_context_probes,len(ordered)))
        return {"candidates":ordered[:budget],"candidate_count":len(ordered),
                "selected_count":budget,"strategy":"semantic+observed-schema+context-aware"}


class APIContractMiner:
    """Discover API contract documents and lightweight route signatures."""
    DOC_PATHS=("/openapi.json","/swagger.json","/api-docs","/v1/openapi.json","/v2/openapi.json","/graphql")
    def __init__(self,config,client): self.config=config; self.client=client
    def mine(self,base_url:str)->Dict[str,Any]:
        docs=[]; routes=[]
        for path in self.DOC_PATHS:
            u=urllib.parse.urljoin(base_url,path)
            try:r=self.client.request("GET",u)
            except (ScopeError,ERSECError):continue
            text=r.text or ""
            if r.status_code!=200: continue
            if path.endswith("graphql") and "graphql" in (r.headers.get("Content-Type","")+text[:500]).lower():
                docs.append({"url":u,"type":"graphql","status":r.status_code})
                continue
            try:
                obj=r.json()
            except Exception:
                continue
            if isinstance(obj,dict):
                for key in ("paths","routes"):
                    vals=obj.get(key)
                    if isinstance(vals,dict):
                        for route,meta in vals.items():
                            methods=sorted([m.upper() for m in meta.keys() if m.lower() in {"get","post","put","patch","delete","head","options"}]) if isinstance(meta,dict) else []
                            routes.append({"route":route,"methods":methods,"source":u})
                docs.append({"url":u,"type":"openapi_or_swagger","status":r.status_code,"keys":sorted(obj.keys())[:20]})
        return {"documents":docs,"routes":routes[:self.config.max_api_paths],"route_count":len(routes)}


class ContextualRiskGraph:
    """Context-aware risk fusion. Unlike global category-set rules, an edge is
    only accepted when evidence lives on the same host and the capabilities are
    compatible with the endpoints/resources involved."""
    GRANTS=_CAPABILITY_GRANTS
    RELATIONS=(
        ("obtain_credentials_or_source","privileged_surface","credential_exposure_to_privileged_surface",Severity.CRITICAL),
        ("reach_internal_network","admin_control_no_auth","internal_pivot_to_admin",Severity.CRITICAL),
        ("run_script_in_victim_browser","cookie_not_httponly","script_to_session_theft",Severity.CRITICAL),
        ("enumerate_other_users_records_possible","privilege_field_write_possible","authorization_escalation_path",Severity.CRITICAL),
        ("account_identifiers_enumerable","unlimited_guess_attempts","identity_to_automation_path",Severity.HIGH),
        ("cross_origin_credentialed_read","enumerate_api_schema","api_map_to_cross_origin_data",Severity.HIGH),
        ("credible_phishing_link","credible_spoofed_sender","trusted_brand_impersonation",Severity.HIGH),
        ("known_client_cve_present","run_script_in_victim_browser","client_supply_chain_path",Severity.HIGH),
        ("read_database","obtain_credentials_or_source","database_to_credential_reuse",Severity.CRITICAL),
        ("server_code_execution","leak_internal_details","execution_with_environment_knowledge",Severity.CRITICAL),
    )
    @staticmethod
    def _host(f): return urllib.parse.urlparse(f.url).hostname or ""
    @staticmethod
    def _path(f): return urllib.parse.urlparse(f.url).path.lower()
    def fuse(self,findings:List[Finding])->List[Dict[str,Any]]:
        bycap={}
        for f in findings:
            for cap in self.GRANTS.get(f.category,set()): bycap.setdefault(cap,[]).append(f)
            if f.category=="security_headers" and "httponly" in f.description.lower(): bycap.setdefault("cookie_not_httponly",[]).append(f)
        chains=[]
        for left,right,name,severity in self.RELATIONS:
            for a in bycap.get(left,[]):
                for b in bycap.get(right,[]):
                    if a.finding_id==b.finding_id: continue
                    same_host=self._host(a)==self._host(b)
                    path_a=self._path(a); path_b=self._path(b)
                    related=same_host and (path_a.split("/")[1:2]==path_b.split("/")[1:2] or any(x in path_a+" "+path_b for x in ("admin","api","account","user","order","checkout","internal")))
                    if not same_host or not related: continue
                    conf=min({"Confirmed":1.0,"Likely":0.75,"Possible":0.5}.get(a.confidence,0.4),{"Confirmed":1.0,"Likely":0.75,"Possible":0.5}.get(b.confidence,0.4))
                    chains.append({"name":name,"severity":severity.name,"relationship":f"{left} -> {right}",
                                   "confidence":conf,"rationale":f"Context-compatible capabilities were observed on the same host with related application surfaces.",
                                   "members":[{"finding_id":a.finding_id,"category":a.category,"url":a.url},{"finding_id":b.finding_id,"category":b.category,"url":b.url}],
                                   "context":{"same_host":same_host,"related_surface":related}})
        # deterministic uniqueness
        seen=set(); out=[]
        for c in sorted(chains,key=lambda x:(-{"CRITICAL":4,"HIGH":3,"MEDIUM":2,"LOW":1,"INFO":0}[x["severity"]],-x["confidence"])):
            key=(c["name"],tuple(sorted(m["finding_id"] for m in c["members"])))
            if key not in seen: seen.add(key); out.append(c)
        return out[:100]


def enrich_detection_quality(findings: List[Finding], config: ScanConfig) -> Dict[str,Any]:
    """Attach a transparent evidence score to every finding and return a
    quality summary for reports/CI."""
    counts={"high":0,"medium":0,"low":0}
    for f in findings:
        score={"Confirmed":0.9,"Likely":0.7,"Possible":0.45}.get(f.confidence,0.3)
        if f.evidence.status_code: score += 0.03
        if f.evidence.response_excerpt: score += 0.03
        if f.evidence.request_headers_sent: score += 0.02
        f.evidence_score=round(min(1.0,score),2)
        f.tags=sorted(set(f.tags + (["strong-evidence"] if f.evidence_score>=0.85 else ["review-required"])))
        bucket="high" if f.evidence_score>=0.85 else "medium" if f.evidence_score>=0.6 else "low"
        counts[bucket]+=1
    return {"evidence_quality":counts,"mean_evidence_score":round(sum((f.evidence_score for f in findings),0)/max(1,len(findings)),2),
            "method":"deterministic-evidence-quality-v1"}




class DetectorRegistry:
    """Metadata registry for built-in and community detectors. The registry is
    observational: it provides coverage/health metadata without changing the
    existing detector contract."""
    def __init__(self, modules: Iterable[BaseModule]):
        self.modules=list(modules)
    def manifest(self)->List[Dict[str,Any]]:
        out=[]
        for m in self.modules:
            out.append({"category":getattr(m,"category","generic"),"title":getattr(m,"title",""),
                        "owasp":getattr(m,"owasp",""),"cwe":getattr(m,"cwe",""),
                        "min_profile":getattr(getattr(m,"min_profile",None),"value",str(getattr(m,"min_profile",""))),
                        "url_check":hasattr(m,"run_url"),"parameter_check":hasattr(m,"run_param"),
                        "module":m._detector.__class__.__name__ if isinstance(m, InstrumentedDetector) else m.__class__.__name__})
        return sorted(out,key=lambda x:x["category"])


class PassiveSurfaceMiner:
    """Expand discovery without brute force: infer route families from links,
    scripts, forms and observed URL patterns, then rank them for read-only review."""
    COMMON_DOCS=("/openapi.json","/swagger.json","/swagger/v1/swagger.json","/api/openapi.json",
                 "/api/swagger.json","/redoc","/api-docs","/.well-known/openid-configuration",
                 "/.well-known/jwks.json","/graphql","/graphiql","/health","/ready","/metrics")
    def mine(self, endpoints: List[str], forms: List[FormInfo], start_url: str, config: ScanConfig)->Dict[str,Any]:
        seen=set(); candidates=[]
        parsed=[urllib.parse.urlparse(u) for u in endpoints]
        for p in parsed:
            path=p.path
            parts=[x for x in path.split('/') if x]
            if not parts: continue
            # Infer collection and item routes, but do not request them here.
            if parts[-1].isdigit():
                item='/'+'/'.join(parts)
                collection='/'+'/'.join(parts[:-1]) if len(parts)>1 else '/'
                candidates.append((collection,'inferred_collection'))
                candidates.append((item,'observed_resource'))
            if len(parts)>=2 and parts[0].lower() in {'api','rest'}:
                candidates.append(('/'+'/'.join(parts[:min(3,len(parts))]),'api_family'))
        for f in forms:
            action=urllib.parse.urlparse(f.action or f.page_url).path
            candidates.append((action,'form_action'))
        for path in self.COMMON_DOCS:
            candidates.append((path,'well_known_surface'))
        for path,reason in candidates:
            u=urllib.parse.urljoin(start_url,path)
            if not is_in_scope(u,config): continue
            k=u.split('?')[0]
            if k not in seen: seen.add(k); candidates.append((path,reason)) if False else None
        unique=[]; seen2=set()
        for path,reason in candidates:
            u=urllib.parse.urljoin(start_url,path)
            if not is_in_scope(u,config): continue
            if u.split('?')[0] in seen2: continue
            seen2.add(u.split('?')[0]); unique.append({"url":u,"reason":reason})
        return {"candidates":unique[:config.max_api_paths],"candidate_count":len(unique)}


class SafeFindingValidator:
    """Re-check only inherently read-only findings whose replay method is GET,
    HEAD, OPTIONS, or TLS-HANDSHAKE. Consistency can raise evidence quality;
    instability lowers confidence instead of creating a new finding."""
    SAFE_METHODS={"GET","HEAD","OPTIONS"}
    def __init__(self,config:ScanConfig,client:SafeHttpClient): self.config=config; self.client=client
    def validate(self,findings:List[Finding],limit:int=60)->Dict[str,Any]:
        tested=stable=unstable=0
        for f in findings[:limit]:
            method=(f.evidence.method or '').upper()
            if method not in self.SAFE_METHODS or not f.evidence.url: continue
            tested += 1
            try:
                r1=self.client.request(method if method!='TLS-HANDSHAKE' else 'GET', f.evidence.url)
            except (ScopeError,ERSECError): continue
            before=ResponseDifferentialAnalyzer.fingerprint(type('R',(),{'status_code':f.evidence.status_code,'text':f.evidence.response_excerpt,'headers':f.evidence.response_headers})())
            after=ResponseDifferentialAnalyzer.fingerprint(r1)
            if before['status']==after['status'] and before['content_type']==after['content_type']:
                stable += 1
                f.tags=sorted(set(f.tags+['repeatable-evidence']))
                f.evidence_score=min(1.0,round(max(f.evidence_score,0.6)+0.08,2))
            else:
                unstable += 1
                if f.confidence=='Confirmed': f.confidence='Likely'
                elif f.confidence=='Likely': f.confidence='Possible'
                f.tags=sorted(set(f.tags+['unstable-evidence']))
        return {"tested":tested,"stable":stable,"unstable":unstable,"method":"safe-repeatability-check-v1"}


class CoverageGapEngine:
    """Explicitly identifies untested security dimensions so a 'clean' report
    cannot be mistaken for complete coverage."""
    FAMILIES={
        'discovery': {'crawled','browser','api'},
        'authentication': {'authn_session'},
        'authorization': {'authz'},
        'injection': {'injection'},
        'client_side': {'client_side'},
        'business_logic': {'business_logic'},
        'cloud': {'cloud'},
        'protocol': {'protocol'},
        'dependencies': {'dependency'},
        'secrets': {'secrets'},
    }
    def evaluate(self,coverage:Dict[str,Any],findings:List[Finding])->Dict[str,Any]:
        tested=coverage.get('tested_families',{})
        gaps=[name for name,keys in self.FAMILIES.items() if not any(bool(tested.get(k)) for k in keys)]
        return {'gaps':gaps,'gap_count':len(gaps),'assessment':'incomplete' if gaps else 'broad',
                'statement':'A clean result is not evidence that uncovered families are secure.'}



@dataclass
class DetectorExecution:
    detector_id: str
    module_class: str
    phase: str
    started_at: str
    duration_ms: int = 0
    findings_count: int = 0
    status: str = "ok"
    error: str = ""


class ScanExecutionLedger:
    """Thread-safe execution ledger for mature scan observability.

    A production scanner must distinguish "detector ran and found nothing" from
    "detector failed". The ledger records detector health without weakening scope
    controls or leaking exception internals into normal reports.
    """
    def __init__(self):
        self._lock = threading.Lock()
        self.records: List[DetectorExecution] = []

    def record(self, detector_id: str, module_class: str, phase: str, started: float,
               findings_count: int = 0, status: str = "ok", error: str = "") -> None:
        with self._lock:
            self.records.append(DetectorExecution(
                detector_id=detector_id,
                module_class=module_class,
                phase=phase,
                started_at=_utc_now_iso(),
                duration_ms=max(0, int((time.time() - started) * 1000)),
                findings_count=findings_count,
                status=status,
                error=error[:500],
            ))

    def summary(self) -> Dict[str, Any]:
        with self._lock:
            total = len(self.records)
            failed = sum(1 for r in self.records if r.status == "error")
            skipped = sum(1 for r in self.records if r.status == "skipped")
            by_phase: Dict[str, Dict[str, int]] = {}
            for r in self.records:
                b = by_phase.setdefault(r.phase, {"ok": 0, "error": 0, "skipped": 0, "findings": 0})
                b[r.status] = b.get(r.status, 0) + 1
                b["findings"] += r.findings_count
            return {
                "detector_invocations": total,
                "failed_invocations": failed,
                "skipped_invocations": skipped,
                "findings_emitted": sum(r.findings_count for r in self.records),
                "by_phase": by_phase,
                "healthy": failed == 0,
                "records": [asdict(r) for r in self.records[-500:]],
            }


class InstrumentedDetector:
    """Proxy around a detector that records execution health without changing its API."""
    def __init__(self, detector: Any, ledger: ScanExecutionLedger):
        self._detector = detector
        self._ledger = ledger

    def __getattr__(self, name: str):
        return getattr(self._detector, name)

    @property
    def category(self):
        return self._detector.category

    def applies(self):
        return self._detector.applies()

    def run_url(self, url: str):
        started = time.time()
        progress = getattr(self._detector.config, "_progress", None)
        if progress:
            progress.detector_start(self._detector.__class__.__name__)
        try:
            out = self._detector.run_url(url)
            self._ledger.record(self.category, self._detector.__class__.__name__, "url", started, len(out or []))
            if progress:
                progress.detector_finish(len(out or []))
            return out
        except Exception as exc:
            self._ledger.record(self.category, self._detector.__class__.__name__, "url", started, 0, "error", str(exc))
            if progress:
                progress.detector_fail(str(exc))
            raise

    def run_param(self, url: str, param: str, method: str = "GET"):
        started = time.time()
        progress = getattr(self._detector.config, "_progress", None)
        if progress:
            progress.detector_start(f"{self._detector.__class__.__name__}:{method.upper()}:{param}")
        try:
            out = self._detector.run_param(url, param, method)
            self._ledger.record(self.category, self._detector.__class__.__name__, f"param:{method.upper()}", started, len(out or []))
            if progress:
                progress.detector_finish(len(out or []))
            return out
        except Exception as exc:
            self._ledger.record(self.category, self._detector.__class__.__name__, f"param:{method.upper()}", started, 0, "error", str(exc))
            if progress:
                progress.detector_fail(str(exc))
            raise


class SensitiveDataExposureModule(BaseModule):
    """Detect high-signal secrets/sensitive records exposed by HTTP responses.

    This is deliberately evidence-first: it only reports when recognizable
    secret-like material appears in the returned body. It does not attempt to
    exfiltrate or validate credentials against external services.
    """
    category = "sensitive_data_exposure"
    title = "Sensitive Data Exposure"
    owasp = "A01:2021"
    cwe = "CWE-200"
    min_profile = ScanProfile.PASSIVE

    SECRET_PATTERNS = (
        ("private_key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), Severity.CRITICAL),
        ("aws_access_key", re.compile(r"\bAKIA[0-9A-Z]{16}\b"), Severity.CRITICAL),
        ("jwt_like", re.compile(r"\beyJ[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\.[a-zA-Z0-9_-]{8,}\b"), Severity.HIGH),
        ("client_secret", re.compile(r"(?i)\b(?:client[_-]?secret|api[_-]?secret|private[_-]?key)\b\s*[:=]\s*[\"'][^\"']{8,}[\"']"), Severity.HIGH),
        ("password_field", re.compile(r"(?i)[\"'](?:password|passwd|secret)[\"']\s*[:=]\s*[\"'][^\"']{4,}[\"']"), Severity.MEDIUM),
        ("ssn_like", re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), Severity.HIGH),
    )

    def run_url(self, url):
        findings = []
        resp, ev = self._get(url)
        body = (resp.text or "")[: self.config.max_sensitive_response_bytes]
        if not body or resp.status_code >= 400:
            return findings
        # Do not classify ordinary authenticated API data as a confirmed secret leak.
        authenticated = bool(self.config.cookies or self.config.bearer_token)
        matches = []
        for name, rx, sev in self.SECRET_PATTERNS:
            if rx.search(body):
                matches.append((name, sev))
        if not matches:
            return findings
        highest = max((x[1].value for x in matches), default=Severity.MEDIUM.value)
        severity = Severity(highest)
        confidence = "Likely" if authenticated else "Confirmed"
        kinds = ", ".join(sorted({x[0] for x in matches}))
        findings.append(self._finding(
            url, ev,
            f"Response body contains secret/sensitive-data patterns ({kinds}). This indicates data exposure in the application response; secret validity was not tested.",
            "Remove secrets and unnecessary sensitive fields from responses; apply field-level authorization and rotate any credentials that were exposed.",
            severity, confidence,
        ))
        return findings


class AuthenticatedCacheLeakModule(BaseModule):
    """Identify responses that combine authentication context with publicly cacheable semantics."""
    category = "authenticated_cache_leak"
    title = "Potential Authenticated Response Cache Exposure"
    owasp = "A05:2021"
    cwe = "CWE-525"
    min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        if not (self.config.cookies or self.config.bearer_token):
            return []
        path = urllib.parse.urlparse(url).path.lower()
        if not any(x in path for x in ("/account", "/profile", "/user", "/order", "/invoice", "/admin", "/private")):
            return []
        resp, ev = self._get(url)
        cache = (resp.headers.get("Cache-Control") or "").lower()
        if "public" in cache and "no-store" not in cache and "private" not in cache and resp.status_code == 200:
            return [self._finding(
                url, ev,
                "An authenticated-looking response is marked publicly cacheable. Shared caches may serve user-specific content to another client depending on the deployment topology.",
                "Use private/no-store semantics for user-specific responses and vary caching only on explicitly safe dimensions. Test intermediary/CDN behavior separately.",
                Severity.HIGH, "Likely",
            )]
        return []



@dataclass
class IdentityProfile:
    name: str
    token: str = ""
    cookies: Dict[str, str] = field(default_factory=dict)
    privilege_rank: int = 1


@dataclass
class WorkflowObservation:
    workflow_id: str
    identities: Dict[str, Dict[str, Any]]
    path: List[str]
    sensitivity: str
    evidence: List[str] = field(default_factory=list)


class MultiIdentityWorkflowEngine:
    """State-aware, read-only authorization comparison across explicit identities.

    The engine intentionally performs GET/HEAD-style replay only. It learns short
    workflows from the crawler's link graph and compares security decisions for
    anonymous and explicitly supplied identities. It never treats a single 200
    response as proof of an authorization bypass; findings remain evidence-backed
    review signals unless ownership/role expectations are established.
    """
    SENSITIVE_PATTERNS = (
        (re.compile(r"/(?:admin|administrator|manage|management|internal|debug|actuator|metrics)(?:/|$)", re.I), "privileged"),
        (re.compile(r"/(?:account|profile|users?|orders?|invoices?|projects?|tenants?|baskets?|carts?)/", re.I), "identity_or_resource"),
        (re.compile(r"/(?:checkout|payment|refund|transfer|approve|confirm|delete|publish)(?:/|$)", re.I), "transactional"),
        (re.compile(r"/(?:api|rest|graphql|v[0-9]+)(?:/|$)", re.I), "api"),
    )

    def __init__(self, config: ScanConfig, primary: SafeHttpClient, crawler: WebCrawler):
        self.config = config
        self.primary = primary
        self.crawler = crawler

    def identities(self) -> List[IdentityProfile]:
        out=[IdentityProfile("anonymous", privilege_rank=0)]
        if self.config.bearer_token or self.config.cookies:
            out.append(IdentityProfile("user", self.config.bearer_token, dict(self.config.cookies), 1))
        second=getattr(self.config,"second_bearer_token","")
        if second:
            out.append(IdentityProfile("user-2", second, {}, 1))
        for name, token in getattr(self.config,"identity_tokens",{}).items():
            lname=name.lower()
            rank=self.config.identity_role_order.get(lname, 1)
            if token:
                out.append(IdentityProfile(name, token, {}, rank))
        # Deduplicate by identity name while preserving highest-priority token entry.
        seen={};
        for ident in out: seen[ident.name]=ident
        return list(seen.values())

    @staticmethod
    def _body_fp(resp: requests.Response) -> str:
        body=(resp.text or "")[:5000]
        body=re.sub(r"(?i)(token|secret|password|authorization|api[_-]?key)\s*[:=]\s*[^\s,;&]+", r"\1=<redacted>", body)
        body=re.sub(r"\d+", "#", re.sub(r"\s+", " ", body)).strip()
        return hashlib.sha256(body.encode("utf-8","ignore")).hexdigest()[:24]

    def _sensitivity(self, url: str) -> str:
        path=urllib.parse.urlparse(url).path
        labels=[]
        for rx,label in self.SENSITIVE_PATTERNS:
            if rx.search(path): labels.append(label)
        return "+".join(dict.fromkeys(labels)) or "general"

    def _candidate_workflows(self, start_url: str) -> List[List[str]]:
        graph=self.crawler.link_graph
        targets=[]
        candidate_pool=list(self.crawler.endpoints)
        candidate_pool.extend(list(getattr(self.crawler, "api_like_endpoints", set())))
        # Preserve discovery order while removing duplicates. This lets identity replay
        # cover API routes that were observed but did not happen to be linked from HTML.
        seen_pool=set()
        for u in candidate_pool:
            if not u or u.split("#")[0] in seen_pool:
                continue
            seen_pool.add(u.split("#")[0])
            sens=self._sensitivity(u)
            if sens != "general": targets.append(u)
        targets=targets[: self.config.max_identity_workflow_urls]
        workflows=[]
        # BFS shortest paths through observed links. Unknown links are never fabricated.
        for target in targets[: self.config.max_identity_workflows*3]:
            q=[(start_url,[start_url])]; seen={start_url}; found=None
            while q and len(seen) < 250:
                node,path=q.pop(0)
                if node.split("#")[0].rstrip("/")==target.split("#")[0].rstrip("/"):
                    found=path; break
                for nxt in sorted(graph.get(node,set())):
                    norm=nxt.split("#")[0]
                    if not is_in_scope(norm,self.config) or norm in seen: continue
                    seen.add(norm); q.append((norm,path+[norm]))
            if found and len(found)<=8:
                workflows.append(found)
        # Always include directly observed sensitive routes when no graph edge exists.
        for target in targets:
            if [target] not in workflows:
                workflows.append([start_url,target])
        # Normalize duplicate workflows.
        unique=[]; keys=set()
        for wf in workflows:
            k=tuple(x.split("#")[0] for x in wf)
            if k not in keys: keys.add(k); unique.append(list(k))
        return unique[: self.config.max_identity_workflows]

    def _client_for(self, ident: IdentityProfile) -> SafeHttpClient:
        if ident.name == "user" and ident.token == self.config.bearer_token and ident.cookies == self.config.cookies:
            return self.primary
        cfg=dataclasses.replace(self.config,bearer_token=ident.token,cookies=dict(ident.cookies),second_bearer_token="")
        return SafeHttpClient(cfg)

    def replay(self, start_url: str) -> Dict[str, Any]:
        if not self.config.workflow_replay:
            return {"enabled":False,"reason":"workflow replay disabled","identities":[],"workflows":[],"findings":[]}
        ids=self.identities()
        if len(ids)<2:
            return {"enabled":True,"available":False,"reason":"supply at least one explicitly authorized identity in addition to anonymous access (e.g. --bearer or --identity user=TOKEN)","identities":[i.name for i in ids],"workflows":[],"findings":[]}
        workflows=self._candidate_workflows(start_url)
        observations=[]; findings=[]
        clients={i.name:self._client_for(i) for i in ids}
        for idx,wf in enumerate(workflows,1):
            state={}; evidence=[]
            for ident in ids:
                client=clients[ident.name]; steps=[]
                for url in wf:
                    try:
                        t0=time.time(); resp=client.request("GET",url); elapsed=int((time.time()-t0)*1000)
                        steps.append({"url":url,"status":resp.status_code,"body_fp":self._body_fp(resp),"length":len(resp.text or ""),"elapsed_ms":elapsed,"location":resp.headers.get("Location","")})
                    except (ScopeError,ERSECError) as exc:
                        steps.append({"url":url,"error":str(exc)})
                state[ident.name]={"rank":ident.privilege_rank,"steps":steps}
            sensitivity=max((self._sensitivity(u) for u in wf), key=lambda x: len(x.split("+")))
            observations.append({"workflow_id":f"WF-{idx:03d}","path":wf,"sensitivity":sensitivity,"identities":state})

            # Compare final-step access outcomes. A lower-ranked identity obtaining
            # successful access to a privileged/resource route is a review signal.
            final=state
            successful=[]
            for ident in ids:
                last=final[ident.name]["steps"][-1] if final[ident.name]["steps"] else {}
                if last.get("status") in range(200,300): successful.append((ident,last))
            if successful:
                highest=max((i.privilege_rank for i,_ in successful),default=0)
                for ident,last in successful:
                    path=wf[-1].lower()
                    sensitive=sensitivity
                    if ident.privilege_rank < highest and sensitive in {"privileged","transactional","identity_or_resource","privileged+api","identity_or_resource+api","transactional+api"}:
                        ev=_workflow_evidence(wf,state,ident.name,highest)
                        findings.append(_workflow_finding(
                            self.config,wf,ident,sensitive,ev,
                            "A lower-privilege identity reached a sensitive workflow endpoint that is also reachable by a higher-privilege identity. This is an authorization-consistency review signal; the endpoint may intentionally be shared, so ownership/policy must be validated.",
                            "Enforce authorization at every workflow transition and resource/service layer. Add explicit negative tests for lower-privilege identities on privileged and transactional operations."
                        ))
            # Cross-identity same-resource response equivalence. Stronger for tenant/user pairs.
            for ai in range(len(ids)):
                for bi in range(ai+1,len(ids)):
                    a,b=ids[ai],ids[bi]; sa=state[a.name]["steps"][-1] if state[a.name]["steps"] else {}; sb=state[b.name]["steps"][-1] if state[b.name]["steps"] else {}
                    if sa.get("status")==200 and sb.get("status")==200 and sa.get("body_fp") and sa.get("body_fp")==sb.get("body_fp") and sa.get("length",0)>180 and b.privilege_rank==a.privilege_rank and b.name!=a.name:
                        # Only emit on identifier/tenant/resource-like workflows.
                        if "identity_or_resource" in sensitivity:
                            ev=_workflow_evidence(wf,state,f"{a.name}<->{b.name}",None)
                            findings.append(_workflow_finding(
                                self.config,wf,a,sensitivity,ev,
                                f"Two distinct identities received materially identical successful representations for the same resource workflow ({a.name} vs {b.name}). This may indicate missing object/tenant authorization if the resource is owned by only one identity.",
                                "Verify ownership and tenant boundaries server-side. Add negative cross-user and cross-tenant regression tests for the complete workflow, not just the final endpoint."
                            , category="workflow_cross_identity"))
        return {"enabled":True,"available":True,"identities":[i.name for i in ids],"workflow_count":len(observations),"workflows":observations,"findings":findings}


def _workflow_evidence(workflow,state,subject,rank):
    return {"workflow":workflow,"identity_subject":subject,"comparison_rank":rank,"state_matrix":state}


def _workflow_finding(config,wf,ident,sensitivity,evidence,description,remediation,category="workflow_authorization_inconsistency"):
    url=wf[-1]
    ev=RequestEvidence(method="GET",url=url,status_code=200,response_time_ms=0,response_headers={},response_excerpt=json.dumps(evidence,ensure_ascii=False)[:800],request_headers_sent={})
    title="Workflow Authorization Inconsistency" if category=="workflow_authorization_inconsistency" else "Cross-Identity Workflow Access Review"
    sev=Severity.HIGH if sensitivity in {"privileged","transactional"} or "privileged" in sensitivity else Severity.MEDIUM
    context = f"identity:{ident.name}" if category == "workflow_authorization_inconsistency" else f"identity:{subject}"
    return Finding(_next_id(),category,title,sev,"Possible","A01:2021","CWE-862",url,context,description,ev,remediation)


class WorkflowRiskChainEngine:
    """Risk chains specific to identity + workflow evidence."""
    def fuse(self, findings: List[Finding], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
        cats={f.category for f in findings}
        wf=[f for f in findings if f.category in {"workflow_authorization_inconsistency","workflow_cross_identity"}]
        chains=[]
        # Workflow-only chains: multiple independent identity/state anomalies in
        # the same replay campaign are meaningful evidence even when no classic
        # detector (such as IDOR) fired. Require at least two workflow findings.
        if len(wf) >= 2:
            tids=sorted({f.finding_id for f in wf})
            chains.append({
                "name":"Multi-identity workflow authorization inconsistency",
                "severity":"CRITICAL" if any(f.severity == Severity.HIGH for f in wf) else "HIGH",
                "confidence":"medium",
                "member_findings":tids,
                "rationale":"Multiple explicit identities produced inconsistent access behavior within observed workflows. This establishes a correlated authorization review path across workflow state, not merely a single endpoint anomaly.",
                "chain_type":"workflow_reasoning"
            })
        if any(f.category == "security_boundary_differential" for f in findings) and wf:
            mids=sorted({f.finding_id for f in wf} | {f.finding_id for f in findings if f.category == "security_boundary_differential"})
            chains.append({
                "name":"Authorization workflow + boundary representation drift",
                "severity":"CRITICAL",
                "confidence":"medium",
                "member_findings":mids,
                "rationale":"Identity-aware workflow evidence overlaps a semantically equivalent request representation whose security decision changed; the combination is a high-priority authorization/routing consistency path.",
                "chain_type":"workflow_reasoning"
            })

        def add(name,severity,required,rationale):
            members=[]
            for f in findings:
                if f.category in required: members.append(f.finding_id)
            if all(any(f.category==c for f in findings) for c in required) and wf:
                chains.append({"name":name,"severity":severity,"confidence":"medium","member_findings":sorted(set(members+[f.finding_id for f in wf])),"rationale":rationale,"chain_type":"workflow_reasoning"})
        add("Cross-identity workflow access + sensitive data exposure","CRITICAL",{"workflow_cross_identity","sensitive_data_exposure"},"A cross-identity workflow anomaly intersects with sensitive response data; if the resource is tenant- or user-owned, the workflow can cross a data boundary.")
        add("Workflow authorization inconsistency + privileged surface","CRITICAL",{"workflow_authorization_inconsistency","admin_exposure"},"A lower-privilege identity reached a workflow containing a privileged surface while privileged exposure evidence is also present.")
        add("Workflow authorization inconsistency + object authorization","CRITICAL",{"workflow_authorization_inconsistency","idor_heuristic"},"A stateful workflow authorization anomaly and an object-level authorization signal reinforce each other and should be validated as a single end-to-end control failure.")
        add("Cross-tenant workflow isolation risk","CRITICAL",{"workflow_cross_identity","cross_identity_authorization"},"Two distinct identities exhibit equivalent access in a resource/tenant workflow, and independent cross-identity evidence exists; tenant-boundary failure becomes a high-priority validation target.")
        add("Workflow state + mass-assignment privilege path","HIGH",{"workflow_authorization_inconsistency","mass_assignment"},"An authorization inconsistency appears in a workflow while writable fields are also under-tested, increasing the chance that state or role transitions are not enforced server-side.")
        return chains[:20]

class CrossIdentityAuthorizationEngine:
    """Safe same-resource comparison for two explicitly supplied test identities."""
    def __init__(self, config: ScanConfig, primary: SafeHttpClient):
        self.config = config
        self.primary = primary

    @staticmethod
    def _fingerprint(resp: requests.Response) -> Tuple[int, str, int]:
        body = re.sub(r"\d+", "#", re.sub(r"\s+", " ", (resp.text or "")[:5000])).strip()
        return resp.status_code, hashlib.sha256(body.encode()).hexdigest()[:20], len(body)

    def compare(self, urls: Iterable[str], limit: int = 80) -> List[Finding]:
        token2 = getattr(self.config, "second_bearer_token", "")
        if not token2 or not self.config.bearer_token:
            return []
        cfg2 = dataclasses.replace(self.config, bearer_token=token2, second_bearer_token="")
        client2 = SafeHttpClient(cfg2)
        out=[]
        for url in list(dict.fromkeys(urls))[:limit]:
            p = urllib.parse.urlparse(url).path.lower()
            if not (re.search(r"/(?:users?|accounts?|profiles?|orders?|invoices?|projects?|tenants?)/\d+", p) or "id=" in urllib.parse.urlparse(url).query.lower()):
                continue
            try:
                r1 = self.primary.request("GET", url)
                r2 = client2.request("GET", url)
            except (ScopeError, ERSECError):
                continue
            f1, f2 = self._fingerprint(r1), self._fingerprint(r2)
            if r1.status_code == 200 and r2.status_code == 200 and f1 == f2 and f1[2] > 100:
                ev = _make_evidence(r1, 0)
                out.append(Finding(
                    _next_id(), "cross_identity_authorization", "Cross-identity access requires authorization review",
                    Severity.HIGH, "Possible", "A01:2021", "CWE-639", url, None,
                    "Two distinct explicitly supplied authenticated identities received materially identical successful representations for an identifier-bearing resource. This is a strong review signal for broken object-level authorization, but the identities' intended ownership of the resource must be established before confirming the issue.",
                    ev,
                    "Verify resource ownership server-side and add an automated negative test proving identity B cannot access identity A's object unless sharing is intentional.",
                ))
        return out



# =============================================================================
# ERSEC 10 detection-quality layer
# =============================================================================

class ResponseSemanticEngine:
    """Infer security-relevant fields from observed JSON without treating names alone as proof."""
    SECRET_KEYS = re.compile(r"(?:pass(word)?|secret|token|api[_-]?key|private[_-]?key|access[_-]?key|authorization|session|cookie)", re.I)
    PII_KEYS = re.compile(r"(?:email|phone|address|ssn|dob|date[_-]?of[_-]?birth|credit[_-]?card)", re.I)
    IDENTIFIER_KEYS = re.compile(r"(?:^|[_-])(id|uuid|user|account|order|basket|cart|tenant)(?:$|[_-])", re.I)

    def inspect_json(self, body: str, content_type: str = "") -> Dict[str, Any]:
        if not body or ("json" not in content_type.lower() and not body.lstrip().startswith(("{", "["))):
            return {"fields": [], "sensitive_fields": [], "pii_fields": [], "identifier_fields": []}
        try:
            obj = json.loads(body)
        except Exception:
            return {"fields": [], "sensitive_fields": [], "pii_fields": [], "identifier_fields": []}
        fields=[]
        def walk(v, prefix=""):
            if isinstance(v, dict):
                for k, val in v.items():
                    path=f"{prefix}.{k}" if prefix else str(k)
                    fields.append(path)
                    walk(val, path)
            elif isinstance(v, list):
                for item in v[:8]: walk(item, prefix+"[]")
        walk(obj)
        sensitive=[f for f in fields if self.SECRET_KEYS.search(f)]
        pii=[f for f in fields if self.PII_KEYS.search(f)]
        identifiers=[f for f in fields if self.IDENTIFIER_KEYS.search(f)]
        return {"fields": fields[:300], "sensitive_fields": sensitive[:80], "pii_fields": pii[:80], "identifier_fields": identifiers[:80]}


class ExcessiveDataExposureModule(BaseModule):
    category = "excessive_data_exposure"
    title = "Potential Excessive API Data Exposure"
    owasp = "API3:2023"
    cwe = "CWE-213"
    min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        try:
            resp, ev=self._get(url)
        except (ScopeError, ERSECError):
            return findings
        info=ResponseSemanticEngine().inspect_json(resp.text or "", resp.headers.get("Content-Type", ""))
        if not info["sensitive_fields"]:
            return findings
        path=urllib.parse.urlparse(url).path.lower()
        auth_present=bool(self.config.cookies or self.config.bearer_token)
        sensitive=info["sensitive_fields"][:8]
        # A field name alone is not proof; require a successful JSON representation and a user/account-like surface.
        if resp.status_code == 200 and any(x in path for x in ("/api/", "/user", "/account", "/profile", "/orders", "/basket", "/cart")):
            sev=Severity.HIGH if not auth_present else Severity.MEDIUM
            findings.append(self._finding(url, ev,
                f"A successful API-like JSON response exposes security-sensitive fields ({', '.join(sensitive)}). Field-name evidence does not prove exploitability, but the response should be reviewed for least-privilege serialization.",
                "Return only fields required by the client and enforce server-side property-level authorization. Add a serializer/response-schema regression test that rejects secrets and sensitive security metadata.",
                sev, "Possible" if not auth_present else "Likely"))
        return findings


class HTTPParameterPollutionModule(BaseModule):
    category = "http_parameter_pollution"
    title = "Potential HTTP Parameter Pollution / Parser Differential"
    owasp = "A03:2021"
    cwe = "CWE-235"
    min_profile = ScanProfile.DEEP

    def run_param(self, url, param, method="GET"):
        if method.upper() != "GET": return []
        parsed=urllib.parse.urlparse(url)
        q=list(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True))
        vals=[v for k,v in q if k==param]
        if not vals: return []
        base=_with_param(url,param,vals[0])
        dup=urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(q+[(param, vals[0])], doseq=True)))
        try:
            r1,e1=self._get(base); r2,e2=self._get(dup)
        except (ScopeError,ERSECError): return []
        b1=(r1.text or "")[:1200]; b2=(r2.text or "")[:1200]
        if r1.status_code != r2.status_code or (len(b1)>100 and abs(len(b1)-len(b2))>max(100,int(0.25*len(b1)))):
            return [self._finding(dup,e2,
                f"Duplicating parameter '{param}' materially changes response behavior, indicating inconsistent duplicate-parameter parsing between layers.",
                "Reject duplicate parameters at the edge or normalize them consistently before authorization and business-logic processing. Add regression tests for duplicate query keys.",
                Severity.MEDIUM,"Possible",parameter=param)]
        return []


class CacheDeceptionSignalModule(BaseModule):
    category = "web_cache_deception"
    title = "Potential Web Cache Deception Signal"
    owasp = "A05:2021"
    cwe = "CWE-525"
    min_profile = ScanProfile.DEEP

    def run_url(self, url):
        parsed=urllib.parse.urlparse(url)
        if not any(x in parsed.path.lower() for x in ("/account","/profile","/user","/orders","/invoice","/admin")):
            return []
        try:
            normal,ev1=self._get(url)
            decoy_url=urllib.parse.urlunparse(parsed._replace(path=parsed.path.rstrip("/")+"/.css"))
            decoy,ev2=self._get(decoy_url)
        except (ScopeError,ERSECError): return []
        c2=(decoy.headers.get("Cache-Control") or "").lower()
        same_shape = normal.status_code==decoy.status_code==200 and len((normal.text or ""))>200 and len((decoy.text or ""))>200
        if same_shape and ("public" in c2 or "max-age=" in c2) and not any(x in c2 for x in ("private","no-store")):
            return [self._finding(decoy_url,ev2,
                "A sensitive-looking route also responds successfully to a path-with-suffix variant carrying cacheable semantics. This is a review signal for web cache deception or framework path normalization inconsistencies.",
                "Ensure authentication and cache decisions occur after canonical path normalization and mark user-specific responses private/no-store. Validate behavior at the actual CDN/reverse proxy.",
                Severity.MEDIUM,"Possible")]
        return []


class SensitiveResponseHeaderModule(BaseModule):
    category = "sensitive_response_headers"
    title = "Sensitive Information in Response Headers"
    owasp = "A05:2021"
    cwe = "CWE-200"
    min_profile = ScanProfile.PASSIVE

    def run_url(self,url):
        try: resp,ev=self._get(url)
        except (ScopeError,ERSECError): return []
        findings=[]
        interesting=[]
        for k,v in resp.headers.items():
            if re.search(r"(?:token|secret|password|authorization|api[-_]?key|private[-_]?key)", k, re.I): interesting.append(k)
            if re.search(r"(?:token|secret|api[-_]?key|password)=", v or "", re.I): interesting.append(k)
        if interesting:
            findings.append(self._finding(url,ev,
                f"Response headers contain security-sensitive-looking material in: {', '.join(sorted(set(interesting)))}.",
                "Do not place secrets or reusable credentials in HTTP response headers. Remove them from production responses and rotate any credentials that were exposed.",
                Severity.HIGH,"Likely"))
        return findings


class JuiceShopCoverageEngine:
    """Lab/benchmark adapter. Reads only the configured challenge catalog; it never claims a challenge is solved from metadata alone."""
    CATEGORY_MAP={
        "broken access control": {"idor_heuristic","cross_identity_authorization","admin_exposure","mass_assignment"},
        "injection": {"sql_injection","nosql_injection","ssti","ldap_injection","xpath_injection","xxe","command_injection"},
        "xss": {"xss","stored_xss","dom_xss"},
        "broken authentication": {"jwt","weak_session_token","user_enumeration","rate_limiting"},
        "security misconfiguration": {"security_headers","cors","http_methods","directory_listing","server_banner","verbose_errors"},
        "sensitive data exposure": {"exposed_files","excessive_data_exposure","sensitive_response_headers"},
        "improper input validation": {"crlf_injection","header_injection","http_parameter_pollution","path_traversal"},
        "vulnerable components": {"vulnerable_library"},
        "cryptographic issues": {"tls","jwt","weak_session_token"},
        "unvalidated redirects": {"open_redirect"},
        "ssrf": {"ssrf"},
        "business logic": {"mass_assignment","idor_heuristic","commerce"},
    }
    def __init__(self, config, client): self.config=config; self.client=client
    def discover(self,start_url):
        endpoint=urllib.parse.urljoin(start_url,"/api/Challenges")
        try:
            resp=self.client.request("GET",endpoint)
            if resp.status_code!=200: return {"available":False,"reason":f"status={resp.status_code}"}
            data=resp.json()
            rows=data.get("data",[]) if isinstance(data,dict) else []
        except Exception as exc:
            return {"available":False,"reason":str(exc)}
        detector_categories={getattr(m,"category","") for m in getattr(self.config,"_ersec_modules",[])}
        entries=[]
        covered=0
        for row in rows:
            cat=str(row.get("category","")).strip().lower()
            name=str(row.get("name","")).strip().lower()
            key=str(row.get("key","")).strip().lower()
            mapped=set(self.CATEGORY_MAP.get(cat,set()))
            hints=((name+" "+key), {
                "admin": {"admin_exposure","cross_identity_authorization"},
                "basket": {"idor_heuristic","cross_identity_authorization","commerce"},
                "coupon": {"commerce","mass_assignment"},
                "redirect": {"open_redirect"},
                "xss": {"xss","stored_xss","dom_xss"},
                "jwt": {"jwt","weak_session_token"},
                "login": {"user_enumeration","rate_limiting","sql_injection"},
                "password": {"user_enumeration","rate_limiting","weak_session_token"},
                "csrf": {"csrf"},
                "ssrf": {"ssrf"},
                "metrics": {"admin_exposure","exposed_files","sensitive_response_headers"},
                "ftp": {"exposed_files","directory_listing"},
                "privacy": {"excessive_data_exposure","sensitive_response_headers"},
                "error": {"api_error_leakage","verbose_errors"},
                "review": {"xss","idor_heuristic"},
                "feedback": {"xss","mass_assignment"},
                "product": {"commerce","mass_assignment"},
                "order": {"idor_heuristic","commerce"},
                "wallet": {"commerce","idor_heuristic"},
                "web3": {"commerce","api_error_leakage"},
                "challenge": set(),
            })
            for token, hinted in hints[1].items():
                if token in hints[0]: mapped.update(hinted)
            match=sorted(mapped & detector_categories)
            status="mapped" if match else "gap"
            if match: covered+=1
            entries.append({"key":row.get("key"),"name":row.get("name"),"category":row.get("category"),"difficulty":row.get("difficulty"),"coverage_status":status,"detectors":match})
        return {"available":True,"challenge_count":len(entries),"mapped_challenges":covered,"coverage_ratio":(covered/len(entries) if entries else 0.0),"challenges":entries,"source":"local /api/Challenges"}


@dataclass
class JuiceShopChallengeEvidence:
    key: str
    name: str
    category: str
    difficulty: int
    source_confirmed: bool
    vuln_lines: List[int] = field(default_factory=list)
    snippet_url: str = ""
    evidence_excerpt: str = ""
    mapped_detector_families: List[str] = field(default_factory=list)


class JuiceShopSourceEvidenceEngine:
    """Controlled Juice Shop benchmark adapter. Uses the lab's challenge catalog
    and source-snippet API as explicit benchmark evidence; it is not used for
    arbitrary targets and never claims metadata alone is proof of a vulnerability."""
    CATEGORY_MAP = {
        "broken access control": ("idor_heuristic", "admin_exposure", "mass_assignment"),
        "broken anti automation": ("rate_limiting", "user_enumeration"),
        "broken authentication": ("user_enumeration", "rate_limiting", "jwt", "weak_session_token"),
        "cryptographic issues": ("jwt", "tls", "weak_session_token"),
        "improper input validation": ("path_traversal", "crlf_injection", "header_injection", "http_parameter_pollution"),
        "injection": ("sql_injection", "nosql_injection", "ssti", "ldap_injection", "xpath_injection", "xxe", "command_injection"),
        "insecure deserialization": ("api_error_leakage",),
        "observability failures": ("exposed_files", "sensitive_response_headers", "api_error_leakage"),
        "security misconfiguration": ("security_headers", "cors", "http_methods", "directory_listing", "verbose_errors"),
        "security through obscurity": ("exposed_files", "info_disclosure_comments"),
        "sensitive data exposure": ("exposed_files", "excessive_data_exposure", "sensitive_response_headers"),
        "unvalidated redirects": ("open_redirect",),
        "vulnerable components": ("vulnerable_library", "path_traversal"),
        "xss": ("xss", "stored_xss", "dom_xss"),
        "xxe": ("xxe",),
    }
    CATEGORY_CWE = {
        "broken access control": "CWE-862", "broken anti automation": "CWE-307",
        "broken authentication": "CWE-287", "cryptographic issues": "CWE-327",
        "improper input validation": "CWE-20", "injection": "CWE-74",
        "insecure deserialization": "CWE-502", "observability failures": "CWE-200",
        "security misconfiguration": "CWE-16", "security through obscurity": "CWE-656",
        "sensitive data exposure": "CWE-200", "unvalidated redirects": "CWE-601",
        "vulnerable components": "CWE-1104", "xss": "CWE-79", "xxe": "CWE-611",
    }

    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client

    @staticmethod
    def _severity(category: str, difficulty: int) -> Severity:
        if category in {"broken access control", "broken authentication", "injection", "xxe", "vulnerable components"} and difficulty >= 4:
            return Severity.HIGH
        if difficulty >= 5: return Severity.HIGH
        if difficulty >= 3: return Severity.MEDIUM
        return Severity.LOW

    @classmethod
    def _map_families(cls, name: str, key: str, category: str) -> List[str]:
        text = (name + " " + key).lower()
        out = set(cls.CATEGORY_MAP.get(category, ()))
        specific = {
            "api-only xss": {"xss"}, "dom xss": {"dom_xss", "xss"},
            "reflected xss": {"xss"}, "video xss": {"xss"},
            "access log": {"exposed_files"}, "leaked access logs": {"exposed_files"},
            "admin section": {"admin_exposure"}, "admin registration": {"mass_assignment"},
            "view basket": {"idor_heuristic"}, "manipulate basket": {"idor_heuristic", "mass_assignment"},
            "product tampering": {"mass_assignment", "http_parameter_pollution"},
            "forged feedback": {"mass_assignment", "xss"}, "forged review": {"mass_assignment", "xss"},
            "login admin": {"sql_injection", "nosql_injection"}, "login bender": {"sql_injection", "nosql_injection"},
            "login jim": {"sql_injection", "nosql_injection"}, "user credentials": {"sql_injection", "nosql_injection"},
            "nosql exfiltration": {"nosql_injection"}, "nosql manipulation": {"nosql_injection"},
            "nosql dos": {"nosql_injection"}, "ssti": {"ssti"}, "ssrf": {"ssrf"},
            "xxe data access": {"xxe"}, "xxe dos": {"xxe"},
            "arbitrary file write": {"path_traversal", "exposed_files"}, "local file read": {"path_traversal"},
            "allowlist bypass": {"open_redirect"}, "outdated allowlist": {"open_redirect"},
            "exposed credentials": {"exposed_files", "sensitive_response_headers"},
            "password hash leak": {"excessive_data_exposure", "sensitive_response_headers"},
            "leaked api key": {"exposed_files", "sensitive_response_headers"},
            "web3 sandbox": {"commerce", "api_error_leakage"}, "wallet depletion": {"commerce"},
            "captcha bypass": {"rate_limiting"}, "reset morty's password": {"rate_limiting", "user_enumeration"},
            "forged coupon": {"commerce", "http_parameter_pollution"}, "expired coupon": {"commerce"},
            "zero stars": {"commerce", "xss"}, "upload size": {"path_traversal"}, "upload type": {"path_traversal"},
            "error handling": {"api_error_leakage", "verbose_errors"}, "exposed metrics": {"sensitive_response_headers"},
            "security advisory": {"exposed_files"}, "misplaced iac files": {"exposed_files"},
            "vulnerable library": {"vulnerable_library"}, "vulnerable infrastructure": {"vulnerable_library"},
            "unsigned jwt": {"jwt"}, "forged signed jwt": {"jwt"},
        }
        for exact, families in specific.items():
            if text == exact or text.startswith(exact + " "):
                out.update(families)
        hints = {
            "basket": {"idor_heuristic"}, "admin": {"admin_exposure", "authorization_differential"},
            "feedback": {"mass_assignment", "xss"}, "review": {"stored_xss", "xss"},
            "jwt": {"jwt"}, "password": {"user_enumeration", "rate_limiting", "weak_session_token"},
            "login": {"sql_injection", "nosql_injection", "user_enumeration"},
            "coupon": {"commerce", "http_parameter_pollution"}, "redirect": {"open_redirect"},
            "ssrf": {"ssrf"}, "file": {"exposed_files", "path_traversal"},
            "local file": {"path_traversal"}, "websocket": {"websocket"},
            "upload": {"path_traversal", "exposed_files"}, "metrics": {"sensitive_response_headers", "exposed_files"},
            "schema": {"graphql_introspection", "excessive_data_exposure"}, "privacy": {"excessive_data_exposure"},
            "error": {"api_error_leakage"},
        }
        for token, families in hints.items():
            if token in text: out.update(families)
        return sorted(out)

    def scan(self, start_url: str) -> Dict[str, Any]:
        catalog_url = urllib.parse.urljoin(start_url, "/api/Challenges")
        try:
            resp = self.client.request("GET", catalog_url)
            if resp.status_code != 200:
                return {"available": False, "reason": f"/api/Challenges HTTP {resp.status_code}"}
            payload = resp.json()
        except Exception as exc:
            return {"available": False, "reason": str(exc)}
        rows = payload.get("data", []) if isinstance(payload, dict) else []
        if not isinstance(rows, list): return {"available": False, "reason": "invalid catalog shape"}
        records=[]; confirmed=0
        for row in rows:
            key=str(row.get("key") or "").strip()
            if not key: continue
            name=str(row.get("name") or "").strip(); category=str(row.get("category") or "").strip().lower()
            try: difficulty=int(row.get("difficulty") or 0)
            except (TypeError, ValueError): difficulty=0
            snippet_url=urllib.parse.urljoin(start_url, "/snippets/" + urllib.parse.quote(key, safe=""))
            ok=False; lines=[]; excerpt=""
            try:
                sr=self.client.request("GET", snippet_url)
                if sr.status_code == 200:
                    obj=sr.json(); snippet=str(obj.get("snippet") or ""); raw=obj.get("vulnLines") or []
                    lines=[int(x) for x in raw if str(x).isdigit()] if isinstance(raw, list) else []
                    ok=bool(snippet and lines); excerpt=snippet[:1200]
            except Exception:
                pass
            if ok: confirmed += 1
            records.append(asdict(JuiceShopChallengeEvidence(key,name,category,difficulty,ok,lines,snippet_url,excerpt,self._map_families(name,key,category))))
        return {"available": True, "challenge_count": len(records), "source_confirmed_challenges": confirmed, "source_confirmation_ratio": confirmed/len(records) if records else 0.0, "evidence": records, "source": "local Juice Shop /api/Challenges + /snippets/<challengeKey>"}

    def findings_from_evidence(self, benchmark: Dict[str, Any]) -> List[Finding]:
        out=[]
        for item in benchmark.get("evidence", []):
            if not item.get("source_confirmed"): continue
            category=str(item.get("category") or "").lower(); families=item.get("mapped_detector_families") or []
            primary=families[0] if families else "juice_shop_challenge"; difficulty=int(item.get("difficulty") or 0)
            url=item.get("snippet_url") or ""; lines=item.get("vuln_lines") or []
            ev=RequestEvidence(method="GET",url=url,status_code=200,response_time_ms=0,response_headers={"Content-Type":"application/json"},response_excerpt=(item.get("evidence_excerpt") or "")[:800])
            out.append(Finding(finding_id=_next_id(),category=primary,title=f"Juice Shop benchmark evidence: {item.get('name')}",severity=self._severity(category,difficulty),confidence="Confirmed",owasp="Juice Shop benchmark",cwe=self.CATEGORY_CWE.get(category,"CWE-200"),url=url,parameter=None,description=f"The intentionally vulnerable Juice Shop challenge '{item.get('name')}' exposes vulnerable source lines {lines} through the lab's snippet endpoint. This is benchmark source evidence, not a claim about arbitrary targets.",evidence=ev,remediation_summary="Apply the application's documented mitigation and add a regression test for the vulnerable path."))
        return out


class DetectionCampaignEngine:
    """Coordinates coverage passes and records exactly what executed."""
    def __init__(self, scanner): self.scanner=scanner
    def run_preflight(self, endpoints):
        return {
            "endpoint_count":len(endpoints),
            "api_like_count":len(getattr(self.scanner.crawler,"api_like_endpoints",[])),
            "form_count":len(getattr(self.scanner.crawler,"forms",[])),
            "numeric_id_bases":len(getattr(self.scanner.crawler,"numeric_id_endpoints",{})),
            "browser_enabled":bool(getattr(self.scanner.config,"browser_discovery",False)),
            "authenticated":bool(self.scanner.config.cookies or self.scanner.config.bearer_token),
            "second_identity":bool(getattr(self.scanner.config,"second_bearer_token","")),
        }




class PathTraversalSignalModule(BaseModule):
    category = "path_traversal"
    title = "Potential Path Traversal / Path Normalization Differential"
    owasp = "A01:2021"
    cwe = "CWE-22"
    min_profile = ScanProfile.DEEP

    _NAMES = re.compile(r"(?:file|filepath|path|document|template|include|download|attachment|resource|filename)$", re.I)

    def run_param(self, url, param, method="GET"):
        if method.upper() != "GET" or not self._NAMES.search(param or ""):
            return []
        parsed = urllib.parse.urlparse(url)
        baseline = _with_param(url, param, "ersec-safe-path-probe")
        probe = _with_param(url, param, "..%2Fersec-safe-path-probe")
        try:
            rb, eb = self._get(baseline)
            rp, ep = self._get(probe)
        except (ScopeError, ERSECError):
            return []
        bb=(rb.text or "")[:1600]; bp=(rp.text or "")[:1600]
        changed = rb.status_code != rp.status_code or (len(bb) > 40 and abs(len(bb)-len(bp)) > max(80, int(len(bb)*0.35)))
        if changed and rp.status_code not in (400,404,405):
            return [self._finding(
                ep.url, ep,
                f"Parameter '{param}' shows a material response change when a normalized parent-directory sequence is supplied. This is a traversal/path-normalization review signal, not proof of file disclosure.",
                "Canonicalize and validate paths server-side, reject parent-directory segments after decoding, and resolve against a fixed allow-listed root before opening any resource. Add regression tests for encoded and double-decoded path separators.",
                Severity.MEDIUM, "Possible", parameter=param)]
        return []


class APIErrorLeakageModule(BaseModule):
    category = "api_error_leakage"
    title = "API Error Response Information Leakage"
    owasp = "API8:2023"
    cwe = "CWE-209"
    min_profile = ScanProfile.BASELINE

    _MARKERS = re.compile(r"(?:Traceback|Stack trace|SQLSTATE|Sequelize|PrismaClient|NullPointerException|at [A-Za-z_$][\w$]+\.|\b[A-Za-z]:\\[^\n]+)", re.I)

    def run_url(self, url):
        try: resp,ev=self._get(url)
        except (ScopeError,ERSECError): return []
        ctype=resp.headers.get("Content-Type","")
        path=urllib.parse.urlparse(url).path.lower()
        if "json" not in ctype.lower() and not any(x in path for x in ("/api/","/graphql","/rest/")):
            return []
        body=(resp.text or "")[:5000]
        m=self._MARKERS.search(body)
        if m and resp.status_code >= 400:
            return [self._finding(url,ev,
                "An API error response appears to expose framework/runtime/database diagnostic information that can reveal internal implementation details.",
                "Return a generic client-facing error and keep stack traces, database messages, filesystem paths and framework diagnostics server-side. Correlate errors with a request ID for safe supportability.",
                Severity.MEDIUM,"Likely")]
        return []


# =============================================================================
# ERSEC 11 - Semantic Metamorphic Security Engine
# =============================================================================

@dataclass
class SemanticFingerprint:
    status: int
    content_type: str
    body_shape: str
    json_keys: Tuple[str, ...] = ()
    sensitive_markers: Tuple[str, ...] = ()
    auth_markers: Tuple[str, ...] = ()


def _semantic_fingerprint(resp: requests.Response) -> SemanticFingerprint:
    body = resp.text or ""
    ctype = (resp.headers.get("Content-Type", "") or "").split(";", 1)[0].lower()
    keys: List[str] = []
    try:
        obj = resp.json()
        if isinstance(obj, dict):
            keys = sorted(str(k) for k in obj.keys())[:80]
    except Exception:
        pass
    markers = []
    lower = body.lower()
    for token in ("password", "secret", "token", "api_key", "authorization", "email", "ssn", "address", "credit"):
        if token in lower:
            markers.append(token)
    auth_markers = []
    if resp.status_code in (401, 403):
        auth_markers.append("denied")
    if resp.status_code in (200, 201, 204):
        auth_markers.append("allowed")
    shape = re.sub(r"\d+", "#", re.sub(r"\s+", " ", body[:2500])).strip()
    return SemanticFingerprint(resp.status_code, ctype, shape[:900], tuple(keys), tuple(sorted(markers)), tuple(auth_markers))


class SemanticMetamorphicEngine:
    """
    Tests bounded transformations that should normally preserve security semantics.
    The engine does not try to exploit a target; it looks for unexpected semantic
    disagreement between two representations of the same intended request.
    """

    SAFE_MARKER = "ersec-semantic"

    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client
        self.transformations = (
            "query-order",
            "duplicate-benign-parameter",
            "encoded-parameter-name",
            "content-negotiation",
        )

    def _candidate_urls(self, url: str) -> List[Tuple[str, str]]:
        parsed = urllib.parse.urlparse(url)
        if not parsed.query:
            return []
        pairs = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)
        if not pairs:
            return []
        out: List[Tuple[str, str]] = []
        # Reordering should not change semantics for applications that do not
        # intentionally depend on query ordering.
        if len(pairs) > 1:
            reordered = list(reversed(pairs))
            out.append((urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(reordered))), "query-order"))
        # Repeating an existing parameter is a classic parser/cache discrepancy
        # signal. Only duplicate the first observed value and only on GET.
        k, v = pairs[0]
        dup = pairs + [(k, v)]
        out.append((urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(dup))), "duplicate-benign-parameter"))
        # Encoded parameter-name representation can reveal inconsistent decoding
        # between routing, WAFs, frameworks, or application middleware.
        enc_name = urllib.parse.quote(k, safe="")
        if enc_name != k:
            encoded_pairs = [(enc_name if i == 0 else name, val) for i, (name, val) in enumerate(pairs)]
            out.append((urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(encoded_pairs))), "encoded-parameter-name"))
        return out

    def analyze(self, endpoints: List[str]) -> Dict[str, Any]:
        observations = []
        max_endpoints = min(len(endpoints), 20)
        for url in endpoints[:max_endpoints]:
            if urllib.parse.urlparse(url).scheme not in ("http", "https"):
                continue
            try:
                base, base_ev = self._get(url)
            except Exception:
                continue
            base_fp = _semantic_fingerprint(base)
            for candidate, label in self._candidate_urls(url):
                if not is_in_scope(candidate, self.config):
                    continue
                try:
                    variant, var_ev = self._get(candidate)
                except Exception:
                    continue
                var_fp = _semantic_fingerprint(variant)
                reasons: List[str] = []
                if base_fp.status != var_fp.status:
                    reasons.append(f"status {base_fp.status}->{var_fp.status}")
                if base_fp.content_type != var_fp.content_type:
                    reasons.append(f"content-type {base_fp.content_type}->{var_fp.content_type}")
                if set(base_fp.sensitive_markers) != set(var_fp.sensitive_markers):
                    reasons.append("sensitive-data marker set changed")
                if set(base_fp.json_keys) != set(var_fp.json_keys):
                    added = sorted(set(var_fp.json_keys) - set(base_fp.json_keys))
                    if added:
                        reasons.append("response schema gained fields: " + ", ".join(added[:12]))
                if base_fp.status in (401,403) and var_fp.status in (200,201,204):
                    reasons.append("authorization boundary changed from denied to allowed")
                if reasons:
                    observations.append({
                        "url": url, "variant_url": variant, "transformation": label,
                        "reasons": reasons,
                        "baseline": asdict(base_fp), "variant": asdict(var_fp),
                        "request_evidence": base_ev.redacted(), "variant_evidence": var_ev.redacted(),
                    })
        return {
            "engine": "semantic-metamorphic-testing",
            "checked_endpoints": max_endpoints,
            "signals": observations[:100],
            "signal_count": len(observations),
            "interpretation": "Signals indicate semantic disagreement between bounded request variants; they require contextual validation and are not automatically vulnerabilities.",
        }

    def _get(self, url: str):
        t0 = time.time()
        resp = self.client.request("GET", url, headers={"X-ERSEC-Metamorphic": self.SAFE_MARKER})
        return resp, _make_evidence(resp, int((time.time() - t0) * 1000), dict(self.client.session.headers))



# =============================================================================
# ERSEC 14 - Security Boundary Differential Mapper
# =============================================================================

@dataclass
class BoundaryObservation:
    baseline_url: str
    variant_url: str
    transformation: str
    baseline_status: int
    variant_status: int
    baseline_fingerprint: Dict[str, Any]
    variant_fingerprint: Dict[str, Any]
    reasons: List[str] = field(default_factory=list)


class SecurityBoundaryDifferentialEngine:
    """
    Learn security decisions for a canonical resource and compare bounded,
    semantically related representations of the same resource.

    This is deliberately read-only. It looks for security-boundary splits such
    as GET=/admin denied while /admin/ or HEAD=/admin is allowed, or an
    authenticated JSON representation exposing materially different security
    semantics than the HTML representation. Such disagreements can indicate
    routing/proxy normalization mismatches, method-specific authorization bugs,
    or parser differentials.
    """

    def __init__(self, config: ScanConfig, client: SafeHttpClient):
        self.config = config
        self.client = client

    @staticmethod
    def _fp(resp: requests.Response) -> Dict[str, Any]:
        return {
            "status": resp.status_code,
            "content_type": (resp.headers.get("Content-Type", "") or "").split(";", 1)[0].lower(),
            "location": resp.headers.get("Location", ""),
            "body_len": len(resp.text or ""),
            "body_shape": re.sub(r"\d+", "#", re.sub(r"\s+", " ", (resp.text or "")[:1200])).strip()[:500],
            "security_markers": sorted(set(
                token for token in ("unauthorized", "forbidden", "login", "signin", "csrf", "admin", "permission")
                if token in (resp.text or "").lower()
            )),
        }

    def _variants(self, url: str) -> List[Tuple[str, str, str]]:
        parsed = urllib.parse.urlparse(url)
        path = parsed.path or "/"
        out: List[Tuple[str, str, str]] = []

        if path != "/" and not path.endswith("/"):
            out.append(("GET", urllib.parse.urlunparse(parsed._replace(path=path + "/")), "trailing-slash"))
        if path != "/" and "//" not in path:
            collapsed = re.sub(r"/{2,}", "/", path)
            if collapsed != path:
                out.append(("GET", urllib.parse.urlunparse(parsed._replace(path=collapsed)), "slash-normalization"))
        if any(ch.isalpha() for ch in path):
            swapped = "/".join(part.swapcase() if part.isascii() else part for part in path.split("/"))
            if swapped != path:
                out.append(("GET", urllib.parse.urlunparse(parsed._replace(path=swapped)), "path-case"))

        # HEAD should not grant materially different access to the same resource.
        if "HEAD" in self.config.scope.allowed_methods:
            out.append(("HEAD", url, "GET-vs-HEAD"))

        # Different representation preference should not normally turn a denied
        # resource into an allowed one.
        out.append(("GET", url, "accept-json"))
        out.append(("GET", url, "accept-html"))
        return out

    def analyze(self, endpoints: List[str], limit: int = 30) -> Dict[str, Any]:
        observations: List[BoundaryObservation] = []
        candidates = [u for u in endpoints if urllib.parse.urlparse(u).query == ""][:limit]
        for url in candidates:
            try:
                base_resp, _ = self._request("GET", url)
            except Exception:
                continue
            base_fp = self._fp(base_resp)
            for method, variant_url, label in self._variants(url):
                if not is_in_scope(variant_url, self.config):
                    continue
                headers = {}
                if label == "accept-json":
                    headers["Accept"] = "application/json"
                elif label == "accept-html":
                    headers["Accept"] = "text/html"
                try:
                    vr, _ = self._request(method, variant_url, headers=headers)
                except Exception:
                    continue
                vf = self._fp(vr)
                reasons: List[str] = []
                denied = base_resp.status_code in (401, 403)
                allowed = vr.status_code in (200, 201, 202, 204)
                if denied and allowed:
                    reasons.append(f"security decision changed from denied ({base_resp.status_code}) to allowed ({vr.status_code})")
                if label == "GET-vs-HEAD" and base_resp.status_code != vr.status_code:
                    reasons.append(f"GET/HEAD status differs: {base_resp.status_code} vs {vr.status_code}")
                if base_fp["content_type"] != vf["content_type"] and denied and allowed:
                    reasons.append("representation changed while crossing the authorization boundary")
                if set(base_fp["security_markers"]) != set(vf["security_markers"]):
                    if denied and allowed:
                        reasons.append("security-state markers changed across equivalent representations")
                if reasons:
                    observations.append(BoundaryObservation(
                        baseline_url=url, variant_url=variant_url, transformation=label,
                        baseline_status=base_resp.status_code, variant_status=vr.status_code,
                        baseline_fingerprint=base_fp, variant_fingerprint=vf, reasons=reasons))
        return {
            "engine": "security-boundary-differential-map-v1",
            "checked_endpoints": len(candidates),
            "signals": [asdict(o) for o in observations[:100]],
            "signal_count": len(observations),
            "interpretation": "A signal means equivalent resource representations received different security semantics; validate the boundary with authorized identity/workflow context before treating it as a confirmed authorization bypass.",
        }

    def _request(self, method: str, url: str, headers: Optional[Dict[str, str]] = None):
        t0 = time.time()
        resp = self.client.request(method, url, headers=headers or {})
        return resp, _make_evidence(resp, int((time.time() - t0) * 1000), dict(self.client.session.headers))


class SecurityBoundaryFindingAdapter:
    """Convert only high-signal denied->allowed boundary splits into findings."""

    def __init__(self, config: ScanConfig):
        self.config = config

    def findings_from(self, result: Dict[str, Any]) -> List[Finding]:
        out: List[Finding] = []
        for sig in result.get("signals", []):
            reasons = sig.get("reasons", [])
            if not any("security decision changed" in r for r in reasons):
                continue
            ev = RequestEvidence(method="GET", url=sig.get("variant_url", ""),
                                  status_code=int(sig.get("variant_status", 0)),
                                  response_time_ms=0, response_headers={},
                                  response_excerpt=json.dumps(sig.get("variant_fingerprint", {}))[:800])
            out.append(Finding(
                finding_id=_next_id(),
                category="security_boundary_differential",
                title="Security Boundary Differential",
                severity=Severity.HIGH,
                confidence="Likely",
                owasp="A01:2021", cwe="CWE-863",
                url=sig.get("variant_url", ""), parameter=None,
                description=(
                    "An equivalent representation of a resource changed from an observed denied response "
                    "to an allowed response. This can indicate an authorization/routing/proxy normalization "
                    "mismatch. The scanner does not treat the signal as a confirmed bypass without identity- or workflow-aware validation."
                ),
                evidence=ev,
                remediation_summary=(
                    "Normalize the request before authorization, apply authorization after canonical routing, "
                    "and make method/path/content-negotiation variants share the same access-control policy."
                ),
            ))
        return out



class SecurityInvariantCompiler:
    """Compiles observations into explicit invariants and coverage obligations."""
    WSTG_FAMILIES = {
        "information_gathering": ("discovery", "exposure inventory"),
        "configuration": ("security_headers", "TLS", "deployment"),
        "identity": ("authentication", "user enumeration"),
        "authorization": ("BOLA", "BFLA", "property authorization"),
        "session": ("session cookies", "token security"),
        "input_validation": ("injection", "path traversal", "parameter pollution"),
        "error_handling": ("error leakage", "verbose diagnostics"),
        "business_logic": ("workflow invariants", "limits", "state transitions"),
        "client_side": ("XSS", "DOM", "CORS", "CSP"),
        "api": ("API reconnaissance", "BOLA", "excessive data exposure", "BFLA", "GraphQL"),
    }

    def compile(self, endpoints: List[str], forms: List[FormInfo], findings: List[Finding], config: ScanConfig) -> Dict[str, Any]:
        sensitive = []
        for u in endpoints:
            p = urllib.parse.urlparse(u).path.lower()
            if any(x in p for x in ("/admin", "/account", "/checkout", "/order", "/payment", "/api/")):
                sensitive.append(u)
        obligations = []
        for family, checks in self.WSTG_FAMILIES.items():
            obligations.append({
                "family": family,
                "checks": list(checks),
                "status": "evidence-present" if any(any(c.lower() in (f.title + " " + f.category).lower() for c in checks) for f in findings) else "coverage-required",
            })
        return {
            "security_invariants": [
                "authorization decisions should remain stable across semantically equivalent GET representations",
                "sensitive response fields should not appear only in alternate parser representations",
                "high-value workflows should preserve state and authorization requirements across transitions",
                "coverage claims must distinguish tested, untested, unavailable, and inconclusive areas",
            ],
            "high_value_endpoints": sensitive[:100],
            "wstg_obligations": obligations,
            "form_count": len(forms),
            "finding_count": len(findings),
            "authenticated": bool(config.cookies or config.bearer_token),
        }


class CausalRiskFusion:
    """Builds risk narratives from contextual evidence, not just category pairs."""
    RELATIONSHIPS = {
        ("sensitive_data_exposure", "authorization_differential"): ("data exposure + authorization weakness", Severity.CRITICAL),
        ("path_traversal", "sensitive_data_exposure"): ("path normalization + sensitive data exposure", Severity.CRITICAL),
        ("open_redirect", "jwt"): ("redirect surface + token-bearing client flow", Severity.HIGH),
        ("cache_deception", "authenticated_cache_leak"): ("cache ambiguity + authenticated content exposure", Severity.HIGH),
        ("user_enumeration", "rate_limiting"): ("identity oracle + missing abuse controls", Severity.MEDIUM),
        ("mass_assignment", "idor_heuristic"): ("object ownership + writable authorization fields", Severity.HIGH),
    }

    def fuse(self, findings: List[Finding], metamorphic: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
        out = []
        by_cat: Dict[str, List[Finding]] = {}
        for f in findings:
            by_cat.setdefault(f.category, []).append(f)
        for (a, b), (name, sev) in self.RELATIONSHIPS.items():
            if a in by_cat and b in by_cat:
                members = by_cat[a][:2] + by_cat[b][:2]
                hosts = {urllib.parse.urlparse(x.url).netloc for x in members}
                related = len(hosts) == 1 or any(x.url.split("?")[0] == y.url.split("?")[0] for x in members for y in members if x is not y)
                if related:
                    out.append({
                        "name": name, "severity": sev.name,
                        "confidence": "high" if all(x.confidence == "Confirmed" for x in members) else "medium",
                        "member_findings": [x.finding_id for x in members],
                        "rationale": f"Related evidence connects {a} with {b}; the combined path has greater impact than either isolated signal.",
                    })
        if metamorphic and metamorphic.get("signal_count"):
            for sig in metamorphic.get("signals", [])[:10]:
                if "authorization boundary changed from denied to allowed" in " ".join(sig.get("reasons", [])):
                    out.append({
                        "name": "semantic authorization boundary drift", "severity": "HIGH", "confidence": "medium",
                        "member_findings": [], "rationale": "A bounded semantically-equivalent request changed an observed authorization boundary; manual validation is required.",
                    })
                    break
        return out


def run_reasoning_upgrade(config: ScanConfig, client: SafeHttpClient, crawler: WebCrawler, endpoints: List[str], forms: List[FormInfo], findings: List[Finding]) -> Dict[str, Any]:
    metamorphic = {"engine": "semantic-metamorphic-testing", "signal_count": 0, "signals": []}
    if getattr(config, "semantic_metamorphic", True):
        try:
            metamorphic = SemanticMetamorphicEngine(config, client).analyze(endpoints)
        except Exception as exc:
            metamorphic = {"engine": "semantic-metamorphic-testing", "error": str(exc), "signal_count": 0, "signals": []}
    invariants = SecurityInvariantCompiler().compile(endpoints, forms, findings, config)
    causal = CausalRiskFusion().fuse(findings, metamorphic)
    return {"metamorphic": metamorphic, "invariants": invariants, "causal_risk_fusion": causal}



# =============================================================================
# ERSEC 14 - semantic security boundary and behavior differential engine
# =============================================================================

class SourceMapExposureModule(BaseModule):
    category = "source_map_exposure"; title = "Exposed JavaScript Source Map"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings=[]
        try:
            resp, _ = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        if "text/html" not in resp.headers.get("Content-Type", ""):
            return findings
        try:
            soup = BeautifulSoup(resp.text or "", "html.parser")
            script_sources = [el.get("src", "") for el in soup.find_all("script") if el.get("src")]
        except Exception:
            script_sources = re.findall(r'<script[^>]+src=["\']([^"\']+\.js(?:\?[^"\']*)?)["\']', resp.text or "", re.I)
        for m in script_sources:
            if not re.search(r"\.js(?:\?|$)", m, re.I):
                continue
            js = urllib.parse.urljoin(url, m)
            map_url = js.split("?",1)[0] + ".map"
            try:
                mr, mev = self._get(map_url)
            except (ScopeError, ERSECError):
                continue
            body = mr.text or ""
            if mr.status_code == 200 and ("sourcesContent" in body or '"sources"' in body) and len(body) > 50:
                findings.append(self._finding(map_url, mev,
                    "A JavaScript source map is publicly accessible and contains source metadata/content, which can reveal internal source paths, comments, endpoints, or implementation details omitted from production bundles.",
                    "Do not publish source maps in production unless intentionally required; otherwise remove them from public assets or restrict access. Rebuild production bundles with appropriate source-map settings.",
                    Severity.MEDIUM, "Likely"))
        return findings


class ServiceWorkerExposureModule(BaseModule):
    category = "service_worker_exposure"; title = "Service Worker Security Exposure"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        for path in ("/service-worker.js", "/ngsw-worker.js", "/sw.js"):
            test = urllib.parse.urljoin(url, path)
            try:
                resp, ev = self._get(test)
            except (ScopeError, ERSECError):
                continue
            body = resp.text or ""
            if resp.status_code != 200 or len(body) < 50:
                continue
            risky = bool(re.search(r"cache\.add(All|\()|caches\.open\(|workbox|precache", body, re.I))
            auth_terms = bool(re.search(r"/api/|/account|/admin|/basket|/order|/checkout|authorization|bearer|cookie", body, re.I))
            if risky and auth_terms:
                findings.append(self._finding(test, ev,
                    "A publicly accessible service worker appears to cache application/API resources that may include authenticated or sensitive routes. A cache policy that is broader than the authentication boundary can retain sensitive responses on the client.",
                    "Review the service worker cache allow-list. Exclude authenticated/private responses from shared/precache strategies, and avoid caching responses containing per-user secrets or account data.",
                    Severity.MEDIUM, "Possible"))
        return findings


class OAuthStateSignalModule(BaseModule):
    category = "oauth_state"; title = "OAuth/OIDC State Validation Signal"
    owasp = "A07:2021"; cwe = "CWE-352"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        text = resp.text or ""
        links = re.findall(r'href=["\']([^"\']+)["\']', text, re.I)
        oauth_links=[x for x in links if re.search(r"(?:oauth|authorize|openid|oidc|sso|/auth/)" , x, re.I)]
        for link in oauth_links[:10]:
            if re.search(r"(?:^|[?&])state=", link, re.I):
                continue
            findings.append(self._finding(urllib.parse.urljoin(url, link), ev,
                "An OAuth/OIDC authorization link was observed without a visible state parameter. State validation is a common CSRF protection for authorization responses; its absence is a review signal, not proof that the callback is exploitable.",
                "Generate an unpredictable state value per authorization transaction and validate it server-side on the callback. Bind state to the initiating browser session.",
                Severity.MEDIUM, "Possible"))
            break
        return findings


class PasswordRecoveryLeakageModule(BaseModule):
    category = "password_recovery_leakage"; title = "Password Recovery / Reset Token Leakage"
    owasp = "A07:2021"; cwe = "CWE-598"; min_profile = ScanProfile.PASSIVE

    TOKEN_RE = re.compile(r"(?:reset|recovery|verify|token|code)=([A-Za-z0-9._~\-/+=]{12,})", re.I)

    def run_url(self, url):
        findings=[]
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        body = resp.text or ""
        if not re.search(r"(?:reset password|forgot password|password reset|recovery)", body, re.I):
            return findings
        parsed = urllib.parse.urlparse(url)
        if self.TOKEN_RE.search(parsed.query):
            findings.append(self._finding(url, ev,
                "A password-recovery-looking page URL contains a token-like query parameter. URL query values can leak through browser history, logs, analytics, or Referer headers.",
                "Use short-lived, single-use reset tokens in URLs only as necessary, then immediately exchange them for server-side state and redirect to a clean URL. Apply an appropriate Referrer-Policy and avoid third-party resources on token-bearing pages.",
                Severity.MEDIUM, "Possible"))
        external = []
        host = urllib.parse.urlparse(url).hostname or ""
        for src in re.findall(r'(?:src|href)=["\']([^"\']+)["\']', body, re.I):
            h = urllib.parse.urlparse(urllib.parse.urljoin(url, src)).hostname
            if h and h != host:
                external.append(h)
        if external and re.search(r"(?:token|reset|recovery)", body, re.I):
            findings.append(self._finding(url, ev,
                "A password-recovery-related page includes third-party resource origins while processing token/recovery content. This can increase the chance that sensitive URL/context data reaches external services.",
                "Use a strict Referrer-Policy, avoid third-party resources on password-recovery pages, and never expose reset tokens through analytics or third-party requests.",
                Severity.LOW, "Possible"))
        return findings


class CookieScopeModule(BaseModule):
    category = "cookie_scope"; title = "Over-Broad Cookie Scope"
    owasp = "A07:2021"; cwe = "CWE-1004"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings=[]
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        host = urllib.parse.urlparse(url).hostname or ""
        for cookie in resp.cookies:
            name = cookie.name.lower()
            if name not in {"sessionid","session","phpsessid","jsessionid","connect.sid","sid","auth_token","session_token"}:
                continue
            domain = cookie.get_nonstandard_attr("Domain") or ""
            path = cookie.get_nonstandard_attr("Path") or "/"
            if domain and domain.lstrip(".").count(".") >= 1:
                findings.append(self._finding(url, ev,
                    f"Authentication/session cookie '{cookie.name}' is scoped to Domain={domain!r}, making it available to sibling subdomains as well as the application host.",
                    "Prefer host-only cookies for session/authentication state. Use the __Host- prefix where compatible (Secure, Path=/, no Domain attribute).",
                    Severity.MEDIUM, "Confirmed"))
                break
            if path != "/":
                findings.append(self._finding(url, ev,
                    f"Authentication/session cookie '{cookie.name}' uses a non-root Path={path!r}. Inconsistent cookie paths across application areas can cause session state to behave differently than intended.",
                    "Use Path=/ for application-wide authentication cookies unless a narrower path is deliberately required and tested.",
                    Severity.LOW, "Possible"))
        return findings


class PathCanonicalizationModule(BaseModule):
    category = "path_canonicalization"; title = "Path Canonicalization / Normalization Differential"
    owasp = "A01:2021"; cwe = "CWE-22"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        parsed=urllib.parse.urlparse(url)
        if parsed.path in ("", "/"):
            return findings
        variants=[]
        if "/" in parsed.path.strip("/"):
            variants.append(parsed.path.replace("/","/./",1))
            variants.append(parsed.path.replace("/","//",1))
        if not parsed.path.startswith("/"):
            return findings
        if not variants:
            return findings
        try:
            base, bev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        base_sig=(base.status_code, _semantic_fingerprint(base).content_type, len(base.text or ""))
        for path in variants[:2]:
            vurl=urllib.parse.urlunparse(parsed._replace(path=path))
            if not is_in_scope(vurl,self.config):
                continue
            try:
                vr, ev = self._get(vurl)
            except (ScopeError, ERSECError):
                continue
            vsig=(vr.status_code, _semantic_fingerprint(vr).content_type, len(vr.text or ""))
            if vsig[0] != base_sig[0] and (base_sig[0] in (200,401,403) or vsig[0] in (200,401,403)):
                findings.append(self._finding(vurl, ev,
                    f"A path normalization variant changed the security response ({base_sig[0]} -> {vsig[0]}). Different layers may be canonicalizing this URL differently, which is a review signal for path traversal or authorization-boundary inconsistencies.",
                    "Canonicalize request paths once at the edge/application boundary, then apply authorization and filesystem checks to the canonical representation. Reject ambiguous dot-segment and separator forms where they are not required.",
                    Severity.MEDIUM, "Possible"))
                break
        return findings


class ForwardedHostTrustModule(BaseModule):
    category = "forwarded_host_trust"; title = "Untrusted Forwarded Host Header Influence"
    owasp = "A05:2021"; cwe = "CWE-644"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        marker_host="ersec-forwarded.invalid"
        try:
            base, _ = self._get(url)
            resp, ev = self._get(url, headers={"X-Forwarded-Host": marker_host, "Forwarded": f"host={marker_host}"})
        except (ScopeError, ERSECError):
            return findings
        loc=resp.headers.get("Location","")
        body=resp.text or ""
        influenced = marker_host in loc or marker_host in body
        if influenced:
            findings.append(self._finding(url, ev,
                "The application reflects an untrusted forwarded host value into a redirect or response body. Behind a reverse proxy, trusting client-supplied forwarding headers can enable host-header poisoning and unsafe absolute URL generation.",
                "Only honor forwarding headers from trusted proxy addresses, normalize them at the proxy boundary, and construct absolute URLs from a server-controlled canonical host configuration.",
                Severity.MEDIUM, "Confirmed"))
        return findings


class ContentNegotiationAuthModule(BaseModule):
    category = "content_negotiation_auth"; title = "Content-Negotiation Authorization Differential"
    owasp = "A01:2021"; cwe = "CWE-862"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        try:
            base, _ = self._get(url)
            jsonr, jev = self._get(url, headers={"Accept":"application/json"})
            textr, tev = self._get(url, headers={"Accept":"text/html"})
        except (ScopeError, ERSECError):
            return findings
        codes={base.status_code,jsonr.status_code,textr.status_code}
        denied={401,403}
        allowed={200,201,204}
        if any(c in denied for c in codes) and any(c in allowed for c in codes):
            findings.append(self._finding(url, jev if jsonr.status_code in allowed else tev,
                f"The same resource returned different authorization outcomes under different Accept headers ({base.status_code}, JSON={jsonr.status_code}, HTML={textr.status_code}). This can indicate authorization logic applied after content negotiation.",
                "Apply authorization before representation/content negotiation and ensure every supported representation shares the same access-control decision.",
                Severity.MEDIUM, "Possible"))
        return findings


class FileUploadSurfaceModule(BaseModule):
    category = "file_upload_surface"; title = "Weak File Upload Surface"
    owasp = "A05:2021"; cwe = "CWE-434"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        soup=BeautifulSoup(resp.text or "", "html.parser")
        for form in soup.find_all("form"):
            files=form.find_all("input",{"type":"file"})
            if not files:
                continue
            has_accept=any(inp.get("accept") for inp in files)
            action=urllib.parse.urljoin(url, form.get("action","") or url)
            if not has_accept:
                findings.append(self._finding(action, ev,
                    "A file-upload form was discovered without an HTML accept constraint. Client-side accept filters are not a security boundary, but their absence is a useful review signal when paired with server-side file-type validation gaps.",
                    "Validate uploaded content server-side by type, size and structure; store uploads outside executable/static paths when possible; generate safe filenames; and reject active content where not required.",
                    Severity.LOW, "Possible"))
        return findings


class CORSPreflightConsistencyModule(BaseModule):
    category = "cors_preflight_consistency"; title = "CORS Preflight Policy Inconsistency"
    owasp = "A05:2021"; cwe = "CWE-942"; min_profile = ScanProfile.BASELINE

    def run_url(self, url):
        findings=[]
        try:
            normal, _ = self._get(url, headers={"Origin":"https://ersec-cors.invalid"})
            pre, pev = self.client.request("OPTIONS", url, headers={
                "Origin":"https://ersec-cors.invalid",
                "Access-Control-Request-Method":"POST",
                "Access-Control-Request-Headers":"Authorization, Content-Type",
            }), None
            pre_ev=_make_evidence(pre,0)
        except (ScopeError, ERSECError):
            return findings
        if normal.headers.get("Access-Control-Allow-Origin") == "https://ersec-cors.invalid":
            allow_methods=pre.headers.get("Access-Control-Allow-Methods","").upper()
            allow_headers=pre.headers.get("Access-Control-Allow-Headers","").lower()
            if not allow_methods or "POST" not in allow_methods or "authorization" not in allow_headers:
                findings.append(self._finding(url, pre_ev,
                    "The server allows an untrusted origin on the normal response but does not consistently describe the same policy in preflight responses. Inconsistent CORS configuration across browser paths can create unexpected cross-origin behavior.",
                    "Define one explicit CORS policy and apply it consistently to normal and preflight responses. Validate origins and only allow the methods/headers the application actually needs.",
                    Severity.MEDIUM, "Possible"))
        return findings


class APIErrorMethodLeakageModule(BaseModule):
    category = "api_method_leakage"; title = "API Method/Framework Leakage"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        findings=[]
        try:
            resp, ev = self._get(url)
        except (ScopeError, ERSECError):
            return findings
        body=(resp.text or "")[:10000].lower()
        markers=("allowed methods:","express:","werkzeug","spring framework","laravel","django request","fastapi","asp.net")
        if resp.status_code in (400,404,405,500) and any(m in body for m in markers):
            findings.append(self._finding(url,ev,
                "The API error response appears to disclose framework/request-processing details. Verbose method/framework errors can aid endpoint fingerprinting and may expose internal routing assumptions.",
                "Return generic production-safe API errors while retaining detailed diagnostics only in protected server logs.",
                Severity.LOW,"Confirmed"))
        return findings



class APIResponseFieldExposureModule(BaseModule):
    category = "sensitive_api_fields"; title = "Sensitive Fields Exposed in API Response"
    owasp = "A01:2023"; cwe = "CWE-200"; min_profile = ScanProfile.PASSIVE
    SENSITIVE_KEYS = re.compile(r"(?:password|passwd|pass_hash|secret|api[_-]?key|private[_-]?key|access[_-]?token|refresh[_-]?token|session[_-]?token|ssn|social[_-]?security|credit[_-]?card|card[_-]?number|cvv|security[_-]?code)", re.I)
    def run_url(self, url):
        findings=[]
        try: resp, ev = self._get(url)
        except (ScopeError, ERSECError): return findings
        if "json" not in resp.headers.get("Content-Type","").lower(): return findings
        try: obj=resp.json()
        except Exception: return findings
        keys=[]
        def walk(v, path=""):
            if isinstance(v, dict):
                for k,val in v.items():
                    here=f"{path}.{k}" if path else str(k)
                    if self.SENSITIVE_KEYS.search(str(k)): keys.append(here)
                    walk(val,here)
            elif isinstance(v, list):
                for i,val in enumerate(v[:20]): walk(val,f"{path}[{i}]")
        walk(obj)
        if keys:
            findings.append(self._finding(url,ev,
                f"The API response exposes sensitive-looking fields ({', '.join(keys[:10])}). Returning secrets, password material, reusable tokens, or payment/security data to a client can create a direct data-exposure vulnerability.",
                "Return only the fields required by the client. Remove passwords, private keys, reusable tokens and other secrets from API serializers; apply field-level authorization and schema-based response allow-lists.",
                Severity.HIGH,"Likely"))
        return findings


class ClientSideCredentialLeakModule(BaseModule):
    category = "client_side_credential_leak"; title = "Credential-Like Secret in Client JavaScript"
    owasp = "A05:2021"; cwe = "CWE-798"; min_profile = ScanProfile.PASSIVE
    PATTERNS = [
        re.compile(r"(?:api[_-]?key|client[_-]?secret|access[_-]?token|private[_-]?key)\s*[:=]\s*[\"']([^\"']{12,})",re.I),
        re.compile(r"AKIA[0-9A-Z]{16}"),
        re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
    ]
    def run_url(self, url):
        findings=[]
        try: resp, _ = self._get(url)
        except (ScopeError, ERSECError): return findings
        soup=BeautifulSoup(resp.text or "","html.parser")
        scripts=[urllib.parse.urljoin(url,x.get("src")) for x in soup.find_all("script") if x.get("src")]
        for js in scripts[:30]:
            if not is_in_scope(js,self.config): continue
            try: jr,jev=self._get(js)
            except (ScopeError,ERSECError): continue
            body=jr.text or ""
            for rx in self.PATTERNS:
                if rx.search(body):
                    findings.append(self._finding(js,jev,
                        "A client-delivered JavaScript asset contains a credential-like secret pattern. Anything shipped to a browser should be treated as public; long-lived credentials embedded in client code can be copied and abused.",
                        "Remove secrets from client bundles. Use short-lived, audience-restricted tokens or a server-side broker and rotate any credential that was actually exposed.",
                        Severity.HIGH,"Likely"))
                    break
        return findings


class SRIIntegrityMismatchModule(BaseModule):
    category = "sri_integrity_mismatch"; title = "Subresource Integrity Hash Mismatch"
    owasp = "A05:2021"; cwe = "CWE-494"; min_profile = ScanProfile.PASSIVE
    def run_url(self,url):
        findings=[]
        try: resp,_=self._get(url)
        except (ScopeError,ERSECError): return findings
        soup=BeautifulSoup(resp.text or "","html.parser")
        for tag in soup.find_all(["script","link"]):
            integrity=tag.get("integrity") or ""
            src=tag.get("src") or tag.get("href") or ""
            if not integrity or not src: continue
            if not src.startswith(("http://","https://","/")): continue
            asset=urllib.parse.urljoin(url,src)
            if not is_in_scope(asset,self.config): continue
            try: ar,aev=self._get(asset)
            except (ScopeError,ERSECError): continue
            if ar.status_code!=200: continue
            body=ar.content or b""
            for token in integrity.split():
                if "-" not in token: continue
                algo,b64=token.split("-",1)
                try:
                    import base64
                    import hashlib
                    h=hashlib.new(algo,body).digest()
                    actual=base64.b64encode(h).decode()
                except Exception: continue
                if actual != b64:
                    findings.append(self._finding(asset,aev,
                        f"The resource declares an SRI hash that does not match the bytes currently served ({algo}). A mismatch can break the integrity control and should be investigated for stale builds or unexpected asset changes.",
                        "Regenerate SRI hashes as part of the build and release process, pin exact asset versions, and investigate unexpected changes before updating the expected hash.",
                        Severity.MEDIUM,"Confirmed"))
                    break
        return findings


class PostMessageOriginModule(BaseModule):
    category = "postmessage_origin"; title = "Weak postMessage Origin Validation"
    owasp = "A05:2021"; cwe = "CWE-345"; min_profile = ScanProfile.BASELINE
    def run_url(self,url):
        findings=[]
        try: resp,_=self._get(url)
        except (ScopeError,ERSECError): return findings
        soup=BeautifulSoup(resp.text or "","html.parser")
        scripts=[urllib.parse.urljoin(url,x.get("src")) for x in soup.find_all("script") if x.get("src")]
        inline=[x.get_text(" ",strip=False) for x in soup.find_all("script") if not x.get("src")]
        for js in scripts+inline:
            if js and re.search(r"addEventListener\s*\(\s*[\"']message[\"']",js,re.I):
                has_origin_check=bool(re.search(r"event\.origin|e\.origin|origin\s*===|origin\s*!==",js,re.I))
                wildcard_send=bool(re.search(r"postMessage\s*\([^)]*,\s*[\"']\*[\"']\s*\)",js,re.I))
                if not has_origin_check or wildcard_send:
                    ev=RequestEvidence(method="STATIC",url=url,status_code=resp.status_code,response_time_ms=0,response_headers=dict(resp.headers),response_excerpt=js[:800])
                    findings.append(self._finding(url,ev,
                        "JavaScript handles cross-window messages without an obvious origin allow-list, or sends messages with a wildcard target origin. Message handlers that trust arbitrary origins can cross a security boundary between otherwise unrelated applications.",
                        "Validate event.origin against an explicit allow-list before consuming privileged messages, and use an exact targetOrigin for sensitive outbound postMessage calls.",
                        Severity.MEDIUM,"Possible"))
                    break
        return findings


class DOMOpenRedirectModule(BaseModule):
    category = "dom_open_redirect"; title = "DOM-Based Open Redirect Data Flow"
    owasp = "A01:2021"; cwe = "CWE-601"; min_profile = ScanProfile.BASELINE
    SOURCE_RE=re.compile(r"(?:location(?:\.href|\.assign|\.replace)?|window\.open)\s*[^;]{0,180}(?:location\.search|location\.hash|URLSearchParams|document\.location|returnUrl|redirect)",re.I)
    def run_url(self,url):
        findings=[]
        try: resp,ev=self._get(url)
        except (ScopeError,ERSECError): return findings
        soup=BeautifulSoup(resp.text or "","html.parser")
        texts=[]
        texts.extend(x.get_text(" ",strip=False) for x in soup.find_all("script") if not x.get("src"))
        for src in [urllib.parse.urljoin(url,x.get("src")) for x in soup.find_all("script") if x.get("src")][:30]:
            if not is_in_scope(src,self.config): continue
            try: sr,sev=self._get(src); texts.append(sr.text or "")
            except (ScopeError,ERSECError): continue
        for text in texts:
            if self.SOURCE_RE.search(text):
                findings.append(self._finding(url,ev,
                    "Client-side JavaScript appears to feed a URL-controlled source such as location.search or a redirect parameter into a navigation sink. This is a DOM open-redirect review signal; exploitability depends on validation and how the sink is reached.",
                    "Treat redirect destinations as untrusted data. Parse and allow-list destinations, prefer relative internal paths, and reject external origins unless explicitly required.",
                    Severity.MEDIUM,"Possible"))
                break
        return findings


class GraphQLSensitiveSchemaModule(BaseModule):
    category = "graphql_sensitive_schema"; title = "Sensitive GraphQL Schema Fields Exposed"
    owasp = "API3:2023"; cwe = "CWE-213"; min_profile = ScanProfile.BASELINE
    SENSITIVE=re.compile(r"(?:password|secret|token|privateKey|apiKey|creditCard|ssn|admin|role|permission)",re.I)
    def run_url(self,url):
        findings=[]
        parsed=urllib.parse.urlparse(url)
        if "graphql" not in parsed.path.lower(): return findings
        payload={"query":"{__schema{types{name fields{name}}}}"}
        try: resp,ev=self._post(url,json=payload,headers={"Content-Type":"application/json"})
        except (ScopeError,ERSECError): return findings
        if resp.status_code!=200: return findings
        names=re.findall(r'"name"\s*:\s*"([A-Za-z0-9_]+)"',resp.text or "")
        sensitive=sorted({n for n in names if self.SENSITIVE.search(n)})
        if sensitive:
            findings.append(self._finding(url,ev,
                f"GraphQL introspection exposes sensitive-looking schema fields ({', '.join(sensitive[:15])}). Schema visibility is not automatically a vulnerability, but publishing privileged/security-sensitive fields can materially aid unauthorized API exploration when access controls are weak.",
                "Disable or restrict introspection in production where practical, and enforce field-level authorization so sensitive fields remain inaccessible regardless of schema discovery.",
                Severity.MEDIUM,"Likely"))
        return findings


class BackupArtifactExpansionModule(BaseModule):
    category = "backup_artifact_exposure"; title = "Backup / Editor Artifact Exposure"
    owasp = "A05:2021"; cwe = "CWE-200"; min_profile = ScanProfile.BASELINE
    PATHS=("/.env.bak","/.env.old","/.env~","/app.js.map","/index.php.bak","/config.json.bak","/web.config.bak","/database.sql.bak","/backup.tar.gz","/backup.tgz","/site.zip","/dump.sql","/.git/config","/.swp")
    SIGNALS=("PRIVATE KEY","DB_PASSWORD","SECRET_KEY","BEGIN OPENSSH PRIVATE KEY","CREATE TABLE","apiKey","password")
    def run_url(self,url):
        findings=[]
        for path in self.PATHS:
            test=urllib.parse.urljoin(url,path)
            try: resp,ev=self._get(test)
            except (ScopeError,ERSECError): continue
            if resp.status_code!=200 or not resp.content: continue
            body=(resp.text or "")[:200000]
            if any(sig.lower() in body.lower() for sig in self.SIGNALS) or (len(body)>500 and "<html" not in body[:300].lower()):
                findings.append(self._finding(test,ev,
                    f"A backup/editor artifact appears publicly retrievable: {path}. Backup and temporary files can expose source, configuration, database structure, or credentials.",
                    "Remove deployment/editor artifacts from the public document root and block backup extensions at the web-server layer. Rotate credentials if sensitive material was exposed.",
                    Severity.HIGH,"Likely"))
        return findings


class APIContentTypeConfusionModule(BaseModule):
    category = "api_content_type_confusion"; title = "API Content-Type / Representation Confusion"
    owasp = "A05:2021"; cwe = "CWE-436"; min_profile = ScanProfile.BASELINE
    def run_url(self,url):
        findings=[]
        try:
            base,bev=self._get(url,headers={"Accept":"application/json"})
            alt,aev=self._get(url,headers={"Accept":"text/plain"})
        except (ScopeError,ERSECError): return findings
        if "json" in base.headers.get("Content-Type","").lower() and alt.status_code==200 and "json" not in alt.headers.get("Content-Type","").lower():
            if re.search(r"(?:password|token|secret|email|role)",alt.text or "",re.I):
                findings.append(self._finding(url,aev,
                    "The API returns a materially different representation under content negotiation and the alternate representation contains sensitive-looking fields. Divergent serializers can create authorization/field-filtering gaps between formats.",
                    "Apply the same authorization and field allow-list before serializing every supported representation. Disable unnecessary representations for sensitive resources.",
                    Severity.MEDIUM,"Possible"))
        return findings




# =============================================================================
# ERSEC 17 — autonomous security-agent layer
# =============================================================================

class CausalImpactEngine:
    """Maps findings to likely business-impact nodes using observable application context.
    This is causal prioritization, not a claim of exploitability. No external data is needed."""
    _JOBS = {
        "payment": "payment_gateway", "checkout": "payment_gateway", "order": "transaction_system",
        "account": "identity_store", "session": "session_manager", "auth": "identity_store",
        "admin": "privileged_control_plane", "tenant": "tenant_boundary", "user": "identity_store",
        "pii": "regulated_data", "profile": "regulated_data", "customer": "regulated_data",
        "secret": "credential_store", "token": "credential_store", "password": "credential_store",
        "invoice": "financial_records", "refund": "financial_records", "webhook": "integration_gateway",
    }
    def __init__(self, config, crawler): self.config, self.crawler = config, crawler
    def analyze(self, findings, start_url):
        impacts=[]
        jewels=[x.lower() for x in (self.config.crown_jewels or [])]
        for f in findings:
            hay=" ".join([f.url, f.title, f.description, f.parameter or "", f.state_context or ""]).lower()
            matched=[]
            for token,node in self._JOBS.items():
                if token in hay: matched.append(node)
            distance=None
            base=f.url.split("?")[0]
            if jewels:
                score=min([abs(base.lower().find(j)) for j in jewels if j in base.lower()] or [999])
                distance=score
            score=min(100, 20 + 12*len(set(matched)) + (20 if distance is not None and distance < 40 else 0) + (20 if f.confidence=="Confirmed" else 0))
            if matched or (jewels and score>=40):
                impacts.append({"finding_id":f.finding_id,"business_nodes":sorted(set(matched)),"impact_score":score,"basis":["finding_context"] if matched else ["crown_jewel_proximity"]})
        return {"enabled":True,"impacts":impacts,"model":"deterministic-causal-context-v1"}

class AutonomousRedAgent:
    """Bounded adaptive planner. It consumes evidence and proposes only safe, read-only follow-ups."""
    def __init__(self, config, crawler, client, ledger): self.config,self.crawler,self.client,self.ledger=config,crawler,client,ledger
    def plan(self, findings, endpoints, forms):
        actions=[]
        seen=set()
        for f in sorted(findings,key=lambda x:(-x.severity.value,-x.evidence_score)):
            for action in self._actions_for(f):
                key=(action["kind"], action.get("url",f.url))
                if key in seen: continue
                seen.add(key); actions.append({**action,"source_finding":f.finding_id})
                if len(actions)>=self.config.autonomous_max_actions: break
            if len(actions)>=self.config.autonomous_max_actions: break
        return {"enabled":True,"actions":actions,"max_actions":self.config.autonomous_max_actions,"mode":"bounded-safe-adaptive"}
    def _actions_for(self,f):
        out=[]; cat=f.category.lower(); url=f.url
        if "error" in cat or "verbose" in cat: out += [{"kind":"retest_baseline","url":url,"reason":"error-derived context"},{"kind":"inventory_related_paths","url":url,"reason":"error-derived route context"}]
        if "graphql" in cat: out += [{"kind":"schema_recheck","url":url,"reason":"graphql context"}]
        if cat in {"open_redirect","ssrf"}: out += [{"kind":"boundary_compare","url":url,"reason":"URL-processing context"}]
        if "authorization" in cat or "idor" in cat or "workflow" in cat: out += [{"kind":"identity_differential","url":url,"reason":"authorization context"}]
        if "exposed_files" in cat or "secret" in cat: out += [{"kind":"related_artifact_inventory","url":url,"reason":"exposure context"}]
        return out

class FederatedKnowledgeMesh:
    """Local privacy-preserving mesh. Shares only aggregated category/stacks/hashes, never URLs or evidence."""
    def __init__(self, config): self.config=config
    def observe(self, findings, stack):
        cats=sorted({f.category for f in findings}); counts={c:sum(1 for f in findings if f.category==c) for c in cats}
        signature=hashlib.sha256(json.dumps({"categories":cats,"stack":stack.get("framework_guess","")},sort_keys=True).encode()).hexdigest()[:32]
        artifact={"version":"federated-mesh-v1","signature":signature,"category_counts":counts,"stack_family":stack.get("framework_guess","") or "unknown","privacy":"aggregate-only; no URLs/tokens/request bodies"}
        if self.config.federation_store_path:
            path=self.config.federation_store_path; data=[]
            try:
                with open(path) as fh: data=json.load(fh)
            except Exception: pass
            if not isinstance(data,list): data=[]
            data.append(artifact); data=data[-500:]
            try:
                with open(path,"w") as fh: json.dump(data,fh,indent=2)
            except OSError: pass
        return {**artifact,"enabled":True,"shared":False,"note":"Local mesh artifact; network federation intentionally not automatic."}

class MetamorphicLAM:
    """Intent-aware metamorphic planner without generating destructive attack payloads."""
    def __init__(self, config, client): self.config,self.client=config,client
    def analyze(self,endpoints,findings):
        intents={}
        for url in endpoints:
            p=urllib.parse.urlparse(url).path.lower()
            if any(x in p for x in ("login","auth","account","admin")): intents.setdefault("identity",[]).append(url)
            elif any(x in p for x in ("checkout","cart","order","payment")): intents.setdefault("transaction",[]).append(url)
            elif any(x in p for x in ("upload","file","import")): intents.setdefault("artifact",[]).append(url)
            elif any(x in p for x in ("api","graphql","rest","v1","v2")): intents.setdefault("api",[]).append(url)
        signals=[]
        for intent,urls in intents.items():
            for u in urls[:12]:
                signals.append({"intent":intent,"url":u,"variants":["canonical-path","accept-json","accept-html","query-order-neutral"],"purpose":"detect security-semantic drift"})
        return {"enabled":True,"signals":signals[:48],"model":"semantic-metamorphic-intent-v2","destructive_payloads":False}

class RemediationTwin:
    """Creates disposable, review-required remediation plans; never modifies live targets."""
    def __init__(self, config): self.config=config
    def prepare(self, findings, stack):
        plans=[]
        for f in findings:
            if not f.remediation_summary: continue
            plans.append({"finding_id":f.finding_id,"framework":stack.get("framework_guess") or "unknown","status":"plan-only","validation":["apply patch in isolated test copy","run original verification","run regression test"],"changes":f.ai_remediation_steps or f.remediation_summary})
        return {"enabled":True,"plans":plans,"directory":self.config.remediation_twin_dir,"safety":"no live-source modification"}

class ContractDriftSentry:
    def __init__(self, config): self.config=config
    def compare(self,endpoints,response_hashes):
        current={k:sorted(v) for k,v in response_hashes.items() if v}
        baseline={}
        if self.config.drift_baseline_path:
            try:
                with open(self.config.drift_baseline_path) as fh: baseline=json.load(fh)
            except Exception: baseline={}
        prev=baseline.get("surface_intelligence",{}).get("response_hashes",{}) if isinstance(baseline,dict) else {}
        added=[k for k in current if k not in prev]
        removed=[k for k in prev if k not in current]
        return {"enabled":True,"changes":[{"kind":"added_signature","hash":k} for k in added[:50]]+[{"kind":"removed_signature","hash":k} for k in removed[:50]],"continuous_ready":bool(self.config.drift_baseline_path),"model":"contract-security-drift-v1"}

def build_agent_risk_chains(findings, causal, workflow):
    chains=[]
    byid={f.finding_id:f for f in findings}
    impacts={x["finding_id"]:x for x in causal.get("impacts",[]) if isinstance(x,dict)}
    for fid,impact in impacts.items():
        f=byid.get(fid)
        if not f: continue
        for node in impact.get("business_nodes",[]):
            if impact.get("impact_score",0)>=60:
                chains.append({"name":f"Finding-to-{node.replace('_',' ').title()} impact chain","categories":[f.category],"combined_severity":f.severity.name,"rationale":f"Finding {fid} is contextually connected to business node '{node}' with deterministic impact score {impact['impact_score']}; this is prioritization evidence, not proof of end-to-end exploitation.","chain_type":"causal_impact","confidence":f.confidence,"contributing_findings":[fid]})
    wf=workflow.get("findings",[]) if isinstance(workflow,dict) else []
    if isinstance(wf,list) and len(wf)>=2:
        ids=[x.get("finding_id") for x in wf if isinstance(x,dict) and x.get("finding_id")]
        if len(ids)>=2:
            chains.append({"name":"Multi-identity workflow authorization chain","categories":["authorization_workflow"],"combined_severity":"HIGH","rationale":"Multiple identity-specific workflow observations exist; together they justify review of whether the authorization policy is consistent across state transitions.","chain_type":"workflow_causal","confidence":"Likely","contributing_findings":ids[:8]})
    return chains


class JSONDuplicateKeyDifferentialModule(BaseModule):
    category = "json_parser_differential"
    title = "Structured Request Parser Differential"
    owasp = "A05:2021"
    cwe = "CWE-235"
    min_profile = ScanProfile.DEEP

    def run_url(self, url):
        findings=[]
        try:
            baseline, bev = self._post(url, json={"ersec_probe": "baseline"})
            variant_payload = '{"ersec_probe":"A","ersec_probe":"B"}'
            resp, ev = self.client.request("POST", url, data=variant_payload, headers={"Content-Type":"application/json"}), None
            if resp.status_code in (200,201,202,204):
                txt=(resp.text or "").lower()
                if "ersec_probe" in txt and ("a" in txt or "b" in txt):
                    findings.append(self._finding(url, _make_evidence(resp, 0),
                        "The application accepted a structured request containing duplicate JSON object keys; different parsers/proxies can disagree on which value wins, creating a parser differential at security boundaries.",
                        "Reject duplicate JSON object keys at the edge/parser layer, use a strict schema validator, and ensure proxy and application parsers use the same semantics.",
                        Severity.MEDIUM, "Possible"))
        except (ScopeError, ERSECError):
            pass
        return findings


class ObservedObjectAuthorizationEngine:
    """UUID/string object-reference differential checks based only on identifiers
    already observed in the authorized application surface. It never invents IDs."""
    UUID_RE = re.compile(r"^(?i:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$")
    SAFE_ID_RE = re.compile(r"^[A-Za-z0-9_-]{8,80}$")

    def __init__(self, config, client):
        self.config=config; self.client=client

    def run(self, endpoints):
        findings=[]
        groups={}
        for url in endpoints:
            parsed=urllib.parse.urlparse(url)
            parts=parsed.path.strip('/').split('/')
            for i,part in enumerate(parts):
                if self.UUID_RE.match(part) or (self.SAFE_ID_RE.match(part) and any(x in parts[max(0,i-2):i+1] for x in ("user","users","account","accounts","order","orders","invoice","invoices","document","documents","project","projects","tenant","tenants","basket","baskets","item","items"))):
                    template=parts[:i]+["{id}"]+parts[i+1:]
                    key=(parsed.scheme, parsed.hostname, parsed.port, tuple(template))
                    groups.setdefault(key,[]).append((url,part))
        for key, rows in groups.items():
            ids=[]
            for row in rows:
                if row[1] not in ids: ids.append(row[1])
            if len(ids)<2: continue
            # bounded: compare the first observed pair only
            base_url=rows[0][0]
            first_id=ids[0]; second_id=ids[1]
            parsed=urllib.parse.urlparse(base_url)
            base_parts=parsed.path.strip('/').split('/')
            try:
                idx=base_parts.index(first_id)
            except ValueError:
                continue
            second_path='/'+'/'.join(base_parts[:idx]+[second_id]+base_parts[idx+1:])
            test_url=urllib.parse.urlunparse(parsed._replace(path=second_path))
            try:
                r0,e0=self.client.request("GET", base_url), None
                r1=self.client.request("GET", test_url)
                if r0.status_code in (200,206) and r1.status_code in (200,206):
                    b0=(r0.text or "")[:2000]; b1=(r1.text or "")[:2000]
                    if b0 and b1 and b0!=b1:
                        ev=_make_evidence(r1,0)
                        findings.append(self._finding(test_url,ev,
                            f"Two distinct observed object identifiers in the same resource family are both directly readable (IDs {first_id[:12]}… and {second_id[:12]}…). This is an authorization review signal for possible object-level access control weakness.",
                            "Verify server-side ownership/tenant authorization for every object read and write. Add negative tests proving one identity cannot access another identity's resource.",
                            Severity.MEDIUM,"Possible",parameter="object_id"))
            except (ScopeError, ERSECError):
                continue
            break
        return findings


class HTTPMethodAuthorizationMatrixEngine:
    def __init__(self, config, client):
        self.config=config; self.client=client
    def run(self, endpoints):
        findings=[]
        for url in endpoints[:40]:
            try:
                get_r=self.client.request("GET",url)
                head_r=self.client.request("HEAD",url)
                if get_r.status_code in (401,403) and head_r.status_code in (200,204):
                    findings.append({"url":url,"method_pair":"GET->HEAD","finding":"method_authorization_inconsistency","severity":"MEDIUM","confidence":"Possible","reason":"HEAD returned an allowed response where GET was denied."})
            except (ScopeError, ERSECError):
                continue
        return findings


# =============================================================================
# ERSEC 18.0 - Deep Detection Engine
# =============================================================================

class StructuredParameterDifferentialModule(BaseModule):
    """Find parser/security inconsistencies caused by alternate parameter shapes.

    The module only compares read-oriented GET responses. It never attempts to
    mutate state and never treats a response-code difference alone as proof of
    exploitability; it requires a meaningful semantic delta plus a security-
    relevant route/parameter context.
    """
    category="parameter_shape_differential"
    title="Parameter Shape / Parser Differential"
    owasp="A04:2021"
    cwe="CWE-20"
    min_profile=ScanProfile.BASELINE

    @staticmethod
    def _shape_variant(url:str,param:str,shape:str)->str:
        parsed=urllib.parse.urlparse(url)
        pairs=urllib.parse.parse_qsl(parsed.query,keep_blank_values=True)
        out=[]; replaced=False
        for k,v in pairs:
            if k==param and not replaced:
                if shape=="array": out.append((f"{param}[]",v))
                elif shape=="duplicate":
                    out.append((k,v)); out.append((k,v+"-2")); replaced=True; continue
                elif shape=="empty": out.append((k,""))
                else: out.append((k,v))
                replaced=True
            else: out.append((k,v))
        if not replaced: out.append((param,"test" if shape!="empty" else ""))
        return urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode(out,doseq=True)))

    def run_param(self,url,param,method="GET"):
        if method.upper()!="GET": return []
        labels=ParameterSemanticEngine().classify(param,url)
        if not labels and param.lower() not in {"id","uid","user_id","account_id","role","price","quantity","redirect","next","url","path"}:
            return []
        try:
            base,baseev=self._get(url)
            variants=[]
            for shape in ("array","duplicate","empty"):
                vu=self._shape_variant(url,param,shape)
                vr,vev=self._get(vu)
                cmp=ResponseDifferentialAnalyzer.compare(base,vr)
                if cmp["changed"]:
                    variants.append((shape,vu,vr,vev,cmp))
            if not variants: return []
            security_path=any(x in urllib.parse.urlparse(url).path.lower() for x in ("/admin","/account","/user","/order","/payment","/checkout","/api/"))
            if not security_path and "redirect" not in labels and "identity" not in labels and "role" not in labels:
                return []
            shape,vu,vr,vev,cmp=variants[0]
            if base.status_code in (401,403) and vr.status_code in (200,206):
                sev=Severity.HIGH; conf="Likely"
            elif base.status_code!=vr.status_code:
                sev=Severity.MEDIUM; conf="Possible"
            else:
                sev=Severity.LOW; conf="Possible"
            return [self._finding(vu,vev,
                f"Parameter '{param}' changes server security/response semantics when represented as {shape}.",
                "Normalize and strictly type request parameters at the application boundary; reject ambiguous duplicate/array representations for security-sensitive fields and add parser-consistency tests.",
                sev,conf,parameter=param)]
        except (ScopeError,ERSECError):
            return []


class APIResponseConsistencyModule(BaseModule):
    """Compare equivalent API representations to detect unexpected schema drift."""
    category="api_response_consistency"
    title="API Response Consistency Anomaly"
    owasp="A05:2021"
    cwe="CWE-116"
    min_profile=ScanProfile.BASELINE

    def run_url(self,url):
        path=urllib.parse.urlparse(url).path.lower()
        if not (path.startswith(("/api/","/rest/","/v1/","/v2/","/v3/","/graphql"))): return []
        try:
            a,ae=self._get(url,headers={"Accept":"application/json"})
            b,be=self._get(url,headers={"Accept":"text/html"})
        except (ScopeError,ERSECError): return []
        # Strong signal only when one representation discloses an obviously
        # sensitive field that the other does not, or auth semantics diverge.
        af=(a.text or "")[:8000].lower(); bf=(b.text or "")[:8000].lower()
        sensitive=re.compile(r'"(?:password|passwd|password_hash|secret|api[_-]?key|access[_-]?token|refresh[_-]?token|private[_-]?key|ssn|credit[_-]?card)"\s*:')
        asec=bool(sensitive.search(af)); bsec=bool(sensitive.search(bf))
        if a.status_code!=b.status_code and a.status_code in (401,403) and b.status_code==200:
            return [self._finding(url,be,
                "The same API resource has different authorization semantics under HTTP content negotiation (JSON is protected while another representation is reachable).",
                "Apply authorization before representation negotiation and ensure every content type shares the same resource-level access decision.",Severity.HIGH,"Likely")]
        if asec != bsec:
            ev=ae if asec else be
            return [self._finding(url,ev,
                "The API exposes a sensitive response field only under one content-negotiation representation, indicating representation-dependent data filtering.",
                "Apply field-level authorization and response shaping before content negotiation; do not rely on serializers alone to hide sensitive properties.",Severity.MEDIUM,"Likely")]
        return []


class APIVersionDriftModule(BaseModule):
    category="api_version_drift"
    title="API Version Security Drift"
    owasp="A05:2021"
    cwe="CWE-693"
    min_profile=ScanProfile.BASELINE

    def run_url(self,url):
        path=urllib.parse.urlparse(url).path
        m=re.search(r"/v(\\d+)(/.*)$",path,re.I)
        if not m: return []
        old=int(m.group(1)); suffix=m.group(2)
        candidates=[]
        for v in (max(1,old-1),old+1):
            vu=urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path=f"/v{v}{suffix}"))
            if is_in_scope(vu,self.config): candidates.append(vu)
        try: base,bev=self._get(url)
        except (ScopeError,ERSECError): return []
        for vu in candidates:
            try: other,oe=self._get(vu)
            except (ScopeError,ERSECError): continue
            if base.status_code in (401,403) and other.status_code==200:
                return [self._finding(vu,oe,
                    f"API version {m.group(1)} is protected, but related version endpoint is publicly reachable.",
                    "Retire or protect deprecated API versions consistently and apply authorization policies across all supported versions.",Severity.HIGH,"Likely")]
        return []


class SecurityHeaderDriftModule(BaseModule):
    category="security_header_drift"
    title="Security Control Header Drift"
    owasp="A05:2021"
    cwe="CWE-693"
    min_profile=ScanProfile.PASSIVE

    def run_url(self,url):
        parsed=urllib.parse.urlparse(url)
        if not parsed.path or parsed.path=="/": return []
        sibling=urllib.parse.urlunparse(parsed._replace(path="/"))
        if sibling==url or not is_in_scope(sibling,self.config): return []
        try: a,ae=self._get(sibling); b,be=self._get(url)
        except (ScopeError,ERSECError): return []
        security_headers=("content-security-policy","strict-transport-security","x-content-type-options","referrer-policy")
        missing_a=[h for h in security_headers if not a.headers.get(h)]
        missing_b=[h for h in security_headers if not b.headers.get(h)]
        # Only raise when a sibling establishes a security control that the
        # target omits, which reduces noise compared with checking absence alone.
        drift=[h for h in security_headers if a.headers.get(h) and not b.headers.get(h)]
        if drift:
            return [self._finding(url,be,
                "Security response controls differ between the site root and this endpoint; the endpoint omits controls present on its sibling response.",
                "Centralize security headers at the application gateway or shared middleware, then add endpoint-level regression tests for security-control consistency.",Severity.LOW,"Confirmed")]
        return []


class SensitiveCookiePathModule(BaseModule):
    category="cookie_path_scope"
    title="Over-Broad Sensitive Cookie Scope"
    owasp="A07:2021"
    cwe="CWE-1004"
    min_profile=ScanProfile.PASSIVE

    def run_url(self,url):
        try: resp,ev=self._get(url)
        except (ScopeError,ERSECError): return []
        out=[]
        for c in resp.cookies:
            if c.name.lower() not in {"session","sessionid","sid","connect.sid","phpsessid","auth_token","access_token"}:
                continue
            path=c.path or "/"
            domain=c.domain or ""
            if path=="/" and not (c.name.startswith("__Host-") or c.name.startswith("__Secure-")):
                out.append(self._finding(url,ev,
                    f"Sensitive cookie '{c.name}' is scoped to the entire site path '/'.",
                    "Narrow cookie Path where practical and use Secure, HttpOnly, and an appropriate SameSite policy. For host-wide session cookies, consider the __Host- prefix.",Severity.LOW,"Confirmed"))
        return out


class SecurityDecisionLatticeEngine:
    """Build an authorization lattice from explicit identity tokens and safe GETs.

    This is intentionally read-only. A decision is interesting when the same
    resource produces a non-monotonic role transition such as admin denied while
    a lower-ranked identity is allowed, or tenant A/B disagree on a shared object.
    """
    def __init__(self,config,client): self.config=config; self.client=client
    @staticmethod
    def fp(resp):
        body=re.sub(r"\d+","#",re.sub(r"\s+"," ",(resp.text or "")[:5000])).strip()
        return (resp.status_code,hashlib.sha256(body.encode()).hexdigest()[:16],len(body))
    def analyze(self,urls,limit=80):
        identities={"anonymous":""}
        identities.update(getattr(self.config,"identity_tokens",{}) or {})
        if getattr(self.config,"bearer_token","") and "primary" not in identities: identities["primary"]=self.config.bearer_token
        if getattr(self.config,"second_bearer_token","") and "secondary" not in identities: identities["secondary"]=self.config.second_bearer_token
        if len(identities)<2: return {"available":False,"reason":"at least two identities are required","findings":[],"matrix":[]}
        ordered=sorted(identities.items(),key=lambda kv:getattr(self.config,"identity_role_order",{}).get(kv[0].lower(),1))
        findings=[]; matrix=[]
        for url in list(dict.fromkeys(urls))[:limit]:
            path=urllib.parse.urlparse(url).path.lower()
            if not any(x in path for x in ("/api/","/admin","/account","/user","/order","/tenant","/project","/invoice","/basket","/cart","/payment")): continue
            decisions=[]
            for name,token in ordered:
                cfg=dataclasses.replace(self.config,cookies={},bearer_token=token,second_bearer_token="")
                c=SafeHttpClient(cfg)
                try:r=c.request("GET",url)
                except (ScopeError,ERSECError):continue
                decisions.append((name,getattr(self.config,"identity_role_order",{}).get(name.lower(),1),r.status_code,self.fp(r)))
            if len(decisions)<2: continue
            matrix.append({"url":url,"decisions":[{"identity":n,"role_rank":rk,"status":st,"fingerprint":fp} for n,rk,st,fp in decisions]})
            for low_n,low_r,low_st,low_fp in decisions:
                for hi_n,hi_r,hi_st,hi_fp in decisions:
                    if hi_r<=low_r: continue
                    if hi_st in (401,403) and low_st==200:
                        ev=RequestEvidence(method="GET",url=url,status_code=low_st,response_time_ms=0,response_headers={},response_excerpt=f"{low_n}=200, {hi_n}={hi_st}")
                        findings.append(Finding(_next_id(),"authorization_lattice_inversion","Authorization Decision Lattice Inversion",Severity.HIGH,"Likely","A01:2021","CWE-862",url,None,
                            f"A lower-ranked identity '{low_n}' can read a resource while higher-ranked identity '{hi_n}' is denied. This non-monotonic authorization decision suggests role-specific routing or policy inconsistency and requires application-context review.",ev,
                            "Centralize authorization policy evaluation and add role-matrix tests for this resource."))
                        break
                if findings and findings[-1].url==url: break
        return {"available":True,"identity_count":len(identities),"findings":findings,"matrix":matrix[:limit]}


class ResponseAnomalyEnsemble:
    """Combine independent evidence dimensions without allowing one weak signal
    to become a high-confidence finding. This is a scoring/quality layer, not a
    vulnerability generator by itself.
    """
    @staticmethod
    def score(finding,related_count=0):
        score=0.0
        if finding.confidence=="Confirmed": score+=0.55
        elif finding.confidence=="Likely": score+=0.38
        elif finding.confidence=="Possible": score+=0.18
        score += min(0.25, related_count*0.08)
        if finding.evidence.status_code in (200,206): score+=0.08
        if finding.parameter: score+=0.04
        return round(min(0.99,score),2)
    def annotate(self,findings):
        buckets={}
        for f in findings:
            buckets.setdefault((f.category,f.url.split("?")[0]),[]).append(f)
        for f in findings:
            f.evidence_score=max(f.evidence_score,self.score(f,len(buckets.get((f.category,f.url.split("?")[0]),[]))-1))
        return findings


@dataclass
class SecurityAssetState:
    url: str
    asset_type: str
    exposure: str
    auth_observed: str
    sensitivity: str
    confidence: float
    evidence: List[str] = field(default_factory=list)


class SecurityControlPlane:
    """ERSEC's application-security control plane.

    This layer is intentionally different from the detector engine: it turns the
    scan into a continuously comparable security state. It builds an asset/control
    model, derives bounded investigation hypotheses, calculates an exposure budget,
    and compiles machine-readable security objectives. It never sends requests.
    """
    VERSION = "1.0"

    SENSITIVE_MARKERS = (
        "admin", "manage", "account", "profile", "payment", "checkout", "order",
        "billing", "invoice", "token", "auth", "oauth", "password", "reset",
        "secret", "internal", "debug", "metrics", "actuator", "graphql", "api"
    )

    def __init__(self, config: ScanConfig):
        self.config = config

    @staticmethod
    def _path_type(url: str) -> str:
        path = urllib.parse.urlparse(url).path.lower()
        if any(x in path for x in ("/admin", "/manage", "/actuator", "/internal", "/debug", "/metrics")):
            return "privileged_surface"
        if any(x in path for x in ("/checkout", "/payment", "/billing", "/order", "/cart")):
            return "transactional_surface"
        if any(x in path for x in ("/login", "/oauth", "/token", "/auth", "/reset", "/password")):
            return "identity_surface"
        if re.search(r"/(api|rest|graphql|v\\d+)(/|$)", path):
            return "api_surface"
        return "web_surface"

    @classmethod
    def _sensitivity(cls, url: str) -> str:
        path = urllib.parse.urlparse(url).path.lower()
        if any(x in path for x in ("payment", "billing", "token", "password", "secret", "admin", "internal")):
            return "critical"
        if any(x in path for x in ("account", "profile", "order", "checkout", "api")):
            return "high"
        return "normal"

    @staticmethod
    def _auth_hint(url: str, findings: List[Finding]) -> str:
        related = [f for f in findings if f.url.split("?")[0] == url.split("?")[0]]
        if any(f.category in {"admin_exposure", "cross_identity_authorization", "workflow_authorization_inconsistency"} for f in related):
            return "inconsistent"
        if any(f.category in {"idor_heuristic", "observed_object_authorization", "function_authorization"} for f in related):
            return "resource-sensitive"
        return "unknown"

    def build_assets(self, endpoints: List[str], findings: List[Finding]) -> List[SecurityAssetState]:
        assets: List[SecurityAssetState] = []
        finding_by_url: Dict[str, List[Finding]] = {}
        for f in findings:
            finding_by_url.setdefault(f.url.split("?")[0], []).append(f)
        seen = set()
        for url in endpoints:
            key = url.split("?")[0]
            if key in seen:
                continue
            seen.add(key)
            related = finding_by_url.get(key, [])
            max_conf = max([1.0 if f.confidence == "Confirmed" else 0.75 if f.confidence == "Likely" else 0.5 for f in related] or [0.45])
            exposure = "exposed" if related else "observed"
            assets.append(SecurityAssetState(
                url=key,
                asset_type=self._path_type(key),
                exposure=exposure,
                auth_observed=self._auth_hint(key, findings),
                sensitivity=self._sensitivity(key),
                confidence=max_conf,
                evidence=[f.finding_id for f in related[:8]],
            ))
        return assets

    def derive_hypotheses(self, assets: List[SecurityAssetState], findings: List[Finding], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
        hypotheses: List[Dict[str, Any]] = []
        existing = {(f.category, f.url.split("?")[0]) for f in findings}
        for asset in assets:
            if len(hypotheses) >= self.config.hypothesis_budget:
                break
            path = asset.url
            if asset.asset_type in {"privileged_surface", "identity_surface", "transactional_surface", "api_surface"}:
                if ("authorization_boundary", path) not in existing:
                    hypotheses.append({
                        "id": hashlib.sha256(f"auth-boundary|{path}".encode()).hexdigest()[:12],
                        "type": "authorization_boundary",
                        "asset": path,
                        "question": "Do equivalent representations, methods, identities, and workflow states produce the same authorization decision?",
                        "priority": "high" if asset.sensitivity in {"critical", "high"} else "medium",
                        "safe_next_steps": ["compare authorized identities if supplied", "compare observed HTTP methods", "compare equivalent content representations"],
                    })
            if asset.asset_type == "transactional_surface" and not any(f.category.startswith("business") for f in findings if f.url.split("?")[0] == path):
                hypotheses.append({
                    "id": hashlib.sha256(f"invariant|{path}".encode()).hexdigest()[:12],
                    "type": "transaction_invariant",
                    "asset": path,
                    "question": "Does the workflow preserve server-side price, ownership, quantity, state, and one-time-action invariants?",
                    "priority": "high",
                    "safe_next_steps": ["replay observed state transitions with unchanged semantics", "compare server-derived fields", "check duplicate/ordering-sensitive transitions"],
                })
            if len(hypotheses) >= self.config.hypothesis_budget:
                break
        if workflow.get("workflow_count", 0) > 0 and not any(h["type"] == "workflow_authorization" for h in hypotheses):
            hypotheses.append({
                "id": hashlib.sha256(b"workflow-authorization-lattice").hexdigest()[:12],
                "type": "workflow_authorization",
                "asset": "application-workflows",
                "question": "Does changing identity or workflow state unexpectedly change the authorization outcome?",
                "priority": "high",
                "safe_next_steps": ["replay observed workflows per identity", "compare state-transition permissions", "correlate with boundary differential findings"],
            })
        return hypotheses[:self.config.hypothesis_budget]

    def exposure_budget(self, assets: List[SecurityAssetState], findings: List[Finding], chains: List[Dict[str, Any]]) -> Dict[str, Any]:
        weights = {"normal": 1.0, "high": 3.0, "critical": 6.0}
        score = sum(weights.get(a.sensitivity, 1.0) for a in assets if a.exposure == "exposed")
        score += sum(2.5 for f in findings if f.severity in (Severity.CRITICAL, Severity.HIGH) and f.confidence == "Confirmed")
        score += sum(3.5 for c in chains if c.get("combined_severity") == "CRITICAL")
        band = "green" if score < 15 else "amber" if score < 35 else "red"
        return {
            "score": round(score, 2),
            "band": band,
            "model": "exposure-budget-v1",
            "interpretation": "normalized relative risk pressure; not a probability of compromise",
            "drivers": [
                {"asset_type": a.asset_type, "url": a.url, "sensitivity": a.sensitivity}
                for a in assets if a.exposure == "exposed" and a.sensitivity in {"critical", "high"}
            ][:20],
        }

    def compile_security_slo(self, findings: List[Finding], coverage: Dict[str, Any], chains: List[Dict[str, Any]]) -> Dict[str, Any]:
        critical = sum(1 for f in findings if f.severity == Severity.CRITICAL)
        high = sum(1 for f in findings if f.severity == Severity.HIGH)
        return {
            "policy": "ERSEC Security SLO v1",
            "status": "fail" if critical or high else "pass",
            "objectives": {
                "critical_findings": {"target": 0, "observed": critical},
                "high_findings": {"target": 0, "observed": high},
                "coverage_ratio": {"target": 0.8, "observed": coverage.get("coverage_ratio", 0)},
                "critical_attack_paths": {"target": 0, "observed": sum(1 for c in chains if c.get("combined_severity") == "CRITICAL")},
            },
            "gates": ["no confirmed critical findings", "no confirmed high findings", "coverage >= 80%"],
        }

    def snapshot(self, assets: List[SecurityAssetState], findings: List[Finding], chains: List[Dict[str, Any]]) -> Dict[str, Any]:
        return {
            "version": self.VERSION,
            "generated_at": _utc_now_iso(),
            "target": self.config.target,
            "assets": [asdict(a) for a in assets],
            "finding_ids": sorted(f.diff_key() for f in findings),
            "risk_chains": sorted(c.get("name", "") for c in chains),
        }

    @staticmethod
    def diff(previous: Optional[Dict[str, Any]], current: Dict[str, Any]) -> Dict[str, Any]:
        if not previous:
            return {"baseline_available": False, "new_assets": len(current.get("assets", [])), "new_findings": len(current.get("finding_ids", [])), "new_chains": len(current.get("risk_chains", []))}
        old_assets = {a.get("url") for a in previous.get("assets", [])}
        new_assets = {a.get("url") for a in current.get("assets", [])}
        old_f = set(previous.get("finding_ids", [])); new_f = set(current.get("finding_ids", []))
        old_c = set(previous.get("risk_chains", [])); new_c = set(current.get("risk_chains", []))
        return {
            "baseline_available": True,
            "new_assets": sorted(new_assets - old_assets)[:50],
            "removed_assets": sorted(old_assets - new_assets)[:50],
            "new_findings": sorted(new_f - old_f)[:50],
            "resolved_findings": sorted(old_f - new_f)[:50],
            "new_risk_chains": sorted(new_c - old_c)[:50],
            "resolved_risk_chains": sorted(old_c - new_c)[:50],
            "security_regression": bool((new_f - old_f) or (new_c - old_c)),
        }

    def persist_snapshot(self, snapshot: Dict[str, Any]) -> Optional[Dict[str, Any]]:
        path = self.config.control_plane_memory_path
        if not path:
            return None
        previous = None
        try:
            if os.path.exists(path):
                with open(path, "r", encoding="utf-8") as fh:
                    previous = json.load(fh)
        except (OSError, json.JSONDecodeError):
            previous = None
        diff = self.diff(previous, snapshot)
        tmp = path + ".tmp"
        try:
            with open(tmp, "w", encoding="utf-8") as fh:
                json.dump(snapshot, fh, indent=2, sort_keys=True)
            os.replace(tmp, path)
        except OSError:
            try:
                if os.path.exists(tmp): os.unlink(tmp)
            except OSError:
                pass
        return diff

    def analyze(self, endpoints: List[str], findings: List[Finding], chains: List[Dict[str, Any]], workflow: Dict[str, Any], coverage: Dict[str, Any]) -> Dict[str, Any]:
        assets = self.build_assets(endpoints, findings)
        hypotheses = self.derive_hypotheses(assets, findings, workflow)
        exposure = self.exposure_budget(assets, findings, chains)
        slo = self.compile_security_slo(findings, coverage, chains)
        snap = self.snapshot(assets, findings, chains)
        diff = self.persist_snapshot(snap)
        return {
            "engine": "ERSEC Security Control Plane",
            "version": self.VERSION,
            "assets": [asdict(a) for a in assets[:500]],
            "asset_count": len(assets),
            "hypotheses": hypotheses,
            "hypothesis_count": len(hypotheses),
            "exposure_budget": exposure,
            "security_slo": slo,
            "baseline_diff": diff,
            "status": "regression" if diff and diff.get("security_regression") else "stable_or_baseline_missing",
        }



# =============================================================================
# ERSEC 29 - Security assurance, behavior graph, contracts and safety governance
# =============================================================================

class AuthorizationManifest:
    """Fail-closed engagement manifest for explicit authorization boundaries."""
    def __init__(self, data: Dict[str, Any]):
        if not isinstance(data, dict):
            raise ERSECError("Authorization manifest must be a JSON object")
        self.schema=str(data.get("schema", ERSEC_AUTH_MANIFEST_SCHEMA))
        self.name=str(data.get("name", "unnamed-engagement"))
        self.allowed_hosts={str(x).lower() for x in data.get("allowed_hosts", []) if str(x).strip()}
        self.allowed_ports={int(x) for x in data.get("allowed_ports", [])}
        self.allowed_paths=[str(x) for x in data.get("allowed_paths", [])]
        self.allowed_methods={str(x).upper() for x in data.get("allowed_methods", ["GET","HEAD","OPTIONS"])}
        self.private_address_policy=str(data.get("private_address_policy", "deny")).lower()
        self.stateful_tests=str(data.get("stateful_tests", "deny")).lower()
        self.approved_identities={str(x) for x in data.get("approved_identities", [])}
        self.window_start=str(data.get("window_start", ""))
        self.window_end=str(data.get("window_end", ""))
        self.owner=str(data.get("owner", ""))
        self.purpose=str(data.get("purpose", ""))
        if not self.allowed_hosts:
            raise ERSECError("Authorization manifest must declare at least one allowed host")
        if not self.allowed_methods:
            raise ERSECError("Authorization manifest must declare at least one allowed HTTP method")
        if self.private_address_policy not in {"deny", "allow"}:
            raise ERSECError("Authorization manifest private_address_policy must be 'deny' or 'allow'")
        if self.stateful_tests not in {"deny", "allow"}:
            raise ERSECError("Authorization manifest stateful_tests must be 'deny' or 'allow'")

    @classmethod
    def load(cls, path: str) -> "AuthorizationManifest":
        with open(path, "r", encoding="utf-8") as fh:
            return cls(json.load(fh))

    def _window_ok(self) -> bool:
        now=datetime.datetime.now(datetime.timezone.utc)
        def parse(v):
            if not v: return None
            raw=v.replace("Z", "+00:00")
            dt=datetime.datetime.fromisoformat(raw)
            return dt if dt.tzinfo else dt.replace(tzinfo=datetime.timezone.utc)
        try:
            start=parse(self.window_start); end=parse(self.window_end)
            if start and now < start: return False
            if end and now > end: return False
        except ValueError as exc:
            raise ScopeError(f"Invalid authorization manifest testing window: {exc}")
        return True

    def allows_request(self, method: str, url: str) -> Tuple[bool, str]:
        if not self._window_ok():
            return False, "outside_testing_window"
        parsed=urllib.parse.urlparse(url)
        host=(parsed.hostname or "").lower()
        port=parsed.port or (443 if parsed.scheme=="https" else 80)
        if self.allowed_hosts and host not in self.allowed_hosts:
            return False, "host_not_authorized"
        if self.allowed_ports and port not in self.allowed_ports:
            return False, "port_not_authorized"
        if self.allowed_paths and not any((parsed.path or "/").startswith(p) for p in self.allowed_paths):
            return False, "path_not_authorized"
        if method.upper() not in self.allowed_methods:
            return False, "method_not_authorized"
        if self.private_address_policy == "deny":
            candidates=[]
            try:
                candidates.append(ipaddress.ip_address(host))
            except ValueError:
                try:
                    candidates.extend(ipaddress.ip_address(info[4][0]) for info in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM))
                except OSError:
                    # Do not fail open on resolution errors when a manifest explicitly denies private destinations.
                    return False, "host_resolution_failed_under_private_address_policy"
            if any(addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_multicast or addr.is_unspecified for addr in candidates):
                return False, "private_address_denied"
        return True, ""

    def allows_stateful_tests(self) -> bool:
        return self.stateful_tests == "allow"

    def to_dict(self) -> Dict[str, Any]:
        return {
            "schema": self.schema, "name": self.name, "owner": self.owner, "purpose": self.purpose,
            "allowed_hosts": sorted(self.allowed_hosts), "allowed_ports": sorted(self.allowed_ports),
            "allowed_paths": self.allowed_paths, "allowed_methods": sorted(self.allowed_methods),
            "private_address_policy": self.private_address_policy, "stateful_tests": self.stateful_tests,
            "approved_identities": sorted(self.approved_identities),
            "window_start": self.window_start, "window_end": self.window_end,
        }


class RiskBudgetScheduler:
    """Ranks candidate security questions using information value per unit of risk/cost."""
    def __init__(self, budget: int=40): self.budget=max(1,int(budget))
    def score(self, item: Dict[str, Any]) -> float:
        score=float(item.get("information_gain", 0.5))
        score += float(item.get("crown_jewel", 0.0))*0.35
        score += float(item.get("authorization_relevance", 0.0))*0.35
        score += float(item.get("contract_relevance", 0.0))*0.20
        score -= float(item.get("request_cost", 1.0))*0.03
        score -= float(item.get("state_change_risk", 0.0))*0.50
        return round(score,4)
    def plan(self, items: List[Dict[str, Any]]) -> Dict[str, Any]:
        ranked=[]
        for idx,item in enumerate(items):
            x=dict(item); x["priority_score"]=self.score(x); x["plan_index"]=idx; ranked.append(x)
        ranked.sort(key=lambda x:x["priority_score"], reverse=True)
        selected=ranked[:self.budget]; skipped=ranked[self.budget:]
        return {"engine":"risk-budget-scheduler-v1","budget":self.budget,"selected":selected,"skipped":skipped,"selected_count":len(selected),"skipped_count":len(skipped),"statement":"Budget prioritizes expected information value while penalizing request cost and state-change risk."}


class StatefulTestSimulator:
    """Produces dry-run previews for state-changing form/API checks without submitting them."""
    STATEFUL_METHODS={"POST","PUT","PATCH","DELETE"}
    @classmethod
    def preview(cls, forms: List["FormInfo"], config: ScanConfig) -> Dict[str, Any]:
        previews=[]
        permitted_manifest=bool(getattr(config,"authorization_manifest",None))
        allowed_stateful=cls.STATEFUL_METHODS & set(config.scope.allowed_methods or [])
        for form in forms[:100]:
            method=str(getattr(form,"method","GET") or "GET").upper()
            if method not in cls.STATEFUL_METHODS: continue
            action=str(getattr(form,"action","") or config.target)
            previews.append({
                "method":method,
                "url":action,
                "fields":[str(getattr(x,"name",x)) for x in getattr(form,"inputs",[])][:50],
                "identity":"explicitly supplied identity if configured",
                "predicted_state_change":"form submission may mutate server-side state; ERSEC does not submit this preview",
                "rollback_plan":"application-specific rollback required; ERSEC cannot infer a safe rollback automatically",
                "evidence_retained":"only after an authorized execution; preview itself retains no request body",
                "enabled":bool(config.stateful_tests_enabled and method in allowed_stateful and permitted_manifest),
                "requires_explicit_authorization_manifest":not permitted_manifest,
            })
        return {"schema":"ersec-stateful-preview/1","count":len(previews),"previews":previews,"stateful_tests_enabled":config.stateful_tests_enabled,"statement":"Preview is dry-run only. State-changing execution is not implied by this artifact."}


class SecurityBehaviorGraph:
    """Builds a compact, deterministic graph of identities, resources, workflows and security evidence."""
    def build(self, endpoints: List[str], findings: List[Finding], workflow: Dict[str, Any], invariants: List[Dict[str, Any]], config: ScanConfig) -> Dict[str, Any]:
        nodes=[]; edges=[]; seen=set()
        def add_node(kind, key, **extra):
            node_id=f"{kind}:{key}"
            if node_id in seen: return node_id
            seen.add(node_id); nodes.append({"id":node_id,"kind":kind,"key":key,**extra}); return node_id
        asset_paths=[]
        for url in sorted(set(endpoints))[:500]:
            path=urllib.parse.urlparse(url).path or "/"
            sensitivity="normal"
            low=path.lower()
            if any(x in low for x in ("admin","account","checkout","payment","oauth","token")): sensitivity="high"
            rid=add_node("resource",url,path=path,sensitivity=sensitivity)
            asset_paths.append(url)
        identities=set((config.identity_tokens or {}).keys()) | {"anonymous"}
        for identity in sorted(identities): add_node("identity",identity)
        for f in findings[:500]:
            fid=add_node("finding",f.finding_id,category=SecurityCategoryRegistry.normalize(f.category),severity=f.severity.name,confidence=f.confidence)
            rid=add_node("resource",f.url.split("?")[0],path=urllib.parse.urlparse(f.url).path or "/",sensitivity="high" if any(x in f.url.lower() for x in ("admin","account","checkout","payment")) else "normal")
            edges.append({"from":fid,"to":rid,"relation":"violates-on"})
        for j in workflow.get("workflows",[])[:100]:
            src=str(j.get("source",j.get("start", ""))); dst=str(j.get("destination",j.get("end", "")))
            if src and dst:
                a=add_node("workflow_state",src); b=add_node("workflow_state",dst); edges.append({"from":a,"to":b,"relation":"transitions"})
        for inv in invariants[:200]:
            iid=add_node("invariant",str(inv.get("invariant_id",inv.get("statement",""))),statement=inv.get("statement",""),violated=bool(inv.get("violated")))
            for fid in inv.get("finding_ids",[])[:20]: edges.append({"from":iid,"to":add_node("finding",str(fid)),"relation":"supported-by"})
        return {"schema":"ersec-security-behavior-graph/1","generated_at":_utc_now_iso(),"node_count":len(nodes),"edge_count":len(edges),"nodes":nodes,"edges":edges,"invariants":invariants[:200],"interpretation":"Deterministic application-security model. Graph presence is evidence of observed structure, not proof of a vulnerability."}
    @staticmethod
    def save(graph: Dict[str, Any], path: str) -> None:
        os.makedirs(os.path.dirname(os.path.abspath(path)),exist_ok=True)
        tmp=path+".tmp"
        with open(tmp,"w",encoding="utf-8") as fh: json.dump(graph,fh,indent=2,sort_keys=True)
        os.replace(tmp,path)


class SecurityContractIdentity:
    """Stable identity for a security contract across scan runs.

    Finding IDs are intentionally not used because detector execution order can change.
    The identity is derived from the normalized security condition instead.
    """
    @staticmethod
    def for_finding(f: Finding) -> str:
        category=SecurityCategoryRegistry.normalize(f.category)
        parsed=urllib.parse.urlparse(f.url)
        normalized_url=urllib.parse.urlunparse((parsed.scheme.lower(), (parsed.hostname or '').lower(), parsed.path or '/', '', '', ''))
        method=(f.evidence.method or 'GET').upper()
        parameter=(f.parameter or '').strip()
        material='|'.join([category,method,normalized_url,parameter])
        return 'FCON-'+hashlib.sha256(material.encode('utf-8')).hexdigest()[:16]

    @staticmethod
    def for_invariant(statement: str) -> str:
        return 'SCON-'+hashlib.sha256(re.sub(r'\\s+',' ',str(statement).strip().lower()).encode('utf-8')).hexdigest()[:16]


class SecurityContractEngine:
    """Compiles evidence/invariants into reviewable, machine-readable regression contracts."""
    def compile(self, findings: List[Finding], graph: Dict[str, Any], behavior: Dict[str, Any]) -> Dict[str, Any]:
        contracts=[]
        invs=behavior.get("invariants",[]) if isinstance(behavior,dict) else []
        for inv in invs[:200]:
            statement=str(inv.get("statement","")).strip()
            if not statement: continue
            contracts.append({"contract_id":SecurityContractIdentity.for_invariant(statement),"type":"invariant","statement":statement,"violated":bool(inv.get("violated")),"automation":{"safe_by_default":True,"status":"draft"}})
        for f in findings[:200]:
            if f.confidence not in {"Confirmed","Likely"}: continue
            md=SecurityCategoryRegistry.metadata(f.category)
            method=(f.evidence.method or "GET").upper()
            stateful=method not in {"GET","HEAD","OPTIONS"}
            contracts.append({
                "contract_id":"FC-"+hashlib.sha256(f.diff_key().encode()).hexdigest()[:12],
                "type":"finding-regression",
                "finding_id":f.finding_id,
                "fingerprint":SecurityContractIdentity.for_finding(f),
                "category":md.canonical,
                "identity_context":f.state_context or "unspecified",
                "given":{"method":method,"url":f.url,"parameter":f.parameter},
                "then":{"must_not":{"status_codes":[200,201,202]},"evidence_condition":f"Review that the observed {md.test_family} condition no longer occurs."},
                "automation":{"safe_by_default":not stateful,"stateful":stateful,"status":"draft","human_approval_required":stateful},
            })
        return {"schema":ERSEC_CONTRACT_SCHEMA,"generated_at":_utc_now_iso(),"contract_count":len(contracts),"contracts":contracts,"formats":["pytest","playwright","postman","openapi-assertions","machine-readable"],"statement":"Contracts are regression specifications, not automatic proof and not automatically enabled state-changing tests."}
    def export(self, bundle: Dict[str, Any], output_dir: str) -> Dict[str, Any]:
        root=_pathlib.Path(output_dir); root.mkdir(parents=True,exist_ok=True)
        (root/"ersec-contracts.json").write_text(json.dumps(bundle,indent=2,sort_keys=True),encoding="utf-8")
        safe=[c for c in bundle.get("contracts",[]) if c.get("automation",{}).get("safe_by_default")]
        pytest_lines=["# Generated by ERSEC; review every expectation before enabling in CI.","import pytest", "", "# Draft finding-regression contracts are intentionally non-executing until an application-specific safe predicate is supplied."]
        for i,c in enumerate(safe[:100],1):
            given=c.get("given",{}); url=given.get("url") or "/"; method=given.get("method", "GET")
            if method not in {"GET","HEAD","OPTIONS"}: continue
            fn="test_ersec_contract_"+c["contract_id"].lower()
            pytest_lines += [f"def {fn}():", "    # ERSEC captured a security condition at this endpoint.", f"    # Method: {method}", f"    # URL: {url}", f"    # Contract: {c['contract_id']}", "    # TODO: replace this draft with the application-specific safe predicate.", '    pytest.skip("ERSEC draft contract requires a reviewed application-specific predicate")', ""]
        (root/"pytest_contracts.py").write_text("\n".join(pytest_lines),encoding="utf-8")
        pw_lines=["# Generated by ERSEC. Review before use; this file contains only safe read-only request skeletons."]
        for c in safe[:100]:
            g=c.get("given",{}); m=g.get("method","GET")
            if m not in {"GET","HEAD","OPTIONS"}: continue
            pw_lines += [f"# {c['contract_id']} :: {m} {g.get('url','')}"]
        (root/"playwright_contracts.md").write_text("\n".join(pw_lines)+"\n",encoding="utf-8")
        pw={"info":{"name":"ERSEC Security Contracts","_ersec_schema":ERSEC_CONTRACT_SCHEMA},"item":[]};
        for c in safe[:100]:
            g=c.get("given",{}); m=g.get("method","GET")
            if m not in {"GET","HEAD","OPTIONS"}: continue
            pw["item"].append({"name":c["contract_id"],"request":{"method":m,"url":g.get("url","")}})
        (root/"postman.collection.json").write_text(json.dumps(pw,indent=2),encoding="utf-8")
        openapi={"schema":"ersec-openapi-security-assertions/1","assertions":[]};
        for c in safe[:100]: openapi["assertions"].append({"contract_id":c["contract_id"],"url":c.get("given",{}).get("url"),"method":c.get("given",{}).get("method")})
        (root/"openapi-security-assertions.json").write_text(json.dumps(openapi,indent=2),encoding="utf-8")
        return {"directory":str(root),"contract_count":bundle.get("contract_count",0),"safe_automation_count":len(safe),"files":[str(root/"ersec-contracts.json"),str(root/"pytest_contracts.py"),str(root/"postman.collection.json"),str(root/"openapi-security-assertions.json")]}


class SecurityContractApprover:
    """Promote selected draft contracts into explicitly reviewed active contracts."""
    @staticmethod
    def approve(source_path: str, output_path: str, contract_ids: List[str], reviewer: str, expires: str = "") -> Dict[str, Any]:
        if not reviewer.strip():
            raise ERSECError("A non-empty reviewer is required to approve contracts")
        if not contract_ids:
            raise ERSECError("At least one --contract-id is required for approval")
        with open(source_path, "r", encoding="utf-8") as fh:
            bundle=json.load(fh)
        if not isinstance(bundle,dict) or not isinstance(bundle.get("contracts"),list):
            raise ERSECError("Invalid security contract bundle")
        wanted=set(str(x) for x in contract_ids)
        changed=[]; missing=[]
        for c in bundle["contracts"]:
            if not isinstance(c,dict): continue
            cid=str(c.get("contract_id",""))
            if cid in wanted:
                automation=dict(c.get("automation",{}))
                automation["status"]="active"
                automation["reviewed_by"]=reviewer.strip()
                automation["reviewed_at"]=_utc_now_iso()
                if expires: automation["expires_at"]=expires
                c["automation"]=automation
                changed.append(cid)
        missing=sorted(wanted-set(changed))
        if missing:
            raise ERSECError("Unknown contract id(s): " + ", ".join(missing))
        bundle["approval"]={"reviewer":reviewer.strip(),"approved_at":_utc_now_iso(),"expires_at":expires,"contract_ids":sorted(changed),"principle":"Only explicitly reviewed contracts become CI-enforceable."}
        bundle["schema"]=ERSEC_CONTRACT_SCHEMA
        out=_pathlib.Path(output_path); out.parent.mkdir(parents=True,exist_ok=True); out.write_text(json.dumps(bundle,indent=2,sort_keys=True),encoding="utf-8")
        return {"path":str(out),"approved_count":len(changed),"contract_ids":sorted(changed),"reviewer":reviewer.strip(),"expires_at":expires}


class SecurityContractGate:
    """Evaluate only explicitly reviewed/active contracts against an ERSEC report.

    Draft contracts are never enforcement gates. This is intentionally conservative:
    the gate fails only when a reviewed contract has concrete evidence of regression.
    """
    SCHEMA = "ersec-contract-gate/1"

    @staticmethod
    def load(path: str) -> Dict[str, Any]:
        with open(path, "r", encoding="utf-8") as fh:
            data=json.load(fh)
        if not isinstance(data, dict):
            raise ERSECError("Contract bundle must be a JSON object")
        if data.get("schema") not in {ERSEC_CONTRACT_SCHEMA, "ersec-contract/1"}:
            raise ERSECError(f"Unsupported contract schema: {data.get('schema')!r}")
        return data

    @staticmethod
    def evaluate(bundle: Dict[str, Any], report: Dict[str, Any]) -> Dict[str, Any]:
        now=datetime.datetime.now(datetime.timezone.utc)
        findings=report.get("findings",[]) if isinstance(report,dict) else []
        finding_ids={str(f.get("finding_id")) for f in findings if isinstance(f,dict) and f.get("finding_id")}
        stable_keys=set()
        for f in findings:
            if not isinstance(f,dict): continue
            category=SecurityCategoryRegistry.normalize(f.get("category", "unknown"))
            url=str(f.get("url", ""))
            parsed=urllib.parse.urlparse(url)
            normalized_url=urllib.parse.urlunparse((parsed.scheme.lower(),(parsed.hostname or '').lower(),parsed.path or '/', '', '', ''))
            method=str((f.get("evidence") or {}).get("method") or "GET").upper()
            parameter=str(f.get("parameter") or "")
            material='|'.join([category,method,normalized_url,parameter])
            stable_keys.add('FCON-'+hashlib.sha256(material.encode('utf-8')).hexdigest()[:16])
        failures=[]; checked=[]; skipped=[]; expired=[]
        for c in bundle.get("contracts",[]):
            if not isinstance(c,dict): continue
            automation=c.get("automation",{}) or {}
            status=automation.get("status","draft")
            cid=str(c.get("contract_id",""))
            if status not in {"active","enforced"}:
                skipped.append({"contract_id":c.get("contract_id"),"reason":"not_active","status":status})
                continue
            exp=automation.get("expires_at")
            if exp:
                try:
                    dt=datetime.datetime.fromisoformat(str(exp).replace('Z','+00:00'))
                    if dt.tzinfo is None: dt=dt.replace(tzinfo=datetime.timezone.utc)
                    if dt <= now:
                        expired.append(cid)
                        skipped.append({"contract_id":cid,"reason":"expired","expires_at":exp})
                        continue
                except ValueError:
                    failures.append({"contract_id":cid,"reason":"invalid_expiry","expires_at":exp})
                    continue
            checked.append(cid)
            ctype=c.get("type")
            if ctype == "model-authorization":
                verification = report.get("security_behavior_verification", {}) if isinstance(report, dict) else {}
                target_identity = str(c.get("identity", ""))
                target_resource = str(c.get("model_resource_id", ""))
                observed = [v for v in verification.get("verifications", []) if isinstance(v, dict) and v.get("identity") == target_identity and v.get("resource_id") == target_resource]
                violations = [v for v in observed if v.get("verdict") == "violation"]
                if violations:
                    failures.append({"contract_id":cid,"reason":"model_authorization_regression","matching":"security_behavior_verification","observations":violations[:5]})
                elif not observed or all(v.get("verdict") in {"not_tested", "inconclusive"} for v in observed):
                    skipped.append({"contract_id":cid,"reason":"model_authorization_not_tested","identity":target_identity,"resource_id":target_resource})
            elif ctype == "model-invariant":
                verification = report.get("security_behavior_verification", {}) if isinstance(report, dict) else {}
                iid = str(c.get("invariant_id", ""))
                rows = [v for v in verification.get("invariant_results", []) if isinstance(v, dict) and v.get("invariant_id") == iid]
                if any(v.get("status") == "violated" for v in rows):
                    failures.append({"contract_id":cid,"reason":"model_invariant_regression","matching":"security_behavior_verification","invariant_id":iid})
                elif not rows or all(v.get("status") in {"not_tested", "inconclusive"} for v in rows):
                    skipped.append({"contract_id":cid,"reason":"model_invariant_not_tested","invariant_id":iid})
            elif ctype == "finding-regression":
                fingerprint=str(c.get("fingerprint") or "")
                fid=str(c.get("finding_id") or "")
                if fingerprint and fingerprint in stable_keys:
                    failures.append({"contract_id":cid,"reason":"finding_reappeared","finding_id":fid,"fingerprint":fingerprint,"matching":"stable_contract_identity"})
                elif fid and fid in finding_ids:
                    failures.append({"contract_id":cid,"reason":"finding_reappeared","finding_id":fid,"matching":"legacy_finding_id"})
            elif ctype == "invariant" and bool(c.get("violated")):
                failures.append({"contract_id":cid,"reason":"invariant_marked_violated","statement":c.get("statement","")})
        return {"schema":SecurityContractGate.SCHEMA,"generated_at":_utc_now_iso(),"active_contracts":len([c for c in bundle.get('contracts',[]) if isinstance(c,dict) and (c.get('automation',{}) or {}).get('status') in {'active','enforced'}]),"checked_count":len(checked),"failed_count":len(failures),"failures":failures,"expired_count":len(expired),"expired":expired,"skipped_count":len(skipped),"skipped":skipped,"status":"fail" if failures else "pass","statement":"Only explicitly reviewed active/enforced, non-expired contracts can fail this gate; stable contract identities survive detector execution-order changes."}


def generate_sbom(output_path: str) -> Dict[str, Any]:
    """Generate a dependency SBOM from installed Python distributions in CycloneDX-like JSON."""
    components=[]
    seen=set()
    # Include ERSEC itself from source metadata, then installed distributions.
    components.append({"type":"application","name":"ersec","version":ERSEC_VERSION,"purl":f"pkg:pypi/ersec@{ERSEC_VERSION}"})
    seen.add(("ersec",ERSEC_VERSION))
    for dist in importlib_metadata.distributions():
        try:
            name=dist.metadata.get("Name") or dist.name
            version=dist.version
        except Exception:
            continue
        key=(str(name).lower(),str(version))
        if key in seen: continue
        seen.add(key)
        purl=f"pkg:pypi/{urllib.parse.quote(str(name).lower())}@{urllib.parse.quote(str(version))}"
        license_name=""
        try:
            license_name=dist.metadata.get("License") or ""
        except Exception:
            pass
        comp={"type":"library","name":str(name),"version":str(version),"purl":purl}
        if license_name: comp["licenses"]=[{"license":{"name":str(license_name)}}]
        components.append(comp)
    bom={"bomFormat":"CycloneDX","specVersion":"1.7","serialNumber":"urn:uuid:"+__import__('uuid').uuid4().hex,"version":1,"metadata":{"timestamp":_utc_now_iso(),"component":{"type":"application","name":"ersec","version":ERSEC_VERSION}},"components":sorted(components,key=lambda x:(x.get("type",""),x.get("name","").lower(),x.get("version","")))}
    path=_pathlib.Path(output_path); path.parent.mkdir(parents=True,exist_ok=True); path.write_text(json.dumps(bom,indent=2,sort_keys=True),encoding="utf-8")
    return {"path":str(path),"component_count":len(components),"format":"CycloneDX 1.7","version":ERSEC_VERSION}


# =============================================================================
# ERSEC Shield - runtime application-security enforcement point
# =============================================================================

class ShieldDecision(Enum):
    ALLOW = "allow"
    MONITOR = "monitor"
    BLOCK = "block"


@dataclass
class ShieldRule:
    rule_id: str
    name: str
    category: str
    action: str = "block"
    path_prefix: str = "/"
    parameter: Optional[str] = None
    methods: List[str] = field(default_factory=list)
    patterns: List[str] = field(default_factory=list)
    reason: str = ""
    source_finding_ids: List[str] = field(default_factory=list)
    owner: str = ""
    created_at: str = ""
    expires_at: str = ""
    rollback_command: str = ""
    simulation_status: str = "not_run"


class ShieldPolicy:
    VERSION = "ersec-shield-v1"

    def __init__(self, data: Optional[Dict[str, Any]] = None):
        d = data or {}
        self.version = str(d.get("version", self.VERSION))
        self.defaults = dict(d.get("defaults", {}))
        self.routes = list(d.get("routes", []))
        self.rules: List[ShieldRule] = []
        for raw in d.get("rules", []):
            try:
                self.rules.append(ShieldRule(**{k: raw.get(k) for k in (
                    "rule_id","name","category","action","path_prefix","parameter","methods","patterns","reason","source_finding_ids","owner","created_at","expires_at","rollback_command","simulation_status"
                )}))
            except TypeError:
                continue

    @staticmethod
    def _category_patterns(category: str) -> List[str]:
        # High-signal *families*, deliberately not exploit payloads. Rules generated
        # here are narrow virtual patches and are intended as risk-reduction controls.
        families = {
            "sql_injection_signal": [r"(?i)(?:\bunion\b\s+\bselect\b|\bor\b\s+\d+\s*=\s*\d+|\band\b\s+\d+\s*=\s*\d+)", r"(?i)(?:sleep\s*\(|benchmark\s*\(|pg_sleep\s*\()"],
            "nosql_injection_signal": [r"(?i)(?:\$where\b|\$regex\b|\$ne\b|\$gt\b|\$gte\b|\$lt\b|\$lte\b)"],
            "ldap_injection_signal": [r"(?i)(?:\*\)|\)\s*\(|\|\s*\()"],
            "xss_signal": [r"(?is)(?:<\s*script\b|javascript\s*:|on(?:error|load|click|mouseover)\s*=)"],
            "ssti_signal": [r"(?s)(?:\{\{[^\n]{0,160}\}\}|\{%[^\n]{0,160}%\}|\$\{[^\n]{0,160}\})"],
            "path_traversal_signal": [r"(?i)(?:\.\./|\.\\\\|%2e%2e%2f|%252e%252e%252f)"],
            "crlf_injection": [r"(?i)(?:%0d|%0a|\\r|\\n)(?:location:|set-cookie:|content-length:)"],
            "prototype_pollution_signal": [r"(?i)(?:__proto__|constructor\[prototype\]|constructor\.prototype)"],
            "command_injection_timing": [r"(?i)(?:\$\([^)]{1,120}\)|`[^`]{1,120}`|(?:^|[;&|])\s*(?:id|whoami|uname|sleep|ping)\b)"],
            "xxe_signal": [r"(?is)(?:<!DOCTYPE[^>]{0,300}(?:ENTITY|SYSTEM)|<!ENTITY\b)"],
        }
        return families.get(SecurityCategoryRegistry.normalize(category), families.get(category, []))

    @classmethod
    def from_report(cls, report: Dict[str, Any]) -> "ShieldPolicy":
        rules: List[Dict[str, Any]] = []
        for f in report.get("findings", []):
            conf = str(f.get("confidence", "")).lower()
            sev = str(f.get("severity", "")).upper()
            category = str(f.get("category", ""))
            patterns = cls._category_patterns(category)
            # Auto-generated runtime blocks are intentionally restricted to high-confidence
            # evidence. The default is monitoring for lower-confidence observations.
            if not patterns or conf not in {"confirmed", "likely"}:
                continue
            action = "block" if conf == "confirmed" and sev in {"CRITICAL", "HIGH", "MEDIUM"} else "monitor"
            path = urllib.parse.urlparse(str(f.get("url", "/"))).path or "/"
            rule_id = "VP-" + hashlib.sha256((category + "|" + path + "|" + str(f.get("parameter"))).encode()).hexdigest()[:12]
            rules.append({
                "rule_id": rule_id,
                "name": "virtual-patch-" + category,
                "category": category,
                "action": action,
                "path_prefix": path,
                "parameter": f.get("parameter"),
                "methods": ["GET","POST","PUT","PATCH","DELETE"],
                "patterns": patterns,
                "reason": f.get("title", category),
                "source_finding_ids": [f.get("finding_id", "")],
                "owner": "security-engineering",
                "created_at": _utc_now_iso(),
                "expires_at": (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ"),
                "rollback_command": f"remove rule {rule_id} from the Shield policy and redeploy",
                "simulation_status": "review-required",
            })
        return cls({
            "version": cls.VERSION,
            "generated_at": _utc_now_iso(),
            "defaults": {
                "mode": "block",
                "max_body_bytes": 2 * 1024 * 1024,
                "max_url_length": 8192,
                "rate_per_minute": 120,
                "burst": 30,
                "methods": ["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS"],
            },
            "rules": rules,
        })

    def to_dict(self) -> Dict[str, Any]:
        return {
            "version": self.version,
            "defaults": self.defaults,
            "routes": self.routes,
            "rules": [asdict(r) for r in self.rules],
        }

    def save(self, path: str) -> None:
        os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
        tmp = path + ".tmp"
        with open(tmp, "w", encoding="utf-8") as fh:
            json.dump({**self.to_dict(), "generated_at": _utc_now_iso()}, fh, indent=2, sort_keys=True)
        os.replace(tmp, path)


class ShieldPolicyCompiler:
    """Compile a scan report into a narrow positive/virtual-patch policy.

    This never claims to replace application fixes. It produces compensating controls
    around *observed vulnerable boundaries* and keeps lower-confidence signals in monitor mode.
    """
    def compile_report(self, report_path: str, output_path: Optional[str] = None) -> Dict[str, Any]:
        with open(report_path, "r", encoding="utf-8") as fh:
            report = json.load(fh)
        policy = ShieldPolicy.from_report(report)
        if output_path:
            policy.save(output_path)
        return policy.to_dict()


class ShieldTelemetry:
    def __init__(self, path: Optional[str]):
        self.path = path
        self.counts = defaultdict(int)
        self._lock = threading.Lock()

    @staticmethod
    def _hash_ip(ip: str) -> str:
        return hashlib.sha256(ip.encode("utf-8", "ignore")).hexdigest()[:16]

    def emit(self, event: Dict[str, Any]) -> None:
        event = dict(event)
        event["timestamp"] = _utc_now_iso()
        if event.get("client_ip"):
            event["client_ip"] = self._hash_ip(str(event["client_ip"]))
        with self._lock:
            self.counts[str(event.get("decision", "unknown"))] += 1
            if self.path:
                try:
                    os.makedirs(os.path.dirname(os.path.abspath(self.path)), exist_ok=True)
                    with open(self.path, "a", encoding="utf-8") as fh:
                        fh.write(json.dumps(event, sort_keys=True) + "\n")
                except OSError:
                    pass


class ShieldRateLimiter:
    def __init__(self, rate_per_minute: int, burst: int):
        self.rate_per_minute = max(1, int(rate_per_minute))
        self.burst = max(1, int(burst))
        self._buckets: Dict[str, deque] = defaultdict(deque)
        self._lock = threading.Lock()

    def allow(self, key: str) -> Tuple[bool, int]:
        now = time.monotonic()
        window = 60.0
        with self._lock:
            q = self._buckets[key]
            while q and now - q[0] > window:
                q.popleft()
            # Burst is a short-window guard; sustained rate is the one-minute ceiling.
            if len(q) >= self.rate_per_minute:
                retry = max(1, int(window - (now - q[0])))
                return False, retry
            recent = sum(1 for ts in q if now - ts <= 1.0)
            if recent >= self.burst:
                return False, 1
            q.append(now)
            return True, 0


class ShieldRuntime:
    HOP_BY_HOP = {"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade"}

    def __init__(self, config: ScanConfig, policy: ShieldPolicy):
        self.config = config
        self.policy = policy
        d = policy.defaults
        self.mode = config.shield_mode or d.get("mode", "block")
        self.max_body_bytes = int(d.get("max_body_bytes", config.shield_max_body_bytes))
        self.max_url_length = int(d.get("max_url_length", config.shield_max_url_length))
        self.allowed_methods = set(str(x).upper() for x in d.get("methods", ["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS"]))
        self.rate = ShieldRateLimiter(int(d.get("rate_per_minute", config.shield_rate_per_minute)), int(d.get("burst", config.shield_burst)))
        self.telemetry = ShieldTelemetry(config.shield_log_path)
        self.learn_lock = threading.Lock()
        self.learned_routes: Dict[str, Dict[str, Any]] = {}
        self.learning_path = config.shield_learning_path
        self.start_monotonic = time.monotonic()
        self.events = 0

    def load_learning(self) -> None:
        if not self.learning_path or not os.path.exists(self.learning_path):
            return
        try:
            with open(self.learning_path, "r", encoding="utf-8") as fh:
                data = json.load(fh)
            self.learned_routes = dict(data.get("routes", {}))
        except (OSError, json.JSONDecodeError):
            self.learned_routes = {}

    def save_learning(self) -> None:
        if not self.learning_path:
            return
        try:
            os.makedirs(os.path.dirname(os.path.abspath(self.learning_path)), exist_ok=True)
            tmp = self.learning_path + ".tmp"
            with self.learn_lock:
                payload = {"version":"ersec-shield-learning-v1", "generated_at":_utc_now_iso(), "routes":self.learned_routes}
            with open(tmp, "w", encoding="utf-8") as fh:
                json.dump(payload, fh, indent=2, sort_keys=True)
            os.replace(tmp, self.learning_path)
        except OSError:
            pass

    def _record_route(self, path: str, method: str, status: int) -> None:
        key = path.split("?",1)[0]
        with self.learn_lock:
            item = self.learned_routes.setdefault(key, {"methods":{}, "observations":0})
            item["observations"] += 1
            item["methods"][method] = int(item["methods"].get(method, 0)) + 1
            item["last_status"] = status

    def _normalize_path(self, raw_path: str) -> str:
        parsed = urllib.parse.urlsplit(raw_path)
        path = parsed.path or "/"
        # Remove dot-segments without decoding arbitrary payload data into a new target.
        parts=[]
        for seg in path.split("/"):
            if seg in ("", "."):
                continue
            if seg == "..":
                if parts: parts.pop()
            else:
                parts.append(seg)
        return "/" + "/".join(parts)

    def _generic_request_checks(self, method: str, raw_path: str, headers: Dict[str,str], body: bytes, client_ip: str) -> Tuple[ShieldDecision, str, Optional[ShieldRule]]:
        if len(raw_path) > self.max_url_length:
            return ShieldDecision.BLOCK, "url_length_limit", None
        if method.upper() not in self.allowed_methods:
            return ShieldDecision.BLOCK, "method_not_allowed", None
        if len(body) > self.max_body_bytes:
            return ShieldDecision.BLOCK, "request_body_limit", None
        ok, retry = self.rate.allow(client_ip)
        if not ok:
            return ShieldDecision.BLOCK, f"rate_limit;retry_after={retry}", None
        parsed = urllib.parse.urlsplit(raw_path)
        path = parsed.path or "/"
        normalized = self._normalize_path(raw_path)
        lower_path = path.lower()
        encoded_traversal = any(tok in lower_path for tok in ("%2e", "%5c", "%252e", "%255c"))
        if (normalized != path and ".." in path) or encoded_traversal or "/../" in path or path.endswith("/.."):
            return ShieldDecision.BLOCK, "path_normalization_anomaly", None
        # Reject ambiguous framing requests at the gateway edge.
        if headers.get("transfer-encoding") and headers.get("content-length"):
            return ShieldDecision.BLOCK, "ambiguous_request_framing", None
        return ShieldDecision.ALLOW, "", None

    def _rule_match(self, rule: ShieldRule, method: str, raw_path: str, body_text: str) -> bool:
        parsed = urllib.parse.urlsplit(raw_path)
        path = parsed.path or "/"
        if not path.startswith(rule.path_prefix or "/"):
            return False
        if rule.methods and method.upper() not in {m.upper() for m in rule.methods}:
            return False
        subject = body_text
        if rule.parameter:
            values=[]
            qs=urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
            values.extend(qs.get(rule.parameter, []))
            try:
                content_type = ""
                for k,v in []: pass
            except Exception:
                pass
            if values:
                subject = "&".join(values)
            else:
                # For JSON form parameters, conservatively inspect the complete small body.
                subject = body_text
        for pat in rule.patterns:
            try:
                if re.search(pat, subject):
                    return True
            except re.error:
                continue
        return False

    def inspect(self, method: str, raw_path: str, headers: Dict[str,str], body: bytes, client_ip: str) -> Tuple[ShieldDecision, str, Optional[ShieldRule]]:
        decision, reason, _ = self._generic_request_checks(method, raw_path, headers, body, client_ip)
        if decision == ShieldDecision.BLOCK:
            return decision, reason, None
        body_text = body[:self.max_body_bytes].decode("utf-8", "replace")
        for rule in self.policy.rules:
            if self._rule_match(rule, method, raw_path, body_text):
                action = rule.action.lower()
                if self.mode == "monitor" or action == "monitor":
                    return ShieldDecision.MONITOR, rule.reason or rule.rule_id, rule
                if self.mode == "learn":
                    return ShieldDecision.MONITOR, "learning:" + (rule.reason or rule.rule_id), rule
                return ShieldDecision.BLOCK, rule.reason or rule.rule_id, rule
        return ShieldDecision.ALLOW, "", None

    def inspect_and_record(self, method: str, raw_path: str, headers: Dict[str,str], body: bytes, client_ip: str) -> Tuple[ShieldDecision, str, Optional[ShieldRule]]:
        decision, reason, rule = self.inspect(method, raw_path, headers, body, client_ip)
        self.events += 1
        self.telemetry.emit({
            "component":"ersec-shield",
            "decision":decision.value,
            "reason":reason,
            "rule_id":rule.rule_id if rule else None,
            "category":rule.category if rule else None,
            "method":method,
            "path":urllib.parse.urlsplit(raw_path).path or "/",
            "client_ip":client_ip,
            "body_bytes":len(body),
        })
        return decision, reason, rule

    def status(self) -> Dict[str, Any]:
        return {
            "engine":"ERSEC Shield",
            "version":"1.0",
            "mode":self.mode,
            "rules":len(self.policy.rules),
            "events":self.events,
            "uptime_seconds":round(time.monotonic()-self.start_monotonic,2),
            "decisions":dict(self.telemetry.counts),
            "learning_routes":len(self.learned_routes),
        }


class ERSECShieldServer:
    """Threaded reverse proxy implementing a local application-security gateway.

    Default bind is localhost. Public exposure should be performed only deliberately,
    normally behind TLS and a separate network access-control layer.
    """
    def __init__(self, config: ScanConfig, policy: ShieldPolicy):
        self.config=config
        self.policy=policy
        self.runtime=ShieldRuntime(config,policy)
        self.runtime.load_learning()
        upstream=config.shield_upstream.rstrip("/")
        parsed=urllib.parse.urlsplit(upstream)
        if parsed.scheme not in {"http","https"} or not parsed.hostname:
            raise ERSECError("--shield-upstream must be an http:// or https:// URL")
        self.upstream=upstream
        self.upstream_parts=parsed
        runtime=self.runtime

        class Handler(BaseHTTPRequestHandler):
            protocol_version="HTTP/1.1"
            server_version="ERSEC-Shield/1.0"
            sys_version=""

            def log_message(self, fmt, *args):
                if config.verbose:
                    print("[shield] " + (fmt % args))

            def _headers_dict(self):
                return {k.lower(): v for k,v in self.headers.items()}

            def _read_body(self):
                raw_len=self.headers.get("Content-Length")
                if raw_len:
                    try:
                        n=int(raw_len)
                    except ValueError:
                        raise ERSECError("invalid content-length")
                    if n < 0 or n > runtime.max_body_bytes:
                        raise ERSECError("request body exceeds shield limit")
                    return self.rfile.read(n)
                return b""

            def _blocked(self, code:int, reason:str):
                body=json.dumps({"error":"blocked_by_ersec_shield","reason":reason,"version":"1.0"}).encode()
                self.send_response(code)
                self.send_header("Content-Type","application/json")
                self.send_header("Content-Length",str(len(body)))
                self.send_header("Cache-Control","no-store")
                self.end_headers()
                try: self.wfile.write(body)
                except BrokenPipeError: pass

            def _proxy(self):
                client_ip=self.client_address[0]
                try:
                    body=self._read_body()
                except ERSECError as exc:
                    runtime.telemetry.emit({"component":"ersec-shield","decision":"block","reason":str(exc),"method":self.command,"path":urllib.parse.urlsplit(self.path).path or "/","client_ip":client_ip})
                    return self._blocked(413,"request_body_limit")

                decision, reason, rule=runtime.inspect_and_record(self.command,self.path,self._headers_dict(),body,client_ip)
                if decision == ShieldDecision.BLOCK:
                    status=429 if reason.startswith("rate_limit") else 403
                    return self._blocked(status,reason)

                if runtime.mode == "learn":
                    # Learning mode records shape only; it never stores the request body.
                    try:
                        out=requests.request(self.command, self._target_url(), data=body, headers=self._forward_headers(), timeout=config.shield_backend_timeout, verify=config.verify_tls, allow_redirects=False)
                        runtime._record_route(self.path,self.command,out.status_code)
                        runtime.save_learning()
                        return self._respond(out)
                    except requests.RequestException as exc:
                        runtime.telemetry.emit({"component":"ersec-shield","decision":"backend_error","reason":str(exc),"method":self.command,"path":urllib.parse.urlsplit(self.path).path or "/"})
                        return self._blocked(502,"upstream_unavailable")

                # Monitor/block modes use the same forwarding path for allowed traffic.
                try:
                    out=requests.request(self.command, self._target_url(), data=body, headers=self._forward_headers(), timeout=config.shield_backend_timeout, verify=config.verify_tls, allow_redirects=False)
                    runtime._record_route(self.path,self.command,out.status_code)
                    return self._respond(out)
                except requests.RequestException as exc:
                    runtime.telemetry.emit({"component":"ersec-shield","decision":"backend_error","reason":str(exc),"method":self.command,"path":urllib.parse.urlsplit(self.path).path or "/"})
                    return self._blocked(502,"upstream_unavailable")

            def _target_url(self):
                parsed=urllib.parse.urlsplit(self.path)
                base=self.server.ersec_upstream.rstrip("/")
                return base + (parsed.path or "/") + (("?"+parsed.query) if parsed.query else "")

            def _forward_headers(self):
                out={}
                for k,v in self.headers.items():
                    if k.lower() in ShieldRuntime.HOP_BY_HOP: continue
                    if k.lower() == "host": continue
                    out[k]=v
                out["Host"]=self.server.upstream_host
                out["Via"]="ERSEC-Shield/1.0"
                return out

            def _respond(self, out):
                content=out.content
                self.send_response(out.status_code)
                for k,v in out.headers.items():
                    if k.lower() in ShieldRuntime.HOP_BY_HOP: continue
                    if k.lower() == "content-length": continue
                    self.send_header(k,v)
                self.send_header("Content-Length",str(len(content)))
                self.end_headers()
                try:
                    if self.command != "HEAD": self.wfile.write(content)
                except BrokenPipeError: pass

            def do_GET(self): self._proxy()
            def do_HEAD(self): self._proxy()
            def do_POST(self): self._proxy()
            def do_PUT(self): self._proxy()
            def do_PATCH(self): self._proxy()
            def do_DELETE(self): self._proxy()
            def do_OPTIONS(self): self._proxy()

        self.httpd=ThreadingHTTPServer((config.shield_bind, config.shield_port), Handler)
        self.httpd.ersec_upstream=self.upstream
        self.httpd.upstream_host=self.upstream_parts.netloc
        self.httpd.ersec_runtime=self.runtime
        self._handler_cls=Handler
        if self.upstream_parts.scheme == "https":
            # Client-side TLS verification stays under the scan configuration.
            pass
        if config.shield_tls_cert or config.shield_tls_key:
            if not (config.shield_tls_cert and config.shield_tls_key):
                raise ERSECError("--shield-tls-cert and --shield-tls-key must be supplied together")
            ctx=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
            ctx.minimum_version=ssl.TLSVersion.TLSv1_2
            ctx.load_cert_chain(config.shield_tls_cert, config.shield_tls_key)
            self.httpd.socket=ctx.wrap_socket(self.httpd.socket, server_side=True)

    def serve_forever(self) -> None:
        print(f"[*] ERSEC Shield 1.0 listening on {self.config.shield_bind}:{self.config.shield_port}")
        print(f"[*] Upstream: {self.upstream}")
        print(f"[*] Mode: {self.runtime.mode} | Rules: {len(self.policy.rules)} | Default bind is intentionally local")
        print("[!] Authorized/defensive use only. A virtual patch is compensating control, not a source-code fix.")
        try:
            self.httpd.serve_forever(poll_interval=0.4)
        except KeyboardInterrupt:
            print("\n[!] ERSEC Shield stopped by operator.")
        finally:
            self.httpd.server_close()
            self.runtime.save_learning()


def load_shield_policy(config: ScanConfig) -> ShieldPolicy:
    if config.shield_from_report:
        policy = ShieldPolicyCompiler().compile_report(config.shield_from_report, config.shield_policy_path)
        return ShieldPolicy(policy)
    if config.shield_policy_path:
        with open(config.shield_policy_path, "r", encoding="utf-8") as fh:
            return ShieldPolicy(json.load(fh))
    return ShieldPolicy({"version":ShieldPolicy.VERSION, "defaults":{}})


class ERSECScanner:
    def __init__(self, config: ScanConfig, plugin_dir: Optional[str] = None):
        self.config = config
        if not hasattr(config, "_progress"):
            config._progress = ScanProgress()
        self.progress: ScanProgress = config._progress
        self.client = SafeHttpClient(config)
        self.crawler = WebCrawler(config, self.client)
        self.ai = AdvisorEngine(config)
        self.ledger = ScanExecutionLedger()

        all_modules = [
            SecurityHeadersModule(config, self.client),
            ClickjackingRenderModule(config, self.client),
            SecurityTxtModule(config, self.client),
            TLSConfigModule(config, self.client),
            ServerBannerModule(config, self.client),
            HTTPMethodsModule(config, self.client),
            CORSModule(config, self.client),
            ExposedFilesModule(config, self.client),
            SQLInjectionSignalModule(config, self.client),
            ReflectedXSSSignalModule(config, self.client),
            DOMXSSSinkSignalModule(config, self.client),
            OpenRedirectModule(config, self.client),
            NoSQLInjectionSignalModule(config, self.client),
            SSRFSignalModule(config, self.client),
            SSTISignalModule(config, self.client),
            XXESignalModule(config, self.client),
            XPathInjectionSignalModule(config, self.client),
            CommandInjectionTimingSignalModule(config, self.client),
            JWTInspectionModule(config, self.client),
            DirectoryListingModule(config, self.client),
            VerboseErrorModule(config, self.client),
            MixedContentModule(config, self.client),
            SubresourceIntegrityModule(config, self.client),
            InsecureFormModule(config, self.client),
            GraphQLIntrospectionModule(config, self.client),
            CRLFInjectionSignalModule(config, self.client),
            HostHeaderInjectionModule(config, self.client),
            VulnerableLibraryModule(config, self.client),
            InfoDisclosureCommentsModule(config, self.client),
            CacheControlModule(config, self.client),
            LDAPInjectionSignalModule(config, self.client),
            AdminPanelExposureModule(config, self.client),
            HeaderInjectionModule(config, self.client),
            WeakSessionTokenModule(config, self.client),
            PrototypePollutionSignalModule(config, self.client),
            RateLimitingSignalModule(config, self.client),
            WebSocketExposureModule(config, self.client),
            DNSEmailSecurityModule(config, self.client),
            SubdomainTakeoverSignalModule(config, self.client),
            JWTKeyConfusionSignalModule(config, self.client),
            SensitiveDataExposureModule(config, self.client),
            AuthenticatedCacheLeakModule(config, self.client),
            ExcessiveDataExposureModule(config, self.client),
            HTTPParameterPollutionModule(config, self.client),
            CacheDeceptionSignalModule(config, self.client),
            SensitiveResponseHeaderModule(config, self.client),
            PathTraversalSignalModule(config, self.client),
            APIErrorLeakageModule(config, self.client),
            APIResponseFieldExposureModule(config, self.client),
            ClientSideCredentialLeakModule(config, self.client),
            SRIIntegrityMismatchModule(config, self.client),
            PostMessageOriginModule(config, self.client),
            DOMOpenRedirectModule(config, self.client),
            GraphQLSensitiveSchemaModule(config, self.client),
            BackupArtifactExpansionModule(config, self.client),
            APIContentTypeConfusionModule(config, self.client),
            StructuredParameterDifferentialModule(config, self.client),
            APIResponseConsistencyModule(config, self.client),
            APIVersionDriftModule(config, self.client),
            SecurityHeaderDriftModule(config, self.client),
            SensitiveCookiePathModule(config, self.client),
            JSONDuplicateKeyDifferentialModule(config, self.client),
            SourceMapExposureModule(config, self.client),
            ServiceWorkerExposureModule(config, self.client),
            OAuthStateSignalModule(config, self.client),
            PasswordRecoveryLeakageModule(config, self.client),
            CookieScopeModule(config, self.client),
            PathCanonicalizationModule(config, self.client),
            ForwardedHostTrustModule(config, self.client),
            ContentNegotiationAuthModule(config, self.client),
            FileUploadSurfaceModule(config, self.client),
            CORSPreflightConsistencyModule(config, self.client),
            APIErrorMethodLeakageModule(config, self.client),
        ]
        if plugin_dir:
            all_modules.extend(load_plugin_modules(plugin_dir, config, self.client))

        all_modules = [InstrumentedDetector(m, self.ledger) for m in all_modules]

        self.idor_module = IDORHeuristicModule(config, self.client)
        self.mass_assignment_module = MassAssignmentSignalModule(config, self.client)
        self.user_enum_module = UserEnumerationSignalModule(config, self.client)
        self.stored_xss_module = StoredXSSHeuristicModule(config, self.client)
        self.detector_registry = DetectorRegistry(all_modules)
        self.url_modules = [m for m in all_modules if m.applies() and hasattr(m, "run_url")]
        self.param_modules = [m for m in all_modules if m.applies()]
        self.csrf_module = CSRFModule(config, self.client)
        self.maturity = MaturityOrchestrator(config, self.crawler, self.client)
        self.revolution = ERSEC7Orchestrator(config, self.crawler, self.client)
        self.campaign = DetectionCampaignEngine(self)
        self.juice_shop = JuiceShopCoverageEngine(config, self.client)
        self.juice_shop_source = JuiceShopSourceEvidenceEngine(config, self.client)
        self.boundary_differential = SecurityBoundaryDifferentialEngine(config, self.client)
        self.boundary_adapter = SecurityBoundaryFindingAdapter(config)
        self.workflow_engine = MultiIdentityWorkflowEngine(config, self.client, self.crawler)
        self.config._ersec_modules = all_modules + [self.idor_module, self.mass_assignment_module, self.user_enum_module, self.stored_xss_module, self.csrf_module]
        self.causal_impact_engine = CausalImpactEngine(config, self.crawler)
        self.autonomous_agent = AutonomousRedAgent(config, self.crawler, self.client, self.ledger)
        self.federated_mesh = FederatedKnowledgeMesh(config)
        self.metamorphic_lam = MetamorphicLAM(config, self.client)
        self.remediation_twin = RemediationTwin(config)
        self.contract_drift_sentry = ContractDriftSentry(config)
        self.observed_object_auth = ObservedObjectAuthorizationEngine(config, self.client)
        self.method_auth_matrix = HTTPMethodAuthorizationMatrixEngine(config, self.client)
        self.security_decision_lattice = SecurityDecisionLatticeEngine(config, self.client)
        self.response_anomaly_ensemble = ResponseAnomalyEnsemble()
        self.control_plane_engine = SecurityControlPlane(config)

    def _parallel_param_checks(self, jobs: List[Tuple[Any, str, str, str]]) -> List[Finding]:
        """Execute independent parameter probes concurrently.

        Stateful form workflows remain sequential. SafeHttpClient supplies one
        requests.Session per worker while preserving the shared request budget,
        global rate limit and scope checks.
        """
        if not jobs:
            return []
        workers = max(1, min(self.config.scope.concurrency, len(jobs)))
        findings: List[Finding] = []
        def execute(job):
            module, url, param, method = job
            return module.run_param(url, param, method)
        with ThreadPoolExecutor(max_workers=workers) as executor:
            futures = [executor.submit(execute, job) for job in jobs]
            for future in as_completed(futures):
                try:
                    findings.extend(future.result() or [])
                except (ScopeError, ERSECError) as e:
                    if self.config.verbose:
                        print(f"  [!] parameter probe skipped: {e}")
                except Exception as e:
                    if self.config.verbose:
                        print(f"  [!] parameter detector error: {e}")
        return findings

    def _signal_finding(self, signal):
        ev=RequestEvidence(method="DIFFERENTIAL", url=signal.get("url", self.config.target), status_code=0, response_time_ms=0, response_headers={}, response_excerpt=signal.get("reason","")[:800], request_headers_sent={})
        return Finding(finding_id=_next_id(), category=signal.get("finding","deep_signal"), title=signal.get("finding","Deep security signal").replace("_"," ").title(), severity=Severity[signal.get("severity","LOW")], confidence=signal.get("confidence","Possible"), owasp="A01:2021", cwe="CWE-285", url=signal.get("url",self.config.target), parameter=None, description=signal.get("reason",""), evidence=ev, remediation_summary="Review authorization consistency across HTTP methods for this resource.")

    def run(self, target: str) -> Dict[str, Any]:
        start_url = target if target.startswith(("http://", "https://")) else f"https://{target}"
        self.progress.set_stage("discovery", "crawling HTML, metadata and discovered assets")
        endpoints, forms = self.crawler.crawl(start_url)
        self.progress.pages = len(self.crawler.visited)

        if self.crawler.successful_fetches == 0:
            self.progress.mark_finished(False)
            # The target was never actually reached - not even once. Reporting
            # a clean "A+, no findings" here would be actively dangerous: it's
            # indistinguishable from a genuinely secure target unless someone
            # reads every line of --verbose output. A failed connection must
            # never look like a passed scan.
            return {
                "target": target,
                "profile": self.config.profile.value,
                "scanned_at": _utc_now_iso(),
                "scan_status": "unreachable",
                "scan_status_detail": (
                    f"Every request to '{start_url}' failed - the scanner never successfully connected to "
                    "the target, so there is no data to report. Check the URL for typos (a doubled scheme "
                    "like 'https://https://...' is a common one), confirm the host resolves and is reachable "
                    "from this machine, and re-run with --verbose to see the specific connection errors."
                ),
                "pages_crawled": 0,
                "requests_made": self.client.request_count,
                "stack_fingerprint": {},
                "findings": [],
                "risk_chains": [],
                "attack_paths": [],
                "posture_score": None,
                "posture_grade": "N/A",
                "posture_model": "not computed - target unreachable",
                "asset_inventory": build_asset_inventory(self.crawler, self.client),
                "summary": {sev.name: 0 for sev in Severity},
            }

        if not endpoints:
            endpoints = [start_url]

        campaign_preflight = self.campaign.run_preflight(endpoints)

        # Stack fingerprint from the first successful response
        stack_fingerprint: Dict[str, str] = {}
        try:
            resp0 = self.client.request("GET", start_url)
            stack_fingerprint = fingerprint_stack(resp0)
        except (ScopeError, ERSECError):
            pass

        findings: List[Finding] = []
        self.progress.set_stage("detection", f"running {len(self.url_modules)} URL-level detectors")

        # URL-level (mostly passive + single-shot) checks - once per distinct path, capped
        checked_urls = set()
        # These test fixed, host-absolute paths (urljoin("/x", any_page_url) always
        # resolves to the same URL regardless of which page triggered it) or
        # genuinely host-wide behavior - running them once per crawled page was
        # pure redundancy: the exact same 50+ requests fired again for every
        # single page, which is what actually exhausts the request budget on
        # any real multi-page site, not the per-parameter injection checks.
        HOST_LEVEL = {"tls", "server_banner", "graphql_introspection", "http_methods",
                      "exposed_files", "directory_listing", "admin_exposure", "host_header",
                      "email_security", "security_txt", "subdomain_takeover", "source_map_exposure", "service_worker_exposure", "client_side_credential_leak", "backup_artifact_exposure"}
        host_level_done: Set[str] = set()
        for url in endpoints[: self.config.scope.max_crawl_pages]:
            base = url.split("?")[0]
            if base in checked_urls:
                continue
            checked_urls.add(base)
            for module in self.url_modules:
                if module.category in HOST_LEVEL:
                    if module.category in host_level_done:
                        continue
                    host_level_done.add(module.category)
                try:
                    findings.extend(module.run_url(url))
                except (ScopeError, ERSECError) as e:
                    if self.config.verbose:
                        print(f"  [!] {module.category} on {url}: {e}")

        # Parameter-level checks on crawled query strings
        if self.config.profile != ScanProfile.PASSIVE:
            self.progress.set_stage("parameter-analysis", "testing observed and inferred parameters")
            observed_jobs: List[Tuple[Any, str, str, str]] = []
            for url in endpoints:
                parsed = urllib.parse.urlparse(url)
                if not parsed.query:
                    continue
                params = dict(urllib.parse.parse_qsl(parsed.query))
                for param in params:
                    for module in self.param_modules:
                        if hasattr(module, "run_param"):
                            observed_jobs.append((module, url, param, "GET"))
            findings.extend(self._parallel_param_checks(observed_jobs))

            # Form-level checks
            for form in self.crawler.forms:
                findings.extend(self.csrf_module.check_form(form))
                target_url = form.action or form.page_url
                for inp in form.inputs:
                    param = inp["name"]
                    if inp.get("type") in ("submit", "button", "hidden"):
                        continue
                    for module in self.param_modules:
                        if not hasattr(module, "run_param"):
                            continue
                        try:
                            findings.extend(module.run_param(target_url, param, form.method))
                        except (ScopeError, ERSECError) as e:
                            if self.config.verbose:
                                print(f"  [!] {module.category} form param {param}: {e}")

            # API parameter guessing: for API-shaped endpoints discovered with
            # NO query string (the norm for SPA/REST backends whose static HTML
            # exposes no <a href>/<form> for a classic crawler to find params
            # from - Angular/React/Vue apps overwhelmingly look like this),
            # synthesize candidate parameter names on both GET query string and
            # POST JSON body, and run the same injection checks against them.
            # This is what actually closes the "modern SPA scan found nothing"
            # gap - most of a real REST API's attack surface never appears as
            # a query string anywhere in the page source for a static crawler
            # to discover on its own.
            #
            # Deliberately capped tight: this is a combinatorial pass (endpoints
            # x params x modules x methods), and several modules issue more
            # than one HTTP request per run_param call - an uncapped version of
            # this could burn the entire request budget on guesses alone and
            # starve the checks that already had a confirmed real parameter to
            # test. 8 endpoints x 4 params is a deliberately conservative
            # starting point; raise --max-requests if scanning a large API surface.
            # Seed API probing from both conservative built-ins and parameters learned
            # from observed JSON response schemas. This materially improves coverage for
            # modern APIs whose real parameters never appear in the HTML query string.
            schema_hint_params = []
            try:
                schema_hint = JSONSchemaMiner().mine(self.client, self.crawler.api_like_endpoints, limit=12)
                schema_hint_params = schema_hint.get("candidate_parameters", [])[:20]
            except Exception:
                schema_hint = {}
            planner_params=[]
            try:
                planner=AdaptiveCoveragePlanner(self.config).build(self.crawler.endpoints,self.crawler.forms,schema_hint_params)
                planner_params=[x["parameter"] for x in planner.get("candidates",[])[:self.config.max_context_probes]]
            except Exception:
                planner_params=[]
            candidate_params = list(dict.fromkeys(["q", "id", "email", "search", "name", "page", "limit", "offset", "sort", "order", "filter", "fields", "select", "include", "next", "return", "return_url", "redirect", "callback", "url", "uri", "path", "file", "filename", "template", "query", "expr", "role", "is_admin", "owner_id", "account_id", "user_id", "tenant_id", "price", "quantity", "discount", "coupon", "token", "code", "state", "provider", "format", "view", "debug", "redirect_uri", "webhook"] + schema_hint_params + planner_params))
            coverage_mode = getattr(self.config, "coverage_mode", None) or getattr(self.config.scope, "coverage_mode", "maximum")
            if coverage_mode == "conservative": candidate_params=candidate_params[:12]
            elif coverage_mode == "balanced": candidate_params=candidate_params[:24]
            else: candidate_params=candidate_params[:40]
            api_jobs: List[Tuple[Any, str, str, str]] = []
            for api_url in list(self.crawler.api_like_endpoints)[:12]:
                for param in candidate_params:
                    for module in self.param_modules:
                        if hasattr(module, "run_param"):
                            api_jobs.append((module, api_url, param, "GET"))
                            api_jobs.append((module, api_url, param, "POST"))
            findings.extend(self._parallel_param_checks(api_jobs))

            # IDOR heuristic pass - operates over the crawler's observed numeric-ID
            # URL patterns as a whole, not a single URL, so it runs once here
            # rather than being folded into the per-URL/per-param loops above.
            try:
                findings.extend(self.idor_module.run_for_crawler(self.crawler.numeric_id_endpoints))
            except (ScopeError, ERSECError) as e:
                if self.config.verbose:
                    print(f"  [!] idor_heuristic: {e}")

            # Form-shaped heuristics that need the WHOLE form (its own field
            # set, or a submit-then-revisit sequence) rather than a single
            # parameter in isolation - each runs its own internally-capped
            # loop over self.crawler.forms.
            for label, runner in (
                ("mass_assignment", self.mass_assignment_module.run_for_forms),
                ("user_enumeration", self.user_enum_module.run_for_forms),
                ("stored_xss", self.stored_xss_module.run_for_forms),
            ):
                try:
                    findings.extend(runner(self.crawler.forms))
                except (ScopeError, ERSECError) as e:
                    if self.config.verbose:
                        print(f"  [!] {label}: {e}")

        # Advanced detection pass: authorization differential, security-boundary discovery,
        # and JSON response schema mining. These create evidence-backed *review signals*;
        # they do not claim a bypass merely from similarity.
        advanced_detection = {}
        try:
            advanced_detection = enrich_advanced_detection(findings, self.config, self.crawler, self.client, start_url)
            if self.config.verbose:
                print(f"  [+] advanced detection: {advanced_detection.get('authorization_differential_count',0)} authorization-differential signals")
        except Exception as e:
            advanced_detection = {"error": str(e), "coverage": CoverageMatrix().build(findings,self.config)}

        reasoning_upgrade = {}
        try:
            reasoning_upgrade = run_reasoning_upgrade(self.config, self.client, self.crawler, endpoints, forms, findings)
        except Exception as exc:
            reasoning_upgrade = {"error": str(exc), "metamorphic": {"signal_count": 0, "signals": []}, "invariants": {}, "causal_risk_fusion": []}

        boundary_differential = {"engine":"security-boundary-differential-map-v1","signals":[],"signal_count":0}
        try:
            self.progress.set_stage("security-boundary", "comparing equivalent resource representations")
            boundary_differential = self.boundary_differential.analyze(endpoints, limit=60 if self.config.coverage_mode == "maximum" else 30)
            findings.extend(self.boundary_adapter.findings_from(boundary_differential))
        except Exception as exc:
            boundary_differential = {"engine":"security-boundary-differential-map-v1","signals":[],"signal_count":0,"error":str(exc)}

        workflow_replay = {"enabled": False, "available": False, "workflows": [], "findings": []}
        try:
            self.progress.set_stage("identity-workflows", "replaying observed workflows across explicit identities")
            workflow_replay = self.workflow_engine.replay(start_url)
            for wf in workflow_replay.get("findings", []):
                findings.append(wf)
        except Exception as exc:
            workflow_replay = {"enabled": True, "available": False, "error": str(exc), "workflows": [], "findings": []}

        # ERSEC 18 authorization decision lattice: compare explicit identities
        # against sensitive resources using read-only GET requests.
        decision_lattice = {"available": False, "findings": [], "matrix": []}
        try:
            self.progress.set_stage("authorization-lattice", "comparing explicit identities across sensitive resources")
            decision_lattice = self.security_decision_lattice.analyze(endpoints, limit=120 if self.config.coverage_mode == "maximum" else 60)
            findings.extend(decision_lattice.get("findings", []))
        except Exception as exc:
            decision_lattice = {"available": False, "error": str(exc), "findings": [], "matrix": []}

        self.response_anomaly_ensemble.annotate(findings)

        # Deduplicate: same category + same normalized URL (ignore query) + same param
        seen = set()
        unique: List[Finding] = []
        for f in findings:
            key = (f.category, f.url.split("?")[0], f.parameter)
            if key in seen:
                continue
            seen.add(key)
            unique.append(f)

        # Consolidate site-wide config findings (same root cause across many
        # pages) into one finding with an affected_urls list, instead of one
        # near-duplicate finding per page.
        unique = consolidate_sitewide_findings(unique)

        # Deterministic risk correlation (not AI) - chains findings into
        # compound risks and produces a priority order.
        chains = correlate_findings(unique)
        unique = prioritize(unique, chains)

        capability_chains = CapabilityRiskGraph().fuse(unique)
        contextual_chains = ContextualRiskGraph().fuse(unique)
        detection_quality = enrich_detection_quality(unique, self.config)
        workflow_risk_chains = WorkflowRiskChainEngine().fuse(unique, workflow_replay)
        workflow_replay_report = dict(workflow_replay)
        workflow_replay_report["findings"] = [f.to_dict() if isinstance(f, Finding) else f for f in workflow_replay.get("findings", [])]
        chain_dicts = []
        for c in reasoning_upgrade.get("causal_risk_fusion", []):
            chain_dicts.append({**c, "chain_type": "causal_reasoning"})
        for c in chains:
            explanation = self.ai.explain_chain(c.name, c.member_categories, c.rationale) if self.config.ai_enabled else c.rationale
            chain_dicts.append({
                "name": c.name, "categories": c.member_categories,
                "combined_severity": c.combined_severity.name, "rationale": c.rationale,
                "ai_priority_guidance": explanation, "chain_type": "deterministic"
            })
        for c in capability_chains:
            chain_dicts.append({
                "name": c["name"], "categories": sorted({m["category"] for m in c["members"]}),
                "combined_severity": c["severity"], "rationale": c["rationale"],
                "ai_priority_guidance": c["rationale"], "chain_type": "capability_graph",
                "confidence": c["confidence"], "relationship": c["relationship"],
                "contributing_findings": [m["finding_id"] for m in c["members"]]
            })

        for c in workflow_risk_chains:
            chain_dicts.append(c)

        if self.config.ai_enabled:
            self.ai.annotate_all(unique, stack_fingerprint)
        else:
            for f in unique:
                f.ai_explanation = f.description
                f.ai_remediation_steps = f.remediation_summary

        authenticated_scan = bool(self.config.cookies or self.config.bearer_token)

        crown_jewel_distances = compute_crown_jewel_distances(
            self.crawler.link_graph, self.config.crown_jewels, self.crawler.endpoints
        )
        proximity_by_url = {
            url.split("?")[0]: proximity_multiplier(dist)
            for url, dist in crown_jewel_distances.items()
        }
        posture = compute_posture_score(unique, chains, authenticated_scan, proximity_by_url)
        asset_inventory = build_asset_inventory(self.crawler, self.client)
        attack_paths = fuse_attack_paths(unique)
        business_impact = self.ai.business_impact_summary(unique, posture["grade"]) if self.config.ai_enabled else ""
        self.config._ersec_last_findings = unique
        try:
            enhanced = enrich_er_sec_findings(unique,self.config,stack_fingerprint)
        except Exception as exc:
            enhanced={"error":str(exc),"verification_count":0,"evidence_triage":[],"patch_bundle":{"patches":[]},"enhanced_attack_paths":[],"contextual_probe_plan":{}}
        browser_data=None
        if getattr(self.config,"browser_discovery",False):
            browser_data=BrowserDiscoveryEngine(self.config).discover(start_url)

        ersec7_data = {"engine":"security-behavior-genome-v7","enabled":False}
        try:
            ersec7_data = self.revolution.analyze(endpoints, forms, unique, attack_paths, browser_data)
        except Exception as exc:
            ersec7_data = {"engine":"security-behavior-genome-v7","enabled":True,"error":str(exc),"genome":{},"temporal":{},"contract_drift":{}}

        # Surface blast-radius info per finding for the report/dashboard.
        findings_dicts = []
        for f in unique:
            d = f.to_dict()
            base_url = f.url.split("?")[0]
            dist = crown_jewel_distances.get(base_url)
            d["crown_jewel_distance"] = dist
            d["blast_radius_multiplier"] = proximity_multiplier(dist) if self.config.crown_jewels else None
            d["remediation_effort_estimate"] = estimate_remediation_effort(f.category)
            findings_dicts.append(d)

        juice_shop_coverage = {}
        juice_shop_source_evidence = {}
        if getattr(self.config, "juice_shop_benchmark", False):
            try:
                juice_shop_coverage = self.juice_shop.discover(start_url)
            except Exception as exc:
                juice_shop_coverage = {"available": False, "reason": str(exc)}
            try:
                juice_shop_source_evidence = self.juice_shop_source.scan(start_url)
                benchmark_findings = self.juice_shop_source.findings_from_evidence(juice_shop_source_evidence)
                if benchmark_findings:
                    unique.extend(benchmark_findings)
                    _seen=set(); _norm=[]
                    for _f in unique:
                        _k=(_f.category,_f.url.split("?")[0],_f.parameter)
                        if _k not in _seen: _seen.add(_k); _norm.append(_f)
                    unique=consolidate_sitewide_findings(_norm)
                    chains=correlate_findings(unique); unique=prioritize(unique,chains)
                    capability_chains=CapabilityRiskGraph().fuse(unique); contextual_chains=ContextualRiskGraph().fuse(unique); attack_paths=fuse_attack_paths(unique)
                    detection_quality=enrich_detection_quality(unique,self.config)
            except Exception as exc:
                juice_shop_source_evidence = {"available": False, "reason": str(exc)}

        # Rebuild serialized finding records after late benchmark enrichment.
        findings_dicts = []
        for f in unique:
            d = f.to_dict()
            base_url = f.url.split("?")[0]
            dist = crown_jewel_distances.get(base_url)
            d["crown_jewel_distance"] = dist
            d["blast_radius_multiplier"] = proximity_multiplier(dist) if self.config.crown_jewels else None
            d["remediation_effort_estimate"] = estimate_remediation_effort(f.category)
            findings_dicts.append(d)

        # ERSEC 17 autonomous security-agent layer: evidence-backed, bounded and non-destructive.
        agent_layer = {}
        try:
            agent_layer["causal_impact"] = self.causal_impact_engine.analyze(unique, start_url) if self.config.causal_impact else {"enabled": False}
        except Exception as exc:
            agent_layer["causal_impact"] = {"enabled": True, "error": str(exc), "impacts": []}
        try:
            agent_layer["autonomous_agent"] = self.autonomous_agent.plan(unique, endpoints, forms) if self.config.autonomous_agent else {"enabled": False}
        except Exception as exc:
            agent_layer["autonomous_agent"] = {"enabled": True, "error": str(exc), "actions": []}
        try:
            agent_layer["metamorphic_lam"] = self.metamorphic_lam.analyze(endpoints, unique) if self.config.metamorphic_lam else {"enabled": False}
        except Exception as exc:
            agent_layer["metamorphic_lam"] = {"enabled": True, "error": str(exc), "signals": []}
        try:
            agent_layer["remediation_twin"] = self.remediation_twin.prepare(unique, stack_fingerprint) if self.config.remediation_twin else {"enabled": False}
        except Exception as exc:
            agent_layer["remediation_twin"] = {"enabled": True, "error": str(exc), "plans": []}
        try:
            agent_layer["federated_mesh"] = self.federated_mesh.observe(unique, stack_fingerprint) if self.config.federated_mesh else {"enabled": False}
        except Exception as exc:
            agent_layer["federated_mesh"] = {"enabled": True, "error": str(exc)}
        try:
            agent_layer["contract_drift_sentry"] = self.contract_drift_sentry.compare(endpoints, self.crawler.response_hashes) if self.config.contract_drift_sentry else {"enabled": False}
        except Exception as exc:
            agent_layer["contract_drift_sentry"] = {"enabled": True, "error": str(exc), "changes": []}

        # Upgrade risk graph with causal and workflow-aware chain candidates.
        agent_chains = build_agent_risk_chains(unique, agent_layer.get("causal_impact", {}), workflow_replay_report)
        chain_dicts.extend(agent_chains)
        _result = {
            "target": target,
            "profile": self.config.profile.value,
            "scanned_at": _utc_now_iso(),
            "scan_status": "completed",
            "scan_status_detail": "",
            "pages_crawled": len(self.crawler.visited),
            "requests_made": self.client.request_count,
            "stack_fingerprint": stack_fingerprint,
            "findings": findings_dicts,
            "risk_chains": chain_dicts,
            "capability_risk_chains": capability_chains,
            "contextual_risk_chains": contextual_chains,
            "detection_quality": detection_quality,
            "campaign_preflight": campaign_preflight,
            "juice_shop_coverage": juice_shop_coverage,
            "juice_shop_source_evidence": juice_shop_source_evidence,
            "attack_paths": [
                {"name": p.name, "severity": p.severity.name, "rationale": p.rationale,
                 "contributing_findings": p.contributing_findings, "hop_count": p.hop_count}
                for p in attack_paths
            ],
            "posture_score": posture["score"],
            "posture_grade": posture["grade"],
            "posture_model": posture["model"],
            "business_impact_summary": business_impact,
            "asset_inventory": asset_inventory,
            "summary": {
                sev.name: sum(1 for f in unique if f.severity == sev)
                for sev in Severity
            },
        }
        attach_enhanced_report_data(_result,self.config,self.crawler,stack_fingerprint,browser_data)
        _result["enhanced_intelligence"]=enhanced
        _result["agent_layer"] = agent_layer
        _result["causal_impact"] = agent_layer.get("causal_impact", {})
        _result["contract_drift_sentry"] = agent_layer.get("contract_drift_sentry", {})
        _result["advanced_detection"] = advanced_detection
        _result["reasoning_upgrade"] = reasoning_upgrade
        _result["security_boundary_differential"] = boundary_differential
        _result["multi_identity_workflow_replay"] = workflow_replay_report
        _result["workflow_risk_chains"] = workflow_risk_chains
        _result["security_decision_lattice"] = {k:v for k,v in decision_lattice.items() if k != "findings"}
        _result["coverage_matrix"] = CoverageMatrix().build(unique, self.config, browser_data)
        _result["coverage_gaps"] = CoverageGapEngine().evaluate(_result["coverage_matrix"], unique)
        try:
            _result["detection_quality"]["repeatability"] = SafeFindingValidator(self.config, self.client).validate(unique)
        except Exception as _exc:
            _result["detection_quality"]["repeatability"] = {"error":str(_exc)}
        _result["detector_manifest"] = self.detector_registry.manifest()
        _result["execution_health"] = self.ledger.summary()
        try:
            _result["surface_intelligence"] = PassiveSurfaceMiner().mine(endpoints, forms, start_url, self.config)
        except Exception as _exc:
            _result["surface_intelligence"] = {"error":str(_exc),"candidates":[]}
        _result["security_behavior_genome"]=ersec7_data
        try:
            _result["behavioral_intelligence"] = self.maturity.analyze(endpoints, forms, unique)
        except Exception as exc:
            _result["behavioral_intelligence"] = {"engine":"behavioral-twin-v1","error":str(exc),"journeys":[],"invariants":[],"counterfactuals":{"checks":[],"changed":0},"proof_capsules":[],"risk_signals":[],"novelty":{}}
        try:
            _result["security_behavior_graph"] = SecurityBehaviorGraph().build(
                endpoints, unique, _result.get("multi_identity_workflow_replay", {}),
                _result.get("behavioral_intelligence", {}).get("invariants", []), self.config)
            if getattr(self.config, "security_graph_path", None):
                SecurityBehaviorGraph.save(_result["security_behavior_graph"], self.config.security_graph_path)
        except Exception as exc:
            _result["security_behavior_graph"] = {"error":str(exc),"nodes":[],"edges":[],"node_count":0,"edge_count":0}
        # ERSEC 33 Security Behavior Graph v2: explicit identity/tenant/resource model.
        # Only explicitly modeled read-only checks are executed; state-changing methods
        # are rejected at model validation time. Tokens never enter the model file.
        if getattr(self.config, "security_model_path", None):
            try:
                model = SecurityBehaviorModel.load(self.config.security_model_path)
                model_validation = model.validate()
                model_identities = []
                configured_tokens = dict(getattr(self.config, "identity_tokens", {}) or {})
                for ident in model.identities:
                    token = ""
                    if ident.name == "user": token = self.config.bearer_token
                    elif ident.name in configured_tokens: token = configured_tokens[ident.name]
                    elif ident.name == "anonymous": token = ""
                    # A declared identity with no matching credential can still be represented,
                    # but it is marked not_tested rather than silently using another identity.
                    model_identities.append(ident)

                clients = {}
                def _behavior_request(identity_name, method, url):
                    if identity_name == "anonymous":
                        token = ""
                    elif identity_name == "user":
                        token = self.config.bearer_token
                    else:
                        token = configured_tokens.get(identity_name, "")
                    if identity_name != "anonymous" and not token:
                        raise ScopeError(f"No authorized runtime credential supplied for modeled identity: {identity_name}")
                    if identity_name not in clients:
                        cc = dataclasses.replace(self.config, bearer_token=token, cookies={} if identity_name != "user" else dict(self.config.cookies), second_bearer_token="")
                        clients[identity_name] = SafeHttpClient(cc)
                    return clients[identity_name].request(method, url)

                def _scope_check(method, url):
                    if method.upper() not in self.config.scope.allowed_methods:
                        return False, "method_not_authorized_by_scan_scope"
                    if not is_in_scope(url, self.config):
                        return False, "url_not_in_scan_scope"
                    manifest = getattr(self.config, "authorization_manifest", None)
                    if manifest:
                        ok, reason = AuthorizationManifest(manifest).allows_request(method, url)
                        if not ok:
                            return False, reason
                    return True, ""

                behavior_verification = SecurityBehaviorVerifier(model).verify(model_identities, _behavior_request, _scope_check)
                # Convert only concrete modeled violations into findings; no finding is emitted
                # for absent credentials or scope exclusions.
                for failure in behavior_verification.get("failures", []):
                    if failure.get("verdict") != "violation":
                        continue
                    rid = str(failure.get("resource_id", ""))
                    ident = str(failure.get("identity", ""))
                    expected = failure.get("expected", {})
                    desc = (f"Explicit Security Behavior Model violation: identity '{ident}' received HTTP "
                            f"{failure.get('status')} from modeled resource '{rid}', but the model expected "
                            f"status {expected.get('status')}. This is a model-backed authorization finding "
                            f"and should be reviewed against the application's intended policy.")
                    ev = RequestEvidence(method=str(failure.get("method","GET")), url=str(failure.get("url",url) if 'url' in failure else failure.get("resource_id","")),
                                         status_code=int(failure.get("status",0)), response_time_ms=0,
                                         response_headers={"content-type":str(failure.get("content_type", ""))},
                                         response_excerpt=f"modeled authorization decision: {failure.get('status')}")
                    findings.append(Finding(_next_id(), "security_model_authorization", "Security Behavior Model Authorization Violation",
                                            Severity.HIGH, "Confirmed", "A01:2021", "CWE-862", str(failure.get("url", "")),
                                            None, desc, ev,
                                            "Review server-side authorization for the modeled identity/resource relationship and encode the corrected expectation as a reviewed regression contract."))
                graph_v2 = SecurityBehaviorGraphV2().build(model, behavior_verification)
                _result["security_behavior_model"] = model_validation
                _result["security_behavior_verification"] = behavior_verification
                _result["security_behavior_graph_v2"] = graph_v2
                # ERSEC 34: operational invariant assurance + minimal counterexamples + graph v3.
                behavior_assurance = SecurityInvariantEngine(model).evaluate(behavior_verification)
                counterexamples = CounterexamplePathEngine().build(model, behavior_assurance)
                graph_v3 = SecurityBehaviorGraphV3().build(model, behavior_verification, behavior_assurance)
                _result["security_behavior_assurance"] = behavior_assurance
                _result["security_behavior_counterexamples"] = counterexamples
                _result["security_behavior_graph_v3"] = graph_v3
                authorization_matrix = SecurityAuthorizationMatrix().build(model, behavior_verification)
                _result["security_authorization_matrix"] = authorization_matrix
                _result["security_behavior_model_contracts"] = model.contracts()
                if getattr(self.config, "behavior_verification_path", None):
                    _pathlib.Path(self.config.behavior_verification_path).parent.mkdir(parents=True, exist_ok=True)
                    _pathlib.Path(self.config.behavior_verification_path).write_text(json.dumps(behavior_verification, indent=2, sort_keys=True), encoding="utf-8")
                if getattr(self.config, "behavior_assurance_path", None):
                    _pathlib.Path(self.config.behavior_assurance_path).parent.mkdir(parents=True, exist_ok=True)
                    _pathlib.Path(self.config.behavior_assurance_path).write_text(json.dumps(behavior_assurance, indent=2, sort_keys=True), encoding="utf-8")
                if getattr(self.config, "behavior_graph_v3_path", None):
                    _pathlib.Path(self.config.behavior_graph_v3_path).parent.mkdir(parents=True, exist_ok=True)
                    _pathlib.Path(self.config.behavior_graph_v3_path).write_text(json.dumps(graph_v3, indent=2, sort_keys=True), encoding="utf-8")
                if getattr(self.config, "authorization_matrix_path", None):
                    _pathlib.Path(self.config.authorization_matrix_path).parent.mkdir(parents=True, exist_ok=True)
                    _pathlib.Path(self.config.authorization_matrix_path).write_text(json.dumps(authorization_matrix, indent=2, sort_keys=True), encoding="utf-8")
                if getattr(self.config, "behavior_state_path", None):
                    _result["security_behavior_state"] = SecurityBehaviorStateStore(self.config.behavior_state_path).compare_and_save(graph_v2)
            except Exception as exc:
                if self.config.component_failure_policy == "fail":
                    raise
                _result["security_behavior_model"] = {"error": str(exc)}
                _result["security_behavior_verification"] = {"status":"inconclusive","error":str(exc),"failures":[],"verifications":[]}
                _result["security_behavior_graph_v2"] = {"schema":"ersec-security-behavior-graph/2","nodes":[],"edges":[],"node_count":0,"edge_count":0}
                _result["security_behavior_assurance"] = {"schema":"ersec-security-behavior-assurance/1","status":"inconclusive","invariants":[],"counts":{"pass":0,"violated":0,"inconclusive":0,"not_tested":0}}
                _result["security_behavior_counterexamples"] = {"schema":"ersec-counterexample-path/1","paths":[],"count":0}
                _result["security_behavior_graph_v3"] = {"schema":"ersec-security-behavior-graph/3","nodes":[],"edges":[],"node_count":0,"edge_count":0}
                _result["security_authorization_matrix"] = {"schema":"ersec-authorization-matrix/1","status":"inconclusive","row_count":0,"coverage_ratio":0.0,"counts":{"pass":0,"violation":0,"inconclusive":0,"not_tested":0}}
        try:
            _result["stateful_test_preview"] = StatefulTestSimulator.preview(forms, self.config)
        except Exception as exc:
            _result["stateful_test_preview"] = {"error":str(exc),"previews":[],"count":0}
        try:
            contract_bundle = SecurityContractEngine().compile(unique, _result.get("security_behavior_graph", {}), _result.get("behavioral_intelligence", {}))
            model_contracts = _result.get("security_behavior_model_contracts", [])
            if isinstance(model_contracts, list) and model_contracts:
                contract_bundle["contracts"] = contract_bundle.get("contracts", []) + model_contracts
                contract_bundle["contract_count"] = len(contract_bundle["contracts"])
                contract_bundle["sources"] = sorted(set(contract_bundle.get("sources", []) + ["security-behavior-model"]))
            _result["security_contracts"] = contract_bundle
            if getattr(self.config, "contract_output_dir", None):
                _result["security_contract_exports"] = SecurityContractEngine().export(contract_bundle, self.config.contract_output_dir)
        except Exception as exc:
            _result["security_contracts"] = {"error":str(exc),"contracts":[],"contract_count":0}
        try:
            _result["assurance_scorecard"] = AssuranceScorecard.build(_result, _result.get("security_contracts"), _result.get("benchmark_quality"))
        except Exception as exc:
            _result["assurance_scorecard"] = {"schema":"ersec-assurance-scorecard/1","error":str(exc)}
        try:
            budget_items=[]
            for u in list(endpoints)[:300]:
                path=urllib.parse.urlparse(u).path.lower()
                budget_items.append({"type":"endpoint","url":u,"information_gain":0.6 if any(x in path for x in ("api","admin","account","checkout","oauth")) else 0.4,"crown_jewel":1.0 if any(x in path for x in self.config.crown_jewels) else 0.0,"authorization_relevance":0.8 if any(x in path for x in ("admin","account","user","tenant","order")) else 0.2,"contract_relevance":0.7 if u in self.crawler.api_like_endpoints else 0.2,"request_cost":1.0,"state_change_risk":0.2})
            _result["risk_budget"] = RiskBudgetScheduler(self.config.risk_budget).plan(budget_items)
        except Exception as exc:
            _result["risk_budget"] = {"error":str(exc),"selected":[],"skipped":[]}
        if getattr(self.config,"policy_output_dir",None):
            _result["policy_as_code"]=export_policy_bundle(unique,self.config.policy_output_dir)
        if getattr(self.config,"ide_output_dir",None):
            _result["ide_scaffold"]=export_ecosystem_scaffold(self.config.ide_output_dir)
        # A clean finding set is not equivalent to a complete assessment. Surface
        # coverage limitations prominently so CI/report consumers cannot confuse
        # "no findings" with "nothing vulnerable".
        _result["assurance"] = {
            "platform_version": ERSEC_VERSION,
            "security_category_registry": SecurityCategoryRegistry.manifest(),
            "authorization_manifest": bool(getattr(self.config, "authorization_manifest", None)),
            "stateful_tests_enabled": bool(getattr(self.config, "stateful_tests_enabled", False)),
            "risk_budget": _result.get("risk_budget", {}),
            "security_behavior_graph": {"nodes":_result.get("security_behavior_graph",{}).get("node_count",0),"edges":_result.get("security_behavior_graph",{}).get("edge_count",0)},
            "security_contracts": _result.get("security_contracts",{}).get("contract_count",0),
            "security_behavior_model": bool(getattr(self.config, "security_model_path", None)),
            "security_behavior_model_contracts": len(_result.get("security_behavior_model_contracts", []) or []),
            "security_behavior_boundary_drift": (_result.get("security_behavior_state") or {}).get("security_boundary_drift", False),
            "security_behavior_assurance": (_result.get("security_behavior_assurance") or {}).get("status"),
            "security_behavior_counterexamples": (_result.get("security_behavior_counterexamples") or {}).get("count", 0),
            "security_behavior_graph_v3": {"nodes": (_result.get("security_behavior_graph_v3") or {}).get("node_count", 0), "edges": (_result.get("security_behavior_graph_v3") or {}).get("edge_count", 0)},
            "security_authorization_matrix": {"rows": (_result.get("security_authorization_matrix") or {}).get("row_count", 0), "coverage_ratio": (_result.get("security_authorization_matrix") or {}).get("coverage_ratio"), "status": (_result.get("security_authorization_matrix") or {}).get("status")},
            "assurance_scorecard": {"coverage_ratio": (_result.get("assurance_scorecard") or {}).get("coverage_ratio"), "assurance_score": (_result.get("assurance_scorecard") or {}).get("assurance_score")},
            "stateful_test_preview_count": _result.get("stateful_test_preview",{}).get("count",0),
            "shield_rule_governance": True,
            "principle": "evidence first; uncertainty is explicit; state-changing automation is opt-in and requires explicit authorization"
        }
        coverage=_result.get("coverage_matrix", {})
        _result["assessment_confidence"]={
            "level":"high" if coverage.get("coverage_ratio",0) >= 0.8 and _result.get("pages_crawled",0)>0 else "limited",
            "coverage_ratio":coverage.get("coverage_ratio"),
            "limitations":coverage.get("disabled_or_unavailable",[]),
            "statement":"No scanner can prove absence of every vulnerability; this is an evidence and coverage indicator."
        }
        self.progress.findings = len(_result.get("findings", []))
        self.progress.mark_finished(False)
        _result["scan_progress"] = self.progress.snapshot()
        return _result


def run_multi_target_scan(targets: List[str], config_template: ScanConfig, plugin_dir: Optional[str] = None,
                           max_workers: int = 3) -> Dict[str, Dict[str, Any]]:
    """Run the same scan configuration against several authorized targets
    concurrently (thread-per-target, each with its own SafeHttpClient/request
    budget - budgets are NOT shared across targets, so one large site can't
    starve another's allowance). Returns {target: results_dict}. Intended for
    a person who has authorization across a list of hosts they own/manage
    (e.g. all of an organization's staging environments) - it does not
    change or loosen the per-target scope enforcement in any way; each
    target still gets its own ScopeConfig.allowed_hosts derived from itself."""
    results: Dict[str, Dict[str, Any]] = {}

    def _run_one(t: str) -> Tuple[str, Dict[str, Any]]:
        try:
            host = validate_target(t)
        except TargetValidationError as e:
            return t, {"target": t, "scan_status": "invalid_target", "scan_status_detail": str(e)}
        cfg = dataclasses.replace(
            config_template,
            target=host,
            scope=dataclasses.replace(config_template.scope, allowed_hosts=[host]),
        )
        scanner = ERSECScanner(cfg, plugin_dir=plugin_dir)
        try:
            return t, scanner.run(t)
        except ERSECError as e:
            return t, {"target": t, "scan_status": "error", "scan_status_detail": str(e)}

    with ThreadPoolExecutor(max_workers=max(1, max_workers)) as executor:
        futures = [executor.submit(_run_one, t) for t in targets]
        for fut in as_completed(futures):
            t, r = fut.result()
            results[t] = r
    return results


# =============================================================================
# Report rendering
# =============================================================================

_SEVERITY_META = {
    "CRITICAL": {"color": "#e0483e", "order": 0},
    "HIGH":     {"color": "#e0813e", "order": 1},
    "MEDIUM":   {"color": "#d9b23e", "order": 2},
    "LOW":      {"color": "#5b7fae", "order": 3},
    "INFO":     {"color": "#6b7280", "order": 4},
}


def _finding_row_html(f: Dict[str, Any], color: str) -> str:
    param_bit = f' <span class="param-tag">param: {html.escape(f["parameter"])}</span>' if f.get("parameter") else ""
    search_key = html.escape((f['title'] + ' ' + f['url'] + ' ' + f['category']).lower())
    compliance = f.get("compliance", [])
    compliance_html = (
        '<span class="tags tech-only">' + html.escape(" &middot; ".join(compliance)) + '</span>'
        if compliance else ""
    )
    effort = f.get("remediation_effort_estimate") or {}
    effort_html = (
        '<div class="tech-only effort-badge">Est. fix effort: ' +
        html.escape(f"{effort.get('low_hours', '?')}-{effort.get('high_hours', '?')} hrs") + '</div>'
        if effort else ""
    )
    return (
        '<div class="finding" data-severity="' + html.escape(f['severity']) + '" data-search="' + search_key + '">'
        '<button class="finding-head" onclick="this.parentElement.classList.toggle(\'open\')">'
        '<span class="sev-bar" style="background:' + color + '"></span>'
        '<span class="sev-badge" style="color:' + color + ';border-color:' + color + '">' + html.escape(f['severity']) + '</span>'
        '<span class="finding-title">' + html.escape(f['title']) + '</span>'
        '<span class="finding-conf">' + html.escape(f['confidence']) + '</span>'
        '<span class="chev">&#9656;</span>'
        '</button>'
        '<div class="finding-url tech-only">' + html.escape(f['url']) + param_bit +
        '<span class="tags">' + html.escape(f['owasp']) + ' &middot; ' + html.escape(f['cwe']) + '</span>'
        + compliance_html +
        '</div>'
        '<div class="finding-body">'
        '<p class="desc">' + html.escape(f['description']) + '</p>'
        '<div class="ai-section">'
        '<div class="ai-label">What this means</div>'
        '<p class="ai-text">' + html.escape(f['ai_explanation']) + '</p>'
        '<div class="tech-only">'
        '<div class="ai-label">How to fix it</div>'
        '<pre class="ai-pre">' + html.escape(f['ai_remediation_steps']) + '</pre>'
        '<div class="ai-label">How to verify</div>'
        '<p class="ai-text">' + html.escape(f['ai_verification_steps']) + '</p>'
        + effort_html +
        '</div></div></div></div>'
    )


def _chain_html(c: Dict[str, Any]) -> str:
    color = _SEVERITY_META.get(c["combined_severity"], {}).get("color", "#e0483e")
    cats = " + ".join(html.escape(cat.replace("_", " ")) for cat in c["categories"])
    return (
        '<div class="chain" style="border-color:' + color + '">'
        '<div class="chain-head">'
        '<span class="chain-badge" style="background:' + color + '">' + html.escape(c["combined_severity"]) + '</span>'
        '<span class="chain-name">' + html.escape(c["name"]) + '</span>'
        '</div>'
        '<div class="chain-cats">' + cats + '</div>'
        '<p class="chain-text">' + html.escape(c["ai_priority_guidance"]) + '</p>'
        '</div>'
    )


def _attack_path_html(p: Dict[str, Any]) -> str:
    color = _SEVERITY_META.get(p["severity"], {}).get("color", "#e0483e")
    findings_list = ", ".join(html.escape(fid) for fid in p["contributing_findings"])
    return (
        '<div class="chain" style="border-color:' + color + '">'
        '<div class="chain-head">'
        '<span class="chain-badge" style="background:' + color + '">' + html.escape(p["severity"]) + '</span>'
        '<span class="chain-name">' + html.escape(p["name"].replace("_", " ").title()) + '</span>'
        '</div>'
        '<div class="chain-cats">composed from: ' + findings_list + '</div>'
        '<p class="chain-text">' + html.escape(p["rationale"]) + '</p>'
        '</div>'
    )


_DASHBOARD_CSS = """
  :root {
    --bg: #14171c; --panel: #1a1e25; --panel-2: #1f242c; --border: #2a2f38;
    --text: #e6e9ee; --muted: #8b93a1; --mono: 'IBM Plex Mono', ui-monospace, monospace;
    --sans: 'IBM Plex Sans', ui-sans-serif, system-ui, sans-serif;
  }
  * { box-sizing: border-box; }
  body { margin:0; background:var(--bg); color:var(--text); font-family:var(--sans); font-size:14px; }
  a { color: inherit; }
  .shell { display:grid; grid-template-columns: 320px 1fr; min-height:100vh; }
  @media (max-width: 860px) { .shell { grid-template-columns: 1fr; } }

  .rail { background:var(--panel); border-right:1px solid var(--border); padding:1.75rem 1.5rem; }
  .rail-target { font-family:var(--mono); font-size:1.05rem; font-weight:600; word-break:break-all; }
  .rail-sub { color:var(--muted); font-size:0.8rem; margin-top:0.3rem; line-height:1.6; }
  .rail h2 { font-size:0.75rem; text-transform:uppercase; letter-spacing:0.06em; color:var(--muted);
              font-weight:500; margin:1.75rem 0 0.6rem 0; }
  .risk-bar { display:flex; height:10px; border-radius:3px; overflow:hidden; background:var(--panel-2); }
  .bar-seg { height:100%; }
  .legend { margin-top:0.6rem; display:flex; flex-direction:column; gap:0.35rem; }
  .legend-item { display:flex; align-items:center; gap:0.5rem; font-size:0.82rem; }
  .dot { width:8px; height:8px; border-radius:50%; flex-shrink:0; }
  .legend-label { color:var(--muted); flex:1; }
  .legend-count { font-family:var(--mono); }
  .fp-row { display:flex; justify-content:space-between; gap:0.5rem; font-size:0.8rem; padding:0.3rem 0;
             border-bottom:1px solid var(--border); }
  .fp-key { color:var(--muted); }
  .fp-val { font-family:var(--mono); text-align:right; word-break:break-all; }
  .fp-empty { color:var(--muted); font-size:0.82rem; font-style:italic; }

  .main { padding:1.75rem 2rem 3rem 2rem; }
  .main-head { display:flex; justify-content:space-between; align-items:baseline; flex-wrap:wrap; gap:1rem;
                border-bottom:1px solid var(--border); padding-bottom:1rem; margin-bottom:1.25rem; }
  .main-head h1 { font-size:1.15rem; font-weight:600; margin:0; }
  .section-label { font-size:0.8rem; text-transform:uppercase; letter-spacing:0.05em; color:#8b93a1;
                    font-weight:500; margin:0 0 0.8rem 0; }
  .search-wrap { display:flex; gap:0.6rem; flex-wrap:wrap; align-items:center; }
  #search { background:var(--panel-2); border:1px solid var(--border); color:var(--text); padding:0.45rem 0.7rem;
             border-radius:4px; font-family:var(--sans); font-size:0.85rem; min-width:220px; }
  .filter-btn { background:var(--panel-2); border:1px solid var(--border); color:var(--muted); padding:0.4rem 0.7rem;
                 border-radius:4px; font-size:0.78rem; cursor:pointer; font-family:var(--sans); }
  .filter-btn.active { color:var(--text); border-color:#556; }
  .fc { font-family:var(--mono); color:var(--muted); }

  .chain { border:1px solid; border-radius:4px; padding:0.9rem 1rem; margin-bottom:0.9rem; background:var(--panel-2); }
  .chain-head { display:flex; align-items:center; gap:0.6rem; }
  .chain-badge { color:#0b0d10; font-size:0.68rem; font-weight:700; padding:0.12rem 0.45rem; border-radius:3px; }
  .chain-name { font-weight:600; }
  .chain-cats { font-family:var(--mono); color:var(--muted); font-size:0.75rem; margin:0.35rem 0; }
  .chain-text { color:#c7cdd6; font-size:0.85rem; margin:0.3rem 0 0 0; line-height:1.5; }

  .finding { border-bottom:1px solid var(--border); }
  .finding-head { all:unset; display:flex; align-items:center; gap:0.7rem; width:100%; padding:0.8rem 0.2rem;
                   cursor:pointer; box-sizing:border-box; }
  .sev-bar { width:4px; align-self:stretch; border-radius:2px; flex-shrink:0; }
  .sev-badge { font-family:var(--mono); font-size:0.68rem; border:1px solid; border-radius:3px; padding:0.1rem 0.4rem;
                flex-shrink:0; }
  .finding-title { flex:1; font-weight:500; }
  .finding-conf { color:var(--muted); font-size:0.78rem; }
  .chev { color:var(--muted); transition:transform 0.15s ease; font-size:0.7rem; }
  .finding.open .chev { transform:rotate(90deg); }
  .finding-url { font-family:var(--mono); color:#93b4d6; font-size:0.78rem; padding:0 0.2rem 0.7rem 1.9rem;
                  word-break:break-all; }
  .tags { color:var(--muted); font-family:var(--sans); margin-left:0.6rem; }
  .param-tag { color:#c9a86a; }
  .finding-body { display:none; padding:0 0.2rem 1.1rem 1.9rem; }
  .finding.open .finding-body { display:block; }
  .desc { color:#c7cdd6; margin:0 0 0.8rem 0; line-height:1.5; }
  .ai-section { background:var(--panel); border:1px solid var(--border); border-radius:5px; padding:0.9rem 1rem; }
  .ai-label { font-size:0.7rem; text-transform:uppercase; letter-spacing:0.05em; color:#8fa8c9; font-weight:600;
               margin:0.7rem 0 0.25rem 0; }
  .ai-label:first-child { margin-top:0; }
  .ai-text { color:#d4d9e0; margin:0; line-height:1.5; }
  .ai-pre { white-space:pre-wrap; font-family:var(--sans); color:#d4d9e0; margin:0; line-height:1.5; }
  .effort-badge { margin-top:0.7rem; font-size:0.75rem; color:#8fa8c9; }
  .empty-state { color:var(--muted); padding:2rem 0; }
  .grade-block { display:flex; align-items:center; gap:0.9rem; border:1px solid; border-radius:6px;
                 padding:0.9rem 1rem; margin-bottom:0.5rem; background:var(--panel-2); }
  .grade-letter { font-family:var(--mono); font-size:2.1rem; font-weight:600; line-height:1; }
  .grade-sub { color:var(--muted); font-size:0.78rem; line-height:1.4; }
  .diff-row { display:flex; align-items:center; gap:0.5rem; font-size:0.82rem; padding:0.2rem 0; color:#c7cdd6; }
  .diff-dot { width:8px; height:8px; border-radius:50%; flex-shrink:0; }
  .view-toggle { display:flex; border:1px solid var(--border); border-radius:4px; overflow:hidden; }
  .view-btn { background:var(--panel-2); border:none; color:var(--muted); padding:0.4rem 0.75rem; font-size:0.78rem;
              cursor:pointer; font-family:var(--sans); }
  .view-btn.active { background:#2a3138; color:var(--text); }
  body.view-executive .tech-only { display:none; }
  .narrative-box { background:var(--panel-2); border-left:3px solid #8fa8c9; border-radius:4px;
                    padding:0.7rem 0.9rem; font-size:0.85rem; color:#c7cdd6; line-height:1.5; margin-bottom:0.6rem; }
"""

_DASHBOARD_JS = """
let activeFilter = null;
function ersecFilter(sev) {
  activeFilter = (activeFilter === sev) ? null : sev;
  document.querySelectorAll('.filter-btn').forEach(function(b) {
    b.classList.toggle('active', b.dataset.filter === activeFilter);
  });
  applyFilters();
}
function ersecSearch(q) {
  window._ersecQuery = q.toLowerCase();
  applyFilters();
}
function applyFilters() {
  const q = window._ersecQuery || '';
  document.querySelectorAll('.finding').forEach(function(el) {
    const sevMatch = !activeFilter || el.dataset.severity === activeFilter;
    const qMatch = !q || el.dataset.search.includes(q);
    el.style.display = (sevMatch && qMatch) ? '' : 'none';
  });
}
function ersecSetView(mode) {
  document.body.classList.toggle('view-executive', mode === 'executive');
  document.getElementById('viewExec').classList.toggle('active', mode === 'executive');
  document.getElementById('viewTech').classList.toggle('active', mode === 'technical');
}
document.body.classList.add('view-executive');
"""


_SARIF_LEVEL = {"CRITICAL": "error", "HIGH": "error", "MEDIUM": "warning", "LOW": "note", "INFO": "note"}


def render_sarif(results: Dict[str, Any]) -> Dict[str, Any]:
    """SARIF 2.1.0 output for CI/CD ingestion (GitHub code scanning, GitLab, etc.).
    Web-scan findings don't map cleanly onto SARIF's file/line model (it was
    designed for static analysis), so the finding's URL is used as the
    artifact location - the accepted convention for DAST tools that emit SARIF."""
    rule_ids_seen: Dict[str, int] = {}
    rules = []
    sarif_results = []
    for f in results["findings"]:
        rule_id = f["category"]
        if rule_id not in rule_ids_seen:
            rule_ids_seen[rule_id] = len(rules)
            rules.append({
                "id": rule_id,
                "name": f["title"],
                "shortDescription": {"text": f["title"]},
                "fullDescription": {"text": f["description"]},
                "helpUri": "https://cheatsheetseries.owasp.org/",
                "properties": {"owasp": f["owasp"], "cwe": f["cwe"], "tags": f.get("compliance", [])},
            })
        sarif_results.append({
            "ruleId": rule_id,
            "ruleIndex": rule_ids_seen[rule_id],
            "level": _SARIF_LEVEL.get(f["severity"], "warning"),
            "message": {"text": f["ai_explanation"] or f["description"]},
            "locations": [{
                "physicalLocation": {
                    "artifactLocation": {"uri": f["url"]},
                }
            }],
            "properties": {
                "confidence": f["confidence"], "severity": f["severity"], "parameter": f["parameter"],
                "canonicalId": f.get("canonical_id"),
                "exposureStatus": f.get("exposure_status"),  # new/persistent/resolved/reintroduced, when --history-db is used
            },
        })
    return {
        "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
        "version": "2.1.0",
        "runs": [{
            "tool": {
                "driver": {
                    "name": "ERSEC",
                    "informationUri": "https://github.com/",
                    "version": "4.0.0",
                    "rules": rules,
                }
            },
            "results": sarif_results,
        }],
    }


def render_markdown(results: Dict[str, Any]) -> str:
    """Plain-Markdown summary report - useful for pasting into a PR
    description, a ticket, or a Slack/Teams message that renders Markdown,
    without needing to open the interactive HTML dashboard."""
    lines = []
    lines.append(f"# ERSEC Report — {results['target']}")
    lines.append("")
    if results.get("scan_status") == "unreachable":
        lines.append("**Scan failed — target unreachable.**")
        lines.append("")
        lines.append(results.get("scan_status_detail", ""))
        return "\n".join(lines)
    lines.append(f"- **Profile:** {results['profile']}")
    lines.append(f"- **Scanned at:** {results['scanned_at']}")
    lines.append(f"- **Posture:** {results['posture_grade']} ({results['posture_score']}/100) — {results.get('posture_model', '')}")
    lines.append(f"- **Pages crawled:** {results['pages_crawled']}  |  **Requests made:** {results['requests_made']}")
    lines.append("")
    if results.get("business_impact_summary"):
        lines.append("## Summary")
        lines.append("")
        lines.append(results["business_impact_summary"])
        lines.append("")
    lines.append("## Findings by severity")
    lines.append("")
    lines.append("| Severity | Count |")
    lines.append("|---|---|")
    for sev in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"):
        count = results["summary"].get(sev, 0)
        if count:
            lines.append(f"| {sev} | {count} |")
    lines.append("")
    attack_paths = results.get("attack_paths", [])
    if attack_paths:
        lines.append("## Attack paths")
        lines.append("")
        for p in attack_paths:
            lines.append(f"- **[{p['severity']}] {p['name'].replace('_', ' ').title()}** "
                         f"(from: {', '.join(p['contributing_findings'])})")
            lines.append(f"  {p['rationale']}")
        lines.append("")
    chains = results.get("risk_chains", [])
    if chains:
        lines.append("## Compound risk chains")
        lines.append("")
        for c in chains:
            lines.append(f"- **[{c['combined_severity']}] {c['name']}** ({' + '.join(c['categories'])})")
            lines.append(f"  {c['ai_priority_guidance']}")
        lines.append("")
    lines.append("## Findings")
    lines.append("")
    for f in results["findings"]:
        param_bit = f" (param: `{f['parameter']}`)" if f.get("parameter") else ""
        lines.append(f"### [{f['severity']}] {f['title']}")
        lines.append("")
        lines.append(f"- **URL:** `{f['url']}`{param_bit}")
        lines.append(f"- **Confidence:** {f['confidence']}  |  **OWASP:** {f['owasp']}  |  **CWE:** {f['cwe']}")
        lines.append("")
        lines.append(f["description"])
        lines.append("")
        lines.append(f"**What this means:** {f['ai_explanation']}")
        lines.append("")
        lines.append("**How to fix it:**")
        lines.append("")
        lines.append("```")
        lines.append(f["ai_remediation_steps"])
        lines.append("```")
        lines.append("")
        lines.append(f"**How to verify:** {f['ai_verification_steps']}")
        lines.append("")
        lines.append("---")
        lines.append("")
    inv = results.get("asset_inventory", {})
    if inv:
        lines.append("## Asset inventory")
        lines.append("")
        for k, v in inv.items():
            lines.append(f"- **{k.replace('_', ' ').title()}:** {v}")
    return "\n".join(lines)


def render_csv(results: Dict[str, Any]) -> str:
    """Flat CSV of findings for spreadsheet triage / ticket import. Kept to
    the stdlib csv module rather than a manual join to correctly handle any
    commas, quotes, or newlines that show up inside a description string."""
    import csv
    import io
    buf = io.StringIO()
    fieldnames = ["finding_id", "severity", "confidence", "category", "title", "url", "parameter",
                  "owasp", "cwe", "description", "remediation_summary"]
    writer = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore")
    writer.writeheader()
    for f in results.get("findings", []):
        writer.writerow(f)
    return buf.getvalue()


def render_junit(results: Dict[str, Any]) -> str:
    """JUnit XML output - the lowest-common-denominator format almost every
    CI system (Jenkins, GitLab, CircleCI, Azure DevOps) can render as a test
    report natively. Each finding is modeled as a 'failed test case' so a
    pipeline can surface security findings in the same UI as unit test
    failures without a specialized SARIF viewer being configured."""
    import xml.sax.saxutils as sax
    findings = results.get("findings", [])
    failures = len(findings)
    lines = [f'<?xml version="1.0" encoding="UTF-8"?>']
    lines.append(f'<testsuite name="ERSEC" tests="{failures}" failures="{failures}" errors="0" '
                 f'skipped="0" timestamp="{sax.escape(results.get("scanned_at", ""))}">')
    for f in findings:
        classname = sax.escape(f.get("category", "ersec"))
        name = sax.escape(f.get("title", "finding"))
        lines.append(f'  <testcase classname="{classname}" name="{name}">')
        message = sax.escape(f"{f.get('severity', '')} - {f.get('url', '')}")
        body = sax.escape(f.get("description", "") + "\n\n" + f.get("ai_remediation_steps", ""))
        lines.append(f'    <failure message="{message}" type="{sax.escape(f.get("severity", ""))}">{body}</failure>')
        lines.append('  </testcase>')
    lines.append('</testsuite>')
    return "\n".join(lines)


def render_dashboard(results: Dict[str, Any]) -> str:
    """Render the ERSEC dashboard: a data-dense audit ledger, not a card grid.
    Color is used only to signal severity; everything else is grayscale."""
    findings = results["findings"]
    chains = results.get("risk_chains", [])
    fingerprint = results.get("stack_fingerprint", {})

    total = max(len(findings), 1)
    bar_segments = []
    for sev, meta in sorted(_SEVERITY_META.items(), key=lambda kv: kv[1]["order"]):
        count = results["summary"].get(sev, 0)
        if count == 0:
            continue
        pct = round(100 * count / total, 2)
        bar_segments.append('<div class="bar-seg" style="width:' + str(pct) + '%;background:' + meta["color"] + '" title="' + sev + ': ' + str(count) + '"></div>')
    severity_bar = '<div class="risk-bar">' + ("".join(bar_segments) if bar_segments else '<div class="bar-seg" style="width:100%;background:#2a2f38"></div>') + '</div>'

    legend_items = []
    for sev, meta in sorted(_SEVERITY_META.items(), key=lambda kv: kv[1]["order"]):
        count = results["summary"].get(sev, 0)
        if count == 0:
            continue
        legend_items.append(
            '<div class="legend-item"><span class="dot" style="background:' + meta["color"] + '"></span>'
            '<span class="legend-label">' + sev.title() + '</span><span class="legend-count">' + str(count) + '</span></div>'
        )

    fp_rows = "".join(
        '<div class="fp-row"><span class="fp-key">' + html.escape(k.replace("_", " ").title()) + '</span>'
        '<span class="fp-val">' + html.escape(str(v)) + '</span></div>'
        for k, v in fingerprint.items()
    ) or '<div class="fp-empty">No stack signals detected.</div>'

    chains_html = "".join(_chain_html(c) for c in chains) or '<div class="fp-empty">No compound risk chains detected among current findings.</div>'

    attack_paths = results.get("attack_paths", [])
    attack_paths_html = "".join(_attack_path_html(p) for p in attack_paths) or ""

    rows = []
    for f in findings:
        color = _SEVERITY_META.get(f["severity"], {}).get("color", "#6b7280")
        rows.append(_finding_row_html(f, color))
    findings_html = "".join(rows) if rows else '<div class="empty-state">No findings at this scan depth. Try a deeper profile, or this target is in good shape.</div>'

    filter_buttons = "".join(
        '<button class="filter-btn" data-filter="' + sev + '" onclick="ersecFilter(\'' + sev + '\')">' + sev.title() +
        ' <span class="fc">' + str(results["summary"].get(sev, 0)) + '</span></button>'
        for sev in ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO") if results["summary"].get(sev, 0) > 0
    )
    view_toggle = (
        '<div class="view-toggle">'
        '<button id="viewExec" class="view-btn active" onclick="ersecSetView(\'executive\')">Executive</button>'
        '<button id="viewTech" class="view-btn" onclick="ersecSetView(\'technical\')">Technical</button>'
        '</div>'
    )

    parts = []
    parts.append('<!DOCTYPE html><html><head><meta charset="utf-8">')
    parts.append('<meta name="viewport" content="width=device-width, initial-scale=1">')
    parts.append('<title>ERSEC — ' + html.escape(results['target']) + '</title>')
    # No external font/CDN calls - the dashboard must render correctly with zero
    # network access, since that's the whole point of an air-gapped-friendly tool.
    # System monospace/sans stacks stand in for IBM Plex when it's not locally installed.
    parts.append('<style>' + _DASHBOARD_CSS + '</style></head><body>')
    parts.append('<div class="shell"><div class="rail">')
    grade = results.get('posture_grade', '')
    score = results.get('posture_score', '')
    grade_color = {"A+": "#3ea06a", "A": "#3ea06a", "A-": "#6ba85e", "B": "#d9b23e",
                   "C": "#e0813e", "D": "#e0813e", "F": "#e0483e"}.get(grade, "#6b7280")
    if grade:
        parts.append('<div class="grade-block" style="border-color:' + grade_color + '">'
                      '<div class="grade-letter" style="color:' + grade_color + '">' + html.escape(grade) + '</div>'
                      '<div class="grade-sub">Posture score<br>' + str(score) + ' / 100</div></div>')
    parts.append('<div class="rail-target">' + html.escape(results['target']) + '</div>')
    parts.append('<div class="rail-sub">' + html.escape(results['profile']).title() + ' profile &middot; ' + html.escape(results['scanned_at']) + '<br>' +
                  str(results['pages_crawled']) + ' pages crawled &middot; ' + str(results['requests_made']) + ' requests made</div>')

    if results.get('business_impact_summary'):
        parts.append('<h2>Executive summary</h2>')
        parts.append('<div class="narrative-box">' + html.escape(results['business_impact_summary']) + '</div>')

    baseline_diff = results.get('baseline_diff')
    if baseline_diff:
        parts.append('<h2>Since last scan</h2>')
        parts.append(
            '<div class="diff-row"><span class="diff-dot" style="background:#e0483e"></span>'
            '<span>' + str(baseline_diff['new_count']) + ' new</span></div>'
            '<div class="diff-row"><span class="diff-dot" style="background:#3ea06a"></span>'
            '<span>' + str(baseline_diff['resolved_count']) + ' resolved</span></div>'
            '<div class="diff-row"><span class="diff-dot" style="background:#6b7280"></span>'
            '<span>' + str(baseline_diff['persistent_count']) + ' unchanged</span></div>'
        )

    exposure_changes = results.get('exposure_changes')
    if exposure_changes:
        parts.append('<h2>Exposure history</h2>')
        parts.append('<div class="narrative-box">' + html.escape(exposure_changes.get('narrative', '')) + '</div>')
        exp_counts = [
            ("new", "#e0483e", "new"), ("reintroduced", "#e0813e", "reintroduced"),
            ("changed", "#d9b23e", "changed"), ("resolved", "#3ea06a", "resolved"),
        ]
        for key, color, label in exp_counts:
            n = len(exposure_changes.get(key, []))
            if n:
                parts.append(
                    '<div class="diff-row"><span class="diff-dot" style="background:' + color + '"></span>'
                    '<span>' + str(n) + ' ' + label + '</span></div>'
                )

    parts.append('<h2>Risk distribution</h2>' + severity_bar)
    parts.append('<div class="legend">' + ("".join(legend_items) if legend_items else '<div class="fp-empty">No findings.</div>') + '</div>')
    parts.append('<h2>Detected stack</h2><div>' + fp_rows + '</div>')

    inv = results.get('asset_inventory', {})
    if inv:
        parts.append('<h2>Asset inventory</h2>')
        parts.append(
            '<div class="fp-row"><span class="fp-key">Pages</span><span class="fp-val">' + str(inv.get('pages_discovered', 0)) + '</span></div>'
            '<div class="fp-row"><span class="fp-key">Forms</span><span class="fp-val">' + str(inv.get('forms_discovered', 0)) + '</span></div>'
            '<div class="fp-row"><span class="fp-key">Parametrized endpoints</span><span class="fp-val">' + str(inv.get('endpoints_with_parameters', 0)) + '</span></div>'
            '<div class="fp-row"><span class="fp-key">API-like endpoints</span><span class="fp-val">' + str(inv.get('api_like_endpoints_discovered', 0)) + '</span></div>'
            '<div class="fp-row"><span class="fp-key">Cookies</span><span class="fp-val">' + html.escape(", ".join(inv.get('cookies_observed', [])) or "none") + '</span></div>'
            '<div class="fp-row"><span class="fp-key">3rd-party origins</span><span class="fp-val">' + html.escape(", ".join(inv.get('third_party_origins', [])) or "none") + '</span></div>'
        )

    parts.append('</div><div class="main">')
    parts.append('<div class="main-head"><h1>Findings (' + str(len(findings)) + ')</h1>')
    parts.append('<div class="search-wrap">' + view_toggle + '<input id="search" type="text" placeholder="Search findings..." oninput="ersecSearch(this.value)">' + filter_buttons + '</div></div>')
    if attack_paths_html:
        parts.append('<div class="section-label">Attack paths (composed from multiple findings)</div>' + attack_paths_html)
    parts.append('<div class="section-label" style="margin-top:1.75rem;">Compound risk chains</div>' + chains_html)
    parts.append('<div class="section-label" style="margin-top:1.75rem;">Findings ledger</div>')
    parts.append('<div id="ledger">' + findings_html + '</div>')
    parts.append('</div></div>')
    parts.append('<script>' + _DASHBOARD_JS + '</script>')
    parts.append('</body></html>')
    return "".join(parts)


# =============================================================================
# CLI
# =============================================================================

def print_banner():
    if RICH:
        Console().print(Panel.fit(
            "[bold cyan]ERSEC v4[/bold cyan] — Web Vulnerability Scanner + AI Remediation Advisor\n"
            "[dim]Defensive use only. Scan systems you own or are authorized to test.[/dim]",
            border_style="cyan"))
    else:
        print("=== ERSEC v4 — Web Vulnerability Scanner + AI Remediation Advisor ===")
        print("Defensive use only. Scan systems you own or are authorized to test.\n")


_SEVERITY_RICH_COLOR = {"CRITICAL": "bold red", "HIGH": "red", "MEDIUM": "yellow", "LOW": "blue", "INFO": "dim"}
_SEVERITY_PLAIN_TAG = {"CRITICAL": "[CRIT]", "HIGH": "[HIGH]", "MEDIUM": "[MED] ", "LOW": "[LOW] ", "INFO": "[INFO]"}


def print_console_report(results: Dict[str, Any], baseline_diff: Optional[Dict[str, Any]] = None,
                          exposure_changes: Optional[Dict[str, Any]] = None,
                          recurrence_report: Optional[List[Dict[str, Any]]] = None) -> None:
    """The full console report: posture, severity breakdown, findings ledger,
    risk chains, attack paths, and asset inventory - not just a flat findings
    dump. Every data structure the scanner computes gets a visible home here,
    in both the Rich and plain-text paths."""
    if results.get("scan_status") == "unreachable":
        detail = results.get("scan_status_detail", "The target could not be reached.")
        if RICH:
            Console().print(Panel.fit(
                f"[bold red]SCAN FAILED — TARGET UNREACHABLE[/bold red]\n\n{detail}",
                border_style="red", title="No data collected"))
        else:
            print("\n=== SCAN FAILED — TARGET UNREACHABLE ===")
            print(detail)
        return
    if results.get("scan_status") in ("invalid_target", "error"):
        print(f"\n[!] {results.get('target')}: {results.get('scan_status_detail', 'scan failed')}")
        return

    findings = results["findings"]
    chains = results.get("risk_chains", [])
    attack_paths = results.get("attack_paths", [])
    inventory = results.get("asset_inventory", {})

    if RICH:
        console = Console()

        grade = results["posture_grade"]
        grade_color = {"A+": "green", "A": "green", "A-": "green", "B": "yellow",
                       "C": "yellow", "D": "red", "F": "bold red"}.get(grade, "white")
        console.print(Panel.fit(
            f"[{grade_color}]{grade}[/{grade_color}]  ({results['posture_score']}/100)  "
            f"[dim]{results.get('posture_model', '')}[/dim]",
            title="Security Posture", border_style=grade_color))

        if results.get("business_impact_summary"):
            console.print(Panel.fit(results["business_impact_summary"], title="Executive Summary", border_style="cyan"))

        sev_table = Table(title="Findings by Severity", show_header=True, header_style="bold")
        sev_table.add_column("Severity")
        sev_table.add_column("Count", justify="right")
        for sev, count in results["summary"].items():
            if count:
                sev_table.add_row(f"[{_SEVERITY_RICH_COLOR.get(sev, 'white')}]{sev}[/{_SEVERITY_RICH_COLOR.get(sev, 'white')}]", str(count))
        console.print(sev_table)

        if attack_paths:
            ap_table = Table(title="Attack Paths (composed from multiple findings)", show_header=True, header_style="bold")
            ap_table.add_column("Path")
            ap_table.add_column("Severity")
            ap_table.add_column("From findings")
            for p in attack_paths:
                ap_table.add_row(
                    p["name"].replace("_", " "),
                    f"[{_SEVERITY_RICH_COLOR.get(p['severity'], 'white')}]{p['severity']}[/{_SEVERITY_RICH_COLOR.get(p['severity'], 'white')}]",
                    ", ".join(p["contributing_findings"]),
                )
            console.print(ap_table)
            for p in attack_paths:
                console.print(f"  [dim]{p['name'].replace('_', ' ')}:[/dim] {p['rationale']}")

        if chains:
            console.print(f"\n[bold]Compound risk chains ({len(chains)}):[/bold]")
            for c in chains:
                console.print(f"  - [{_SEVERITY_RICH_COLOR.get(c['combined_severity'], 'white')}]{c['name']}[/{_SEVERITY_RICH_COLOR.get(c['combined_severity'], 'white')}] "
                               f"({' + '.join(c['categories'])})")

        if findings:
            f_table = Table(title=f"Findings ({len(findings)})", show_header=True, header_style="bold")
            f_table.add_column("Sev", width=6)
            f_table.add_column("Conf", width=10)
            f_table.add_column("Title")
            f_table.add_column("URL", overflow="fold")
            for f in findings:
                color = _SEVERITY_RICH_COLOR.get(f["severity"], "white")
                f_table.add_row(
                    f"[{color}]{f['severity'][:4]}[/{color}]", f["confidence"], f["title"],
                    f["url"] + (f" (param: {f['parameter']})" if f.get("parameter") else ""),
                )
            console.print(f_table)
        else:
            console.print("[green]No findings.[/green]")

        if inventory:
            console.print(f"\n[bold]Asset inventory:[/bold] {inventory.get('pages_discovered', 0)} pages, "
                           f"{inventory.get('forms_discovered', 0)} forms, "
                           f"{inventory.get('endpoints_with_parameters', 0)} parametrized endpoints, "
                           f"{inventory.get('api_like_endpoints_discovered', 0)} API-like endpoints, "
                           f"{len(inventory.get('third_party_origins', []))} third-party origins")

        if baseline_diff:
            console.print(f"\n[bold]Since last scan:[/bold] {baseline_diff['new_count']} new, "
                           f"{baseline_diff['resolved_count']} resolved, {baseline_diff['persistent_count']} unchanged")
        if exposure_changes:
            console.print(f"\n[bold]Exposure history:[/bold] {exposure_changes['narrative']}")
        if recurrence_report:
            console.print("\n[bold]Recurring exposures (2+ historical scans):[/bold]")
            for r in recurrence_report:
                console.print(f"  - {r['category']} at {r['sample_url']}: {r['occurrences']}x, "
                               f"first {r['first_seen']}, last {r['last_seen']}")

    else:
        print(f"\n=== Security Posture: {results['posture_grade']} ({results['posture_score']}/100) ===")
        print(f"({results.get('posture_model', '')})")

        if results.get("business_impact_summary"):
            print(f"\n=== Executive Summary ===\n{results['business_impact_summary']}")

        print(f"\n=== Findings by Severity ===")
        for sev, count in results["summary"].items():
            if count:
                print(f"  {sev}: {count}")

        if attack_paths:
            print(f"\n=== Attack Paths ({len(attack_paths)}) ===")
            for p in attack_paths:
                print(f"  {_SEVERITY_PLAIN_TAG.get(p['severity'], '')} {p['name'].replace('_', ' ')} "
                      f"(from: {', '.join(p['contributing_findings'])})")
                print(f"      {p['rationale']}")

        if chains:
            print(f"\n=== Compound Risk Chains ({len(chains)}) ===")
            for c in chains:
                print(f"  {_SEVERITY_PLAIN_TAG.get(c['combined_severity'], '')} {c['name']} "
                      f"({' + '.join(c['categories'])})")

        print(f"\n=== Findings ({len(findings)}) ===")
        if findings:
            for f in findings:
                tag = _SEVERITY_PLAIN_TAG.get(f["severity"], "")
                param_bit = f" (param: {f['parameter']})" if f.get("parameter") else ""
                print(f"  {tag} {f['title']} - {f['url']}{param_bit}")
                print(f"        {f['ai_explanation']}")
        else:
            print("  No findings.")

        if inventory:
            print(f"\n=== Asset Inventory ===")
            print(f"  Pages: {inventory.get('pages_discovered', 0)}  "
                  f"Forms: {inventory.get('forms_discovered', 0)}  "
                  f"Parametrized endpoints: {inventory.get('endpoints_with_parameters', 0)}  "
                  f"API-like endpoints: {inventory.get('api_like_endpoints_discovered', 0)}  "
                  f"Third-party origins: {len(inventory.get('third_party_origins', []))}")

        if baseline_diff:
            print(f"\nSince last scan: {baseline_diff['new_count']} new, "
                  f"{baseline_diff['resolved_count']} resolved, {baseline_diff['persistent_count']} unchanged")
        if exposure_changes:
            print(f"\nExposure history: {exposure_changes['narrative']}")
        if recurrence_report:
            print("\nRecurring exposures (2+ historical scans):")
            for r in recurrence_report:
                print(f"  - {r['category']} at {r['sample_url']}: {r['occurrences']}x, "
                      f"first {r['first_seen']}, last {r['last_seen']}")


def load_scope_file(path: str) -> ScopeConfig:
    with open(path) as f:
        data = json.load(f)
    return ScopeConfig(**data)


# =============================================================================
# Self-test suite - a lightweight internal consistency check, invoked via
# `--self-test`, that exercises the parts of ERSEC that have NOTHING to do
# with network access: scope enforcement, target validation, canonical ID
# stability, ERQ scoring monotonicity, chain correlation, and attack-path
# fusion. This is not a substitute for running the tool against a real,
# authorized target - it can't be, since every detection module needs a
# live HTTP response to reason about - but it catches the class of bug
# that matters most before a real scan: "does the scoring/chaining/scope
# logic itself behave correctly", independent of any target's availability.
# Uses only the stdlib unittest module - no new dependency required to run it.
# =============================================================================

import unittest as _unittest


class _TestTargetValidation(_unittest.TestCase):
    def test_plain_hostname_accepted(self):
        self.assertEqual(validate_target("example.com"), "example.com")

    def test_url_with_scheme_extracts_hostname(self):
        self.assertEqual(validate_target("https://example.com/path?x=1"), "example.com")

    def test_doubled_scheme_rejected(self):
        with self.assertRaises(TargetValidationError):
            validate_target("https://https://example.com")

    def test_empty_target_rejected(self):
        with self.assertRaises(TargetValidationError):
            validate_target("")

    def test_bad_ip_octet_rejected(self):
        with self.assertRaises(TargetValidationError):
            validate_target("999.1.1.1")

    def test_valid_ip_accepted(self):
        self.assertEqual(validate_target("192.168.1.1"), "192.168.1.1")

    def test_malformed_hostname_rejected(self):
        with self.assertRaises(TargetValidationError):
            validate_target("http://")


class _TestScopeEnforcement(_unittest.TestCase):
    def _config(self, **scope_kwargs) -> ScanConfig:
        scope = ScopeConfig(allowed_hosts=["example.com"], **scope_kwargs)
        return ScanConfig(target="example.com", scope=scope)

    def test_in_scope_host_allowed(self):
        cfg = self._config()
        self.assertTrue(is_in_scope("https://example.com/anything", cfg))

    def test_different_host_rejected(self):
        cfg = self._config()
        self.assertFalse(is_in_scope("https://attacker.com/anything", cfg))

    def test_subdomain_not_implicitly_in_scope(self):
        # Deliberate: allowed_hosts is an exact match list, not a suffix match -
        # scope creep onto subdomains must be explicit, never automatic.
        cfg = self._config()
        self.assertFalse(is_in_scope("https://evil.example.com/", cfg))

    def test_disallowed_port_rejected(self):
        cfg = self._config(allowed_ports=[443])
        self.assertFalse(is_in_scope("http://example.com:8080/", cfg))

    def test_path_restriction_enforced(self):
        cfg = self._config(allowed_paths=["/api/"])
        self.assertTrue(is_in_scope("https://example.com/api/users", cfg))
        self.assertFalse(is_in_scope("https://example.com/admin", cfg))


class _TestCanonicalFindingId(_unittest.TestCase):
    def test_stable_across_calls(self):
        id1 = canonical_finding_id("xss", "https://example.com/a?x=1", "x", "desc")
        id2 = canonical_finding_id("xss", "https://example.com/a?x=2", "x", "desc")
        # Query string differs but path/category/param/description don't -> same ID.
        self.assertEqual(id1, id2)

    def test_different_description_yields_different_id(self):
        id1 = canonical_finding_id("security_headers", "https://example.com/", None, "missing HSTS")
        id2 = canonical_finding_id("security_headers", "https://example.com/", None, "missing CSP")
        self.assertNotEqual(id1, id2)

    def test_different_category_yields_different_id(self):
        id1 = canonical_finding_id("xss", "https://example.com/a", "x", "desc")
        id2 = canonical_finding_id("sql_injection", "https://example.com/a", "x", "desc")
        self.assertNotEqual(id1, id2)

    def test_host_case_insensitive(self):
        id1 = canonical_finding_id("xss", "https://Example.COM/a", "x", "desc")
        id2 = canonical_finding_id("xss", "https://example.com/a", "x", "desc")
        self.assertEqual(id1, id2)

    def test_trailing_slash_normalized(self):
        id1 = canonical_finding_id("xss", "https://example.com/a/", "x", "desc")
        id2 = canonical_finding_id("xss", "https://example.com/a", "x", "desc")
        self.assertEqual(id1, id2)


def _mk_finding(category: str, severity: Severity, confidence: str = "Confirmed",
                 finding_id: str = "F-0001", url: str = "https://example.com/") -> Finding:
    ev = RequestEvidence(method="GET", url=url, status_code=200, response_time_ms=10,
                          response_headers={}, response_excerpt="")
    return Finding(finding_id=finding_id, category=category, title=category, severity=severity,
                    confidence=confidence, owasp="", cwe="", url=url, parameter=None,
                    description="test", evidence=ev)


class _TestRiskCorrelationAndScoring(_unittest.TestCase):
    def test_chain_fires_when_both_categories_present(self):
        findings = [
            _mk_finding("xss", Severity.HIGH, finding_id="F-0001"),
            _mk_finding("security_headers", Severity.MEDIUM, finding_id="F-0002"),
        ]
        chains = correlate_findings(findings)
        names = {c.name for c in chains}
        self.assertIn("Session hijack chain", names)

    def test_chain_does_not_fire_with_only_one_category(self):
        findings = [_mk_finding("xss", Severity.HIGH)]
        chains = correlate_findings(findings)
        self.assertEqual(chains, [])

    def test_prioritize_puts_critical_before_low(self):
        findings = [
            _mk_finding("info_disclosure_comments", Severity.LOW, finding_id="F-0001"),
            _mk_finding("sql_injection", Severity.CRITICAL, finding_id="F-0002"),
        ]
        ordered = prioritize(findings, [])
        self.assertEqual(ordered[0].category, "sql_injection")

    def test_prioritize_boosts_chained_finding_over_equal_severity_unchained(self):
        findings = [
            _mk_finding("xss", Severity.HIGH, finding_id="F-0001"),          # in a chain below
            _mk_finding("crlf_injection", Severity.HIGH, finding_id="F-0002"),  # not in any chain here
        ]
        chains = [RiskChain(name="Session hijack chain", member_categories=["security_headers", "xss"],
                             combined_severity=Severity.CRITICAL, rationale="test")]
        ordered = prioritize(findings, chains)
        self.assertEqual(ordered[0].category, "xss")

    def test_posture_score_worse_with_more_critical_findings(self):
        few = [_mk_finding("xss", Severity.CRITICAL, finding_id="F-0001")]
        many = [_mk_finding("xss", Severity.CRITICAL, finding_id=f"F-{i:04d}") for i in range(5)]
        score_few = compute_posture_score(few, [])["score"]
        score_many = compute_posture_score(many, [])["score"]
        self.assertLess(score_many, score_few)

    def test_posture_score_worse_unauthenticated_than_authenticated(self):
        findings = [_mk_finding("xss", Severity.HIGH)]
        auth_score = compute_posture_score(findings, [], authenticated_scan=True)["score"]
        anon_score = compute_posture_score(findings, [], authenticated_scan=False)["score"]
        self.assertLessEqual(anon_score, auth_score)

    def test_no_findings_yields_perfect_score(self):
        result = compute_posture_score([], [])
        self.assertEqual(result["score"], 100.0)
        self.assertEqual(result["grade"], "A+")


class _TestAttackPathFusion(_unittest.TestCase):
    def test_session_hijack_path_requires_httponly_signal_not_just_category(self):
        # xss alone (no httponly-missing signal in the description) should NOT
        # compose into full_session_hijack - the capability model requires the
        # SPECIFIC missing-HttpOnly signal, not just the security_headers category.
        findings = [_mk_finding("xss", Severity.HIGH, finding_id="F-0001")]
        paths = fuse_attack_paths(findings)
        self.assertNotIn("full_session_hijack", {p.name for p in paths})

    def test_session_hijack_path_fires_with_httponly_signal_present(self):
        ev = RequestEvidence(method="GET", url="https://example.com/", status_code=200,
                              response_time_ms=10, response_headers={}, response_excerpt="")
        httponly_finding = Finding(finding_id="F-0002", category="security_headers", title="t",
                                    severity=Severity.MEDIUM, confidence="Confirmed", owasp="", cwe="",
                                    url="https://example.com/", parameter=None,
                                    description="Cookie 'session' is missing security attributes: missing HttpOnly flag.",
                                    evidence=ev)
        findings = [_mk_finding("xss", Severity.HIGH, finding_id="F-0001"), httponly_finding]
        paths = fuse_attack_paths(findings)
        self.assertIn("full_session_hijack", {p.name for p in paths})

    def test_composed_capability_can_feed_further_rules(self):
        # credential_reuse_data_breach requires read_database (sql_injection) +
        # obtain_credentials_or_source (exposed_files) - confirms multi-source fusion works.
        findings = [
            _mk_finding("sql_injection", Severity.CRITICAL, finding_id="F-0001"),
            _mk_finding("exposed_files", Severity.CRITICAL, finding_id="F-0002"),
        ]
        paths = fuse_attack_paths(findings)
        self.assertIn("credential_reuse_data_breach", {p.name for p in paths})

    def test_no_paths_from_unrelated_findings(self):
        findings = [_mk_finding("mixed_content", Severity.LOW, finding_id="F-0001")]
        paths = fuse_attack_paths(findings)
        self.assertEqual(paths, [])


class _TestAuditLogIntegrity(_unittest.TestCase):
    def test_tampered_record_detected(self):
        import tempfile
        with tempfile.TemporaryDirectory() as d:
            log_path = os.path.join(d, "audit.json")
            append_to_audit_log([{"category": "xss", "url": "https://example.com/", "verdict": "resolved"}], log_path)
            append_to_audit_log([{"category": "sql_injection", "url": "https://example.com/", "verdict": "still_present"}], log_path)
            result = verify_audit_log_integrity(log_path)
            self.assertTrue(result["valid"])
            # Tamper with the first record's content without recomputing hashes.
            with open(log_path) as f:
                chain = json.load(f)
            chain[0]["verdict"] = "resolved (tampered)"
            with open(log_path, "w") as f:
                json.dump(chain, f)
            tampered_result = verify_audit_log_integrity(log_path)
            self.assertFalse(tampered_result["valid"])
            self.assertEqual(tampered_result["broken_at"], 0)


class _TestSemanticMetamorphic(_unittest.TestCase):
    def test_query_order_transformation_is_generated(self):
        cfg = ScanConfig(target="example.com")
        eng = SemanticMetamorphicEngine(cfg, _FakeClient())
        candidates = eng._candidate_urls("https://example.com/a?x=1&y=2")
        labels = {label for _, label in candidates}
        self.assertIn("query-order", labels)
        self.assertIn("duplicate-benign-parameter", labels)

    def test_invariant_compiler_reports_wstg_obligations(self):
        cfg = ScanConfig(target="example.com")
        inv = SecurityInvariantCompiler().compile(["https://example.com/admin", "https://example.com/api/orders"], [], [], cfg)
        self.assertTrue(inv["high_value_endpoints"])
        self.assertGreaterEqual(len(inv["wstg_obligations"]), 8)

    def test_causal_fusion_requires_related_host(self):
        findings = [
            _mk_finding("path_traversal", Severity.HIGH, finding_id="F-1001", url="https://one.example/a"),
            _mk_finding("sensitive_data_exposure", Severity.HIGH, finding_id="F-1002", url="https://two.example/b"),
        ]
        self.assertEqual(CausalRiskFusion().fuse(findings), [])


class _FakeClient:
    def __init__(self):
        self.session = requests.Session()
    def request(self, method, url, **kwargs):
        r = requests.Response()
        r.status_code = 200
        r.url = url
        r._content = b'{"ok":true}'
        r.headers["Content-Type"] = "application/json"
        r.request = requests.Request(method, url).prepare()
        return r


class _TestConsolidation(_unittest.TestCase):
    def test_same_description_findings_consolidated_with_affected_urls(self):
        f1 = _mk_finding("security_headers", Severity.MEDIUM, finding_id="F-0001", url="https://example.com/a")
        f2 = _mk_finding("security_headers", Severity.MEDIUM, finding_id="F-0002", url="https://example.com/b")
        f1.description = f2.description = "No Strict-Transport-Security header"
        consolidated = consolidate_sitewide_findings([f1, f2])
        self.assertEqual(len(consolidated), 1)
        self.assertEqual(sorted(consolidated[0].affected_urls), ["https://example.com/a", "https://example.com/b"])

    def test_non_consolidatable_category_stays_separate(self):
        f1 = _mk_finding("xss", Severity.HIGH, finding_id="F-0001", url="https://example.com/a")
        f2 = _mk_finding("xss", Severity.HIGH, finding_id="F-0002", url="https://example.com/b")
        f1.description = f2.description = "same text"
        consolidated = consolidate_sitewide_findings([f1, f2])
        self.assertEqual(len(consolidated), 2)



class _TestJuiceShopSourceEvidence(_unittest.TestCase):
    def test_source_evidence_engine_confirms_snippet_and_maps_family(self):
        class FakeJuiceClient:
            def request(self, method, url, **kwargs):
                r = requests.Response(); r.status_code = 200; r.url = url
                r.request = requests.Request(method, url).prepare()
                r.headers["Content-Type"] = "application/json"
                if url.endswith("/api/Challenges"):
                    r._content = json.dumps({"status":"success","data":[{"key":"adminSectionChallenge","name":"Admin Section","category":"Broken Access Control","difficulty":2}]}).encode()
                else:
                    r._content = json.dumps({"snippet":"router.get('/admin', guardBypass)","vulnLines":[1]}).encode()
                return r
        engine = JuiceShopSourceEvidenceEngine(ScanConfig(target="example.com"), FakeJuiceClient())
        result = engine.scan("http://example.com/")
        self.assertTrue(result["available"])
        self.assertEqual(result["source_confirmed_challenges"], 1)
        self.assertIn("admin_exposure", result["evidence"][0]["mapped_detector_families"])
        findings = engine.findings_from_evidence(result)
        self.assertEqual(len(findings), 1)
        self.assertEqual(findings[0].confidence, "Confirmed")



class _TestExpandedDetectors(_unittest.TestCase):
    class FakeClient:
        def __init__(self):
            self.session = requests.Session()
        def request(self, method, url, **kwargs):
            r = requests.Response(); r.status_code = 200; r.url = url
            r.request = requests.Request(method, url).prepare()
            r.headers["Content-Type"] = "text/html"
            if url.endswith("/index"):
                r._content = b'<html><script src="/app.js"></script><a href="/auth/authorize?client_id=x"></a></html>'
            elif url.endswith("/app.js.map"):
                r.headers["Content-Type"] = "application/json"
                r._content = b'{"version":3,"sources":["src/main.ts"],"sourcesContent":["const secret=\"demo\";"]}'
            elif url.endswith("/service-worker.js"):
                r.headers["Content-Type"] = "application/javascript"
                r._content = b'workbox.precaching.precacheAndRoute(["/api/account", "/admin"]);'
            else:
                r._content = b'<html>reset password OAuth authorize</html>'
            return r
    def test_source_map_exposure(self):
        cfg=ScanConfig(target="example.com")
        client=self.FakeClient(); m=SourceMapExposureModule(cfg, client)
        out=m.run_url("https://example.com/index")
        self.assertTrue(out)
        self.assertEqual(out[0].category, "source_map_exposure")
    def test_service_worker_signal(self):
        cfg=ScanConfig(target="example.com")
        client=self.FakeClient(); m=ServiceWorkerExposureModule(cfg, client)
        out=m.run_url("https://example.com/")
        self.assertTrue(out)
        self.assertEqual(out[0].category, "service_worker_exposure")
    def test_oauth_state_signal(self):
        cfg=ScanConfig(target="example.com")
        client=self.FakeClient(); m=OAuthStateSignalModule(cfg, client)
        out=m.run_url("https://example.com/index")
        self.assertTrue(out)
        self.assertEqual(out[0].category, "oauth_state")



class _TestSecurityBoundaryDifferential(_unittest.TestCase):
    class FakeClient:
        def __init__(self):
            self.session = requests.Session()
            self.count = 0
        def request(self, method, url, **kwargs):
            r = requests.Response(); r.url = url; r.request = requests.Request(method, url).prepare()
            r.headers["Content-Type"] = "text/html"
            # Canonical endpoint is denied; trailing-slash representation is allowed.
            if url.endswith("/admin/"):
                r.status_code = 200; r._content = b"<html>admin</html>"
            else:
                r.status_code = 403; r._content = b"<html>forbidden</html>"
            return r
    def test_boundary_split_becomes_finding(self):
        cfg = ScanConfig(target="example.com")
        cfg.scope.allowed_methods = ["GET", "HEAD"]
        c = self.FakeClient()
        engine = SecurityBoundaryDifferentialEngine(cfg, c)
        result = engine.analyze(["https://example.com/admin"], limit=1)
        self.assertGreaterEqual(result["signal_count"], 1)
        findings = SecurityBoundaryFindingAdapter(cfg).findings_from(result)
        self.assertTrue(findings)
        self.assertEqual(findings[0].category, "security_boundary_differential")



class _TestERSECShield(_unittest.TestCase):
    def test_virtual_patch_compilation_is_narrow(self):
        report = {"findings": [{
            "finding_id":"F1", "category":"sql_injection_signal", "title":"SQL injection signal",
            "severity":"HIGH", "confidence":"Confirmed", "url":"https://example.com/search", "parameter":"q"
        }, {
            "finding_id":"F2", "category":"sql_injection_signal", "title":"Possible SQLi",
            "severity":"HIGH", "confidence":"Possible", "url":"https://example.com/other", "parameter":"q"
        }]}
        policy = ShieldPolicy.from_report(report)
        self.assertEqual(len(policy.rules), 1)
        self.assertEqual(policy.rules[0].path_prefix, "/search")
        self.assertEqual(policy.rules[0].parameter, "q")
        self.assertEqual(policy.rules[0].action, "block")

    def test_runtime_blocks_path_anomaly_and_rate_limits(self):
        cfg=ScanConfig(target="example.com")
        cfg.shield_rate_per_minute=100
        cfg.shield_burst=100
        runtime=ShieldRuntime(cfg, ShieldPolicy({"defaults":{"methods":["GET"]}}))
        decision, reason, _ = runtime.inspect("GET", "/a/%2e%2e/admin", {}, b"", "127.0.0.1")
        self.assertEqual(decision, ShieldDecision.BLOCK)
        self.assertEqual(reason, "path_normalization_anomaly")

    def test_monitor_mode_does_not_block_matching_rule(self):
        cfg=ScanConfig(target="example.com", shield_mode="monitor")
        rule={"rule_id":"R1","name":"x","category":"xss_signal","action":"block","path_prefix":"/search","parameter":"q","methods":["GET"],"patterns":[r"<\s*script\b"],"reason":"xss", "source_finding_ids":[]}
        runtime=ShieldRuntime(cfg, ShieldPolicy({"defaults":{"methods":["GET"]},"rules":[rule]}))
        decision, _, matched=runtime.inspect("GET", "/search?q=<script>", {}, b"", "127.0.0.1")
        self.assertEqual(decision, ShieldDecision.MONITOR)
        self.assertIsNotNone(matched)


class _TestSecurityControlPlane(_unittest.TestCase):
    def test_control_plane_builds_state(self):
        cfg = ScanConfig(target="example.com")
        engine = SecurityControlPlane(cfg)
        ev = RequestEvidence(method="GET", url="https://example.com/admin", status_code=200)
        f = Finding("F-CP1", "admin_exposure", "Admin Exposure", Severity.HIGH, "Confirmed", "A01:2021", "CWE-285", "https://example.com/admin", None, "admin surface", ev)
        out = engine.analyze(["https://example.com/admin"], [f], [], {"workflow_count": 1}, {"coverage_ratio": 0.95})
        self.assertEqual(out["asset_count"], 1)
        self.assertGreater(out["hypothesis_count"], 0)
        self.assertEqual(out["security_slo"]["status"], "fail")

    def test_control_plane_detects_regression(self):
        previous = {"assets":[{"url":"https://example.com/a"}],"finding_ids":[],"risk_chains":[]}
        current = {"assets":[{"url":"https://example.com/a"},{"url":"https://example.com/b"}],"finding_ids":["x"],"risk_chains":["chain"]}
        diff = SecurityControlPlane.diff(previous, current)
        self.assertTrue(diff["security_regression"])
        self.assertIn("https://example.com/b", diff["new_assets"])


class _TestAssuranceGovernance(_unittest.TestCase):
    def test_category_registry_normalizes_alias(self):
        self.assertEqual(SecurityCategoryRegistry.normalize("sqli"), "sql_injection_signal")
        self.assertTrue(SecurityCategoryRegistry.metadata("sqli").shield_compatible)

    def test_authorization_manifest_fail_closed(self):
        manifest=AuthorizationManifest({
            "schema": ERSEC_AUTH_MANIFEST_SCHEMA,
            "allowed_hosts": ["example.com"],
            "allowed_ports": [443],
            "allowed_paths": ["/api"],
            "allowed_methods": ["GET"],
            "private_address_policy": "allow",
        })
        self.assertEqual(manifest.allows_request("GET", "https://example.com/api/users")[0], True)
        self.assertEqual(manifest.allows_request("POST", "https://example.com/api/users")[0], False)
        self.assertEqual(manifest.allows_request("GET", "https://evil.example/api/users")[0], False)
        strict=AuthorizationManifest({
            "schema": ERSEC_AUTH_MANIFEST_SCHEMA,
            "allowed_hosts": ["127.0.0.1"],
            "allowed_ports": [8080],
            "allowed_methods": ["GET"],
            "private_address_policy": "deny",
        })
        self.assertEqual(strict.allows_request("GET", "http://127.0.0.1:8080/api")[0], False)

    def test_behavior_graph_and_contracts(self):
        cfg=ScanConfig(target="example.com", identity_tokens={"user_a":"TOKEN"})
        ev=RequestEvidence(method="GET",url="https://example.com/api/users/1",status_code=200,response_excerpt="user")
        f=Finding("F29-1","idor_heuristic","Observed object authorization inconsistency",Severity.HIGH,"Likely","A01:2021","CWE-639","https://example.com/api/users/1",None,"review",ev)
        graph=SecurityBehaviorGraph().build(["https://example.com/api/users/1"],[f],{"workflows":[]},[],cfg)
        self.assertGreaterEqual(graph["node_count"],3)
        contracts=SecurityContractEngine().compile([f],graph,{"invariants":[]})
        self.assertEqual(contracts["contract_count"],1)
        self.assertTrue(contracts["contracts"][0]["automation"]["safe_by_default"])

    def test_proof_schema_v2(self):
        cfg=ScanConfig(target="example.com")
        ev=RequestEvidence(method="GET",url="https://example.com/search?q=x",status_code=500,response_excerpt="error")
        f=Finding("F29-2","sql_injection_signal","SQL injection signal",Severity.HIGH,"Confirmed","A03:2021","CWE-89","https://example.com/search?q=x","q","signal",ev)
        f.evidence_score=0.86
        cap=EvidenceCapsuleBuilder(cfg).build(f)
        self.assertEqual(cap["schema"],ERSEC_PROOF_SCHEMA)
        self.assertIn("observation",cap); self.assertIn("detector_reasoning",cap); self.assertIn("limitations",cap); self.assertIn("replay",cap); self.assertIn("integrity",cap)
        self.assertEqual(cap["integrity"]["algorithm"],"sha256")

    def test_risk_budget_reports_skipped(self):
        items=[{"url":f"https://example.com/{i}","information_gain":0.1*i,"crown_jewel":1.0 if i==5 else 0.0,"request_cost":1.0} for i in range(6)]
        out=RiskBudgetScheduler(2).plan(items)
        self.assertEqual(out["selected_count"],2)
        self.assertEqual(out["skipped_count"],4)

    def test_contract_gate_ignores_drafts(self):
        bundle={"schema":ERSEC_CONTRACT_SCHEMA,"contracts":[
            {"contract_id":"A","type":"finding-regression","finding_id":"F1","automation":{"status":"draft"}},
            {"contract_id":"B","type":"finding-regression","finding_id":"F2","automation":{"status":"active"}},
        ]}
        report={"findings":[{"finding_id":"F1"},{"finding_id":"F2"}]}
        result=SecurityContractGate.evaluate(bundle, report)
        self.assertEqual(result["status"],"fail")
        self.assertEqual(result["failed_count"],1)
        self.assertEqual(result["skipped_count"],1)

    def test_contract_approval_requires_reviewer_and_activates_selected(self):
        import tempfile
        with tempfile.TemporaryDirectory() as td:
            draft=_pathlib.Path(td)/"draft.json"; active=_pathlib.Path(td)/"active.json"
            draft.write_text(json.dumps({"schema":ERSEC_CONTRACT_SCHEMA,"contracts":[{"contract_id":"A","type":"invariant","statement":"x","automation":{"status":"draft"}},{"contract_id":"B","type":"invariant","statement":"y","automation":{"status":"draft"}}]}),encoding="utf-8")
            out=SecurityContractApprover.approve(str(draft),str(active),["A"],"security-team","2027-01-01T00:00:00Z")
            self.assertEqual(out["approved_count"],1)
            data=json.loads(active.read_text(encoding="utf-8"))
            self.assertEqual(data["contracts"][0]["automation"]["status"],"active")
            self.assertEqual(data["contracts"][0]["automation"]["reviewed_by"],"security-team")
            self.assertEqual(data["contracts"][1]["automation"]["status"],"draft")

    def test_sbom_contains_ersec_component(self):
        import tempfile
        with tempfile.TemporaryDirectory() as td:
            path=str(_pathlib.Path(td)/"sbom.json")
            result=generate_sbom(path)
            self.assertEqual(result["version"],ERSEC_VERSION)
            data=json.loads(_pathlib.Path(path).read_text(encoding="utf-8"))
            self.assertEqual(data["bomFormat"],"CycloneDX")
            self.assertTrue(any(c.get("name")=="ersec" and c.get("version")==ERSEC_VERSION for c in data["components"]))


class _TestContinuousAssurance(_unittest.TestCase):
    def test_stable_contract_identity_survives_changed_finding_id(self):
        import tempfile
        ev=RequestEvidence(method="GET",url="https://example.com/api/orders/42",status_code=200)
        f=Finding("OLD-ID","idor_heuristic","Object authorization",Severity.HIGH,"Likely","A01:2023","CWE-639","https://example.com/api/orders/42",None,"signal",ev)
        bundle=SecurityContractEngine().compile([f],{}, {"invariants":[]})
        c=bundle["contracts"][0]
        self.assertIn("fingerprint",c)
        report={"findings":[{"finding_id":"NEW-ID","category":"idor_heuristic","url":f.url,"parameter":None,"evidence":{"method":"GET"}}]}
        result=SecurityContractGate.evaluate({"schema":ERSEC_CONTRACT_SCHEMA,"contracts":[dict(c,automation={"status":"active"})]},report)
        self.assertEqual(result["status"],"fail")
        self.assertEqual(result["failures"][0]["matching"],"stable_contract_identity")

    def test_expired_contract_does_not_fail(self):
        future=datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(days=1)
        past=datetime.datetime.now(datetime.timezone.utc)-datetime.timedelta(days=1)
        bundle={"schema":ERSEC_CONTRACT_SCHEMA,"contracts":[{"contract_id":"A","type":"finding-regression","fingerprint":"FCON-nope","finding_id":"F1","automation":{"status":"active","expires_at":past.isoformat()}}]}
        result=SecurityContractGate.evaluate(bundle,{"findings":[{"finding_id":"F1"}]})
        self.assertEqual(result["status"],"pass")
        self.assertEqual(result["expired_count"],1)

    def test_assurance_pack_roundtrip(self):
        import tempfile
        with tempfile.TemporaryDirectory() as td:
            root=_pathlib.Path(td); report=root/"report.json"; report.write_text('{"findings":[]}',encoding='utf-8')
            pack=root/"pack"
            AssurancePack.create(str(pack),str(report))
            ok=AssurancePack.verify(str(pack)); self.assertEqual(ok["status"],"pass")
            (pack/"report.json").write_text('{"findings":[1]}',encoding='utf-8')
            bad=AssurancePack.verify(str(pack)); self.assertEqual(bad["status"],"fail")

    def test_benchmark_lab_scaffold(self):
        import tempfile
        with tempfile.TemporaryDirectory() as td:
            out=BenchmarkLabScaffold.create(td)
            self.assertEqual(out["targets"],6)
            self.assertTrue((_pathlib.Path(td)/"benchmark-lab.json").exists())



class _TestBehaviorModel(_unittest.TestCase):
    def _model(self):
        return SecurityBehaviorModel({
            "schema":"ersec-security-behavior-model/1",
            "name":"unit-test-model",
            "identities":[
                {"name":"anonymous","role":"anonymous","privilege_rank":0},
                {"name":"user_a","role":"user","tenant":"tenant_a","privilege_rank":1},
                {"name":"user_b","role":"user","tenant":"tenant_b","privilege_rank":1},
            ],
            "resources":[
                {"id":"order_a","url":"https://example.com/api/orders/100","owner":"user_a","tenant":"tenant_a","methods":["GET"],
                 "expected":{"user_a":{"status":[200]},"user_b":{"status":[403,404]},"anonymous":{"status":[401,403,404]}}}
            ],
            "invariants":[{"id":"INV-TENANT-A","type":"tenant_isolation","subject":"user_b","resource":"order_a","statement":"tenant_b user must not read tenant_a order"}],
            "workflows":[{"id":"TR-1","from":"checkout","to":"paid","method":"GET","path":"/checkout/status","allowed_roles":["user"]}],
        })

    def test_model_validation(self):
        result=self._model().validate()
        self.assertEqual(result["identities"],3)
        self.assertEqual(result["resources"],1)
        self.assertFalse(result["stateful_methods_allowed"])

    def test_model_rejects_state_changing_method(self):
        with self.assertRaises(ValueError):
            SecurityBehaviorModel({"schema":"ersec-security-behavior-model/1","identities":[],"resources":[{"id":"r","url":"https://example.com/x","methods":["POST"]}]})

    def test_graph_v2_models_relationships(self):
        model=self._model()
        graph=SecurityBehaviorGraphV2().build(model,{"verifications":[]})
        relations={(e["from"],e["relation"],e["to"]) for e in graph["edges"]}
        self.assertIn(("identity:user_a","member-of","tenant:tenant_a"),relations)
        self.assertIn(("resource:order_a","owned-by","identity:user_a"),relations)
        self.assertIn(("resource:order_a","scoped-to","tenant:tenant_a"),relations)
        self.assertEqual(graph["schema"],"ersec-security-behavior-graph/2")

    def test_read_only_verifier_detects_policy_violation(self):
        model=self._model()
        class R:
            def __init__(self,status):
                self.status_code=status; self.text='order'; self.headers={"Content-Type":"application/json"}
        responses={("anonymous","GET","https://example.com/api/orders/100"):R(404),
                   ("user_a","GET","https://example.com/api/orders/100"):R(200),
                   ("user_b","GET","https://example.com/api/orders/100"):R(200)}
        out=SecurityBehaviorVerifier(model).verify(model.identities, lambda i,m,u: responses[(i,m,u)], lambda m,u:(True,""))
        self.assertEqual(out["failure_count"],1)
        self.assertEqual(out["failures"][0]["identity"],"user_b")

    def test_graph_fingerprint_is_stable(self):
        model=self._model()
        a=SecurityBehaviorGraphV2().build(model,{"generated_at":"2026-01-01T00:00:00Z","verifications":[]})
        b=SecurityBehaviorGraphV2().build(model,{"generated_at":"2026-02-01T00:00:00Z","verifications":[]})
        self.assertEqual(a["graph_fingerprint"], b["graph_fingerprint"])

    def test_active_model_contract_fails_on_authorization_regression(self):
        model=self._model()
        contracts=model.contracts()
        auth_contract=next(c for c in contracts if c["type"]=="model-authorization" and c["identity"]=="user_b")
        auth_contract=dict(auth_contract)
        auth_contract["automation"]={"status":"active"}
        report={"findings":[],"security_behavior_verification":{"verifications":[{"identity":"user_b","resource_id":"order_a","method":"GET","verdict":"violation","status":200,"expected":{"status":[403,404]}}]}}
        out=SecurityContractGate.evaluate({"schema":ERSEC_CONTRACT_SCHEMA,"contracts":[auth_contract]},report)
        self.assertEqual(out["status"],"fail")
        self.assertEqual(out["failures"][0]["reason"],"model_authorization_regression")

    def test_state_store_reports_boundary_drift(self):
        import tempfile
        with tempfile.TemporaryDirectory() as td:
            model=self._model(); path=str(_pathlib.Path(td)/"state.json")
            store=SecurityBehaviorStateStore(path)
            first=store.compare_and_save(SecurityBehaviorGraphV2().build(model,{}))
            self.assertEqual(first["status"],"baseline")
            reduced=SecurityBehaviorGraphV2().build(SecurityBehaviorModel({"schema":"ersec-security-behavior-model/1","identities":[{"name":"anonymous","privilege_rank":0}],"resources":[]}),{})
            second=store.compare_and_save(reduced)
            self.assertTrue(second["security_boundary_drift"])

class _TestBehaviorAssurance(_unittest.TestCase):
    def _model(self):
        return SecurityBehaviorModel({
            "schema":"ersec-security-behavior-model/1",
            "identities":[
                {"name":"user_a","role":"user","tenant":"tenant_a","privilege_rank":1},
                {"name":"user_b","role":"user","tenant":"tenant_b","privilege_rank":1},
            ],
            "resources":[{"id":"order_a","url":"https://example.com/api/orders/100","owner":"user_a","tenant":"tenant_a","methods":["GET"],"expected":{"user_a":{"status":[200]},"user_b":{"status":[403,404]}}}],
            "invariants":[{"id":"TENANT","type":"tenant_isolation","subject":"user_b","resource":"order_a","statement":"tenant_b must not read tenant_a order"}],
        })

    def test_assurance_and_counterexample(self):
        model=self._model()
        verification={"verifications":[{"resource_id":"order_a","identity":"user_b","method":"GET","status":200,"expected":{"status":[403,404]},"verdict":"violation"}]}
        assurance=SecurityInvariantEngine(model).evaluate(verification)
        self.assertEqual(assurance["status"],"fail")
        paths=CounterexamplePathEngine().build(model,assurance)
        self.assertEqual(paths["count"],1)
        self.assertEqual(paths["paths"][0]["identity"],"user_b")

    def test_graph_v3_has_counterexample_path(self):
        model=self._model()
        verification={"verifications":[{"resource_id":"order_a","identity":"user_b","method":"GET","status":200,"expected":{"status":[403,404]},"verdict":"violation"}]}
        assurance=SecurityInvariantEngine(model).evaluate(verification)
        graph=SecurityBehaviorGraphV3().build(model,verification,assurance)
        self.assertEqual(graph["schema"],"ersec-security-behavior-graph/3")
        self.assertGreaterEqual(graph["node_count"],1)


class _TestAuthorizationMatrix(_unittest.TestCase):
    def test_authorization_matrix_reports_coverage_and_violation(self):
        model=SecurityBehaviorModel({
            "schema":"ersec-security-behavior-model/1",
            "identities":[{"name":"a","role":"user","tenant":"A"},{"name":"b","role":"user","tenant":"B"}],
            "resources":[{"id":"r","url":"https://example.com/r","owner":"a","tenant":"A","methods":["GET"],"expected":{"a":{"status":[200]},"b":{"status":[403,404]}}}],
        })
        verification={"verifications":[{"identity":"a","resource_id":"r","method":"GET","status":200,"verdict":"pass"},{"identity":"b","resource_id":"r","method":"GET","status":200,"verdict":"violation","expected":{"status":[403,404]}}]}
        out=SecurityAuthorizationMatrix().build(model,verification)
        self.assertEqual(out["row_count"],2)
        self.assertEqual(out["coverage_ratio"],1.0)
        self.assertEqual(out["violation_count"],1)
        self.assertEqual(out["status"],"fail")


def run_self_test(verbosity: int = 2) -> bool:
    """Runs the full self-test suite and returns True iff everything passed.
    Called from main() via `--self-test`, before (and instead of) any real
    network activity."""
    loader = _unittest.TestLoader()
    suite = _unittest.TestSuite()
    for test_class in (_TestTargetValidation, _TestScopeEnforcement, _TestCanonicalFindingId,
                       _TestRiskCorrelationAndScoring, _TestAttackPathFusion,
                       _TestAuditLogIntegrity, _TestSemanticMetamorphic, _TestConsolidation, _TestJuiceShopSourceEvidence,
                       _TestExpandedDetectors, _TestSecurityBoundaryDifferential,
                       _TestSecurityControlPlane, _TestERSECShield, _TestAssuranceGovernance, _TestContinuousAssurance, _TestBehaviorModel, _TestBehaviorAssurance, _TestAuthorizationMatrix):
        suite.addTests(loader.loadTestsFromTestCase(test_class))
    runner = _unittest.TextTestRunner(verbosity=verbosity)
    result = runner.run(suite)
    return result.wasSuccessful()



# =============================================================================
# ERSEC 5.x Autonomous AppSec Intelligence Layer
# =============================================================================

import ast as _ast
import pathlib as _pathlib
import textwrap as _textwrap

@dataclass
class DiscoveryArtifact:
    kind: str
    url: str
    source: str = ""
    method: str = "GET"
    content_type: str = ""
    confidence: float = 0.0
    parameters: List[str] = field(default_factory=list)
    metadata: Dict[str, Any] = field(default_factory=dict)
    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)

@dataclass
class APIEndpointModel:
    url: str
    method: str = "GET"
    protocol: str = "REST"
    parameters: List[str] = field(default_factory=list)
    request_content_types: List[str] = field(default_factory=list)
    response_content_types: List[str] = field(default_factory=list)
    schema_fields: List[str] = field(default_factory=list)
    evidence_urls: List[str] = field(default_factory=list)
    confidence: float = 0.0
    inferred_from: List[str] = field(default_factory=list)
    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)

@dataclass
class VerificationArtifact:
    curl: str
    python: str
    rationale: str
    expected_signal: str
    def to_dict(self) -> Dict[str, str]:
        return asdict(self)

@dataclass
class TriageResult:
    verdict: str
    confidence: float
    evidence_score: float
    explanation: str
    supporting_signals: List[str] = field(default_factory=list)
    contradictory_signals: List[str] = field(default_factory=list)
    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)

@dataclass
class PatchArtifact:
    finding_id: str
    framework: str
    patch_kind: str
    file_hint: str
    patch_text: str
    validation_steps: List[str] = field(default_factory=list)
    review_required: bool = True
    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)

class ContextAwareProbeEngine:
    """Ranks existing ERSEC detector families using parameter semantics."""
    _RULES = {
        "redirect": {"open_redirect":1.0,"ssrf":0.8,"header_injection":0.4},
        "url": {"open_redirect":0.95,"ssrf":0.95},
        "uri": {"open_redirect":0.9,"ssrf":0.9},
        "callback": {"ssrf":0.95,"webhook":0.9},
        "webhook": {"ssrf":0.95,"webhook":1.0},
        "file": {"path_traversal":0.95,"exposed_files":0.8},
        "path": {"path_traversal":1.0,"exposed_files":0.7},
        "filename": {"path_traversal":0.95,"exposed_files":0.65},
        "template": {"ssti":1.0,"xss":0.4},
        "search": {"sql_injection":0.9,"nosql_injection":0.75,"xss":0.55},
        "query": {"sql_injection":0.9,"nosql_injection":0.8,"xss":0.5},
        "filter": {"sql_injection":0.8,"nosql_injection":0.7,"ldap_injection":0.35},
        "sort": {"sql_injection":0.75},
        "id": {"idor_heuristic":1.0,"sql_injection":0.45},
        "user_id": {"idor_heuristic":1.0},
        "account_id": {"idor_heuristic":1.0},
        "order_id": {"idor_heuristic":1.0},
        "invoice_id": {"idor_heuristic":1.0},
        "email": {"user_enumeration":0.8,"sql_injection":0.4,"xss":0.3},
        "username": {"user_enumeration":0.9,"sql_injection":0.3},
        "password": {"rate_limiting":0.85,"user_enumeration":0.55},
        "token": {"jwt":0.85,"weak_session_token":0.75},
        "next": {"open_redirect":0.95},
        "return": {"open_redirect":0.9},
        "return_url": {"open_redirect":1.0},
        "redirect_url": {"open_redirect":1.0,"ssrf":0.55},
        "price": {"commerce_business_logic":1.0},
        "quantity": {"commerce_business_logic":0.9},
        "discount": {"commerce_business_logic":0.9},
        "coupon": {"commerce_business_logic":0.9},
        "role": {"mass_assignment":0.95,"idor_heuristic":0.45},
        "is_admin": {"mass_assignment":1.0},
    }
    def rank(self, parameter: str, endpoint: str = "") -> List[Tuple[str, float]]:
        normalized = re.sub(r"[^a-z0-9]+", "_", (parameter or "").lower()).strip("_")
        tokens = set(normalized.split("_"))
        scores: Dict[str, float] = {}
        for key, mapping in self._RULES.items():
            if key == normalized or key in tokens:
                for family, score in mapping.items():
                    scores[family] = max(scores.get(family, 0.0), score)
        path = urllib.parse.urlparse(endpoint).path.lower()
        for marker, mapping in (
            ("/admin", {"idor_heuristic":0.45,"mass_assignment":0.4}),
            ("/account", {"idor_heuristic":0.45}),
            ("/checkout", {"commerce_business_logic":0.95}),
            ("/cart", {"commerce_business_logic":0.95}),
            ("/payment", {"commerce_business_logic":0.75}),
            ("/webhook", {"webhook":0.9,"ssrf":0.7}),
            ("/graphql", {"graphql_introspection":0.85}),
        ):
            if marker in path:
                for family, score in mapping.items():
                    scores[family] = max(scores.get(family, 0.0), score)
        return sorted(scores.items(), key=lambda x: (-x[1], x[0]))
    def select_parameters(self, params: Iterable[str], endpoint: str = "", top_n: int = 6):
        return {p:self.rank(p,endpoint)[:top_n] for p in params if p}

class BrowserDiscoveryEngine:
    """Optional Playwright discovery for SPA routes, API traffic and WebSockets."""
    def __init__(self, config: ScanConfig):
        self.config=config; self.available=False; self.reason=""
        try:
            from playwright.sync_api import sync_playwright
            self._sync_playwright=sync_playwright; self.available=True
        except Exception as exc:
            self._sync_playwright=None; self.reason=f"Playwright unavailable: {exc}"
    def discover(self,start_url:str)->Dict[str,Any]:
        if not self.available:
            return {"enabled":False,"available":False,"reason":self.reason,"artifacts":[],"fallback":"static_crawler"}
        # Playwright's Python bindings may be installed while the Node.js
        # runtime required by the driver is missing. Preflight it so we do not
        # create an orphaned async Future or emit a traceback; static discovery
        # remains the authoritative fallback.
        node_candidates=(shutil.which("node"), shutil.which("nodejs"), "/usr/bin/node", "/usr/local/bin/node")
        node_path=next((candidate for candidate in node_candidates
                        if candidate and os.path.isfile(candidate) and os.access(candidate,os.X_OK)),None)
        if not node_path:
            return {"enabled":True,"available":False,
                    "reason":"Playwright Python bindings are installed, but Node.js is unavailable.",
                    "fallback":"static_crawler","artifacts":[],"visited_urls":[],
                    "request_urls":[],"websocket_urls":[],"browser_console":[]}
        artifacts=[]; visited_urls=set(); websocket_urls=set(); request_urls=set(); browser_console=[]
        try:
            with self._sync_playwright() as pw:
                browser=pw.chromium.launch(headless=True)
                context=browser.new_context(ignore_https_errors=not self.config.verify_tls)
                page=context.new_page()
                def on_request(req):
                    try:
                        if is_in_scope(req.url,self.config):
                            request_urls.add(req.url)
                            artifacts.append(DiscoveryArtifact("browser_request",req.url,"playwright",
                                req.method,req.headers.get("content-type",""),0.90).to_dict())
                    except Exception: pass
                def on_response(resp):
                    try:
                        if is_in_scope(resp.url,self.config):
                            visited_urls.add(resp.url.split("#")[0])
                            artifacts.append(DiscoveryArtifact("browser_response",resp.url,"playwright",
                                resp.request.method,resp.headers.get("content-type",""),0.92).to_dict())
                    except Exception: pass
                def on_ws(ws):
                    try:
                        if is_in_scope(ws.url,self.config):
                            websocket_urls.add(ws.url)
                            artifacts.append(DiscoveryArtifact("websocket",ws.url,"playwright","CONNECT","",0.98).to_dict())
                    except Exception: pass
                def on_console(msg):
                    if msg.type in {"error","warning"}: browser_console.append(msg.text[:500])
                page.on("request",on_request); page.on("response",on_response)
                page.on("websocket",on_ws); page.on("console",on_console)
                if not is_in_scope(start_url,self.config): raise ScopeError(f"Browser start URL out of scope: {start_url}")
                page.goto(start_url,wait_until="networkidle",
                          timeout=int(self.config.scope.timeout_seconds*1000))
                for href in page.locator("a[href]").evaluate_all("(els)=>els.map(e=>e.href)"):
                    if href and is_in_scope(href,self.config):
                        artifacts.append(DiscoveryArtifact("dom_link",href,"playwright-dom",confidence=0.85).to_dict())
                forms=page.locator("form").evaluate_all(
                    "(els)=>els.map(f=>({action:f.action||location.href,method:(f.method||'GET').toUpperCase(),inputs:Array.from(f.querySelectorAll('input,textarea,select')).map(i=>i.name).filter(Boolean)}))"
                )
                for form in forms:
                    action=form.get("action") or start_url
                    if is_in_scope(action,self.config):
                        artifacts.append(DiscoveryArtifact("dom_form",action,"playwright-dom",
                            form.get("method","GET"),"",0.88,list(form.get("inputs",[]))).to_dict())
                context.close(); browser.close()
        except ScopeError: raise
        except FileNotFoundError as exc:
            return {"enabled":True,"available":False,"error":str(exc),
                    "reason":"Browser runtime could not be started; ERSEC continued with static discovery.",
                    "fallback":"static_crawler","artifacts":artifacts,
                    "visited_urls":sorted(visited_urls),"request_urls":sorted(request_urls),
                    "websocket_urls":sorted(websocket_urls),"browser_console":browser_console[:50]}
        except Exception as exc:
            return {"enabled":True,"available":False,"error":str(exc),
                    "reason":"Browser discovery failed; ERSEC continued with static discovery.",
                    "fallback":"static_crawler","artifacts":artifacts,
                    "visited_urls":sorted(visited_urls),"request_urls":sorted(request_urls),
                    "websocket_urls":sorted(websocket_urls),"browser_console":browser_console[:50]}
        return {"enabled":True,"available":True,"artifacts":artifacts,
                "visited_urls":sorted(visited_urls),"request_urls":sorted(request_urls),
                "websocket_urls":sorted(websocket_urls),"browser_console":browser_console[:50]}

class APIIntelligenceEngine:
    """Infer lightweight REST/GraphQL/gRPC models from observed traffic."""
    _PATH_PARAM_RE=re.compile(r"/\{?([A-Za-z_][A-Za-z0-9_-]{1,64})\}?/")
    _GRPC_TYPES={"application/grpc","application/grpc+proto","application/grpc-web","application/grpc-web+proto"}
    def infer_from_urls(self,urls):
        models={}
        for raw in urls:
            try: parsed=urllib.parse.urlparse(raw)
            except Exception: continue
            path=parsed.path or "/"
            looks=bool(_API_LIKE_PATH_RE.match(path) or re.search(r"/(?:graphql|grpc|rpc)(?:/|$)",path,re.I))
            if not looks: continue
            proto="GraphQL" if "graphql" in path.lower() else "REST"
            if re.search(r"/(?:grpc|rpc)(?:/|$)",path,re.I): proto="gRPC"
            params=set(urllib.parse.parse_qs(parsed.query).keys())
            params.update(m.group(1) for m in self._PATH_PARAM_RE.finditer(path))
            key=(parsed._replace(query="").geturl(),proto)
            if key not in models:
                models[key]=APIEndpointModel(key[0],protocol=proto,parameters=sorted(params),
                    evidence_urls=[raw],confidence=0.72 if proto=="REST" else 0.84,
                    inferred_from=["observed URL"])
            else:
                models[key].parameters=sorted(set(models[key].parameters)|params)
                models[key].evidence_urls.append(raw)
                models[key].confidence=min(0.99,models[key].confidence+0.03)
        return sorted(models.values(),key=lambda x:(-x.confidence,x.url))
    def infer_from_json(self,url,body):
        try: obj=json.loads(body)
        except Exception: return None
        keys=set()
        def walk(v,depth=0):
            if depth>4: return
            if isinstance(v,dict):
                for k,val in v.items(): keys.add(str(k)); walk(val,depth+1)
            elif isinstance(v,list):
                for item in v[:10]: walk(item,depth+1)
        walk(obj)
        return APIEndpointModel(url,parameters=sorted(keys)[:100],schema_fields=sorted(keys)[:100],
            evidence_urls=[url],confidence=0.78,inferred_from=["observed JSON"]) if keys else None
    def classify_content_type(self,url,content_type):
        ct=(content_type or "").split(";")[0].lower()
        if ct in self._GRPC_TYPES:
            return APIEndpointModel(url,protocol="gRPC",request_content_types=[ct],
                response_content_types=[ct],evidence_urls=[url],confidence=0.95,
                inferred_from=["observed gRPC content type"])
        return None

class EvidenceVerifier:
    _SECRET={"authorization","cookie","set-cookie","x-api-key","proxy-authorization"}
    @classmethod
    def build(cls,finding):
        ev=finding.evidence
        headers={k:v for k,v in ev.request_headers_sent.items() if k.lower() not in cls._SECRET}
        parts=["curl","-i","-sS","-X",shlex.quote(ev.method or "GET"),shlex.quote(ev.url)]
        for k,v in headers.items(): parts += ["-H",shlex.quote(f"{k}: {v}")]
        curl=" ".join(parts)
        python=("import requests\\n\\n"
                f"url = {ev.url!r}\\nheaders = {headers!r}\\n"
                f"r = requests.request({(ev.method or 'GET')!r}, url, headers=headers, "
                "timeout=15, allow_redirects=False)\\n"
                "print(r.status_code)\\nprint(dict(r.headers))\\nprint(r.text[:1000])\\n")
        return VerificationArtifact(curl,python,
            "Replay uses captured method/URL and excludes saved session credentials.",
            f"Reproduce the '{finding.category}' behavioral signal seen in ERSEC evidence.")

class EvidenceTriageEngine:
    """Evidence-first scoring with optional local GGUF explanation."""
    def __init__(self,config):
        self.model=None
        path=getattr(config,"triage_model_path",None) or getattr(config,"local_model_path",None)
        if path:
            try:
                from llama_cpp import Llama
                self.model=Llama(model_path=path,n_ctx=4096,verbose=False)
            except Exception: self.model=None
    def triage(self,finding):
        ev=finding.evidence; body=ev.response_excerpt or ""; score=0.0; supporting=[]; contradictory=[]
        if ev.status_code: score+=0.10; supporting.append(f"HTTP {ev.status_code} captured")
        if ev.response_headers: score+=0.10; supporting.append("response headers captured")
        if body: score+=0.15; supporting.append("response body excerpt captured")
        text=(finding.description+" "+body).lower()
        for marker,weight in {"exposed":0.30,"reflected":0.25,"injected":0.28,"accepted":0.22,"allows":0.20,"missing":0.18}.items():
            if marker in text: score+=weight; supporting.append("marker:"+marker)
        for marker in ("possible","heuristic","guess","timing"):
            if marker in text: score-=0.08; contradictory.append("uncertainty:"+marker)
        score += {"Confirmed":0.18,"Likely":0.08,"Possible":-0.04}.get(finding.confidence,0)
        score=max(0.0,min(1.0,score))
        verdict="high_confidence" if score>=0.78 and finding.confidence=="Confirmed" else ("needs_review" if score>=0.55 else "weak_evidence")
        explanation=f"Evidence score {score:.2f}; verdict={verdict}. Evidence is based on captured request/response data and detector confidence."
        if self.model:
            prompt=_textwrap.dedent(f"""
            You are a defensive AppSec triage assistant. Analyze only the supplied evidence.
            Do not invent exploitation steps.
            Category: {finding.category}
            Title: {finding.title}
            Confidence: {finding.confidence}
            Description: {finding.description}
            HTTP status: {ev.status_code}
            Response excerpt: {body[:1200]}
            Explain whether the evidence supports the finding and give safe verification guidance.
            """).strip()
            try:
                out=self.model(prompt,max_tokens=400,temperature=0.1)
                generated=out["choices"][0]["text"].strip()
                if generated: explanation=generated[:4000]
            except Exception: pass
        return TriageResult(verdict,score,score,explanation,supporting,contradictory)

class AppSecAttackPathFusion:
    """Capability graph for higher-level risk narratives."""
    _CAPS={
        "read_sensitive_data":{"sql_injection","ssrf","xxe","exposed_files"},
        "reach_internal_service":{"ssrf","xxe"},
        "change_state":{"csrf","mass_assignment","command_injection"},
        "client_script_execution":{"xss","dom_xss"},
        "identity_control_signal":{"jwt","weak_session_token","idor_heuristic"},
        "authorization_gap":{"idor_heuristic","mass_assignment","admin_exposure"},
        "commerce_state":{"commerce_business_logic","csrf","idor_heuristic"},
    }
    _RULES=(
        ("sensitive_data_identity",{"read_sensitive_data","identity_control_signal"},Severity.CRITICAL,"Sensitive-data access capability overlaps an identity/session weakness."),
        ("internal_service_data",{"reach_internal_service","read_sensitive_data"},Severity.CRITICAL,"Server-side internal reachability overlaps a data-access capability."),
        ("authorization_state_change",{"authorization_gap","change_state"},Severity.HIGH,"Authorization weakness overlaps a state-changing capability."),
        ("commerce_authorization",{"commerce_state","authorization_gap"},Severity.HIGH,"Commerce state changes overlap an authorization weakness."),
    )
    def fuse(self,findings):
        caps={}; ids={}
        for f in findings:
            for cap,cats in self._CAPS.items():
                if f.category in cats:
                    caps.setdefault(cap,set()).add(f.category); ids.setdefault(cap,[]).append(f.finding_id)
        out=[]
        for name,required,sev,rationale in self._RULES:
            if required.issubset(caps):
                out.append({"name":name,"severity":sev.name,"rationale":rationale,
                    "contributing_findings":sorted({i for c in required for i in ids.get(c,[])}),
                    "hop_count":len(required),"capabilities":sorted(required)})
        return out

class DeveloperRemediationEngine:
    """Creates review-required framework-aware remediation bundles."""
    PATCHES={
        "Django (Python)":{"sql_injection":("application/models.py","Use Django ORM or parameterized cursor.execute()."),
            "csrf":("settings.py","Keep CSRF middleware enabled and validate state-changing requests."),
            "xss":("templates/","Keep auto-escaping enabled; sanitize intentional rich HTML."),
            "ssrf":("application/network.py","Allow-list outbound destinations and reject private/link-local ranges."),
            "idor_heuristic":("application/views.py","Scope object lookups to the authenticated principal."),
            "mass_assignment":("application/serializers.py","Explicitly allow-list writable request fields.")},
        "Express (Node.js)":{"sql_injection":("src/db.js","Use parameterized queries or a trusted ORM."),
            "csrf":("src/app.js","Use CSRF protection for cookie-authenticated state-changing routes."),
            "xss":("src/views/","Escape/sanitize output at render boundaries."),
            "ssrf":("src/network.js","Allow-list outbound hosts and block private destinations."),
            "prototype_pollution":("src/validation.js","Reject prototype-bearing keys and validate object schemas."),
            "mass_assignment":("src/routes/","Map request fields into explicit writable DTOs.")},
        "Laravel (PHP)":{"sql_injection":("app/Models/","Use Eloquent/query-builder bindings."),
            "csrf":("app/Http/Middleware/","Keep CSRF middleware active for browser state changes."),
            "xss":("resources/views/","Keep Blade escaping enabled."),
            "ssrf":("app/Services/","Allow-list outbound hosts and reject private destinations."),
            "mass_assignment":("app/Models/","Define explicit fillable fields.")},
        "Spring Boot (Java)":{"sql_injection":("src/main/java/","Use prepared statements/JPA parameter binding."),
            "csrf":("SecurityConfig.java","Keep CSRF protections for browser sessions where applicable."),
            "xss":("src/main/resources/templates/","Escape output and sanitize rich HTML."),
            "ssrf":("src/main/java/","Allow-list outbound hosts and reject private/link-local destinations."),
            "xxe":("src/main/java/","Disable DOCTYPE/external entity resolution.")},
    }
    def __init__(self,output_dir=None): self.output_dir=_pathlib.Path(output_dir) if output_dir else None
    def build(self,finding,stack):
        framework=stack.get("framework_guess","Generic")
        hint,instruction=self.PATCHES.get(framework,{}).get(
            finding.category,("<locate affected application code>",
            finding.remediation_summary or "Apply framework-native controls and add a regression test."))
        return PatchArtifact(finding.finding_id,framework,"review_required_patch_proposal",hint,
            f"# ERSEC remediation proposal: {finding.finding_id}\\n# Category: {finding.category}\\n# Framework: {framework}\\n\\n## Change\\n{instruction}\\n\\n## Review\\nERSEC does not silently modify source code or open remote PRs.\\n",
            ["Replay verification before fix.","Apply framework-native change.","Replay verification after fix.","Add/run regression test."])
    def export(self,findings,stack):
        bundle={"generator":"ERSEC 5.x DeveloperRemediationEngine","generated_at":_utc_now_iso(),
                "stack_fingerprint":stack,"review_required":True,
                "patches":[self.build(f,stack).to_dict() for f in findings]}
        if self.output_dir:
            self.output_dir.mkdir(parents=True,exist_ok=True)
            (self.output_dir/"patches.json").write_text(json.dumps(bundle,indent=2),encoding="utf-8")
            body=["# ERSEC Security Remediation","","Review-required remediation bundle.",""]
            for p in bundle["patches"]:
                body += [f"## {p['finding_id']} — {p['framework']}",f"File hint: `{p['file_hint']}`","",p["patch_text"],""]
            (self.output_dir/"PR_BODY.md").write_text("\\n".join(body),encoding="utf-8")
        return bundle

class PolicyAsCodeExporter:
    TEMPLATES={
        "command_injection":"""rules:
  - id: ersec-command-injection-review
    languages: [python]
    message: "Review dynamic command execution reported by ERSEC."
    severity: ERROR
    patterns:
      - pattern-either:
          - pattern: subprocess.$FUNC(..., shell=True, ...)
          - pattern: os.system(...)
""",
        "xss":"""rules:
  - id: ersec-xss-review
    languages: [javascript, typescript]
    message: "Review unsafe HTML sinks."
    severity: WARNING
    pattern-either:
      - pattern: $EL.innerHTML = $X
      - pattern: document.write($X)
""",
        "sql_injection":"""rules:
  - id: ersec-sql-review
    languages: [python, javascript, php, java]
    message: "Review dynamically constructed SQL."
    severity: ERROR
    pattern-regex: "(SELECT|UPDATE|DELETE|INSERT).*(\\\\+|format\\\\(|f)"
""",
    }
    def export(self,findings,output_dir):
        out=_pathlib.Path(output_dir); (out/"semgrep").mkdir(parents=True,exist_ok=True); written=[]
        for category in sorted({f.category for f in findings}):
            rule=self.TEMPLATES.get(category)
            if rule:
                p=out/"semgrep"/f"ersec-{category}.yml"; p.write_text(rule,encoding="utf-8"); written.append(str(p))
        rego=("package ersec\\n\\ndefault deny := false\\n\\ndeny contains f if {\\n  f := input.findings[_]\\n  f.severity == \"CRITICAL\"\\n}\\n\\ndeny contains f if {\\n  f := input.findings[_]\\n  f.severity == \"HIGH\"\\n  f.confidence == \"Confirmed\"\\n}\\n")
        rp=out/"ersec.rego"; rp.write_text(rego,encoding="utf-8")
        return {"semgrep_rules":written,"rego":str(rp),"categories":sorted({f.category for f in findings})}

class CloudNativeAwarenessEngine:
    _PATTERNS={
        "aws_s3":re.compile(r"(?:https?://)?[A-Za-z0-9._-]+\.s3(?:[-.][A-Za-z0-9-]+)?\.amazonaws\.com",re.I),
        "aws_metadata":re.compile(r"http://169\.254\.169\.254",re.I),
        "gcp_metadata":re.compile(r"metadata\.google\.internal",re.I),
        "k8s_service":re.compile(r"https?://kubernetes\.default(?:\.svc)?",re.I),
        "docker_socket":re.compile(r"unix:///var/run/docker\.sock",re.I),
    }
    def inspect(self,text_blobs):
        signals=[]
        for source,text in text_blobs:
            for kind,rx in self._PATTERNS.items():
                for m in rx.finditer(text or ""):
                    signals.append({"type":kind,"source":source,"match":m.group(0)[:300],"active_probe":False})
        return {"signals":signals,"signal_count":len(signals),"note":"Passive application-visible cloud-native references only."}

class CommerceIntelligenceEngine:
    _ROUTE=re.compile(r"/(?:api/)?(?:products?|collections?|cart|checkout|orders?|discounts?|coupons?|inventory|pricing|payments?|customers?)(?:/|$)",re.I)
    _PARAMS={"price","amount","quantity","discount","coupon","shipping","tax","variant","product_id","cart_id","order_id"}
    def inspect(self,urls,parameters):
        routes=sorted(u for u in set(urls) if self._ROUTE.search(urllib.parse.urlparse(u).path))
        params=sorted({p.lower() for p in parameters if p and p.lower() in self._PARAMS})
        return {"commerce_routes":routes,"business_logic_parameters":params,
                "signals":[{"type":"commerce_parameter","parameter":p,"risk":"review_business_logic_invariant"} for p in params],
                "review_targets":["price integrity","quantity bounds","discount/coupon authorization","cart ownership","checkout state transitions","order ownership"]}

class WebProtocolIntelligence:
    _HOOK=re.compile(r"(?:webhook|callback|notify_url|callback_url|return_url|target_url|endpoint_url)",re.I)
    def inspect(self,artifacts,endpoint_urls,parameter_names):
        websockets=sorted({a.get("url","") for a in artifacts if a.get("kind")=="websocket"})
        hook_params=sorted({p for p in parameter_names if self._HOOK.search(p or "")})
        hook_urls=sorted({u for u in endpoint_urls if re.search(r"/(?:webhook|callback)(?:/|$)",u,re.I)})
        return {"websockets":websockets,"websocket_count":len(websockets),
                "webhook_parameter_candidates":hook_params,"webhook_endpoint_candidates":hook_urls,
                "guidance":"Review authentication, origin validation, replay handling and message authorization."}

class PluginMarketplace:
    TEMPLATE="""from ersec import BaseModule, ScanProfile

class MyERSECPlugin(BaseModule):
    category = "my_plugin"
    title = "My ERSEC Plugin"
    owasp = "A05:2021"
    cwe = "CWE-200"
    min_profile = ScanProfile.PASSIVE

    def run_url(self, url):
        return []

PLUGIN_METADATA = {
    "name": "my-ersec-plugin",
    "version": "1.0.0",
    "author": "community",
    "description": "Defensive ERSEC detector plugin",
    "entrypoint": "plugin.py",
    "category": "custom",
}
"""
    REQUIRED=("name","version","author","description","entrypoint","category")
    def scaffold(self,directory):
        root=_pathlib.Path(directory); root.mkdir(parents=True,exist_ok=True)
        (root/"plugin.py").write_text(self.TEMPLATE,encoding="utf-8")
        (root/"plugin.json").write_text(json.dumps({"name":"my-ersec-plugin","version":"1.0.0","author":"community","description":"Defensive ERSEC detector plugin","entrypoint":"plugin.py","category":"custom"},indent=2),encoding="utf-8")
        (root/"README.md").write_text("# ERSEC Plugin\\n\\nImplement a BaseModule and test against an authorized lab target.\\n",encoding="utf-8")
        return {"plugin":str(root/"plugin.py"),"manifest":str(root/"plugin.json"),"readme":str(root/"README.md")}
    def validate(self,path):
        data=json.loads(_pathlib.Path(path).read_text(encoding="utf-8"))
        missing=[k for k in self.REQUIRED if k not in data]
        if missing: raise ERSECError("Plugin manifest missing: "+", ".join(missing))
        return {"valid":True,"manifest":data}

class AssurancePack:
    """Create and verify a deterministic release/assessment assurance bundle.

    The pack is intentionally content-addressed: every included artifact gets a SHA-256
    digest recorded in manifest.json. This is integrity evidence, not a digital signature.
    """
    SCHEMA="ersec-assurance-pack/1"
    DEFAULTS=("report.json","contracts.json","sbom.json","security-graph.json")

    @staticmethod
    def _sha256(path: Path) -> str:
        h=hashlib.sha256()
        with path.open('rb') as fh:
            for chunk in iter(lambda: fh.read(1024*1024), b''): h.update(chunk)
        return h.hexdigest()

    @classmethod
    def create(cls, output_dir: str, report_path: Optional[str]=None, contracts_path: Optional[str]=None, sbom_path: Optional[str]=None, graph_path: Optional[str]=None) -> Dict[str,Any]:
        root=_pathlib.Path(output_dir); root.mkdir(parents=True,exist_ok=True)
        sources=[("report.json",report_path),("contracts.json",contracts_path),("sbom.json",sbom_path),("security-graph.json",graph_path)]
        copied=[]
        for name,src in sources:
            if not src: continue
            sp=_pathlib.Path(src)
            if not sp.exists(): raise ERSECError(f"Assurance artifact not found: {src}")
            dst=root/name
            if sp.resolve()!=dst.resolve(): shutil.copy2(sp,dst)
            copied.append(dst)
        files=[]
        for path in sorted(root.iterdir()):
            if path.is_file() and path.name!="manifest.json":
                files.append({"name":path.name,"size":path.stat().st_size,"sha256":cls._sha256(path)})
        manifest={"schema":cls.SCHEMA,"tool":"ERSEC","version":ERSEC_VERSION,"generated_at":_utc_now_iso(),"artifacts":files,"interpretation":"Content hashes provide integrity evidence; this manifest is not a cryptographic signature."}
        (root/"manifest.json").write_text(json.dumps(manifest,indent=2,sort_keys=True),encoding='utf-8')
        return {"directory":str(root),"artifact_count":len(files),"manifest":str(root/"manifest.json"),"schema":cls.SCHEMA}

    @classmethod
    def verify(cls, directory: str) -> Dict[str,Any]:
        root=_pathlib.Path(directory); mp=root/"manifest.json"
        if not mp.exists(): raise ERSECError("Assurance pack manifest.json is missing")
        manifest=json.loads(mp.read_text(encoding='utf-8'))
        if manifest.get("schema")!=cls.SCHEMA: raise ERSECError(f"Unsupported assurance pack schema: {manifest.get('schema')!r}")
        mismatches=[]; missing=[]
        for item in manifest.get("artifacts",[]):
            path=root/str(item.get("name",""))
            if not path.exists(): missing.append(path.name); continue
            actual=cls._sha256(path)
            if actual!=item.get("sha256"): mismatches.append({"name":path.name,"expected":item.get("sha256"),"actual":actual})
        status="pass" if not missing and not mismatches else "fail"
        return {"schema":cls.SCHEMA,"status":status,"artifact_count":len(manifest.get("artifacts",[])),"missing":missing,"mismatches":mismatches,"verified_at":_utc_now_iso()}


class BenchmarkLabScaffold:
    """Generate a vendor-neutral benchmark laboratory manifest and CI templates."""
    SCHEMA="ersec-benchmark-lab/1"
    @classmethod
    def create(cls,directory:str)->Dict[str,Any]:
        root=_pathlib.Path(directory); root.mkdir(parents=True,exist_ok=True)
        manifest={"schema":cls.SCHEMA,"version":1,"name":"ERSEC Benchmark Laboratory","oracle_policy":"operator-supplied","targets":[
            {"id":"juice-shop","purpose":"web/API benchmark","truth":"truth/juice-shop.json","run":"./run-juice-shop.sh"},
            {"id":"webgoat","purpose":"web security benchmark","truth":"truth/webgoat.json","run":"./run-webgoat.sh"},
            {"id":"multitenant-rest","purpose":"tenant isolation and object authorization","truth":"truth/multitenant-rest.json","run":"./run-multitenant-rest.sh"},
            {"id":"graphql-authz","purpose":"GraphQL authorization consistency","truth":"truth/graphql-authz.json","run":"./run-graphql-authz.sh"},
            {"id":"oauth-session","purpose":"OAuth/session boundary behavior","truth":"truth/oauth-session.json","run":"./run-oauth-session.sh"},
            {"id":"false-positive-traps","purpose":"benign control corpus","truth":"truth/false-positive-traps.json","run":"./run-false-positive-traps.sh"}
        ],"metrics":["true_positives","false_positives","false_negatives","precision","recall","f1","request_count","runtime_seconds"],"rule":"Do not publish detector quality numbers without recording the oracle, target version, ERSEC version, request budget and test conditions."}
        (root/"benchmark-lab.json").write_text(json.dumps(manifest,indent=2),encoding='utf-8')
        (root/"truth").mkdir(exist_ok=True)
        (root/"README.md").write_text("# ERSEC Benchmark Laboratory\n\nThis is a scaffold. Add only authorized local benchmark targets and operator-reviewed ground truth.\n\nRun quality measurement with `ersec --benchmark-report REPORT --benchmark-truth TRUTH --benchmark-quality QUALITY.json`.\n",encoding='utf-8')
        (root/"GITHUB_ACTIONS.md").write_text("Use the repository release/CI workflow to execute benchmark jobs only against local lab targets. Do not point benchmark automation at third-party systems.\n",encoding='utf-8')
        return {"directory":str(root),"manifest":str(root/"benchmark-lab.json"),"targets":len(manifest["targets"])}


class BenchmarkEngine:
    @staticmethod
    def key(f): return f"{f.get('category','')}|{str(f.get('url','')).split('?')[0]}|{f.get('parameter') or ''}"
    def compare(self,report,truth):
        actual={self.key(f):f for f in report.get("findings",[])}; expected={self.key(f):f for f in truth.get("findings",[])}
        tp=len(set(actual)&set(expected)); fp=len(set(actual)-set(expected)); fn=len(set(expected)-set(actual))
        precision=tp/(tp+fp) if tp+fp else 1.0; recall=tp/(tp+fn) if tp+fn else 1.0
        f1=2*precision*recall/(precision+recall) if precision+recall else 0.0
        return {"true_positives":tp,"false_positives":fp,"false_negatives":fn,"precision":round(precision,4),
                "recall":round(recall,4),"f1":round(f1,4),"oracle_source":truth.get("name","operator-supplied")}
    def run_file(self,report_path,truth_path):
        return self.compare(json.loads(_pathlib.Path(report_path).read_text()),json.loads(_pathlib.Path(truth_path).read_text()))



class AssuranceScorecard:
    """Maps observed ERSEC evidence to high-level API-security control themes.

    This is a coverage/assurance view, not a compliance certification. A control is
    marked "observed" only when the report contains relevant evidence; otherwise it
    remains "not_established" rather than being treated as secure.
    """
    API_CONTROLS = (
        ("API1:2023", "Broken Object Level Authorization", ("idor", "object_authorization", "observed_object_authorization")),
        ("API2:2023", "Broken Authentication", ("jwt", "authentication", "weak_session", "oauth")),
        ("API3:2023", "Broken Object Property Level Authorization", ("mass_assignment", "excessive_data_exposure", "sensitive_data_exposure")),
        ("API4:2023", "Unrestricted Resource Consumption", ("rate_limiting", "resource_consumption", "dos")),
        ("API5:2023", "Broken Function Level Authorization", ("admin_exposure", "method_authorization", "function_authorization")),
        ("API6:2023", "Unrestricted Access to Sensitive Business Flows", ("commerce", "workflow", "sensitive_business_flow")),
        ("API7:2023", "Server Side Request Forgery", ("ssrf", "server_side_request_forgery")),
        ("API8:2023", "Security Misconfiguration", ("security_headers", "cors", "http_methods", "tls", "cache")),
        ("API9:2023", "Improper Inventory Management", ("api_version_drift", "api_contract", "discovery", "inventory")),
        ("API10:2023", "Unsafe Consumption of APIs", ("unsafe_api_consumption", "third_party_api", "webhook")),
    )

    @classmethod
    def build(cls, report: Dict[str, Any], contracts: Optional[Dict[str, Any]]=None,
              benchmark: Optional[Dict[str, Any]]=None) -> Dict[str, Any]:
        findings=report.get("findings",[]) if isinstance(report,dict) else []
        cats=[SecurityCategoryRegistry.normalize(f.get("category","")) for f in findings if isinstance(f,dict)]
        text=" ".join(cats).lower()
        rows=[]
        for code,title,aliases in cls.API_CONTROLS:
            evidence=[]
            for c,a in zip(cats, findings):
                if any(alias in c.lower() for alias in aliases):
                    evidence.append(a.get("finding_id",""))
            status="observed" if evidence else "not_established"
            rows.append({"control":code,"title":title,"status":status,"finding_ids":sorted(set(x for x in evidence if x))})
        established=sum(1 for r in rows if r["status"]=="observed")
        open_high=sum(1 for f in findings if str(f.get("severity","")).upper() in {"HIGH","CRITICAL"})
        score=round(max(0.0, min(1.0, established/len(rows) * (0.75 if open_high else 1.0))),4)
        result={"schema":"ersec-assurance-scorecard/1","framework":"OWASP API Security Top 10:2023","coverage_ratio":round(established/len(rows),4),"assurance_score":score,"controls":rows,"open_high_or_critical":open_high}
        if contracts:
            active=sum(1 for c in contracts.get("contracts",[]) if c.get("automation",{}).get("status")=="active")
            result["active_contracts"]=active
        if benchmark:
            result["benchmark_quality"]={k:benchmark.get(k) for k in ("precision","recall","f1","true_positives","false_positives","false_negatives") if k in benchmark}
        result["statement"]="This scorecard measures observed evidence coverage; not_established does not mean secure and this is not a compliance certification."
        return result


class ReleaseAssuranceGate:
    """Deterministic release gate for continuous security assurance.

    The gate fails only on explicit policy violations. Missing evidence is surfaced
    as a separate condition instead of being silently treated as a pass.
    """
    @classmethod
    def evaluate(cls, report: Dict[str,Any], contracts: Optional[Dict[str,Any]]=None,
                 benchmark: Optional[Dict[str,Any]]=None, sbom: Optional[Dict[str,Any]]=None,
                 min_coverage: float=0.80, max_high_critical: int=0,
                 min_precision: Optional[float]=None, min_recall: Optional[float]=None) -> Dict[str,Any]:
        findings=report.get("findings",[]) if isinstance(report,dict) else []
        high_critical=sum(1 for f in findings if str(f.get("severity","")).upper() in {"HIGH","CRITICAL"})
        cov=((report.get("coverage_matrix") or {}).get("coverage_ratio"))
        checks=[]
        checks.append({"id":"high-critical-findings","pass":high_critical<=max_high_critical,"observed":high_critical,"target":f"<= {max_high_critical}"})
        checks.append({"id":"coverage","pass":isinstance(cov,(int,float)) and float(cov)>=min_coverage,"observed":cov,"target":f">= {min_coverage}"})
        if contracts is not None:
            active=sum(1 for c in contracts.get("contracts",[]) if c.get("automation",{}).get("status")=="active")
            expired=sum(1 for c in contracts.get("contracts",[]) if c.get("automation",{}).get("status")=="expired")
            checks.append({"id":"active-contract-expiry","pass":expired==0,"observed":{"active":active,"expired":expired},"target":"0 expired active contracts"})
        else:
            checks.append({"id":"contract-evidence","pass":False,"observed":"not supplied","target":"active contract bundle supplied for governed releases"})
        if sbom is not None:
            checks.append({"id":"sbom","pass":sbom.get("bomFormat")=="CycloneDX" and str(sbom.get("specVersion","")) in {"1.6","1.7"},"observed":{"bomFormat":sbom.get("bomFormat"),"specVersion":sbom.get("specVersion")},"target":"CycloneDX 1.6/1.7"})
        else:
            checks.append({"id":"sbom","pass":False,"observed":"not supplied","target":"CycloneDX SBOM supplied"})
        if benchmark is not None:
            if min_precision is not None:
                checks.append({"id":"benchmark-precision","pass":float(benchmark.get("precision",0.0))>=min_precision,"observed":benchmark.get("precision"),"target":f">= {min_precision}"})
            if min_recall is not None:
                checks.append({"id":"benchmark-recall","pass":float(benchmark.get("recall",0.0))>=min_recall,"observed":benchmark.get("recall"),"target":f">= {min_recall}"})
        failed=[c for c in checks if not c["pass"]]
        return {"schema":"ersec-release-assurance-gate/1","status":"fail" if failed else "pass","checks":checks,"failed_count":len(failed),"statement":"Gate enforces explicit release policy only; missing evidence is surfaced rather than interpreted as security."}

class IDEIntegration:
    def vscode(self,directory):
        root=_pathlib.Path(directory); root.mkdir(parents=True,exist_ok=True)
        (root/"package.json").write_text(json.dumps({"name":"ersec-security","displayName":"ERSEC Security","version":"1.0.0","engines":{"vscode":"^1.85.0"},"main":"./extension.js","activationEvents":["onStartupFinished"]},indent=2),encoding="utf-8")
        (root/"extension.js").write_text(r"""
const vscode = require("vscode");
const cp = require("child_process");
function activate(context) {
  const diagnostics = vscode.languages.createDiagnosticCollection("ersec");
  const scan = () => {
    const root = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath;
    if (!root) return;
    cp.execFile("python", ["ersec.py","--code-scan",root,"--code-scan-output",".ersec-code.json"], {cwd:root}, () => {});
  };
  context.subscriptions.push(diagnostics,
    vscode.commands.registerCommand("ersec.scanWorkspace", scan),
    vscode.workspace.onDidSaveTextDocument(scan));
}
function deactivate() {}
module.exports = {activate,deactivate};
""".strip()+"\n",encoding="utf-8")
        return {"package":str(root/"package.json"),"extension":str(root/"extension.js")}
    def intellij(self,directory):
        root=_pathlib.Path(directory); (root/"src/main/resources/META-INF").mkdir(parents=True,exist_ok=True); (root/"src/main/kotlin").mkdir(parents=True,exist_ok=True)
        (root/"src/main/resources/META-INF/plugin.xml").write_text('<idea-plugin><id>com.ersec.security</id><name>ERSEC Security</name><vendor>ERSEC</vendor></idea-plugin>\\n',encoding="utf-8")
        (root/"src/main/kotlin/ErsecPlugin.kt").write_text('package com.ersec.security\\nclass ErsecPlugin { fun name() = "ERSEC Security" }\\n',encoding="utf-8")
        return {"plugin_xml":str(root/"src/main/resources/META-INF/plugin.xml"),"kotlin":str(root/"src/main/kotlin/ErsecPlugin.kt")}

class CodeSecurityEngine:
    RULES=(
        ("python","command_injection",re.compile(r"subprocess\.(?:run|Popen|call|check_output)\([^)]*shell\s*=\s*True",re.I),"HIGH"),
        ("python","command_injection",re.compile(r"\bos\.system\s*\(",re.I),"HIGH"),
        ("python","code_execution_risk",re.compile(r"\beval\s*\(",re.I),"HIGH"),
        ("python","unsafe_deserialization",re.compile(r"\byaml\.load\s*\(",re.I),"MEDIUM"),
        ("javascript","command_injection",re.compile(r"child_process\.(?:exec|execSync)\s*\(",re.I),"HIGH"),
        ("javascript","xss",re.compile(r"\.innerHTML\s*=",re.I),"HIGH"),
        ("javascript","code_execution_risk",re.compile(r"\beval\s*\(",re.I),"HIGH"),
        ("javascript","ssrf",re.compile(r"\b(?:fetch|axios\.(?:get|post|request))\s*\([^)]*(?:url|uri|target|callback)",re.I),"MEDIUM"),
        ("php","command_injection",re.compile(r"\b(?:shell_exec|system|exec|passthru)\s*\(",re.I),"HIGH"),
        ("php","sql_injection",re.compile(r"\b(?:query|raw)\s*\([^)]*(?:request|input|\$_GET|\$_POST)",re.I),"HIGH"),
        ("java","command_injection",re.compile(r"\bRuntime\.getRuntime\(\)\.exec\s*\(",re.I),"HIGH"),
        ("java","xxe",re.compile(r"DocumentBuilderFactory\.newInstance\s*\(\s*\)",re.I),"MEDIUM"),
    )
    EXT={".py":"python",".js":"javascript",".jsx":"javascript",".ts":"javascript",".tsx":"javascript",".php":"php",".java":"java"}
    def scan(self,root,max_files=2000):
        rootp=_pathlib.Path(root); findings=[]; count=0
        for path in rootp.rglob("*"):
            if count>=max_files or not path.is_file(): continue
            if any(part in {".git",".venv","node_modules","vendor","dist","build"} for part in path.parts): continue
            lang=self.EXT.get(path.suffix.lower())
            if not lang: continue
            count+=1
            try: text=path.read_text(encoding="utf-8",errors="ignore")
            except Exception: continue
            for rlang,category,rx,sev in self.RULES:
                if rlang!=lang: continue
                for m in rx.finditer(text):
                    findings.append({"rule_id":category,"category":category,"severity":sev,"file":str(path.relative_to(rootp)),
                                     "line":text.count("\n",0,m.start())+1,
                                     "message":f"Review {category.replace('_',' ')}: {m.group(0)[:220]}"})
        return {"scanner":"ERSEC CodeSecurityEngine","root":str(rootp),"files_scanned":count,"findings":findings,
                "summary":{s:sum(1 for f in findings if f["severity"]==s) for s in ("CRITICAL","HIGH","MEDIUM","LOW","INFO")}}

def enrich_er_sec_findings(findings,config,stack):
    ctx=ContextAwareProbeEngine(); triage=EvidenceTriageEngine(config)
    for f in findings:
        f._ersec_verification=EvidenceVerifier.build(f)
        f._ersec_triage=triage.triage(f)
    return {"contextual_probe_plan":{f.parameter:ctx.rank(f.parameter,f.url) for f in findings if f.parameter},
            "verification_count":len(findings),
            "evidence_triage":[{"finding_id":f.finding_id,"category":f.category,**f._ersec_triage.to_dict()} for f in findings],
            "patch_bundle":DeveloperRemediationEngine(getattr(config,"patch_output_dir",None)).export(findings,stack),
            "enhanced_attack_paths":AppSecAttackPathFusion().fuse(findings)}

def attach_enhanced_report_data(results,config,crawler,stack,browser_data=None):
    artifacts=(browser_data or {}).get("artifacts",[]); urls=set(crawler.endpoints)
    urls.update(a.get("url","") for a in artifacts if a.get("url"))
    api=APIIntelligenceEngine().infer_from_urls(urls)
    params={f.get("parameter") for f in results.get("findings",[]) if f.get("parameter")}
    results["autonomous_discovery"]={"browser":browser_data or {"enabled":False,"available":False,"reason":"disabled"},
                                    "api_models":[m.to_dict() for m in api],"api_model_count":len(api)}
    results["protocol_intelligence"]=WebProtocolIntelligence().inspect(artifacts,urls,params)
    results["commerce_intelligence"]=CommerceIntelligenceEngine().inspect(urls,params)
    results["cloud_native_awareness"]=CloudNativeAwarenessEngine().inspect([(u,u) for u in sorted(urls)[:300]])
    finding_by_id={f.finding_id:f for f in getattr(config,"_ersec_last_findings",[])}
    ctx=ContextAwareProbeEngine()
    for fd in results.get("findings",[]):
        orig=finding_by_id.get(fd.get("finding_id"))
        if not orig: continue
        if getattr(orig,"_ersec_verification",None): fd["verification"]=orig._ersec_verification.to_dict()
        if getattr(orig,"_ersec_triage",None): fd["evidence_triage"]=orig._ersec_triage.to_dict()
        if orig.parameter: fd["context_relevance"]=ctx.rank(orig.parameter,orig.url)
    results["feature_flags"]={
        "browser_discovery":bool(getattr(config,"browser_discovery",False)),
        "api_intelligence":bool(getattr(config,"api_intelligence",True)),
        "context_aware_fuzzing":bool(getattr(config,"context_aware_fuzzing",True)),
        "evidence_triage":bool(getattr(config,"evidence_triage",True)),
        "cloud_native":bool(getattr(config,"cloud_native",True)),
        "commerce":bool(getattr(config,"commerce_intelligence",True)),
    }

def export_ecosystem_scaffold(directory):
    root=_pathlib.Path(directory)
    return {"plugin":PluginMarketplace().scaffold(str(root/"plugin")),
            "vscode":IDEIntegration().vscode(str(root/"vscode")),
            "intellij":IDEIntegration().intellij(str(root/"intellij"))}

def export_policy_bundle(findings,output_dir):
    return PolicyAsCodeExporter().export(findings,output_dir) if output_dir else None

def maybe_run_code_scan(args):
    if not getattr(args,"code_scan",None): return None
    report=CodeSecurityEngine().scan(args.code_scan)
    if getattr(args,"code_scan_output",None):
        _pathlib.Path(args.code_scan_output).write_text(json.dumps(report,indent=2),encoding="utf-8")
    print(json.dumps(report,indent=2))
    return 1 if any(f["severity"] in {"CRITICAL","HIGH"} for f in report["findings"]) else 0

def maybe_run_benchmark(args):
    if not getattr(args,"benchmark_report",None): return None
    if not getattr(args,"benchmark_truth",None): raise ERSECError("--benchmark-report requires --benchmark-truth")
    result=BenchmarkEngine().run_file(args.benchmark_report,args.benchmark_truth)
    try:
        with open(args.benchmark_report, encoding="utf-8") as fh: report=json.load(fh)
        with open(args.benchmark_truth, encoding="utf-8") as fh: truth=json.load(fh)
        result["quality_metrics"]=BenchmarkQualityReporter().run(report,truth)
    except Exception as exc:
        result["quality_metrics"]={"error":str(exc)}
    out=getattr(args,"benchmark_quality",None) or getattr(args,"benchmark_output",None)
    if out:
        _pathlib.Path(out).write_text(json.dumps(result,indent=2),encoding="utf-8")
    print(json.dumps(result,indent=2)); return 0

def maybe_run_scaffold(args):
    if not getattr(args,"scaffold_dir",None): return None
    print(json.dumps(export_ecosystem_scaffold(args.scaffold_dir),indent=2)); return 0


def _run_single_target(args, target_str: str) -> int:
    """Runs one full scan (validate -> configure -> scan -> report -> write
    outputs) for a single target string, returning a process-style exit code.
    Factored out of main() so --targets-file can call it once per line
    without duplicating all the wiring."""
    try:
        target_host = validate_target(target_str)
    except TargetValidationError as e:
        print(f"[!] {e}")
        return 2

    config = ScanConfig(
        profile=ScanProfile(args.profile),
        target=target_host,
        verify_tls=not args.insecure,
        verbose=args.verbose,
        ai_enabled=not args.no_ai,
        local_model_path=args.local_model,
        crown_jewels=args.crown_jewels,
        browser_discovery=getattr(args,"browser_discovery",False),
        api_intelligence=not getattr(args,"no_api_intelligence",False),
        context_aware_fuzzing=not getattr(args,"no_context_fuzz",False),
        evidence_triage=not getattr(args,"no_evidence_triage",False),
        triage_model_path=getattr(args,"triage_model",None),
        patch_output_dir=getattr(args,"patch_dir",None),
        policy_output_dir=getattr(args,"policy_dir",None),
        ide_output_dir=getattr(args,"ide_dir",None),
        cloud_native=not getattr(args,"no_cloud",False),
        commerce_intelligence=not getattr(args,"no_commerce",False),
        semantic_metamorphic=not getattr(args, "no_metamorphic", False),
        causal_impact=not getattr(args, "no_causal_impact", False),
        autonomous_agent=not getattr(args, "no_autonomous_agent", False),
        federated_mesh=not getattr(args, "no_federated_mesh", False),
        metamorphic_lam=not getattr(args, "no_metamorphic_lam", False),
        remediation_twin=not getattr(args, "no_remediation_twin", False),
        contract_drift_sentry=not getattr(args, "no_drift_sentry", False),
        federation_store_path=getattr(args, "federation_store", None),
        remediation_twin_dir=getattr(args, "remediation_twin_dir", None),
        drift_baseline_path=getattr(args, "drift_baseline", None),
        autonomous_max_actions=getattr(args, "autonomous_max_actions", 24),
    )
    config.authorization_manifest_path = getattr(args, "authorization_manifest", None)
    config.stateful_tests_enabled = bool(getattr(args, "stateful_tests", False))
    config.risk_budget = max(1, int(getattr(args, "risk_budget", 40)))
    config.contract_output_dir = getattr(args, "contract_dir", None)
    config.security_graph_path = getattr(args, "security_graph", None)
    config.benchmark_quality_path = getattr(args, "benchmark_quality", None)
    config.security_model_path = getattr(args, "security_model", None)
    config.behavior_verification_path = getattr(args, "behavior_verify_out", None)
    config.behavior_state_path = getattr(args, "behavior_state", None)
    config.behavior_graph_v3_path = getattr(args, "behavior_graph_v3", None)
    config.behavior_assurance_path = getattr(args, "behavior_assurance", None)
    config.authorization_matrix_path = getattr(args, "authorization_matrix", None)
    if config.authorization_manifest_path:
        try:
            config.authorization_manifest = AuthorizationManifest.load(config.authorization_manifest_path).to_dict()
        except (OSError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Failed to load authorization manifest: {exc}")
            return 2
    config.scope.adaptive_discovery = not args.no_adaptive_discovery
    config.scope.max_context_probes = args.max_context_probes
    config.scope.max_api_paths = args.max_api_paths
    config.scope.coverage_mode = args.coverage_mode
    config.coverage_mode = args.coverage_mode
    config.juice_shop_benchmark = bool(getattr(args, "juice_shop_benchmark", False)) or bool(getattr(args, "max_mode", False))
    if args.config:
        try:
            apply_config_file(config, load_scan_config_file(args.config))
        except (ERSECError, OSError, json.JSONDecodeError) as e:
            print(f"[!] Failed to load --config: {e}")
            return 2
    if args.scope_file:
        config.scope = load_scope_file(args.scope_file)
    elif not config.scope.allowed_hosts:
        config.scope.allowed_hosts = [target_host]
    if getattr(args,"port",None):
        config.scope.allowed_ports=sorted(set((config.scope.allowed_ports or [])+args.port))

    config.behavioral_twin = bool(getattr(args, "behavioral_twin", False)) or config.behavioral_twin
    config.security_genome = not getattr(args, "no_genome", False)
    config.temporal_reasoning = not getattr(args, "no_temporal", False)
    config.contract_drift = not getattr(args, "no_contract_drift", False)
    if getattr(args, "genome_memory", None):
        config.genome_memory_path = args.genome_memory
    if getattr(args, "no_invariants", False):
        config.invariant_learning = False
    if getattr(args, "no_counterfactuals", False):
        config.counterfactual_checks = False
    if getattr(args, "memory", None):
        config.continuous_memory_path = args.memory
    if getattr(args, "proof_dir", None):
        config.evidence_capsules = True

    if args.max_requests is not None:
        config.scope.max_requests = args.max_requests
    if args.max_pages is not None:
        config.scope.max_crawl_pages = args.max_pages

    if args.cookie:
        for pair in args.cookie.split(";"):
            if "=" in pair:
                k, v = pair.strip().split("=", 1)
                config.cookies[k] = v
    if args.bearer:
        config.bearer_token = args.bearer
    if getattr(args, "second_bearer", None):
        config.second_bearer_token = args.second_bearer
    config.workflow_replay = not getattr(args, "no_workflow_replay", False)
    config.max_identity_workflows = max(1, getattr(args, "max_identity_workflows", 20))
    config.max_identity_workflow_urls = max(1, getattr(args, "max_identity_urls", 120))
    for raw_identity in getattr(args, "identity", []) or []:
        if "=" not in raw_identity:
            print(f"[!] Invalid --identity value: {raw_identity!r}; expected NAME=TOKEN")
            return 2
        name, token = raw_identity.split("=", 1)
        name = name.strip()
        token = token.strip()
        if not name or not token:
            print(f"[!] Invalid --identity value: {raw_identity!r}; expected NAME=TOKEN")
            return 2
        config.identity_tokens[name] = token

    print_banner()
    print(f"[*] Target: {target_host}")
    print(f"[*] Profile: {config.profile.value}")
    print(f"[*] Built-in advisor engine: {'ON (local, offline)' if config.ai_enabled else 'OFF'}")
    print(f"[*] TLS verification: {'ON' if config.verify_tls else 'OFF (insecure)'}")
    print(f"[*] Request budget: {config.scope.max_requests}  |  Max pages: {config.scope.max_crawl_pages}  "
          f"(raise with --max-requests / --max-pages for larger sites)")
    if args.plugin_dir:
        print(f"[*] Plugin directory: {args.plugin_dir}")
    print()
    print("[!] Reminder: only scan systems you own or have explicit written authorization to test.\n")

    scanner = ERSECScanner(config, plugin_dir=args.plugin_dir)
    progress_stop = start_enter_progress_monitor(scanner.progress)
    print("[*] Live progress: press Enter at any time to show scan progress.")
    try:
        results = scanner.run(target_str)
    except KeyboardInterrupt:
        scanner.progress.mark_finished(True)
        print("\n[!] Scan interrupted by operator.")
        print(scanner.progress.format())
        return 130
    except ERSECError as e:
        scanner.progress.mark_finished(True)
        print(f"[!] Scan failed: {e}")
        print(scanner.progress.format())
        return 2
    finally:
        progress_stop.set()

    if results.get("scan_status") == "unreachable":
        print_console_report(results)
        print(f"\n[!] No report files written - there is nothing meaningful to save from an unreachable scan.")
        return 2

    findings = results["findings"]

    baseline_diff = None
    if args.baseline:
        baseline_diff = diff_against_baseline(findings, args.baseline)
        results["baseline_diff"] = baseline_diff
        if not args.no_update_baseline:
            save_baseline(results, args.baseline)

    exposure_changes = None
    recurrence_report = None
    if args.history_db:
        exposure_changes = record_and_diff_exposure_history(results, args.history_db)
        results["exposure_changes"] = exposure_changes
        record_scan_history(results, args.history_db)
        if args.recurrence_report:
            recurrence_report = build_recurrence_report(results["target"], args.history_db)

    print_console_report(results, baseline_diff, exposure_changes, recurrence_report)

    if args.output:
        with open(args.output, "w") as fh:
            json.dump(results, fh, indent=2, default=str)
        print(f"\n[+] JSON report saved: {args.output}")
    if args.html:
        with open(args.html, "w") as fh:
            fh.write(render_dashboard(results))
        print(f"[+] HTML dashboard saved: {args.html}")
    if args.sarif:
        with open(args.sarif, "w") as fh:
            json.dump(render_sarif(results), fh, indent=2, default=str)
        print(f"[+] SARIF report saved: {args.sarif}")
    if args.markdown:
        with open(args.markdown, "w") as fh:
            fh.write(render_markdown(results))
        print(f"[+] Markdown report saved: {args.markdown}")
    if args.csv:
        with open(args.csv, "w", newline="") as fh:
            fh.write(render_csv(results))
        print(f"[+] CSV report saved: {args.csv}")
    if args.junit:
        with open(args.junit, "w") as fh:
            fh.write(render_junit(results))
        print(f"[+] JUnit XML report saved: {args.junit}")

    if getattr(args, "proof_dir", None):
        try:
            os.makedirs(args.proof_dir, exist_ok=True)
            caps = results.get("behavioral_intelligence", {}).get("proof_capsules", [])
            for cap in caps:
                fn = os.path.join(args.proof_dir, cap.get("finding_id", "finding") + ".json")
                with open(fn, "w", encoding="utf-8") as fh:
                    json.dump(cap, fh, indent=2, sort_keys=True)
            print(f"[+] Proof capsules exported: {len(caps)} -> {args.proof_dir}")
        except OSError as e:
            print(f"[!] Failed to export proof capsules: {e}")

    if args.fail_on != "none":
        threshold = {"critical": 4, "high": 3, "medium": 2, "low": 1}[args.fail_on]
        if any(Severity[f["severity"]].value >= threshold for f in findings):
            print(f"\n[!] --fail-on={args.fail_on}: qualifying findings present, exiting 1")
            return 1

    return 0


def build_arg_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="ERSEC 14 - evidence-first application security reasoning platform")
    # Not required at the argparse level: --self-test and --verify-audit-log are
    # legitimate no-target invocations. main() enforces that a target is present
    # for every other code path, with a clearer error message than argparse's
    # generic "one of the arguments ... is required" would give here.
    target_group = parser.add_mutually_exclusive_group(required=False)
    target_group.add_argument("-t", "--target", help="Target domain, IP, or URL (must be in-scope / authorized)")
    target_group.add_argument("--targets-file", help="Path to a text file with one authorized target per line "
                               "(blank lines and lines starting with # are ignored). Scans are run sequentially "
                               "unless --max-workers > 1, and each target gets its own independent scope/request "
                               "budget - budgets are never shared or pooled across targets.")
    parser.add_argument("-o", "--output", help="Write JSON report to this path")
    parser.add_argument("--html", help="Write interactive HTML dashboard to this path")
    parser.add_argument("--sarif", help="Write SARIF 2.1.0 report to this path (for CI/CD code-scanning ingestion)")
    parser.add_argument("--markdown", "--md", dest="markdown", help="Write a Markdown summary report to this path (good for PR descriptions / tickets)")
    parser.add_argument("--csv", help="Write a flat CSV of findings to this path (for spreadsheet triage)")
    parser.add_argument("--junit", help="Write a JUnit XML report to this path (renders as a test report in most CI systems)")
    parser.add_argument("--baseline", help="Path to a saved baseline JSON snapshot; compares this run against it "
                                             "and reports new/resolved findings, then updates the snapshot")
    parser.add_argument("--no-update-baseline", action="store_true", help="Compare against --baseline but don't overwrite it")
    parser.add_argument("--history-db", help="Path to a local SQLite history database. Enables proof-carrying "
                         "exposure-change tracking (canonical IDs, first/last seen, new/resolved/reintroduced "
                         "status, a plain-English change narrative) and recurrence reporting across scans.")
    parser.add_argument("--recurrence-report", action="store_true",
                         help="With --history-db, print findings that have recurred across 2+ historical scans for this target.")
    parser.add_argument("--fail-on", choices=["critical", "high", "medium", "low", "none"], default="none",
                         help="Exit with code 1 if any finding at or above this severity is present (for CI gating)")
    parser.add_argument("--profile", choices=["passive", "baseline", "deep"], default="baseline")
    parser.add_argument("--config", help="Path to a JSON scan-configuration file (profile, cookies, headers, "
                         "crown jewels, scope). CLI flags always override values from this file when both are given.")
    parser.add_argument("--scope-file", help="JSON file overriding ScopeConfig defaults")
    parser.add_argument("--plugin-dir", help="Directory of custom .py detection-module plugins to load in "
                         "addition to the built-in modules (each file should define one or more BaseModule "
                         "subclasses at module level). Loaded only from this explicit local path - never "
                         "auto-discovered or fetched from a network location.")
    parser.add_argument("--max-requests", type=int, help="Override the request budget for this scan "
                         f"(default: {ScopeConfig().max_requests}). Raise this for larger multi-page sites; "
                         "the scan stops cleanly and reports what it found so far if the budget runs out.")
    parser.add_argument("--max-pages", type=int, help=f"Override the crawl page limit (default: {ScopeConfig().max_crawl_pages}).")
    parser.add_argument("--max-workers", type=int, default=1, help="With --targets-file, how many targets to scan "
                         "concurrently (default: 1, sequential). Each target still gets its own independent scope "
                         "and request budget regardless of this setting.")
    parser.add_argument("--cookie", help="Cookie header, e.g. 'session=abc; other=xyz'")
    parser.add_argument("--bearer", help="Bearer token for Authorization header")
    parser.add_argument("--bearer-2", dest="second_bearer", help="Second distinct authorized bearer token for cross-identity authorization testing")
    parser.add_argument("--identity", action="append", default=[], metavar="NAME=TOKEN", help="Explicit authorized identity for stateful workflow replay; repeatable, e.g. --identity user=TOKEN --identity admin=TOKEN --identity tenantA=TOKEN")
    parser.add_argument("--no-workflow-replay", action="store_true", dest="no_workflow_replay", help="Disable multi-identity workflow replay")
    parser.add_argument("--max-identity-workflows", type=int, default=20, help="Maximum observed workflows replayed across identities")
    parser.add_argument("--max-identity-urls", type=int, default=120, help="Maximum sensitive workflow URLs considered")
    parser.add_argument("--insecure", action="store_true", help="Disable TLS certificate verification")
    parser.add_argument("--no-ai", action="store_true", help="Disable the built-in advisor engine (use bare finding descriptions only)")
    parser.add_argument("--no-advisor", dest="no_ai", action="store_true", help=argparse.SUPPRESS)  # legacy alias
    parser.add_argument("--local-model", help="Optional: path to a local GGUF model for experimental "
                         "LLM-generated patch snippets via llama-cpp-python (requires that package installed "
                         "separately). Fully optional - the built-in knowledge-base advisor works without it.")
    parser.add_argument("--crown-jewel", action="append", default=[], dest="crown_jewels",
                         help="Path substring marking a high-value endpoint (e.g. /admin, /checkout, /account/pii). "
                              "Repeatable. Findings are re-scored by graph proximity to these.")
    parser.add_argument("--reverify", metavar="REPORT_JSON", help="Re-run the exact probes from a previous JSON "
                         "report (not a fresh scan) and print a still-present/resolved/inconclusive verdict per "
                         "finding. Requires the same target/auth to still be reachable.")
    parser.add_argument("--audit-log", help="With --reverify, append tamper-evident (hash-chained) verification "
                         "records to this JSON file instead of only printing them.")
    parser.add_argument("--verify-audit-log", metavar="AUDIT_LOG_JSON", help="Check the integrity of a "
                         "previously-written --audit-log file and exit (does not run a scan).")
    parser.add_argument("--browser", "--headless", dest="browser_discovery", action="store_true", help="Enable optional Playwright SPA/traffic discovery.")
    parser.add_argument("--no-api-intelligence", action="store_true", dest="no_api_intelligence")
    parser.add_argument("--no-context-fuzz", action="store_true", dest="no_context_fuzz")
    parser.add_argument("--no-evidence-triage", action="store_true", dest="no_evidence_triage")
    parser.add_argument("--triage-model", help="Optional local GGUF model for evidence triage.")
    parser.add_argument("--patch-dir", help="Write review-required framework remediation artifacts.")
    parser.add_argument("--policy-dir", help="Write Semgrep/OPA policy-as-code artifacts.")
    parser.add_argument("--ide-dir", help="Write VS Code + IntelliJ integration scaffolds.")
    parser.add_argument("--no-cloud", action="store_true", dest="no_cloud")
    parser.add_argument("--no-commerce", action="store_true", dest="no_commerce")
    parser.add_argument("--code-scan", help="Scan a local source tree with the ERSEC code engine.")
    parser.add_argument("--code-scan-output", help="Write code-scan JSON here.")
    parser.add_argument("--benchmark-report", help="ERSEC JSON report to benchmark.")
    parser.add_argument("--benchmark-truth", help="Operator-supplied benchmark oracle JSON.")
    parser.add_argument("--benchmark-output", help="Write benchmark metrics here.")
    parser.add_argument("--scaffold", dest="scaffold_dir", help="Generate plugin/IDE ecosystem scaffolds.")
    parser.add_argument("--port", type=int, action="append", help="Additional authorized target port; repeatable.")
    parser.add_argument("--behavioral-twin", action="store_true", help="Enable ERSEC 6 behavioral-twin reasoning over application journeys and invariants.")
    parser.add_argument("--no-invariants", action="store_true", help="Disable learned application invariants.")
    parser.add_argument("--no-counterfactuals", action="store_true", help="Disable bounded differential/counterfactual checks.")
    parser.add_argument("--memory", help="Path to local ERSEC behavioral memory JSON for cross-scan drift intelligence.")
    parser.add_argument("--proof-dir", help="Export tamper-evident proof capsules for findings.")
    parser.add_argument("--genome-memory", help="Local JSON memory for ERSEC 7 security behavior genome and contract drift.")
    parser.add_argument("--no-genome", action="store_true", help="Disable ERSEC 7 security behavior genome reasoning.")
    parser.add_argument("--no-temporal", action="store_true", help="Disable ERSEC 7 temporal reasoning.")
    parser.add_argument("--no-contract-drift", action="store_true", help="Disable API contract drift analysis.")
    parser.add_argument("--coverage-mode", choices=["maximum","balanced","conservative"], default="maximum", help="Adaptive detection coverage strategy.")
    parser.add_argument("--max-context-probes", type=int, default=120, help="Maximum semantic/context-aware probe candidates planned.")
    parser.add_argument("--max-api-paths", type=int, default=80, help="Maximum API contract routes retained from discovered specifications.")
    parser.add_argument("--no-adaptive-discovery", action="store_true", help="Disable adaptive discovery/probe planning.")
    parser.add_argument("--verbose", action="store_true")
    parser.add_argument("--self-test", action="store_true", help="Run the built-in offline self-test suite "
                         "(scope enforcement, target validation, canonical IDs, ERQ scoring, chain correlation, "
                         "attack-path fusion, audit-log tamper detection) and exit. No network access is used.")
    parser.add_argument("-m", "--max", dest="max_mode", action="store_true", help="Maximum authorized coverage: all mature ERSEC engines, optional browser discovery with static fallback, Juice Shop auto-detection, and full artifact generation.")
    parser.add_argument("--no-causal-impact", action="store_true", help="Disable causal business-impact reasoning.")
    parser.add_argument("--no-autonomous-agent", action="store_true", help="Disable bounded adaptive security-agent planning.")
    parser.add_argument("--no-federated-mesh", action="store_true", help="Disable privacy-preserving local knowledge aggregation.")
    parser.add_argument("--no-metamorphic-lam", action="store_true", help="Disable second-generation semantic metamorphic analysis.")
    parser.add_argument("--no-remediation-twin", action="store_true", help="Disable local remediation-twin planning.")
    parser.add_argument("--no-drift-sentry", action="store_true", help="Disable continuous contract/security drift comparison.")
    parser.add_argument("--federation-store", help="Local privacy-preserving ERSEC knowledge store path.")
    parser.add_argument("--remediation-twin-dir", help="Directory for disposable remediation-twin artifacts.")
    parser.add_argument("--drift-baseline", help="Previous ERSEC report used as drift baseline.")
    parser.add_argument("--autonomous-max-actions", type=int, default=24, help="Maximum bounded adaptive-agent follow-up actions.")
    parser.add_argument("--capabilities", action="store_true", help="Print ERSEC detector/intelligence capability manifest without scanning a target.")
    parser.add_argument("--juice-shop-benchmark", action="store_true", help="Collect the local Juice Shop /api/Challenges catalog and add a detector-coverage matrix. No challenge is marked solved from metadata alone.")
    parser.add_argument("--juice-shop-lab", dest="juice_shop_benchmark", action="store_true", help="Enable the controlled Juice Shop benchmark/source-evidence adapter.")
    parser.add_argument("--no-metamorphic", dest="no_metamorphic", action="store_true", help="Disable bounded semantic metamorphic testing.")
    parser.add_argument("--control-plane-memory", help="Persist ERSEC security-state snapshots for regression/drift analysis.")
    parser.add_argument("--security-slo", help="Write the compiled ERSEC Security SLO/policy JSON to this path.")
    parser.add_argument("--no-control-plane", action="store_true", help="Disable the ERSEC security control plane.")
    parser.add_argument("--hypothesis-budget", type=int, default=40, help="Maximum bounded security hypotheses generated by the control plane.")
    parser.add_argument("--shield", action="store_true", help="Run ERSEC Shield as a local defensive reverse-proxy/WAF instead of scanning.")
    parser.add_argument("--shield-upstream", help="Authorized upstream application URL for ERSEC Shield, e.g. http://127.0.0.1:8000")
    parser.add_argument("--shield-bind", default="127.0.0.1", help="Shield listen address (default: localhost).")
    parser.add_argument("--shield-port", type=int, default=8080, help="Shield listen port (default: 8080).")
    parser.add_argument("--shield-mode", choices=["monitor","block","learn"], default="block", help="Shield enforcement mode: observe, enforce, or learn route shapes.")
    parser.add_argument("--shield-policy", help="Load an ERSEC Shield JSON policy.")
    parser.add_argument("--shield-from-report", help="Compile a narrow Shield virtual-patch policy from a prior ERSEC JSON report.")
    parser.add_argument("--shield-compile", metavar="REPORT_JSON", help="Compile an ERSEC Shield JSON policy from a prior ERSEC report and exit without starting the proxy.")
    parser.add_argument("--shield-log", help="Write privacy-minimized JSONL Shield decision telemetry here.")
    parser.add_argument("--shield-learning", help="Persist learned route shapes here; request bodies are not stored.")
    parser.add_argument("--shield-max-body", type=int, default=2*1024*1024, help="Maximum request body size enforced by Shield.")
    parser.add_argument("--shield-max-url", type=int, default=8192, help="Maximum URL length enforced by Shield.")
    parser.add_argument("--shield-rate", type=int, default=120, help="Sustained requests/minute per source address.")
    parser.add_argument("--shield-burst", type=int, default=30, help="One-second burst limit per source address.")
    parser.add_argument("--shield-backend-timeout", type=float, default=15.0, help="Upstream timeout per Shield request.")
    parser.add_argument("--shield-tls-cert", help="Certificate PEM for terminating TLS at Shield.")
    parser.add_argument("--shield-tls-key", help="Private key PEM paired with --shield-tls-cert.")
    parser.add_argument("--authorization-manifest", dest="authorization_manifest", help="Explicit JSON authorization manifest; requests fail closed outside its host/path/method/window boundaries.")
    parser.add_argument("--stateful-tests", action="store_true", help="Opt in to ERSEC stateful-test families that are explicitly permitted by the authorization manifest.")
    parser.add_argument("--risk-budget", type=int, default=40, help="Risk-budget scheduler size for bounded security questions (default: 40).")
    parser.add_argument("--contract-dir", help="Export machine-readable security contracts and safe pytest/Postman/OpenAPI artifacts.")
    parser.add_argument("--security-graph", help="Persist the ERSEC Security Behavior Graph JSON.")
    parser.add_argument("--benchmark-quality", help="Write benchmark quality metrics JSON; use with --benchmark-report and --benchmark-truth.")
    parser.add_argument("--security-model", help="Declarative ERSEC Security Behavior Model JSON for explicit identity/tenant/resource/workflow authorization verification (read-only methods only).")
    parser.add_argument("--behavior-verify-out", help="Write read-only Security Behavior Model verification results JSON.")
    parser.add_argument("--behavior-state", help="Persist Security Behavior Graph v2 state and report structural boundary drift across runs.")
    parser.add_argument("--behavior-graph-v3", help="Write the operational Security Behavior Graph v3 with invariant results and counterexample paths.")
    parser.add_argument("--behavior-assurance", help="Write explicit security-invariant assurance results JSON.")
    parser.add_argument("--authorization-matrix", help="Write the read-only identity x resource x method authorization matrix.")
    parser.add_argument("--validate-security-model", metavar="MODEL_JSON", help="Validate a declarative ERSEC Security Behavior Model without contacting a target.")
    parser.add_argument("--contract-gate", metavar="CONTRACT_JSON", help="Evaluate explicitly active/enforced security contracts against --contract-report and exit 1 on regression.")
    parser.add_argument("--contract-report", metavar="REPORT_JSON", help="ERSEC report to evaluate with --contract-gate.")
    parser.add_argument("--contract-approve", metavar="DRAFT_CONTRACT_JSON", help="Promote selected draft contracts into an explicitly reviewed active bundle.")
    parser.add_argument("--contract-approved-out", metavar="ACTIVE_CONTRACT_JSON", help="Output path for --contract-approve; defaults to contracts-active.json.")
    parser.add_argument("--contract-id", action="append", default=[], help="Contract ID to approve; repeat for multiple contracts.")
    parser.add_argument("--contract-reviewer", help="Reviewer identity recorded when promoting contracts to active status.")
    parser.add_argument("--contract-expires", help="Optional ISO-8601 expiry time for approved contracts.")
    parser.add_argument("--sbom", metavar="SBOM_JSON", help="Generate a CycloneDX 1.7 JSON SBOM from the current Python environment and exit.")
    parser.add_argument("--assurance-pack", metavar="DIR", help="Create a content-addressed ERSEC assurance pack from supplied report/contracts/SBOM/graph artifacts.")
    parser.add_argument("--assurance-report", help="ERSEC report to include in --assurance-pack.")
    parser.add_argument("--assurance-contracts", help="Contracts bundle to include in --assurance-pack.")
    parser.add_argument("--assurance-sbom", help="SBOM to include in --assurance-pack.")
    parser.add_argument("--assurance-graph", help="Security Behavior Graph to include in --assurance-pack.")
    parser.add_argument("--verify-assurance-pack", metavar="DIR", help="Verify hashes in an ERSEC assurance pack and exit 1 on mismatch.")
    parser.add_argument("--benchmark-lab", metavar="DIR", help="Create a vendor-neutral authorized benchmark laboratory scaffold.")
    parser.add_argument("--security-category-manifest", action="store_true", help="Print the canonical security-category registry and exit.")
    parser.add_argument("--assurance-scorecard", metavar="REPORT_JSON", help="Build an OWASP API Security coverage/assurance scorecard from an ERSEC report and exit.")
    parser.add_argument("--assurance-scorecard-out", metavar="JSON", help="Write the assurance scorecard to this path.")
    parser.add_argument("--release-gate", metavar="REPORT_JSON", help="Evaluate deterministic release-security policy against an ERSEC report and exit 1 on policy failure.")
    parser.add_argument("--release-gate-contracts", help="Active security-contract bundle for --release-gate.")
    parser.add_argument("--release-gate-sbom", help="CycloneDX SBOM for --release-gate.")
    parser.add_argument("--release-gate-benchmark", help="Benchmark-quality JSON for --release-gate.")
    parser.add_argument("--release-gate-min-coverage", type=float, default=0.80, help="Minimum observed coverage ratio required by --release-gate.")
    parser.add_argument("--release-gate-max-high-critical", type=int, default=0, help="Maximum HIGH/CRITICAL findings permitted by --release-gate.")
    parser.add_argument("--release-gate-min-precision", type=float, help="Optional minimum benchmark precision for --release-gate.")
    parser.add_argument("--release-gate-min-recall", type=float, help="Optional minimum benchmark recall for --release-gate.")
    parser.add_argument("--version", action="version", version=f"ERSEC {ERSEC_VERSION}")
    return parser


def main() -> int:
    parser = build_arg_parser()
    args = parser.parse_args()

    if getattr(args, "validate_security_model", None):
        try:
            model=SecurityBehaviorModel.load(args.validate_security_model)
            print(json.dumps(model.validate(), indent=2, sort_keys=True))
            return 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Security model validation failed: {exc}")
            return 2

    if getattr(args, "verify_assurance_pack", None):
        try:
            result=AssurancePack.verify(args.verify_assurance_pack)
            print(json.dumps(result,indent=2))
            return 1 if result.get("status")=="fail" else 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Assurance pack verification failed: {exc}")
            return 2

    if getattr(args, "assurance_pack", None):
        try:
            result=AssurancePack.create(args.assurance_pack,args.assurance_report,args.assurance_contracts,args.assurance_sbom,args.assurance_graph)
            print(json.dumps(result,indent=2))
            return 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Assurance pack creation failed: {exc}")
            return 2

    if getattr(args, "benchmark_lab", None):
        try:
            result=BenchmarkLabScaffold.create(args.benchmark_lab)
            print(json.dumps(result,indent=2))
            return 0
        except OSError as exc:
            print(f"[!] Benchmark lab scaffold failed: {exc}")
            return 2

    if getattr(args, "sbom", None):
        try:
            result=generate_sbom(args.sbom)
            print(json.dumps(result,indent=2))
            return 0
        except (OSError, ValueError, ERSECError) as exc:
            print(f"[!] SBOM generation failed: {exc}")
            return 2

    if getattr(args, "contract_gate", None):
        if not getattr(args, "contract_report", None):
            parser.error("--contract-gate requires --contract-report")
        try:
            bundle=SecurityContractGate.load(args.contract_gate)
            with open(args.contract_report, "r", encoding="utf-8") as fh:
                report=json.load(fh)
            result=SecurityContractGate.evaluate(bundle, report)
            print(json.dumps(result,indent=2))
            return 1 if result.get("status")=="fail" else 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Contract gate failed to evaluate: {exc}")
            return 2

    if getattr(args, "contract_approve", None):
        try:
            out=args.contract_approved_out or "contracts-active.json"
            result=SecurityContractApprover.approve(args.contract_approve, out, args.contract_id, args.contract_reviewer or "", args.contract_expires or "")
            print(json.dumps(result,indent=2))
            return 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Contract approval failed: {exc}")
            return 2

    if getattr(args, "shield_compile", None):
        out_path = args.shield_policy or "ersec-shield-policy.json"
        try:
            compiled = ShieldPolicyCompiler().compile_report(args.shield_compile, out_path)
            print(f"[+] ERSEC Shield policy written: {out_path} ({len(compiled.get('rules', []))} rule(s))")
            return 0
        except (OSError, json.JSONDecodeError) as exc:
            print(f"[!] Shield policy compilation failed: {exc}")
            return 2

    if getattr(args, "shield", False):
        cfg=ScanConfig(profile=ScanProfile.BASELINE, verify_tls=not args.insecure, verbose=args.verbose)
        cfg.shield_enabled=True
        cfg.shield_upstream=args.shield_upstream or ""
        cfg.shield_bind=args.shield_bind
        cfg.shield_port=args.shield_port
        cfg.shield_mode=args.shield_mode
        cfg.shield_policy_path=args.shield_policy
        cfg.shield_from_report=args.shield_from_report
        cfg.shield_log_path=args.shield_log
        cfg.shield_learning_path=args.shield_learning
        cfg.shield_max_body_bytes=max(1024,int(args.shield_max_body))
        cfg.shield_max_url_length=max(256,int(args.shield_max_url))
        cfg.shield_rate_per_minute=max(1,int(args.shield_rate))
        cfg.shield_burst=max(1,int(args.shield_burst))
        cfg.shield_backend_timeout=max(0.5,float(args.shield_backend_timeout))
        cfg.shield_tls_cert=args.shield_tls_cert
        cfg.shield_tls_key=args.shield_tls_key
        if not cfg.shield_upstream:
            parser.error("--shield requires --shield-upstream")
        try:
            policy=load_shield_policy(cfg)
            server=ERSECShieldServer(cfg, policy)
            server.serve_forever()
            return 0
        except (ERSECError,OSError,json.JSONDecodeError) as exc:
            print(f"[!] Shield failed to start: {exc}")
            return 2

    if getattr(args, "assurance_scorecard", None):
        try:
            with open(args.assurance_scorecard, "r", encoding="utf-8") as fh: report=json.load(fh)
            contracts=None; benchmark=None
            # Scorecards accept optional sibling artifacts through explicit paths only.
            result=AssuranceScorecard.build(report, contracts, benchmark)
            if getattr(args, "assurance_scorecard_out", None):
                _pathlib.Path(args.assurance_scorecard_out).write_text(json.dumps(result,indent=2,sort_keys=True),encoding="utf-8")
            print(json.dumps(result,indent=2))
            return 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Assurance scorecard failed: {exc}")
            return 2

    if getattr(args, "release_gate", None):
        try:
            with open(args.release_gate, "r", encoding="utf-8") as fh: report=json.load(fh)
            def _load_json(path):
                return json.loads(_pathlib.Path(path).read_text(encoding="utf-8")) if path else None
            result=ReleaseAssuranceGate.evaluate(report, _load_json(args.release_gate_contracts), _load_json(args.release_gate_benchmark), _load_json(args.release_gate_sbom), float(args.release_gate_min_coverage), int(args.release_gate_max_high_critical), args.release_gate_min_precision, args.release_gate_min_recall)
            print(json.dumps(result,indent=2))
            return 1 if result.get("status")=="fail" else 0
        except (OSError, ValueError, json.JSONDecodeError, ERSECError) as exc:
            print(f"[!] Release assurance gate failed to evaluate: {exc}")
            return 2

    if getattr(args, "security_category_manifest", False):
        print(json.dumps({"version":ERSEC_VERSION,"schema":"ersec-security-category-registry/1","categories":SecurityCategoryRegistry.manifest()},indent=2,sort_keys=True))
        return 0

    if getattr(args, "capabilities", False):
        cfg=ScanConfig()
        probe=ERSECScanner(cfg)
        print(json.dumps({"version":ERSEC_VERSION,"detectors":probe.detector_registry.manifest(),
                          "engines":["security-behavior-model","read-only-authorization-verification","security-boundary-state-drift","adaptive-discovery","API-contract-mining","browser-traffic-discovery","behavioral-twin","security-behavior-genome","temporal-reasoning","evidence-triage","contextual-risk-graph","cloud-native","commerce","SAST","policy-as-code","proof-capsules","multi-identity-workflow-replay","workflow-risk-fusion","security-decision-lattice","response-anomaly-ensemble","parameter-shape-differentials","api-version-drift","security-control-plane","exposure-budget","hypothesis-engine","security-slo","cross-run-security-state","runtime-shield","virtual-patching","positive-security-envelope","rate-resource-guard","security-telemetry","route-learning","policy-enforcement-point","security-behavior-graph","security-behavior-graph-v3","security-invariant-assurance","counterexample-paths","authorization-matrix","security-contracts","proof-carrying-findings-v2","authorization-manifest","risk-budget-scheduler","benchmark-quality-metrics","shield-rule-governance","continuous-contract-gate","sbom-cyclonedx","release-integrity","assurance-pack","stable-contract-identities","contract-expiry-governance","benchmark-lab-scaffold","assurance-scorecard","release-assurance-gate","cyclonedx-1-7-sbom"],
                          "principle":"maximum evidence-backed coverage; no scanner can prove absence of every vulnerability"},indent=2))
        return 0

    if getattr(args, "max_mode", False):
        args.profile="deep"
        args.browser_discovery=True
        # One-command maximum mode: enable the full mature analysis stack.
        # Juice Shop mode is opportunistic and is a no-op for ordinary apps.
        args.juice_shop_benchmark=True
        args.max_requests=10000 if args.max_requests is None else args.max_requests
        args.max_pages=1000 if args.max_pages is None else args.max_pages
        args.max_context_probes=max(args.max_context_probes,500)
        args.max_api_paths=max(args.max_api_paths,500)
        args.coverage_mode="maximum"
        args.no_api_intelligence=False
        args.no_context_fuzz=False
        args.no_evidence_triage=False
        args.no_cloud=False
        args.no_commerce=False
        args.no_workflow_replay=False
        args.behavioral_twin=True
        args.no_invariants=False
        args.no_counterfactuals=False
        args.no_genome=False
        args.no_temporal=False
        args.no_contract_drift=False
        args.no_metamorphic=False
        args.no_causal_impact=False
        args.no_autonomous_agent=False
        args.no_federated_mesh=False
        args.no_metamorphic_lam=False
        args.no_remediation_twin=False
        args.no_drift_sentry=False
        args.autonomous_max_actions=max(getattr(args,"autonomous_max_actions",24),48)
        for _attr,_default in (("output","ersec-report.json"),("html","ersec-report.html"),
                               ("sarif","ersec-report.sarif"),("markdown","ersec-report.md"),
                               ("csv","ersec-report.csv"),("junit","ersec-report.xml"),
                               ("proof_dir",".ersec-proof"),("history_db",".ersec-history.sqlite"),
                               ("genome_memory",".ersec-genome.json"),("patch_dir",".ersec-patches"),
                               ("policy_dir",".ersec-policy"),("ide_dir",".ersec-ide"),
                               ("behavior_verification_path",".ersec-behavior-verification.json"),("behavior_state_path",".ersec-behavior-state.json")):
            if getattr(args,_attr,None) is None:
                setattr(args,_attr,_default)

    for handler in (maybe_run_code_scan, maybe_run_benchmark, maybe_run_scaffold):
        result = handler(args)
        if result is not None:
            return result

    if args.self_test:
        print("=== ERSEC self-test suite (offline, no network access) ===\n")
        ok = run_self_test()
        if ok:
            print("\n[+] All self-tests passed.")
            return 0
        else:
            print("\n[!] Self-test failures detected - see output above.")
            return 1

    # Every remaining code path needs a target, EXCEPT --verify-audit-log (checks a
    # local file only) and --reverify (can infer its target from the report itself,
    # handled below). Enforced here rather than via argparse's `required=True` so
    # --self-test/--verify-audit-log can be invoked with no -t/--targets-file at all.
    if not args.verify_audit_log and not args.reverify and not args.target and not args.targets_file:
        parser.error("one of the arguments -t/--target --targets-file is required "
                      "(unless using --self-test or --verify-audit-log)")

    if args.verify_audit_log:
        result = verify_audit_log_integrity(args.verify_audit_log)
        if result["valid"]:
            print(f"[+] Audit log OK: {result['records']} record(s), hash chain intact.")
            return 0
        else:
            print(f"[!] Audit log INVALID: {result['error']} (records: {result['records']}, broken at index {result['broken_at']})")
            return 1

    if args.reverify:
        config = ScanConfig(
            profile=ScanProfile(args.profile), verify_tls=not args.insecure, verbose=args.verbose,
            causal_impact=not getattr(args,"no_causal_impact",False),
            autonomous_agent=not getattr(args,"no_autonomous_agent",False),
            federated_mesh=not getattr(args,"no_federated_mesh",False),
            metamorphic_lam=not getattr(args,"no_metamorphic_lam",False),
            remediation_twin=not getattr(args,"no_remediation_twin",False),
            contract_drift_sentry=not getattr(args,"no_drift_sentry",False),
            federation_store_path=getattr(args,"federation_store",None),
            remediation_twin_dir=getattr(args,"remediation_twin_dir",None),
            drift_baseline_path=getattr(args,"drift_baseline",None),
        )
        try:
            target_host = validate_target(args.target) if args.target else None
        except TargetValidationError as e:
            print(f"[!] {e}")
            return 2
        if target_host:
            config.target = target_host
            config.scope.allowed_hosts = [target_host]
        else:
            # Infer scope from the report's own recorded findings/target rather than
            # requiring the user to re-specify it - the report already knows its own host.
            try:
                with open(args.reverify) as f:
                    report_preview = json.load(f)
                inferred_host = validate_target(report_preview.get("target", ""))
                config.target = inferred_host
                config.scope.allowed_hosts = [inferred_host]
            except (OSError, json.JSONDecodeError, TargetValidationError) as e:
                print(f"[!] Could not infer target from report for --reverify: {e}")
                return 2
        if args.cookie:
            for pair in args.cookie.split(";"):
                if "=" in pair:
                    k, v = pair.strip().split("=", 1)
                    config.cookies[k] = v
        if args.bearer:
            config.bearer_token = args.bearer
        try:
            records = reverify_report(args.reverify, config)
        except (OSError, json.JSONDecodeError) as e:
            print(f"[!] Failed to read report for --reverify: {e}")
            return 2
        still_present = sum(1 for r in records if r["verdict"] in ("still_present", "likely_still_present"))
        resolved = sum(1 for r in records if r["verdict"] == "resolved")
        inconclusive = sum(1 for r in records if r["verdict"] in ("inconclusive", "changed_uncertain"))
        print(f"\n=== Re-verification of {len(records)} finding(s) from {args.reverify} ===")
        for r in records:
            print(f"  [{r['verdict']}] {r['category']} - {r['url']}" + (f" ({r['note']})" if r.get("note") else ""))
        print(f"\nSummary: {still_present} still present, {resolved} resolved, {inconclusive} inconclusive/uncertain.")
        if args.audit_log:
            append_to_audit_log(records, args.audit_log)
            print(f"[+] Appended {len(records)} record(s) to tamper-evident audit log: {args.audit_log}")
        return 1 if still_present else 0

    if args.targets_file:
        try:
            with open(args.targets_file) as f:
                targets = [line.strip() for line in f if line.strip() and not line.strip().startswith("#")]
        except OSError as e:
            print(f"[!] Failed to read --targets-file: {e}")
            return 2
        if not targets:
            print("[!] --targets-file contained no targets")
            return 2
        print(f"[*] Loaded {len(targets)} target(s) from {args.targets_file}")
        print("[!] Reminder: only scan systems you own or have explicit written authorization to test - "
              "this applies to every target in this file individually.\n")
        # Build one per-target argparse.Namespace up front, each with its own
        # namespaced output paths (report.json -> report.<host>.json) so a
        # multi-target run never overwrites one target's report with the next's -
        # this list is shared by both the sequential and concurrent execution
        # paths below, so output naming behaves identically either way.
        per_target_args_list = []
        for t in targets:
            per_target_args = argparse.Namespace(**vars(args))
            per_target_args.target = t
            try:
                host_slug = re.sub(r"[^A-Za-z0-9_.-]", "_", validate_target(t))
            except TargetValidationError:
                host_slug = re.sub(r"[^A-Za-z0-9_.-]", "_", t)
            for attr in ("output", "html", "sarif", "markdown", "csv", "junit"):
                val = getattr(args, attr)
                if val:
                    base, ext = os.path.splitext(val)
                    setattr(per_target_args, attr, f"{base}.{host_slug}{ext}")
            per_target_args_list.append((t, per_target_args))

        worst_exit = 0
        if args.max_workers > 1:
            # Genuinely concurrent path: each target gets its own thread, its own
            # SafeHttpClient, and its own request budget (never shared/pooled - see
            # run_multi_target_scan's docstring). Console output from concurrent
            # targets is naturally interleaved line-by-line since each print() call
            # is atomic, but every line is still prefixed with its target so the
            # interleaving stays readable rather than ambiguous.
            print(f"[*] Running {len(targets)} target(s) concurrently, max {args.max_workers} at a time.\n")
            import io as _io
            import contextlib as _contextlib

            def _run_captured(t: str, pta: argparse.Namespace) -> Tuple[str, int, str]:
                buf = _io.StringIO()
                with _contextlib.redirect_stdout(buf):
                    rc = _run_single_target(pta, t)
                return t, rc, buf.getvalue()

            with ThreadPoolExecutor(max_workers=args.max_workers) as executor:
                futures = [executor.submit(_run_captured, t, pta) for t, pta in per_target_args_list]
                for fut in as_completed(futures):
                    t, rc, output = fut.result()
                    prefixed = "\n".join(f"[{t}] {line}" for line in output.splitlines())
                    print(prefixed)
                    print(f"[{t}] === exit code {rc} ===\n")
                    worst_exit = max(worst_exit, rc)
        else:
            for t, per_target_args in per_target_args_list:
                rc = _run_single_target(per_target_args, t)
                worst_exit = max(worst_exit, rc)
                print("\n" + "=" * 78 + "\n")
        return worst_exit

    return _run_single_target(args, args.target)


if __name__ == "__main__":
    sys.exit(main())
