# Fichier: python_cheats/cheatsheets/prometheus.txt
# Cheatsheet Prometheus & Grafana - Guide Complet Python


[OK] INTRODUCTION PROMETHEUS

# Prometheus est un système de monitoring et alerting open-source
# Conçu pour la fiabilité et l'observabilité des systèmes
# Modèle de données: time series avec métriques et labels
# Pull-based: Prometheus scrape les endpoints HTTP des applications

# Architecture:
# Application -> Prometheus Client -> Endpoint /metrics -> Prometheus Server -> Grafana

# Types de métriques:
# - Counter: Compteur qui ne fait qu'augmenter (requêtes, erreurs)
# - Gauge: Valeur qui peut monter/descendre (température, mémoire)
# - Histogram: Distribution de valeurs (latences, tailles)
# - Summary: Quantiles de valeurs (similaire à histogram)


[OK] INSTALLATION

# Installer prometheus-client
pip install prometheus-client

# Installer avec extras
pip install prometheus-client[twisted]

# Pour Grafana (optionnel)
pip install grafana-api

# Dépendances recommandées
pip install prometheus-client flask gunicorn
pip install prometheus-client django
pip install prometheus-client fastapi uvicorn


[OK] PROMETHEUS CLIENT - BASICS

from prometheus_client import Counter, Gauge, Histogram, Summary
from prometheus_client import start_http_server, generate_latest
from prometheus_client import CollectorRegistry, push_to_gateway
from prometheus_client import multiprocess, REGISTRY
import time

# === COUNTER (Compteur monotone) ===

# Créer un counter simple
requests_total = Counter('http_requests_total', 'Total HTTP requests')

# Counter avec labels
requests_by_method = Counter(
    'http_requests_by_method_total',
    'HTTP requests by method',
    ['method', 'endpoint']
)

# Counter avec namespace et subsystem
app_requests = Counter(
    'requests_total',
    'Total requests',
    namespace='myapp',
    subsystem='api',
    labelnames=['method', 'status']
)
# Métrique exposée: myapp_api_requests_total

# Utilisation
requests_total.inc()                           # Incrémenter de 1
requests_total.inc(5)                          # Incrémenter de 5
requests_by_method.labels(method='GET', endpoint='/api').inc()
requests_by_method.labels('POST', '/users').inc(2)

# Obtenir valeur actuelle (debug uniquement)
print(requests_total._value.get())

# === GAUGE (Valeur variable) ===

# Créer un gauge
active_connections = Gauge('active_connections', 'Active connections')
temperature = Gauge('temperature_celsius', 'Temperature in Celsius')

# Gauge avec labels
memory_usage = Gauge(
    'memory_usage_bytes',
    'Memory usage in bytes',
    ['process', 'type']
)

# Utilisation
active_connections.set(42)                     # Définir valeur
active_connections.inc()                       # Incrémenter de 1
active_connections.inc(10)                     # Incrémenter de 10
active_connections.dec()                       # Décrémenter de 1
active_connections.dec(5)                      # Décrémenter de 5

# Set avec labels
memory_usage.labels(process='web', type='rss').set(1024000)

# Gauge comme décorateur pour mesurer durée
processing_time = Gauge('task_processing_seconds', 'Time spent processing')

@processing_time.time()
def process_task():
    time.sleep(2)
    return "done"

# Gauge avec contexte manager
with processing_time.time():
    time.sleep(1)
    # Traitement...

# Gauge pour fonctions (callback)
def get_cpu_usage():
    import psutil
    return psutil.cpu_percent()

cpu_gauge = Gauge('cpu_usage_percent', 'CPU usage', get_cpu_usage)

# Gauge pour timestamp
last_update = Gauge('last_update_timestamp', 'Last update time')
last_update.set_to_current_time()

# === HISTOGRAM (Distribution de valeurs) ===

# Créer histogram avec buckets par défaut
request_duration = Histogram(
    'http_request_duration_seconds',
    'HTTP request duration in seconds'
)

# Histogram avec buckets personnalisés
response_size = Histogram(
    'http_response_size_bytes',
    'HTTP response size in bytes',
    buckets=[100, 500, 1000, 5000, 10000, 50000, 100000]
)

# Histogram avec labels
api_latency = Histogram(
    'api_latency_seconds',
    'API latency',
    ['method', 'endpoint'],
    buckets=[0.001, 0.01, 0.1, 0.5, 1.0, 5.0, 10.0]
)

# Utilisation - Observer valeur
request_duration.observe(0.5)                  # Observer 500ms
api_latency.labels('GET', '/users').observe(0.23)

# Histogram comme décorateur
@request_duration.time()
def handle_request():
    time.sleep(0.1)
    return "OK"

# Histogram avec contexte manager
with request_duration.time():
    # Traitement...
    time.sleep(0.2)

# Métriques générées par histogram:
# http_request_duration_seconds_bucket{le="0.005"} 0
# http_request_duration_seconds_bucket{le="0.01"} 0
# http_request_duration_seconds_bucket{le="+Inf"} 3
# http_request_duration_seconds_sum 1.5
# http_request_duration_seconds_count 3

# === SUMMARY (Quantiles) ===

# Créer summary
request_latency = Summary(
    'request_latency_seconds',
    'Request latency in seconds'
)

# Summary avec quantiles personnalisés
latency_summary = Summary(
    'latency_seconds',
    'Latency',
    ['service'],
    # Quantiles: 50th, 90th, 99th percentile
    quantiles=[0.5, 0.9, 0.99]
)

# Utilisation
request_latency.observe(0.3)
latency_summary.labels(service='api').observe(0.15)

# Summary comme décorateur
@request_latency.time()
def process():
    time.sleep(0.1)

# Métriques générées par summary:
# request_latency_seconds{quantile="0.5"} 0.15
# request_latency_seconds{quantile="0.9"} 0.3
# request_latency_seconds{quantile="0.99"} 0.5
# request_latency_seconds_sum 1.5
# request_latency_seconds_count 10


[OK] EXPOSER LES MÉTRIQUES

# === Serveur HTTP Simple ===

from prometheus_client import start_http_server, Counter
import time

request_count = Counter('app_requests_total', 'Total requests')

# Démarrer serveur sur port 8000
start_http_server(8000)
print("Prometheus metrics available at http://localhost:8000/metrics")

# Application continue de tourner
while True:
    request_count.inc()
    time.sleep(1)

# === Avec Flask ===

from flask import Flask
from prometheus_client import make_wsgi_app, Counter, Histogram
from werkzeug.middleware.dispatcher import DispatcherMiddleware

app = Flask(__name__)

# Métriques
REQUEST_COUNT = Counter(
    'flask_requests_total',
    'Total Flask requests',
    ['method', 'endpoint', 'status']
)

REQUEST_LATENCY = Histogram(
    'flask_request_duration_seconds',
    'Flask request duration',
    ['method', 'endpoint']
)

@app.before_request
def before_request():
    from flask import request, g
    g.start_time = time.time()

@app.after_request
def after_request(response):
    from flask import request, g
    
    # Enregistrer métriques
    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.path,
        status=response.status_code
    ).inc()
    
    if hasattr(g, 'start_time'):
        duration = time.time() - g.start_time
        REQUEST_LATENCY.labels(
            method=request.method,
            endpoint=request.path
        ).observe(duration)
    
    return response

@app.route('/')
def index():
    return "Hello World"

@app.route('/api/users')
def users():
    time.sleep(0.1)  # Simuler traitement
    return {"users": []}

# Ajouter endpoint /metrics
app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {
    '/metrics': make_wsgi_app()
})

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)

# === Avec FastAPI ===

from fastapi import FastAPI, Request
from prometheus_client import make_asgi_app, Counter, Histogram
import time

app = FastAPI()

# Métriques
REQUEST_COUNT = Counter(
    'fastapi_requests_total',
    'Total requests',
    ['method', 'endpoint', 'status']
)

REQUEST_DURATION = Histogram(
    'fastapi_request_duration_seconds',
    'Request duration',
    ['method', 'endpoint']
)

@app.middleware("http")
async def prometheus_middleware(request: Request, call_next):
    start_time = time.time()
    
    response = await call_next(request)
    
    duration = time.time() - start_time
    
    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.url.path,
        status=response.status_code
    ).inc()
    
    REQUEST_DURATION.labels(
        method=request.method,
        endpoint=request.url.path
    ).observe(duration)
    
    return response

@app.get("/")
def read_root():
    return {"message": "Hello World"}

@app.get("/users")
def read_users():
    time.sleep(0.1)
    return {"users": []}

# Monter endpoint metrics
metrics_app = make_asgi_app()
app.mount("/metrics", metrics_app)

# Lancer: uvicorn main:app --host 0.0.0.0 --port 8000

# === Avec Django ===

# settings.py
INSTALLED_APPS = [
    # ...
    'django_prometheus',
]

MIDDLEWARE = [
    'django_prometheus.middleware.PrometheusBeforeMiddleware',
    # ... autres middlewares ...
    'django_prometheus.middleware.PrometheusAfterMiddleware',
]

# urls.py
from django.urls import path, include

urlpatterns = [
    # ...
    path('metrics/', include('django_prometheus.urls')),
]

# Ou manuel:
from django.http import HttpResponse
from prometheus_client import generate_latest, REGISTRY

def metrics(request):
    return HttpResponse(
        generate_latest(REGISTRY),
        content_type='text/plain; charset=utf-8'
    )

# === Endpoint Manuel ===

from prometheus_client import generate_latest, CONTENT_TYPE_LATEST

def metrics_endpoint():
    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}

# Avec Flask
@app.route('/metrics')
def metrics():
    return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}


[OK] LABELS & CARDINALITÉ

# === Bonnes pratiques labels ===

# [OK] BON: Labels avec cardinalité limitée
http_requests = Counter(
    'http_requests_total',
    'Total requests',
    ['method', 'endpoint', 'status']  # Peu de valeurs possibles
)

# [X] MAUVAIS: Labels avec cardinalité illimitée
bad_counter = Counter(
    'requests_total',
    'Requests',
    ['user_id', 'session_id']  # Millions de valeurs possibles!
)

# === Labels dynamiques ===

# Définir labels à l'avance
request_counter = Counter(
    'app_requests',
    'Requests',
    ['method', 'path', 'status']
)

# Utiliser avec labels
request_counter.labels(method='GET', path='/api', status='200').inc()
request_counter.labels('POST', '/users', '201').inc()

# === Child metrics (pour réutilisation) ===

get_requests = request_counter.labels(method='GET', path='/api', status='200')
get_requests.inc()
get_requests.inc()

# === Labels par défaut ===

from prometheus_client import Counter

# Pas de labels par défaut natifs, mais pattern:
def create_labeled_counter(default_labels):
    base_counter = Counter('my_counter', 'Description', list(default_labels.keys()))
    
    def inc_with_defaults(**kwargs):
        labels = {**default_labels, **kwargs}
        base_counter.labels(**labels).inc()
    
    return inc_with_defaults

counter = create_labeled_counter({'app': 'myapp', 'env': 'prod'})
counter(method='GET')  # Ajoute app et env automatiquement

# === Validation labels ===

# Les labels ne peuvent pas:
# - Commencer par __
# - Contenir autre chose que [a-zA-Z0-9_]

# Valide:
valid = Counter('metric', 'desc', ['method', 'endpoint_name', 'status_code'])

# Invalide:
# invalid = Counter('metric', 'desc', ['__reserved', 'invalid-label'])


[OK] REGISTRY & COLLECTORS

# === Registry par défaut ===

from prometheus_client import REGISTRY, Counter

# Par défaut, toutes les métriques utilisent REGISTRY
counter = Counter('my_counter', 'Description')

# Lister toutes les métriques
for collector in REGISTRY._collector_to_names.keys():
    print(collector)

# === Registry personnalisé ===

from prometheus_client import CollectorRegistry, Counter, generate_latest

# Créer registry isolé
custom_registry = CollectorRegistry()

# Créer métrique avec ce registry
counter = Counter(
    'custom_counter',
    'Description',
    registry=custom_registry
)

# Générer métriques seulement de ce registry
metrics_output = generate_latest(custom_registry)

# === Désactiver métriques par défaut ===

from prometheus_client import CollectorRegistry, PLATFORM_COLLECTOR, PROCESS_COLLECTOR, GC_COLLECTOR

# Registry sans métriques par défaut
registry = CollectorRegistry(auto_describe=False)

# Ou désactiver collectors spécifiques
registry = CollectorRegistry()
registry.unregister(PLATFORM_COLLECTOR)
registry.unregister(PROCESS_COLLECTOR)
registry.unregister(GC_COLLECTOR)

# === Custom Collector ===

from prometheus_client.core import GaugeMetricFamily
from prometheus_client import CollectorRegistry

class CustomCollector:
    def collect(self):
        # Créer métrique dynamiquement
        gauge = GaugeMetricFamily(
            'custom_metric',
            'Description',
            labels=['label1', 'label2']
        )
        
        # Ajouter valeurs
        gauge.add_metric(['value1', 'value2'], 42)
        gauge.add_metric(['value1', 'value3'], 100)
        
        yield gauge

# Enregistrer collector
registry = CollectorRegistry()
registry.register(CustomCollector())

# === Collector pour metrics externes ===

class DatabaseStatsCollector:
    def __init__(self, db_connection):
        self.db = db_connection
    
    def collect(self):
        # Gauge pour connexions actives
        connections = GaugeMetricFamily(
            'db_active_connections',
            'Active database connections'
        )
        
        # Requête DB
        count = self.db.execute("SELECT COUNT(*) FROM pg_stat_activity").fetchone()[0]
        connections.add_metric([], count)
        
        yield connections
        
        # Gauge pour taille DB
        db_size = GaugeMetricFamily(
            'db_size_bytes',
            'Database size in bytes',
            labels=['database']
        )
        
        for row in self.db.execute("SELECT datname, pg_database_size(datname) FROM pg_database"):
            db_size.add_metric([row[0]], row[1])
        
        yield db_size

# Utilisation
# REGISTRY.register(DatabaseStatsCollector(db_conn))


[OK] MÉTRIQUES AVANCÉES

# === Info Metric ===

from prometheus_client import Info

# Info pour métadonnées statiques
app_info = Info('app', 'Application info')
app_info.info({
    'version': '1.2.3',
    'environment': 'production',
    'build_date': '2024-01-15'
})

# === Enum Metric ===

from prometheus_client import Enum

# État avec valeurs prédéfinies
app_state = Enum(
    'app_state',
    'Application state',
    states=['starting', 'running', 'stopping', 'stopped']
)

app_state.state('running')

# === Multiprocess Mode (Gunicorn/uWSGI) ===

# 1. Configurer variable d'environnement
# export PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_multiproc

# 2. Créer dossier
import os
os.makedirs('/tmp/prometheus_multiproc', exist_ok=True)

# 3. Utiliser registry multiprocess
from prometheus_client import CollectorRegistry, generate_latest, Counter
from prometheus_client import multiprocess, values

# Créer métriques
counter = Counter('worker_requests', 'Requests per worker')

# Endpoint metrics (dans app)
def metrics():
    registry = CollectorRegistry()
    multiprocess.MultiProcessCollector(registry)
    return generate_latest(registry)

# 4. Nettoyer à la fermeture du worker
from prometheus_client import multiprocess

def child_exit(server, worker):
    multiprocess.mark_process_dead(worker.pid)

# gunicorn.conf.py
from prometheus_client import multiprocess

def child_exit(server, worker):
    multiprocess.mark_process_dead(worker.pid)

# === Timestamp personnalisé ===

from prometheus_client import Counter
import time

counter = Counter('my_counter', 'Description')

# Normalement:
counter.inc()

# Avec timestamp (pas supporté directement, utiliser Pushgateway)

# === Exemplars (OpenMetrics) ===

from prometheus_client import Histogram
from prometheus_client.openmetrics.exposition import generate_latest

histogram = Histogram('request_duration', 'Duration', registry=registry)

# Observer avec exemplar
histogram.observe(0.5, exemplar={'trace_id': 'abc123'})

# Générer avec format OpenMetrics
output = generate_latest(registry)


[OK] PUSH GATEWAY

# Push Gateway pour jobs batch ou short-lived

from prometheus_client import CollectorRegistry, Counter, Gauge, push_to_gateway
from prometheus_client import pushadd_to_gateway, delete_from_gateway

registry = CollectorRegistry()

# Créer métriques
duration = Gauge('job_duration_seconds', 'Job duration', registry=registry)
processed = Counter('records_processed_total', 'Records processed', registry=registry)

# Job batch
def run_batch_job():
    start = time.time()
    
    for i in range(1000):
        # Traiter...
        processed.inc()
    
    duration.set(time.time() - start)
    
    # Push vers gateway
    push_to_gateway(
        'localhost:9091',
        job='batch_job',
        registry=registry
    )

# === Push avec labels ===

push_to_gateway(
    'localhost:9091',
    job='batch_job',
    registry=registry,
    grouping_key={'instance': 'server1', 'environment': 'prod'}
)

# === Pushadd (ajoute sans écraser) ===

pushadd_to_gateway(
    'localhost:9091',
    job='batch_job',
    registry=registry
)

# === Delete métriques ===

delete_from_gateway(
    'localhost:9091',
    job='batch_job',
    grouping_key={'instance': 'server1'}
)

# === Avec authentification ===

from prometheus_client import push_to_gateway
import requests

def custom_handler(url, method, timeout, headers, data):
    return requests.request(
        method=method,
        url=url,
        data=data,
        headers=headers,
        timeout=timeout,
        auth=('username', 'password')
    )

push_to_gateway(
    'localhost:9091',
    job='job',
    registry=registry,
    handler=custom_handler
)

# === Contexte manager ===

from prometheus_client import push_to_gateway, CollectorRegistry, Gauge
import time

registry = CollectorRegistry()
duration = Gauge('job_duration_seconds', 'Duration', registry=registry)

with duration.time():
    # Job...
    time.sleep(2)

push_to_gateway('localhost:9091', job='myjob', registry=registry)


[OK] MÉTRIQUES SYSTÈME

# === Métriques processus (automatiques) ===

from prometheus_client import REGISTRY, ProcessCollector, PlatformCollector, GCCollector

# Activées par défaut dans REGISTRY:
# - process_cpu_seconds_total
# - process_resident_memory_bytes
# - process_virtual_memory_bytes
# - process_open_fds
# - process_max_fds
# - process_start_time_seconds

# Désactiver si besoin:
from prometheus_client import PROCESS_COLLECTOR, PLATFORM_COLLECTOR, GC_COLLECTOR
REGISTRY.unregister(PROCESS_COLLECTOR)
REGISTRY.unregister(PLATFORM_COLLECTOR)
REGISTRY.unregister(GC_COLLECTOR)

# === Métriques custom système ===

import psutil
from prometheus_client import Gauge

# CPU
cpu_percent = Gauge('system_cpu_percent', 'CPU usage percent')
cpu_count = Gauge('system_cpu_count', 'CPU count')

def update_cpu_metrics():
    cpu_percent.set(psutil.cpu_percent(interval=1))
    cpu_count.set(psutil.cpu_count())

# Mémoire
memory_usage = Gauge('system_memory_bytes', 'Memory usage', ['type'])

def update_memory_metrics():
    mem = psutil.virtual_memory()
    memory_usage.labels(type='total').set(mem.total)
    memory_usage.labels(type='available').set(mem.available)
    memory_usage.labels(type='used').set(mem.used)
    memory_usage.labels(type='free').set(mem.free)

# Disque
disk_usage = Gauge('system_disk_bytes', 'Disk usage', ['path', 'type'])

def update_disk_metrics():
    for partition in psutil.disk_partitions():
        usage = psutil.disk_usage(partition.mountpoint)
        disk_usage.labels(path=partition.mountpoint, type='total').set(usage.total)
        disk_usage.labels(path=partition.mountpoint, type='used').set(usage.used)
        disk_usage.labels(path=partition.mountpoint, type='free').set(usage.free)

# Réseau
network_bytes = Counter('system_network_bytes_total', 'Network bytes', ['interface', 'direction'])

def update_network_metrics():
    net = psutil.net_io_counters(pernic=True)
    for interface, stats in net.items():
        network_bytes.labels(interface=interface, direction='sent').inc(stats.bytes_sent)
        network_bytes.labels(interface=interface, direction='received').inc(stats.bytes_recv)

# === Collector automatique ===

from prometheus_client.core import GaugeMetricFamily

class SystemCollector:
    def collect(self):
        # CPU
        cpu = GaugeMetricFamily('system_cpu_percent', 'CPU usage')
        cpu.add_metric([], psutil.cpu_percent(interval=1))
        yield cpu
        
        # Mémoire
        mem = GaugeMetricFamily('system_memory_bytes', 'Memory', labels=['type'])
        memory = psutil.virtual_memory()
        mem.add_metric(['total'], memory.total)
        mem.add_metric(['available'], memory.available)
        mem.add_metric(['used'], memory.used)
        yield mem

REGISTRY.register(SystemCollector())


[OK] MÉTRIQUES APPLICATIVES

# === Web Application ===

from prometheus_client import Counter, Histogram, Gauge
from functools import wraps
import time

# Requêtes HTTP
http_requests_total = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status']
)

http_request_duration_seconds = Histogram(
    'http_request_duration_seconds',
    'HTTP request duration',
    ['method', 'endpoint'],
    buckets=[0.01, 0.05, 0.1, 0.5, 1.0, 5.0]
)

http_requests_in_progress = Gauge(
    'http_requests_in_progress',
    'HTTP requests in progress'
)

# Décorateur pour mesurer requêtes
def track_request(method, endpoint):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            http_requests_in_progress.inc()
            start = time.time()
            
            try:
                result = f(*args, **kwargs)
                status = getattr(result, 'status_code', 200)
                return result
            except Exception as e:
                status = 500
                raise
            finally:
                duration = time.time() - start
                
                http_requests_total.labels(
                    method=method,
                    endpoint=endpoint,
                    status=status
                ).inc()
                
                http_request_duration_seconds.labels(
                    method=method,
                    endpoint=endpoint
                ).observe(duration)
                
                http_requests_in_progress.dec()
        
        return wrapper
    return decorator

# Utilisation
@track_request('GET', '/api/users')
def get_users():
    return {"users": []}

# === Database ===

# Connexions DB
db_connections = Gauge(
    'db_connections',
    'Database connections',
    ['state']
)

# Requêtes DB
db_queries_total = Counter(
    'db_queries_total',
    'Total database queries',
    ['operation', 'table']
)

db_query_duration_seconds = Histogram(
    'db_query_duration_seconds',
    'Database query duration',
    ['operation', 'table'],
    buckets=[0.001, 0.01, 0.1, 0.5, 1.0]
)

# Wrapper pour requêtes
def track_query(operation, table):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            start = time.time()
            
            try:
                return f(*args, **kwargs)
            finally:
                duration = time.time() - start
                
                db_queries_total.labels(
                    operation=operation,
                    table=table
                ).inc()
                
                db_query_duration_seconds.labels(
                    operation=operation,
                    table=table
                ).observe(duration)
        
        return wrapper
    return decorator

@track_query('SELECT', 'users')
def get_user(user_id):
    # Query DB...
    pass

# === Cache ===

cache_hits_total = Counter('cache_hits_total', 'Cache hits')
cache_misses_total = Counter('cache_misses_total', 'Cache misses')
cache_size = Gauge('cache_size_bytes', 'Cache size')

# === Queue ===

queue_size = Gauge('queue_size', 'Queue size', ['queue'])
queue_processing_duration = Histogram(
    'queue_processing_duration_seconds',
    'Task processing duration',
    ['queue', 'task_type']
)

# === Business Metrics ===

# E-commerce
orders_total = Counter('orders_total', 'Total orders', ['status'])
revenue_total = Counter('revenue_cents_total', 'Total revenue in cents')
active_users = Gauge('active_users', 'Active users')
cart_value = Histogram(
    'cart_value_cents',
    'Cart value',
    buckets=[1000, 5000, 10000, 50000, 100000]
)

# Événements métier
user_registrations = Counter('user_registrations_total', 'User registrations')
login_attempts = Counter('login_attempts_total', 'Login attempts', ['result'])
password_resets = Counter('password_resets_total', 'Password resets')

# === Erreurs ===

errors_total = Counter(
    'errors_total',
    'Total errors',
    ['type', 'severity']
)

exceptions_total = Counter(
    'exceptions_total',
    'Total exceptions',
    ['exception_type']
)

# Wrapper exception
def track_exceptions(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except Exception as e:
            exceptions_total.labels(
                exception_type=type(e).__name__
            ).inc()
            raise
    return wrapper


[OK] BONNES PRATIQUES

# === Nommage ===

# Format: <namespace>_<subsystem>_<name>_<unit>_<suffix>

# Bon:
http_requests_total                    # Counter avec _total
http_request_duration_seconds          # Histogram avec unité
process_cpu_seconds_total              # Process metrics
db_connections_active                  # État actuel

# Unités courantes:
# - seconds, milliseconds, microseconds
# - bytes, kilobytes, megabytes
# - celsius, fahrenheit
# - ratio (0-1), percent (0-100)

# Suffixes:
# - _total pour Counter
# - _count, _sum, _bucket pour Histogram
# - Pas de suffixe pour Gauge

# === Labels ===

# [OK] BON: Cardinalité basse
http_requests.labels(method='GET', endpoint='/api', status='200')

# [X] MAUVAIS: Cardinalité haute
# bad.labels(user_id='123456')  # Éviter user_id
# bad.labels(timestamp='2024-01-15')  # Jamais de timestamp
# bad.labels(url='http://...')  # URLs complètes

# Regrouper endpoints:
# /api/users/123 -> /api/users/{id}
# /api/orders/456 -> /api/orders/{id}

# === Performance ===

# Éviter calculs lourds dans collect()
class BadCollector:
    def collect(self):
        # [X] MAUVAIS: Calcul lourd à chaque scrape
        data = expensive_database_query()
        gauge = GaugeMetricFamily('metric', 'Description')
        gauge.add_metric([], data)
        yield gauge

# [OK] BON: Cache ou mise à jour périodique
class GoodCollector:
    def __init__(self):
        self.cached_value = 0
        self.last_update = 0
    
    def collect(self):
        now = time.time()
        if now - self.last_update > 60:  # Cache 60s
            self.cached_value = expensive_database_query()
            self.last_update = now
        
        gauge = GaugeMetricFamily('metric', 'Description')
        gauge.add_metric([], self.cached_value)
        yield gauge

# === Cardinalité ===

# Surveiller nombre de séries temporelles
# each unique label combination = new time series

# Exemple:
# 5 methods × 100 endpoints × 10 status codes = 5000 time series
# Avec user_id: 5 × 100 × 10 × 10000 users = 50M time series! [X]

# Limiter à < 10 valeurs par label si possible
# Maximum recommandé: < 10000 time series par métrique

# === Documentation ===

# Toujours documenter métriques
well_documented = Counter(
    'api_requests_total',
    'Total number of API requests received since application start',
    ['method', 'endpoint', 'status_code']
)

# === Consistance ===

# Utiliser même naming scheme
http_requests_total          # [OK]
http_request_count          # [X] (pas cohérent)

# Utiliser mêmes labels
requests.labels(method='GET', endpoint='/api', status='200')
requests.labels(method='POST', endpoint='/users', status='201')
# [X] requests.labels(http_method='GET', path='/api', http_status='200')


[OK] INTÉGRATIONS FRAMEWORKS

# === Flask-Prometheus ===

from flask import Flask
from prometheus_flask_exporter import PrometheusMetrics

app = Flask(__name__)
metrics = PrometheusMetrics(app)

# Métriques automatiques activées:
# - flask_http_request_duration_seconds
# - flask_http_request_total
# - flask_http_request_exceptions_total

@app.route('/api/users')
def users():
    return {"users": []}

# Métriques personnalisées
@app.route('/api/orders')
@metrics.counter('orders_requests', 'Orders endpoint requests')
def orders():
    return {"orders": []}

# Histogram personnalisé
@app.route('/api/slow')
@metrics.histogram('slow_request_duration', 'Slow request duration')
def slow_endpoint():
    time.sleep(2)
    return "Done"

# Exclure endpoints
metrics = PrometheusMetrics(app, excluded_paths=['/health', '/metrics'])

# === Django-Prometheus ===

# pip install django-prometheus

# settings.py
INSTALLED_APPS = [
    'django_prometheus',
    # ... autres apps
]

MIDDLEWARE = [
    'django_prometheus.middleware.PrometheusBeforeMiddleware',
    # ... autres middlewares
    'django_prometheus.middleware.PrometheusAfterMiddleware',
]

# Database monitoring
DATABASES = {
    'default': {
        'ENGINE': 'django_prometheus.db.backends.postgresql',
        'NAME': 'mydb',
        # ...
    }
}

# Cache monitoring
CACHES = {
    'default': {
        'BACKEND': 'django_prometheus.cache.backends.redis.RedisCache',
        'LOCATION': 'redis://127.0.0.1:6379/1',
    }
}

# urls.py
urlpatterns = [
    path('', include('django_prometheus.urls')),
    # ...
]

# Métriques exposées:
# - django_http_requests_total
# - django_http_requests_latency_seconds
# - django_db_query_duration_seconds
# - django_cache_hits_total

# === FastAPI avec middleware ===

from fastapi import FastAPI
from prometheus_fastapi_instrumentator import Instrumentator

app = FastAPI()

# Auto-instrumentation
Instrumentator().instrument(app).expose(app)

# Personnalisé
instrumentator = Instrumentator(
    should_group_status_codes=False,
    should_ignore_untemplated=True,
    should_respect_env_var=True,
    should_instrument_requests_inprogress=True,
    excluded_handlers=["/metrics", "/health"],
    env_var_name="ENABLE_METRICS",
    inprogress_name="http_requests_inprogress",
    inprogress_labels=True,
)

instrumentator.instrument(app).expose(app, endpoint="/metrics")

# === Celery ===

from celery import Celery
from prometheus_client import Counter, Histogram

app = Celery('tasks', broker='redis://localhost:6379/0')

# Métriques Celery
task_started = Counter('celery_task_started_total', 'Tasks started', ['task'])
task_succeeded = Counter('celery_task_succeeded_total', 'Tasks succeeded', ['task'])
task_failed = Counter('celery_task_failed_total', 'Tasks failed', ['task'])
task_duration = Histogram('celery_task_duration_seconds', 'Task duration', ['task'])

@app.task(bind=True)
def process_data(self, data):
    task_name = self.name
    task_started.labels(task=task_name).inc()
    
    start = time.time()
    try:
        # Process...
        result = data * 2
        task_succeeded.labels(task=task_name).inc()
        return result
    except Exception as e:
        task_failed.labels(task=task_name).inc()
        raise
    finally:
        duration = time.time() - start
        task_duration.labels(task=task_name).observe(duration)

# Ou utiliser celery-prometheus-exporter
# pip install celery-prometheus-exporter

# === SQLAlchemy ===

from sqlalchemy import event, create_engine
from prometheus_client import Counter, Histogram

query_counter = Counter('sqlalchemy_queries_total', 'Total queries', ['operation'])
query_duration = Histogram('sqlalchemy_query_duration_seconds', 'Query duration')

engine = create_engine('postgresql://user:pass@localhost/db')

@event.listens_for(engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    conn.info.setdefault('query_start_time', []).append(time.time())

@event.listens_for(engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    total = time.time() - conn.info['query_start_time'].pop(-1)
    query_duration.observe(total)
    
    # Détecter type opération
    operation = statement.strip().split()[0].upper()
    query_counter.labels(operation=operation).inc()

# === Redis ===

from redis import Redis
from prometheus_client import Counter, Histogram

redis_commands = Counter('redis_commands_total', 'Redis commands', ['command'])
redis_duration = Histogram('redis_command_duration_seconds', 'Command duration', ['command'])

class InstrumentedRedis(Redis):
    def execute_command(self, *args, **kwargs):
        command = args[0] if args else 'UNKNOWN'
        
        start = time.time()
        try:
            result = super().execute_command(*args, **kwargs)
            return result
        finally:
            duration = time.time() - start
            redis_commands.labels(command=command).inc()
            redis_duration.labels(command=command).observe(duration)

redis_client = InstrumentedRedis(host='localhost', port=6379)

# === AWS Boto3 ===

import boto3
from prometheus_client import Counter, Histogram

boto_calls = Counter('boto_api_calls_total', 'Boto3 API calls', ['service', 'operation'])
boto_duration = Histogram('boto_api_duration_seconds', 'API call duration', ['service', 'operation'])

def instrument_boto_client(client):
    original_make_request = client._make_request
    
    def instrumented_make_request(operation_model, request_dict, *args, **kwargs):
        service = client._service_model.service_name
        operation = operation_model.name
        
        start = time.time()
        try:
            return original_make_request(operation_model, request_dict, *args, **kwargs)
        finally:
            duration = time.time() - start
            boto_calls.labels(service=service, operation=operation).inc()
            boto_duration.labels(service=service, operation=operation).observe(duration)
    
    client._make_request = instrumented_make_request
    return client

# Utilisation
s3 = instrument_boto_client(boto3.client('s3'))


[OK] ALERTING

# Prometheus supporte les alertes via alertmanager
# Définir règles dans prometheus.yml

# === Exemples de règles d'alerte ===

# prometheus_rules.yml
groups:
  - name: application_alerts
    interval: 30s
    rules:
      # Taux d'erreur élevé
      - alert: HighErrorRate
        expr: |
          rate(http_requests_total{status=~"5.."}[5m]) 
          / 
          rate(http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"
      
      # Latence élevée
      - alert: HighLatency
        expr: |
          histogram_quantile(0.95, 
            rate(http_request_duration_seconds_bucket[5m])
          ) > 1.0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High latency detected"
          description: "P95 latency is {{ $value }}s"
      
      # Service down
      - alert: ServiceDown
        expr: up{job="myapp"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.instance }} is down"
      
      # Utilisation mémoire élevée
      - alert: HighMemoryUsage
        expr: |
          (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) 
          / 
          node_memory_MemTotal_bytes > 0.90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
      
      # Disque plein
      - alert: DiskSpaceLow
        expr: |
          (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Disk space low on {{ $labels.instance }}"

# === Exposer métriques pour alertes ===

from prometheus_client import Gauge

# État de santé
health_status = Gauge('app_health_status', 'Application health (1=healthy, 0=unhealthy)')

def check_health():
    try:
        # Vérifications...
        db_ok = check_database()
        cache_ok = check_cache()
        
        if db_ok and cache_ok:
            health_status.set(1)
        else:
            health_status.set(0)
    except Exception:
        health_status.set(0)

# SLO (Service Level Objective)
slo_compliance = Gauge('slo_compliance_ratio', 'SLO compliance ratio')

def calculate_slo():
    # Exemple: 99.9% des requêtes doivent réussir
    total = http_requests_total._value.get()
    errors = http_requests_total.labels(status='500')._value.get()
    
    if total > 0:
        success_rate = (total - errors) / total
        slo_compliance.set(success_rate)


[OK] GRAFANA INTEGRATION

# === Installation Grafana ===

# Docker
docker run -d -p 3000:3000 --name=grafana grafana/grafana

# Accès: http://localhost:3000
# Login: admin / admin

# === Configuration Data Source ===

# UI: Configuration -> Data Sources -> Add Prometheus
# URL: http://prometheus:9090 (ou localhost:9090)

# Ou via provisioning:
# /etc/grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true

# === Créer Dashboard ===

# Via UI ou API
import requests
import json

GRAFANA_URL = "http://localhost:3000"
API_KEY = "your-api-key"

headers = {
    "Authorization": f"Bearer {API_KEY}",
    "Content-Type": "application/json"
}

dashboard = {
    "dashboard": {
        "title": "Application Metrics",
        "tags": ["python", "application"],
        "timezone": "browser",
        "panels": [
            {
                "id": 1,
                "title": "Request Rate",
                "type": "graph",
                "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8},
                "targets": [
                    {
                        "expr": "rate(http_requests_total[5m])",
                        "legendFormat": "{{method}} {{endpoint}}",
                        "refId": "A"
                    }
                ]
            },
            {
                "id": 2,
                "title": "Error Rate",
                "type": "graph",
                "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8},
                "targets": [
                    {
                        "expr": "rate(http_requests_total{status=~\"5..\"}[5m])",
                        "legendFormat": "Errors",
                        "refId": "A"
                    }
                ]
            }
        ]
    },
    "overwrite": True
}

response = requests.post(
    f"{GRAFANA_URL}/api/dashboards/db",
    headers=headers,
    data=json.dumps(dashboard)
)

# === Dashboard Python avec grafana-api ===

from grafana_api.grafana_face import GrafanaFace

grafana = GrafanaFace(
    auth="admin:admin",
    host="localhost",
    port=3000
)

# Créer dashboard
dashboard_json = {
    "dashboard": {
        "title": "Python App Dashboard",
        "panels": [
            {
                "id": 1,
                "title": "Request Rate",
                "type": "timeseries",
                "targets": [
                    {
                        "expr": "rate(http_requests_total[5m])",
                        "legendFormat": "{{method}} {{endpoint}}"
                    }
                ]
            }
        ]
    },
    "folderId": 0,
    "overwrite": True
}

grafana.dashboard.update_dashboard(dashboard_json)

# === Panels recommandés ===

# 1. Request Rate (QPS)
rate(http_requests_total[5m])

# 2. Error Rate
rate(http_requests_total{status=~"5.."}[5m])

# 3. Success Rate (%)
(
  rate(http_requests_total{status!~"5.."}[5m])
  /
  rate(http_requests_total[5m])
) * 100

# 4. Latency (P50, P95, P99)
histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# 5. Requests in Progress
http_requests_in_progress

# 6. Memory Usage
process_resident_memory_bytes

# 7. CPU Usage
rate(process_cpu_seconds_total[5m]) * 100

# 8. Active Connections
db_connections{state="active"}

# === Variables Grafana ===

# Créer variable pour filtrer:
# Name: environment
# Query: label_values(http_requests_total, environment)

# Utiliser dans query:
rate(http_requests_total{environment="$environment"}[5m])

# === Annotations ===

# Marquer deployments
deployment_timestamp

# Query dans Grafana:
# Type: Tags
# Query: deployment

# === Alerting dans Grafana ===

# Créer alerte sur panel
alert_condition = {
    "conditions": [
        {
            "evaluator": {
                "params": [0.05],
                "type": "gt"
            },
            "operator": {"type": "and"},
            "query": {"params": ["A", "5m", "now"]},
            "reducer": {"type": "avg"},
            "type": "query"
        }
    ],
    "executionErrorState": "alerting",
    "frequency": "60s",
    "handler": 1,
    "name": "High Error Rate",
    "noDataState": "no_data",
    "notifications": []
}

# === Templates Dashboard ===

# Dashboard JSON complet
dashboard_template = {
    "dashboard": {
        "title": "Python Application Overview",
        "tags": ["python", "monitoring"],
        "timezone": "browser",
        "schemaVersion": 16,
        "version": 0,
        "refresh": "30s",
        "panels": [
            # Row 1: Overview
            {
                "id": 1,
                "title": "Request Rate",
                "type": "stat",
                "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
                "targets": [
                    {
                        "expr": "sum(rate(http_requests_total[5m]))",
                        "refId": "A"
                    }
                ],
                "options": {
                    "graphMode": "area",
                    "colorMode": "value"
                }
            },
            {
                "id": 2,
                "title": "Error Rate",
                "type": "stat",
                "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
                "targets": [
                    {
                        "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m]))",
                        "refId": "A"
                    }
                ],
                "options": {
                    "colorMode": "value"
                }
            },
            # Row 2: Detailed metrics
            {
                "id": 3,
                "title": "Requests by Endpoint",
                "type": "timeseries",
                "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
                "targets": [
                    {
                        "expr": "sum by(endpoint) (rate(http_requests_total[5m]))",
                        "legendFormat": "{{endpoint}}",
                        "refId": "A"
                    }
                ]
            },
            {
                "id": 4,
                "title": "Latency (P95)",
                "type": "timeseries",
                "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
                "targets": [
                    {
                        "expr": "histogram_quantile(0.95, sum by(le) (rate(http_request_duration_seconds_bucket[5m])))",
                        "legendFormat": "P95",
                        "refId": "A"
                    }
                ]
            }
        ]
    }
}


[OK] EXEMPLES COMPLETS

# === Exemple 1: API Flask complète ===

from flask import Flask, request
from prometheus_client import Counter, Histogram, Gauge, generate_latest
from functools import wraps
import time

app = Flask(__name__)

# Métriques
REQUEST_COUNT = Counter(
    'flask_request_count',
    'App Request Count',
    ['method', 'endpoint', 'http_status']
)

REQUEST_LATENCY = Histogram(
    'flask_request_latency_seconds',
    'Request latency',
    ['method', 'endpoint']
)

REQUEST_IN_PROGRESS = Gauge(
    'flask_requests_in_progress',
    'Requests in progress'
)

DB_CONNECTIONS = Gauge(
    'database_connections',
    'Database connections',
    ['state']
)

# Middleware
@app.before_request
def before_request_func():
    request.start_time = time.time()
    REQUEST_IN_PROGRESS.inc()

@app.after_request
def after_request_func(response):
    REQUEST_IN_PROGRESS.dec()
    
    latency = time.time() - request.start_time
    
    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.endpoint or 'unknown',
        http_status=response.status_code
    ).inc()
    
    REQUEST_LATENCY.labels(
        method=request.method,
        endpoint=request.endpoint or 'unknown'
    ).observe(latency)
    
    return response

# Routes
@app.route('/')
def index():
    return {"message": "Hello World"}

@app.route('/api/users')
def get_users():
    time.sleep(0.1)  # Simuler DB query
    DB_CONNECTIONS.labels(state='active').inc()
    
    try:
        users = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
        return {"users": users}
    finally:
        DB_CONNECTIONS.labels(state='active').dec()

@app.route('/api/error')
def error_endpoint():
    raise Exception("Test error")

@app.route('/metrics')
def metrics():
    return generate_latest()

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)

# === Exemple 2: Worker asynchrone ===

import asyncio
from prometheus_client import Counter, Histogram, Gauge, start_http_server
import time

# Métriques
tasks_processed = Counter('worker_tasks_processed_total', 'Tasks processed', ['status'])
task_duration = Histogram('worker_task_duration_seconds', 'Task duration', ['task_type'])
queue_size = Gauge('worker_queue_size', 'Queue size')
active_workers = Gauge('worker_active_count', 'Active workers')

async def process_task(task):
    """Traiter une tâche"""
    task_type = task.get('type', 'unknown')
    
    active_workers.inc()
    start = time.time()
    
    try:
        # Simuler traitement
        await asyncio.sleep(task.get('duration', 1))
        
        tasks_processed.labels(status='success').inc()
        return True
    except Exception as e:
        tasks_processed.labels(status='error').inc()
        return False
    finally:
        duration = time.time() - start
        task_duration.labels(task_type=task_type).observe(duration)
        active_workers.dec()

async def worker(queue):
    """Worker qui consomme la queue"""
    while True:
        task = await queue.get()
        queue_size.set(queue.qsize())
        
        await process_task(task)
        queue.task_done()

async def main():
    # Démarrer serveur metrics
    start_http_server(8000)
    print("Metrics available at http://localhost:8000/metrics")
    
    # Créer queue et workers
    queue = asyncio.Queue()
    
    workers = [asyncio.create_task(worker(queue)) for _ in range(5)]
    
    # Simuler ajout de tâches
    for i in range(100):
        await queue.put({'type': 'processing', 'duration': 0.5})
        await asyncio.sleep(0.1)
    
    await queue.join()

if __name__ == '__main__':
    asyncio.run(main())

# === Exemple 3: Monitoring système complet ===

import psutil
import time
from prometheus_client import Gauge, CollectorRegistry, start_http_server

# Registry personnalisé
registry = CollectorRegistry()

# Métriques système
cpu_percent = Gauge('system_cpu_percent', 'CPU usage', registry=registry)
memory_percent = Gauge('system_memory_percent', 'Memory usage', registry=registry)
disk_percent = Gauge('system_disk_percent', 'Disk usage', ['mount'], registry=registry)
network_bytes = Gauge('system_network_bytes', 'Network bytes', ['interface', 'direction'], registry=registry)

# Métriques processus
process_cpu = Gauge('process_cpu_percent', 'Process CPU', ['pid', 'name'], registry=registry)
process_memory = Gauge('process_memory_mb', 'Process memory MB', ['pid', 'name'], registry=registry)

def collect_metrics():
    """Collecter toutes les métriques"""
    # CPU
    cpu_percent.set(psutil.cpu_percent(interval=1))
    
    # Mémoire
    mem = psutil.virtual_memory()
    memory_percent.set(mem.percent)
    
    # Disque
    for partition in psutil.disk_partitions():
        try:
            usage = psutil.disk_usage(partition.mountpoint)
            disk_percent.labels(mount=partition.mountpoint).set(usage.percent)
        except:
            pass
    
    # Réseau
    net = psutil.net_io_counters(pernic=True)
    for interface, stats in net.items():
        network_bytes.labels(interface=interface, direction='sent').set(stats.bytes_sent)
        network_bytes.labels(interface=interface, direction='received').set(stats.bytes_recv)
    
    # Top processus
    for proc in psutil.process_iter(['pid', 'name', 'cpu_percent', 'memory_info']):
        try:
            info = proc.info
            process_cpu.labels(pid=info['pid'], name=info['name']).set(info['cpu_percent'])
            process_memory.labels(pid=info['pid'], name=info['name']).set(
                info['memory_info'].rss / 1024 / 1024  # MB
            )
        except:
            pass

def main():
    # Démarrer serveur
    start_http_server(8000, registry=registry)
    print("System metrics available at http://localhost:8000/metrics")
    
    # Boucle collection
    while True:
        collect_metrics()
        time.sleep(15)  # Toutes les 15s

if __name__ == '__main__':
    main()


[OK] PROMETHEUS SERVER CONFIG

# prometheus.yml - Configuration serveur Prometheus

global:
  scrape_interval: 15s      # Scrape toutes les 15s
  evaluation_interval: 15s  # Évaluer règles toutes les 15s
  external_labels:
    cluster: 'production'
    region: 'us-east-1'

# Alertmanager
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - localhost:9093

# Règles d'alerte
rule_files:
  - 'alerts/*.yml'
  - 'recording_rules/*.yml'

# Scrape configs
scrape_configs:
  # Prometheus lui-même
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  # Application Python
  - job_name: 'python-app'
    static_configs:
      - targets: ['localhost:5000']
        labels:
          environment: 'production'
          service: 'api'
  
  # Plusieurs instances
  - job_name: 'python-workers'
    static_configs:
      - targets:
          - 'worker1:8000'
          - 'worker2:8000'
          - 'worker3:8000'
        labels:
          role: 'worker'
  
  # Service discovery (Kubernetes)
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
  
  # Pushgateway
  - job_name: 'pushgateway'
    honor_labels: true
    static_configs:
      - targets: ['localhost:9091']

# Recording rules (pour optimiser queries)
# recording_rules.yml
groups:
  - name: api_rules
    interval: 30s
    rules:
      - record: api:http_requests:rate5m
        expr: rate(http_requests_total[5m])
      
      - record: api:http_requests:rate5m:by_endpoint
        expr: sum by (endpoint) (rate(http_requests_total[5m]))
      
      - record: api:http_request_duration:p95
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))


[OK] QUERIES PROMQL

# === Basics ===

# Valeur instantanée
http_requests_total

# Avec labels
http_requests_total{method="GET"}
http_requests_total{method="GET", status="200"}

# Regex labels
http_requests_total{status=~"2.."}        # 2xx status
http_requests_total{status!~"5.."}        # Pas 5xx
http_requests_total{endpoint=~"/api/.*"}  # Commence par /api/

# === Rate & Increase ===

# Rate (par seconde sur 5min)
rate(http_requests_total[5m])

# Increase (total sur 5min)
increase(http_requests_total[5m])

# irate (instantaneous rate)
irate(http_requests_total[5m])

# === Aggregation ===

# Sum
sum(http_requests_total)
sum by(method) (http_requests_total)
sum without(instance) (http_requests_total)

# Avg
avg(http_request_duration_seconds)
avg by(endpoint) (http_request_duration_seconds)

# Min/Max
min(http_request_duration_seconds)
max(http_request_duration_seconds)

# Count
count(http_requests_total)
count by(status) (http_requests_total)

# Stddev/Stdvar
stddev(http_request_duration_seconds)
stdvar(http_request_duration_seconds)

# Topk/Bottomk (top N)
topk(5, http_requests_total)
bottomk(3, http_request_duration_seconds)

# === Histogram & Summary ===

# P95 latency
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# P99 latency
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# P50 (median)
histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))

# Par endpoint
histogram_quantile(
  0.95,
  sum by(le, endpoint) (rate(http_request_duration_seconds_bucket[5m]))
)

# Average from histogram
rate(http_request_duration_seconds_sum[5m])
/
rate(http_request_duration_seconds_count[5m])

# === Math Operations ===

# Addition
http_requests_total + 100

# Multiplication
memory_bytes * 1024

# Division (error rate %)
rate(http_requests_total{status="500"}[5m])
/
rate(http_requests_total[5m])
* 100

# Subtraction
memory_total_bytes - memory_available_bytes

# === Comparison ===

# Greater than
http_requests_total > 1000

# Less than or equal
memory_usage_percent <= 90

# Equal
http_requests_total{status="200"} == 100

# Not equal
http_requests_total != 0

# === Logical Operations ===

# AND
http_requests_total > 100 and rate(http_requests_total[5m]) > 10

# OR
http_requests_total{status="500"} or http_requests_total{status="503"}

# UNLESS (exclusion)
http_requests_total unless http_requests_total{status="200"}

# === Time Functions ===

# Timestamp
timestamp(http_requests_total)

# Day of month (1-31)
day_of_month()

# Day of week (0-6, 0=Sunday)
day_of_week()

# Hour
hour()

# Minute
minute()

# Time offset
http_requests_total offset 5m     # 5 minutes ago
http_requests_total offset 1h     # 1 hour ago

# === Predictions ===

# Predict linear
predict_linear(disk_usage_bytes[1h], 4 * 3600)  # Predict 4h ahead

# Deriv (rate of change)
deriv(disk_usage_bytes[1h])

# === Changes ===

# Delta (difference)
delta(cpu_temp_celsius[5m])

# Increase
increase(http_requests_total[1h])

# Changes (number of changes)
changes(app_version[1d])

# Resets (counter resets)
resets(http_requests_total[1d])

# === Sorting ===

# Sort ascending
sort(http_requests_total)

# Sort descending
sort_desc(http_requests_total)

# === Missing Data ===

# Absent (returns 1 if no data)
absent(http_requests_total{job="myapp"})

# Absent over time
absent_over_time(http_requests_total[5m])

# === Vector Matching ===

# One-to-one
method:http_requests:rate5m / on(method) group_left method:http_requests:total

# Many-to-one
sum by(instance) (rate(http_requests_total[5m]))
/
on(instance) group_left
instance:node_cpu:ratio

# Ignoring labels
sum without(status) (rate(http_requests_total[5m]))

# === Advanced Examples ===

# Success rate
sum(rate(http_requests_total{status!~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))

# Availability (SLA)
(
  1 - (
    sum(rate(http_requests_total{status=~"5.."}[30d]))
    /
    sum(rate(http_requests_total[30d]))
  )
) * 100

# Request rate per instance
sum by(instance) (rate(http_requests_total[5m]))

# Top 10 endpoints
topk(10, sum by(endpoint) (rate(http_requests_total[5m])))

# Memory available %
(
  node_memory_MemAvailable_bytes
  /
  node_memory_MemTotal_bytes
) * 100

# Disk usage %
(
  node_filesystem_size_bytes - node_filesystem_free_bytes
)
/
node_filesystem_size_bytes
* 100

# CPU usage %
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Network throughput
rate(node_network_receive_bytes_total[5m]) * 8  # bits/sec


[OK] DOCKER & KUBERNETES

# === Docker Compose ===

# docker-compose.yml
version: '3'

services:
  app:
    build: .
    ports:
      - "5000:5000"
    environment:
      - PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus
    volumes:
      - prom-data:/tmp/prometheus
  
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
  
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    depends_on:
      - prometheus

volumes:
  prom-data:
  prometheus-data:
  grafana-data:

# === Kubernetes Deployment ===

# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: python-app
  labels:
    app: python-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: python-app
  template:
    metadata:
      labels:
        app: python-app
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "5000"
        prometheus.io/path: "/metrics"
    spec:
      containers:
      - name: app
        image: myapp:latest
        ports:
        - containerPort: 5000
          name: http
        - containerPort: 8000
          name: metrics
        env:
        - name: PROMETHEUS_MULTIPROC_DIR
          value: /tmp/prometheus
        livenessProbe:
          httpGet:
            path: /health
            port: 5000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            Port: 5000
          initialDelaySeconds: 5
          periodSeconds: 5

---
apiVersion: v1
kind: Service
metadata:
  name: python-app
  labels:
    app: python-app
spec:
  type: ClusterIP
  ports:
  - port: 5000
    targetPort: 5000
    name: http
  - port: 8000
    targetPort: 8000
    name: metrics
  selector:
    app: python-app

# === ServiceMonitor (Prometheus Operator) ===

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: python-app
  labels:
    app: python-app
spec:
  selector:
    matchLabels:
      app: python-app
  endpoints:
  - port: metrics
    interval: 30s
    path: /metrics

# === PodMonitor ===

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: python-app
spec:
  selector:
    matchLabels:
      app: python-app
  podMetricsEndpoints:
  - port: metrics
    interval: 30s

# === Dockerfile avec Prometheus ===

FROM python:3.11-slim

WORKDIR /app

# Install dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy app
COPY . .

# Create prometheus multiproc dir
RUN mkdir -p /tmp/prometheus

# Expose ports
EXPOSE 5000 8000

# Environment
ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus

# Run with gunicorn
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "app:app"]


[OK] TESTING

# === Tests unitaires ===

import pytest
from prometheus_client import REGISTRY, Counter, Gauge
from prometheus_client.core import CollectorRegistry

def test_counter_increment():
    """Test counter increment"""
    registry = CollectorRegistry()
    counter = Counter('test_counter', 'Test counter', registry=registry)
    
    counter.inc()
    assert counter._value.get() == 1
    
    counter.inc(5)
    assert counter._value.get() == 6

def test_gauge_operations():
    """Test gauge set/inc/dec"""
    registry = CollectorRegistry()
    gauge = Gauge('test_gauge', 'Test gauge', registry=registry)
    
    gauge.set(10)
    assert gauge._value.get() == 10
    
    gauge.inc()
    assert gauge._value.get() == 11
    
    gauge.dec(5)
    assert gauge._value.get() == 6

def test_labels():
    """Test labels"""
    registry = CollectorRegistry()
    counter = Counter('test', 'Test', ['method', 'endpoint'], registry=registry)
    
    counter.labels(method='GET', endpoint='/api').inc()
    counter.labels('POST', '/users').inc(2)
    
    # Verify values
    metrics = list(registry.collect())
    assert len(metrics) > 0

def test_histogram():
    """Test histogram observations"""
    registry = CollectorRegistry()
    histogram = Histogram('test_histogram', 'Test', registry=registry)
    
    histogram.observe(0.5)
    histogram.observe(1.0)
    histogram.observe(2.0)
    
    # Check count
    metrics = list(registry.collect())
    for metric in metrics:
        for sample in metric.samples:
            if sample.name == 'test_histogram_count':
                assert sample.value == 3

def test_metrics_endpoint():
    """Test Flask metrics endpoint"""
    from flask import Flask
    from prometheus_client import generate_latest
    
    app = Flask(__name__)
    
    @app.route('/metrics')
    def metrics():
        return generate_latest()
    
    client = app.test_client()
    response = client.get('/metrics')
    
    assert response.status_code == 200
    assert b'python_gc_objects_collected_total' in response.data

# === Tests d'intégration ===

def test_prometheus_scrape():
    """Test que Prometheus peut scraper l'endpoint"""
    import requests
    
    response = requests.get('http://localhost:5000/metrics')
    assert response.status_code == 200
    assert 'http_requests_total' in response.text

def test_metrics_format():
    """Test format des métriques"""
    import requests
    
    response = requests.get('http://localhost:5000/metrics')
    lines = response.text.split('\n')
    
    # Vérifier format
    for line in lines:
        if line and not line.startswith('#'):
            # Doit contenir nom métrique et valeur
            assert ' ' in line

# === Mock pour tests ===

from unittest.mock import Mock, patch

def test_with_mock_collector():
    """Test avec collector mocké"""
    mock_collector = Mock()
    mock_collector.collect.return_value = []
    
    registry = CollectorRegistry()
    registry.register(mock_collector)
    
    list(registry.collect())
    mock_collector.collect.assert_called_once()


[OK] PERFORMANCE & OPTIMISATION

# === Optimisations ===

# 1. Limiter cardinalité
# [X] MAUVAIS
bad_counter = Counter('requests', 'Requests', ['user_id', 'url'])

# [OK] BON
good_counter = Counter('requests', 'Requests', ['method', 'endpoint', 'status'])

# 2. Utiliser recording rules
# Pré-calculer queries complexes dans Prometheus
# recording_rules.yml
# - record: api:request_rate:5m
#   expr: rate(http_requests_total[5m])

# 3. Réutiliser child metrics
request_counter = Counter('requests', 'Requests', ['method', 'endpoint'])

# [X] Lent
for _ in range(1000):
    request_counter.labels(method='GET', endpoint='/api').inc()

# [OK] Rapide
get_api_counter = request_counter.labels(method='GET', endpoint='/api')
for _ in range(1000):
    get_api_counter.inc()

# 4. Batch updates avec contexte
from prometheus_client import Gauge
import time

gauge = Gauge('processing_time', 'Processing time')

# Au lieu de mesurer chaque micro-opération
with gauge.time():
    # Tout le traitement
    for i in range(100):
        process_item(i)

# 5. Éviter collect() coûteux
class OptimizedCollector:
    def __init__(self):
        self.cache = {}
        self.cache_ttl = 60
        self.last_update = 0
    
    def collect(self):
        now = time.time()
        
        # Cache les résultats
        if now - self.last_update > self.cache_ttl:
            self.cache = self._fetch_data()
            self.last_update = now
        
        gauge = GaugeMetricFamily('metric', 'Description')
        gauge.add_metric([], self.cache.get('value', 0))
        yield gauge
    
    def _fetch_data(self):
        # Opération coûteuse
        return {'value': 42}

# 6. Multiprocess mode pour Gunicorn
# Utiliser PROMETHEUS_MULTIPROC_DIR pour éviter conflits

# 7. Registry personnalisé pour isolation
custom_registry = CollectorRegistry()
# Seulement les métriques nécessaires

# 8. Désactiver métriques inutiles
from prometheus_client import REGISTRY, PROCESS_COLLECTOR
REGISTRY.unregister(PROCESS_COLLECTOR)

# === Monitoring des métriques ===

# Métriques sur les métriques elles-mêmes
prometheus_metrics_count = Gauge(
    'prometheus_metrics_count',
    'Number of metrics'
)

def count_metrics():
    """Compter nombre de métriques exposées"""
    from prometheus_client import REGISTRY
    
    count = 0
    for collector in REGISTRY._collector_to_names.values():
        for name in collector:
            count += 1
    
    prometheus_metrics_count.set(count)

# Taille des métriques exposées
from prometheus_client import generate_latest

metrics_size = Gauge('prometheus_metrics_size_bytes', 'Size of metrics output')

def measure_metrics_size():
    data = generate_latest()
    metrics_size.set(len(data))


[OK] SÉCURITÉ

# === Authentification Basic ===

# Flask avec auth
from flask import Flask, request, Response
from functools import wraps
from prometheus_client import generate_latest

def check_auth(username, password):
    return username == 'admin' and password == 'secret'

def authenticate():
    return Response(
        'Authentication required',
        401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'}
    )

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

@app.route('/metrics')
@requires_auth
def metrics():
    return generate_latest()

# === Authentification Token ===

from flask import request, abort

METRICS_TOKEN = 'secret-token-123'

@app.route('/metrics')
def metrics():
    token = request.headers.get('X-Auth-Token')
    if token != METRICS_TOKEN:
        abort(403)
    return generate_latest()

# === IP Whitelist ===

ALLOWED_IPS = ['127.0.0.1', '10.0.0.0/8']

@app.route('/metrics')
def metrics():
    client_ip = request.remote_addr
    
    if client_ip not in ALLOWED_IPS:
        abort(403)
    
    return generate_latest()

# === TLS/HTTPS ===

# Utiliser reverse proxy (nginx, traefik)
# Ou gunicorn avec certfile

# gunicorn --certfile=cert.pem --keyfile=key.pem app:app

# === Filtrer métriques sensibles ===

from prometheus_client import CollectorRegistry, generate_latest

def generate_filtered_metrics():
    """Générer métriques sans données sensibles"""
    registry = CollectorRegistry()
    
    # Copier seulement métriques publiques
    for collector in REGISTRY._collector_to_names.keys():
        # Filter logic
        registry.register(collector)
    
    return generate_latest(registry)


[OK] DEBUGGING

# === Afficher métriques en console ===

from prometheus_client import REGISTRY

def print_metrics():
    """Debug: afficher toutes les métriques"""
    for metric in REGISTRY.collect():
        print(f"\n=== {metric.name} ===")
        print(f"Type: {metric.type}")
        print(f"Documentation: {metric.documentation}")
        
        for sample in metric.samples:
            print(f"  {sample.name}{sample.labels} = {sample.value}")

# === Vérifier valeurs métriques ===

def debug_counter(counter):
    """Debug counter value"""
    print(f"Counter value: {counter._value.get()}")
    
    # Avec labels
    if hasattr(counter, '_metrics'):
        for labels, child in counter._metrics.items():
            print(f"  {labels}: {child._value.get()}")

# === Logger appels ===

import logging
from prometheus_client import Counter

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

class DebugCounter(Counter):
    def inc(self, amount=1):
        logger.debug(f"Counter {self._name} incremented by {amount}")
        super().inc(amount)

# === Valider format métriques ===

def validate_metrics_format():
    """Valider format exposition metrics"""
    from prometheus_client import generate_latest
    
    output = generate_latest().decode('utf-8')
    
    for line in output.split('\n'):
        if line and not line.startswith('#'):
            # Vérifier format: metric_name{labels} value
            if ' ' not in line:
                print(f"Invalid line: {line}")
            
            parts = line.rsplit(' ', 1)
            if len(parts) != 2:
                print(f"Invalid format: {line}")
            
            try:
                float(parts[1])
            except ValueError:
                print(f"Invalid value: {parts[1]}")

# === Test endpoint localement ===

def test_metrics_endpoint():
    """Test endpoint /metrics"""
    import requests
    
    response = requests.get('http://localhost:5000/metrics')
    
    print(f"Status: {response.status_code}")
    print(f"Content-Type: {response.headers.get('Content-Type')}")
    print(f"Size: {len(response.content)} bytes")
    print(f"\nFirst 500 chars:\n{response.text[:500]}")

# === Simuler scrape Prometheus ===

def simulate_prometheus_scrape():
    """Simuler scrape Prometheus"""
    import requests
    import time
    
    for i in range(5):
        start = time.time()
        response = requests.get('http://localhost:5000/metrics')
        duration = time.time() - start
        
        print(f"Scrape {i+1}:")
        print(f"  Duration: {duration:.3f}s")
        print(f"  Status: {response.status_code}")
        print(f"  Metrics: {len(response.text.split('\\n'))} lines")
        
        time.sleep(15)


[OK] ERREURS COURANTES

# === Erreur 1: Labels non définis ===

# [X] ERREUR
counter = Counter('requests', 'Requests')
counter.labels(method='GET').inc()  # KeyError!

# [OK] SOLUTION
counter = Counter('requests', 'Requests', ['method'])
counter.labels(method='GET').inc()

# === Erreur 2: Nom métrique invalide ===

# [X] ERREUR
counter = Counter('http-requests', 'Requests')  # Tiret invalide!

# [OK] SOLUTION
counter = Counter('http_requests', 'Requests')  # Underscore

# === Erreur 3: Métrique dupliquée ===

# [X] ERREUR
counter1 = Counter('requests', 'Requests')
counter2 = Counter('requests', 'Other')  # ValueError: Duplicate!

# [OK] SOLUTION: Utiliser même instance ou registry différent
counter = Counter('requests', 'Requests')
# Réutiliser counter partout

# Ou:
registry1 = CollectorRegistry()
counter1 = Counter('requests', 'Requests', registry=registry1)

registry2 = CollectorRegistry()
counter2 = Counter('requests', 'Other', registry=registry2)

# === Erreur 4: Observer valeur négative dans Histogram ===

# [X] ERREUR
histogram = Histogram('duration', 'Duration')
histogram.observe(-1)  # Valeurs négatives pas recommandées

# [OK] SOLUTION: Utiliser valeur absolue ou gauge
histogram.observe(abs(duration))
# Ou
gauge = Gauge('duration', 'Duration')
gauge.set(duration)  # Peut être négatif

# === Erreur 5: Multiprocess sans PROMETHEUS_MULTIPROC_DIR ===

# [X] ERREUR avec gunicorn
# gunicorn app:app  # Métriques incohérentes!

# [OK] SOLUTION
# export PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus
# mkdir -p /tmp/prometheus
# gunicorn app:app

# === Erreur 6: Cardinalité trop élevée ===

# [X] ERREUR
counter = Counter('requests', 'Requests', ['user_id', 'request_id'])
# Millions de combinaisons!

# [OK] SOLUTION
counter = Counter('requests', 'Requests', ['method', 'status'])
# Peu de combinaisons

# === Erreur 7: Oublier d'exposer métriques ===

# [X] ERREUR: Métriques créées mais jamais exposées
counter = Counter('requests', 'Requests')
counter.inc()
# Pas d'endpoint /metrics!

# [OK] SOLUTION
from prometheus_client import start_http_server
start_http_server(8000)
# Ou endpoint Flask/FastAPI

# === Erreur 8: Collector coûteux ===

# [X] ERREUR
class SlowCollector:
    def collect(self):
        time.sleep(5)  # Scrape timeout!
        yield GaugeMetricFamily('metric', 'desc')

# [OK] SOLUTION: Cache ou async
class FastCollector:
    def __init__(self):
        self.cached = None
        self.last_update = 0
    
    def collect(self):
        if time.time() - self.last_update > 60:
            self.cached = fetch_data()  # Une fois/minute
            self.last_update = time.time()
        yield self.cached


[OK] RESSOURCES

# Documentation officielle:
# Prometheus: https://prometheus.io/docs/
# Prometheus Client Python: https://github.com/prometheus/client_python
# PromQL: https://prometheus.io/docs/prometheus/latest/querying/basics/
# Grafana: https://grafana.com/docs/

# Best Practices:
# Prometheus Naming: https://prometheus.io/docs/practices/naming/
# Instrumentation: https://prometheus.io/docs/practices/instrumentation/
# Histograms vs Summaries: https://prometheus.io/docs/practices/histograms/

# Exporters:
# Node Exporter: https://github.com/prometheus/node_exporter
# Blackbox Exporter: https://github.com/prometheus/blackbox_exporter
# PostgreSQL Exporter: https://github.com/prometheus-community/postgres_exporter

# Outils:
# Promtool: Validation config et règles
# PromLens: Query builder et visualizer
# Promlinter: Linter pour PromQL

# Exemples:
# https://github.com/prometheus/client_python/tree/master/examples
# https://github.com/grafana/grafana/tree/main/public/app/plugins/datasource/prometheus

# Formations:
# Prometheus Up & Running (O'Reilly)
# Grafana Tutorials: https://grafana.com/tutorials/

# Communauté:
# Prometheus Users: https://groups.google.com/g/prometheus-users
# CNCF Slack: #prometheus


[OK] CHECKLIST PRODUCTION

# [OK] Configuration
[WHITE_SQUARE] Variables d'environnement pour config
[WHITE_SQUARE] Logging configuré
[WHITE_SQUARE] Health check endpoints (/health, /ready)
[WHITE_SQUARE] Graceful shutdown

# [OK] Métriques
[WHITE_SQUARE] Métriques RED exposées (Rate, Errors, Duration)
[WHITE_SQUARE] Métriques USE exposées (Utilization, Saturation, Errors)
[WHITE_SQUARE] Cardinalité limitée (< 10 valeurs par label)
[WHITE_SQUARE] Labels consistants
[WHITE_SQUARE] Documentation métriques

# [OK] Sécurité
[WHITE_SQUARE] Authentification endpoint /metrics
[WHITE_SQUARE] TLS/HTTPS activé
[WHITE_SQUARE] IP whitelist si nécessaire
[WHITE_SQUARE] Métriques sensibles filtrées

# [OK] Performance
[WHITE_SQUARE] Multiprocess mode configuré (Gunicorn)
[WHITE_SQUARE] Registry optimisé
[WHITE_SQUARE] Collectors cachés si coûteux
[WHITE_SQUARE] Scrape interval approprié (15-60s)

# [OK] Monitoring
[WHITE_SQUARE] Alertes configurées
[WHITE_SQUARE] Dashboard Grafana créé
[WHITE_SQUARE] Recording rules définies
[WHITE_SQUARE] SLO/SLA définis

# [OK] Tests
[WHITE_SQUARE] Tests unitaires métriques
[WHITE_SQUARE] Tests intégration endpoint
[WHITE_SQUARE] Load testing
[WHITE_SQUARE] Validation format métriques

# [OK] Documentation
[WHITE_SQUARE] README avec instructions
[WHITE_SQUARE] Dashboard JSON versionné
[WHITE_SQUARE] Alertes documentées
[WHITE_SQUARE] Runbook pour incidents

# [OK] Déploiement
[WHITE_SQUARE] Docker/K8s manifests
[WHITE_SQUARE] Service discovery configuré
[WHITE_SQUARE] Prometheus scrape config
[WHITE_SQUARE] Retention data appropriée


[OK] TEMPLATES DE DASHBOARDS

# === Dashboard Application Overview ===
{
  "title": "Application Overview",
  "panels": [
    {
      "title": "Request Rate",
      "targets": [{
        "expr": "sum(rate(http_requests_total[5m]))"
      }]
    },
    {
      "title": "Error Rate %",
      "targets": [{
        "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100"
      }]
    },
    {
      "title": "P95 Latency",
      "targets": [{
        "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))"
      }]
    },
    {
      "title": "Active Connections",
      "targets": [{
        "expr": "sum(http_requests_in_progress)"
      }]
    }
  ]
}

# === Dashboard RED Metrics ===
# Rate - Errors - Duration
{
  "title": "RED Metrics",
  "panels": [
    {
      "title": "Request Rate by Endpoint",
      "targets": [{
        "expr": "sum by(endpoint) (rate(http_requests_total[5m]))"
      }]
    },
    {
      "title": "Error Rate by Endpoint",
      "targets": [{
        "expr": "sum by(endpoint) (rate(http_requests_total{status=~\"5..\"}[5m]))"
      }]
    },
    {
      "title": "Request Duration (P50, P95, P99)",
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "P50"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "P95"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "P99"
        }
      ]
    }
  ]
}


[OK] SCRIPTS UTILES

# === Script collecte métriques système ===
#!/usr/bin/env python3
"""
Collecte métriques système et expose via Prometheus
Usage: python system_metrics.py --port 8000
"""

import argparse
import psutil
import time
from prometheus_client import start_http_server, Gauge, CollectorRegistry

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--port', type=int, default=8000)
    parser.add_argument('--interval', type=int, default=15)
    args = parser.parse_args()
    
    registry = CollectorRegistry()
    
    # Métriques
    cpu = Gauge('system_cpu_percent', 'CPU', registry=registry)
    mem = Gauge('system_memory_percent', 'Memory', registry=registry)
    disk = Gauge('system_disk_percent', 'Disk', ['mount'], registry=registry)
    
    start_http_server(args.port, registry=registry)
    print(f"Metrics server started on port {args.port}")
    
    while True:
        cpu.set(psutil.cpu_percent(interval=1))
        mem.set(psutil.virtual_memory().percent)
        
        for part in psutil.disk_partitions():
            try:
                usage = psutil.disk_usage(part.mountpoint)
                disk.labels(mount=part.mountpoint).set(usage.percent)
            except:
                pass
        
        time.sleep(args.interval)

if __name__ == '__main__':
    main()

# === Script test endpoint metrics ===
#!/usr/bin/env python3
"""
Test endpoint Prometheus
Usage: python test_metrics.py http://localhost:5000/metrics
"""

import sys
import requests

def test_metrics(url):
    print(f"Testing {url}...")
    
    try:
        response = requests.get(url, timeout=5)
        
        print(f"[OK] Status: {response.status_code}")
        print(f"[OK] Content-Type: {response.headers.get('Content-Type')}")
        print(f"[OK] Size: {len(response.content)} bytes")
        
        # Parse metrics
        lines = response.text.split('\n')
        metrics = {}
        
        for line in lines:
            if line and not line.startswith('#'):
                parts = line.split()
                if len(parts) >= 2:
                    metric_name = parts[0].split('{')[0]
                    metrics[metric_name] = metrics.get(metric_name, 0) + 1
        
        print(f"\n[OK] Found {len(metrics)} unique metrics:")
        for name, count in sorted(metrics.items())[:10]:
            print(f"  - {name}: {count} series")
        
        if len(metrics) > 10:
            print(f"  ... and {len(metrics) - 10} more")
        
        # Validate format
        errors = []
        for i, line in enumerate(lines):
            if line and not line.startswith('#'):
                if ' ' not in line:
                    errors.append(f"Line {i}: Missing space")
        
        if errors:
            print(f"\n[X] Format errors:")
            for error in errors[:5]:
                print(f"  - {error}")
        else:
            print("\n[OK] Format validation passed")
        
        return True
        
    except Exception as e:
        print(f"[X] Error: {e}")
        return False

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print("Usage: python test_metrics.py <metrics_url>")
        sys.exit(1)
    
    success = test_metrics(sys.argv[1])
    sys.exit(0 if success else 1)

# === Script génération dashboard Grafana ===
#!/usr/bin/env python3
"""
Génère dashboard Grafana depuis template
Usage: python generate_dashboard.py --app myapp --output dashboard.json
"""

import json
import argparse

def generate_dashboard(app_name, metrics_prefix=''):
    dashboard = {
        "dashboard": {
            "title": f"{app_name} - Application Metrics",
            "tags": [app_name, "prometheus"],
            "timezone": "browser",
            "refresh": "30s",
            "time": {
                "from": "now-1h",
                "to": "now"
            },
            "panels": [
                # Row 1: Overview Stats
                {
                    "id": 1,
                    "title": "Total Requests/s",
                    "type": "stat",
                    "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4},
                    "targets": [{
                        "expr": f"sum(rate({metrics_prefix}http_requests_total[5m]))",
                        "refId": "A"
                    }],
                    "options": {
                        "graphMode": "area",
                        "colorMode": "value",
                        "unit": "reqps"
                    }
                },
                {
                    "id": 2,
                    "title": "Error Rate %",
                    "type": "stat",
                    "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4},
                    "targets": [{
                        "expr": f"sum(rate({metrics_prefix}http_requests_total{{status=~\"5..\"}}[5m])) / sum(rate({metrics_prefix}http_requests_total[5m])) * 100",
                        "refId": "A"
                    }],
                    "options": {
                        "colorMode": "value",
                        "unit": "percent",
                        "thresholds": {
                            "mode": "absolute",
                            "steps": [
                                {"value": 0, "color": "green"},
                                {"value": 1, "color": "yellow"},
                                {"value": 5, "color": "red"}
                            ]
                        }
                    }
                },
                {
                    "id": 3,
                    "title": "P95 Latency",
                    "type": "stat",
                    "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4},
                    "targets": [{
                        "expr": f"histogram_quantile(0.95, sum(rate({metrics_prefix}http_request_duration_seconds_bucket[5m])) by (le))",
                        "refId": "A"
                    }],
                    "options": {
                        "graphMode": "area",
                        "colorMode": "value",
                        "unit": "s"
                    }
                },
                {
                    "id": 4,
                    "title": "Active Requests",
                    "type": "stat",
                    "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4},
                    "targets": [{
                        "expr": f"sum({metrics_prefix}http_requests_in_progress)",
                        "refId": "A"
                    }],
                    "options": {
                        "colorMode": "value"
                    }
                },
                
                # Row 2: Detailed Graphs
                {
                    "id": 5,
                    "title": "Request Rate by Endpoint",
                    "type": "timeseries",
                    "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8},
                    "targets": [{
                        "expr": f"sum by(endpoint) (rate({metrics_prefix}http_requests_total[5m]))",
                        "legendFormat": "{{{{endpoint}}}}",
                        "refId": "A"
                    }],
                    "options": {
                        "legend": {"displayMode": "table", "calcs": ["mean", "max"]}
                    }
                },
                {
                    "id": 6,
                    "title": "Latency Percentiles",
                    "type": "timeseries",
                    "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8},
                    "targets": [
                        {
                            "expr": f"histogram_quantile(0.50, sum(rate({metrics_prefix}http_request_duration_seconds_bucket[5m])) by (le))",
                            "legendFormat": "P50",
                            "refId": "A"
                        },
                        {
                            "expr": f"histogram_quantile(0.95, sum(rate({metrics_prefix}http_request_duration_seconds_bucket[5m])) by (le))",
                            "legendFormat": "P95",
                            "refId": "B"
                        },
                        {
                            "expr": f"histogram_quantile(0.99, sum(rate({metrics_prefix}http_request_duration_seconds_bucket[5m])) by (le))",
                            "legendFormat": "P99",
                            "refId": "C"
                        }
                    ],
                    "fieldConfig": {
                        "defaults": {
                            "unit": "s"
                        }
                    }
                },
                
                # Row 3: System Metrics
                {
                    "id": 7,
                    "title": "Memory Usage",
                    "type": "timeseries",
                    "gridPos": {"x": 0, "y": 12, "w": 8, "h": 6},
                    "targets": [{
                        "expr": f"{metrics_prefix}process_resident_memory_bytes",
                        "legendFormat": "RSS",
                        "refId": "A"
                    }],
                    "fieldConfig": {
                        "defaults": {
                            "unit": "bytes"
                        }
                    }
                },
                {
                    "id": 8,
                    "title": "CPU Usage",
                    "type": "timeseries",
                    "gridPos": {"x": 8, "y": 12, "w": 8, "h": 6},
                    "targets": [{
                        "expr": f"rate({metrics_prefix}process_cpu_seconds_total[5m]) * 100",
                        "legendFormat": "CPU %",
                        "refId": "A"
                    }],
                    "fieldConfig": {
                        "defaults": {
                            "unit": "percent"
                        }
                    }
                },
                {
                    "id": 9,
                    "title": "Status Code Distribution",
                    "type": "piechart",
                    "gridPos": {"x": 16, "y": 12, "w": 8, "h": 6},
                    "targets": [{
                        "expr": f"sum by(status) (rate({metrics_prefix}http_requests_total[5m]))",
                        "legendFormat": "{{{{status}}}}",
                        "refId": "A"
                    }]
                }
            ]
        },
        "overwrite": True
    }
    
    return dashboard

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--app', required=True, help='Application name')
    parser.add_argument('--prefix', default='', help='Metrics prefix')
    parser.add_argument('--output', default='dashboard.json', help='Output file')
    args = parser.parse_args()
    
    dashboard = generate_dashboard(args.app, args.prefix)
    
    with open(args.output, 'w') as f:
        json.dump(dashboard, f, indent=2)
    
    print(f"[OK] Dashboard generated: {args.output}")
    print(f"  Import in Grafana: Configuration -> Dashboards -> Import")

if __name__ == '__main__':
    main()

# === Script backup métriques ===
#!/usr/bin/env python3
"""
Backup snapshot des métriques Prometheus
Usage: python backup_metrics.py --url http://localhost:9090
"""

import argparse
import requests
import json
from datetime import datetime

def backup_metrics(prometheus_url):
    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
    output_file = f"metrics_backup_{timestamp}.json"
    
    print(f"Backing up metrics from {prometheus_url}...")
    
    # Query all metrics
    response = requests.get(
        f"{prometheus_url}/api/v1/label/__name__/values"
    )
    
    if response.status_code != 200:
        print(f"[X] Error: {response.status_code}")
        return False
    
    metric_names = response.json()['data']
    print(f"Found {len(metric_names)} metrics")
    
    # Fetch current values
    backup_data = {
        'timestamp': timestamp,
        'prometheus_url': prometheus_url,
        'metrics': {}
    }
    
    for metric in metric_names:
        try:
            response = requests.get(
                f"{prometheus_url}/api/v1/query",
                params={'query': metric}
            )
            
            if response.status_code == 200:
                result = response.json()['data']['result']
                backup_data['metrics'][metric] = result
                print(f"  [OK] {metric}: {len(result)} series")
        except Exception as e:
            print(f"  [X] {metric}: {e}")
    
    # Save to file
    with open(output_file, 'w') as f:
        json.dump(backup_data, f, indent=2)
    
    print(f"\n[OK] Backup saved: {output_file}")
    return True

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--url', default='http://localhost:9090')
    args = parser.parse_args()
    
    success = backup_metrics(args.url)
    exit(0 if success else 1)

if __name__ == '__main__':
    main()


[OK] PATTERNS AVANCÉS

# === Pattern 1: Contexte Manager pour métriques ===

from contextlib import contextmanager
from prometheus_client import Histogram, Counter
import time

class MetricsContext:
    def __init__(self, operation_name):
        self.operation = operation_name
        self.duration = Histogram(
            f'{operation_name}_duration_seconds',
            f'{operation_name} duration'
        )
        self.counter = Counter(
            f'{operation_name}_total',
            f'{operation_name} total',
            ['status']
        )
    
    @contextmanager
    def track(self):
        start = time.time()
        status = 'success'
        
        try:
            yield
        except Exception as e:
            status = 'error'
            raise
        finally:
            duration = time.time() - start
            self.duration.observe(duration)
            self.counter.labels(status=status).inc()

# Usage
db_metrics = MetricsContext('database_query')

with db_metrics.track():
    # Execute query
    result = db.execute("SELECT * FROM users")

# === Pattern 2: Décorateur métrique générique ===

from functools import wraps
from prometheus_client import Counter, Histogram
import time

def track_metrics(operation_name):
    """Décorateur pour tracker métriques automatiquement"""
    
    duration_metric = Histogram(
        f'{operation_name}_duration_seconds',
        f'Duration of {operation_name}'
    )
    
    count_metric = Counter(
        f'{operation_name}_total',
        f'Total {operation_name} calls',
        ['status']
    )
    
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            start = time.time()
            status = 'success'
            
            try:
                result = func(*args, **kwargs)
                return result
            except Exception as e:
                status = 'error'
                raise
            finally:
                duration = time.time() - start
                duration_metric.observe(duration)
                count_metric.labels(status=status).inc()
        
        return wrapper
    return decorator

# Usage
@track_metrics('process_payment')
def process_payment(amount):
    # Process...
    return True

# === Pattern 3: Métriques par classe ===

from prometheus_client import Counter, Histogram

class MetricsTrackedService:
    """Service avec métriques intégrées"""
    
    def __init__(self, service_name):
        self.service_name = service_name
        
        # Créer métriques
        self.calls = Counter(
            f'{service_name}_calls_total',
            f'{service_name} calls',
            ['method', 'status']
        )
        
        self.duration = Histogram(
            f'{service_name}_duration_seconds',
            f'{service_name} duration',
            ['method']
        )
    
    def _track_call(self, method_name):
        """Décorateur interne pour tracker méthodes"""
        def decorator(func):
            @wraps(func)
            def wrapper(*args, **kwargs):
                start = time.time()
                status = 'success'
                
                try:
                    result = func(*args, **kwargs)
                    return result
                except Exception:
                    status = 'error'
                    raise
                finally:
                    duration = time.time() - start
                    self.calls.labels(method=method_name, status=status).inc()
                    self.duration.labels(method=method_name).observe(duration)
            
            return wrapper
        return decorator
    
    def get_user(self, user_id):
        # Cette méthode sera automatiquement trackée
        pass

# Usage
class UserService(MetricsTrackedService):
    def __init__(self):
        super().__init__('user_service')
    
    @MetricsTrackedService._track_call('get_user')
    def get_user(self, user_id):
        # Implementation
        return {"id": user_id}

# === Pattern 4: Registry par environnement ===

from prometheus_client import CollectorRegistry, Counter

class EnvironmentMetrics:
    """Métriques isolées par environnement"""
    
    def __init__(self, environment):
        self.environment = environment
        self.registry = CollectorRegistry()
        
        # Créer métriques avec ce registry
        self.requests = Counter(
            'requests_total',
            'Requests',
            ['method', 'status'],
            registry=self.registry
        )
    
    def get_metrics(self):
        from prometheus_client import generate_latest
        return generate_latest(self.registry)

# Usage
prod_metrics = EnvironmentMetrics('production')
dev_metrics = EnvironmentMetrics('development')

prod_metrics.requests.labels(method='GET', status='200').inc()
dev_metrics.requests.labels(method='POST', status='201').inc()

# === Pattern 5: Métriques conditionnelles ===

class ConditionalMetrics:
    """Métriques activées/désactivées dynamiquement"""
    
    def __init__(self, enabled=True):
        self.enabled = enabled
        
        if self.enabled:
            self.counter = Counter('requests', 'Requests')
        else:
            # No-op counter
            self.counter = type('obj', (object,), {
                'inc': lambda self, *args, **kwargs: None,
                'labels': lambda self, *args, **kwargs: self
            })()
    
    def track(self, *args, **kwargs):
        if self.enabled:
            self.counter.labels(*args, **kwargs).inc()

# Usage
import os
metrics_enabled = os.getenv('METRICS_ENABLED', 'true').lower() == 'true'
metrics = ConditionalMetrics(enabled=metrics_enabled)

# === Pattern 6: Batch metrics update ===

from prometheus_client import Gauge
import time

class BatchMetricsUpdater:
    """Update métriques par batch"""
    
    def __init__(self, interval=60):
        self.interval = interval
        self.metrics = {}
        self.last_update = 0
    
    def register_metric(self, name, callback):
        """Register metric avec callback"""
        gauge = Gauge(name, f'Metric {name}')
        self.metrics[name] = (gauge, callback)
    
    def update_all(self):
        """Update toutes les métriques"""
        now = time.time()
        
        if now - self.last_update < self.interval:
            return  # Skip si trop récent
        
        for name, (gauge, callback) in self.metrics.items():
            try:
                value = callback()
                gauge.set(value)
            except Exception as e:
                print(f"Error updating {name}: {e}")
        
        self.last_update = now

# Usage
updater = BatchMetricsUpdater(interval=30)

def get_db_connections():
    return db.get_active_connections()

def get_queue_size():
    return queue.qsize()

updater.register_metric('db_connections', get_db_connections)
updater.register_metric('queue_size', get_queue_size)

# Dans une boucle ou scheduler
while True:
    updater.update_all()
    time.sleep(10)

# === Pattern 7: Métriques hiérarchiques ===

class HierarchicalMetrics:
    """Métriques organisées hiérarchiquement"""
    
    def __init__(self, namespace):
        self.namespace = namespace
        self.subsystems = {}
    
    def add_subsystem(self, name):
        """Ajouter sous-système"""
        subsystem = {
            'requests': Counter(
                'requests_total',
                'Requests',
                namespace=self.namespace,
                subsystem=name
            ),
            'duration': Histogram(
                'duration_seconds',
                'Duration',
                namespace=self.namespace,
                subsystem=name
            )
        }
        self.subsystems[name] = subsystem
        return subsystem
    
    def get_subsystem(self, name):
        if name not in self.subsystems:
            return self.add_subsystem(name)
        return self.subsystems[name]

# Usage
app_metrics = HierarchicalMetrics('myapp')

api_metrics = app_metrics.get_subsystem('api')
api_metrics['requests'].inc()

db_metrics = app_metrics.get_subsystem('database')
db_metrics['requests'].inc()

# Génère:
# myapp_api_requests_total
# myapp_api_duration_seconds
# myapp_database_requests_total
# myapp_database_duration_seconds


[OK] MIGRATION & UPGRADE

# === Migration de collectd vers Prometheus ===

# Avant (collectd)
# collectd.conf
# LoadPlugin python
# <Plugin python>
#   ModulePath "/opt/collectd"
#   Import "my_metrics"
# </Plugin>

# Après (Prometheus)
from prometheus_client import Gauge, start_http_server

cpu_usage = Gauge('cpu_usage_percent', 'CPU usage')
memory_usage = Gauge('memory_usage_bytes', 'Memory')

def collect():
    cpu_usage.set(psutil.cpu_percent())
    memory_usage.set(psutil.virtual_memory().used)

start_http_server(8000)

# === Migration de StatsD vers Prometheus ===

# Avant (StatsD)
# import statsd
# c = statsd.StatsClient('localhost', 8125)
# c.incr('requests')
# c.timing('response.time', 500)

# Après (Prometheus)
from prometheus_client import Counter, Histogram

requests_counter = Counter('requests_total', 'Requests')
response_time = Histogram('response_duration_seconds', 'Response time')

requests_counter.inc()
response_time.observe(0.5)

# === Upgrade prometheus-client ===

# Vérifier version actuelle
pip show prometheus-client

# Upgrade
pip install --upgrade prometheus-client

# Changements breaking changes v0.x -> v1.0:
# - REGISTRY devient CollectorRegistry()
# - Certains collectors renommés

# Migration code:
# Avant:
from prometheus_client import REGISTRY
counter = Counter('metric', 'desc', registry=REGISTRY)

# Après (optionnel, REGISTRY existe toujours):
from prometheus_client import CollectorRegistry, Counter
registry = CollectorRegistry()
counter = Counter('metric', 'desc', registry=registry)


[OK] TROUBLESHOOTING AVANCÉ

# === Diagnostic: Métriques manquantes ===

def diagnose_missing_metrics(metric_name):
    """Diagnostiquer pourquoi une métrique n'apparaît pas"""
    from prometheus_client import REGISTRY
    
    print(f"Diagnosing metric: {metric_name}\n")
    
    # 1. Vérifier si métrique existe dans registry
    found = False
    for collector in REGISTRY._collector_to_names.keys():
        names = REGISTRY._collector_to_names[collector]
        if metric_name in names:
            found = True
            print(f"[OK] Metric found in registry")
            break
    
    if not found:
        print(f"[X] Metric NOT found in registry")
        print("  Possible causes:")
        print("  - Metric not created yet")
        print("  - Using different registry")
        print("  - Typo in metric name")
        return
    
    # 2. Vérifier si métrique a des valeurs
    metrics_output = generate_latest().decode('utf-8')
    
    if metric_name in metrics_output:
        print(f"[OK] Metric appears in output")
        
        # Count occurrences
        count = metrics_output.count(metric_name)
        print(f"  Found {count} time series")
        
    else:
        print(f"[X] Metric NOT in output")
        print("  Possible causes:")
        print("  - No values recorded yet")
        print("  - Metric has labels but no label values set")
        print("  - Metric filtered by registry")
    
    # 3. Vérifier type métrique
    for metric in REGISTRY.collect():
        if metric.name == metric_name:
            print(f"[OK] Metric type: {metric.type}")
            print(f"  Documentation: {metric.documentation}")
            
            if hasattr(metric, 'samples'):
                print(f"  Samples: {len(metric.samples)}")
                for sample in metric.samples[:5]:
                    print(f"    - {sample.name}{sample.labels} = {sample.value}")

# Usage
diagnose_missing_metrics('http_requests_total')

# === Diagnostic: Cardinalité élevée ===

def analyze_cardinality():
    """Analyser cardinalité des métriques"""
    from prometheus_client import REGISTRY
    from collections import defaultdict
    
    cardinality = defaultdict(int)
    label_values = defaultdict(set)
    
    for metric in REGISTRY.collect():
        for sample in metric.samples:
            cardinality[metric.name] += 1
            
            # Track unique label values
            for label_key, label_value in sample.labels.items():
                label_values[f"{metric.name}:{label_key}"].add(label_value)
    
    print("=== Cardinality Analysis ===\n")
    
    # Sort by cardinality
    sorted_metrics = sorted(cardinality.items(), key=lambda x: x[1], reverse=True)
    
    print("Top 10 metrics by cardinality:")
    for metric_name, count in sorted_metrics[:10]:
        print(f"  {metric_name}: {count} time series")
        
        # Show label cardinality
        for key, values in label_values.items():
            if key.startswith(metric_name + ":"):
                label_name = key.split(":", 1)[1]
                print(f"    - {label_name}: {len(values)} unique values")
    
    # Warnings
    print("\n=== Warnings ===")
    for metric_name, count in sorted_metrics:
        if count > 1000:
            print(f"[ATTENTION] {metric_name}: {count} series (high cardinality!)")
        
        for key, values in label_values.items():
            if key.startswith(metric_name + ":"):
                if len(values) > 100:
                    label_name = key.split(":", 1)[1]
                    print(f"[ATTENTION] {metric_name}.{label_name}: {len(values)} values (high!)")

# === Diagnostic: Performance scrape ===

def benchmark_scrape():
    """Benchmark performance de génération métriques"""
    import time
    from prometheus_client import generate_latest
    
    iterations = 100
    durations = []
    
    print(f"Benchmarking scrape performance ({iterations} iterations)...\n")
    
    for i in range(iterations):
        start = time.time()
        data = generate_latest()
        duration = time.time() - start
        durations.append(duration)
    
    avg = sum(durations) / len(durations)
    p50 = sorted(durations)[len(durations) // 2]
    p95 = sorted(durations)[int(len(durations) * 0.95)]
    p99 = sorted(durations)[int(len(durations) * 0.99)]
    
    print(f"Metrics size: {len(data)} bytes")
    print(f"Average duration: {avg*1000:.2f}ms")
    print(f"P50: {p50*1000:.2f}ms")
    print(f"P95: {p95*1000:.2f}ms")
    print(f"P99: {p99*1000:.2f}ms")
    
    if avg > 0.1:
        print("\n[ATTENTION] Scrape is slow (>100ms)!")
        print("  Consider:")
        print("  - Reducing number of metrics")
        print("  - Caching expensive collectors")
        print("  - Using custom registry")


[OK] CONCLUSION

# Prometheus + Grafana = Stack monitoring moderne et puissant

# Points clés:
# [OK] 4 types métriques: Counter, Gauge, Histogram, Summary
# [OK] Labels pour dimensionner métriques (attention cardinalité!)
# [OK] Exposition via endpoint HTTP /metrics
# [OK] Prometheus scrape périodiquement
# [OK] Grafana pour visualisation
# [OK] Alerting via Alertmanager

# Best practices:
# [OK] Nommer métriques avec unité (_seconds, _bytes, _total)
# [OK] Limiter cardinalité (< 10 valeurs par label)
# [OK] Documenter métriques
# [OK] Utiliser RED (Rate, Errors, Duration) ou USE (Utilization, Saturation, Errors)
# [OK] Tester endpoint /metrics
# [OK] Monitorer les métriques elles-mêmes

# Éviter:
# [X] Labels avec cardinalité illimitée (user_id, url complète)
# [X] Calculs lourds dans collect()
# [X] Métriques dupliquées
# [X] Noms avec tirets ou espaces
# [X] Timestamps personnalisés (utiliser Pushgateway)

# Checklist déploiement:
# [WHITE_SQUARE] Métriques RED/USE implémentées
# [WHITE_SQUARE] Labels documentés et limités
# [WHITE_SQUARE] Tests unitaires écrits
# [WHITE_SQUARE] Dashboard Grafana créé
# [WHITE_SQUARE] Alertes configurées
# [WHITE_SQUARE] Documentation à jour
# [WHITE_SQUARE] Multiprocess configuré (Gunicorn)
# [WHITE_SQUARE] Sécurité endpoint /metrics
# [WHITE_SQUARE] Monitoring production testé

# Métriques essentielles à avoir:
# - http_requests_total (Counter)
# - http_request_duration_seconds (Histogram)
# - http_requests_in_progress (Gauge)
# - errors_total (Counter)
# - process_resident_memory_bytes (Gauge)
# - process_cpu_seconds_total (Counter)

# Architecture recommandée:
# Application -> prometheus_client -> /metrics endpoint
#     v
# Prometheus Server (scrape + store)
#     v
# Grafana (visualisation) + Alertmanager (alertes)

# Pour aller plus loin:
# - Service mesh metrics (Istio, Linkerd)
# - Distributed tracing (Jaeger, Zipkin)
# - Log aggregation (Loki, ELK)
# - APM complet (Datadog, New Relic)

# Happy monitoring! [GRAPHIQUE]


[OK] EXEMPLES PAR CAS D'USAGE

# === E-commerce Application ===

from prometheus_client import Counter, Histogram, Gauge

# Business metrics
orders_total = Counter('orders_total', 'Total orders', ['status'])
revenue_cents = Counter('revenue_cents_total', 'Total revenue in cents')
cart_value = Histogram(
    'cart_value_cents',
    'Shopping cart value',
    buckets=[1000, 5000, 10000, 50000, 100000, 500000]
)
active_sessions = Gauge('active_sessions', 'Active user sessions')
inventory_level = Gauge('inventory_level', 'Product inventory', ['product_id'])

# Usage
def complete_order(order):
    orders_total.labels(status='completed').inc()
    revenue_cents.inc(order.total_cents)
    cart_value.observe(order.total_cents)

# === API Gateway ===

from prometheus_client import Counter, Histogram, Gauge

# API metrics
api_requests = Counter(
    'api_requests_total',
    'API requests',
    ['service', 'method', 'endpoint', 'status']
)

api_latency = Histogram(
    'api_latency_seconds',
    'API latency',
    ['service', 'endpoint']
)

api_rate_limit = Counter(
    'api_rate_limit_exceeded_total',
    'Rate limit exceeded',
    ['client_id']
)

api_quota = Gauge(
    'api_quota_remaining',
    'Remaining API quota',
    ['client_id']
)

# === Background Worker ===

from prometheus_client import Counter, Histogram, Gauge

# Worker metrics
tasks_processed = Counter(
    'tasks_processed_total',
    'Tasks processed',
    ['task_type', 'status']
)

task_duration = Histogram(
    'task_duration_seconds',
    'Task duration',
    ['task_type'],
    buckets=[1, 5, 10, 30, 60, 300, 600]
)

queue_size = Gauge('queue_size', 'Queue size', ['queue_name'])
active_workers = Gauge('workers_active', 'Active workers')

task_retries = Counter(
    'task_retries_total',
    'Task retries',
    ['task_type']
)

# === Microservice ===

from prometheus_client import Counter, Histogram, Gauge, Info

# Service info
service_info = Info('service', 'Service information')
service_info.info({
    'version': '1.2.3',
    'environment': 'production'
})

# Inter-service calls
external_calls = Counter(
    'external_calls_total',
    'External service calls',
    ['target_service', 'operation', 'status']
)

external_call_duration = Histogram(
    'external_call_duration_seconds',
    'External call duration',
    ['target_service', 'operation']
)

circuit_breaker_state = Gauge(
    'circuit_breaker_state',
    'Circuit breaker state (0=closed, 1=open, 2=half-open)',
    ['target_service']
)

# === Machine Learning Model ===

from prometheus_client import Counter, Histogram, Gauge, Summary

# ML metrics
predictions_total = Counter(
    'ml_predictions_total',
    'Predictions made',
    ['model_version']
)

prediction_latency = Histogram(
    'ml_prediction_latency_seconds',
    'Prediction latency',
    ['model_version']
)

model_accuracy = Gauge(
    'ml_model_accuracy',
    'Model accuracy',
    ['model_version']
)

prediction_confidence = Summary(
    'ml_prediction_confidence',
    'Prediction confidence score',
    ['model_version']
)

model_load_time = Gauge(
    'ml_model_load_time_seconds',
    'Model load time'
)

# === IoT/Sensor Data ===

from prometheus_client import Gauge, Counter

# Sensor metrics
sensor_value = Gauge(
    'sensor_value',
    'Sensor reading',
    ['sensor_id', 'type']
)

sensor_errors = Counter(
    'sensor_errors_total',
    'Sensor errors',
    ['sensor_id', 'error_type']
)

sensor_last_update = Gauge(
    'sensor_last_update_timestamp',
    'Last sensor update',
    ['sensor_id']
)

# Usage
def update_sensor_data(sensor_id, value):
    sensor_value.labels(sensor_id=sensor_id, type='temperature').set(value)
    sensor_last_update.labels(sensor_id=sensor_id).set_to_current_time()

# === Batch Processing ===

from prometheus_client import Counter, Histogram, Gauge

# Batch metrics
batch_jobs = Counter(
    'batch_jobs_total',
    'Batch jobs',
    ['job_name', 'status']
)

batch_duration = Histogram(
    'batch_duration_seconds',
    'Batch job duration',
    ['job_name'],
    buckets=[60, 300, 600, 1800, 3600, 7200]
)

batch_records_processed = Counter(
    'batch_records_processed_total',
    'Records processed',
    ['job_name']
)

batch_last_success = Gauge(
    'batch_last_success_timestamp',
    'Last successful batch',
    ['job_name']
)

# Usage with Pushgateway
from prometheus_client import CollectorRegistry, push_to_gateway

registry = CollectorRegistry()
duration = Histogram('batch_duration', 'Duration', registry=registry)

with duration.time():
    # Process batch
    pass

push_to_gateway('localhost:9091', job='batch_job', registry=registry)


[OK] INTÉGRATION CI/CD

# === GitHub Actions ===

# .github/workflows/metrics.yml
name: Metrics Test

on: [push, pull_request]

jobs:
  test-metrics:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.11'
      
      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          pip install pytest requests
      
      - name: Start application
        run: |
          python app.py &
          sleep 5
      
      - name: Test metrics endpoint
        run: |
          curl -f http://localhost:5000/metrics
          python -c "
          import requests
          r = requests.get('http://localhost:5000/metrics')
          assert r.status_code == 200
          assert 'http_requests_total' in r.text
          print('[OK] Metrics endpoint OK')
          "
      
      - name: Run metrics tests
        run: pytest tests/test_metrics.py

# === GitLab CI ===

# .gitlab-ci.yml
stages:
  - test
  - deploy

test-metrics:
  stage: test
  image: python:3.11
  script:
    - pip install -r requirements.txt
    - python -m pytest tests/test_metrics.py
    - python -c "from prometheus_client import REGISTRY; print(f'Metrics: {len(list(REGISTRY.collect()))}')"
  
deploy-production:
  stage: deploy
  script:
    - docker build -t myapp:latest .
    - docker push myapp:latest
    - kubectl apply -f k8s/
  only:
    - main

# === Dockerfile multi-stage ===

# Dockerfile
FROM python:3.11-slim as builder

WORKDIR /app
COPY requirements.txt .
RUN pip install --user -r requirements.txt

FROM python:3.11-slim

WORKDIR /app

# Copy dependencies
COPY --from=builder /root/.local /root/.local
ENV PATH=/root/.local/bin:$PATH

# Copy application
COPY . .

# Create prometheus multiproc dir
RUN mkdir -p /tmp/prometheus
ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus

# Health check
HEALTHCHECK --interval=30s --timeout=3s \
  CMD python -c "import requests; requests.get('http://localhost:5000/health')" || exit 1

# Expose ports
EXPOSE 5000 8000

# Run with gunicorn
CMD ["gunicorn", \
     "--workers", "4", \
     "--bind", "0.0.0.0:5000", \
     "--timeout", "120", \
     "--access-logfile", "-", \
     "--error-logfile", "-", \
     "app:app"]


[OK] MONITORING AS CODE

# === Prometheus config as code ===

import yaml

def generate_prometheus_config(services):
    """Générer configuration Prometheus"""
    
    config = {
        'global': {
            'scrape_interval': '15s',
            'evaluation_interval': '15s'
        },
        'scrape_configs': []
    }
    
    for service in services:
        scrape_config = {
            'job_name': service['name'],
            'static_configs': [{
                'targets': service['targets'],
                'labels': service.get('labels', {})
            }]
        }
        
        if service.get('metrics_path'):
            scrape_config['metrics_path'] = service['metrics_path']
        
        config['scrape_configs'].append(scrape_config)
    
    return yaml.dump(config)

# Usage
services = [
    {
        'name': 'api',
        'targets': ['api-1:5000', 'api-2:5000'],
        'labels': {'env': 'prod', 'service': 'api'}
    },
    {
        'name': 'workers',
        'targets': ['worker-1:8000', 'worker-2:8000'],
        'labels': {'env': 'prod', 'service': 'worker'}
    }
]

config = generate_prometheus_config(services)
with open('prometheus.yml', 'w') as f:
    f.write(config)

# === Alerting rules as code ===

def generate_alert_rules(app_name, thresholds):
    """Générer règles d'alerting"""
    
    rules = {
        'groups': [{
            'name': f'{app_name}_alerts',
            'rules': []
        }]
    }
    
    # High error rate
    rules['groups'][0]['rules'].append({
        'alert': 'HighErrorRate',
        'expr': f'''
            rate(http_requests_total{{job="{app_name}",status=~"5.."}}[5m])
            /
            rate(http_requests_total{{job="{app_name}"}}[5m])
            > {thresholds['error_rate']}
        ''',
        'for': '5m',
        'labels': {'severity': 'critical'},
        'annotations': {
            'summary': f'High error rate on {app_name}',
            'description': 'Error rate is above threshold'
        }
    })
    
    # High latency
    rules['groups'][0]['rules'].append({
        'alert': 'HighLatency',
        'expr': f'''
            histogram_quantile(0.95,
                rate(http_request_duration_seconds_bucket{{job="{app_name}"}}[5m])
            ) > {thresholds['latency']}
        ''',
        'for': '10m',
        'labels': {'severity': 'warning'},
        'annotations': {
            'summary': f'High latency on {app_name}',
            'description': 'P95 latency is above threshold'
        }
    })
    
    return yaml.dump(rules)

# Usage
thresholds = {
    'error_rate': 0.05,  # 5%
    'latency': 1.0       # 1 second
}

alert_rules = generate_alert_rules('myapp', thresholds)
with open('alerts.yml', 'w') as f:
    f.write(alert_rules)

# === Grafana dashboard as code ===

def generate_grafana_dashboard(app_name, panels_config):
    """Générer dashboard Grafana programmatiquement"""
    
    dashboard = {
        "dashboard": {
            "title": f"{app_name} Dashboard",
            "panels": [],
            "refresh": "30s",
            "time": {"from": "now-1h", "to": "now"}
        }
    }
    
    panel_id = 1
    y_pos = 0
    
    for panel_config in panels_config:
        panel = {
            "id": panel_id,
            "title": panel_config['title'],
            "type": panel_config.get('type', 'timeseries'),
            "gridPos": {
                "x": 0,
                "y": y_pos,
                "w": panel_config.get('width', 24),
                "h": panel_config.get('height', 8)
            },
            "targets": [{
                "expr": panel_config['query'],
                "legendFormat": panel_config.get('legend', ''),
                "refId": "A"
            }]
        }
        
        dashboard['dashboard']['panels'].append(panel)
        panel_id += 1
        y_pos += panel_config.get('height', 8)
    
    return dashboard

# Usage
panels = [
    {
        'title': 'Request Rate',
        'query': 'sum(rate(http_requests_total[5m]))',
        'type': 'stat'
    },
    {
        'title': 'Error Rate',
        'query': 'sum(rate(http_requests_total{status=~"5.."}[5m]))',
        'type': 'stat'
    },
    {
        'title': 'Latency',
        'query': 'histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))',
        'legend': 'P95'
    }
]

dashboard = generate_grafana_dashboard('myapp', panels)

# Push to Grafana
import requests
response = requests.post(
    'http://grafana:3000/api/dashboards/db',
    json=dashboard,
    headers={'Authorization': 'Bearer YOUR_API_KEY'}
)


[OK] TERRAFORM POUR PROMETHEUS/GRAFANA

# === main.tf ===

# Provider AWS
provider "aws" {
  region = "us-east-1"
}

# Security group
resource "aws_security_group" "monitoring" {
  name = "monitoring-sg"
  
  ingress {
    from_port   = 9090
    to_port     = 9090
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  
  ingress {
    from_port   = 3000
    to_port     = 3000
    protocol    = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

# EC2 pour Prometheus
resource "aws_instance" "prometheus" {
  ami           = "ami-0c55b159cbfafe1f0"
  instance_type = "t3.medium"
  
  user_data = <<-EOF
              #!/bin/bash
              docker run -d -p 9090:9090 \
                -v /prometheus.yml:/etc/prometheus/prometheus.yml \
                prom/prometheus
              EOF
  
  tags = {
    Name = "prometheus-server"
  }
}

# EC2 pour Grafana
resource "aws_instance" "grafana" {
  ami           = "ami-0c55b159cbfafe1f0"
  instance_type = "t3.small"
  
  user_data = <<-EOF
              #!/bin/bash
              docker run -d -p 3000:3000 \
                -e "GF_SECURITY_ADMIN_PASSWORD=admin" \
                grafana/grafana
              EOF
  
  tags = {
    Name = "grafana-server"
  }
}

# Output
output "prometheus_url" {
  value = "http://${aws_instance.prometheus.public_ip}:9090"
}

output "grafana_url" {
  value = "http://${aws_instance.grafana.public_ip}:3000"
}


[OK] FIN DU CHEATSHEET

# Ce cheatsheet couvre:
# [OK] Installation et configuration
# [OK] Types de métriques (Counter, Gauge, Histogram, Summary)
# [OK] Labels et cardinalité
# [OK] Exposition des métriques
# [OK] Intégrations frameworks (Flask, Django, FastAPI, Celery)
# [OK] Multiprocess mode
# [OK] Push Gateway
# [OK] Queries PromQL
# [OK] Grafana dashboards
# [OK] Alerting
# [OK] Docker & Kubernetes
# [OK] Testing et debugging
# [OK] Performance et optimisation
# [OK] Sécurité
# [OK] Patterns avancés
# [OK] Exemples par cas d'usage
# [OK] CI/CD
# [OK] Infrastructure as Code

# Pour toute question: https://prometheus.io/docs/
# Happy monitoring! [RAPIDE][GRAPHIQUE]