# Fichier: python_cheats/cheatsheets/urllib.txt
# Python urllib - HTTP Library Standard (Built-in)


[OK] MODULES ET IMPORT


[OK] MODULES URLLIB
    # urllib est divisé en plusieurs modules
    import urllib.request   # Ouvrir et lire URLs
    import urllib.parse     # Parser et encoder URLs
    import urllib.error     # Exceptions
    import urllib.robotparser  # Parser robots.txt

[OK] IMPORT COURANTS
    from urllib.request import urlopen, Request, urlretrieve
    from urllib.parse import urlencode, urlparse, parse_qs, quote, unquote
    from urllib.error import URLError, HTTPError
    import json


[OK] REQUÊTES HTTP BASIQUES


[OK] GET REQUEST SIMPLE
    import urllib.request
    
    # Méthode la plus simple
    with urllib.request.urlopen('https://api.example.com/data') as response:
        html = response.read()
        print(html)
    
    # Lire en texte
    with urllib.request.urlopen('https://api.example.com/data') as response:
        text = response.read().decode('utf-8')
        print(text)
    
    # Sans context manager
    response = urllib.request.urlopen('https://api.example.com/data')
    data = response.read()
    response.close()

[OK] GET AVEC QUERY PARAMETERS
    from urllib.parse import urlencode
    
    # Encoder les paramètres
    params = {
        'search': 'python',
        'page': 1,
        'limit': 10
    }
    query_string = urlencode(params)
    # Résultat: 'search=python&page=1&limit=10'
    
    # URL complète
    url = f'https://api.example.com/search?{query_string}'
    
    with urllib.request.urlopen(url) as response:
        data = response.read()
    
    # Avec listes (multiple values)
    params = [('tag', 'python'), ('tag', 'urllib'), ('tag', 'api')]
    query_string = urlencode(params)
    # Résultat: 'tag=python&tag=urllib&tag=api'

[OK] POST REQUEST
    import urllib.request
    import urllib.parse
    
    # Données à envoyer
    data = {
        'username': 'john',
        'password': 'secret'
    }
    
    # Encoder en bytes
    data_encoded = urllib.parse.urlencode(data).encode('utf-8')
    
    # Créer la requête
    req = urllib.request.Request('https://api.example.com/login', 
                                  data=data_encoded,
                                  method='POST')
    
    # Envoyer
    with urllib.request.urlopen(req) as response:
        result = response.read()

[OK] POST JSON
    import urllib.request
    import json
    
    # Données JSON
    data = {
        'name': 'John Doe',
        'email': 'john@example.com'
    }
    
    # Convertir en JSON et encoder
    json_data = json.dumps(data).encode('utf-8')
    
    # Créer requête avec headers JSON
    req = urllib.request.Request(
        'https://api.example.com/user',
        data=json_data,
        headers={'Content-Type': 'application/json'},
        method='POST'
    )
    
    with urllib.request.urlopen(req) as response:
        result = json.loads(response.read().decode('utf-8'))

[OK] PUT REQUEST
    import urllib.request
    
    data = {'email': 'newemail@example.com'}
    json_data = json.dumps(data).encode('utf-8')
    
    req = urllib.request.Request(
        'https://api.example.com/user/1',
        data=json_data,
        headers={'Content-Type': 'application/json'},
        method='PUT'
    )
    
    with urllib.request.urlopen(req) as response:
        result = response.read()

[OK] DELETE REQUEST
    req = urllib.request.Request(
        'https://api.example.com/user/1',
        method='DELETE'
    )
    
    with urllib.request.urlopen(req) as response:
        result = response.read()

[OK] HEAD REQUEST
    req = urllib.request.Request(
        'https://api.example.com/resource',
        method='HEAD'
    )
    
    with urllib.request.urlopen(req) as response:
        print(response.headers)
        print(response.status)


[OK] REQUEST OBJECT ET HEADERS


[OK] CRÉER UN REQUEST OBJECT
    from urllib.request import Request
    
    # Basique
    req = Request('https://api.example.com/data')
    
    # Avec méthode
    req = Request('https://api.example.com/data', method='GET')
    
    # Avec data (POST automatique)
    data = b'key=value'
    req = Request('https://api.example.com/data', data=data)

[OK] HEADERS PERSONNALISÉS
    # Headers au moment de la création
    headers = {
        'User-Agent': 'MyApp/1.0.0',
        'Authorization': 'Bearer token123',
        'Accept': 'application/json'
    }
    
    req = urllib.request.Request(
        'https://api.example.com/data',
        headers=headers
    )
    
    # Ajouter headers après création
    req.add_header('Custom-Header', 'custom-value')
    req.add_header('Accept-Language', 'en-US')
    
    # Envoyer
    with urllib.request.urlopen(req) as response:
        data = response.read()

[OK] USER-AGENT COURANTS
    # Browser-like user agent
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    
    # Python default
    headers = {
        'User-Agent': 'Python-urllib/3.11'
    }

[OK] MODIFIER HEADERS
    req = urllib.request.Request('https://api.example.com')
    
    # Ajouter header
    req.add_header('Authorization', 'Bearer token')
    
    # Supprimer header
    req.remove_header('User-Agent')
    
    # Vérifier si header existe
    if req.has_header('Authorization'):
        print("Header exists")
    
    # Obtenir header
    auth = req.get_header('Authorization')


[OK] RESPONSE OBJECT


[OK] PROPRIÉTÉS RESPONSE
    with urllib.request.urlopen('https://api.example.com') as response:
        # Status code
        print(response.status)        # 200, 404, 500, etc.
        print(response.reason)        # 'OK', 'Not Found', etc.
        
        # URL (après redirections)
        print(response.url)
        
        # Headers
        print(response.headers)
        
        # Version HTTP
        print(response.version)       # 11 pour HTTP/1.1

[OK] LIRE LE CONTENU
    with urllib.request.urlopen('https://api.example.com') as response:
        # Tout le contenu (bytes)
        content = response.read()
        
        # Lire N bytes
        chunk = response.read(1024)
        
        # Lire ligne par ligne
        for line in response:
            print(line.decode('utf-8'))
        
        # Décoder en texte
        text = response.read().decode('utf-8')
        
        # Parser JSON
        data = json.loads(response.read().decode('utf-8'))

[OK] HEADERS DE RÉPONSE
    with urllib.request.urlopen('https://api.example.com') as response:
        # Tous les headers
        headers = response.headers
        print(headers)
        
        # Header spécifique (insensible à la casse)
        content_type = response.headers['Content-Type']
        content_type = response.headers.get('Content-Type')
        
        # Headers courants
        print(response.headers['Content-Length'])
        print(response.headers['Server'])
        print(response.headers['Date'])
        print(response.headers['Last-Modified'])
        
        # Itérer sur headers
        for name, value in response.headers.items():
            print(f"{name}: {value}")

[OK] INFO RESPONSE
    with urllib.request.urlopen('https://api.example.com') as response:
        # Obtenir objet HTTPMessage
        info = response.info()
        
        # Type de contenu
        content_type = info.get_content_type()
        
        # Charset
        charset = info.get_content_charset()
        
        # Sous-type
        subtype = info.get_content_subtype()


[OK] TIMEOUT ET OPTIONS


[OK] TIMEOUT
    import urllib.request
    from urllib.error import URLError
    
    # Timeout en secondes
    try:
        with urllib.request.urlopen('https://api.example.com', 
                                     timeout=5) as response:
            data = response.read()
    except URLError as e:
        if isinstance(e.reason, socket.timeout):
            print("Request timed out")
        else:
            print(f"URL Error: {e.reason}")

[OK] CONTEXT (SSL, PROXY, ETC.)
    import urllib.request
    import ssl
    
    # Désactiver vérification SSL ([ATTENTION] non sécurisé!)
    context = ssl._create_unverified_context()
    
    with urllib.request.urlopen('https://api.example.com', 
                                 context=context) as response:
        data = response.read()

[OK] CAFILE (CERTIFICAT PERSONNALISÉ)
    import ssl
    
    context = ssl.create_default_context(cafile='/path/to/certfile.pem')
    
    with urllib.request.urlopen('https://api.example.com', 
                                 context=context) as response:
        data = response.read()

[OK] DONNÉES ET METHOD
    # POST avec data
    data = b'key=value'
    req = urllib.request.Request('https://api.example.com', data=data)
    
    # Méthode explicite
    req = urllib.request.Request('https://api.example.com', 
                                  data=data,
                                  method='PUT')


[OK] AUTHENTIFICATION


[OK] BASIC AUTH
    import urllib.request
    import base64
    
    # Méthode 1: Header manuel
    credentials = f'{username}:{password}'
    encoded = base64.b64encode(credentials.encode('utf-8')).decode('utf-8')
    
    req = urllib.request.Request('https://api.example.com')
    req.add_header('Authorization', f'Basic {encoded}')
    
    with urllib.request.urlopen(req) as response:
        data = response.read()
    
    # Méthode 2: HTTPPasswordMgrWithDefaultRealm
    password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, 'https://api.example.com', 
                              'username', 'password')
    
    handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
    opener = urllib.request.build_opener(handler)
    
    response = opener.open('https://api.example.com')
    data = response.read()

[OK] DIGEST AUTH
    import urllib.request
    
    password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, 'https://api.example.com',
                              'username', 'password')
    
    handler = urllib.request.HTTPDigestAuthHandler(password_mgr)
    opener = urllib.request.build_opener(handler)
    
    response = opener.open('https://api.example.com')
    data = response.read()

[OK] BEARER TOKEN
    req = urllib.request.Request('https://api.example.com')
    req.add_header('Authorization', 'Bearer eyJhbGciOiJIUzI1NiIs...')
    
    with urllib.request.urlopen(req) as response:
        data = response.read()

[OK] API KEY
    # Dans headers
    req = urllib.request.Request('https://api.example.com')
    req.add_header('X-API-Key', 'your-api-key-here')
    
    # Dans query params
    from urllib.parse import urlencode
    params = urlencode({'api_key': 'your-api-key-here'})
    url = f'https://api.example.com?{params}'
    
    with urllib.request.urlopen(url) as response:
        data = response.read()


[OK] COOKIES


[OK] GÉRER LES COOKIES
    import urllib.request
    from http.cookiejar import CookieJar
    
    # Créer un cookie jar
    cookie_jar = CookieJar()
    
    # Créer un handler pour cookies
    handler = urllib.request.HTTPCookieProcessor(cookie_jar)
    
    # Créer un opener
    opener = urllib.request.build_opener(handler)
    
    # Utiliser l'opener
    response = opener.open('https://api.example.com/login')
    
    # Les cookies sont automatiquement stockés
    response = opener.open('https://api.example.com/profile')
    
    # Voir les cookies
    for cookie in cookie_jar:
        print(f"{cookie.name}: {cookie.value}")

[OK] COOKIES PERSISTANTS (FICHIER)
    from http.cookiejar import MozillaCookieJar
    
    # Créer cookie jar avec fichier
    cookie_jar = MozillaCookieJar('cookies.txt')
    
    # Charger cookies existants
    try:
        cookie_jar.load(ignore_discard=True, ignore_expires=True)
    except FileNotFoundError:
        pass
    
    handler = urllib.request.HTTPCookieProcessor(cookie_jar)
    opener = urllib.request.build_opener(handler)
    
    response = opener.open('https://api.example.com')
    
    # Sauvegarder cookies
    cookie_jar.save(ignore_discard=True, ignore_expires=True)

[OK] ENVOYER COOKIES MANUELLEMENT
    from http.cookies import SimpleCookie
    
    # Créer cookies
    cookies = SimpleCookie()
    cookies['session_id'] = '12345'
    cookies['user'] = 'john'
    
    # Ajouter au header
    req = urllib.request.Request('https://api.example.com')
    req.add_header('Cookie', cookies.output(header='', sep='; '))
    
    with urllib.request.urlopen(req) as response:
        data = response.read()


[OK] REDIRECTIONS


[OK] GÉRER LES REDIRECTIONS
    # Par défaut, urllib suit automatiquement les redirections
    
    with urllib.request.urlopen('https://example.com/redirect') as response:
        print(response.url)  # URL finale après redirections
        print(response.status)

[OK] DÉSACTIVER REDIRECTIONS
    import urllib.request
    
    class NoRedirectHandler(urllib.request.HTTPRedirectHandler):
        def redirect_request(self, req, fp, code, msg, headers, newurl):
            return None
    
    opener = urllib.request.build_opener(NoRedirectHandler)
    
    try:
        response = opener.open('https://example.com/redirect')
    except urllib.error.HTTPError as e:
        if e.code in (301, 302, 303, 307, 308):
            print(f"Redirect to: {e.headers['Location']}")

[OK] LIMITER NOMBRE DE REDIRECTIONS
    class LimitedRedirectHandler(urllib.request.HTTPRedirectHandler):
        max_redirects = 3
        
        def redirect_request(self, req, fp, code, msg, headers, newurl):
            if hasattr(req, 'redirect_count'):
                req.redirect_count += 1
                if req.redirect_count >= self.max_redirects:
                    raise urllib.error.HTTPError(
                        req.full_url, code, 
                        f"Too many redirects ({req.redirect_count})",
                        headers, fp
                    )
            else:
                req.redirect_count = 1
            
            return super().redirect_request(req, fp, code, msg, headers, newurl)


[OK] PROXIES


[OK] PROXY HTTP/HTTPS
    import urllib.request
    
    # Définir proxy
    proxy = urllib.request.ProxyHandler({
        'http': 'http://proxy.example.com:3128',
        'https': 'http://proxy.example.com:3128'
    })
    
    opener = urllib.request.build_opener(proxy)
    
    response = opener.open('https://api.example.com')
    data = response.read()

[OK] PROXY AVEC AUTHENTIFICATION
    proxy = urllib.request.ProxyHandler({
        'http': 'http://user:pass@proxy.example.com:3128',
        'https': 'http://user:pass@proxy.example.com:3128'
    })
    
    opener = urllib.request.build_opener(proxy)
    response = opener.open('https://api.example.com')

[OK] PROXY DEPUIS ENVIRONNEMENT
    # Urllib utilise automatiquement HTTP_PROXY et HTTPS_PROXY
    import os
    
    os.environ['HTTP_PROXY'] = 'http://proxy.example.com:3128'
    os.environ['HTTPS_PROXY'] = 'http://proxy.example.com:3128'
    
    with urllib.request.urlopen('https://api.example.com') as response:
        data = response.read()

[OK] DÉSACTIVER PROXY
    # Créer un handler sans proxy
    no_proxy = urllib.request.ProxyHandler({})
    opener = urllib.request.build_opener(no_proxy)
    
    response = opener.open('https://api.example.com')


[OK] DOWNLOAD DE FICHIERS


[OK] DOWNLOAD SIMPLE
    import urllib.request
    
    # Méthode 1: urlretrieve
    urllib.request.urlretrieve('https://example.com/file.pdf', 
                               'downloaded.pdf')
    
    # Méthode 2: urlopen
    with urllib.request.urlopen('https://example.com/file.pdf') as response:
        with open('downloaded.pdf', 'wb') as f:
            f.write(response.read())

[OK] DOWNLOAD AVEC PROGRESS
    def download_progress(block_num, block_size, total_size):
        downloaded = block_num * block_size
        percent = (downloaded / total_size) * 100
        print(f'Downloaded: {percent:.2f}%', end='\r')
    
    urllib.request.urlretrieve(
        'https://example.com/file.zip',
        'file.zip',
        reporthook=download_progress
    )

[OK] DOWNLOAD PAR CHUNKS
    url = 'https://example.com/bigfile.zip'
    
    with urllib.request.urlopen(url) as response:
        with open('bigfile.zip', 'wb') as f:
            while True:
                chunk = response.read(8192)  # 8KB chunks
                if not chunk:
                    break
                f.write(chunk)

[OK] DOWNLOAD AVEC RESUME
    import os
    import urllib.request
    
    url = 'https://example.com/file.zip'
    filename = 'file.zip'
    
    # Vérifier taille déjà téléchargée
    if os.path.exists(filename):
        resume_pos = os.path.getsize(filename)
        req = urllib.request.Request(url)
        req.add_header('Range', f'bytes={resume_pos}-')
    else:
        resume_pos = 0
        req = urllib.request.Request(url)
    
    with urllib.request.urlopen(req) as response:
        mode = 'ab' if resume_pos > 0 else 'wb'
        with open(filename, mode) as f:
            while True:
                chunk = response.read(8192)
                if not chunk:
                    break
                f.write(chunk)


[OK] UPLOAD DE FICHIERS


[OK] UPLOAD SIMPLE (MULTIPART)
    import urllib.request
    import mimetypes
    import io
    
    def encode_multipart_formdata(fields, files):
        """Encoder des fichiers en multipart/form-data"""
        boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW'
        body = io.BytesIO()
        
        # Champs normaux
        for key, value in fields.items():
            body.write(f'--{boundary}\r\n'.encode())
            body.write(f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode())
            body.write(f'{value}\r\n'.encode())
        
        # Fichiers
        for key, filepath in files.items():
            filename = filepath.split('/')[-1]
            mimetype = mimetypes.guess_type(filepath)[0] or 'application/octet-stream'
            
            with open(filepath, 'rb') as f:
                file_content = f.read()
            
            body.write(f'--{boundary}\r\n'.encode())
            body.write(f'Content-Disposition: form-data; name="{key}"; filename="{filename}"\r\n'.encode())
            body.write(f'Content-Type: {mimetype}\r\n\r\n'.encode())
            body.write(file_content)
            body.write(b'\r\n')
        
        body.write(f'--{boundary}--\r\n'.encode())
        
        content_type = f'multipart/form-data; boundary={boundary}'
        return content_type, body.getvalue()
    
    # Utilisation
    fields = {'title': 'My Document', 'description': 'Important file'}
    files = {'file': 'document.pdf'}
    
    content_type, body = encode_multipart_formdata(fields, files)
    
    req = urllib.request.Request('https://api.example.com/upload')
    req.add_header('Content-Type', content_type)
    req.data = body
    
    with urllib.request.urlopen(req) as response:
        result = response.read()

[OK] UPLOAD BINAIRE SIMPLE
    # Upload fichier en POST simple
    with open('file.bin', 'rb') as f:
        data = f.read()
    
    req = urllib.request.Request(
        'https://api.example.com/upload',
        data=data,
        headers={'Content-Type': 'application/octet-stream'}
    )
    
    with urllib.request.urlopen(req) as response:
        result = response.read()


[OK] URL PARSING ET ENCODING


[OK] PARSER UNE URL
    from urllib.parse import urlparse, parse_qs
    
    url = 'https://user:pass@example.com:8080/path/to/page?key=value&foo=bar#section'
    
    parsed = urlparse(url)
    
    print(parsed.scheme)    # 'https'
    print(parsed.netloc)    # 'user:pass@example.com:8080'
    print(parsed.hostname)  # 'example.com'
    print(parsed.port)      # 8080
    print(parsed.username)  # 'user'
    print(parsed.password)  # 'pass'
    print(parsed.path)      # '/path/to/page'
    print(parsed.query)     # 'key=value&foo=bar'
    print(parsed.fragment)  # 'section'
    
    # Parser query string
    params = parse_qs(parsed.query)
    print(params)  # {'key': ['value'], 'foo': ['bar']}

[OK] ENCODER/DECODER URL
    from urllib.parse import quote, unquote, quote_plus, unquote_plus
    
    # Quote (encoder caractères spéciaux)
    text = 'Hello World! @#$%'
    encoded = quote(text)
    # Résultat: 'Hello%20World%21%20%40%23%24%25'
    
    # Quote plus (espace = +)
    encoded = quote_plus(text)
    # Résultat: 'Hello+World%21+%40%23%24%25'
    
    # Unquote (décoder)
    decoded = unquote('Hello%20World%21')
    # Résultat: 'Hello World!'
    
    decoded = unquote_plus('Hello+World%21')
    # Résultat: 'Hello World!'

[OK] CONSTRUIRE QUERY STRING
    from urllib.parse import urlencode
    
    # Dictionnaire simple
    params = {'search': 'python', 'page': 1, 'limit': 10}
    query = urlencode(params)
    # Résultat: 'search=python&page=1&limit=10'
    
    # Liste de tuples (pour doublons)
    params = [('tag', 'python'), ('tag', 'urllib'), ('tag', 'api')]
    query = urlencode(params)
    # Résultat: 'tag=python&tag=urllib&tag=api'
    
    # Avec quote_plus
    params = {'message': 'Hello World!'}
    query = urlencode(params, quote_via=quote_plus)
    # Résultat: 'message=Hello+World%21'

[OK] JOINDRE URL
    from urllib.parse import urljoin
    
    base = 'https://example.com/path/to/page'
    
    # Chemin relatif
    url = urljoin(base, 'subpage')
    # Résultat: 'https://example.com/path/to/subpage'
    
    # Chemin absolu
    url = urljoin(base, '/other/page')
    # Résultat: 'https://example.com/other/page'
    
    # URL complète (remplace)
    url = urljoin(base, 'https://other.com/page')
    # Résultat: 'https://other.com/page'

[OK] SPLIT QUERY
    from urllib.parse import parse_qs, parse_qsl
    
    query = 'key=value&foo=bar&tag=python&tag=urllib'
    
    # parse_qs: dict avec listes
    params = parse_qs(query)
    # {'key': ['value'], 'foo': ['bar'], 'tag': ['python', 'urllib']}
    
    # parse_qsl: liste de tuples
    params = parse_qsl(query)
    # [('key', 'value'), ('foo', 'bar'), ('tag', 'python'), ('tag', 'urllib')]


[OK] GESTION D'ERREURS


[OK] ERREURS BASIQUES
    import urllib.request
    from urllib.error import URLError, HTTPError
    
    try:
        with urllib.request.urlopen('https://api.example.com') as response:
            data = response.read()
    
    except HTTPError as e:
        # Erreur HTTP (4xx, 5xx)
        print(f"HTTP Error: {e.code} - {e.reason}")
        print(f"URL: {e.url}")
        print(f"Headers: {e.headers}")
        print(f"Body: {e.read()}")
    
    except URLError as e:
        # Erreur de connexion, timeout, DNS, etc.
        print(f"URL Error: {e.reason}")
    
    except Exception as e:
        print(f"Other error: {e}")

[OK] VÉRIFICATION DU STATUS CODE
    try:
        with urllib.request.urlopen('https://api.example.com') as response:
            if response.status == 200:
                data = response.read()
            else:
                print(f"Unexpected status: {response.status}")
    
    except HTTPError as e:
        if e.code == 404:
            print("Resource not found")
        elif e.code == 401:
            print("Unauthorized")
        elif e.code >= 500:
            print("Server error")

[OK] TIMEOUT ERROR
    import socket
    from urllib.error import URLError
    
    try:
        with urllib.request.urlopen('https://api.example.com', 
                                     timeout=5) as response:
            data = response.read()
    
    except URLError as e:
        if isinstance(e.reason, socket.timeout):
            print("Connection timed out")
        else:
            print(f"URL Error: {e.reason}")

[OK] RETRY MANUEL
    import time
    from urllib.error import URLError
    
    def urlopen_with_retry(url, max_retries=3, delay=1):
        """Retry avec backoff exponentiel"""
        for attempt in range(max_retries):
            try:
                with urllib.request.urlopen(url, timeout=5) as response:
                    return response.read()
            
            except (URLError, socket.timeout) as e:
                if attempt < max_retries - 1:
                    print(f"Attempt {attempt + 1} failed. Retrying in {delay}s...")
                    time.sleep(delay)
                    delay *= 2  # Exponential backoff
                else:
                    raise
    
    data = urlopen_with_retry('https://api.example.com')


[OK] OPENER PERSONNALISÉ


[OK] BUILD OPENER
    import urllib.request
    
    # Créer des handlers
    cookie_handler = urllib.request.HTTPCookieProcessor()
    redirect_handler = urllib.request.HTTPRedirectHandler()
    
    # Build opener avec handlers
    opener = urllib.request.build_opener(cookie_handler, redirect_handler)
    
    # Utiliser opener
    response = opener.open('https://api.example.com')
    data = response.read()
    
    # Installer comme opener par défaut
    urllib.request.install_opener(opener)
    
    # Maintenant urlopen utilise cet opener
    with urllib.request.urlopen('https://api.example.com') as response:
        data = response.read()

[OK] HANDLERS DISPONIBLES
    # HTTPHandler: HTTP basique
    http_handler = urllib.request.HTTPHandler()
    
    # HTTPSHandler: HTTPS
    https_handler = urllib.request.HTTPSHandler()
    
    # HTTPCookieProcessor: Gestion cookies
    from http.cookiejar import CookieJar
    cookie_jar = CookieJar()
    cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
    
    # HTTPBasicAuthHandler: Authentification basique
    password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
    
    # HTTPDigestAuthHandler: Authentification digest
    digest_handler = urllib.request.HTTPDigestAuthHandler(password_mgr)
    
    # ProxyHandler: Support proxy
    proxy_handler = urllib.request.ProxyHandler({
        'http': 'http://proxy.example.com:3128'
    })
    
    # HTTPRedirectHandler: Gestion redirections
    redirect_handler = urllib.request.HTTPRedirectHandler()
    
    # HTTPErrorProcessor: Gestion erreurs HTTP
    error_handler = urllib.request.HTTPErrorProcessor()

[OK] CUSTOM HANDLER
    class CustomHandler(urllib.request.BaseHandler):
        """Handler personnalisé"""
        
        def http_request(self, request):
            # Modifier la requête avant l'envoi
            request.add_header('Custom-Header', 'value')
            return request
        
        def http_response(self, request, response):
            # Traiter la réponse
            print(f"Response status: {response.status}")
            return response
        
        https_request = http_request
        https_response = http_response
    
    # Utiliser le handler
    opener = urllib.request.build_opener(CustomHandler())
    response = opener.open('https://api.example.com')

[OK] OPENER AVEC MULTIPLES HANDLERS
    from http.cookiejar import CookieJar
    
    # Configurer plusieurs handlers
    cookie_jar = CookieJar()
    cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
    
    proxy = urllib.request.ProxyHandler({
        'http': 'http://proxy.example.com:3128'
    })
    
    password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, 'https://api.example.com',
                              'user', 'pass')
    auth_handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
    
    # Combiner tous les handlers
    opener = urllib.request.build_opener(
        cookie_handler,
        proxy,
        auth_handler
    )
    
    response = opener.open('https://api.example.com')


[OK] ROBOTS.TXT PARSER


[OK] VÉRIFIER ROBOTS.TXT
    from urllib.robotparser import RobotFileParser
    
    # Créer parser
    rp = RobotFileParser()
    rp.set_url('https://example.com/robots.txt')
    rp.read()
    
    # Vérifier si URL peut être crawlée
    user_agent = 'MyBot/1.0'
    
    if rp.can_fetch(user_agent, 'https://example.com/page'):
        print("Can crawl this page")
    else:
        print("Cannot crawl this page")
    
    # Obtenir crawl delay
    delay = rp.crawl_delay(user_agent)
    if delay:
        print(f"Crawl delay: {delay} seconds")
    
    # Obtenir request rate
    rate = rp.request_rate(user_agent)
    if rate:
        print(f"Request rate: {rate.requests}/{rate.seconds} seconds")

[OK] PARSER MANUEL
    import urllib.request
    
    # Télécharger robots.txt
    with urllib.request.urlopen('https://example.com/robots.txt') as response:
        robots_txt = response.read().decode('utf-8')
    
    # Parser manuel
    rp = RobotFileParser()
    rp.parse(robots_txt.splitlines())
    
    # Vérifier permissions
    if rp.can_fetch('*', 'https://example.com/page'):
        print("Allowed")


[OK] PATTERNS ET BONNES PRATIQUES


[OK] CLIENT API SIMPLE
    import urllib.request
    import json
    from urllib.parse import urlencode
    from urllib.error import HTTPError, URLError
    
    class APIClient:
        def __init__(self, base_url, api_key=None, timeout=10):
            self.base_url = base_url.rstrip('/')
            self.timeout = timeout
            
            # Créer opener avec cookies
            from http.cookiejar import CookieJar
            cookie_jar = CookieJar()
            cookie_handler = urllib.request.HTTPCookieProcessor(cookie_jar)
            self.opener = urllib.request.build_opener(cookie_handler)
            
            # Headers par défaut
            self.default_headers = {}
            if api_key:
                self.default_headers['X-API-Key'] = api_key
        
        def _request(self, method, endpoint, data=None, params=None, headers=None):
            # Construire URL
            url = f"{self.base_url}/{endpoint.lstrip('/')}"
            if params:
                url = f"{url}?{urlencode(params)}"
            
            # Préparer headers
            req_headers = self.default_headers.copy()
            if headers:
                req_headers.update(headers)
            
            # Préparer data
            if data:
                if isinstance(data, dict):
                    data = json.dumps(data).encode('utf-8')
                    req_headers['Content-Type'] = 'application/json'
                elif isinstance(data, str):
                    data = data.encode('utf-8')
            
            # Créer requête
            req = urllib.request.Request(
                url,
                data=data,
                headers=req_headers,
                method=method
            )
            
            # Envoyer
            try:
                with self.opener.open(req, timeout=self.timeout) as response:
                    content = response.read()
                    
                    # Parser JSON si applicable
                    if 'application/json' in response.headers.get('Content-Type', ''):
                        return json.loads(content.decode('utf-8'))
                    return content.decode('utf-8')
            
            except HTTPError as e:
                print(f"HTTP Error {e.code}: {e.reason}")
                raise
            except URLError as e:
                print(f"URL Error: {e.reason}")
                raise
        
        def get(self, endpoint, params=None, headers=None):
            return self._request('GET', endpoint, params=params, headers=headers)
        
        def post(self, endpoint, data=None, headers=None):
            return self._request('POST', endpoint, data=data, headers=headers)
        
        def put(self, endpoint, data=None, headers=None):
            return self._request('PUT', endpoint, data=data, headers=headers)
        
        def delete(self, endpoint, headers=None):
            return self._request('DELETE', endpoint, headers=headers)
    
    # Utilisation
    client = APIClient('https://api.example.com', api_key='secret')
    
    # GET
    users = client.get('users', params={'page': 1})
    
    # POST
    new_user = client.post('users', data={'name': 'John', 'email': 'john@example.com'})
    
    # PUT
    updated = client.put('users/1', data={'email': 'newemail@example.com'})
    
    # DELETE
    client.delete('users/1')

[OK] RATE LIMITING
    import time
    from collections import deque
    
    class RateLimitedOpener:
        def __init__(self, requests_per_second=10):
            self.requests_per_second = requests_per_second
            self.requests = deque()
            self.opener = urllib.request.build_opener()
        
        def open(self, *args, **kwargs):
            now = time.time()
            
            # Nettoyer requêtes anciennes (> 1 seconde)
            while self.requests and self.requests[0] < now - 1:
                self.requests.popleft()
            
            # Attendre si limite atteinte
            if len(self.requests) >= self.requests_per_second:
                sleep_time = 1 - (now - self.requests[0])
                if sleep_time > 0:
                    time.sleep(sleep_time)
            
            # Enregistrer requête
            self.requests.append(time.time())
            
            # Exécuter
            return self.opener.open(*args, **kwargs)
    
    # Utilisation
    opener = RateLimitedOpener(requests_per_second=5)
    
    for i in range(20):
        with opener.open('https://api.example.com/data') as response:
            data = response.read()
            print(f"Request {i+1} completed")

[OK] SESSION AVEC CONTEXT MANAGER
    class URLSession:
        """Session réutilisable avec cookies et headers"""
        
        def __init__(self, headers=None):
            from http.cookiejar import CookieJar
            
            self.cookie_jar = CookieJar()
            cookie_handler = urllib.request.HTTPCookieProcessor(self.cookie_jar)
            self.opener = urllib.request.build_opener(cookie_handler)
            
            self.default_headers = headers or {}
        
        def __enter__(self):
            return self
        
        def __exit__(self, *args):
            # Cleanup si nécessaire
            pass
        
        def request(self, url, method='GET', data=None, headers=None):
            req_headers = self.default_headers.copy()
            if headers:
                req_headers.update(headers)
            
            if data and isinstance(data, dict):
                data = json.dumps(data).encode('utf-8')
                req_headers['Content-Type'] = 'application/json'
            
            req = urllib.request.Request(
                url,
                data=data,
                headers=req_headers,
                method=method
            )
            
            with self.opener.open(req) as response:
                return response.read().decode('utf-8')
        
        def get(self, url, **kwargs):
            return self.request(url, method='GET', **kwargs)
        
        def post(self, url, **kwargs):
            return self.request(url, method='POST', **kwargs)
    
    # Utilisation
    with URLSession(headers={'User-Agent': 'MyApp/1.0'}) as session:
        # Login
        session.post('https://api.example.com/login',
                    data={'user': 'john', 'pass': 'secret'})
        
        # Les cookies sont persistés
        profile = session.get('https://api.example.com/profile')
        print(profile)

[OK] DOWNLOAD MANAGER
    import os
    import hashlib
    from pathlib import Path
    
    class DownloadManager:
        def __init__(self, download_dir='downloads'):
            self.download_dir = Path(download_dir)
            self.download_dir.mkdir(exist_ok=True)
        
        def download(self, url, filename=None, chunk_size=8192, resume=True):
            """Download avec resume et progress"""
            
            # Déterminer filename
            if not filename:
                filename = url.split('/')[-1] or 'download'
            
            filepath = self.download_dir / filename
            
            # Vérifier resume
            resume_pos = 0
            if resume and filepath.exists():
                resume_pos = filepath.stat().st_size
            
            # Créer requête
            req = urllib.request.Request(url)
            if resume_pos > 0:
                req.add_header('Range', f'bytes={resume_pos}-')
            
            # Download
            try:
                with urllib.request.urlopen(req) as response:
                    total_size = int(response.headers.get('Content-Length', 0))
                    if resume_pos > 0:
                        total_size += resume_pos
                    
                    mode = 'ab' if resume_pos > 0 else 'wb'
                    
                    with open(filepath, mode) as f:
                        downloaded = resume_pos
                        
                        while True:
                            chunk = response.read(chunk_size)
                            if not chunk:
                                break
                            
                            f.write(chunk)
                            downloaded += len(chunk)
                            
                            # Progress
                            if total_size > 0:
                                percent = (downloaded / total_size) * 100
                                print(f'\rDownload: {percent:.1f}%', end='')
                    
                    print()  # Nouvelle ligne
                    return filepath
            
            except Exception as e:
                print(f"\nDownload failed: {e}")
                raise
        
        def verify_checksum(self, filepath, expected_md5):
            """Vérifier checksum MD5"""
            md5 = hashlib.md5()
            
            with open(filepath, 'rb') as f:
                while chunk := f.read(8192):
                    md5.update(chunk)
            
            return md5.hexdigest() == expected_md5
    
    # Utilisation
    manager = DownloadManager(download_dir='downloads')
    
    filepath = manager.download('https://example.com/file.zip')
    print(f"Downloaded to: {filepath}")
    
    # Avec vérification
    if manager.verify_checksum(filepath, 'expected_md5_hash'):
        print("Checksum verified!")


[OK] COMPARAISON URLLIB VS REQUESTS


[OK] DIFFÉRENCES PRINCIPALES
    # URLLIB (Built-in)
    # [OK] Avantages:
    # - Inclus dans Python (pas d'installation)
    # - Léger et rapide
    # - Bon pour scripts simples
    
    # [X] Inconvénients:
    # - API plus verbeuse
    # - Moins de fonctionnalités
    # - Gestion manuelle de beaucoup de choses
    
    # REQUESTS (Externe)
    # [OK] Avantages:
    # - API simple et élégante
    # - Beaucoup de fonctionnalités
    # - Meilleure gestion des sessions
    # - Retry automatique possible
    
    # [X] Inconvénients:
    # - Dépendance externe
    # - Plus lourd

[OK] EXEMPLES COMPARATIFS
    # GET simple
    # ═══════════════════════════════════════════════
    
    # urllib
    import urllib.request
    with urllib.request.urlopen('https://api.example.com') as response:
        data = response.read().decode('utf-8')
    
    # requests
    import requests
    response = requests.get('https://api.example.com')
    data = response.text
    
    
    # POST JSON
    # ═══════════════════════════════════════════════
    
    # urllib
    import urllib.request
    import json
    
    data = {'key': 'value'}
    json_data = json.dumps(data).encode('utf-8')
    req = urllib.request.Request(
        'https://api.example.com',
        data=json_data,
        headers={'Content-Type': 'application/json'}
    )
    with urllib.request.urlopen(req) as response:
        result = response.read()
    
    # requests
    import requests
    data = {'key': 'value'}
    response = requests.post('https://api.example.com', json=data)
    result = response.json()
    
    
    # Authentification
    # ═══════════════════════════════════════════════
    
    # urllib
    import urllib.request
    import base64
    
    credentials = base64.b64encode(b'user:pass').decode('utf-8')
    req = urllib.request.Request('https://api.example.com')
    req.add_header('Authorization', f'Basic {credentials}')
    with urllib.request.urlopen(req) as response:
        data = response.read()
    
    # requests
    import requests
    response = requests.get('https://api.example.com',
                           auth=('user', 'pass'))
    
    
    # Sessions
    # ═══════════════════════════════════════════════
    
    # urllib
    from http.cookiejar import CookieJar
    import urllib.request
    
    cookie_jar = CookieJar()
    handler = urllib.request.HTTPCookieProcessor(cookie_jar)
    opener = urllib.request.build_opener(handler)
    
    response1 = opener.open('https://api.example.com/login')
    response2 = opener.open('https://api.example.com/profile')
    
    # requests
    import requests
    session = requests.Session()
    response1 = session.get('https://api.example.com/login')
    response2 = session.get('https://api.example.com/profile')

[OK] QUAND UTILISER URLLIB
    # [OK] Utilisez urllib si:
    # - Vous ne voulez pas de dépendances externes
    # - Script simple avec peu de requêtes
    # - Environnement restreint (pas de pip)
    # - Performance critique (urllib est plus léger)
    # - Manipulation avancée d'URLs (urllib.parse)

[OK] QUAND UTILISER REQUESTS
    # [OK] Utilisez requests si:
    # - Vous faites beaucoup de requêtes HTTP
    # - Vous avez besoin de sessions complexes
    # - Vous voulez une API simple et lisible
    # - Vous utilisez déjà d'autres packages externes
    # - Vous avez besoin de retry automatique


[OK] TIPS ET ASTUCES


[OK] HEADERS USER-AGENT RÉALISTES
    # Éviter blocage par user-agent
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }

[OK] RESPECTER ROBOTS.TXT
    from urllib.robotparser import RobotFileParser
    import time
    
    def polite_crawler(url, user_agent='MyBot/1.0'):
        # Vérifier robots.txt
        rp = RobotFileParser()
        domain = urllib.parse.urlparse(url).netloc
        rp.set_url(f'https://{domain}/robots.txt')
        rp.read()
        
        if not rp.can_fetch(user_agent, url):
            print(f"Not allowed to crawl: {url}")
            return None
        
        # Respecter crawl delay
        delay = rp.crawl_delay(user_agent)
        if delay:
            time.sleep(delay)
        
        # Crawler
        req = urllib.request.Request(url)
        req.add_header('User-Agent', user_agent)
        
        with urllib.request.urlopen(req) as response:
            return response.read()

[OK] GESTION ENCODAGE
    # Détecter et utiliser le bon encodage
    with urllib.request.urlopen('https://example.com') as response:
        # Obtenir encoding depuis headers
        encoding = response.headers.get_content_charset('utf-8')
        
        # Décoder avec bon encoding
        text = response.read().decode(encoding)
        
        # Alternative: utiliser chardet pour détection
        # pip install chardet
        import chardet
        
        content = response.read()
        detected = chardet.detect(content)
        text = content.decode(detected['encoding'])

[OK] CACHE SIMPLE
    import os
    import hashlib
    import pickle
    from pathlib import Path
    
    class SimpleCache:
        def __init__(self, cache_dir='.cache', ttl=3600):
            self.cache_dir = Path(cache_dir)
            self.cache_dir.mkdir(exist_ok=True)
            self.ttl = ttl
        
        def _get_cache_path(self, url):
            url_hash = hashlib.md5(url.encode()).hexdigest()
            return self.cache_dir / f"{url_hash}.cache"
        
        def get(self, url):
            cache_path = self._get_cache_path(url)
            
            if not cache_path.exists():
                return None
            
            # Vérifier TTL
            age = time.time() - cache_path.stat().st_mtime
            if age > self.ttl:
                cache_path.unlink()
                return None
            
            with open(cache_path, 'rb') as f:
                return pickle.load(f)
        
        def set(self, url, data):
            cache_path = self._get_cache_path(url)
            with open(cache_path, 'wb') as f:
                pickle.dump(data, f)
        
        def urlopen(self, url):
            # Vérifier cache
            cached = self.get(url)
            if cached:
                return cached
            
            # Fetcher et cacher
            with urllib.request.urlopen(url) as response:
                data = response.read()
                self.set(url, data)
                return data
    
    # Utilisation
    cache = SimpleCache(ttl=3600)  # 1 heure
    data = cache.urlopen('https://api.example.com/data')

[OK] LOGGING DES REQUÊTES
    import logging
    
    # Activer logging urllib
    logging.basicConfig(level=logging.DEBUG)
    
    # Créer logger
    logger = logging.getLogger('urllib3')
    logger.setLevel(logging.DEBUG)
    
    # Handler avec détails
    handler = logging.StreamHandler()
    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    )
    handler.setFormatter(formatter)
    logger.addHandler(handler)

[OK] CONNEXION PERSISTANTE
    # urllib réutilise automatiquement les connexions HTTP/1.1
    # avec keep-alive, mais pour plus de contrôle:
    
    opener = urllib.request.build_opener()
    
    # Faire plusieurs requêtes avec même opener
    for url in urls:
        response = opener.open(url)
        data = response.read()
        # Connexion réutilisée automatiquement


# FIN DU CHEATSHEET