# DATA SCIENCE CHEATSHEET COMPLÈTE
# Fonctionnalités complémentaires à NumPy et Pandas
# Guide de référence exhaustif avec commentaires explicatifs


"""
=============================================================================
[GRAPHIQUE] TABLE DES MATIÈRES
=============================================================================
1.  VISUALISATION DE DONNÉES (Matplotlib, Seaborn, Plotly)
2.  MACHINE LEARNING (Scikit-learn)
3.  DEEP LEARNING (TensorFlow, PyTorch bases)
4.  TRAITEMENT DU LANGAGE NATUREL (NLP)
5.  TRAITEMENT D'IMAGES
6.  SÉRIES TEMPORELLES
7.  STATISTIQUES AVANCÉES
8.  OPTIMISATION ET RECHERCHE D'HYPERPARAMÈTRES
9.  FEATURE ENGINEERING
10. RÉDUCTION DE DIMENSIONNALITÉ
11. CLUSTERING
12. SYSTÈMES DE RECOMMANDATION
13. VALIDATION ET MÉTRIQUES
14. GESTION DES DONNÉES DÉSÉQUILIBRÉES
15. PIPELINES ET AUTOMATISATION
16. BIG DATA (PySpark, Dask)
17. WEB SCRAPING
18. APIs ET REQUÊTES
19. DATABASES SQL/NoSQL
20. DÉPLOIEMENT DE MODÈLES
=============================================================================
"""


[OK] 1. VISUALISATION DE DONNÉES


"""
La visualisation est essentielle pour comprendre les données et communiquer
les résultats. Trois bibliothèques principales dominent en Python.
"""

# --- MATPLOTLIB (Base de la visualisation) ---
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib import cm

# Configuration globale
plt.style.use('seaborn-v0_8')  # Style prédéfini
plt.rcParams['figure.figsize'] = (10, 6)  # Taille par défaut
plt.rcParams['font.size'] = 12  # Taille de police

# Graphique linéaire simple
x = [1, 2, 3, 4, 5]
y = [2, 4, 6, 8, 10]
plt.plot(x, y, marker='o', linestyle='--', color='blue', label='Données')
plt.xlabel('Axe X')
plt.ylabel('Axe Y')
plt.title('Graphique Linéaire')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

# Subplots (plusieurs graphiques)
fig, axes = plt.subplots(2, 2, figsize=(12, 10))
axes[0, 0].plot(x, y)  # Graphique en haut à gauche
axes[0, 1].scatter(x, y)  # Graphique en haut à droite
axes[1, 0].bar(x, y)  # Graphique en bas à gauche
axes[1, 1].hist(y, bins=5)  # Graphique en bas à droite
plt.tight_layout()  # Ajuster l'espacement
plt.show()

# Graphiques avancés
# Histogramme avec densité
plt.hist(data, bins=30, density=True, alpha=0.7, edgecolor='black')
plt.xlabel('Valeurs')
plt.ylabel('Fréquence')

# Scatter plot avec couleurs et tailles
sizes = [20, 50, 100, 200, 500]
colors = ['red', 'blue', 'green', 'yellow', 'purple']
plt.scatter(x, y, s=sizes, c=colors, alpha=0.5)

# Boxplot (boîte à moustaches)
data = [np.random.normal(0, std, 100) for std in range(1, 5)]
plt.boxplot(data, labels=['A', 'B', 'C', 'D'])

# Heatmap (carte de chaleur)
matrix = np.random.rand(10, 10)
plt.imshow(matrix, cmap='hot', interpolation='nearest')
plt.colorbar()

# Sauvegarder une figure
plt.savefig('mon_graphique.png', dpi=300, bbox_inches='tight')


# --- SEABORN (Visualisation statistique) ---
import seaborn as sns

# Configuration Seaborn
sns.set_theme(style="darkgrid")  # Style du thème
sns.set_palette("husl")  # Palette de couleurs

# Distribution plot (distribution avec densité)
sns.histplot(data, kde=True, bins=30)  # kde=True ajoute la courbe de densité

# Pairplot (matrice de scatter plots)
# Très utile pour explorer les relations entre variables
iris = sns.load_dataset('iris')
sns.pairplot(iris, hue='species')  # Coloré par espèce

# Heatmap de corrélation
correlation_matrix = iris.corr()
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0)

# Violin plot (combinaison boxplot + distribution)
sns.violinplot(x='species', y='sepal_length', data=iris)

# Catplot (graphiques catégoriels)
sns.catplot(x='species', y='sepal_length', kind='box', data=iris)
# kind peut être: 'strip', 'swarm', 'box', 'violin', 'boxen', 'point', 'bar'

# Regression plot (avec droite de régression)
sns.regplot(x='sepal_length', y='sepal_width', data=iris)

# Joint plot (scatter + distributions marginales)
sns.jointplot(x='sepal_length', y='sepal_width', data=iris, kind='reg')
# kind: 'scatter', 'kde', 'hex', 'reg'

# Count plot (comptage de catégories)
sns.countplot(x='species', data=iris)


# --- PLOTLY (Graphiques interactifs) ---
import plotly.express as px
import plotly.graph_objects as go

# Scatter plot interactif
fig = px.scatter(iris, x='sepal_length', y='sepal_width', 
                 color='species', size='petal_length',
                 hover_data=['petal_width'])
fig.show()  # Ouvre dans le navigateur

# Line plot interactif
fig = px.line(df, x='date', y='value', title='Évolution temporelle')

# Bar chart interactif
fig = px.bar(df, x='category', y='count', color='subcategory')

# 3D Scatter
fig = px.scatter_3d(iris, x='sepal_length', y='sepal_width', z='petal_length',
                    color='species')

# Carte choroplèthe (géographique)
fig = px.choropleth(df, locations='country_code', color='value',
                    hover_name='country')

# Graphique personnalisé avec Graph Objects
fig = go.Figure()
fig.add_trace(go.Scatter(x=x, y=y, mode='lines+markers', name='Série 1'))
fig.update_layout(title='Mon Graphique', xaxis_title='X', yaxis_title='Y')


[OK] 2. MACHINE LEARNING (Scikit-learn)


"""
Scikit-learn est LA bibliothèque de référence pour le Machine Learning
classique en Python. Elle offre des implémentations optimisées et cohérentes.
"""

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.metrics import accuracy_score, classification_report

# --- PRÉPARATION DES DONNÉES ---

# Split train/test (division données d'entraînement/test)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, 
    test_size=0.2,      # 20% pour le test
    random_state=42,    # Reproductibilité
    stratify=y          # Garde les mêmes proportions de classes
)

# Normalisation (mettre les features sur la même échelle)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)  # Apprend et transforme
X_test_scaled = scaler.transform(X_test)        # Transforme seulement

# Encodage de variables catégorielles
from sklearn.preprocessing import LabelEncoder, OneHotEncoder

# Label Encoding (pour variables ordinales)
le = LabelEncoder()
y_encoded = le.fit_transform(y)  # ['A', 'B', 'C'] -> [0, 1, 2]

# One-Hot Encoding (pour variables nominales)
from sklearn.preprocessing import OneHotEncoder
ohe = OneHotEncoder(sparse=False)
X_encoded = ohe.fit_transform(X_categorical)


# --- MODÈLES DE CLASSIFICATION ---

# 1. Régression Logistique
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(C=1.0, max_iter=1000)  # C: régularisation
model.fit(X_train, y_train)
predictions = model.predict(X_test)
probabilities = model.predict_proba(X_test)  # Probabilités

# 2. Decision Tree (Arbre de décision)
from sklearn.tree import DecisionTreeClassifier
model = DecisionTreeClassifier(max_depth=5, min_samples_split=10)
model.fit(X_train, y_train)

# 3. Random Forest (Forêt aléatoire)
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(
    n_estimators=100,      # Nombre d'arbres
    max_depth=10,
    min_samples_split=5,
    random_state=42
)
model.fit(X_train, y_train)
# Feature importance (importance des variables)
importances = model.feature_importances_

# 4. Gradient Boosting
from sklearn.ensemble import GradientBoostingClassifier
model = GradientBoostingClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=3
)

# 5. XGBoost (externe à sklearn mais très utilisé)
import xgboost as xgb
model = xgb.XGBClassifier(
    n_estimators=100,
    learning_rate=0.1,
    max_depth=5,
    use_label_encoder=False
)

# 6. Support Vector Machine (SVM)
from sklearn.svm import SVC
model = SVC(kernel='rbf', C=1.0, gamma='scale')
# kernel: 'linear', 'poly', 'rbf', 'sigmoid'

# 7. K-Nearest Neighbors (KNN)
from sklearn.neighbors import KNeighborsClassifier
model = KNeighborsClassifier(n_neighbors=5, weights='distance')

# 8. Naive Bayes
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()


# --- MODÈLES DE RÉGRESSION ---

# 1. Régression Linéaire
from sklearn.linear_model import LinearRegression
model = LinearRegression()
model.fit(X_train, y_train)
coefficients = model.coef_        # Coefficients
intercept = model.intercept_       # Ordonnée à l'origine

# 2. Ridge Regression (régularisation L2)
from sklearn.linear_model import Ridge
model = Ridge(alpha=1.0)  # alpha: force de régularisation

# 3. Lasso Regression (régularisation L1)
from sklearn.linear_model import Lasso
model = Lasso(alpha=1.0)

# 4. ElasticNet (combine L1 et L2)
from sklearn.linear_model import ElasticNet
model = ElasticNet(alpha=1.0, l1_ratio=0.5)

# 5. Polynomial Regression
from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures(degree=2)
X_poly = poly.fit_transform(X)
model = LinearRegression().fit(X_poly, y)

# 6. Decision Tree Regressor
from sklearn.tree import DecisionTreeRegressor
model = DecisionTreeRegressor(max_depth=5)

# 7. Random Forest Regressor
from sklearn.ensemble import RandomForestRegressor
model = RandomForestRegressor(n_estimators=100)


# --- VALIDATION CROISÉE ---

# Cross-validation (validation croisée k-fold)
from sklearn.model_selection import cross_val_score
scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
# cv=5 : 5 plis (folds)
print(f"Accuracy: {scores.mean():.2f} (+/- {scores.std():.2f})")

# Stratified K-Fold (garde les proportions de classes)
from sklearn.model_selection import StratifiedKFold
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
for train_idx, val_idx in skf.split(X, y):
    X_train, X_val = X[train_idx], X[val_idx]
    y_train, y_val = y[train_idx], y[val_idx]


# --- GRID SEARCH (Recherche d'hyperparamètres) ---

from sklearn.model_selection import GridSearchCV

# Définir la grille de paramètres
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 7, 10],
    'min_samples_split': [2, 5, 10]
}

# Recherche exhaustive
grid_search = GridSearchCV(
    RandomForestClassifier(),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,  # Utilise tous les cores
    verbose=2
)
grid_search.fit(X_train, y_train)

# Meilleurs paramètres
best_params = grid_search.best_params_
best_model = grid_search.best_estimator_


# --- RANDOM SEARCH (Plus rapide que Grid Search) ---

from sklearn.model_selection import RandomizedSearchCV

# Distributions de paramètres
param_distributions = {
    'n_estimators': [50, 100, 200, 300],
    'max_depth': [3, 5, 7, 10, None],
    'min_samples_split': [2, 5, 10, 20]
}

random_search = RandomizedSearchCV(
    RandomForestClassifier(),
    param_distributions,
    n_iter=20,      # 20 combinaisons aléatoires
    cv=5,
    random_state=42,
    n_jobs=-1
)


[OK] 3. MÉTRIQUES D'ÉVALUATION


"""
Choisir la bonne métrique est crucial. Elle doit correspondre à votre
objectif business et aux caractéristiques de vos données.
"""

from sklearn.metrics import *

# --- CLASSIFICATION ---

# Matrice de confusion
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
cm = confusion_matrix(y_test, predictions)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()

# Rapport de classification complet
report = classification_report(y_test, predictions)
print(report)  # Affiche precision, recall, f1-score par classe

# Accuracy (exactitude globale)
accuracy = accuracy_score(y_test, predictions)

# Precision (précision)
# Proportion de vrais positifs parmi les prédictions positives
precision = precision_score(y_test, predictions, average='weighted')

# Recall (rappel / sensibilité)
# Proportion de vrais positifs identifiés parmi tous les positifs réels
recall = recall_score(y_test, predictions, average='weighted')

# F1-Score (moyenne harmonique de precision et recall)
f1 = f1_score(y_test, predictions, average='weighted')

# ROC-AUC (Area Under the ROC Curve)
# Mesure la capacité du modèle à discriminer les classes
from sklearn.metrics import roc_auc_score, roc_curve
roc_auc = roc_auc_score(y_test, probabilities[:, 1])

# Courbe ROC
fpr, tpr, thresholds = roc_curve(y_test, probabilities[:, 1])
plt.plot(fpr, tpr, label=f'ROC curve (AUC = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], 'k--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')

# Log Loss (perte logarithmique)
logloss = log_loss(y_test, probabilities)

# Cohen's Kappa (accord inter-annotateurs)
from sklearn.metrics import cohen_kappa_score
kappa = cohen_kappa_score(y_test, predictions)


# --- RÉGRESSION ---

from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Mean Squared Error (erreur quadratique moyenne)
mse = mean_squared_error(y_test, predictions)

# Root Mean Squared Error
rmse = np.sqrt(mse)

# Mean Absolute Error (erreur absolue moyenne)
mae = mean_absolute_error(y_test, predictions)

# R² Score (coefficient de détermination)
# Mesure la proportion de variance expliquée
r2 = r2_score(y_test, predictions)

# Mean Absolute Percentage Error
def mape(y_true, y_pred):
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100


[OK] 4. FEATURE ENGINEERING (Ingénierie des features)


"""
Le Feature Engineering consiste à créer, transformer et sélectionner
des variables pour améliorer les performances du modèle.
"""

# --- CRÉATION DE FEATURES ---

# Features temporelles depuis datetime
df['year'] = df['date'].dt.year
df['month'] = df['date'].dt.month
df['day'] = df['date'].dt.day
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['is_weekend'] = df['dayofweek'].isin([5, 6]).astype(int)

# Features cycliques (pour capturer la nature cyclique du temps)
df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)
df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)

# Interactions entre features
df['feature_interaction'] = df['feature1'] * df['feature2']

# Binning (discrétisation)
df['age_group'] = pd.cut(df['age'], bins=[0, 18, 35, 50, 100],
                         labels=['Youth', 'Adult', 'Middle', 'Senior'])

# Aggregations groupées
df['mean_by_group'] = df.groupby('category')['value'].transform('mean')

# Lag features (pour séries temporelles)
df['value_lag1'] = df['value'].shift(1)
df['value_lag7'] = df['value'].shift(7)

# Rolling features (moyennes mobiles)
df['rolling_mean_7'] = df['value'].rolling(window=7).mean()
df['rolling_std_7'] = df['value'].rolling(window=7).std()


# --- SÉLECTION DE FEATURES ---

# 1. Variance Threshold (supprimer features à faible variance)
from sklearn.feature_selection import VarianceThreshold
selector = VarianceThreshold(threshold=0.1)
X_selected = selector.fit_transform(X)

# 2. SelectKBest (sélectionner k meilleures features)
from sklearn.feature_selection import SelectKBest, f_classif
selector = SelectKBest(f_classif, k=10)
X_selected = selector.fit_transform(X, y)
selected_features = X.columns[selector.get_support()]

# 3. Recursive Feature Elimination (RFE)
from sklearn.feature_selection import RFE
model = RandomForestClassifier()
rfe = RFE(model, n_features_to_select=10)
X_selected = rfe.fit_transform(X, y)

# 4. Feature Importance depuis modèles
model = RandomForestClassifier()
model.fit(X, y)
importances = pd.DataFrame({
    'feature': X.columns,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)

# 5. Correlation-based selection
correlation_matrix = X.corr().abs()
upper_triangle = correlation_matrix.where(
    np.triu(np.ones(correlation_matrix.shape), k=1).astype(bool)
)
to_drop = [col for col in upper_triangle.columns 
           if any(upper_triangle[col] > 0.95)]


# --- TRANSFORMATION DE FEATURES ---

# Log transformation (pour distributions asymétriques)
df['log_feature'] = np.log1p(df['feature'])  # log(1 + x)

# Box-Cox transformation
from scipy.stats import boxcox
transformed, lambda_param = boxcox(df['feature'] + 1)

# Power transformation (Yeo-Johnson)
from sklearn.preprocessing import PowerTransformer
pt = PowerTransformer(method='yeo-johnson')
df_transformed = pt.fit_transform(df)


[OK] 5. RÉDUCTION DE DIMENSIONNALITÉ


"""
Réduire le nombre de dimensions tout en préservant l'information importante.
Utile pour visualisation, performance, et éviter l'overfitting.
"""

# --- PCA (Principal Component Analysis) ---
from sklearn.decomposition import PCA

# Réduire à n composantes
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)

# Variance expliquée par chaque composante
explained_variance = pca.explained_variance_ratio_

# Choisir le nombre de composantes (garder 95% de variance)
pca = PCA(n_components=0.95)
X_pca = pca.fit_transform(X_scaled)

# Visualiser
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=y)
plt.xlabel('First Principal Component')
plt.ylabel('Second Principal Component')


# --- t-SNE (t-Distributed Stochastic Neighbor Embedding) ---
from sklearn.manifold import TSNE

# Pour visualisation (surtout en 2D/3D)
tsne = TSNE(n_components=2, perplexity=30, random_state=42)
X_tsne = tsne.fit_transform(X_scaled)
# Note: t-SNE ne peut pas transformer de nouvelles données

plt.scatter(X_tsne[:, 0], X_tsne[:, 1], c=y)


# --- UMAP (Uniform Manifold Approximation and Projection) ---
import umap

# Alternative moderne à t-SNE (plus rapide, préserve mieux la structure)
reducer = umap.UMAP(n_components=2, random_state=42)
X_umap = reducer.fit_transform(X_scaled)


# --- Truncated SVD (pour matrices sparse) ---
from sklearn.decomposition import TruncatedSVD

svd = TruncatedSVD(n_components=50)
X_reduced = svd.fit_transform(X_sparse)


# --- Linear Discriminant Analysis (LDA) ---
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# Réduction supervisée (utilise les labels)
lda = LinearDiscriminantAnalysis(n_components=2)
X_lda = lda.fit_transform(X, y)


[OK] 6. CLUSTERING (Apprentissage non supervisé)


"""
Clustering: regrouper des données similaires sans labels.
Utile pour segmentation client, détection d'anomalies, etc.
"""

# --- K-Means ---
from sklearn.cluster import KMeans

# Clustering basique
kmeans = KMeans(n_clusters=3, random_state=42)
clusters = kmeans.fit_predict(X)

# Centres des clusters
centers = kmeans.cluster_centers_

# Méthode du coude (elbow method) pour choisir k
inertias = []
for k in range(1, 11):
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(X)
    inertias.append(kmeans.inertia_)
plt.plot(range(1, 11), inertias, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')


# --- Hierarchical Clustering (clustering hiérarchique) ---
from sklearn.cluster import AgglomerativeClustering
from scipy.cluster.hierarchy import dendrogram, linkage

# Créer le modèle
hierarchical = AgglomerativeClustering(n_clusters=3)
clusters = hierarchical.fit_predict(X)

# Dendrogramme
linkage_matrix = linkage(X, method='ward')
dendrogram(linkage_matrix)


# --- DBSCAN (Density-Based Spatial Clustering) ---
from sklearn.cluster import DBSCAN

# Bon pour formes complexes et détection d'outliers
dbscan = DBSCAN(eps=0.5, min_samples=5)
clusters = dbscan.fit_predict(X)
# -1 indique les outliers


# --- Gaussian Mixture Models (GMM) ---
from sklearn.mixture import GaussianMixture

# Clustering probabiliste
gmm = GaussianMixture(n_components=3, random_state=42)
clusters = gmm.fit_predict(X)
probabilities = gmm.predict_proba(X)


# --- Métriques de clustering ---
from sklearn.metrics import silhouette_score, davies_bouldin_score

# Silhouette Score (entre -1 et 1, plus haut = mieux)
silhouette = silhouette_score(X, clusters)

# Davies-Bouldin Index (plus bas = mieux)
db_index = davies_bouldin_score(X, clusters)


[OK] 7. TRAITEMENT DU LANGAGE NATUREL (NLP)


"""
NLP: traiter et analyser du texte. De la préparation basique au deep learning.
"""

import re
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer, WordNetLemmatizer

# Télécharger ressources NLTK (une fois)
# nltk.download('punkt')
# nltk.download('stopwords')
# nltk.download('wordnet')

# --- PRÉTRAITEMENT DE TEXTE ---

def preprocess_text(text):
    """Pipeline complet de prétraitement"""
    # Minuscules
    text = text.lower()
    
    # Supprimer URLs
    text = re.sub(r'http\S+|www\S+', '', text)
    
    # Supprimer mentions et hashtags
    text = re.sub(r'@\w+|#\w+', '', text)
    
    # Garder seulement lettres et espaces
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    
    # Tokenization
    tokens = word_tokenize(text)
    
    # Supprimer stopwords
    stop_words = set(stopwords.words('english'))
    tokens = [t for t in tokens if t not in stop_words]
    
    # Stemming (racine des mots)
    stemmer = PorterStemmer()
    tokens = [stemmer.stem(t) for t in tokens]
    
    return ' '.join(tokens)

# Appliquer
df['text_clean'] = df['text'].apply(preprocess_text)


# --- VECTORISATION (Convertir texte en nombres) ---

# 1. Bag of Words (sac de mots)
from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer(max_features=1000)  # Garder 1000 mots
X_bow = vectorizer.fit_transform(texts)
vocabulary = vectorizer.get_feature_names_out()

# 2. TF-IDF (Term Frequency-Inverse Document Frequency)
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(max_features=1000, ngram_range=(1, 2))
X_tfidf = tfidf.fit_transform(texts)
# ngram_range=(1,2) : unigrammes et bigrammes

# 3. Word2Vec (embeddings)
from gensim.models import Word2Vec

# Tokeniser d'abord
tokenized_texts = [text.split() for text in texts]

# Entraîner Word2Vec
model = Word2Vec(
    sentences=tokenized_texts,
    vector_size=100,    # Dimension des vecteurs
    window=5,           # Contexte
    min_count=2,        # Fréquence minimale
    workers=4
)

# Obtenir vecteur d'un mot
vector = model.wv['word']

# Mots similaires
similar_words = model.wv.most_similar('word', topn=10)


# --- ANALYSE DE SENTIMENT ---

from textblob import TextBlob

def get_sentiment(text):
    """Retourne polarité (-1 à 1) et subjectivité (0 à 1)"""
    blob = TextBlob(text)
    return blob.sentiment.polarity, blob.sentiment.subjectivity

df['polarity'], df['subjectivity'] = zip(*df['text'].map(get_sentiment))


# --- NAMED ENTITY RECOGNITION (NER) ---

import spacy

# Charger modèle (télécharger d'abord: python -m spacy download en_core_web_sm)
nlp = spacy.load('en_core_web_sm')

doc = nlp("Apple Inc. is headquartered in Cupertino, California.")

# Extraire entités nommées
for entity in doc.ents:
    print(entity.text, entity.label_)
# Apple Inc. -> ORG
# Cupertino -> GPE (Geo-Political Entity)
# California -> GPE


# --- TOPIC MODELING (LDA) ---

from sklearn.decomposition import LatentDirichletAllocation

# Utiliser CountVectorizer d'abord
vectorizer = CountVectorizer(max_features=1000, stop_words='english')
X_bow = vectorizer.fit_transform(texts)

# LDA pour trouver topics
lda = LatentDirichletAllocation(n_components=5, random_state=42)
lda.fit(X_bow)

# Afficher les mots par topic
feature_names = vectorizer.get_feature_names_out()
for topic_idx, topic in enumerate(lda.components_):
    top_words = [feature_names[i] for i in topic.argsort()[-10:]]
    print(f"Topic {topic_idx}: {', '.join(top_words)}")


[OK] 8. TRAITEMENT D'IMAGES


"""
Traitement et analyse d'images: de la manipulation basique au deep learning.
"""

from PIL import Image
import cv2
from skimage import io, filters, transform

# --- CHARGEMENT ET MANIPULATION D'IMAGES ---

# Avec PIL
img = Image.open('image.jpg')
img_resized = img.resize((224, 224))
img_rotated = img.rotate(45)
img_gray = img.convert('L')  # Convertir en niveaux de gris
img.save('output.jpg')

# Avec OpenCV
img = cv2.imread('image.jpg')
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # BGR -> RGB
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
cv2.imwrite('output.jpg', img)

# Avec scikit-image
img = io.imread('image.jpg')
img_resized = transform.resize(img, (224, 224))


# --- PREPROCESSING D'IMAGES ---

# Normalisation (0-1)
img_normalized = img / 255.0

# Standardisation
mean = img.mean()
std = img.std()
img_standardized = (img - mean) / std

# Redimensionnement avec ratio préservé
def resize_with_aspect_ratio(img, target_size):
    h, w = img.shape[:2]
    ratio = target_size / max(h, w)
    new_size = (int(w * ratio), int(h * ratio))
    return cv2.resize(img, new_size)


# --- FILTRES ET TRANSFORMATIONS ---

# Flou gaussien
blurred = cv2.GaussianBlur(img, (5, 5), 0)

# Détection de contours (Canny)
edges = cv2.Canny(img_gray, 100, 200)

# Détection de contours (Sobel)
from skimage import filters
edges_sobel = filters.sobel(img_gray)

# Seuillage (thresholding)
_, binary = cv2.threshold(img_gray, 127, 255, cv2.THRESH_BINARY)

# Seuillage adaptatif
binary_adaptive = cv2.adaptiveThreshold(
    img_gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, 
    cv2.THRESH_BINARY, 11, 2
)

# Dilatation et érosion
kernel = np.ones((5, 5), np.uint8)
dilated = cv2.dilate(binary, kernel, iterations=1)
eroded = cv2.erode(binary, kernel, iterations=1)


# --- DATA AUGMENTATION POUR IMAGES ---

from tensorflow.keras.preprocessing.image import ImageDataGenerator

# Générateur avec augmentations
datagen = ImageDataGenerator(
    rotation_range=20,           # Rotation jusqu'à 20°
    width_shift_range=0.2,       # Décalage horizontal
    height_shift_range=0.2,      # Décalage vertical
    horizontal_flip=True,        # Miroir horizontal
    zoom_range=0.2,              # Zoom
    shear_range=0.2,             # Cisaillement
    fill_mode='nearest'          # Remplissage des pixels
)

# Générer images augmentées
for batch in datagen.flow(images, batch_size=32):
    # Utiliser le batch
    break


# --- DÉTECTION D'OBJETS (avec pré-entraîné) ---

# Charger modèle pré-entraîné (exemple avec MobileNet)
from tensorflow.keras.applications import MobileNetV2
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input, decode_predictions

model = MobileNetV2(weights='imagenet')

# Préparer image
img = Image.open('image.jpg').resize((224, 224))
img_array = np.array(img)
img_array = np.expand_dims(img_array, axis=0)
img_array = preprocess_input(img_array)

# Prédire
predictions = model.predict(img_array)
decoded = decode_predictions(predictions, top=3)[0]


# --- SEGMENTATION D'IMAGE ---

from sklearn.cluster import KMeans

# Segmentation par couleur (K-means)
img_flat = img.reshape((-1, 3))
kmeans = KMeans(n_clusters=5, random_state=42)
labels = kmeans.fit_predict(img_flat)
segmented = kmeans.cluster_centers_[labels].reshape(img.shape)


[OK] 9. SÉRIES TEMPORELLES


"""
Analyse et prévision de données temporelles: finance, météo, IoT, etc.
"""

import pandas as pd
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.holtwinters import ExponentialSmoothing

# --- PRÉPARATION DES SÉRIES TEMPORELLES ---

# Créer index datetime
df['date'] = pd.to_datetime(df['date'])
df.set_index('date', inplace=True)
df = df.sort_index()

# Rééchantillonner (resampling)
df_daily = df.resample('D').mean()      # Moyenne quotidienne
df_weekly = df.resample('W').sum()      # Somme hebdomadaire
df_monthly = df.resample('M').mean()    # Moyenne mensuelle

# Interpolation des valeurs manquantes
df_interpolated = df.interpolate(method='linear')
df_interpolated = df.interpolate(method='time')  # Basé sur le temps


# --- DÉCOMPOSITION ---

# Décomposition additive: Y = Tendance + Saisonnalité + Résidu
decomposition = seasonal_decompose(df['value'], model='additive', period=12)
trend = decomposition.trend
seasonal = decomposition.seasonal
residual = decomposition.resid

# Visualiser
fig, axes = plt.subplots(4, 1, figsize=(12, 10))
decomposition.observed.plot(ax=axes[0], title='Observed')
decomposition.trend.plot(ax=axes[1], title='Trend')
decomposition.seasonal.plot(ax=axes[2], title='Seasonal')
decomposition.resid.plot(ax=axes[3], title='Residual')


# --- TEST DE STATIONNARITÉ ---

# Test Augmented Dickey-Fuller
def adf_test(series):
    """Test de stationnarité"""
    result = adfuller(series.dropna())
    print(f'ADF Statistic: {result[0]}')
    print(f'p-value: {result[1]}')
    print('Critical Values:')
    for key, value in result[4].items():
        print(f'\t{key}: {value}')
    
    if result[1] <= 0.05:
        print("Série stationnaire")
    else:
        print("Série non-stationnaire")

adf_test(df['value'])

# Rendre stationnaire par différenciation
df['value_diff'] = df['value'].diff()
df['value_diff2'] = df['value_diff'].diff()  # Différenciation d'ordre 2


# --- AUTOCORRÉLATION ---

from statsmodels.graphics.tsaplots import plot_acf, plot_pacf

# ACF (AutoCorrelation Function)
plot_acf(df['value'].dropna(), lags=40)

# PACF (Partial AutoCorrelation Function)
plot_pacf(df['value'].dropna(), lags=40)


# --- MODÈLES DE PRÉVISION ---

# 1. Moving Average (Moyenne Mobile)
df['MA_7'] = df['value'].rolling(window=7).mean()
df['MA_30'] = df['value'].rolling(window=30).mean()

# 2. Exponential Smoothing (Lissage exponentiel)
from statsmodels.tsa.holtwinters import SimpleExpSmoothing

model = SimpleExpSmoothing(df['value'])
fitted_model = model.fit()
forecast = fitted_model.forecast(steps=30)

# 3. Holt-Winters (Triple Exponential Smoothing)
model = ExponentialSmoothing(
    df['value'], 
    seasonal_periods=12,    # Période saisonnière
    trend='add',            # Tendance additive
    seasonal='add'          # Saisonnalité additive
)
fitted_model = model.fit()
forecast = fitted_model.forecast(steps=30)

# 4. ARIMA (AutoRegressive Integrated Moving Average)
# ARIMA(p, d, q) où:
# p = ordre autorégressif
# d = degré de différenciation
# q = ordre moyenne mobile

model = ARIMA(df['value'], order=(1, 1, 1))
fitted_model = model.fit()
forecast = fitted_model.forecast(steps=30)

# Trouver les meilleurs paramètres (p, d, q)
import itertools
p = d = q = range(0, 3)
pdq = list(itertools.product(p, d, q))

best_aic = np.inf
best_params = None

for param in pdq:
    try:
        model = ARIMA(df['value'], order=param)
        fitted = model.fit()
        if fitted.aic < best_aic:
            best_aic = fitted.aic
            best_params = param
    except:
        continue

print(f"Best ARIMA{best_params} - AIC: {best_aic}")

# 5. SARIMA (Seasonal ARIMA)
from statsmodels.tsa.statespace.sarimax import SARIMAX

# SARIMA(p,d,q)(P,D,Q,s)
model = SARIMAX(
    df['value'],
    order=(1, 1, 1),           # (p, d, q)
    seasonal_order=(1, 1, 1, 12)  # (P, D, Q, s)
)
fitted_model = model.fit()
forecast = fitted_model.forecast(steps=30)

# 6. Prophet (par Facebook)
from prophet import Prophet

# Préparer les données (colonnes 'ds' et 'y')
df_prophet = df.reset_index()
df_prophet.columns = ['ds', 'y']

# Créer et entraîner le modèle
model = Prophet(
    yearly_seasonality=True,
    weekly_seasonality=True,
    daily_seasonality=False
)
model.fit(df_prophet)

# Prédire
future = model.make_future_dataframe(periods=30)
forecast = model.predict(future)

# Visualiser
model.plot(forecast)
model.plot_components(forecast)


# --- MÉTRIQUES POUR SÉRIES TEMPORELLES ---

from sklearn.metrics import mean_absolute_error, mean_squared_error

def evaluate_forecast(y_true, y_pred):
    """Évaluer les prévisions"""
    mae = mean_absolute_error(y_true, y_pred)
    rmse = np.sqrt(mean_squared_error(y_true, y_pred))
    mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100
    
    print(f"MAE: {mae:.2f}")
    print(f"RMSE: {rmse:.2f}")
    print(f"MAPE: {mape:.2f}%")


[OK] 10. GESTION DES DONNÉES DÉSÉQUILIBRÉES


"""
Classes déséquilibrées: quand une classe est beaucoup plus fréquente.
Problème courant en détection de fraude, diagnostic médical, etc.
"""

from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler, TomekLinks
from imblearn.combine import SMOTETomek

# --- OVERSAMPLING (Sur-échantillonnage) ---

# 1. Random Oversampling (dupliquer aléatoirement minorité)
ros = RandomOverSampler(random_state=42)
X_resampled, y_resampled = ros.fit_resample(X, y)

# 2. SMOTE (Synthetic Minority Over-sampling Technique)
# Crée des exemples synthétiques
smote = SMOTE(random_state=42)
X_resampled, y_resampled = smote.fit_resample(X, y)

# 3. ADASYN (Adaptive Synthetic Sampling)
adasyn = ADASYN(random_state=42)
X_resampled, y_resampled = adasyn.fit_resample(X, y)


# --- UNDERSAMPLING (Sous-échantillonnage) ---

# 1. Random Undersampling (supprimer aléatoirement majorité)
rus = RandomUnderSampler(random_state=42)
X_resampled, y_resampled = rus.fit_resample(X, y)

# 2. Tomek Links (supprimer paires ambiguës)
tomek = TomekLinks()
X_resampled, y_resampled = tomek.fit_resample(X, y)


# --- APPROCHE COMBINÉE ---

# SMOTE + Tomek Links
smote_tomek = SMOTETomek(random_state=42)
X_resampled, y_resampled = smote_tomek.fit_resample(X, y)


# --- CLASS WEIGHTS (Poids de classes) ---

# Donner plus d'importance à la classe minoritaire
from sklearn.utils.class_weight import compute_class_weight

# Calculer poids automatiquement
class_weights = compute_class_weight(
    'balanced',
    classes=np.unique(y),
    y=y
)
class_weight_dict = dict(enumerate(class_weights))

# Utiliser dans le modèle
model = RandomForestClassifier(class_weight='balanced')
# ou
model = RandomForestClassifier(class_weight=class_weight_dict)


# --- THRESHOLD TUNING (Ajuster le seuil de décision) ---

# Au lieu de 0.5, optimiser le seuil
from sklearn.metrics import precision_recall_curve

probabilities = model.predict_proba(X_test)[:, 1]
precision, recall, thresholds = precision_recall_curve(y_test, probabilities)

# Choisir seuil optimal (par exemple, maximiser F1)
f1_scores = 2 * (precision * recall) / (precision + recall)
optimal_threshold = thresholds[np.argmax(f1_scores)]

# Prédire avec nouveau seuil
predictions = (probabilities >= optimal_threshold).astype(int)


[OK] 11. PIPELINES ET AUTOMATISATION


"""
Pipelines: automatiser et standardiser le workflow de ML.
Évite les fuites de données et rend le code reproductible.
"""

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.compose import ColumnTransformer

# --- PIPELINE SIMPLE ---

# Créer pipeline
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('pca', PCA(n_components=10)),
    ('classifier', RandomForestClassifier())
])

# Entraîner
pipeline.fit(X_train, y_train)

# Prédire (applique toutes les étapes automatiquement)
predictions = pipeline.predict(X_test)


# --- COLUMN TRANSFORMER (différents prétraitements par colonne) ---

# Définir colonnes
numeric_features = ['age', 'income', 'score']
categorical_features = ['gender', 'city', 'category']

# Transformers pour chaque type
numeric_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
    ('onehot', OneHotEncoder(handle_unknown='ignore'))
])

# Combiner
preprocessor = ColumnTransformer([
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

# Pipeline complet
full_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier())
])

# Entraîner
full_pipeline.fit(X_train, y_train)


# --- PIPELINE AVEC GRID SEARCH ---

from sklearn.model_selection import GridSearchCV

# Définir grille de paramètres (préfixer par nom de l'étape)
param_grid = {
    'preprocessor__num__imputer__strategy': ['mean', 'median'],
    'classifier__n_estimators': [50, 100, 200],
    'classifier__max_depth': [5, 10, None]
}

# Grid Search
grid_search = GridSearchCV(
    full_pipeline,
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1
)

grid_search.fit(X_train, y_train)
best_model = grid_search.best_estimator_


# --- SAUVEGARDER ET CHARGER UN PIPELINE ---

import joblib

# Sauvegarder
joblib.dump(pipeline, 'model_pipeline.pkl')

# Charger
loaded_pipeline = joblib.load('model_pipeline.pkl')
predictions = loaded_pipeline.predict(X_new)


[OK] 12. INTERPRÉTABILITÉ DES MODÈLES


"""
Comprendre et expliquer les prédictions des modèles.
Crucial pour la confiance, le debug, et la conformité réglementaire.
"""

# --- FEATURE IMPORTANCE (importance des variables) ---

# Pour modèles à base d'arbres
model = RandomForestClassifier()
model.fit(X_train, y_train)

importances = pd.DataFrame({
    'feature': X.columns,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)

# Visualiser
plt.barh(importances['feature'][:10], importances['importance'][:10])
plt.xlabel('Importance')


# --- SHAP (SHapley Additive exPlanations) ---

import shap

# Créer explainer
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)

# Summary plot (importance globale)
shap.summary_plot(shap_values, X_test)

# Force plot (explication d'une prédiction)
shap.force_plot(
    explainer.expected_value[1],
    shap_values[1][0],
    X_test.iloc[0]
)

# Dependence plot (relation entre feature et prédiction)
shap.dependence_plot('feature_name', shap_values[1], X_test)


# --- LIME (Local Interpretable Model-agnostic Explanations) ---

from lime.lime_tabular import LimeTabularExplainer

# Créer explainer
explainer = LimeTabularExplainer(
    X_train.values,
    feature_names=X_train.columns,
    class_names=['Class 0', 'Class 1'],
    mode='classification'
)

# Expliquer une prédiction
explanation = explainer.explain_instance(
    X_test.iloc[0].values,
    model.predict_proba,
    num_features=10
)

# Visualiser
explanation.show_in_notebook()


# --- PARTIAL DEPENDENCE PLOTS ---

from sklearn.inspection import plot_partial_dependence

# Visualiser l'effet d'une feature sur la prédiction
features = [0, 1, (0, 1)]  # Features individuelles et interactions
plot_partial_dependence(
    model, X_train, features,
    feature_names=X_train.columns
)


# --- PERMUTATION IMPORTANCE ---

from sklearn.inspection import permutation_importance

# Importance en permutant les valeurs
perm_importance = permutation_importance(
    model, X_test, y_test,
    n_repeats=10,
    random_state=42
)

sorted_idx = perm_importance.importances_mean.argsort()
plt.barh(X.columns[sorted_idx], perm_importance.importances_mean[sorted_idx])


[OK] 13. DEEP LEARNING (Bases avec TensorFlow/Keras)


"""
Deep Learning: réseaux de neurones pour problèmes complexes.
Images, texte, séries temporelles, etc.
"""

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, models
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

# --- RÉSEAU DE NEURONES SIMPLE (MLP) ---

# Créer modèle séquentiel
model = models.Sequential([
    layers.Dense(128, activation='relu', input_shape=(input_dim,)),
    layers.Dropout(0.3),  # Dropout pour régularisation
    layers.Dense(64, activation='relu'),
    layers.Dropout(0.3),
    layers.Dense(32, activation='relu'),
    layers.Dense(num_classes, activation='softmax')  # Classification
])

# Compiler
model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',  # ou 'binary_crossentropy'
    metrics=['accuracy']
)

# Résumé du modèle
model.summary()

# Callbacks
early_stop = EarlyStopping(
    monitor='val_loss',
    patience=5,
    restore_best_weights=True
)

checkpoint = ModelCheckpoint(
    'best_model.h5',
    monitor='val_loss',
    save_best_only=True
)

# Entraîner
history = model.fit(
    X_train, y_train,
    validation_split=0.2,
    epochs=100,
    batch_size=32,
    callbacks=[early_stop, checkpoint],
    verbose=1
)

# Évaluer
loss, accuracy = model.evaluate(X_test, y_test)

# Prédire
predictions = model.predict(X_test)


# --- RÉSEAU DE NEURONES CONVOLUTIF (CNN) pour images ---

model = models.Sequential([
    # Couches de convolution
    layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(64, (3, 3), activation='relu'),
    layers.MaxPooling2D((2, 2)),
    layers.Conv2D(64, (3, 3), activation='relu'),
    
    # Aplatir et couches denses
    layers.Flatten(),
    layers.Dense(64, activation='relu'),
    layers.Dropout(0.5),
    layers.Dense(10, activation='softmax')
])

model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)


# --- RÉSEAU RÉCURRENT (RNN/LSTM) pour séquences ---

model = models.Sequential([
    layers.LSTM(128, return_sequences=True, input_shape=(timesteps, features)),
    layers.Dropout(0.3),
    layers.LSTM(64),
    layers.Dropout(0.3),
    layers.Dense(32, activation='relu'),
    layers.Dense(1)  # Régression
])

model.compile(
    optimizer='adam',
    loss='mse',
    metrics=['mae']
)


# --- TRANSFER LEARNING (utiliser modèle pré-entraîné) ---

from tensorflow.keras.applications import VGG16

# Charger modèle pré-entraîné (sans la tête de classification)
base_model = VGG16(
    weights='imagenet',
    include_top=False,
    input_shape=(224, 224, 3)
)

# Geler les couches pré-entraînées
base_model.trainable = False

# Ajouter nouvelles couches
model = models.Sequential([
    base_model,
    layers.GlobalAveragePooling2D(),
    layers.Dense(256, activation='relu'),
    layers.Dropout(0.5),
    layers.Dense(num_classes, activation='softmax')
])

model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy']
)


# --- VISUALISER L'ENTRAÎNEMENT ---

# Tracer loss et accuracy
plt.figure(figsize=(12, 4))

plt.subplot(1, 2, 1)
plt.plot(history.history['loss'], label='Train Loss')
plt.plot(history.history['val_loss'], label='Val Loss')
plt.legend()
plt.title('Loss')

plt.subplot(1, 2, 2)
plt.plot(history.history['accuracy'], label='Train Accuracy')
plt.plot(history.history['val_accuracy'], label='Val Accuracy')
plt.legend()
plt.title('Accuracy')


# --- SAUVEGARDER ET CHARGER ---

# Sauvegarder modèle complet
model.save('my_model.h5')

# Charger
loaded_model = keras.models.load_model('my_model.h5')

# Sauvegarder seulement les poids
model.save_weights('model_weights.h5')

# Charger poids
model.load_weights('model_weights.h5')


[OK] 14. WEB SCRAPING


"""
Extraire des données depuis des sites web.
Important: respecter les robots.txt et les conditions d'utilisation.
"""

import requests
from bs4 import BeautifulSoup
import scrapy
from selenium import webdriver

# --- REQUESTS + BEAUTIFULSOUP (sites statiques) ---

# Faire une requête
url = 'https://example.com'
response = requests.get(url)

# Parser le HTML
soup = BeautifulSoup(response.content, 'html.parser')

# Trouver éléments
title = soup.find('title').text
all_links = soup.find_all('a')

# Extraire avec sélecteurs CSS
divs = soup.select('div.class-name')

# Extraire texte
for div in divs:
    print(div.text.strip())

# Extraire attributs
for link in all_links:
    href = link.get('href')
    text = link.text


# --- SCRAPY (framework complet) ---

# Créer spider
class MySpider(scrapy.Spider):
    name = 'my_spider'
    start_urls = ['https://example.com']
    
    def parse(self, response):
        # Extraire données
        for item in response.css('div.item'):
            yield {
                'title': item.css('h2::text').get(),
                'price': item.css('span.price::text').get(),
                'link': item.css('a::attr(href)').get()
            }
        
        # Suivre pagination
        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)


# --- SELENIUM (sites dynamiques avec JavaScript) ---

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Créer driver
driver = webdriver.Chrome()

# Naviguer
driver.get('https://example.com')

# Attendre que l'élément soit chargé
wait = WebDriverWait(driver, 10)
element = wait.until(
    EC.presence_of_element_located((By.CSS_SELECTOR, 'div.content'))
)

# Extraire données
elements = driver.find_elements(By.CLASS_NAME, 'item')
for elem in elements:
    text = elem.text
    
# Interagir (cliquer, remplir formulaire, etc.)
button = driver.find_element(By.ID, 'submit-button')
button.click()

input_field = driver.find_element(By.NAME, 'search')
input_field.send_keys('query')

# Fermer
driver.quit()


# --- REQUÊTES AVEC HEADERS ---

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept': 'text/html,application/xhtml+xml',
    'Accept-Language': 'en-US,en;q=0.9'
}

response = requests.get(url, headers=headers)


# --- GÉRER LA PAGINATION ---

def scrape_all_pages(base_url, max_pages=10):
    """Scraper plusieurs pages"""
    all_data = []
    
    for page in range(1, max_pages + 1):
        url = f"{base_url}?page={page}"
        response = requests.get(url, headers=headers)
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Extraire données de cette page
        items = soup.find_all('div', class_='item')
        for item in items:
            all_data.append({
                'title': item.find('h2').text,
                'price': item.find('span', class_='price').text
            })
        
        # Pause pour ne pas surcharger le serveur
        time.sleep(1)
    
    return all_data


[OK] 15. APIs ET REQUÊTES


"""
Interagir avec des APIs pour récupérer ou envoyer des données.
"""

import requests
import json

# --- GET REQUEST ---

response = requests.get('https://api.example.com/data')

# Vérifier status
if response.status_code == 200:
    data = response.json()  # Parser JSON automatiquement
else:
    print(f"Error: {response.status_code}")

# Paramètres dans l'URL
params = {'key': 'value', 'page': 1, 'limit': 100}
response = requests.get('https://api.example.com/data', params=params)


# --- POST REQUEST ---

# Envoyer données JSON
data = {'name': 'John', 'age': 30}
response = requests.post('https://api.example.com/users', json=data)

# Envoyer données form
form_data = {'username': 'john', 'password': 'secret'}
response = requests.post('https://api.example.com/login', data=form_data)


# --- AUTHENTIFICATION ---

# Basic Auth
from requests.auth import HTTPBasicAuth
response = requests.get(url, auth=HTTPBasicAuth('user', 'pass'))

# Bearer Token (JWT)
headers = {'Authorization': 'Bearer YOUR_TOKEN_HERE'}
response = requests.get(url, headers=headers)

# API Key
headers = {'X-API-Key': 'your_api_key'}
response = requests.get(url, headers=headers)


# --- GÉRER LES ERREURS ---

try:
    response = requests.get(url, timeout=5)
    response.raise_for_status()  # Lève exception si erreur HTTP
    data = response.json()
except requests.exceptions.Timeout:
    print("Request timed out")
except requests.exceptions.HTTPError as e:
    print(f"HTTP error: {e}")
except requests.exceptions.RequestException as e:
    print(f"Error: {e}")


# --- SESSIONS (garder cookies et headers) ---

session = requests.Session()
session.headers.update({'User-Agent': 'My App'})

# Toutes les requêtes utilisent la session
response1 = session.get('https://api.example.com/login')
response2 = session.get('https://api.example.com/data')  # Cookies préservés


# --- RATE LIMITING (limiter le nombre de requêtes) ---

import time
from functools import wraps

def rate_limit(max_calls, time_frame):
    """Decorator pour limiter les appels"""
    calls = []
    
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            now = time.time()
            calls[:] = [c for c in calls if c > now - time_frame]
            
            if len(calls) >= max_calls:
                sleep_time = time_frame - (now - calls[0])
                time.sleep(sleep_time)
            
            calls.append(time.time())
            return func(*args, **kwargs)
        return wrapper
    return decorator

@rate_limit(max_calls=10, time_frame=60)  # 10 appels par minute
def api_call():
    return requests.get('https://api.example.com/data')


# --- TÉLÉCHARGER FICHIERS ---

# Télécharger petit fichier
response = requests.get('https://example.com/file.pdf')
with open('file.pdf', 'wb') as f:
    f.write(response.content)

# Télécharger gros fichier (streaming)
response = requests.get('https://example.com/bigfile.zip', stream=True)
with open('bigfile.zip', 'wb') as f:
    for chunk in response.iter_content(chunk_size=8192):
        f.write(chunk)


[OK] 16. BASES DE DONNÉES SQL


"""
Interagir avec des bases de données relationnelles.
SQLite, PostgreSQL, MySQL, etc.
"""

import sqlite3
import pandas as pd
from sqlalchemy import create_engine

# --- SQLITE (base de données locale) ---

# Connexion
conn = sqlite3.connect('database.db')
cursor = conn.cursor()

# Créer table
cursor.execute('''
    CREATE TABLE IF NOT EXISTS users (
        id INTEGER PRIMARY KEY,
        name TEXT NOT NULL,
        age INTEGER,
        email TEXT UNIQUE
    )
''')

# Insérer données
cursor.execute('''
    INSERT INTO users (name, age, email)
    VALUES (?, ?, ?)
''', ('John Doe', 30, 'john@example.com'))

# Insérer plusieurs lignes
data = [
    ('Alice', 25, 'alice@example.com'),
    ('Bob', 35, 'bob@example.com')
]
cursor.executemany('INSERT INTO users (name, age, email) VALUES (?, ?, ?)', data)

# Commit (sauvegarder les changements)
conn.commit()

# Requête SELECT
cursor.execute('SELECT * FROM users WHERE age > ?', (25,))
results = cursor.fetchall()  # Toutes les lignes
# ou fetchone() pour une ligne
# ou fetchmany(5) pour 5 lignes

# Fermer connexion
conn.close()


# --- PANDAS + SQL ---

# Lire depuis SQL dans DataFrame
conn = sqlite3.connect('database.db')
df = pd.read_sql_query('SELECT * FROM users', conn)

# Écrire DataFrame dans SQL
df.to_sql('users', conn, if_exists='replace', index=False)
# if_exists: 'fail', 'replace', 'append'

conn.close()


# --- SQLALCHEMY (ORM et support multi-DB) ---

from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

# Créer engine (connexion)
# SQLite
engine = create_engine('sqlite:///database.db')

# PostgreSQL
# engine = create_engine('postgresql://user:password@localhost:5432/dbname')

# MySQL
# engine = create_engine('mysql+pymysql://user:password@localhost:3306/dbname')

# Définir modèle (table)
Base = declarative_base()

class User(Base):
    __tablename__ = 'users'
    
    id = Column(Integer, primary_key=True)
    name = Column(String(50), nullable=False)
    age = Column(Integer)
    email = Column(String(100), unique=True)

# Créer tables
Base.metadata.create_all(engine)

# Session
Session = sessionmaker(bind=engine)
session = Session()

# Insérer
new_user = User(name='John', age=30, email='john@example.com')
session.add(new_user)
session.commit()

# Requêtes
users = session.query(User).all()
user = session.query(User).filter_by(name='John').first()
users_over_25 = session.query(User).filter(User.age > 25).all()

# Mettre à jour
user = session.query(User).filter_by(name='John').first()
user.age = 31
session.commit()

# Supprimer
session.delete(user)
session.commit()

# Fermer
session.close()


# --- PANDAS + SQLALCHEMY ---

# Lire
engine = create_engine('sqlite:///database.db')
df = pd.read_sql_table('users', engine)
# ou
df = pd.read_sql('SELECT * FROM users WHERE age > 25', engine)

# Écrire
df.to_sql('users', engine, if_exists='append', index=False)


[OK] 17. BASES DE DONNÉES NoSQL (MongoDB)


"""
Bases de données non-relationnelles (documents JSON).
"""

from pymongo import MongoClient

# Connexion
client = MongoClient('mongodb://localhost:27017/')
db = client['mydatabase']
collection = db['users']

# Insérer document
user = {
    'name': 'John Doe',
    'age': 30,
    'email': 'john@example.com',
    'tags': ['python', 'data science']
}
result = collection.insert_one(user)
print(result.inserted_id)

# Insérer plusieurs
users = [
    {'name': 'Alice', 'age': 25},
    {'name': 'Bob', 'age': 35}
]
collection.insert_many(users)

# Trouver documents
all_users = collection.find()
for user in all_users:
    print(user)

# Trouver avec filtre
user = collection.find_one({'name': 'John Doe'})
users_over_25 = collection.find({'age': {'$gt': 25}})

# Mettre à jour
collection.update_one(
    {'name': 'John Doe'},
    {'$set': {'age': 31}}
)

# Mettre à jour plusieurs
collection.update_many(
    {'age': {'$lt': 30}},
    {'$set': {'status': 'young'}}
)

# Supprimer
collection.delete_one({'name': 'John Doe'})
collection.delete_many({'age': {'$lt': 25}})

# Agrégation
pipeline = [
    {'$match': {'age': {'$gt': 25}}},
    {'$group': {'_id': '$status', 'count': {'$sum': 1}}},
    {'$sort': {'count': -1}}
]
results = collection.aggregate(pipeline)


[OK] 18. BIG DATA (PySpark, Dask)


"""
Traiter des données trop grandes pour la mémoire.
Calcul distribué sur plusieurs machines.
"""

# --- PYSPARK ---

from pyspark.sql import SparkSession
from pyspark.sql import functions as F

# Créer session Spark
spark = SparkSession.builder \
    .appName("MyApp") \
    .master("local[*]") \
    .getOrCreate()

# Lire données
df = spark.read.csv('data.csv', header=True, inferSchema=True)
# ou JSON
df = spark.read.json('data.json')
# ou Parquet
df = spark.read.parquet('data.parquet')

# Afficher
df.show(5)
df.printSchema()

# Sélectionner colonnes
df_selected = df.select('col1', 'col2')

# Filtrer
df_filtered = df.filter(df['age'] > 25)
# ou
df_filtered = df.where(df['age'] > 25)

# Agrégations
df_agg = df.groupBy('category').agg(
    F.count('*').alias('count'),
    F.mean('value').alias('mean_value'),
    F.max('value').alias('max_value')
)

# Jointures
df_joined = df1.join(df2, on='key', how='inner')
# how: 'inner', 'outer', 'left', 'right'

# Ajouter colonne
df_with_col = df.withColumn('new_col', df['col1'] + df['col2'])

# Renommer colonne
df_renamed = df.withColumnRenamed('old_name', 'new_name')

# UDF (User Defined Function)
from pyspark.sql.types import StringType

@F.udf(returnType=StringType())
def custom_function(value):
    return value.upper()

df_transformed = df.withColumn('upper_name', custom_function(df['name']))

# SQL
df.createOrReplaceTempView('users')
result = spark.sql('SELECT * FROM users WHERE age > 25')

# Écrire résultats
df.write.csv('output.csv', mode='overwrite', header=True)
df.write.parquet('output.parquet', mode='overwrite')

# Machine Learning avec Spark
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.classification import RandomForestClassifier

# Préparer features
assembler = VectorAssembler(
    inputCols=['feature1', 'feature2', 'feature3'],
    outputCol='features'
)
df_assembled = assembler.transform(df)

# Entraîner modèle
rf = RandomForestClassifier(labelCol='label', featuresCol='features')
model = rf.fit(df_assembled)

# Prédire
predictions = model.transform(df_test)


# --- DASK (Alternative à Pandas pour gros datasets) ---

import dask.dataframe as dd

# Lire avec Dask (lazy loading)
df = dd.read_csv('large_file.csv')
# ou plusieurs fichiers
df = dd.read_csv('data/*.csv')

# Opérations identiques à Pandas
df_filtered = df[df['age'] > 25]
df_grouped = df.groupby('category')['value'].mean()

# Compute (déclencher le calcul)
result = df_filtered.compute()  # Retourne un Pandas DataFrame

# Opérations parallèles
df_result = df.map_partitions(lambda partition: partition * 2)

# Persist en mémoire
df = df.persist()

# Écrire
df.to_csv('output/*.csv')
df.to_parquet('output.parquet')


[OK] 19. DÉPLOIEMENT DE MODÈLES


"""
Mettre un modèle en production pour être utilisé par des applications.
"""

# --- SAUVEGARDER ET CHARGER UN MODÈLE ---

import joblib
import pickle

# Avec joblib (recommandé pour sklearn)
joblib.dump(model, 'model.pkl')
loaded_model = joblib.load('model.pkl')

# Avec pickle
with open('model.pkl', 'wb') as f:
    pickle.dump(model, f)

with open('model.pkl', 'rb') as f:
    loaded_model = pickle.load(f)


# --- API AVEC FLASK ---

from flask import Flask, request, jsonify
import joblib

app = Flask(__name__)

# Charger modèle au démarrage
model = joblib.load('model.pkl')
scaler = joblib.load('scaler.pkl')

@app.route('/predict', methods=['POST'])
def predict():
    """Endpoint de prédiction"""
    try:
        # Récupérer données JSON
        data = request.get_json()
        features = data['features']
        
        # Prétraiter
        features_scaled = scaler.transform([features])
        
        # Prédire
        prediction = model.predict(features_scaled)
        probability = model.predict_proba(features_scaled)
        
        # Retourner résultat
        return jsonify({
            'prediction': int(prediction[0]),
            'probability': float(probability[0][1]),
            'status': 'success'
        })
    
    except Exception as e:
        return jsonify({
            'error': str(e),
            'status': 'error'
        }), 400

@app.route('/health', methods=['GET'])
def health():
    """Health check"""
    return jsonify({'status': 'healthy'})

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False)


# --- API AVEC FASTAPI (plus moderne) ---

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import joblib

app = FastAPI()

# Charger modèle
model = joblib.load('model.pkl')

# Définir schéma de données
class PredictionRequest(BaseModel):
    features: list[float]

class PredictionResponse(BaseModel):
    prediction: int
    probability: float

@app.post('/predict', response_model=PredictionResponse)
async def predict(request: PredictionRequest):
    """Prédiction"""
    try:
        features = [request.features]
        prediction = model.predict(features)
        probability = model.predict_proba(features)
        
        return PredictionResponse(
            prediction=int(prediction[0]),
            probability=float(probability[0][1])
        )
    except Exception as e:
        raise HTTPException(status_code=400, detail=str(e))

@app.get('/health')
async def health():
    return {'status': 'healthy'}

# Lancer avec: uvicorn main:app --host 0.0.0.0 --port 8000


# --- STREAMLIT (Interface web interactive) ---

import streamlit as st
import joblib
import pandas as pd

# Charger modèle
model = joblib.load('model.pkl')

# Titre
st.title('Prédiction de Modèle ML')

# Sidebar pour inputs
st.sidebar.header('Paramètres')
feature1 = st.sidebar.slider('Feature 1', 0.0, 10.0, 5.0)
feature2 = st.sidebar.slider('Feature 2', 0.0, 10.0, 5.0)
feature3 = st.sidebar.slider('Feature 3', 0.0, 10.0, 5.0)

# Bouton prédiction
if st.button('Prédire'):
    features = [[feature1, feature2, feature3]]
    prediction = model.predict(features)
    probability = model.predict_proba(features)
    
    st.success(f'Prédiction: {prediction[0]}')
    st.write(f'Probabilité: {probability[0][1]:.2%}')
    
    # Visualisation
    st.bar_chart(pd.DataFrame({
        'Classe': ['0', '1'],
        'Probabilité': probability[0]
    }).set_index('Classe'))

# Lancer avec: streamlit run app.py


# --- DOCKER (Containerisation) ---

# Dockerfile
"""
FROM python:3.9-slim

WORKDIR /app

COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

COPY . .

EXPOSE 8000

CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
"""

# Construire image
# docker build -t my-ml-api .

# Lancer container
# docker run -p 8000:8000 my-ml-api


# --- MLflow (Tracking et déploiement) ---

import mlflow
import mlflow.sklearn

# Démarrer run
with mlflow.start_run():
    # Entraîner modèle
    model.fit(X_train, y_train)
    
    # Logger paramètres
    mlflow.log_param('n_estimators', 100)
    mlflow.log_param('max_depth', 10)
    
    # Logger métriques
    accuracy = model.score(X_test, y_test)
    mlflow.log_metric('accuracy', accuracy)
    
    # Logger modèle
    mlflow.sklearn.log_model(model, 'model')
    
    # Logger artifacts (fichiers)
    mlflow.log_artifact('plot.png')

# Charger modèle depuis MLflow
loaded_model = mlflow.sklearn.load_model('runs:/<run_id>/model')


[OK] 20. OPTIMISATION ET PERFORMANCE


"""
Techniques pour accélérer le code et optimiser les calculs.
"""

# --- PROFILING (Mesurer les performances) ---

import cProfile
import pstats

# Profiler une fonction
cProfile.run('my_function()', 'output.stats')

# Analyser résultats
stats = pstats.Stats('output.stats')
stats.sort_stats('cumulative')
stats.print_stats(10)  # Top 10 fonctions les plus lentes

# Avec line_profiler (ligne par ligne)
# @profile decorator puis: kernprof -lv script.py


# --- MEMORY PROFILING ---

from memory_profiler import profile

@profile
def my_function():
    # Code à profiler
    large_list = [i for i in range(1000000)]
    return sum(large_list)


# --- NUMBA (JIT compilation) ---

from numba import jit, prange

@jit(nopython=True)
def fast_function(x):
    """Compilé en machine code, beaucoup plus rapide"""
    total = 0
    for i in range(len(x)):
        total += x[i] ** 2
    return total

# Avec parallélisation
@jit(nopython=True, parallel=True)
def parallel_function(x):
    total = 0
    for i in prange(len(x)):  # prange au lieu de range
        total += x[i] ** 2
    return total


# --- VECTORISATION (éviter les boucles) ---

# [X] Lent - Boucle Python
result = []
for i in range(len(arr)):
    result.append(arr[i] ** 2)

# [OK] Rapide - Vectorisé avec NumPy
result = arr ** 2

# [X] Lent - Apply avec lambda
df['new_col'] = df['col'].apply(lambda x: x ** 2)

# [OK] Rapide - Opération vectorisée
df['new_col'] = df['col'] ** 2


# --- MULTIPROCESSING (calcul parallèle) ---

from multiprocessing import Pool
import multiprocessing as mp

def process_chunk(chunk):
    """Fonction à paralléliser"""
    return chunk ** 2

# Paralléliser sur plusieurs cores
if __name__ == '__main__':
    data = np.arange(1000000)
    chunks = np.array_split(data, mp.cpu_count())
    
    with Pool(mp.cpu_count()) as pool:
        results = pool.map(process_chunk, chunks)
    
    final_result = np.concatenate(results)


# --- JOBLIB (parallélisation simple) ---

from joblib import Parallel, delayed

def process_item(item):
    return item ** 2

# Paralléliser
results = Parallel(n_jobs=-1)(delayed(process_item)(i) for i in range(1000))


# --- CACHING (mise en cache) ---

from functools import lru_cache

@lru_cache(maxsize=128)
def expensive_function(n):
    """Résultat mis en cache"""
    # Calcul coûteux
    return sum(range(n))

# Première fois: calcule
result1 = expensive_function(1000000)

# Deuxième fois: retourne depuis cache (instantané)
result2 = expensive_function(1000000)


# --- OPTIMISATION PANDAS ---

# Utiliser catégories pour strings répétitifs
df['category'] = df['category'].astype('category')

# Downcast numeric types
df['int_col'] = pd.to_numeric(df['int_col'], downcast='integer')
df['float_col'] = pd.to_numeric(df['float_col'], downcast='float')

# Lire CSV par chunks
for chunk in pd.read_csv('large_file.csv', chunksize=10000):
    # Traiter chunk par chunk
    process(chunk)

# Utiliser query() au lieu de boolean indexing
df_filtered = df.query('age > 25 and city == "Paris"')


# --- UTILISER NUMPY AU LIEU DE PANDAS ---

# Pour calculs numériques purs, NumPy est plus rapide
arr = df['column'].values  # Convertir en NumPy array
result = np.mean(arr)  # Plus rapide que df['column'].mean()


[OK] 21. BONNES PRATIQUES ET CONSEILS


"""
Conseils généraux pour un code de qualité en Data Science.
"""

# --- STRUCTURE DE PROJET ---

"""
project/
│
├── data/
│   ├── raw/              # Données brutes (jamais modifier)
│   ├── processed/        # Données nettoyées
│   └── external/         # Données externes
│
├── notebooks/            # Jupyter notebooks pour exploration
│   ├── 01_exploration.ipynb
│   ├── 02_cleaning.ipynb
│   └── 03_modeling.ipynb
│
├── src/                  # Code source
│   ├── __init__.py
│   ├── data/             # Scripts de préparation données
│   ├── features/         # Feature engineering
│   ├── models/           # Code des modèles
│   └── visualization/    # Visualisations
│
├── tests/                # Tests unitaires
│
├── models/               # Modèles sauvegardés
│
├── reports/              # Rapports et figures
│   └── figures/
│
├── requirements.txt      # Dépendances
├── setup.py
├── README.md
└── .gitignore
"""


# --- VERSIONING DES DONNÉES ET MODÈLES ---

# DVC (Data Version Control)
"""
# Initialiser
dvc init

# Tracker fichiers de données
dvc add data/raw/dataset.csv

# Commit
git add data/raw/dataset.csv.dvc .gitignore
git commit -m "Add dataset"

# Push vers remote storage
dvc push
"""


# --- REPRODUCTIBILITÉ ---

# Fixer random seeds
import random
np.random.seed(42)
random.seed(42)
tf.random.set_seed(42)

# Sauvegarder l'environnement
# pip freeze > requirements.txt

# Avec conda
# conda env export > environment.yml


# --- LOGGING (au lieu de print) ---

import logging

# Configuration
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('app.log'),
        logging.StreamHandler()
    ]
)

logger = logging.getLogger(__name__)

# Utilisation
logger.info('Starting training')
logger.warning('Low data quality detected')
logger.error('Model training failed')


# --- TESTS UNITAIRES ---

import unittest

class TestModel(unittest.TestCase):
    
    def setUp(self):
        """Exécuté avant chaque test"""
        self.model = MyModel()
    
    def test_prediction_shape(self):
        """Test la forme des prédictions"""
        X = np.random.rand(10, 5)
        predictions = self.model.predict(X)
        self.assertEqual(predictions.shape[0], 10)
    
    def test_prediction_range(self):
        """Test que les prédictions sont dans [0, 1]"""
        X = np.random.rand(10, 5)
        predictions = self.model.predict(X)
        self.assertTrue((predictions >= 0).all())
        self.assertTrue((predictions <= 1).all())

if __name__ == '__main__':
    unittest.main()


# --- CONFIGURATION AVEC YAML ---

import yaml

# config.yaml
"""
model:
  type: RandomForest
  n_estimators: 100
  max_depth: 10

training:
  test_size: 0.2
  random_state: 42
  cv_folds: 5
"""

# Charger config
with open('config.yaml', 'r') as f:
    config = yaml.safe_load(f)

model = RandomForestClassifier(
    n_estimators=config['model']['n_estimators'],
    max_depth=config['model']['max_depth']
)


# --- GESTION DES ERREURS ---

def safe_predict(model, X):
    """Prédiction avec gestion d'erreurs"""
    try:
        predictions = model.predict(X)
        return predictions, None
    except ValueError as e:
        logger.error(f"ValueError in prediction: {e}")
        return None, str(e)
    except Exception as e:
        logger.error(f"Unexpected error: {e}")
        return None, str(e)


# --- DOCUMENTATION ---

def preprocess_data(df, target_col, categorical_cols=None, numerical_cols=None):
    """
    Prétraite les données pour le modèle.
    
    Parameters
    ----------
    df : pd.DataFrame
        DataFrame contenant les données brutes
    target_col : str
        Nom de la colonne cible
    categorical_cols : list, optional
        Liste des colonnes catégorielles
    numerical_cols : list, optional
        Liste des colonnes numériques
    
    Returns
    -------
    X : pd.DataFrame
        Features prétraitées
    y : pd.Series
        Variable cible
    
    Examples
    --------
    >>> X, y = preprocess_data(df, 'target', categorical_cols=['cat1', 'cat2'])
    """
    # Implementation
    pass


[OK] 22. RESSOURCES ET OUTILS COMPLÉMENTAIRES


"""
Bibliothèques et outils utiles en Data Science.
"""

# --- VISUALISATION AVANCÉE ---

# Plotly Express (graphiques interactifs faciles)
# Altair (grammaire de visualisation)
# Bokeh (dashboards interactifs)
# Holoviews (visualisations complexes)

# --- MACHINE LEARNING ---

# XGBoost, LightGBM, CatBoost (gradient boosting optimisé)
# Optuna, Hyperopt (optimisation d'hyperparamètres)
# TPOT (AutoML avec genetic programming)
# PyCaret (AutoML low-code)

# --- DEEP LEARNING ---

# PyTorch (alternative à TensorFlow)
# Hugging Face Transformers (NLP pré-entraîné)
# FastAI (deep learning simplifié)

# --- TRAITEMENT DE DONNÉES ---

# Polars (alternative ultra-rapide à Pandas)
# Vaex (out-of-core DataFrames)
# Modin (Pandas parallelisé)

# --- FEATURE ENGINEERING ---

# Feature-engine (transformations automatisées)
# Category Encoders (encodages avancés)
# Featuretools (automated feature engineering)

# --- VALIDATION ---

# Great Expectations (validation de données)
# Pandera (schéma validation pour Pandas)

# --- MONITORING ---

# Evidently (ML monitoring)
# WhyLogs (data logging)

# --- DÉPLOIEMENT ---

# BentoML (packaging et déploiement)
# Seldon Core (Kubernetes deployment)
# TensorFlow Serving

# --- NOTEBOOKS ---

# Google Colab (GPU gratuit)
# Kaggle Notebooks
# Databricks Community Edition


"""
=============================================================================
[OBJECTIF] FIN DE LA CHEATSHEET DATA SCIENCE
=============================================================================

Cette cheatsheet couvre les aspects essentiels du Data Science qui complètent
NumPy et Pandas:
- Visualisation (Matplotlib, Seaborn, Plotly)
- Machine Learning (Scikit-learn, modèles, métriques)
- Feature Engineering
- Deep Learning (TensorFlow/Keras)
- NLP (traitement du langage)
- Traitement d'images
- Séries temporelles
- Big Data (PySpark, Dask)
- Bases de données (SQL, NoSQL)
- APIs et Web Scraping
- Déploiement de modèles
- Optimisation et performance
- Bonnes pratiques

Pour aller plus loin:
- Documentation officielle de chaque bibliothèque
- Kaggle competitions et datasets
- Cours en ligne (Coursera, fast.ai, deeplearning.ai)
- Livres: "Hands-On Machine Learning", "Python Data Science Handbook"
- Communautés: Stack Overflow, Reddit r/datascience, Discord servers

[IMPORTANT] Conseil: Commencez par maîtriser les bases (NumPy, Pandas, Scikit-learn)
avant de vous aventurer dans des domaines plus spécialisés.

Bonne chance dans votre parcours Data Science! [RAPIDE]
=============================================================================
"""


[OK] 23. ANALYSE EXPLORATOIRE DE DONNÉES (EDA) - TECHNIQUES AVANCÉES


"""
L'EDA est cruciale pour comprendre les données avant la modélisation.
Techniques systématiques pour explorer un dataset.
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# --- PROFILING AUTOMATIQUE ---

# Pandas Profiling (rapport complet automatique)
from ydata_profiling import ProfileReport

df = pd.read_csv('data.csv')
profile = ProfileReport(df, title='Pandas Profiling Report', explorative=True)
profile.to_file("report.html")  # Génère rapport HTML interactif

# Sweetviz (comparaison de datasets)
import sweetviz as sv

report = sv.analyze(df)
report.show_html('sweetviz_report.html')

# Comparer train et test
report = sv.compare([train_df, "Training"], [test_df, "Test"])


# --- ANALYSE UNIVARIÉE (une variable à la fois) ---

def univariate_analysis(df, column):
    """Analyse complète d'une variable"""
    
    print(f"\n{'='*60}")
    print(f"ANALYSE DE: {column}")
    print(f"{'='*60}")
    
    # Type et infos basiques
    print(f"\nType: {df[column].dtype}")
    print(f"Valeurs uniques: {df[column].nunique()}")
    print(f"Valeurs manquantes: {df[column].isna().sum()} ({df[column].isna().mean()*100:.2f}%)")
    
    if df[column].dtype in ['int64', 'float64']:
        # Variable numérique
        print("\nSTATISTIQUES DESCRIPTIVES:")
        print(df[column].describe())
        
        print(f"\nAsymétrie (Skewness): {df[column].skew():.3f}")
        print(f"Aplatissement (Kurtosis): {df[column].kurtosis():.3f}")
        
        # Visualisations
        fig, axes = plt.subplots(1, 3, figsize=(15, 4))
        
        # Histogramme
        axes[0].hist(df[column].dropna(), bins=30, edgecolor='black')
        axes[0].set_title(f'Distribution de {column}')
        axes[0].set_xlabel(column)
        axes[0].set_ylabel('Fréquence')
        
        # Boxplot
        axes[1].boxplot(df[column].dropna())
        axes[1].set_title(f'Boxplot de {column}')
        axes[1].set_ylabel(column)
        
        # QQ-plot (test de normalité)
        from scipy import stats
        stats.probplot(df[column].dropna(), dist="norm", plot=axes[2])
        axes[2].set_title(f'Q-Q Plot de {column}')
        
        plt.tight_layout()
        plt.show()
        
        # Détection d'outliers
        Q1 = df[column].quantile(0.25)
        Q3 = df[column].quantile(0.75)
        IQR = Q3 - Q1
        outliers = df[(df[column] < Q1 - 1.5*IQR) | (df[column] > Q3 + 1.5*IQR)]
        print(f"\nOutliers détectés: {len(outliers)} ({len(outliers)/len(df)*100:.2f}%)")
    
    else:
        # Variable catégorielle
        print("\nTOP 10 VALEURS:")
        print(df[column].value_counts().head(10))
        
        # Visualisation
        fig, axes = plt.subplots(1, 2, figsize=(15, 5))
        
        # Bar plot
        df[column].value_counts().head(10).plot(kind='bar', ax=axes[0])
        axes[0].set_title(f'Top 10 de {column}')
        axes[0].set_xlabel(column)
        axes[0].set_ylabel('Count')
        
        # Pie chart
        df[column].value_counts().head(5).plot(kind='pie', ax=axes[1], autopct='%1.1f%%')
        axes[1].set_title(f'Distribution de {column} (Top 5)')
        axes[1].set_ylabel('')
        
        plt.tight_layout()
        plt.show()


# --- ANALYSE BIVARIÉE (relations entre 2 variables) ---

def bivariate_analysis(df, col1, col2, target=None):
    """Analyse de la relation entre deux variables"""
    
    num1 = df[col1].dtype in ['int64', 'float64']
    num2 = df[col2].dtype in ['int64', 'float64']
    
    if num1 and num2:
        # Numérique vs Numérique
        print(f"\nCORRÉLATION: {df[[col1, col2]].corr().iloc[0, 1]:.3f}")
        
        # Scatter plot
        plt.figure(figsize=(10, 6))
        if target:
            scatter = plt.scatter(df[col1], df[col2], c=df[target], alpha=0.5)
            plt.colorbar(scatter, label=target)
        else:
            plt.scatter(df[col1], df[col2], alpha=0.5)
        
        plt.xlabel(col1)
        plt.ylabel(col2)
        plt.title(f'{col1} vs {col2}')
        
        # Ajouter ligne de régression
        z = np.polyfit(df[col1].dropna(), df[col2].dropna(), 1)
        p = np.poly1d(z)
        plt.plot(df[col1], p(df[col1]), "r--", alpha=0.8, label='Régression')
        plt.legend()
        plt.show()
    
    elif not num1 and not num2:
        # Catégorielle vs Catégorielle
        crosstab = pd.crosstab(df[col1], df[col2])
        print("\nTABLE DE CONTINGENCE:")
        print(crosstab)
        
        # Chi-squared test
        from scipy.stats import chi2_contingency
        chi2, p_value, dof, expected = chi2_contingency(crosstab)
        print(f"\nChi-squared test: p-value = {p_value:.4f}")
        
        # Heatmap
        plt.figure(figsize=(10, 8))
        sns.heatmap(crosstab, annot=True, fmt='d', cmap='YlOrRd')
        plt.title(f'{col1} vs {col2}')
        plt.show()
    
    else:
        # Catégorielle vs Numérique
        cat_col = col1 if not num1 else col2
        num_col = col2 if not num1 else col1
        
        # Boxplot par catégorie
        plt.figure(figsize=(12, 6))
        df.boxplot(column=num_col, by=cat_col)
        plt.suptitle('')
        plt.title(f'{num_col} par {cat_col}')
        plt.show()
        
        # Test ANOVA
        from scipy.stats import f_oneway
        groups = [df[df[cat_col] == cat][num_col].dropna() 
                  for cat in df[cat_col].unique()]
        f_stat, p_value = f_oneway(*groups)
        print(f"\nANOVA test: p-value = {p_value:.4f}")


# --- MATRICE DE CORRÉLATION AVANCÉE ---

def correlation_analysis(df, method='pearson', threshold=0.7):
    """
    Analyse complète des corrélations
    
    method: 'pearson', 'spearman', 'kendall'
    """
    
    # Sélectionner colonnes numériques
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    
    # Calculer corrélations
    corr_matrix = df[numeric_cols].corr(method=method)
    
    # Visualisation
    fig, axes = plt.subplots(1, 2, figsize=(18, 7))
    
    # Heatmap complète
    sns.heatmap(corr_matrix, annot=True, fmt='.2f', 
                cmap='coolwarm', center=0, 
                square=True, ax=axes[0])
    axes[0].set_title(f'Matrice de Corrélation ({method})')
    
    # Heatmap des corrélations fortes seulement
    mask = np.abs(corr_matrix) < threshold
    sns.heatmap(corr_matrix, annot=True, fmt='.2f',
                cmap='coolwarm', center=0,
                mask=mask, square=True, ax=axes[1])
    axes[1].set_title(f'Corrélations > {threshold}')
    
    plt.tight_layout()
    plt.show()
    
    # Paires fortement corrélées
    print("\nPAIRES FORTEMENT CORRÉLÉES:")
    for i in range(len(corr_matrix.columns)):
        for j in range(i+1, len(corr_matrix.columns)):
            if abs(corr_matrix.iloc[i, j]) > threshold:
                print(f"{corr_matrix.columns[i]} <-> {corr_matrix.columns[j]}: "
                      f"{corr_matrix.iloc[i, j]:.3f}")


# --- DÉTECTION D'ANOMALIES ---

def detect_anomalies(df, column, method='iqr', contamination=0.1):
    """
    Détecter les anomalies dans une colonne
    
    method: 'iqr', 'zscore', 'isolation_forest'
    """
    
    if method == 'iqr':
        # Méthode IQR
        Q1 = df[column].quantile(0.25)
        Q3 = df[column].quantile(0.75)
        IQR = Q3 - Q1
        
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        
        anomalies = df[(df[column] < lower_bound) | (df[column] > upper_bound)]
    
    elif method == 'zscore':
        # Méthode Z-score
        from scipy import stats
        z_scores = np.abs(stats.zscore(df[column].dropna()))
        anomalies = df[z_scores > 3]
    
    elif method == 'isolation_forest':
        # Isolation Forest
        from sklearn.ensemble import IsolationForest
        
        iso_forest = IsolationForest(contamination=contamination, random_state=42)
        predictions = iso_forest.fit_predict(df[[column]].dropna())
        
        anomalies = df[predictions == -1]
    
    print(f"\nANOMALIES DÉTECTÉES ({method}): {len(anomalies)}")
    print(f"Pourcentage: {len(anomalies)/len(df)*100:.2f}%")
    
    # Visualisation
    plt.figure(figsize=(12, 5))
    
    plt.subplot(1, 2, 1)
    plt.scatter(df.index, df[column], alpha=0.5, label='Normal')
    plt.scatter(anomalies.index, anomalies[column], 
                color='red', alpha=0.7, label='Anomalie')
    plt.xlabel('Index')
    plt.ylabel(column)
    plt.legend()
    plt.title('Détection d\'anomalies')
    
    plt.subplot(1, 2, 2)
    plt.boxplot(df[column].dropna())
    plt.ylabel(column)
    plt.title('Boxplot')
    
    plt.tight_layout()
    plt.show()
    
    return anomalies


# --- ANALYSE DE VALEURS MANQUANTES ---

def missing_values_analysis(df):
    """Analyse complète des valeurs manquantes"""
    
    # Comptage
    missing = df.isnull().sum()
    missing_pct = 100 * missing / len(df)
    
    missing_df = pd.DataFrame({
        'Column': missing.index,
        'Missing_Count': missing.values,
        'Missing_Percentage': missing_pct.values
    })
    missing_df = missing_df[missing_df['Missing_Count'] > 0].sort_values(
        'Missing_Count', ascending=False
    )
    
    print("\nVALEURS MANQUANTES PAR COLONNE:")
    print(missing_df)
    
    # Visualisation
    if len(missing_df) > 0:
        fig, axes = plt.subplots(1, 2, figsize=(15, 6))
        
        # Bar plot
        axes[0].bar(missing_df['Column'], missing_df['Missing_Percentage'])
        axes[0].set_xlabel('Colonnes')
        axes[0].set_ylabel('% Manquant')
        axes[0].set_title('Pourcentage de valeurs manquantes')
        axes[0].tick_params(axis='x', rotation=45)
        
        # Heatmap
        sns.heatmap(df.isnull(), yticklabels=False, cbar=True, 
                    cmap='viridis', ax=axes[1])
        axes[1].set_title('Pattern des valeurs manquantes')
        
        plt.tight_layout()
        plt.show()
        
        # Corrélation des patterns manquants
        missing_corr = df.isnull().corr()
        print("\nCORRÉLATION DES PATTERNS MANQUANTS (top 5):")
        
        # Trouver les paires les plus corrélées
        for i in range(len(missing_corr.columns)):
            for j in range(i+1, len(missing_corr.columns)):
                if abs(missing_corr.iloc[i, j]) > 0.5:
                    print(f"{missing_corr.columns[i]} <-> {missing_corr.columns[j]}: "
                          f"{missing_corr.iloc[i, j]:.3f}")


# --- ANALYSE DE DISTRIBUTION ---

def distribution_analysis(df, columns=None):
    """Analyse des distributions et tests de normalité"""
    
    if columns is None:
        columns = df.select_dtypes(include=[np.number]).columns
    
    from scipy import stats
    
    for col in columns:
        print(f"\n{'='*60}")
        print(f"DISTRIBUTION: {col}")
        print(f"{'='*60}")
        
        data = df[col].dropna()
        
        # Tests de normalité
        # Shapiro-Wilk (petit échantillon)
        if len(data) < 5000:
            stat, p_value = stats.shapiro(data)
            print(f"Shapiro-Wilk test: p-value = {p_value:.4f}")
        
        # Kolmogorov-Smirnov
        stat, p_value = stats.kstest(data, 'norm')
        print(f"Kolmogorov-Smirnov test: p-value = {p_value:.4f}")
        
        # Anderson-Darling
        result = stats.anderson(data)
        print(f"Anderson-Darling statistic: {result.statistic:.4f}")
        
        # Statistiques
        print(f"\nSkewness: {stats.skew(data):.3f}")
        print(f"Kurtosis: {stats.kurtosis(data):.3f}")
        
        # Visualisation
        fig, axes = plt.subplots(2, 2, figsize=(15, 10))
        
        # Histogramme + courbe normale
        axes[0, 0].hist(data, bins=30, density=True, alpha=0.7, edgecolor='black')
        mu, sigma = data.mean(), data.std()
        x = np.linspace(data.min(), data.max(), 100)
        axes[0, 0].plot(x, stats.norm.pdf(x, mu, sigma), 'r-', lw=2, label='Normal fit')
        axes[0, 0].set_title(f'Distribution de {col}')
        axes[0, 0].legend()
        
        # Q-Q plot
        stats.probplot(data, dist="norm", plot=axes[0, 1])
        axes[0, 1].set_title('Q-Q Plot')
        
        # Boxplot
        axes[1, 0].boxplot(data)
        axes[1, 0].set_title('Boxplot')
        axes[1, 0].set_ylabel(col)
        
        # KDE (Kernel Density Estimation)
        data.plot(kind='density', ax=axes[1, 1])
        axes[1, 1].set_title('Kernel Density Estimation')
        axes[1, 1].set_xlabel(col)
        
        plt.tight_layout()
        plt.show()


[OK] 24. FEATURE ENGINEERING AVANCÉ


"""
Techniques avancées pour créer des features pertinentes.
"""

# --- TRANSFORMATIONS AUTOMATIQUES ---

def auto_transform_features(df, target_col=None):
    """Applique automatiquement des transformations utiles"""
    
    df_transformed = df.copy()
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    
    for col in numeric_cols:
        if col == target_col:
            continue
        
        # Log transformation (pour distributions asymétriques)
        if df[col].min() > 0 and df[col].skew() > 1:
            df_transformed[f'{col}_log'] = np.log1p(df[col])
        
        # Square root transformation
        if df[col].min() >= 0 and df[col].skew() > 0.5:
            df_transformed[f'{col}_sqrt'] = np.sqrt(df[col])
        
        # Reciprocal transformation
        if df[col].min() > 0:
            df_transformed[f'{col}_reciprocal'] = 1 / (df[col] + 1)
        
        # Polynomial features (carré)
        df_transformed[f'{col}_squared'] = df[col] ** 2
        
    return df_transformed


# --- BINNING AUTOMATIQUE ---

def auto_binning(df, column, n_bins=5, method='quantile'):
    """
    Discrétisation automatique
    
    method: 'quantile', 'uniform', 'kmeans'
    """
    
    if method == 'quantile':
        # Bins de taille égale
        df[f'{column}_binned'] = pd.qcut(df[column], q=n_bins, 
                                          labels=False, duplicates='drop')
    
    elif method == 'uniform':
        # Bins d'intervalle égal
        df[f'{column}_binned'] = pd.cut(df[column], bins=n_bins, labels=False)
    
    elif method == 'kmeans':
        # K-means binning
        from sklearn.cluster import KMeans
        kmeans = KMeans(n_clusters=n_bins, random_state=42)
        df[f'{column}_binned'] = kmeans.fit_predict(df[[column]])
    
    return df


# --- EXTRACTION DE FEATURES TEMPORELLES AVANCÉES ---

def extract_datetime_features(df, date_column):
    """Extraire features riches depuis datetime"""
    
    df[date_column] = pd.to_datetime(df[date_column])
    
    # Features basiques
    df['year'] = df[date_column].dt.year
    df['month'] = df[date_column].dt.month
    df['day'] = df[date_column].dt.day
    df['dayofweek'] = df[date_column].dt.dayofweek
    df['hour'] = df[date_column].dt.hour
    df['minute'] = df[date_column].dt.minute
    
    # Features avancées
    df['quarter'] = df[date_column].dt.quarter
    df['week'] = df[date_column].dt.isocalendar().week
    df['dayofyear'] = df[date_column].dt.dayofyear
    df['is_weekend'] = df['dayofweek'].isin([5, 6]).astype(int)
    df['is_month_start'] = df[date_column].dt.is_month_start.astype(int)
    df['is_month_end'] = df[date_column].dt.is_month_end.astype(int)
    df['is_quarter_start'] = df[date_column].dt.is_quarter_start.astype(int)
    df['is_quarter_end'] = df[date_column].dt.is_quarter_end.astype(int)
    
    # Partie de la journée
    df['part_of_day'] = pd.cut(df['hour'], 
                                bins=[0, 6, 12, 18, 24],
                                labels=['Night', 'Morning', 'Afternoon', 'Evening'],
                                include_lowest=True)
    
    # Features cycliques
    df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12)
    df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)
    df['day_sin'] = np.sin(2 * np.pi * df['day'] / 31)
    df['day_cos'] = np.cos(2 * np.pi * df['day'] / 31)
    df['hour_sin'] = np.sin(2 * np.pi * df['hour'] / 24)
    df['hour_cos'] = np.cos(2 * np.pi * df['hour'] / 24)
    
    return df


# --- TARGET ENCODING (Encodage par moyenne de la cible) ---

def target_encoding(df, categorical_col, target_col, smoothing=10):
    """
    Target encoding avec smoothing pour éviter l'overfitting
    """
    
    # Calculer moyenne globale
    global_mean = df[target_col].mean()
    
    # Calculer statistiques par catégorie
    agg = df.groupby(categorical_col)[target_col].agg(['mean', 'count'])
    
    # Appliquer smoothing
    # smooth_mean = (count * mean + smoothing * global_mean) / (count + smoothing)
    agg['smooth_mean'] = (
        (agg['count'] * agg['mean'] + smoothing * global_mean) / 
        (agg['count'] + smoothing)
    )
    
    # Mapper les valeurs
    df[f'{categorical_col}_target_encoded'] = df[categorical_col].map(
        agg['smooth_mean']
    )
    
    # Remplir valeurs inconnues avec moyenne globale
    df[f'{categorical_col}_target_encoded'].fillna(global_mean, inplace=True)
    
    return df


# --- FEATURE INTERACTION AUTOMATIQUE ---

from sklearn.preprocessing import PolynomialFeatures

def create_interactions(df, degree=2, include_bias=False):
    """Créer automatiquement des interactions entre features"""
    
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    
    poly = PolynomialFeatures(degree=degree, include_bias=include_bias)
    interactions = poly.fit_transform(df[numeric_cols])
    
    # Noms des nouvelles features
    feature_names = poly.get_feature_names_out(numeric_cols)
    
    df_interactions = pd.DataFrame(interactions, columns=feature_names)
    
    return df_interactions


# --- AGGREGATION FEATURES (pour données groupées) ---

def create_aggregation_features(df, group_col, agg_col, agg_funcs=['mean', 'std', 'min', 'max']):
    """Créer features d'agrégation"""
    
    agg_df = df.groupby(group_col)[agg_col].agg(agg_funcs).reset_index()
    
    # Renommer colonnes
    agg_df.columns = [group_col] + [f'{agg_col}_{func}_by_{group_col}' 
                                     for func in agg_funcs]
    
    # Merger avec df original
    df = df.merge(agg_df, on=group_col, how='left')
    
    return df


[OK] 25. MÉTHODES D'ENSEMBLE AVANCÉES


"""
Techniques pour combiner plusieurs modèles et améliorer les performances.
"""

from sklearn.ensemble import VotingClassifier, StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC

# --- VOTING CLASSIFIER (Vote majoritaire) ---

# Hard voting (majorité des prédictions)
voting_clf = VotingClassifier(
    estimators=[
        ('lr', LogisticRegression()),
        ('rf', RandomForestClassifier()),
        ('svm', SVC())
    ],
    voting='hard'
)

# Soft voting (moyenne des probabilités)
voting_clf = VotingClassifier(
    estimators=[
        ('lr', LogisticRegression()),
        ('rf', RandomForestClassifier()),
        ('svm', SVC(probability=True))  # Activer probabilités pour SVM
    ],
    voting='soft'
)

voting_clf.fit(X_train, y_train)


# --- STACKING (Méta-modèle) ---

# Modèles de base
estimators = [
    ('rf', RandomForestClassifier(n_estimators=100)),
    ('gb', GradientBoostingClassifier(n_estimators=100)),
    ('svm', SVC(probability=True))
]

# Méta-modèle
stacking_clf = StackingClassifier(
    estimators=estimators,
    final_estimator=LogisticRegression(),
    cv=5  # Cross-validation pour éviter overfitting
)

stacking_clf.fit(X_train, y_train)


# --- BLENDING (Stacking simplifié) ---

def blending(models, X_train, y_train, X_val, y_val, X_test):
    """
    Blending manuel: entraîner sur train, prédire sur val,
    entraîner méta-modèle sur val, prédire sur test
    """
    
    # Prédictions sur validation
    val_predictions = np.column_stack([
        model.fit(X_train, y_train).predict_proba(X_val)[:, 1]
        for model in models
    ])
    
    # Entraîner méta-modèle
    meta_model = LogisticRegression()
    meta_model.fit(val_predictions, y_val)
    
    # Prédictions sur test
    test_predictions = np.column_stack([
        model.predict_proba(X_test)[:, 1]
        for model in models
    ])
    
    final_predictions = meta_model.predict(test_predictions)
    
    return final_predictions


# --- BAGGING PERSONNALISÉ ---

from sklearn.ensemble import BaggingClassifier

# Bagging avec n'importe quel modèle de base
bagging_clf = BaggingClassifier(
    base_estimator=DecisionTreeClassifier(),
    n_estimators=100,
    max_samples=0.8,  # 80% des échantillons par modèle
    max_features=0.8,  # 80% des features par modèle
    bootstrap=True,
    n_jobs=-1
)

bagging_clf.fit(X_train, y_train)


[OK] 26. CALIBRATION DE PROBABILITÉS


"""
Ajuster les probabilités prédites pour qu'elles reflètent les vraies probabilités.
"""

from sklearn.calibration import CalibratedClassifierCV, calibration_curve

# Calibrer un modèle
model = RandomForestClassifier()
calibrated_model = CalibratedClassifierCV(model, cv=5, method='sigmoid')
# method: 'sigmoid' (Platt scaling) ou 'isotonic'

calibrated_model.fit(X_train, y_train)

# Courbe de calibration
prob_pos = calibrated_model.predict_proba(X_test)[:, 1]
fraction_of_positives, mean_predicted_value = calibration_curve(
    y_test, prob_pos, n_bins=10
)

plt.figure(figsize=(10, 6))
plt.plot(mean_predicted_value, fraction_of_positives, marker='o', label='Calibrated')
plt.plot([0, 1], [0, 1], 'k--', label='Perfect calibration')
plt.xlabel('Mean predicted probability')
plt.ylabel('Fraction of positives')
plt.title('Calibration Curve')
plt.legend()
plt.show()


[COURS] FIN COMPLÈTE DE LA CHEATSHEET DATA SCIENCE


"""
Cette cheatsheet exhaustive couvre maintenant:

[OK] 1-2.   Visualisation et Machine Learning (couvert précédemment)
[OK] 3-9.   Métriques, Feature Engineering, Réduction dimensionnalité, etc.
[OK] 10-20. Big Data, APIs, Déploiement, Optimisation
[OK] 21-22. Bonnes pratiques et ressources
[OK] 23.    Analyse Exploratoire Avancée (EDA)
[OK] 24.    Feature Engineering Avancé
[OK] 25.    Méthodes d'Ensemble
[OK] 26.    Calibration de Probabilités

Cette cheatsheet est maintenant COMPLÈTE et production-ready!

[DOCS] Pour maîtriser la Data Science:
1. Pratiquer sur des datasets réels (Kaggle, UCI ML Repository)
2. Participer à des compétitions Kaggle
3. Contribuer à des projets open source
4. Lire des papers de recherche récents (arXiv, Papers with Code)
5. Construire un portfolio de projets variés
6. Partager vos connaissances (blog, tutoriels, présentation)

[IDEE] Workflow typique en Data Science:
1. Définir le problème et les objectifs business
2. Collecter et explorer les données (EDA)
3. Nettoyer et préparer les données
4. Feature Engineering
5. Sélection et entraînement des modèles
6. Évaluation et optimisation
7. Interprétation des résultats
8. Déploiement et monitoring
9. Itération continue

[OUTIL] Stack technologique recommandée:
- Données: NumPy, Pandas, Polars
- Visualisation: Matplotlib, Seaborn, Plotly
- ML Classique: Scikit-learn, XGBoost, LightGBM
- Deep Learning: TensorFlow/Keras ou PyTorch
- NLP: Hugging Face Transformers, spaCy
- Big Data: PySpark, Dask
- Déploiement: FastAPI, Docker, MLflow
- Version Control: Git, DVC
- Cloud: AWS SageMaker, Google Cloud AI, Azure ML

=============================================================================
"""


[OK] 27. TECHNIQUES DE RÉGULARISATION AVANCÉES


"""
Prévenir l'overfitting et améliorer la généralisation des modèles.
"""

from sklearn.linear_model import Ridge, Lasso, ElasticNet
from sklearn.model_selection import cross_val_score

# --- COMPARAISON DE RÉGULARISATIONS ---

def compare_regularizations(X, y, alphas=[0.001, 0.01, 0.1, 1, 10, 100]):
    """Comparer différentes forces de régularisation"""
    
    results = {
        'alpha': [],
        'ridge_score': [],
        'lasso_score': [],
        'elasticnet_score': []
    }
    
    for alpha in alphas:
        # Ridge (L2)
        ridge = Ridge(alpha=alpha)
        ridge_cv = cross_val_score(ridge, X, y, cv=5, scoring='r2')
        
        # Lasso (L1)
        lasso = Lasso(alpha=alpha, max_iter=10000)
        lasso_cv = cross_val_score(lasso, X, y, cv=5, scoring='r2')
        
        # ElasticNet (L1 + L2)
        elastic = ElasticNet(alpha=alpha, l1_ratio=0.5, max_iter=10000)
        elastic_cv = cross_val_score(elastic, X, y, cv=5, scoring='r2')
        
        results['alpha'].append(alpha)
        results['ridge_score'].append(ridge_cv.mean())
        results['lasso_score'].append(lasso_cv.mean())
        results['elasticnet_score'].append(elastic_cv.mean())
    
    # Visualiser
    plt.figure(figsize=(12, 6))
    plt.plot(results['alpha'], results['ridge_score'], 'o-', label='Ridge')
    plt.plot(results['alpha'], results['lasso_score'], 's-', label='Lasso')
    plt.plot(results['alpha'], results['elasticnet_score'], '^-', label='ElasticNet')
    plt.xscale('log')
    plt.xlabel('Alpha (force de régularisation)')
    plt.ylabel('R² Score (CV)')
    plt.title('Comparaison des régularisations')
    plt.legend()
    plt.grid(True, alpha=0.3)
    plt.show()
    
    return pd.DataFrame(results)


# --- EARLY STOPPING (pour Deep Learning) ---

from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau

# Early stopping: arrêter si pas d'amélioration
early_stop = EarlyStopping(
    monitor='val_loss',
    patience=10,              # Attendre 10 epochs
    restore_best_weights=True,  # Restaurer meilleur modèle
    verbose=1
)

# Réduire learning rate si plateau
reduce_lr = ReduceLROnPlateau(
    monitor='val_loss',
    factor=0.5,               # Diviser par 2
    patience=5,
    min_lr=1e-7,
    verbose=1
)

# Utiliser dans fit
history = model.fit(
    X_train, y_train,
    validation_split=0.2,
    epochs=200,
    callbacks=[early_stop, reduce_lr]
)


# --- DROPOUT (régularisation pour réseaux de neurones) ---

from tensorflow.keras import layers

model = models.Sequential([
    layers.Dense(256, activation='relu', input_shape=(input_dim,)),
    layers.Dropout(0.5),  # Désactive 50% des neurones aléatoirement
    layers.Dense(128, activation='relu'),
    layers.Dropout(0.3),
    layers.Dense(64, activation='relu'),
    layers.Dropout(0.2),
    layers.Dense(num_classes, activation='softmax')
])


# --- BATCH NORMALIZATION ---

model = models.Sequential([
    layers.Dense(256, input_shape=(input_dim,)),
    layers.BatchNormalization(),  # Normaliser les activations
    layers.Activation('relu'),
    layers.Dropout(0.3),
    
    layers.Dense(128),
    layers.BatchNormalization(),
    layers.Activation('relu'),
    layers.Dropout(0.2),
    
    layers.Dense(num_classes, activation='softmax')
])


# --- L1/L2 REGULARIZATION DANS KERAS ---

from tensorflow.keras import regularizers

model = models.Sequential([
    layers.Dense(
        256, 
        activation='relu',
        kernel_regularizer=regularizers.l2(0.01),  # Régularisation L2
        input_shape=(input_dim,)
    ),
    layers.Dense(
        128,
        activation='relu',
        kernel_regularizer=regularizers.l1_l2(l1=0.01, l2=0.01)  # L1 + L2
    ),
    layers.Dense(num_classes, activation='softmax')
])


[OK] 28. CROSS-VALIDATION AVANCÉE


"""
Techniques sophistiquées de validation croisée pour des cas spécifiques.
"""

from sklearn.model_selection import (
    KFold, StratifiedKFold, GroupKFold,
    TimeSeriesSplit, LeaveOneOut, ShuffleSplit
)

# --- STRATIFIED K-FOLD (préserve proportions) ---

skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):
    X_train_fold, X_val_fold = X[train_idx], X[val_idx]
    y_train_fold, y_val_fold = y[train_idx], y[val_idx]
    
    print(f"Fold {fold + 1}")
    print(f"Train: {len(train_idx)}, Val: {len(val_idx)}")
    print(f"Class distribution - Train: {np.bincount(y_train_fold)}")
    print(f"Class distribution - Val: {np.bincount(y_val_fold)}\n")


# --- GROUP K-FOLD (éviter data leakage sur groupes) ---

# Exemple: plusieurs mesures par patient, garder patients séparés
gkf = GroupKFold(n_splits=5)

groups = df['patient_id']  # Groupes à ne pas mélanger

for train_idx, val_idx in gkf.split(X, y, groups):
    # Les données d'un même groupe restent ensemble
    pass


# --- TIME SERIES SPLIT (respecte l'ordre temporel) ---

tscv = TimeSeriesSplit(n_splits=5, gap=30)

for train_idx, val_idx in tscv.split(X):
    # gap=30 : laisser 30 observations entre train et val
    # Évite le data leakage temporel
    pass

# Visualiser les splits
def plot_cv_indices(cv, X, y, group=None, n_splits=5):
    """Visualiser les indices de CV"""
    fig, ax = plt.subplots(figsize=(12, 8))
    
    for i, (train, test) in enumerate(cv.split(X, y, group)):
        indices = np.array([np.nan] * len(X))
        indices[train] = 1
        indices[test] = 0
        
        ax.scatter(
            range(len(indices)), [i] * len(indices),
            c=indices, marker='_', lw=10,
            cmap='coolwarm', vmin=-.2, vmax=1.2
        )
    
    ax.set_xlabel('Sample index')
    ax.set_ylabel('CV iteration')
    ax.set_title('Cross-validation indices')
    plt.show()


# --- NESTED CROSS-VALIDATION (pour sélection de modèle) ---

def nested_cross_validation(X, y, model, param_grid, outer_cv=5, inner_cv=3):
    """
    CV imbriquée: outer loop pour évaluation, inner loop pour tuning
    Évite l'overfitting lors de la sélection d'hyperparamètres
    """
    
    outer_scores = []
    outer_kf = KFold(n_splits=outer_cv, shuffle=True, random_state=42)
    
    for train_idx, test_idx in outer_kf.split(X):
        X_train, X_test = X[train_idx], X[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]
        
        # Inner loop: Grid Search
        inner_kf = KFold(n_splits=inner_cv, shuffle=True, random_state=42)
        grid_search = GridSearchCV(
            model, param_grid, cv=inner_kf, scoring='accuracy'
        )
        grid_search.fit(X_train, y_train)
        
        # Évaluer sur test set du outer loop
        best_model = grid_search.best_estimator_
        score = best_model.score(X_test, y_test)
        outer_scores.append(score)
        
        print(f"Outer fold: {score:.4f}, Best params: {grid_search.best_params_}")
    
    print(f"\nMean score: {np.mean(outer_scores):.4f} (+/- {np.std(outer_scores):.4f})")
    return outer_scores


# --- PURGED K-FOLD (pour séries temporelles avec chevauchement) ---

class PurgedKFold:
    """
    K-Fold qui purge les observations trop proches temporellement
    Utile en finance pour éviter le data leakage
    """
    
    def __init__(self, n_splits=5, embargo=0):
        self.n_splits = n_splits
        self.embargo = embargo  # Nombre d'observations à purger
    
    def split(self, X, y=None, groups=None):
        n_samples = len(X)
        fold_size = n_samples // self.n_splits
        
        for i in range(self.n_splits):
            # Indices de test
            test_start = i * fold_size
            test_end = (i + 1) * fold_size if i < self.n_splits - 1 else n_samples
            test_idx = np.arange(test_start, test_end)
            
            # Indices d'entraînement (en purgeant autour du test set)
            train_idx = np.concatenate([
                np.arange(0, max(0, test_start - self.embargo)),
                np.arange(min(n_samples, test_end + self.embargo), n_samples)
            ])
            
            yield train_idx, test_idx


[OK] 29. TRAITEMENT DES DÉSÉQUILIBRES EXTRÊMES


"""
Techniques pour datasets très déséquilibrés (ex: 1% de classe positive).
"""

# --- STRATÉGIES D'ÉCHANTILLONNAGE AVANCÉES ---

from imblearn.over_sampling import SMOTE, ADASYN, BorderlineSMOTE
from imblearn.under_sampling import (
    TomekLinks, EditedNearestNeighbours, NeighbourhoodCleaningRule
)
from imblearn.combine import SMOTEENN, SMOTETomek

# Borderline SMOTE (sur-échantillonner seulement la frontière)
borderline_smote = BorderlineSMOTE(random_state=42)
X_resampled, y_resampled = borderline_smote.fit_resample(X, y)

# Edited Nearest Neighbours (sous-échantillonner en nettoyant)
enn = EditedNearestNeighbours()
X_resampled, y_resampled = enn.fit_resample(X, y)

# SMOTE + ENN (combinaison)
smote_enn = SMOTEENN(random_state=42)
X_resampled, y_resampled = smote_enn.fit_resample(X, y)


# --- ALGORITHMES SPÉCIALISÉS ---

# Balanced Random Forest (échantillonnage dans chaque arbre)
from imblearn.ensemble import BalancedRandomForestClassifier

brf = BalancedRandomForestClassifier(
    n_estimators=100,
    sampling_strategy='auto',
    replacement=True,
    random_state=42
)
brf.fit(X_train, y_train)

# Easy Ensemble (bagging avec sous-échantillonnage)
from imblearn.ensemble import EasyEnsembleClassifier

eec = EasyEnsembleClassifier(
    n_estimators=10,
    random_state=42
)
eec.fit(X_train, y_train)


# --- MÉTRIQUES POUR CLASSES DÉSÉQUILIBRÉES ---

from sklearn.metrics import (
    precision_recall_curve, average_precision_score,
    roc_auc_score, matthews_corrcoef, cohen_kappa_score
)

# Average Precision (aire sous courbe Precision-Recall)
ap_score = average_precision_score(y_test, y_pred_proba)

# Matthews Correlation Coefficient (robuste au déséquilibre)
mcc = matthews_corrcoef(y_test, y_pred)

# Cohen's Kappa
kappa = cohen_kappa_score(y_test, y_pred)

# Courbe Precision-Recall (plus informative que ROC)
precision, recall, thresholds = precision_recall_curve(y_test, y_pred_proba)

plt.figure(figsize=(10, 6))
plt.plot(recall, precision, marker='.')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title(f'Precision-Recall Curve (AP={ap_score:.3f})')
plt.grid(True, alpha=0.3)
plt.show()


# --- FOCAL LOSS (pour deep learning) ---

import tensorflow.keras.backend as K

def focal_loss(gamma=2., alpha=0.25):
    """
    Focal Loss: met plus de poids sur exemples difficiles à classifier
    Très efficace pour classes déséquilibrées
    """
    
    def focal_loss_fixed(y_true, y_pred):
        epsilon = K.epsilon()
        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
        
        cross_entropy = -y_true * K.log(y_pred)
        weight = alpha * y_true * K.pow((1 - y_pred), gamma)
        
        loss = weight * cross_entropy
        return K.mean(K.sum(loss, axis=1))
    
    return focal_loss_fixed

# Utiliser dans modèle
model.compile(
    optimizer='adam',
    loss=focal_loss(gamma=2., alpha=0.25),
    metrics=['accuracy']
)


[OK] 30. DÉTECTION D'ANOMALIES (ANOMALY DETECTION)


"""
Identifier des observations inhabituelles ou suspectes.
"""

from sklearn.ensemble import IsolationForest
from sklearn.svm import OneClassSVM
from sklearn.neighbors import LocalOutlierFactor
from sklearn.covariance import EllipticEnvelope

# --- ISOLATION FOREST ---

iso_forest = IsolationForest(
    contamination=0.1,  # Proportion attendue d'anomalies
    random_state=42,
    n_estimators=100
)

predictions = iso_forest.fit_predict(X)
# 1 = normal, -1 = anomalie

anomaly_scores = iso_forest.score_samples(X)
# Plus négatif = plus anormal


# --- ONE-CLASS SVM ---

oc_svm = OneClassSVM(
    kernel='rbf',
    gamma='auto',
    nu=0.1  # Borne supérieure sur fraction d'anomalies
)

predictions = oc_svm.fit_predict(X)


# --- LOCAL OUTLIER FACTOR ---

lof = LocalOutlierFactor(
    n_neighbors=20,
    contamination=0.1,
    novelty=False  # False: fit_predict, True: fit puis predict
)

predictions = lof.fit_predict(X)

# Scores d'anomalie (plus négatif = plus anormal)
anomaly_scores = lof.negative_outlier_factor_


# --- ELLIPTIC ENVELOPE (pour données gaussiennes) ---

ee = EllipticEnvelope(
    contamination=0.1,
    random_state=42
)

predictions = ee.fit_predict(X)


# --- AUTOENCODER (Deep Learning pour anomalies) ---

from tensorflow.keras import models, layers

# Créer autoencoder
input_dim = X.shape[1]
encoding_dim = input_dim // 2

# Encoder
encoder = models.Sequential([
    layers.Dense(64, activation='relu', input_shape=(input_dim,)),
    layers.Dense(32, activation='relu'),
    layers.Dense(encoding_dim, activation='relu')
])

# Decoder
decoder = models.Sequential([
    layers.Dense(32, activation='relu', input_shape=(encoding_dim,)),
    layers.Dense(64, activation='relu'),
    layers.Dense(input_dim, activation='sigmoid')
])

# Autoencoder complet
autoencoder = models.Sequential([encoder, decoder])

autoencoder.compile(optimizer='adam', loss='mse')

# Entraîner sur données normales seulement
autoencoder.fit(
    X_normal, X_normal,
    epochs=50,
    batch_size=32,
    validation_split=0.2,
    verbose=0
)

# Détecter anomalies par erreur de reconstruction
reconstructions = autoencoder.predict(X_test)
mse = np.mean(np.square(X_test - reconstructions), axis=1)

# Définir seuil (ex: 95e percentile sur données normales)
threshold = np.percentile(mse, 95)
anomalies = mse > threshold


# --- VISUALISATION DES ANOMALIES ---

def plot_anomalies(X, predictions, title="Anomaly Detection"):
    """Visualiser anomalies (nécessite réduction à 2D)"""
    
    from sklearn.decomposition import PCA
    
    # Réduire à 2D pour visualisation
    if X.shape[1] > 2:
        pca = PCA(n_components=2)
        X_2d = pca.fit_transform(X)
    else:
        X_2d = X
    
    plt.figure(figsize=(10, 6))
    
    # Points normaux
    normal_mask = predictions == 1
    plt.scatter(X_2d[normal_mask, 0], X_2d[normal_mask, 1],
                c='blue', alpha=0.5, label='Normal')
    
    # Anomalies
    anomaly_mask = predictions == -1
    plt.scatter(X_2d[anomaly_mask, 0], X_2d[anomaly_mask, 1],
                c='red', alpha=0.8, label='Anomaly', marker='x', s=100)
    
    plt.xlabel('Component 1')
    plt.ylabel('Component 2')
    plt.title(title)
    plt.legend()
    plt.grid(True, alpha=0.3)
    plt.show()


[OK] 31. SYSTÈMES DE RECOMMANDATION


"""
Recommander items aux utilisateurs basé sur leurs préférences.
"""

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

# --- COLLABORATIVE FILTERING (FILTRAGE COLLABORATIF) ---

# Matrice user-item (lignes=users, colonnes=items, valeurs=ratings)
ratings_matrix = np.array([
    [5, 3, 0, 1],  # User 1
    [4, 0, 0, 1],  # User 2
    [1, 1, 0, 5],  # User 3
    [1, 0, 0, 4],  # User 4
    [0, 1, 5, 4],  # User 5
])

# User-based: similarité entre utilisateurs
user_similarity = cosine_similarity(ratings_matrix)

def recommend_user_based(user_id, ratings_matrix, user_similarity, n_recommendations=3):
    """Recommandations basées sur utilisateurs similaires"""
    
    # Items non notés par l'utilisateur
    user_ratings = ratings_matrix[user_id]
    unrated_items = np.where(user_ratings == 0)[0]
    
    if len(unrated_items) == 0:
        return []
    
    # Prédire ratings pour items non notés
    predictions = []
    for item_id in unrated_items:
        # Utilisateurs ayant noté cet item
        users_who_rated = np.where(ratings_matrix[:, item_id] > 0)[0]
        
        if len(users_who_rated) == 0:
            continue
        
        # Moyenne pondérée par similarité
        numerator = np.sum(
            user_similarity[user_id, users_who_rated] * 
            ratings_matrix[users_who_rated, item_id]
        )
        denominator = np.sum(np.abs(user_similarity[user_id, users_who_rated]))
        
        if denominator > 0:
            predicted_rating = numerator / denominator
            predictions.append((item_id, predicted_rating))
    
    # Trier par rating prédit
    predictions.sort(key=lambda x: x[1], reverse=True)
    
    return predictions[:n_recommendations]


# Item-based: similarité entre items
item_similarity = cosine_similarity(ratings_matrix.T)

def recommend_item_based(user_id, ratings_matrix, item_similarity, n_recommendations=3):
    """Recommandations basées sur items similaires"""
    
    user_ratings = ratings_matrix[user_id]
    rated_items = np.where(user_ratings > 0)[0]
    unrated_items = np.where(user_ratings == 0)[0]
    
    if len(unrated_items) == 0:
        return []
    
    predictions = []
    for item_id in unrated_items:
        # Similarité avec items déjà notés
        numerator = np.sum(
            item_similarity[item_id, rated_items] * 
            user_ratings[rated_items]
        )
        denominator = np.sum(np.abs(item_similarity[item_id, rated_items]))
        
        if denominator > 0:
            predicted_rating = numerator / denominator
            predictions.append((item_id, predicted_rating))
    
    predictions.sort(key=lambda x: x[1], reverse=True)
    
    return predictions[:n_recommendations]


# --- MATRIX FACTORIZATION (SVD) ---

from scipy.sparse.linalg import svds

# Remplir 0 par moyenne
ratings_mean = np.mean(ratings_matrix[ratings_matrix > 0])
ratings_demeaned = ratings_matrix.copy()
ratings_demeaned[ratings_demeaned == 0] = ratings_mean

# SVD
U, sigma, Vt = svds(ratings_demeaned, k=2)  # k = nb de facteurs latents

# Reconstruction
sigma_diag = np.diag(sigma)
predicted_ratings = np.dot(np.dot(U, sigma_diag), Vt)

def recommend_svd(user_id, predicted_ratings, original_ratings, n_recommendations=3):
    """Recommandations basées sur factorisation matricielle"""
    
    user_predictions = predicted_ratings[user_id]
    unrated_items = np.where(original_ratings[user_id] == 0)[0]
    
    # Trier par rating prédit
    recommendations = sorted(
        [(item_id, user_predictions[item_id]) for item_id in unrated_items],
        key=lambda x: x[1],
        reverse=True
    )
    
    return recommendations[:n_recommendations]


# --- CONTENT-BASED FILTERING (basé sur contenu) ---

# Features des items (ex: genre de films)
item_features = np.array([
    [1, 0, 1],  # Item 0: Action, Sci-Fi
    [1, 1, 0],  # Item 1: Action, Comedy
    [0, 1, 0],  # Item 2: Comedy
    [0, 0, 1],  # Item 3: Sci-Fi
])

# Similarité entre items basée sur features
content_similarity = cosine_similarity(item_features)

def recommend_content_based(user_id, ratings_matrix, content_similarity, n_recommendations=3):
    """Recommandations basées sur similarité de contenu"""
    
    user_ratings = ratings_matrix[user_id]
    rated_items = np.where(user_ratings > 0)[0]
    unrated_items = np.where(user_ratings == 0)[0]
    
    # Profil utilisateur (moyenne pondérée des items aimés)
    user_profile = np.average(
        item_features[rated_items],
        axis=0,
        weights=user_ratings[rated_items]
    )
    
    # Similarité entre profil utilisateur et items non notés
    predictions = []
    for item_id in unrated_items:
        similarity = cosine_similarity(
            user_profile.reshape(1, -1),
            item_features[item_id].reshape(1, -1)
        )[0, 0]
        predictions.append((item_id, similarity))
    
    predictions.sort(key=lambda x: x[1], reverse=True)
    
    return predictions[:n_recommendations]


# --- ÉVALUATION DES RECOMMANDATIONS ---

def evaluate_recommendations(true_ratings, predicted_ratings, k=10):
    """Métriques pour systèmes de recommandation"""
    
    from sklearn.metrics import mean_squared_error, mean_absolute_error
    
    # RMSE et MAE (pour ratings)
    mask = true_ratings > 0  # Seulement ratings existants
    rmse = np.sqrt(mean_squared_error(
        true_ratings[mask], 
        predicted_ratings[mask]
    ))
    mae = mean_absolute_error(
        true_ratings[mask],
        predicted_ratings[mask]
    )
    
    print(f"RMSE: {rmse:.3f}")
    print(f"MAE: {mae:.3f}")
    
    # Precision@K et Recall@K (pour ranking)
    # À implémenter selon vos besoins spécifiques


[OK] 32. A/B TESTING ET INFÉRENCE CAUSALE


"""
Tester l'impact de changements et établir des relations causales.
"""

from scipy import stats
import numpy as np

# --- TEST T (comparer deux groupes) ---

def ab_test_ttest(control, treatment, alpha=0.05):
    """Test t pour A/B testing"""
    
    # Test de Welch (variances inégales)
    t_stat, p_value = stats.ttest_ind(treatment, control, equal_var=False)
    
    # Statistiques descriptives
    control_mean = np.mean(control)
    treatment_mean = np.mean(treatment)
    lift = (treatment_mean - control_mean) / control_mean * 100
    
    print(f"Groupe contrôle: {control_mean:.3f} (n={len(control)})")
    print(f"Groupe traitement: {treatment_mean:.3f} (n={len(treatment)})")
    print(f"Lift: {lift:.2f}%")
    print(f"t-statistic: {t_stat:.3f}")
    print(f"p-value: {p_value:.4f}")
    
    if p_value < alpha:
        print(f"[OK] Différence significative (α={alpha})")
    else:
        print(f"[X] Pas de différence significative (α={alpha})")
    
    # Intervalle de confiance
    diff_mean = treatment_mean - control_mean
    diff_se = np.sqrt(
        np.var(treatment, ddof=1) / len(treatment) +
        np.var(control, ddof=1) / len(control)
    )
    ci_lower = diff_mean - 1.96 * diff_se
    ci_upper = diff_mean + 1.96 * diff_se
    
    print(f"95% CI: [{ci_lower:.3f}, {ci_upper:.3f}]")
    
    return {
        'lift': lift,
        'p_value': p_value,
        'significant': p_value < alpha,
        'ci': (ci_lower, ci_upper)
    }


# --- TEST CHI-CARRÉ (pour proportions) ---

def ab_test_proportions(control_success, control_total, 
                        treatment_success, treatment_total, alpha=0.05):
    """Test chi-carré pour taux de conversion"""
    
    # Table de contingence
    observed = np.array([
        [control_success, control_total - control_success],
        [treatment_success, treatment_total - treatment_success]
    ])
    
    chi2, p_value, dof, expected = stats.chi2_contingency(observed)
    
    # Taux de conversion
    control_rate = control_success / control_total
    treatment_rate = treatment_success / treatment_total
    lift = (treatment_rate - control_rate) / control_rate * 100
    
    print(f"Taux contrôle: {control_rate:.2%} ({control_success}/{control_total})")
    print(f"Taux traitement: {treatment_rate:.2%} ({treatment_success}/{treatment_total})")
    print(f"Lift: {lift:.2f}%")
    print(f"Chi-carré: {chi2:.3f}")
    print(f"p-value: {p_value:.4f}")
    
    if p_value < alpha:
        print(f"[OK] Différence significative (α={alpha})")
    else:
        print(f"[X] Pas de différence significative (α={alpha})")
    
    return {
        'lift': lift,
        'p_value': p_value,
        'significant': p_value < alpha
    }


# --- CALCUL DE TAILLE D'ÉCHANTILLON ---

def calculate_sample_size(baseline_rate, mde, alpha=0.05, power=0.8):
    """
    Calculer taille d'échantillon nécessaire pour A/B test
    
    baseline_rate: taux de conversion actuel
    mde: Minimum Detectable Effect (différence minimale à détecter)
    alpha: niveau de signification
    power: puissance statistique (1 - β)
    """
    
    from statsmodels.stats.power import zt_ind_solve_power
    
    # Effet standardisé
    effect_size = mde / np.sqrt(baseline_rate * (1 - baseline_rate))
    
    # Taille par groupe
    n_per_group = zt_ind_solve_power(
        effect_size=effect_size,
        alpha=alpha,
        power=power,
        alternative='two-sided'
    )
    
    total_n = n_per_group * 2
    
    print(f"Taille d'échantillon par groupe: {int(np.ceil(n_per_group))}")
    print(f"Taille totale: {int(np.ceil(total_n))}")
    
    return int(np.ceil(n_per_group))


# --- BAYESIAN A/B TESTING ---

def bayesian_ab_test(control_success, control_total,
                     treatment_success, treatment_total,
                     n_samples=100000):
    """
    A/B test bayésien (plus intuitif que fréquentiste)
    Retourne la probabilité que le traitement soit meilleur
    """
    
    # Prior: Beta(1, 1) = uniforme
    # Posterior: Beta(succès + 1, échecs + 1)
    
    # Échantillonner des posteriors
    control_samples = np.random.beta(
        control_success + 1,
        control_total - control_success + 1,
        n_samples
    )
    
    treatment_samples = np.random.beta(
        treatment_success + 1,
        treatment_total - treatment_success + 1,
        n_samples
    )
    
    # Probabilité que traitement > contrôle
    prob_treatment_better = np.mean(treatment_samples > control_samples)
    
    # Lift attendu
    expected_lift = np.mean(
        (treatment_samples - control_samples) / control_samples * 100
    )
    
    # Intervalle crédible à 95%
    lift_samples = (treatment_samples - control_samples) / control_samples * 100
    ci_lower, ci_upper = np.percentile(lift_samples, [2.5, 97.5])
    
    print(f"Probabilité que traitement soit meilleur: {prob_treatment_better:.2%}")
    print(f"Lift attendu: {expected_lift:.2f}%")
    print(f"95% Intervalle crédible: [{ci_lower:.2f}%, {ci_upper:.2f}%]")
    
    # Visualisation
    plt.figure(figsize=(12, 5))
    
    plt.subplot(1, 2, 1)
    plt.hist(control_samples, bins=50, alpha=0.5, label='Contrôle', density=True)
    plt.hist(treatment_samples, bins=50, alpha=0.5, label='Traitement', density=True)
    plt.xlabel('Taux de conversion')
    plt.ylabel('Densité')
    plt.legend()
    plt.title('Distributions posterieures')
    
    plt.subplot(1, 2, 2)
    plt.hist(lift_samples, bins=50, edgecolor='black')
    plt.axvline(0, color='red', linestyle='--', label='Pas de différence')
    plt.xlabel('Lift (%)')
    plt.ylabel('Fréquence')
    plt.legend()
    plt.title('Distribution du lift')
    
    plt.tight_layout()
    plt.show()
    
    return {
        'prob_better': prob_treatment_better,
        'expected_lift': expected_lift,
        'ci': (ci_lower, ci_upper)
    }


# --- SEQUENTIAL TESTING (monitoring continu) ---

def sequential_test(control_data, treatment_data, alpha=0.05, 
                    checks_per_experiment=20):
    """
    Test séquentiel avec correction pour multiple testing
    Permet de regarder les résultats pendant l'expérience
    """
    
    # Ajuster alpha pour multiple comparisons (correction de Bonferroni)
    adjusted_alpha = alpha / checks_per_experiment
    
    print(f"Alpha ajusté: {adjusted_alpha:.4f}")
    
    # Effectuer le test
    result = ab_test_ttest(control_data, treatment_data, alpha=adjusted_alpha)
    
    return result


[OK] 33. MODÈLES DE SURVIE (SURVIVAL ANALYSIS)


"""
Analyser le temps jusqu'à un événement (churn, conversion, décès, etc.).
Gère les données censurées (observations incomplètes).
"""

from lifelines import KaplanMeierFitter, CoxPHFitter
from lifelines.statistics import logrank_test

# --- KAPLAN-MEIER (estimateur non-paramétrique) ---

# Données exemple
durations = [5, 6, 6, 2.5, 4, 4, 7, 5, 3, 8, 8, 12]  # Temps observés
event_observed = [1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1]  # 1=événement, 0=censuré

# Fitter
kmf = KaplanMeierFitter()
kmf.fit(durations, event_observed)

# Visualiser courbe de survie
kmf.plot_survival_function()
plt.title('Courbe de Survie Kaplan-Meier')
plt.xlabel('Temps')
plt.ylabel('Probabilité de survie')
plt.show()

# Statistiques
print(f"Médiane de survie: {kmf.median_survival_time_}")
print(f"Survie à t=5: {kmf.survival_function_at_times(5).values[0]:.2%}")


# --- COMPARER DEUX GROUPES ---

# Données par groupe
durations_A = [5, 6, 8, 10, 12, 15]
events_A = [1, 1, 1, 0, 1, 1]
durations_B = [3, 4, 5, 6, 7, 9]
events_B = [1, 1, 1, 1, 0, 1]

# Fitter chaque groupe
kmf_A = KaplanMeierFitter()
kmf_A.fit(durations_A, events_A, label='Groupe A')

kmf_B = KaplanMeierFitter()
kmf_B.fit(durations_B, events_B, label='Groupe B')

# Visualiser
plt.figure(figsize=(10, 6))
kmf_A.plot_survival_function()
kmf_B.plot_survival_function()
plt.title('Comparaison des courbes de survie')
plt.show()

# Test log-rank (tester différence entre groupes)
results = logrank_test(durations_A, durations_B, events_A, events_B)
print(f"Test log-rank p-value: {results.p_value:.4f}")


# --- COX PROPORTIONAL HAZARDS (modèle de régression) ---

# DataFrame avec covariables
df_survival = pd.DataFrame({
    'duration': [5, 6, 6, 2.5, 4, 4, 7, 5, 3, 8],
    'event': [1, 1, 0, 1, 1, 0, 1, 1, 1, 0],
    'age': [45, 55, 50, 40, 60, 35, 70, 50, 45, 65],
    'treatment': [0, 1, 0, 1, 1, 0, 1, 0, 1, 0],
    'risk_score': [2.5, 3.0, 2.0, 4.0, 3.5, 1.5, 4.5, 2.8, 3.2, 4.0]
})

# Fitter Cox PH
cph = CoxPHFitter()
cph.fit(df_survival, duration_col='duration', event_col='event')

# Afficher résultats
cph.print_summary()

# Hazard ratios (interprétation)
print("\nHazard Ratios:")
print(np.exp(cph.params_))

# Prédire survie pour un nouveau patient
new_patient = pd.DataFrame({
    'age': [55],
    'treatment': [1],
    'risk_score': [3.0]
})
survival_func = cph.predict_survival_function(new_patient)

plt.figure(figsize=(10, 6))
survival_func.plot()
plt.title('Courbe de survie prédite')
plt.xlabel('Temps')
plt.ylabel('Probabilité de survie')
plt.show()


[OK] 34. REINFORCEMENT LEARNING (Bases)


"""
Apprentissage par renforcement: agent apprend à prendre des décisions
par essai-erreur pour maximiser une récompense.
"""

import numpy as np
import random

# --- Q-LEARNING (algorithme simple) ---

class QLearningAgent:
    """Agent Q-Learning pour environnement discret"""
    
    def __init__(self, n_states, n_actions, learning_rate=0.1,
                 discount_factor=0.95, epsilon=0.1):
        self.n_states = n_states
        self.n_actions = n_actions
        self.lr = learning_rate
        self.gamma = discount_factor
        self.epsilon = epsilon
        
        # Q-table: état × action
        self.q_table = np.zeros((n_states, n_actions))
    
    def select_action(self, state):
        """Epsilon-greedy: exploration vs exploitation"""
        if random.random() < self.epsilon:
            return random.randint(0, self.n_actions - 1)  # Explore
        else:
            return np.argmax(self.q_table[state])  # Exploit
    
    def update(self, state, action, reward, next_state):
        """Mise à jour Q-learning"""
        # Q(s,a) <- Q(s,a) + α[r + γ max Q(s',a') - Q(s,a)]
        best_next_action = np.argmax(self.q_table[next_state])
        td_target = reward + self.gamma * self.q_table[next_state, best_next_action]
        td_error = td_target - self.q_table[state, action]
        self.q_table[state, action] += self.lr * td_error
    
    def train(self, env, n_episodes=1000):
        """Entraîner l'agent"""
        rewards_history = []
        
        for episode in range(n_episodes):
            state = env.reset()
            total_reward = 0
            done = False
            
            while not done:
                action = self.select_action(state)
                next_state, reward, done = env.step(action)
                
                self.update(state, action, reward, next_state)
                
                state = next_state
                total_reward += reward
            
            rewards_history.append(total_reward)
            
            if (episode + 1) % 100 == 0:
                avg_reward = np.mean(rewards_history[-100:])
                print(f"Episode {episode + 1}: Avg Reward = {avg_reward:.2f}")
        
        return rewards_history


# --- ENVIRONNEMENT SIMPLE (Grid World) ---

class GridWorld:
    """Environnement grille simple"""
    
    def __init__(self, size=5):
        self.size = size
        self.n_states = size * size
        self.n_actions = 4  # Haut, Bas, Gauche, Droite
        
        self.goal_state = size * size - 1  # Coin bas-droite
        self.reset()
    
    def reset(self):
        """Réinitialiser au coin haut-gauche"""
        self.state = 0
        return self.state
    
    def step(self, action):
        """Exécuter une action"""
        row = self.state // self.size
        col = self.state % self.size
        
        # Actions: 0=Haut, 1=Bas, 2=Gauche, 3=Droite
        if action == 0 and row > 0:
            row -= 1
        elif action == 1 and row < self.size - 1:
            row += 1
        elif action == 2 and col > 0:
            col -= 1
        elif action == 3 and col < self.size - 1:
            col += 1
        
        self.state = row * self.size + col
        
        # Récompense
        if self.state == self.goal_state:
            reward = 10
            done = True
        else:
            reward = -0.1  # Pénalité pour chaque pas
            done = False
        
        return self.state, reward, done


# --- ENTRAÎNER UN AGENT ---

# Créer environnement et agent
env = GridWorld(size=5)
agent = QLearningAgent(
    n_states=env.n_states,
    n_actions=env.n_actions,
    learning_rate=0.1,
    discount_factor=0.99,
    epsilon=0.1
)

# Entraîner
rewards = agent.train(env, n_episodes=500)

# Visualiser progression
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(rewards)
plt.xlabel('Episode')
plt.ylabel('Total Reward')
plt.title('Progression de l\'apprentissage')

plt.subplot(1, 2, 2)
window_size = 50
smoothed = np.convolve(rewards, np.ones(window_size)/window_size, mode='valid')
plt.plot(smoothed)
plt.xlabel('Episode')
plt.ylabel('Moyenne mobile des récompenses')
plt.title(f'Lissé (fenêtre={window_size})')

plt.tight_layout()
plt.show()

# Afficher politique apprise
print("\nPolitique optimale (Q-table):")
print("Actions: 0=^, 1=v, 2=<-, 3=->")
for i in range(env.size):
    for j in range(env.size):
        state = i * env.size + j
        best_action = np.argmax(agent.q_table[state])
        arrows = ['^', 'v', '<-', '->']
        print(arrows[best_action], end=' ')
    print()


[OK] 35. GRAPH MACHINE LEARNING (Bases)


"""
Apprentissage sur graphes: réseaux sociaux, molécules, systèmes de recommandation.
"""

import networkx as nx

# --- CRÉATION ET MANIPULATION DE GRAPHES ---

# Créer graphe
G = nx.Graph()

# Ajouter nœuds et arêtes
G.add_nodes_from([1, 2, 3, 4, 5])
G.add_edges_from([(1, 2), (1, 3), (2, 4), (3, 4), (4, 5)])

# Attributs
G.nodes[1]['feature'] = 0.5
G.edges[1, 2]['weight'] = 2.0

# Visualiser
plt.figure(figsize=(8, 6))
pos = nx.spring_layout(G, seed=42)
nx.draw(G, pos, with_labels=True, node_color='lightblue',
        node_size=500, font_size=16, font_weight='bold')
plt.title('Graphe Simple')
plt.show()


# --- MÉTRIQUES DE GRAPHE ---

# Centralité (importance des nœuds)
degree_centrality = nx.degree_centrality(G)
betweenness_centrality = nx.betweenness_centrality(G)
closeness_centrality = nx.closeness_centrality(G)
eigenvector_centrality = nx.eigenvector_centrality(G)

print("Centralité de degré:", degree_centrality)
print("Centralité d'intermédiarité:", betweenness_centrality)

# Clustering coefficient
clustering = nx.clustering(G)
print("Coefficient de clustering:", clustering)

# Plus court chemin
shortest_path = nx.shortest_path(G, source=1, target=5)
print("Plus court chemin 1->5:", shortest_path)

# Composantes connexes
connected_components = list(nx.connected_components(G))
print("Composantes connexes:", connected_components)


# --- DÉTECTION DE COMMUNAUTÉS ---

from networkx.algorithms import community

# Louvain (avec python-louvain)
# communities = community.louvain_communities(G)

# Greedy modularity
communities = community.greedy_modularity_communities(G)

print("\nCommunautés détectées:")
for i, comm in enumerate(communities):
    print(f"Communauté {i}: {comm}")

# Visualiser communautés
plt.figure(figsize=(8, 6))
colors = ['red', 'blue', 'green', 'yellow', 'purple']
node_colors = []
for node in G.nodes():
    for i, comm in enumerate(communities):
        if node in comm:
            node_colors.append(colors[i])
            break

nx.draw(G, pos, node_color=node_colors, with_labels=True,
        node_size=500, font_size=16, font_weight='bold')
plt.title('Détection de Communautés')
plt.show()


# --- NODE EMBEDDINGS (Node2Vec) ---

try:
    from node2vec import Node2Vec
    
    # Générer embeddings
    node2vec = Node2Vec(G, dimensions=64, walk_length=30, num_walks=200)
    model = node2vec.fit(window=10, min_count=1)
    
    # Obtenir embedding d'un nœud
    embedding = model.wv[str(1)]
    
    # Nœuds similaires
    similar_nodes = model.wv.most_similar(str(1))
    print("\nNœuds similaires au nœud 1:", similar_nodes)

except ImportError:
    print("Installer node2vec: pip install node2vec")


# --- GRAPH NEURAL NETWORKS (avec PyTorch Geometric) ---

"""
# Exemple avec PyTorch Geometric (installation requise)
import torch
from torch_geometric.nn import GCNConv
from torch_geometric.data import Data

# Créer données de graphe
edge_index = torch.tensor([[0, 1, 1, 2],
                           [1, 0, 2, 1]], dtype=torch.long)
x = torch.tensor([[1], [2], [3]], dtype=torch.float)

data = Data(x=x, edge_index=edge_index)

# Graph Convolutional Network
class GCN(torch.nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels):
        super().__init__()
        self.conv1 = GCNConv(in_channels, hidden_channels)
        self.conv2 = GCNConv(hidden_channels, out_channels)
    
    def forward(self, x, edge_index):
        x = self.conv1(x, edge_index).relu()
        x = self.conv2(x, edge_index)
        return x

model = GCN(in_channels=1, hidden_channels=16, out_channels=2)
"""


[OK] 36. MÉTA-LEARNING ET AUTOML


"""
Automatiser la sélection et l'optimisation de modèles.
"""

# --- AUTO-SKLEARN (AutoML) ---

"""
from autosklearn.classification import AutoSklearnClassifier

# Recherche automatique de modèle
automl = AutoSklearnClassifier(
    time_left_for_this_task=300,  # 5 minutes
    per_run_time_limit=30,
    n_jobs=-1
)

automl.fit(X_train, y_train)

# Meilleur modèle trouvé
print(automl.show_models())

# Prédire
predictions = automl.predict(X_test)
"""


# --- TPOT (Genetic Programming AutoML) ---

"""
from tpot import TPOTClassifier

# Recherche par algorithme génétique
tpot = TPOTClassifier(
    generations=5,
    population_size=20,
    cv=5,
    random_state=42,
    verbosity=2,
    n_jobs=-1
)

tpot.fit(X_train, y_train)

# Score
print(f"Score: {tpot.score(X_test, y_test):.4f}")

# Exporter meilleur pipeline
tpot.export('best_pipeline.py')
"""


# --- OPTUNA (Optimisation d'hyperparamètres avancée) ---

import optuna

def objective(trial):
    """Fonction objectif pour Optuna"""
    
    # Suggérer hyperparamètres
    n_estimators = trial.suggest_int('n_estimators', 50, 300)
    max_depth = trial.suggest_int('max_depth', 3, 15)
    min_samples_split = trial.suggest_int('min_samples_split', 2, 20)
    learning_rate = trial.suggest_float('learning_rate', 0.01, 0.3, log=True)
    
    # Créer et évaluer modèle
    model = GradientBoostingClassifier(
        n_estimators=n_estimators,
        max_depth=max_depth,
        min_samples_split=min_samples_split,
        learning_rate=learning_rate,
        random_state=42
    )
    
    # Cross-validation
    scores = cross_val_score(model, X_train, y_train, cv=3, scoring='accuracy')
    
    return scores.mean()

# Optimiser
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=50, show_progress_bar=True)

# Meilleurs paramètres
print("\nMeilleurs hyperparamètres:")
print(study.best_params)
print(f"Meilleur score: {study.best_value:.4f}")

# Visualiser historique
optuna.visualization.plot_optimization_history(study)
optuna.visualization.plot_param_importances(study)


# --- HYPERBAND (early stopping efficace) ---

from sklearn.model_selection import HalvingGridSearchCV

# Halving Grid Search (élimine progressivement mauvais candidats)
param_grid = {
    'n_estimators': [50, 100, 200, 300],
    'max_depth': [3, 5, 7, 10],
    'learning_rate': [0.01, 0.05, 0.1, 0.2]
}

halving_search = HalvingGridSearchCV(
    GradientBoostingClassifier(),
    param_grid,
    factor=2,  # Réduire ressources par facteur 2 à chaque étape
    cv=5,
    random_state=42
)

halving_search.fit(X_train, y_train)
print(f"Meilleurs params: {halving_search.best_params_}")


[OK] 37. EXPLAINABLE AI (XAI) - TECHNIQUES AVANCÉES


"""
Techniques avancées pour interpréter et expliquer les modèles complexes.
"""

# --- INTEGRATED GRADIENTS (pour Deep Learning) ---

"""
# Avec TensorFlow
import tensorflow as tf

def integrated_gradients(model, input_data, baseline=None, steps=50):
    '''
    Calculer Integrated Gradients pour une prédiction
    Attribue l'importance à chaque feature
    '''
    
    if baseline is None:
        baseline = np.zeros_like(input_data)
    
    # Interpoler entre baseline et input
    alphas = np.linspace(0, 1, steps)
    interpolated = np.array([
        baseline + alpha * (input_data - baseline)
        for alpha in alphas
    ])
    
    # Calculer gradients
    with tf.GradientTape() as tape:
        tape.watch(interpolated)
        predictions = model(interpolated)
    
    gradients = tape.gradient(predictions, interpolated)
    
    # Intégrer gradients
    integrated_grads = (input_data - baseline) * np.mean(gradients, axis=0)
    
    return integrated_grads
"""


# --- COUNTERFACTUAL EXPLANATIONS ---

def generate_counterfactual(model, instance, target_class, 
                           feature_ranges, max_iterations=1000):
    """
    Générer un exemple contrefactuel:
    "Si X avait été différent de cette façon, la prédiction aurait été Y"
    """
    
    counterfactual = instance.copy()
    current_pred = model.predict([counterfactual])[0]
    
    for iteration in range(max_iterations):
        if current_pred == target_class:
            break
        
        # Modifier aléatoirement une feature
        feature_idx = np.random.randint(len(counterfactual))
        min_val, max_val = feature_ranges[feature_idx]
        
        # Perturbation
        counterfactual[feature_idx] += np.random.uniform(-0.1, 0.1) * (max_val - min_val)
        counterfactual[feature_idx] = np.clip(counterfactual[feature_idx], min_val, max_val)
        
        current_pred = model.predict([counterfactual])[0]
    
    if current_pred == target_class:
        print(f"Contrefactuel trouvé après {iteration} itérations")
        print(f"Instance originale: {instance}")
        print(f"Contrefactuel: {counterfactual}")
        print(f"Différences: {counterfactual - instance}")
    else:
        print("Aucun contrefactuel trouvé")
    
    return counterfactual


# --- ANCHORS (explications basées sur règles) ---

"""
from anchor import anchor_tabular

# Créer explainer
explainer = anchor_tabular.AnchorTabularExplainer(
    class_names=['Class 0', 'Class 1'],
    feature_names=feature_names,
    train_data=X_train
)

# Expliquer une prédiction
explanation = explainer.explain_instance(
    X_test[0],
    model.predict,
    threshold=0.95
)

print('Anchor:', explanation.anchor)
print('Precision:', explanation.precision)
print('Coverage:', explanation.coverage)
"""


[OK] 38. PYTORCH - DEEP LEARNING ALTERNATIF


"""
PyTorch: framework de deep learning privilégié par la recherche.
Plus flexible que TensorFlow, excellent pour prototypage et recherche.
Différences clés: définition dynamique du graphe, syntaxe pythonique.
"""

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F

# --- BASES PYTORCH ---

# Tensors (équivalent NumPy mais avec GPU)
x = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32)
y = torch.randn(2, 3)  # Aléatoire gaussien

# Opérations
z = torch.matmul(x, y)  # Multiplication matricielle
mean = x.mean()
std = x.std()

# Conversion NumPy <-> PyTorch
import numpy as np
np_array = x.numpy()  # Tensor -> NumPy
tensor = torch.from_numpy(np_array)  # NumPy -> Tensor

# GPU (CUDA)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Device: {device}")

x_gpu = x.to(device)  # Déplacer sur GPU
x_cpu = x_gpu.cpu()   # Ramener sur CPU


# --- DATASET CUSTOM ---

class CustomDataset(Dataset):
    """Dataset personnalisé pour PyTorch"""
    
    def __init__(self, X, y, transform=None):
        """
        X: features (numpy array ou list)
        y: labels (numpy array ou list)
        transform: transformations optionnelles
        """
        self.X = torch.FloatTensor(X)
        self.y = torch.LongTensor(y)
        self.transform = transform
    
    def __len__(self):
        """Retourne taille du dataset"""
        return len(self.X)
    
    def __getitem__(self, idx):
        """Retourne un échantillon"""
        sample = self.X[idx]
        label = self.y[idx]
        
        if self.transform:
            sample = self.transform(sample)
        
        return sample, label


# Créer DataLoader
dataset = CustomDataset(X_train, y_train)
dataloader = DataLoader(
    dataset,
    batch_size=32,
    shuffle=True,      # Mélanger à chaque epoch
    num_workers=4,     # Chargement parallèle
    pin_memory=True    # Optimisation GPU
)

# Itérer sur batches
for batch_idx, (data, target) in enumerate(dataloader):
    data, target = data.to(device), target.to(device)
    # Entraînement...


# --- RÉSEAU DE NEURONES SIMPLE (MLP) ---

class MLP(nn.Module):
    """Multi-Layer Perceptron"""
    
    def __init__(self, input_dim, hidden_dims, output_dim, dropout=0.3):
        super(MLP, self).__init__()
        
        # Définir couches
        self.fc1 = nn.Linear(input_dim, hidden_dims[0])
        self.fc2 = nn.Linear(hidden_dims[0], hidden_dims[1])
        self.fc3 = nn.Linear(hidden_dims[1], output_dim)
        
        # Batch Normalization
        self.bn1 = nn.BatchNorm1d(hidden_dims[0])
        self.bn2 = nn.BatchNorm1d(hidden_dims[1])
        
        # Dropout
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x):
        """Forward pass"""
        # Couche 1
        x = self.fc1(x)
        x = self.bn1(x)
        x = F.relu(x)
        x = self.dropout(x)
        
        # Couche 2
        x = self.fc2(x)
        x = self.bn2(x)
        x = F.relu(x)
        x = self.dropout(x)
        
        # Couche sortie
        x = self.fc3(x)
        
        return x


# Instancier modèle
model = MLP(
    input_dim=784,          # Ex: MNIST 28x28
    hidden_dims=[256, 128],
    output_dim=10,          # 10 classes
    dropout=0.3
).to(device)

# Voir architecture
print(model)

# Compter paramètres
total_params = sum(p.numel() for p in model.parameters())
trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f"Total params: {total_params:,}")
print(f"Trainable params: {trainable_params:,}")


# --- ENTRAÎNEMENT ---

# Loss et optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)

# Learning rate scheduler
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, mode='min', factor=0.5, patience=5, verbose=True
)

def train_epoch(model, dataloader, criterion, optimizer, device):
    """Entraîner une epoch"""
    model.train()  # Mode entraînement
    
    running_loss = 0.0
    correct = 0
    total = 0
    
    for batch_idx, (data, target) in enumerate(dataloader):
        # Déplacer sur device
        data, target = data.to(device), target.to(device)
        
        # Zero gradients
        optimizer.zero_grad()
        
        # Forward pass
        outputs = model(data)
        loss = criterion(outputs, target)
        
        # Backward pass
        loss.backward()
        
        # Gradient clipping (éviter explosion)
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
        
        # Update weights
        optimizer.step()
        
        # Statistiques
        running_loss += loss.item()
        _, predicted = outputs.max(1)
        total += target.size(0)
        correct += predicted.eq(target).sum().item()
    
    epoch_loss = running_loss / len(dataloader)
    epoch_acc = 100. * correct / total
    
    return epoch_loss, epoch_acc


def validate(model, dataloader, criterion, device):
    """Valider le modèle"""
    model.eval()  # Mode évaluation
    
    running_loss = 0.0
    correct = 0
    total = 0
    
    with torch.no_grad():  # Pas de gradients
        for data, target in dataloader:
            data, target = data.to(device), target.to(device)
            
            outputs = model(data)
            loss = criterion(outputs, target)
            
            running_loss += loss.item()
            _, predicted = outputs.max(1)
            total += target.size(0)
            correct += predicted.eq(target).sum().item()
    
    val_loss = running_loss / len(dataloader)
    val_acc = 100. * correct / total
    
    return val_loss, val_acc


# Boucle d'entraînement complète
n_epochs = 50
best_val_loss = float('inf')
patience_counter = 0
patience = 10

history = {'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': []}

for epoch in range(n_epochs):
    # Entraîner
    train_loss, train_acc = train_epoch(
        model, train_loader, criterion, optimizer, device
    )
    
    # Valider
    val_loss, val_acc = validate(model, val_loader, criterion, device)
    
    # Sauvegarder historique
    history['train_loss'].append(train_loss)
    history['train_acc'].append(train_acc)
    history['val_loss'].append(val_loss)
    history['val_acc'].append(val_acc)
    
    # Learning rate scheduling
    scheduler.step(val_loss)
    
    # Early stopping
    if val_loss < best_val_loss:
        best_val_loss = val_loss
        patience_counter = 0
        # Sauvegarder meilleur modèle
        torch.save(model.state_dict(), 'best_model.pth')
    else:
        patience_counter += 1
    
    # Afficher progression
    print(f"Epoch {epoch+1}/{n_epochs}")
    print(f"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}%")
    print(f"Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.2f}%")
    print(f"LR: {optimizer.param_groups[0]['lr']:.6f}")
    print("-" * 60)
    
    if patience_counter >= patience:
        print(f"Early stopping at epoch {epoch+1}")
        break

# Charger meilleur modèle
model.load_state_dict(torch.load('best_model.pth'))


# --- CNN POUR IMAGES ---

class CNN(nn.Module):
    """Convolutional Neural Network"""
    
    def __init__(self, num_classes=10):
        super(CNN, self).__init__()
        
        # Bloc convolutionnel 1
        self.conv1 = nn.Conv2d(
            in_channels=3,      # RGB
            out_channels=32,
            kernel_size=3,
            padding=1
        )
        self.bn1 = nn.BatchNorm2d(32)
        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
        
        # Bloc convolutionnel 2
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(64)
        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
        
        # Bloc convolutionnel 3
        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, padding=1)
        self.bn3 = nn.BatchNorm2d(128)
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)
        
        # Couches fully connected
        self.fc1 = nn.Linear(128 * 4 * 4, 256)  # Adapter selon taille image
        self.dropout = nn.Dropout(0.5)
        self.fc2 = nn.Linear(256, num_classes)
    
    def forward(self, x):
        # Bloc 1
        x = self.conv1(x)
        x = self.bn1(x)
        x = F.relu(x)
        x = self.pool1(x)
        
        # Bloc 2
        x = self.conv2(x)
        x = self.bn2(x)
        x = F.relu(x)
        x = self.pool2(x)
        
        # Bloc 3
        x = self.conv3(x)
        x = self.bn3(x)
        x = F.relu(x)
        x = self.pool3(x)
        
        # Aplatir
        x = x.view(x.size(0), -1)
        
        # FC layers
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout(x)
        x = self.fc2(x)
        
        return x


# --- RNN/LSTM POUR SÉQUENCES ---

class LSTMModel(nn.Module):
    """LSTM pour séquences (texte, séries temporelles)"""
    
    def __init__(self, vocab_size, embedding_dim, hidden_dim, 
                 num_layers, output_dim, dropout=0.3):
        super(LSTMModel, self).__init__()
        
        # Embedding layer (pour texte)
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        
        # LSTM
        self.lstm = nn.LSTM(
            embedding_dim,
            hidden_dim,
            num_layers=num_layers,
            dropout=dropout if num_layers > 1 else 0,
            batch_first=True,
            bidirectional=True  # LSTM bidirectionnel
        )
        
        # FC layer
        self.fc = nn.Linear(hidden_dim * 2, output_dim)  # *2 car bidirectionnel
        self.dropout = nn.Dropout(dropout)
    
    def forward(self, x):
        # x shape: (batch_size, seq_length)
        
        # Embedding
        embedded = self.embedding(x)  # (batch, seq_len, emb_dim)
        
        # LSTM
        lstm_out, (hidden, cell) = self.lstm(embedded)
        
        # Prendre dernière sortie (ou pooling)
        # Option 1: dernière sortie
        output = lstm_out[:, -1, :]
        
        # Option 2: moyenne sur séquence
        # output = lstm_out.mean(dim=1)
        
        # FC
        output = self.dropout(output)
        output = self.fc(output)
        
        return output


# --- TRANSFER LEARNING ---

import torchvision.models as models
import torchvision.transforms as transforms

# Charger modèle pré-entraîné
resnet = models.resnet50(pretrained=True)

# Geler couches pré-entraînées
for param in resnet.parameters():
    param.requires_grad = False

# Remplacer dernière couche
num_features = resnet.fc.in_features
resnet.fc = nn.Linear(num_features, num_classes)

# Déplacer sur GPU
resnet = resnet.to(device)

# Fine-tuning: dégeler dernières couches
for param in resnet.layer4.parameters():
    param.requires_grad = True

# Optimizer avec learning rates différents
optimizer = optim.Adam([
    {'params': resnet.layer4.parameters(), 'lr': 1e-4},
    {'params': resnet.fc.parameters(), 'lr': 1e-3}
])


# --- DATA AUGMENTATION ---

train_transforms = transforms.Compose([
    transforms.RandomResizedCrop(224),
    transforms.RandomHorizontalFlip(),
    transforms.RandomRotation(15),
    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                        std=[0.229, 0.224, 0.225])
])

val_transforms = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                        std=[0.229, 0.224, 0.225])
])


# --- MIXED PRECISION TRAINING (plus rapide) ---

from torch.cuda.amp import autocast, GradScaler

scaler = GradScaler()

for data, target in train_loader:
    data, target = data.to(device), target.to(device)
    
    optimizer.zero_grad()
    
    # Forward avec mixed precision
    with autocast():
        outputs = model(data)
        loss = criterion(outputs, target)
    
    # Backward avec scaling
    scaler.scale(loss).backward()
    scaler.step(optimizer)
    scaler.update()


# --- SAUVEGARDER ET CHARGER ---

# Sauvegarder
torch.save({
    'epoch': epoch,
    'model_state_dict': model.state_dict(),
    'optimizer_state_dict': optimizer.state_dict(),
    'loss': loss,
    'history': history
}, 'checkpoint.pth')

# Charger
checkpoint = torch.load('checkpoint.pth')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']

# Export vers ONNX (interopérabilité)
dummy_input = torch.randn(1, 3, 224, 224).to(device)
torch.onnx.export(
    model,
    dummy_input,
    "model.onnx",
    export_params=True,
    opset_version=11,
    input_names=['input'],
    output_names=['output']
)


# --- PYTORCH LIGHTNING (simplifie le code) ---

"""
import pytorch_lightning as pl

class LitModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.model = MLP(784, [256, 128], 10)
        self.criterion = nn.CrossEntropyLoss()
    
    def forward(self, x):
        return self.model(x)
    
    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        self.log('train_loss', loss)
        return loss
    
    def validation_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        self.log('val_loss', loss)
        return loss
    
    def configure_optimizers(self):
        return optim.Adam(self.parameters(), lr=0.001)

# Entraîner
trainer = pl.Trainer(max_epochs=10, gpus=1)
trainer.fit(model, train_loader, val_loader)
"""


# --- DEBUGGING ET PROFILING ---

# Vérifier gradients
for name, param in model.named_parameters():
    if param.grad is not None:
        print(f"{name}: grad mean={param.grad.mean():.6f}, "
              f"grad std={param.grad.std():.6f}")

# Profiling (trouver goulots d'étranglement)
with torch.autograd.profiler.profile(use_cuda=True) as prof:
    outputs = model(data)
    loss = criterion(outputs, target)
    loss.backward()

print(prof.key_averages().table(sort_by="cuda_time_total"))

# Détection d'anomalies (NaN, Inf)
torch.autograd.set_detect_anomaly(True)


# --- CONSEILS PYTORCH ---

"""
[OK] BONNES PRATIQUES

1. Toujours utiliser DataLoader avec num_workers > 0
2. Déplacer données sur GPU en batches, pas tout d'un coup
3. Utiliser mixed precision pour entraînement plus rapide
4. Gradient clipping pour stabilité
5. Learning rate warmup pour gros modèles
6. Valider régulièrement pendant entraînement
7. Sauvegarder checkpoints périodiquement

[ATTENTION] PIÈGES COURANTS

1. Oublier model.train() / model.eval()
2. Oublier optimizer.zero_grad()
3. Fuite mémoire: détacher tensors (x.detach())
4. Garder graphe de calcul: utiliser .item() pour scalaires
5. Dimensions incorrectes: utiliser .view() ou .reshape()

[RAPIDE] OPTIMISATIONS

1. torch.backends.cudnn.benchmark = True  # Auto-tune
2. Pin memory dans DataLoader
3. Précharger données pendant entraînement (prefetch)
4. Utiliser torch.jit.script() pour accélérer inférence
5. Quantization pour déploiement mobile
"""


[OK] 39. TRANSFORMERS ET NLP AVANCÉ


"""
Transformers: architecture révolutionnaire pour NLP (et maintenant Vision).
BERT, GPT, T5 pour classification, génération, question-answering, etc.
Hugging Face [HUGGING_FACE] : bibliothèque de référence pour modèles pré-entraînés.
"""

from transformers import (
    AutoTokenizer, AutoModel, AutoModelForSequenceClassification,
    AutoModelForQuestionAnswering, AutoModelForTokenClassification,
    Trainer, TrainingArguments, pipeline
)
import torch
from datasets import load_dataset

# --- UTILISATION SIMPLE AVEC PIPELINES ---

# Classification de sentiment
classifier = pipeline("sentiment-analysis")
result = classifier("I love this product!")
print(result)
# [{'label': 'POSITIVE', 'score': 0.9998}]

# Génération de texte
generator = pipeline("text-generation", model="gpt2")
result = generator("Once upon a time", max_length=50, num_return_sequences=2)

# Question Answering
qa_pipeline = pipeline("question-answering")
result = qa_pipeline(
    question="What is the capital of France?",
    context="France is a country in Europe. Its capital is Paris."
)
print(result)
# {'answer': 'Paris', 'score': 0.98}

# Named Entity Recognition
ner = pipeline("ner", grouped_entities=True)
result = ner("Apple Inc. was founded by Steve Jobs in California.")

# Traduction
translator = pipeline("translation_en_to_fr")
result = translator("Hello, how are you?")

# Résumé
summarizer = pipeline("summarization")
result = summarizer(long_text, max_length=130, min_length=30)

# Zero-shot classification (sans entraînement!)
classifier = pipeline("zero-shot-classification")
result = classifier(
    "This is a course about Python programming",
    candidate_labels=["education", "politics", "business"]
)


# --- CHARGER UN MODÈLE PRÉ-ENTRAÎNÉ ---

# Tokenizer (convertit texte en tokens/IDs)
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# Modèle
model = AutoModel.from_pretrained("bert-base-uncased")

# Exemple de tokenization
text = "Hello, how are you?"
inputs = tokenizer(
    text,
    padding=True,           # Pad à longueur max
    truncation=True,        # Tronquer si trop long
    max_length=512,         # Longueur max
    return_tensors="pt"     # Retourner tensors PyTorch
)

print(inputs)
# {'input_ids': tensor([[...]]), 'attention_mask': tensor([[...]])}

# Obtenir embeddings
with torch.no_grad():
    outputs = model(**inputs)
    embeddings = outputs.last_hidden_state  # (batch, seq_len, hidden_dim)

# Pooling (CLS token ou mean)
cls_embedding = embeddings[:, 0, :]  # [CLS] token
mean_embedding = embeddings.mean(dim=1)  # Moyenne


# --- FINE-TUNING POUR CLASSIFICATION ---

from transformers import AutoModelForSequenceClassification
from torch.utils.data import Dataset, DataLoader

class TextDataset(Dataset):
    """Dataset pour fine-tuning"""
    
    def __init__(self, texts, labels, tokenizer, max_length=128):
        self.texts = texts
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_length = max_length
    
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, idx):
        text = self.texts[idx]
        label = self.labels[idx]
        
        encoding = self.tokenizer(
            text,
            padding='max_length',
            truncation=True,
            max_length=self.max_length,
            return_tensors='pt'
        )
        
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'labels': torch.tensor(label, dtype=torch.long)
        }


# Charger modèle pour classification
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=2  # Binaire: positif/négatif
)

# Préparer données
train_dataset = TextDataset(train_texts, train_labels, tokenizer)
val_dataset = TextDataset(val_texts, val_labels, tokenizer)

# Training Arguments
training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    warmup_steps=500,               # Learning rate warmup
    weight_decay=0.01,              # Régularisation
    logging_dir='./logs',
    logging_steps=100,
    evaluation_strategy="epoch",    # Évaluer à chaque epoch
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
    fp16=True,                      # Mixed precision (si GPU compatible)
)

# Métriques
from sklearn.metrics import accuracy_score, f1_score

def compute_metrics(pred):
    """Calculer métriques d'évaluation"""
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    
    acc = accuracy_score(labels, preds)
    f1 = f1_score(labels, preds, average='weighted')
    
    return {'accuracy': acc, 'f1': f1}

# Trainer (simplifie l'entraînement)
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    compute_metrics=compute_metrics
)

# Entraîner
trainer.train()

# Évaluer
results = trainer.evaluate()
print(results)

# Prédire
predictions = trainer.predict(val_dataset)
pred_labels = predictions.predictions.argmax(-1)


# --- FINE-TUNING POUR NER (Named Entity Recognition) ---

from transformers import AutoModelForTokenClassification

# Modèle pour NER
model = AutoModelForTokenClassification.from_pretrained(
    "bert-base-cased",  # Cased important pour NER
    num_labels=9        # IOB2 tags: O, B-PER, I-PER, B-ORG, I-ORG, etc.
)

# Dataset pour NER (format spécial)
class NERDataset(Dataset):
    def __init__(self, texts, tags, tokenizer, max_length=128):
        self.texts = texts
        self.tags = tags
        self.tokenizer = tokenizer
        self.max_length = max_length
    
    def __getitem__(self, idx):
        words = self.texts[idx]
        word_tags = self.tags[idx]
        
        encoding = self.tokenizer(
            words,
            is_split_into_words=True,  # Important!
            padding='max_length',
            truncation=True,
            max_length=self.max_length,
            return_tensors='pt'
        )
        
        # Aligner tags avec subword tokens
        word_ids = encoding.word_ids()
        aligned_labels = []
        
        for word_id in word_ids:
            if word_id is None:
                aligned_labels.append(-100)  # Ignore padding
            else:
                aligned_labels.append(word_tags[word_id])
        
        return {
            'input_ids': encoding['input_ids'].flatten(),
            'attention_mask': encoding['attention_mask'].flatten(),
            'labels': torch.tensor(aligned_labels, dtype=torch.long)
        }


# --- QUESTION ANSWERING ---

from transformers import AutoModelForQuestionAnswering

model = AutoModelForQuestionAnswering.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")
tokenizer = AutoTokenizer.from_pretrained("bert-large-uncased-whole-word-masking-finetuned-squad")

def answer_question(question, context):
    """Répondre à une question basée sur contexte"""
    
    inputs = tokenizer(
        question,
        context,
        return_tensors="pt",
        truncation=True,
        max_length=512
    )
    
    with torch.no_grad():
        outputs = model(**inputs)
    
    # Positions de début et fin de la réponse
    answer_start = torch.argmax(outputs.start_logits)
    answer_end = torch.argmax(outputs.end_logits) + 1
    
    # Extraire tokens de réponse
    answer_tokens = inputs['input_ids'][0][answer_start:answer_end]
    answer = tokenizer.decode(answer_tokens, skip_special_tokens=True)
    
    # Score de confiance
    start_score = outputs.start_logits[0][answer_start].item()
    end_score = outputs.end_logits[0][answer_end-1].item()
    confidence = (start_score + end_score) / 2
    
    return {
        'answer': answer,
        'confidence': confidence,
        'start': answer_start.item(),
        'end': answer_end.item()
    }

result = answer_question(
    question="What is the capital of France?",
    context="France is a beautiful country in Europe. Its capital city is Paris, known for the Eiffel Tower."
)


# --- GÉNÉRATION DE TEXTE (GPT-2, GPT-3) ---

from transformers import GPT2LMHeadModel, GPT2Tokenizer

tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")

def generate_text(prompt, max_length=100, temperature=0.8, top_k=50, top_p=0.95):
    """Générer texte avec GPT-2"""
    
    inputs = tokenizer(prompt, return_tensors="pt")
    
    outputs = model.generate(
        inputs['input_ids'],
        max_length=max_length,
        temperature=temperature,    # Créativité (0.7-1.0)
        top_k=top_k,               # Top-k sampling
        top_p=top_p,               # Nucleus sampling
        num_return_sequences=3,    # Générer 3 variations
        no_repeat_ngram_size=2,    # Éviter répétitions
        do_sample=True,            # Sampling stochastique
        pad_token_id=tokenizer.eos_token_id
    )
    
    texts = [tokenizer.decode(output, skip_special_tokens=True) 
             for output in outputs]
    
    return texts

results = generate_text("Once upon a time in a distant land,")


# --- EMBEDDINGS POUR SIMILARITÉ SÉMANTIQUE (suite) ---

from sentence_transformers import SentenceTransformer, util
import numpy as np

# Modèle optimisé pour similarité
model = SentenceTransformer('all-MiniLM-L6-v2')

# Encoder phrases
sentences = [
    "The cat sits on the mat",
    "A feline rests on a rug",
    "Python is a programming language",
    "Dogs are loyal animals"
]

embeddings = model.encode(sentences)

# Calculer similarités
from sklearn.metrics.pairwise import cosine_similarity
similarities = cosine_similarity(embeddings)

print("Matrice de similarité:")
for i, sent1 in enumerate(sentences):
    for j, sent2 in enumerate(sentences):
        if i < j:
            print(f"{sent1[:30]}... <-> {sent2[:30]}...")
            print(f"Similarité: {similarities[i][j]:.4f}\n")


# Recherche sémantique
def semantic_search(query, corpus, top_k=5):
    """Trouver phrases les plus similaires"""
    
    query_embedding = model.encode(query)
    corpus_embeddings = model.encode(corpus)
    
    # Calculer similarités
    similarities = cosine_similarity([query_embedding], corpus_embeddings)[0]
    
    # Top-k résultats
    top_indices = np.argsort(similarities)[::-1][:top_k]
    
    results = []
    for idx in top_indices:
        results.append({
            'text': corpus[idx],
            'score': similarities[idx]
        })
    
    return results

corpus = [
    "Machine learning is a subset of AI",
    "Deep learning uses neural networks",
    "Paris is the capital of France",
    "Python is popular for data science"
]

results = semantic_search("What is deep learning?", corpus, top_k=3)


# --- MULTILINGUAL MODELS ---

# Modèle multilingue (100+ langues)
from transformers import AutoTokenizer, AutoModel

tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base")
model = AutoModel.from_pretrained("xlm-roberta-base")

# Encoder textes en différentes langues
texts = [
    "Hello, how are you?",      # Anglais
    "Bonjour, comment allez-vous?",  # Français
    "Hola, ¿cómo estás?",       # Espagnol
    "こんにちは、元気ですか？"    # Japonais
]

for text in texts:
    inputs = tokenizer(text, return_tensors="pt")
    outputs = model(**inputs)
    embedding = outputs.last_hidden_state.mean(dim=1)
    print(f"{text}: shape={embedding.shape}")


# --- TRADUCTION ---

from transformers import MarianMTModel, MarianTokenizer

def translate(text, src_lang="en", tgt_lang="fr"):
    """Traduire texte"""
    
    model_name = f"Helsinki-NLP/opus-mt-{src_lang}-{tgt_lang}"
    tokenizer = MarianTokenizer.from_pretrained(model_name)
    model = MarianMTModel.from_pretrained(model_name)
    
    inputs = tokenizer(text, return_tensors="pt", padding=True)
    translated = model.generate(**inputs)
    
    translation = tokenizer.decode(translated[0], skip_special_tokens=True)
    
    return translation

result = translate("Hello, how are you?", "en", "fr")
print(result)  # "Bonjour, comment allez-vous?"


# --- DISTILLATION (modèles plus petits/rapides) ---

# DistilBERT: 40% plus petit, 60% plus rapide, 97% des performances
from transformers import DistilBertTokenizer, DistilBertForSequenceClassification

tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
model = DistilBertForSequenceClassification.from_pretrained(
    "distilbert-base-uncased",
    num_labels=2
)

# Autres modèles distillés: DistilGPT2, DistilRoBERTa, TinyBERT


# --- OPTIMISATION POUR PRODUCTION ---

# 1. Quantization (réduire poids à int8)
from transformers import AutoModelForSequenceClassification
import torch

model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")

# Quantization dynamique
quantized_model = torch.quantization.quantize_dynamic(
    model,
    {torch.nn.Linear},
    dtype=torch.qint8
)

# Taille réduite de ~4x, inférence plus rapide

# 2. ONNX Export (pour déploiement)
from transformers import convert_graph_to_onnx
from pathlib import Path

convert_graph_to_onnx.convert(
    framework="pt",
    model="bert-base-uncased",
    output=Path("bert-base-uncased.onnx"),
    opset=11
)

# 3. TorchScript (optimisation PyTorch)
traced_model = torch.jit.trace(model, example_inputs)
torch.jit.save(traced_model, "model_traced.pt")


# --- PROMPT ENGINEERING (pour GPT-3/4) ---

def few_shot_learning(prompt, examples, query):
    """Few-shot learning avec exemples"""
    
    full_prompt = prompt + "\n\n"
    
    # Ajouter exemples
    for ex_input, ex_output in examples:
        full_prompt += f"Input: {ex_input}\nOutput: {ex_output}\n\n"
    
    # Ajouter query
    full_prompt += f"Input: {query}\nOutput:"
    
    return full_prompt

examples = [
    ("The movie was great!", "Positive"),
    ("I hated this product.", "Negative"),
    ("It was okay, nothing special.", "Neutral")
]

prompt = few_shot_learning(
    "Classify the sentiment:",
    examples,
    "This is the best thing ever!"
)


# --- RETRIEVAL-AUGMENTED GENERATION (RAG) ---

from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration

def rag_answer(question, knowledge_base):
    """Répondre en récupérant infos pertinentes d'abord"""
    
    # Tokenizer et modèle
    tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq")
    retriever = RagRetriever.from_pretrained(
        "facebook/rag-token-nq",
        index_name="exact",
        use_dummy_dataset=True
    )
    model = RagTokenForGeneration.from_pretrained(
        "facebook/rag-token-nq",
        retriever=retriever
    )
    
    # Encoder question
    inputs = tokenizer(question, return_tensors="pt")
    
    # Générer réponse
    generated = model.generate(input_ids=inputs["input_ids"])
    answer = tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
    
    return answer


# --- ANALYSE D'ATTENTION (visualiser ce que regarde le modèle) ---

from bertviz import head_view, model_view

def visualize_attention(text, model, tokenizer):
    """Visualiser l'attention du modèle"""
    
    inputs = tokenizer(text, return_tensors="pt")
    
    # Obtenir attentions
    outputs = model(**inputs, output_attentions=True)
    attentions = outputs.attentions  # Tuple de (num_layers, num_heads)
    
    tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])
    
    # Visualiser avec bertviz
    head_view(attentions, tokens)
    model_view(attentions, tokens)


# --- CUSTOM TOKENIZER (pour domaine spécifique) ---

from tokenizers import Tokenizer, models, trainers, pre_tokenizers

def train_custom_tokenizer(texts, vocab_size=30000):
    """Entraîner tokenizer personnalisé"""
    
    # BPE tokenizer
    tokenizer = Tokenizer(models.BPE())
    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
    
    # Trainer
    trainer = trainers.BpeTrainer(
        vocab_size=vocab_size,
        special_tokens=["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]
    )
    
    # Entraîner
    tokenizer.train_from_iterator(texts, trainer)
    
    return tokenizer

# Utiliser
custom_tokenizer = train_custom_tokenizer(medical_texts, vocab_size=50000)


# --- CONSEILS TRANSFORMERS ---

"""
[OK] BONNES PRATIQUES

1. Toujours utiliser AutoTokenizer/AutoModel pour flexibilité
2. Faire fine-tuning sur vos données (même peu)
3. Utiliser modèles distillés en production (vitesse)
4. Warmup + learning rate decay pour stabilité
5. Gradient accumulation si GPU limitée
6. Sauvegarder checkpoints régulièrement
7. Monitorer métriques spécifiques à la tâche

[OBJECTIF] CHOIX DE MODÈLE

BERT/RoBERTa: Classification, NER, QA
GPT-2/3: Génération de texte
T5/BART: Résumé, traduction
DistilBERT: Production (vitesse)
XLM-RoBERTa: Multilingue
Sentence-BERT: Similarité sémantique

[GRAPHIQUE] DATASETS POPULAIRES

GLUE: Benchmark NLP général
SQuAD: Question Answering
CoNLL: Named Entity Recognition
IMDB: Sentiment Analysis
WMT: Traduction

[RAPIDE] OPTIMISATIONS PRODUCTION

1. Quantization: 4x plus petit
2. Distillation: 2-3x plus rapide
3. ONNX Runtime: Accélération
4. Batching dynamique
5. Cache des embeddings
6. Modèles plus petits (DistilBERT, TinyBERT)
"""


[OK] 40. MLOps AVANCÉ - KUBEFLOW, AIRFLOW, CI/CD


"""
MLOps: pratiques DevOps appliquées au Machine Learning.
Orchestration, monitoring, versioning, automatisation bout-en-bout.
"""

# --- APACHE AIRFLOW (Orchestration de pipelines) ---

from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.operators.bash import BashOperator
from airflow.utils.dates import days_ago
from datetime import timedelta

# Définir fonctions de tâches
def extract_data(**context):
    """Extraire données depuis source"""
    import pandas as pd
    
    # Simuler extraction
    df = pd.read_csv('s3://bucket/raw_data.csv')
    
    # Sauvegarder temporairement
    df.to_csv('/tmp/extracted_data.csv', index=False)
    
    # Push metadata vers XCom
    context['ti'].xcom_push(key='n_rows', value=len(df))
    
    print(f"Extracted {len(df)} rows")


def transform_data(**context):
    """Transformer données"""
    import pandas as pd
    
    # Récupérer données
    df = pd.read_csv('/tmp/extracted_data.csv')
    
    # Transformations
    df = df.dropna()
    df['feature_engineered'] = df['col1'] * df['col2']
    
    # Sauvegarder
    df.to_csv('/tmp/transformed_data.csv', index=False)
    
    # Récupérer info de tâche précédente
    n_rows_before = context['ti'].xcom_pull(key='n_rows')
    print(f"Rows: {n_rows_before} -> {len(df)}")


def train_model(**context):
    """Entraîner modèle"""
    import pandas as pd
    from sklearn.ensemble import RandomForestClassifier
    import joblib
    
    # Charger données
    df = pd.read_csv('/tmp/transformed_data.csv')
    
    X = df.drop('target', axis=1)
    y = df['target']
    
    # Entraîner
    model = RandomForestClassifier(n_estimators=100)
    model.fit(X, y)
    
    # Sauvegarder
    joblib.dump(model, '/tmp/model.pkl')
    
    # Métriques
    score = model.score(X, y)
    context['ti'].xcom_push(key='model_score', value=score)
    
    print(f"Model trained with score: {score:.4f}")


def deploy_model(**context):
    """Déployer modèle si score suffisant"""
    
    score = context['ti'].xcom_pull(key='model_score')
    
    if score > 0.85:
        # Déployer (simulé)
        print(f"Deploying model (score={score:.4f})")
        # aws s3 cp /tmp/model.pkl s3://models/production/
    else:
        print(f"Model score too low ({score:.4f}), skipping deployment")
        raise ValueError("Model score below threshold")


# Définir DAG
default_args = {
    'owner': 'data-team',
    'depends_on_past': False,
    'email': ['alerts@company.com'],
    'email_on_failure': True,
    'email_on_retry': False,
    'retries': 3,
    'retry_delay': timedelta(minutes=5),
}

dag = DAG(
    'ml_pipeline',
    default_args=default_args,
    description='Pipeline ML end-to-end',
    schedule_interval='@daily',  # Exécuter quotidiennement
    start_date=days_ago(1),
    catchup=False,
    tags=['ml', 'production'],
)

# Définir tâches
task_extract = PythonOperator(
    task_id='extract_data',
    python_callable=extract_data,
    dag=dag,
)

task_transform = PythonOperator(
    task_id='transform_data',
    python_callable=transform_data,
    dag=dag,
)

task_train = PythonOperator(
    task_id='train_model',
    python_callable=train_model,
    dag=dag,
)

task_deploy = PythonOperator(
    task_id='deploy_model',
    python_callable=deploy_model,
    dag=dag,
)

# Tests de qualité
task_data_quality = BashOperator(
    task_id='data_quality_checks',
    bash_command='python /scripts/check_data_quality.py',
    dag=dag,
)

# Définir dépendances (ordre d'exécution)
task_extract >> task_data_quality >> task_transform >> task_train >> task_deploy

# Notifications Slack (optionnel)
from airflow.providers.slack.operators.slack import SlackAPIPostOperator

task_notify = SlackAPIPostOperator(
    task_id='notify_success',
    slack_conn_id='slack_connection',
    text='Pipeline ML terminé avec succès!',
    channel='#ml-alerts',
    dag=dag,
)

task_deploy >> task_notify


# --- SENSORS (attendre événements) ---

from airflow.sensors.filesystem import FileSensor
from airflow.sensors.external_task import ExternalTaskSensor

# Attendre qu'un fichier apparaisse
wait_for_file = FileSensor(
    task_id='wait_for_data',
    filepath='/data/new_data.csv',
    poke_interval=60,  # Vérifier chaque minute
    timeout=3600,      # Timeout après 1h
    dag=dag,
)

# Attendre qu'une autre DAG se termine
wait_for_upstream = ExternalTaskSensor(
    task_id='wait_for_data_ingestion',
    external_dag_id='data_ingestion_dag',
    external_task_id='final_task',
    dag=dag,
)


# --- BRANCHING (exécution conditionnelle) ---

from airflow.operators.python import BranchPythonOperator

def decide_branch(**context):
    """Décider quelle branche exécuter"""
    score = context['ti'].xcom_pull(key='model_score')
    
    if score > 0.90:
        return 'deploy_production'
    elif score > 0.80:
        return 'deploy_staging'
    else:
        return 'retrain_with_more_data'

branching = BranchPythonOperator(
    task_id='decide_deployment',
    python_callable=decide_branch,
    dag=dag,
)

deploy_prod = BashOperator(task_id='deploy_production', bash_command='echo prod', dag=dag)
deploy_stg = BashOperator(task_id='deploy_staging', bash_command='echo staging', dag=dag)
retrain = BashOperator(task_id='retrain_with_more_data', bash_command='echo retrain', dag=dag)

task_train >> branching >> [deploy_prod, deploy_stg, retrain]


# --- KUBEFLOW PIPELINES (ML sur Kubernetes) ---

"""
import kfp
from kfp import dsl
from kfp.components import create_component_from_func

# Définir composants
@create_component_from_func
def preprocess_data(input_path: str, output_path: str):
    import pandas as pd
    
    df = pd.read_csv(input_path)
    df = df.dropna()
    df.to_csv(output_path, index=False)
    
    return output_path

@create_component_from_func
def train_model(data_path: str, model_path: str) -> float:
    import pandas as pd
    from sklearn.ensemble import RandomForestClassifier
    import joblib
    
    df = pd.read_csv(data_path)
    X = df.drop('target', axis=1)
    y = df['target']
    
    model = RandomForestClassifier()
    model.fit(X, y)
    
    joblib.dump(model, model_path)
    
    return model.score(X, y)

@create_component_from_func
def deploy_model(model_path: str, score: float, threshold: float = 0.85):
    if score > threshold:
        # Logique de déploiement
        print(f"Deploying model (score={score})")
    else:
        raise ValueError(f"Score {score} below threshold {threshold}")

# Définir pipeline
@dsl.pipeline(
    name='ML Pipeline',
    description='Pipeline ML avec Kubeflow'
)
def ml_pipeline(
    input_data: str = 's3://bucket/data.csv',
    model_output: str = 's3://bucket/model.pkl'
):
    # Étapes
    preprocess_task = preprocess_data(
        input_path=input_data,
        output_path='/tmp/processed.csv'
    )
    
    train_task = train_model(
        data_path=preprocess_task.output,
        model_path=model_output
    )
    
    deploy_task = deploy_model(
        model_path=model_output,
        score=train_task.output,
        threshold=0.85
    )

# Compiler pipeline
kfp.compiler.Compiler().compile(ml_pipeline, 'pipeline.yaml')

# Soumettre à Kubeflow
client = kfp.Client(host='http://kubeflow-pipeline-host')
client.create_run_from_pipeline_func(
    ml_pipeline,
    arguments={'input_data': 's3://mybucket/data.csv'}
)
"""


# --- MLFLOW TRACKING (Versioning et Tracking) ---

import mlflow
import mlflow.sklearn
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, f1_score

# Configurer tracking server
mlflow.set_tracking_uri("http://mlflow-server:5000")
mlflow.set_experiment("customer-churn-prediction")

def train_and_log_model(params):
    """Entraîner et logger avec MLflow"""
    
    with mlflow.start_run(run_name=f"rf_n{params['n_estimators']}"):
        
        # Logger paramètres
        mlflow.log_params(params)
        
        # Entraîner modèle
        model = RandomForestClassifier(**params)
        model.fit(X_train, y_train)
        
        # Prédire
        y_pred = model.predict(X_test)
        
        # Calculer métriques
        accuracy = accuracy_score(y_test, y_pred)
        f1 = f1_score(y_test, y_pred, average='weighted')
        
        # Logger métriques
        mlflow.log_metric("accuracy", accuracy)
        mlflow.log_metric("f1_score", f1)
        mlflow.log_metric("train_size", len(X_train))
        
        # Logger modèle
        mlflow.sklearn.log_model(
            model,
            "model",
            registered_model_name="ChurnPredictor"
        )
        
        # Logger artifacts (fichiers)
        import matplotlib.pyplot as plt
        from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
        
        cm = confusion_matrix(y_test, y_pred)
        disp = ConfusionMatrixDisplay(cm)
        disp.plot()
        plt.savefig("confusion_matrix.png")
        mlflow.log_artifact("confusion_matrix.png")
        
        # Logger feature importance
        import pandas as pd
        importance_df = pd.DataFrame({
            'feature': X_train.columns,
            'importance': model.feature_importances_
        }).sort_values('importance', ascending=False)
        
        importance_df.to_csv("feature_importance.csv", index=False)
        mlflow.log_artifact("feature_importance.csv")
        
        # Logger dataset
        mlflow.log_input(
            mlflow.data.pandas_dataset.from_pandas(X_train),
            context="training"
        )
        
        print(f"Run ID: {mlflow.active_run().info.run_id}")
        print(f"Accuracy: {accuracy:.4f}, F1: {f1:.4f}")
        
        return model, accuracy

# Expérimenter avec différents hyperparamètres
param_grid = [
    {'n_estimators': 50, 'max_depth': 5},
    {'n_estimators': 100, 'max_depth': 10},
    {'n_estimators': 200, 'max_depth': 15},
]

for params in param_grid:
    train_and_log_model(params)


# Comparer runs
runs = mlflow.search_runs(experiment_names=["customer-churn-prediction"])
print(runs[['metrics.accuracy', 'metrics.f1_score', 'params.n_estimators']])

# Charger meilleur modèle
best_run = runs.loc[runs['metrics.accuracy'].idxmax()]
model_uri = f"runs:/{best_run.run_id}/model"
loaded_model = mlflow.sklearn.load_model(model_uri)


# --- MODEL REGISTRY (Versioning de modèles) ---

from mlflow.tracking import MlflowClient

client = MlflowClient()

# Promouvoir modèle en staging
client.transition_model_version_stage(
    name="ChurnPredictor",
    version=3,
    stage="Staging"
)

# Ajouter description
client.update_model_version(
    name="ChurnPredictor",
    version=3,
    description="Random Forest avec 100 arbres, accuracy=0.92"
)

# Promouvoir en production après validation
client.transition_model_version_stage(
    name="ChurnPredictor",
    version=3,
    stage="Production"
)

# Charger modèle en production
model = mlflow.pyfunc.load_model("models:/ChurnPredictor/Production")
predictions = model.predict(new_data)


# --- CI/CD POUR ML (GitHub Actions exemple) ---

"""
# .github/workflows/ml_pipeline.yml

name: ML Pipeline CI/CD

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    
    steps:
    - uses: actions/checkout@v2
    
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.9'
    
    - name: Install dependencies
      run: |
        pip install -r requirements.txt
        pip install pytest pytest-cov
    
    - name: Run tests
      run: |
        pytest tests/ --cov=src --cov-report=xml
    
    - name: Upload coverage
      uses: codecov/codecov-action@v2
  
  data-validation:
    runs-on: ubuntu-latest
    needs: test
    
    steps:
    - uses: actions/checkout@v2
    
    - name: Validate data schema
      run: |
        python scripts/validate_data.py
    
    - name: Check data drift
      run: |
        python scripts/check_drift.py
  
  train-model:
    runs-on: ubuntu-latest
    needs: data-validation
    
    steps:
    - uses: actions/checkout@v2
    
    - name: Train model
      run: |
        python train.py --config config/prod.yaml
    
    - name: Evaluate model
      run: |
        python evaluate.py --threshold 0.85
    
    - name: Upload model artifact
      uses: actions/upload-artifact@v2
      with:
        name: model
        path: models/model.pkl
  
  deploy:
    runs-on: ubuntu-latest
    needs: train-model
    if: github.ref == 'refs/heads/main'
    
    steps:
    - uses: actions/checkout@v2
    
    - name: Download model
      uses: actions/download-artifact@v2
      with:
        name: model
    
    - name: Deploy to staging
      run: |
        aws s3 cp model.pkl s3://models/staging/
        aws lambda update-function-code \
          --function-name ml-inference-staging \
          --s3-bucket models \
          --s3-key staging/model.pkl
    
    - name: Run smoke tests
      run: |
        python tests/smoke_tests.py --env staging
    
    - name: Deploy to production
      run: |
        aws s3 cp model.pkl s3://models/production/
        aws lambda update-function-code \
          --function-name ml-inference-prod \
          --s3-bucket models \
          --s3-key production/model.pkl
"""


# --- MONITORING EN PRODUCTION ---

import prometheus_client
from prometheus_client import Counter, Histogram, Gauge
import time

# Métriques Prometheus
prediction_counter = Counter(
    'model_predictions_total',
    'Total number of predictions',
    ['model_version', 'outcome']
)

prediction_latency = Histogram(
    'model_prediction_latency_seconds',
    'Prediction latency'
)

model_accuracy = Gauge(
    'model_accuracy',
    'Current model accuracy'
)

data_drift_score = Gauge(
    'data_drift_score',
    'Data drift detection score'
)

def predict_with_monitoring(model, features, model_version="v1.0"):
    """Prédiction avec monitoring"""
    
    start_time = time.time()
    
    try:
        # Prédire
        prediction = model.predict([features])[0]
        
        # Logger métrique
        prediction_counter.labels(
            model_version=model_version,
            outcome=str(prediction)
        ).inc()
        
        # Latence
        latency = time.time() - start_time
        prediction_latency.observe(latency)
        
        return prediction
    
    except Exception as e:
        prediction_counter.labels(
            model_version=model_version,
            outcome="error"
        ).inc()
        raise e


# Détecter data drift
from scipy.stats import ks_2samp

def detect_drift(reference_data, current_data, threshold=0.05):
    """Détecter drift dans les données"""
    
    drift_detected = False
    
    for column in reference_data.columns:
        # Test Kolmogorov-Smirnov
        statistic, p_value = ks_2samp(
            reference_data[column],
            current_data[column]
        )
        
        if p_value < threshold:
            drift_detected = True
            print(f"Drift détecté dans {column}: p-value={p_value:.4f}")
            
            # Logger métrique
            data_drift_score.set(statistic)
    
    return drift_detected


# --- FEATURE STORE (centraliser features) ---

"""
# Avec Feast (Feature Store)

from feast import FeatureStore, Entity, Feature, FeatureView, FileSource
from feast.value_type import ValueType
from datetime import timedelta

# Définir entité
customer = Entity(
    name="customer_id",
    value_type=ValueType.INT64,
    description="Customer ID"
)

# Source de données
customer_features_source = FileSource(
    path="data/customer_features.parquet",
    event_timestamp_column="event_timestamp"
)

# Vue de features
customer_features_view = FeatureView(
    name="customer_features",
    entities=["customer_id"],
    ttl=timedelta(days=1),
    features=[
        Feature(name="age", dtype=ValueType.INT64),
        Feature(name="income", dtype=ValueType.FLOAT),
        Feature(name="credit_score", dtype=ValueType.INT64),
    ],
    online=True,
    batch_source=customer_features_source
)

# Feature Store
fs = FeatureStore(repo_path=".")

# Récupérer features pour prédiction
features = fs.get_online_features(
    features=[
        "customer_features:age",
        "customer_features:income",
        "customer_features:credit_score"
    ],
    entity_rows=[{"customer_id": 12345}]
).to_dict()
"""


# --- TESTS POUR ML ---

import pytest
import pandas as pd
import numpy as np

def test_model_output_shape():
    """Tester forme de sortie du modèle"""
    X_test = np.random.rand(10, 5)
    predictions = model.predict(X_test)
    
    assert predictions.shape[0] == 10, "Nombre de prédictions incorrect"

def test_model_output_range():
    """Tester plage des prédictions"""
    X_test = np.random.rand(10, 5)
    predictions = model.predict_proba(X_test)
    
    assert (predictions >= 0).all() and (predictions <= 1).all(), \

[OBJECTIF] CONCLUSION FINALE


"""
=============================================================================
[COURS] FÉLICITATIONS ! VOUS AVEZ COMPLÉTÉ LA CHEATSHEET DATA SCIENCE !
=============================================================================

Cette cheatsheet exhaustive couvre maintenant TOUS les aspects majeurs :

[GRAPHIQUE] FONDAMENTAUX
[OK] NumPy, Pandas (fichiers séparés)
[OK] Visualisation (Matplotlib, Seaborn, Plotly)
[OK] Statistiques et probabilités

[BOT] MACHINE LEARNING
[OK] Modèles classiques (Scikit-learn)
[OK] Feature Engineering avancé
[OK] Réduction de dimensionnalité
[OK] Clustering et segmentation
[OK] Méthodes d'ensemble avancées
[OK] Régularisation et optimisation
[OK] Cross-validation sophistiquée

[LOGIQUE] DEEP LEARNING
[OK] Réseaux de neurones (TensorFlow/Keras)
[OK] CNN pour images
[OK] RNN/LSTM pour séquences
[OK] Transfer Learning
[OK] Regularisation (Dropout, Batch Norm)

[NOTE] NLP & COMPUTER VISION
[OK] Traitement de texte
[OK] Vectorisation (TF-IDF, Word2Vec)
[OK] Sentiment Analysis
[OK] Traitement d'images (OpenCV)
[OK] Data Augmentation

[TEMPS] SÉRIES TEMPORELLES
[OK] ARIMA, SARIMA, Prophet
[OK] Décomposition et stationnarité
[OK] Prévision

[RECHERCHE] TECHNIQUES SPÉCIALISÉES
[OK] Anomaly Detection
[OK] Recommender Systems
[OK] Survival Analysis
[OK] Reinforcement Learning
[OK] Graph Machine Learning
[OK] A/B Testing et inférence causale

[RAPIDE] PRODUCTION & DEPLOYMENT
[OK] APIs (Flask, FastAPI)
[OK] Docker et containerisation
[OK] MLflow et tracking
[OK] Streamlit pour interfaces
[OK] Monitoring et maintenance

[HAUSSE] OPTIMISATION & AUTOML
[OK] Profiling et performance
[OK] Parallelisation (Numba, multiprocessing)
[OK] AutoML (AutoSklearn, TPOT, Optuna)
[OK] Hyperparameter tuning avancé

[ANALYSE]