# IMPLEMENT LOGISTIC REGRESSION
import pandas as pd
import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, confusion_matrix
import statsmodels.api as sm
import statsmodels.formula.api as smf
# ============================================================
# A) BINARY LOGISTIC REGRESSION
# ============================================================
print("========================================")
print("A) BINARY LOGISTIC REGRESSION")
print("========================================")
# Load Iris dataset
iris = load_iris()
# Convert dataset into DataFrame
df = pd.DataFrame(
    iris.data,
    columns=[
        "sepal_length",
        "sepal_width",
        "petal_length",
        "petal_width"
    ]
)
# Add target column
df["target"] = iris.target
# Select only two classes
df_binary = df[df["target"] != 2].copy()
# Independent variables
X = df_binary[
    [
        "sepal_length",
        "sepal_width",
        "petal_length",
        "petal_width"
    ]
]
# Dependent variable
y = df_binary["target"]
# Split dataset into training and testing
X_train, X_test, y_train, y_test = train_test_split(
    X,
    y,
    test_size=0.2,
    random_state=42
)
# ============================================================
# 1) SCIKIT-LEARN
# ============================================================
print("\n------------------------------")
print("SKLEARN")
print("------------------------------")
# Create Logistic Regression model
model_sklearn = LogisticRegression(
    max_iter=1000
)
# Train model
model_sklearn.fit(
    X_train,
    y_train
)
# Prediction
y_pred_sklearn = model_sklearn.predict(
    X_test
)
# Accuracy
acc_sklearn = accuracy_score(
    y_test,
    y_pred_sklearn
)
# Display result
print("Accuracy:", acc_sklearn)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_sklearn
    )
)
print("\nIntercept:")
print(
    model_sklearn.intercept_
)
print("\nCoefficient:")
print(
    model_sklearn.coef_
)
# ============================================================
# 2) STATSMODELS.API
# ============================================================
print("\n------------------------------")
print("STATSMODELS.API")
print("------------------------------")
# Add constant
X_train_sm = sm.add_constant(
    X_train
)
X_test_sm = sm.add_constant(
    X_test
)
# Create model
model_sm = sm.Logit(
    y_train,
    X_train_sm
)
# Train model
result_sm = model_sm.fit(
    disp=0
)
# Predict probability
y_prob_sm = result_sm.predict(
    X_test_sm
)
# Convert probability into 0 or 1
y_pred_sm = (
    y_prob_sm >= 0.5
).astype(int)
# Accuracy
acc_sm = accuracy_score(
    y_test,
    y_pred_sm
)
# Display result
print("Accuracy:", acc_sm)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_sm
    )
)
print("\nCoefficients:")
print(
    result_sm.params
)
print("\nModel Summary:")
print(
    result_sm.summary()
)
# ============================================================
# 3) STATSMODELS.FORMULA.API
# ============================================================
print("\n------------------------------")
print("STATSMODELS.FORMULA.API")
print("------------------------------")
# Create training dataframe
train_df = X_train.copy()
train_df["target"] = y_train
# Create testing dataframe
test_df = X_test.copy()
test_df["target"] = y_test
# Logistic Regression formula
model_formula = smf.logit(
    formula="""
    target ~ sepal_length
    + sepal_width
    + petal_length
    + petal_width
    """,
    data=train_df
)
# Train model
result_formula = model_formula.fit(
    disp=0
)
# Prediction probability
y_prob_formula = result_formula.predict(
    test_df
)
# Convert into class
y_pred_formula = (
    y_prob_formula >= 0.5
).astype(int)
# Accuracy
acc_formula = accuracy_score(
    y_test,
    y_pred_formula
)
# Display results
print("Accuracy:", acc_formula)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_formula
    )
)
print("\nCoefficients:")
print(
    result_formula.params
)
print("\nModel Summary:")
print(
    result_formula.summary()
)
# ============================================================
# B) MULTI-CLASS LOGISTIC REGRESSION
# ============================================================
print("\n\n========================================")
print("B) MULTI-CLASS LOGISTIC REGRESSION")
print("========================================")
# Reload complete Iris dataset
iris = load_iris()
# Convert to DataFrame
df = pd.DataFrame(
    iris.data,
    columns=[
        "sepal_length",
        "sepal_width",
        "petal_length",
        "petal_width"
    ]
)
# Add target
df["target"] = iris.target
# Independent variables
X = df[
    [
        "sepal_length",
        "sepal_width",
        "petal_length",
        "petal_width"
    ]
]
# Dependent variable
y = df["target"]
# Split dataset
X_train, X_test, y_train, y_test = train_test_split(
    X,
    y,
    test_size=0.2,
    random_state=42
)
# ============================================================
# 1) SCIKIT-LEARN MULTI-CLASS
# ============================================================
print("\n------------------------------")
print("SKLEARN")
print("------------------------------")
# Create multinomial Logistic Regression model
model_sklearn_multi = LogisticRegression(
    multi_class="multinomial",
    max_iter=1000
)
# Train model
model_sklearn_multi.fit(
    X_train,
    y_train
)
# Prediction
y_pred_sklearn_multi = model_sklearn_multi.predict(
    X_test
)
# Accuracy
acc_sklearn_multi = accuracy_score(
    y_test,
    y_pred_sklearn_multi
)
# Display result
print(
    "Accuracy:",
    acc_sklearn_multi
)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_sklearn_multi
    )
)
# ============================================================
# 2) STATSMODELS.API MULTI-CLASS
# ============================================================
print("\n------------------------------")
print("STATSMODELS.API")
print("------------------------------")
# Add constant
X_train_sm = sm.add_constant(
    X_train
)
X_test_sm = sm.add_constant(
    X_test
)
# Multinomial Logistic Regression
model_sm_multi = sm.MNLogit(
    y_train,
    X_train_sm
)
# Train model
result_sm_multi = model_sm_multi.fit(
    disp=0
)
# Predict probabilities
y_prob_sm_multi = result_sm_multi.predict(
    X_test_sm
)
# Select class with highest probability
y_pred_sm_multi = np.asarray(
    y_prob_sm_multi
).argmax(axis=1)
# Accuracy
acc_sm_multi = accuracy_score(
    y_test,
    y_pred_sm_multi
)
# Display result
print(
    "Accuracy:",
    acc_sm_multi
)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_sm_multi
    )
)
print("\nModel Summary:")
print(
    result_sm_multi.summary()
)
# ============================================================
# 3) STATSMODELS.FORMULA.API MULTI-CLASS
# ============================================================
print("\n------------------------------")
print("STATSMODELS.FORMULA.API")
print("------------------------------")
# Training dataframe
train_df = X_train.copy()
train_df["target"] = y_train
# Testing dataframe
test_df = X_test.copy()
test_df["target"] = y_test
# Create multinomial model
model_formula_multi = smf.mnlogit(
    formula="""
    target ~ sepal_length
    + sepal_width
    + petal_length
    + petal_width
    """,
    data=train_df
)
# Train model
result_formula_multi = model_formula_multi.fit(
    disp=0
)
# Predict probabilities
y_prob_formula_multi = result_formula_multi.predict(
    test_df
)
# Select highest probability class
y_pred_formula_multi = np.asarray(
    y_prob_formula_multi
).argmax(axis=1)
# Accuracy
acc_formula_multi = accuracy_score(
    y_test,
    y_pred_formula_multi
)
# Display result
print(
    "Accuracy:",
    acc_formula_multi
)
print("\nConfusion Matrix:")
print(
    confusion_matrix(
        y_test,
        y_pred_formula_multi
    )
)
print("\nModel Summary:")
print(
    result_formula_multi.summary()
)
# ============================================================
# FINAL COMPARISON
# ============================================================
print("\n\n========================================")
print("COMPARISON")
print("========================================")
print("\nBINARY LOGISTIC REGRESSION")
print(
    "Sklearn Accuracy:",
    acc_sklearn
)
print(
    "Statsmodels API Accuracy:",
    acc_sm
)
print(
    "Statsmodels Formula API Accuracy:",
    acc_formula
)
print("\nMULTI-CLASS LOGISTIC REGRESSION")
print(
    "Sklearn Accuracy:",
    acc_sklearn_multi
)
print(
    "Statsmodels API Accuracy:",
    acc_sm_multi
)
print(
    "Statsmodels Formula API Accuracy:",
    acc_formula_multi
)
