import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from haversine import haversine, Unit
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error

# 1. LOAD AND INSPECT DATASET

df = pd.read_csv("uber.csv")

print("Dimensions: (Rows, Columns)", df.shape)
print("First 5 rows: ", df.head())
print("Data types and non-null counts: ")
df.info()
print("Missing values: ", df.isnull().sum())
print("Statistical Values: ")
print(df.describe(include="all").T)

df.columns

# 2. EDA - FARE DISTRIBUTION AND OUTLIERS

plt.hist(df["fare_amount"].dropna(), bins=50)
plt.title("Fare Distribution")
plt.xlabel("Fare Amount")
plt.ylabel("Frequency")
plt.show()

plt.boxplot(df["fare_amount"].dropna())
plt.title("Fare Amount Outliers")
plt.ylabel("Fare Amount")
plt.show()

# 3. DATA CLEANING

df = df.drop(columns=["Unnamed: 0", "key"], errors="ignore")
df = df.drop_duplicates().dropna()

df = df[df["fare_amount"] > 0]
df = df[df["passenger_count"].between(1, 6)]

# Remove invalid latitude and longitude
df = df[
    df["pickup_latitude"].between(-90, 90) &
    df["dropoff_latitude"].between(-90, 90) &
    df["pickup_longitude"].between(-180, 180) &
    df["dropoff_longitude"].between(-180, 180)
].copy()

# Calculate ride distance
df["distance_km"] = df.apply(
    lambda row: haversine(
        (row["pickup_latitude"], row["pickup_longitude"]),
        (row["dropoff_latitude"], row["dropoff_longitude"]),
        unit=Unit.KILOMETERS
    ),

    axis=1
)

df = df[df["distance_km"].between(0.1, 100)].copy()

# Fare outlier thresholds are learned from training labels after the split.
print("Shape after cleaning:", df.shape)

# EDA - RELATIONSHIP BETWEEN DISTANCE AND FARE

plt.scatter(df["distance_km"], df["fare_amount"], s=5)
plt.xlabel("Distance (km)")
plt.ylabel("Fare Amount")
plt.title("Distance vs Fare")
plt.show()

# 4. SELECT FEATURES AND TARGET

features = [
    "pickup_longitude", "pickup_latitude",
    "dropoff_longitude", "dropoff_latitude",
    "passenger_count", "distance_km"
]

X = df[features]

y = df["fare_amount"]

# 5. TRAIN-TEST SPLIT AND STANDARDIZATION

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# Learn IQR bounds from training fares only; keep the test set untouched.
Q1 = y_train.quantile(0.25)
Q3 = y_train.quantile(0.75)
IQR = Q3 - Q1
keep = y_train.between(Q1 - 1.5 * IQR, Q3 + 1.5 * IQR)
print("Training fare outliers removed:", int((~keep).sum()))
X_train = X_train.loc[keep]
y_train = y_train.loc[keep]


scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 6. LINEAR REGRESSION WITHOUT PCA

model1 = LinearRegression()
model1.fit(X_train_scaled, y_train)

pred1 = model1.predict(X_test_scaled)

# 7. PCA USING 95% EXPLAINED VARIANCE

pca = PCA(n_components=0.95)

X_train_pca = pca.fit_transform(X_train_scaled)
X_test_pca = pca.transform(X_test_scaled)

print("Original features:", X.shape[1])
print("PCA components:", pca.n_components_)
print("Explained variance:", pca.explained_variance_ratio_)
print("Total variance retained:", sum(pca.explained_variance_ratio_))

# 8. LINEAR REGRESSION WITH PCA

model2 = LinearRegression()
model2.fit(X_train_pca, y_train)

pred2 = model2.predict(X_test_pca)

# 9. MODEL EVALUATION AND COMPARISON

r2_1 = r2_score(y_test, pred1)
mae_1 = mean_absolute_error(y_test, pred1)
rmse_1 = np.sqrt(mean_squared_error(y_test, pred1))

r2_2 = r2_score(y_test, pred2)
mae_2 = mean_absolute_error(y_test, pred2)
rmse_2 = np.sqrt(mean_squared_error(y_test, pred2))

comparison = pd.DataFrame({
    "Metric": ["R2", "MAE", "RMSE"],
    "Without PCA": [r2_1, mae_1, rmse_1],
    "With PCA": [r2_2, mae_2, rmse_2]
})

print("\nMODEL COMPARISON")
print(comparison)

print("\nINTERPRETATION")
print("R2 change after PCA:", round(r2_2 - r2_1, 4))

if pca.n_components_ < X.shape[1]:
    print("PCA successfully reduced dimensionality.")
else:
    print("PCA did not reduce dimensions at 95% variance.")

if r2_2 >= r2_1:
    print("PCA maintained or improved R2 performance.")
else:
    print("PCA reduced R2 performance. Check the error metrics.")


# ENHANCED: Training ride-distance histogram
X_train["distance_km"].hist(bins=20)
plt.xlabel("Distance (km)")
plt.ylabel("Training rides")
plt.title("Training Ride-Distance Distribution")
plt.show()
