import tensorflow as tf
import matplotlib.pyplot as plt
import pandas as pd
import time
train = pd.read_csv("mnist_train(1).csv") # Fashion: "fashion-mnist_train.csv"
test = pd.read_csv("mnist_test(1).csv") # Fashion: "fashion-mnist_test.csv"
x_train = train.iloc[:,1:].values.astype('float32').reshape(-1,28,28)/255.0
y_train = train.iloc[:,0].values
x_test = test.iloc[:,1:].values.astype('float32').reshape(-1,28,28)/255.0
y_test = test.iloc[:,0].values
print("Training:",x_train.shape)
print("Testing:",x_test.shape)
acts = ['sigmoid','tanh','relu']
hist,times,grads = {},{},{}
for act in acts:
    m = tf.keras.Sequential([
        tf.keras.layers.Input((28,28)),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(128,activation=act),
        tf.keras.layers.Dense(64,activation=act),
        tf.keras.layers.Dense(10,activation='softmax')
    ])
    m.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
    t = time.time()
    hist[act] = m.fit(x_train,y_train,epochs=5,batch_size=32,validation_split=0.1,verbose=0)
    times[act] = time.time()-t
    print(act,"Accuracy:",m.evaluate(x_test,y_test,verbose=0)[1],"Time:",round(times[act],2))
    with tf.GradientTape() as tape:
        loss = tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(y_train[:32],m(x_train[:32])))
    grads[act] = [tf.norm(g).numpy() for g in tape.gradient(loss,m.trainable_variables) if g is not None]
for key in ['accuracy','val_accuracy','loss','val_loss']:
    for act in acts:
        plt.plot(hist[act].history[key],label=act)
    plt.title(key.capitalize())
    plt.xlabel("Epochs")
    plt.legend()
    plt.show()
plt.bar(acts,[times[a] for a in acts])
plt.title("Training Time")
plt.ylabel("Seconds")
plt.show()
for act in acts:
    plt.plot(grads[act],label=act)
plt.title("Gradient Flow")
plt.xlabel("Trainable Variable Index")
plt.ylabel("Gradient Norm")
plt.legend()
plt.show()
