# -- Code Cell --
import pandas as pd
df = pd.read_csv('train.csv')

# -- Code Cell --
wins = df[df['reward'] == 1000.0]
wins['next_state'].value_counts()

# -- Code Cell --
DARWIN = 777
darwin_row = DARWIN // 32  
darwin_col = DARWIN % 32   

# -- Code Cell --
def best_action(s):
    dr = darwin_row - s // 32  
    dc = darwin_col - s % 32    
    if abs(dr) >= abs(dc):
        return 1 if dr > 0 else 0 
    else:
        return 3 if dc > 0 else 2   

# -- Code Cell --
test = pd.read_csv("test.csv")

# -- Code Cell --
submission = pd.DataFrame({
    "state": test["state"],
    "action": test["state"].apply(best_action)
})
submission.to_csv("submission.csv", index=False)

# -- Code Cell --
!python eval.py

# -- Code Cell --
import pandas as pd
import numpy as np

df = pd.read_csv('train.csv')   

GAMMA      = 0.95   
N_STATES   = 1024   
N_ACTIONS  = 4      
N_ITER     = 500   

grouped = {key: grp for key, grp in df.groupby(['state', 'action'])}

# -- Code Cell --
R = df.groupby(['state', 'action'])['reward'].mean()


# -- Code Cell --
counts = df.groupby(['state','action','next_state']).size()
totals = df.groupby(['state','action']).size()
P      = (counts / totals).to_dict()

# -- Code Cell --
N_ACTIONS  = 4 

# -- Code Cell --
V = np.zeros(N_STATES)

for _ in range(N_ITER):
    V_new = np.zeros(N_STATES)
    Q     = {}

    for (s, a), grp in grouped.items():
        # shortcut: mean(r + γ*V[s']) combină R și P într-un singur calcul
        Q[(s, a)] = (grp['reward'].values + GAMMA * V[grp['next_state'].values]).mean()

    for s in range(N_STATES):
        vals = [Q[(s, a)] for a in range(N_ACTIONS) if (s, a) in Q]
        V_new[s] = max(vals) if vals else V[s]

    if np.max(np.abs(V_new - V)) < 1e-6: break
    V = V_new


# -- Code Cell --
def best_action(s):
    opts = [(a, Q.get((s, a), -9999)) for a in range(N_ACTIONS)]
    return max(opts, key=lambda x: x[1])[0]

# -- Code Cell --
ans = test['state'].apply(best_action)

# -- Code Cell --
ans

# -- Code Cell --
