

impoty gymnasium as gym
import numpy as np
import pandas as pd
env = gym.make("M...")
position_bins = np.linspace(-1.2,0.6,20)
velocity_bins = np.linspace(-0.07,0.07,20)

class DiscretizedQLearning:
    def __init__(self,bins, n_actions, alpha=0.1, gamma=0.99, epsilon = 1.0, epsilon_min = 0.01):
        self.bins = bins
        self.Q = np.ones([len(bins[0])+1, len(bins[1]) + 1, n_actions]) *5.0 #facem un array plin de 5.0 de marimea [pozitie][viteza][actiuni]
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_min = epsilon_min
    def discretize(self,state): # ia vitesa si pozitia si spune in ce bins se afla
        return tuple(np.digitize(s,b) for s,b in zip(state, self.bins))
    def choose_action(self, state, n_actions):
        if np.random.rand()<self.epsilon:#ALEGI UN NUMAR RANDOM INTRE 0/1 daca e mai mic ca epsilon atunci alegi o actiune random (EXPLORARE)
            return np.random.randint(n_actions) 
        s = self.discretize(state) # EXPLOATARE
        return np.argmax(self.Q[s])
    def update(self, state, action, reward, next_state):
        s = self.discretize(state)
        s_next = self.discretize(next_state)
        best_next = np.max(self.Q[s_next])
        self.Q[s][action] += self.alpha * (
            reward + self.gamma * best_next - self.Q[s][action]
        )
    def decay(self, eps_decay=0.9995, alpha_decay=0.9999):
        self.epsilon = max(self.epsilon_min, self.epsilon * eps_decay)
        self.alpha = max(0.01, self.alpha * alpha_decay)
def decay(self, eps_decay = 0.9995, alpha_decay = 0.9999):
    self.epsilon = max(self.epsilon_min, self.epsilon * eps_decay)
    self.aplha = max(0.01 , self.alpha * alpha_decay)
def decay(self, eps_decay =0.9995, alhpa_decay = 0.9999):
    self.epsilon = max(self.epsilon_min, self.epsilon*eps_decay)
    self.alpha = max(0.1, self.alpha*alhpa_decay)
def decay(self, eps_decay = 0.9995, alpha_decay = 0.9999):
    self.epsilon = max(self.epsilon_min, self.epslon * eps_decay)
    self.alpha = max(0.1, self.alpha * alpha_decay)
def discretize(self, state):
    return tuple(np.digitize(s,b) for s,b in zip(state,self.bins))
def discretize(self,state):
    return tuple(np.digitize(s,b) for s,b in zip(state,self.bins))
def discretize(self, state):
    return tuple(np.digitize(s,b) for s,b in zip(state, self.bins))
def choose_action(self, n_actions, state):
    if np.random.rand()<self.epsilon:
        return np.random.randint(n_actions)
    s = self.descritize(state)
    return np.argmax(self.Q[s])
def choose_action(self, n_actions,state):
    if np.random.rand()<self.epsilon:
        return np.random.randint(n_actions)
    s = self.disctrize(state)
    return np.argmax(self.Q[s])
def choose_action(self, n_actions, state):
    if np.random.rand()<self.epsilon:
        return np.random.randint(n_actions)
    s = self.disctrize(state)
    return np.armgax(self.Q[s])
def choose_action(self, state, n_action):
    if np.random.rand()<self.epsilon:
        return np.random.randint(n_action)
    s = self.descritize(state)
    return np.argmax(self.Q[s])
def update(self,state,action, reward, next_state):
    s = self.descritize(state)
    s_next = self.descritize(next_state)
    best_next = np.max(self.Q[s_next])
    self.Q[s][action] += self.alpha * (reward + self.gamma * best_next - self.Q[s][action])
def update(self, state, action, reward, next_state):
    s= self.descritieze(state)
    s_next = self.desciritez(next_state)
    best_state = np.max(self.Q[s_next])
    self.Q[s][action] += self.alpha * (reward + self.gamma*best_state-self.Q[s][action])
def update(self, state, next_state, reward, action):
    s = self.descritize(state)
    s_next = self.descritize(next_state)
    best_next = np.max(self.Q[s_next])
    self.Q[s][action] += self.alpha * (reward + self.gamma * best_next - self.Q[s][action])
agent = DiscretizedQLearning(bins = [position_bins, velocity_bins],n_actions=env.action_space.n)
episodes = 50000
reward_window = []
for ep in episodes:
    state, _ = env.reset()
    done=False
    while not done:
        action = agent.choose_action(state,env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        
        agent.update(state,action,reward,next_state) 
        state = next_state
    agent.decay(ep)
for ep in episodes:
    state, _ = env.reset()
    done = False
    while not done:
        action = agent.choose_action(state, env,action_space.n)
        next_state,reward, terminated,truncated, _ = env.step(action)
        done = terminated or truncated
        agent.update(state, next_state, reward, action)
        state = next_state
    agent.decay(ep)
for ep in episodes:
    state, _ = env.reset()
    done = False
    while not done:
        action = agent.choose_action(state,env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        agent.update(state,next_state,reward, action)
        state = next_state
    agent.decay(ep)
for ep in episodes:
    state, _ = env.reset()
    done= False
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ =env.step(action)
        done = truncated or terminated
        agent.update(state, next_state, reward, action)
        state= next_state
    agent.decay(ep)
for ep in episodes:
    state, _ = env.reset()
    done = False
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        agent.update(state, next_state, reward, action)
        state= next_state
    agent.decay(ep)
test = pd.read_csv('test.csv')
total_rewards = []
logs = []
for i in range(len(test)):
    row = test.iloc[i]
    custom_state=np.array([
        row['start_pos'],
        row['start_vel']
    ])
    obs, info = env.reset()
    env.unwrapped.state = custom_state
    state= custom_state.copy()
    done = False
    total_reward = 0
    step_id = 0
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        total_reward += reward
        logs.append({
            "episode_id": i,
            "action": action,
        })
        state= next_state
        step_id +=1
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)
df = pd.DataFrame(logs)
df.to_csv('subi.csv',index=False)
total_rewards = []
logs = []
for i in range(len(test)):
    row = test.iloc[i]
    logs = np.array([
        row['pos'],
        row['vel']
    ])
    env.reset()
    env.unwrapped.state=custom_state
    state = custom_state.copy()
    done= False
    total_reward = 0
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        total_reward += reward
        logs.append({
            "id":i,
            'action':action
        })
        state= next_state
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)
df = .....to_csv('ss')
total_rewards = []
logs = []
for i in range(len(test)):
    done = False
    row = test.iloc[i]
    custom_state = np.array([
        row['pos'],
        row['vel']
    ])
    env.reset()
    env.unwrapped.state=custom_state
    state= custom_state.copy()
    total_reward = 0
    step_id =0
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state,reward,truncated, terminated, _ = env.step(action)
        done = terminated or truncated
        logs.append({
            "episode_id":i,
            'action':action
        })
        total_reward+=reward
        state= next_state
        step_id +=1
        
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)
df = pd.DataFrame(logs).to_csv()
total_rewards = []
logs = []
for i in range(len(test)):
    row = test.iloc[i]
    custom_state = np.array([
        row['pos'],
        row['vel']
    ])
    env.reset()
    env.unwrapped.state = custom_state
    state= custom_state.copy()
    done = False
    total_reward = 0
    step_id = 0
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        step = next_state
        step_id +=1
        logs.append({
            "id":i,
            'action':action
        })
        total_reward += reward
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)
out = pd.DataFrame(logs).to_csv('s.csv',index=False)
N_BINS = 20
position_bins = np.linspace(-1.2, 0.6, N_BINS)
velocity_bins = np.linspace(-0.07, 0.07, N_BINS)
class Clasa():
    def __init__(self, bins,n_actions, epsilon_min=0.01, gamma=0.99, alpha=0.1, epsilon=1.0):
        self.bins = bins
        self.Q = np.ones([len(bins[0])+1, len(bins[1])+1, n_actions]) * 5.0
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_min = epsilon_min
    def descritize(self, state):
        return tuple(np.digitize(s,b) for s,b in zip(state, self.bins))
    def choose_action(self, state, n_actions):
        if np.random.rand()<self.epsilon:
            return np.random.randint(n_actions)
        s = self.descritize(state)
        return np.argmax(self.Q[s])
    def update(self, next_state, state, reward, action):
        s = self.descritize(state)
        s_next= self.descritize(next_state)
        best_next = np.max(self.Q[s_next])
        self.Q[s][action] += self.alpha * ( reward + self.gamma*best_next - self.Q[s][action])
    def decay(self, ep, eps_decay = 0.9999, alpha_decay = 0.9995):
        self.epsilon = max(self.epsilon_min, self.epsilon*eps_decay)
        self.alpha = max(0.1, self.alpha*alpha_decay)
agent = Clasa(bins = [position_bins,velocity_bins], n_actions=env.action_space.n)
episodes = 50000
for ep in range(episodes):
    state, _ = env.reset()
    done = False
    while not done:
        action = agent.choose_action(state,env.action_space.n)
        next_state, reward_window, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        agent.update(next_state,state, reward, action)
        state= next_state
test = pd.read_csv('test.csv')
logs = []
total_rewards = []
for i in range(len(test)):
    row = test.iloc[i]
    current_state = np.arange([
        row['pos'],
        row['vel']
    ])
    env.reset()
    env.unwreapped.state = custom_state
    state = custom_state.copy()
    done= False
    total_reward = 0
    while not done:
        action = agent.choose_action(state,env.action_space.n)
        next_state, reward_window, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        total_reward += reward
        logs.append({
            "id":i,
            "action":action
        })
        state= next_state
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)

# # VALUE ITERATION
GRID = 32
N_STATES= 1024
N_ACTIONS = 4
GOAL_STATE = 1023
GAMMA = 0.95
EPSILON = 1e-4
STEP_REWARDS = -1.0
ACTION_NAMES = ["up",'down','left','right'] # 0 1 2 3
OUTCOME_DIRS = {0:[0,3,2,1], 1:[1,2,3,0], 2:[2,0,1,3], 3:[3,1,0,2]}
DELTAS = {0:(-1,0), 1:(1, 0), 2:(0,-1), 3:(0,1)}
def clip(r,c): return max(0, min(GRID-1,r)) * GRID + max(0, min(GRID-1,c))
def clip(r,c):
    return max(0, min(GRID+1,r)) * GRID + max(0, min(GRID+1,c))
def clip(r,c):
    return max(0, min(GRID-1,r)) * GRID + max(0, min(GRID-1,c))
count_sa = np.zeros((N_STATES, N_ACTIONS), dtype = np.float64)
count_sas = np.zeros((N_STATES,N_ACTIONS,N_STATES), dtype = np.float64)
reward_sa = np.zeros((N_STATES,N_ACTIONS), dtype = np.float64)
for row in train.itertuples(index=False):
    s = int(row.state)
    a = int(row.action)
    r = int(row.reward)
    ns = int(row.next_state)
    count_sa[s,a]+=1
    count_sas[s,a,ns]+=1
    reward_sa[s,a] += r
seen = count_sa>0
for row in trian.itertuples(index=False):
    s = int(row.score)
    a = int(row.action)
    ns = int(row.next_state)
    r = int(row.reward)
    count_sa[s,a] +=1
    count_sas[s,a,ns]+=1
    reward_sa[s,a] +=r
seen = count_sa>0
for row in test.itertuples(index=False):
    s = int(row.state)
    r = int(row.reward)
    a = int(row.action)
    ns = int(row.next_state)
    count_sa[s,a] +=1
    count_sas[s,a,ns] +=1
    reward_sa[s,a] +=r
seen = count_sa>0
P = np.zeros((N_STATES,N_ACTIONS,N_STATES), dtype = np.float64)
for s in range(N_STATES):
    row_s, col_s = divmod(s, GRID)
    for a in range(N_ACTIONS):
        if seen[s,a]: P[s,a] = count_sas[s,a]/count_sa[s,a]
        else:
            dr,dc = DELTAS[a]
            ns_ = clip(row_s+dr, col_s+dc)
            P[s,a,ns_] = 1.0
P =np.zeros((N_STATES,N_ACTIONS, N_STATES),dtype=float64)
for s in range(N_STATES):
    row_s, col_s = divmod(s, GRID)
    for a in range(N_ACTIONS):
        if seen[s,a]: P[s,a] = count_sas[s,a]/count_sa[s,a]
        else:
            dr,dc =DELTAS[a]
            ns_ = clip(row_s + dr,col_s+dc)
            P[s,a,ns_] = 1.0
P =np.zeros((N_STATES,N_ACTIONS, N_STATES),dtype=float64)
for s in range(N_STATES):
    row_s, col_s =divmod(s, GRID)
    for a in range(N_ACTIONS):
        if seen[s,a]: P[s,a] = count_sas[s,a]/count_sa[s,a]
        else:
            dr,dc = DELTAS(a)
            ns_ = clip(row_s + dr, col_s+dc)
            P[s,a,ns_]=1.0
def clip(r,c):
    return max(0, min(GRID-1,r))*GRID + max(0, min(GRID-1,c))
def clip(r,c):
    return max(0, min(GRID-1,r)) * GRID + max(0, min(GRID-1,c))
R = np.where(seen, reward_sa / count_sa, STEP_REWARDS)
R= np.where(seen, reward_sa/count_sa, STEP_REWARDS)
R = np.where(seen, reward_sa/count_sa, STEP_REWARDS)
R = np.where(seen, reward_sa/count_sa, STEP_REWARDS)
V = np.zeros(N_STATES, dtype=np.float64)
Q = np.zeros((N_STATES, N_ACTIONS), dtype = np.float64)
for iterarion in range(1,10001):
    Q = R + GAMMA * (P@V)
    Q[GOAL_STATE,:] = 0.0
    V_new = Q.max(axis=1)
    delta = np.abs(V_new-V).max()
    V = V_new
    if delta < EPSILON:
        print(f"Converged at {EPSILON}")
        break
policy = Q.argmax(axis=1)
V = np.zeros(N_STATES,dtype=np.float64)
Q = np.zeros((N_STATES,N_ACTIONS), dtype=np.float64)
for iteration in range(10001):
    Q = R+GAMMA*(P@V)
    Q[GOAL_STATE] = 0.0
    V_new = Q.max(axis=1)
    delta = np.abs(V_new-V).max()
    if delta<EPSILON:
        print(f"Converged at {EPSILON}")
        break
    V = V_new
policy = Q.argmax(axis=1)
V = np.zeros(N_STATES,dtype=np.float64)
Q = np.zeros((N_STATES,N_ACTIONS), dtype=np.float64)
for iteration in range(10001):
    Q = R + GAMMA* (P@V)
    Q[GOAL_STATE] = 0.0
    V_new =Q.max(axis=1)
    delta = np.abs(V_new-V).max()
    if delta< EPSILON:
        break
    V=V_new
policy = Q.argmax(axis=1)
V = np.zeros(N_STATES,dtype=np.float64)
Q = np.zeros((N_STATES,N_ACTIONS), dtype=np.float64)
for itearation in range(10001):
    Q = R + GAMMA*(P@V)
    Q[GOAL_STATE] = 0.0
    V_new = Q.max(axis=1)
    delta = np.abs(V_new-V).max()
    if delta<EPSILON:
        break
    V = V_new
policy = Q.argmax(axis=1)
test_states = test['state'].values
y_pred = Q[test_states].argmax(axis=1)
# ### Sau daca imi cere V(s)
# 
test_states = test['state'].values
y_pred = V[test_states] = Q.max(axis=1)
# ### Sau daca imi cere Q(s)
y_pred = Q[test_states]
# # AUDIO
import torchaudio.transforms as T
spectograme = []
for i in range(len(train)):
    waveform, sr = torachaudio.load(path)
    mel_spec = T.MelSpectogram(sample_rate = sr, n_mels = 128, n_fft = 1024, hop_length = 512)(waveform)
    spec_db = T.AmplitudeToDB()(mel_spec)
    spectograme.append(spec_db)
spectograme = []
for i in range(len(train)):
    waveform,sr = torchaudio.load(audio)
    mel_spec = T.MelSpectogram(sample_rate = sr, n_mels = 128, n_fft = 1024, hop_legth = 512)(waveform)
    mel_spec_db = T.AmplitudeToDB(mel_spec)
    spectograme.append(mel_spec_db)
import torch.nn.functional as F
from torchvision import transforms
def spec_to_image(spec):
    if spec.dim() == 2:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size=(224,224), mode='bilinear').squeeze(0)
    spec = spec.repeat(3,1,1)
    normalize = transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])
    spec = normalize(spec)
    return spec
def spec_to_image(spec):
    if spec.dim() == 2:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size = (224,224), mode='bilinear').squeeze(0)
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
    spec = normalize(spec)
    return spec
def spec_to_image(spec):
    if spec.dim() == 2:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size=(224,224), mode = 'bilinear')
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
    spec = normalize(spec)
    return spec
def spec_to_image(spec):
    if spec.dim() == 0:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size =(224,224), mode ="bilinear").squeeze(0)
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
    spec =normalize(spec)
    return spec
spectograme = []
for i in range(len(train)):
    waveform,sr = torchaduio.load(audio)
    mel_spec = T.MelSpectogram(sample_rate = sr,hop_length = 512, n_mels =128)(waveform)
    spec = T.AmplitudeToDB()(mel_spec)
    spectograme.append(spec)
def spec_to_image(spec):
    if spec.dim() == 0:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size=(224,224),mode='bilinear').squeeze(0)
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
    spec=normalize(spec)
    return spec
spectograme = []
for i in range(len(train)):
    waveform,sr=torchaudio.load(audio)
    mel_spec = T.MelSpectogram()
def get_dist(v1, v2):
    v1,v2 = np.array(v1), np.array(v2)
    return 1-(v1@v2)/(np.linalg.norm(v1)*np.linalg.norm(v2))
def pad_segment(spec, pad_len):
    spec= np.array(spec)
    if len(spec) > pad_len:
        spec = spec[:pad_len]
    else:
        pad =np.zeros((pad_len-len(spec), pad_len.shape[1]))
        np.vstack([pad,spec])
def preproc_audio(wav_path, target_len=50):
    waveform,sr =torchaudio.load(wav_path)
    mel_spec = T.MelSpectogram(sample_rate = sr, hop_lenth = 512, n_mels = 128,n_fft = 1024)(waveform)
    spec = T.AmplitudeToDB()(mel_spec)
    
    data = spec.T
    noise_base = np.mean(data[:10], axis=0)
    
    dists = []
    for i in range(len(data)):
        dists.append(get_dist(data[i],noise_base))
    dists=np.array(dists)
    thresh = dists.min()+0.1
    curated_data = []
    temp = []
    for i in range(len(dists)):
        if dists[i]>thresh:
            temp.append(data[i])
        else:
            if temp and len(temp) >10:
                padd = pad_segment(temp, target_len)
                curated_data.append(padd)
            temp = []
    if temp and len(temp) >10:
        padd = pad_segment(temp, target_len)
        curated_data.append(padd)   
    return np.array(curated_data)
def preproc_audio(path, target_len):
    waveform,sr =torchaudio.load(path)
    mel_spec = T.MelSpectogram(sample_rate = sr, hop_lenth = 512, n_mels = 128,n_fft = 1024)(waveform)
    spec = T.AmplitudeToDB()(mel_spec)
    data = spec.T
    noise_mean = np.mean(data[:10], axis=0)
    dist = []
    for i in range(len(data)):
        dist.append(get_dist(data[i], noise_mean))
    dist = np.array(dist)
    thresh = dist.min()+0.1
    curated_data =[]
    temp = []
    for i in range(len(dist)):
        if dist[i]>thresh:
            temp.append(data[i])
        else:
            if temp and len(temp)>10:
                padd = pad_segment(temp,target_len)
                curated_data.append(padd)
            temp = []
    if temp and len(temp)>10:
        padd = pad_segment(temp,target_len)
        curated_data.append(padd)
    return np.array(curated_data)
