================================================================================
ML LEARNING NOTES
Generated from LearnIt board.
================================================================================


================================================================================
[general] Criterion
================================================================================

Binary Classification (0/1): BCEWithLogitsLoss
MultiClass Classification: CrossEnthropyLoss


================================================================================
[general] LSTM return for Classification
================================================================================

out = out[:, -1, :]


================================================================================
[general] Transpose, Solve SHAPE MISSMATCH (OR PERMUTE)
================================================================================

x.shape = (batch, seq, features)
x = x.transpose(1, 2)
(batch, channels, seq)
Schimba pozitia 1 cu 2, incepem numarat de la 0

x = x.permute(0, 3, 1, 2)


================================================================================
[general] FineTune Whole Model/Head
================================================================================

for param in resnet.parameters():
    param.requires_grad = True/False
for param in resnet.fc.parameters():
    param.requires_grad = True


================================================================================
[resnet] Resnet Training
================================================================================

model = resnet18(pretrained=True)
in_f = model.fc.in_features
model.fc = nn.Sequential(
    nn.Linear(in_f, 256),   # reduc dimensiunea features
    nn.ReLU(),              # non-linearity
    nn.Dropout(0.5),        # anti-overfitting
    nn.Linear(256, 1)       # output final (binary)
)
--------
model.fc = nn.Linear(in_f, num_classes)   # multiclass
---------
model.fc = nn.Linear(in_f, 1)   # binary
---------
model.fc = nn.Identity() # embeddings


================================================================================
[general] Model preds/probs
================================================================================

outputs = model(x)
preds = outputs.argmax(dim=1)      # clasa finala (0...C-1)
probs = torch.softmax(outputs, dim=1)  # probabilitati pe fiecare clasa


================================================================================
[resnet] Unfreeze another layer Resnet
================================================================================

for param in model.parameters():
    param.requires_grad = False
for param in model.layer4.parameters():
    param.requires_grad = True
for param in model.fc.parameters():
    param.requires_grad = True

optimizer = optim.Adam([
    {'params': model.layer4.parameters(), 'lr': 1e-4},
    {'params': model.fc.parameters(), 'lr': 1e-3}
])


================================================================================
[resnet] Get Embeddings ResNet
================================================================================

encoder = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1)
encoder.fc = nn.Identity()
encoder.eval()
encoder = encoder.to(device)

catalog_embeddings = []
for img in catalog:
    with torch.no_grad():
        img = img.to(device)
        emb = encoder(img)
        catalog_embeddings.append(emb.cpu())

catalog_embeddings = torch.cat(catalog_embeddings)


================================================================================
[important] Cosine Similarity Embeddings
================================================================================

catalog_norm = catalog_embeddings / catalog_embeddings.norm(dim=1, keepdim=True)
test_norm = test_embeddings / test_embeddings.norm(dim=1, keepdim=True)
scores = test_norm @ catalog_norm.T
best_indices = scores.argmax(dim=1)

-------
vector1 = vector1 / np.linalg.norm(vector1)
vector2 = vector2 / np.linalg.norm(vector2)
sim = vector1 @ vector2
pred_indices = np.argmax(sim, axis=1)
pred_labels = [labels[i] for i in pred_indices]

---------
from sklearn.metrics.pairwise import cosine_similarity
sim = cosine_similarity(v1, v2)


================================================================================
[nlp] Embeddings GLOVE
================================================================================

embeddings_index = {}
with open('glove.6B.200d.txt', encoding='utf-8') as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = vector
Prima valoare e cuvantul, dupa sunt valorile, deci faci un dict cu key word si values de la word in dreapta.


================================================================================
[nlp] Embeddings GLOVE PART 2
================================================================================

def sentence_embedding(tokens, embeddings_index, emedding_dim=200):
    vectors = [embeddings_index[word] for word in tokens if word in embeddings_index]
    if len(vectors) == 0:
        return np.zeros(emedding_dim)
    return np.mean(vectors, axis=0)

vectors = np.array([sentence_embeddings(tokens, embeddings_index) for tokens in text_tokenized])

Pentru fiecare word din tokens, daca e in embeddings, il pui ca key in embeddings ca sa iei vector. Altfel umplii cu 0*dim


================================================================================
[general] Load given Model/Weights
================================================================================

model = torch.load("model.pth")
model.eval()
--------------
model = MyModel()
model.load_state_dict(torch.load("weights.pth"))
model.eval()


================================================================================
[lstm] Multi-Label And LSTM
================================================================================

[1, 0, 1, 1],
[0, 0, 0, 0],
[0, 0, 0, 0],
criterion = nn.BCEWithLogitsLoss()
def forward(self,x):
        emb = self.emb(x)
        lstm, _ = self.lstm(emb)
        lstm = lstm.max(dim=1)[0] !!!
        out = self.fc(lstm)
        return out


================================================================================
[lstm] BILSTM
================================================================================

class BILSTM(nn.Module):
    def __init__(self):
        super().__init__()
        self.emb = nn.Embedding(VOCAB_SIZE, 128)
        self.lstm = nn.LSTM(128,128, bidirectional=True, batch_first=True, num_layers=2, dropout=0.3)
        self.fc = nn.Linear(256,4)
    def forward(self,x):
        emb = self.emb(x)
        lstm, _ = self.lstm(emb)
        lstm = lstm.max(dim=1)[0]
        out = self.fc(lstm)
        return out


================================================================================
[nlp] Word2Index Freq Count
================================================================================

from collections import Counter
all_words = [w for comment in train_toknes for w in comment if w != "<PAD>"]
freq = Counter(all_words)
VOCAB_SIZE = 40000
most_commong = freq.most_common(VOCAB_SIZE)
word2index = {"<PAD>":0, "<UNK>":1}
for i,(word,_) in enumerate(most_commong):
    word2index[word] = i+1
def encode(token_list):
    return [word2index.get(w,1)for w in token_list]
X_train = [encode(t) for t in train_toknes]


================================================================================
[nlp] Padding
================================================================================

for i in range(len(train_toknes)):
    if len(train_toknes[i]) > PADDING:
        train_toknes[i] = train_toknes[i][:PADDING]
    else:
        for j in range(PADDING - len(train_toknes[i])):
            train_toknes[i].append("<PAD>")


================================================================================
[general] Class Imbalance(WEIGHTS)
================================================================================

Iau labeluri, apoi sklearn.compuste_weights si in CrossEntropyLos pune class weight


================================================================================
[vision] Imread -> PIL IMAGE
================================================================================

img = Image.fromarray(ndarray/opencv_image)


================================================================================
[important] Pune optimizer.zero_grad()
================================================================================

Pune optimizer.zero_grad() mereu!!!!!


================================================================================
[vision] Crop, fit while keeping aspect ration
================================================================================

def preprocess_symbol_pil(img_pil, out_size=48, pad=4):
    img = np.array(img_pil.convert("L"))

    _, binary = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)

    ys, xs = np.where(binary > 0) #aici iei toti pixeli care sunt elementul
    x1, x2 = xs.min(), xs.max()
    y1, y2 = ys.min(), ys.max()
    crop = binary[y1:y2+1, x1:x2+1]

    h, w = crop.shape
    size = max(h, w) + 2 * pad
    canvas = np.zeros((size, size), dtype=np.uint8)
    y_off = (size - h) // 2
    x_off = (size - w) // 2
    canvas[y_off:y_off+h, x_off:x_off+w] = crop
    canvas = cv2.resize(canvas, (out_size, out_size), interpolation=cv2.INTER_AREA)

    return Image.fromarray(canvas)


================================================================================
[general] Save and Load model on best Epoch
================================================================================

if loss<decoy:
        decoy = loss
        torch.save(model.state_dict(), "model.pth")
LOAD BEST:
model = Model()
model.load_state_dict(torch.load("model.pth"))
model = model.to(device)
model.eval()


================================================================================
[vision] ConnectedComponentsWithStats(For symbol in image separation)
================================================================================

all_sequences = []

for i in range(len(test)):
    path = os.path.join("./starting_kit/", test['datapointID'][i])
    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)

    _, binary = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)

    num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(binary, connectivity=8)

    boxes = []
    for t in range(1, num_labels): #gaseste in docs
        x = stats[t, cv2.CC_STAT_LEFT]
        y = stats[t, cv2.CC_STAT_TOP]
        w = stats[t, cv2.CC_STAT_WIDTH]
        h = stats[t, cv2.CC_STAT_HEIGHT]
        area = stats[t, cv2.CC_STAT_AREA]
        if area > 20:
            boxes.append((x, y, w, h))

    boxes = sorted(boxes, key=lambda b: b[0]) #sorteaza de la stanga la dreapta

    sequence = []
    for (x, y, w, h) in boxes:
        crop = img[y:y+h, x:x+w]
        crop_pil = Image.fromarray(crop)
        crop_pil = preprocess_symbol_pil(crop_pil, out_size=48, pad=4)
        sequence.append(crop_pil)

    all_sequences.append(sequence)


================================================================================
[unet] UNET getitem
================================================================================

def __getitem__(self, index):
        img = Image.open(f"./train/images/img_{index+1:03d}.png")
        mask = Image.open(f"./train/masks/mask_{index+1:03d}.png")
        img = self.img_tf(img)
        mask = self.mask_tf(mask)
        mask = np.array(mask, dtype=np.int64)
        mask_bin = (mask > 1).astype(np.float32)
        mask_bin = torch.tensor(mask_bin).unsqueeze(0)
        return img, mask_bin


================================================================================
[unet] UNET Pred mask loop
================================================================================

index = 0
model.eval()
with torch.no_grad():
    for img in test_loader:
        img = img.to(device)
        output = model(img)
        for j in range(output.shape[0]):
            mask = (torch.sigmoid(output[j]) > 0.5).float()*255
            mask = mask.squeeze(0).cpu().byte()
            pil_img = to_pil_image(mask)
            pil_img.save(f"C:/Users/David/Desktop/ProblemeML/vanatorul de glitchuri/preds/img_{index+1:03d}.png")
            index += 1


================================================================================
[important] CUM IAU PREDICTIILE
================================================================================

#Binary
criterion = BCEWithLogLits()
output = model(img)
pred = (torch.sigmoid(output) > 0.5).float()

#Multiclass
criterion = CrossEnthropyLoss()
output = model(img)
pred = output.argmax(dim=1)


================================================================================
[audio] Spectogram
================================================================================

spectograme = []
for i in range(len(train)):
    waveform, sr = torchaudio.load(f"./audio/S_{i:05d}.wav")
    melspec = T.MelSpectrogram(
        sample_rate=sr,
        n_mels=128,
        n_fft=1024,
        hop_length=512
    )(waveform)
    mel_spec_db = T.AmplitudeToDB()(melspec)
    spectograme.append(mel_spec_db)


================================================================================
[audio] SpecToImage(FOR RESNET)
================================================================================

def spec_to_image(spec):
    if spec.dim() == 2:
        spec = spec.unsqueeze(0)
    spec = F.interpolate(spec.unsqueeze(0), size=(224, 224), mode='bilinear').squeeze(0)
    spec = spec.repeat(3, 1, 1)
    from torchvision import transforms
    normalize = transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
    )
    spec = normalize(spec)
    return spec


================================================================================
[audio] GetDistance, Pad
================================================================================

def get_dist(vect1, vect2):
    v1, v2 = np.array(vect1), np.array(vect2)
    return 1-(v1 @ v2)/(np.linalg.norm(v1)*np.linalg.norm(v2) + 1e-10)
---------------------------------
def pad_segment(segment, target_len):
    segment = np.array(segment)
    if len(segment) < target_len:
        pad =np.zeros((target_len-len(segment), segment.shape[1]))
        segment = np.vstack([segment,pad])
    else:
        segment = segment[:target_len]
    return segment


================================================================================
[audio] PreprocessAudioSegmnetation
================================================================================

def preproc_audio(wav_path, target_len=50):
    sr, data_init = wavfile.read(wav_path)
    f, t, Sxx = signal.spectrogram(data_init, sr, nperseg=1024, noverlap=500)
    Sxx = 10 * np.log10(Sxx + 1e-10)

    data = Sxx.T #fiecare element data[i] devine un frame temporal
    noise_base = np.mean(data[0:10], axis=0) #estimeaza zgomotul de fundal la primele 10 frameuri(asta e fix zgomotul)

    dists = []
    for i in range(len(data)):
        dists.append(get_dist(data[i], noise_base)) #pentru fiecare moemnt al audio compara cu zgomotul

    dists = np.array(dists)
    dists_smooth = gaussian_filter1d(dists.astype(float), sigma=4) #netezeste valorile
    thresh = dists_smooth.min() + 0.1 #alegem un prag

    curated_data = []
    temp = []

    for i in range(len(dists)): #cat timp e peste prag, pune valorile, cand cade sub prag considera ca segmentul s-a terminat, dar daca totusi e destul de lung il pastreaza
        if dists_smooth[i] > thresh:
            temp.append(Sxx.T[i])
        else:
            if temp and len(temp) > 10:
                padd = pad_segment(temp, target_len)
                curated_data.append(padd)
            temp = []

    if temp and len(temp) > 10: #apoi dam pad pentru toate
        padd = pad_segment(temp, target_len)
        curated_data.append(padd)

    return np.array(curated_data)


================================================================================
[rl] ValueIteration P1
================================================================================

GRID = 32
N_STATES= 1024
N_ACTIONS = 4
GOAL_STATE = 1023
GAMMA = 0.95
EPSILON = 1e-4
STEP_REWARDS = -1.0
ACTION_NAMES = ["up",'down','left','right'] # 0 1 2 3
OUTCOME_DIRS = {0:[0,3,2,1], 1:[1,2,3,0], 2:[2,0,1,3], 3:[3,1,0,2]}
DELTAS = {0:(-1,0), 1:(1, 0), 2:(0,-1), 3:(0,1)}
def clip(r,c): return max(0, min(GRID-1,r)) * GRID + max(0, min(GRID-1,c))


================================================================================
[rl] Value Iteartion P2
================================================================================

count_sa = np.zeros((N_STATES, N_ACTIONS), dtype = np.float64)
count_sas = np.zeros((N_STATES,N_ACTIONS,N_STATES), dtype = np.float64)
reward_sa = np.zeros((N_STATES,N_ACTIONS), dtype = np.float64)
for row in train.itertuples(index=False):
    s = int(row.state)
    a = int(row.action)
    r = int(row.reward)
    ns = int(row.next_state)
    count_sa[s,a]+=1
    count_sas[s,a,ns]+=1
    reward_sa[s,a] += r
seen = count_sa>0


================================================================================
[rl] Value Iteration P3
================================================================================

P = np.zeros((N_STATES,N_ACTIONS,N_STATES), dtype = np.float64)
for s in range(N_STATES):
    row_s, col_s = divmod(s, GRID)
    for a in range(N_ACTIONS):
        if seen[s,a]: P[s,a] = count_sas[s,a]/count_sa[s,a]
        else:
            dr,dc = DELTAS[a]
            ns_ = clip(row_s+dr, col_s+dc)
            P[s,a,ns_] = 1.0
R = np.where(seen, reward_sa/count_sa, STEP_REWARDS)
V = np.zeros(N_STATES, dtype=np.float64)
Q = np.zeros((N_STATES, N_ACTIONS), dtype = np.float64)
for iterarion in range(1,10001):
    Q = R + GAMMA * (P@V)
    Q[GOAL_STATE,:] = 0.0
    V_new = Q.max(axis=1)
    delta = np.abs(V_new-V).max()
    V = V_new
    if delta < EPSILON:
        print(f"Converged at {EPSILON}")
        break
policy = Q.argmax(axis=1)
test_states = test['state'].values
y_pred = Q[test_states].argmax(axis=1)


================================================================================
[rl] Q-Learning P1
================================================================================

position_bins = np.linspace(-1.2,0.6,20)
velocity_bins = np.linspace(-0.07,0.07,20)

class DiscretizedQLearning:
    def __init__(self,bins, n_actions, alpha=0.1, gamma=0.99, epsilon = 1.0, epsilon_min = 0.01):
        self.bins = bins
        self.Q = np.ones([len(bins[0])+1, len(bins[1]) + 1, n_actions]) *5.0 #facem un array plin de 5.0 de marimea [pozitie][viteza][actiuni]
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon = epsilon
        self.epsilon_min = epsilon_min
    def discretize(self,state): # ia vitesa si pozitia si spune in ce bins se afla
        return tuple(np.digitize(s,b) for s,b in zip(state, self.bins))
    def choose_action(self, state, n_actions):
        if np.random.rand()<self.epsilon:#ALEGI UN NUMAR RANDOM INTRE 0/1 daca e mai mic ca epsilon atunci alegi o actiune random (EXPLORARE)
            return np.random.randint(n_actions)
        s = self.discretize(state) # EXPLOATARE
        return np.argmax(self.Q[s])
    def update(self, state, action, reward, next_state):
        s = self.discretize(state)
        s_next = self.discretize(next_state)
        best_next = np.max(self.Q[s_next])
        self.Q[s][action] += self.alpha * (
            reward + self.gamma * best_next - self.Q[s][action]
        )
    def decay(self, eps_decay=0.9995, alpha_decay=0.9999):
        self.epsilon = max(self.epsilon_min, self.epsilon * eps_decay)
        self.alpha = max(0.01, self.alpha * alpha_decay)


================================================================================
[rl] Q-Learning P2
================================================================================

for ep in episodes:
    state, _ = env.reset()
    done=False
    while not done:
        action = agent.choose_action(state,env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated

        agent.update(state,action,reward,next_state)
        state = next_state
    agent.decay(ep)


================================================================================
[rl] Q-Learning P3
================================================================================

total_rewards = []
logs = []
for i in range(len(test)):
    row = test.iloc[i]
    custom_state=np.array([
        row['start_pos'],
        row['start_vel']
    ])
    obs, info = env.reset()
    env.unwrapped.state = custom_state
    state= custom_state.copy()
    done = False
    total_reward = 0
    step_id = 0
    while not done:
        action = agent.choose_action(state, env.action_space.n)
        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        total_reward += reward
        logs.append({
            "episode_id": i,
            "action": action,
        })
        state= next_state
        step_id +=1
    total_rewards.append(total_reward)
avg = np.mean(total_rewards)
print(avg)
df = pd.DataFrame(logs)
df.to_csv('subi.csv',index=False)


================================================================================
[lstm] Collate_FN
================================================================================

from torch.nn.utils.rnn import pad_sequence
import torch
def collate_fn(batch):
    inputs, labels = zip(*batch)
    inputs = [torch.tensor(x) for x in inputs]
    inputs_padded = pad_sequence(inputs, batch_first=True)
    labels = torch.tensor(labels)
    return inputs_padded, labels


================================================================================
[lstm] Collate_FN (test)
================================================================================

def collate_fn(batch):
    inputs = [torch.tensor(x) for x in batch]
    return pad_sequence(inputs, batch_first=True)


================================================================================
[general] Daca Vreau Probs
================================================================================

probs=output.sigmoid(output)


================================================================================
[important] MultiThreshold Tunining
================================================================================

all_probs =[]
all_labels =[]
with torch.no_grad():
    for img, label in valid_loader:
        img = img.to(device)
        output = model(img)
        probs = torch.sigmoid(output)
        all_probs.append(probs.cpu())
        all_labels.append(label.cpu())
all_probs = torch.cat(all_probs).numpy()
all_labels = torch.cat(all_labels).numpy()
best_thresholds = [0.0]*4
for cls in range(4):
    best_cls_f1 = 0
    best_cls_threshold = 0.0
    for k in range(11):
        threshold = k/10
        probs = (all_probs[:,cls] > threshold).astype(int)
        f1 = f1_score(all_labels[:,cls], probs)
        if f1>best_cls_f1:
            best_cls_f1 = f1
            best_cls_threshold = threshold
    best_cls_threshold[cls] = best_cls_threshold
final_preds= np.zeros_like(all_probs,dtype=int)
for cls in range(4):
    final_preds[:,cls] = (all_probs[:,cls]>best_thresholds[cls]).astyype(int)
f1 = f1_score(all_labels,final_preds,average='macro')


================================================================================
[nlp] FastText
================================================================================

text = text.apply(preprocess_text)
text = text.apply(simple_tokenizer)
from gensim.models import FastText
ft = FastText(sentences = text, vector_size =300, window = 5)
import torch
embeddings = []
def get_emb(text):
    for x in text:
        if x in ft.wv:
            embeddings.appen(ft.wv[x])
    return torch.tensor(embeddings)


================================================================================
[nlp] Preprocess_text
================================================================================

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
lemmatizer=WordNetLemmatizer()
stopwords = set(stopwords.words("english"))
def preprocess_text(text):
    tokens = word_tokenize(text)
    tokens = [lemmatizer.lemmatize(x) for x in tokens if x not in stopwords]
    return " ".join(tokens)
def simple_tokenizer(text):
    return text.split()


================================================================================
[important] Reset Index
================================================================================

text_test = text_test.reset_index(drop=True)


================================================================================
[nlp] Word2Index TotalCount
================================================================================

word2idx = {"<PAD>":0, "<UNK>":1}
for text in all_texts:
     for word in text.split():
             w = word.lower()
             if w not in word2idx:
                   word2idx[w] = len(word2idx)
index_texts = []
for element in all_texts:
        index_texts.append([word2idx.get(word,1) for word in element.split()])


================================================================================
[lstm] Word2Index nn.Embedding
================================================================================

embedding = torch.nn.Embedding(
    num_embeddings=len(word2idx),
    embedding_dim=8,
    padding_idx=0  # IMPORTANT
)


================================================================================
[important] SingleThreshold Search
================================================================================

all_outputs=[]
    all_masks = []
    with torch.no_grad():
        for img, mask in valid_loader:
            img = img.to(device)
            mask = mask.to(device)
            optimizer.zero_grad()
            output = model(img)
            all_outputs.append(output)
            all_masks.append(mask)

    best_iou = 0
    threshold_best = 0.5
    all_outputs = torch.cat(all_outputs,)
    all_masks = torch.cat(all_masks)
    for i in range(10, 65):
        threshold = i / 100
        preds = (torch.sigmoid(all_outputs) > threshold).int()
        iou_metric.reset()
        iou_metric.update(preds, all_masks.int())
        current_iou = iou_metric.compute().item()
        if current_iou > best_iou:
            best_iou = current_iou
            threshold_best = threshold
    scheduler.step(best_iou)
    if best_iou > decoy:
        decoy = best_iou
        torch.save(model.state_dict(), "modelbun.pth")
        best_threshold = threshold_best


================================================================================
[plots] Plotting
================================================================================

#PCA
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=y)
------
#TSNE
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, random_state=42)
X_tsne = tsne.fit_transform(X)
plt.scatter(X_tsne[:, 0], X_tsne[:, 1])


================================================================================
[important] Gabys Idea(Unsupervised Learning)
================================================================================

Foloseste labels de la task1 pentru a separa imaginile in ceva mai fezabil de clasificat. Si pentru imaginile label == 1 faci embeddings cu un resnet, apoi Kmeans cu 2 clustere si ai clasificat intre ce aveai nevoie
Si dupa faci la fel pentru imaginile label == 2
! S-ar putea sa fie invers labelurile (nu stiu, de aflat)


================================================================================
[important] Shilouette Score Brute Force Unsupervised Learning
================================================================================

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

best_k = None
best_score = -1

for i in range(2, 11):   # k de la 2 la 10
    kmeans = KMeans(n_clusters=i, random_state=42, n_init=10)
    labels = kmeans.fit_predict(X)

    score = silhouette_score(X, labels)
    print(f"k={i}, silhouette={score:.4f}")

    if score > best_score:
        best_score = score
        best_k = i

print("Best k:", best_k)
print("Best silhouette:", best_score)


================================================================================
[important] Model creation rules
================================================================================

Linear - >BatchNorm - > ReLU -> Dropout
nn.Sequential(
    nn.Linear(in_dim, 256),
    nn.BatchNorm1d(256),
    nn.ReLU(),
    nn.Dropout(0.2),

    nn.Linear(256, 128),
    nn.BatchNorm1d(128),
    nn.ReLU(),
    nn.Dropout(0.2),

    nn.Linear(128, out_dim)
)


================================================================================
[important] Shape (FULL TUTORIAL) !Long
================================================================================

# ============================
# SHAPES & DEBUGGING CHEAT SHEET (PYTORCH)
# ============================

# ----------------------------------
# 1. CUM CITESTI SHAPE-URILE
# ----------------------------------

# CNN (imagini)
# [B, C, H, W]
# B = batch size
# C = canale (ex: 3 pentru RGB)
# H, W = dimensiuni imagine

x.shape = torch.Size([32, 3, 224, 224])


# MLP (fully connected)
# [B, features]

x.shape = torch.Size([32, 100])


# LSTM (cu batch_first=True)
# [B, seq_len, input_size]

x.shape = torch.Size([32, 50, 128])


# ----------------------------------
# 2. EROARE: "mat1 and mat2 shapes cannot be multiplied"
# ----------------------------------

# apare la Linear

fc = nn.Linear(128, 64)

# layer-ul asteapta:
# input shape = [B, 128]

# daca dai:
x.shape = [32, 256]

# eroare, pentru ca:
# (32, 256) @ (128, 64) -> nu se potrivesc

# REGULA:
# nn.Linear(in_features, out_features)
# => ultima dimensiune din input trebuie sa fie = in_features


# ----------------------------------
# 3. DEBUGGING IN FORWARD
# ----------------------------------

def forward(self, x):
    print("input", x.shape)

    x = self.conv1(x)
    print("dupa conv1", x.shape)

    x = self.pool(x)
    print("dupa pool", x.shape)

    x = torch.flatten(x, 1)
    print("dupa flatten", x.shape)

    x = self.fc(x)
    print("dupa fc", x.shape)

    return x


# ----------------------------------
# 4. CNN - REGULI
# ----------------------------------

# input:
# [B, C, H, W]

# Conv2d:
conv = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3)

# daca dai:
x.shape = [B, 1, H, W]

# eroare:
# expected input to have 3 channels, but got 1


# dupa conv:
x.shape = [32, 64, 112, 112]

# B = 32
# C = 64
# H = 112
# W = 112


# ----------------------------------
# 5. FLATTEN + LINEAR
# ----------------------------------

x.shape = [32, 64, 7, 7]

x = torch.flatten(x, 1)

# devine:
x.shape = [32, 3136]

# pentru ca:
# 64 * 7 * 7 = 3136

fc = nn.Linear(3136, 100)   # corect

# daca pui:
fc = nn.Linear(2048, 100)

# => eroare mat1 mat2


# ----------------------------------
# 6. RESNET
# ----------------------------------

# asteapta:
# [B, 3, H, W]

# daca dai grayscale:
# [B, 1, H, W]

# => eroare

# solutii:
# - convertesti la 3 canale
# - modifici primul conv


# output final:
# [B, num_classes]


# ----------------------------------
# 7. U-NET
# ----------------------------------

# format:
# [B, C, H, W]

# concatenare:
torch.cat([x1, x2], dim=1)

# dim=1 = canale

# H si W trebuie sa fie egale

# daca:
x1 = [8, 64, 128, 128]
x2 = [8, 64, 127, 127]

# => eroare (nu se potrivesc spatial)


# ----------------------------------
# 8. LSTM
# ----------------------------------

lstm = nn.LSTM(input_size=8, hidden_size=64, batch_first=True)

# input:
# [B, seq_len, input_size]

x.shape = [32, 20, 8]   # corect

# daca dai:
x.shape = [32, 20, 10]

# => eroare (input_size mismatch)


# output:
output.shape = [32, 20, 64]

# interpretare:
# B = 32
# 20 timesteps
# fiecare timestep are vector de 64


# daca vrei un singur output:
x = output[:, -1, :]   # [32, 64]

fc = nn.Linear(64, num_classes)


# ----------------------------------
# 9. EROARE: "expected input..."
# ----------------------------------

# EXEMPLU 1:
# ai dat:
# [B, H, W, C]

# dar PyTorch vrea:
# [B, C, H, W]

x = x.permute(0, 3, 1, 2)


# EXEMPLU 2:
# Conv2d asteapta 4D

# tu ai:
# [C, H, W]

# adaugi batch:
x = x.unsqueeze(0)


# ----------------------------------
# 10. LOSS SHAPES
# ----------------------------------

# CrossEntropyLoss:
# output: [B, num_classes]
# target: [B]

# BCEWithLogitsLoss:
# output: [B, 1] sau [B]
# target: [B, 1] sau [B]


# ----------------------------------
# 11. DEBUG DATALOADER
# ----------------------------------

for batch in loader:
    x, y = batch
    print("x", x.shape)
    print("y", y.shape)
    break


# ----------------------------------
# 12. SCHEME RAPIDE
# ----------------------------------

# Linear:
# [B, features]

# Conv2d:
# [B, C, H, W]

# LSTM:
# [B, seq_len, input_size]

# Output clasificare:
# [B, num_classes]


# ----------------------------------
# 13. GRESELI FRECVENTE
# ----------------------------------

# - Softmax inainte de CrossEntropyLoss
# - Sigmoid inainte de BCEWithLogitsLoss
# - input ne-normalizat
# - shape gresit
# - batch_first uitat la LSTM
# - flatten gresit
# - canale gresite


# ----------------------------------
# 14. REGULA DE AUR
# ----------------------------------

# intreaba-te mereu:
# 1. ce shape asteapta layer-ul?
# 2. ce shape ii dau eu?

# daca astea doua nu se potrivesc => eroare


# ----------------------------------
# 15. CHEAT SHEET ULTRA SCURT
# ----------------------------------

# Linear:    [B, F]
# Conv2d:    [B, C, H, W]
# LSTM:      [B, T, F]
# Output:    [B, num_classes]


================================================================================
[important] Word2Vec
================================================================================

w2v = Word2Vec(sentences=unlabeled_tokenized,vector_size=200,window=5,min_count=1,epochs=10)


================================================================================
[important] Submission for 1,2,1,2,1,2
================================================================================

submission = pd.concat([sub1, sub2])
submission = submission.sort_values(by=["datapointID", "subtaskID"]).reset_index(drop=True)


================================================================================
[datetabulare] CosineSimilarity TabularData
================================================================================

import numpy as np
sims = cosine_similarity(predictions, text_task2_gallery_eng_tf)
best_indexies = np.argmax(sims, axis=1)
answer = []
for i in range(len(best_indexies)):
    answer.append(nlp_test_gallery['candidate_id'].iloc[best_indexies[i]])


================================================================================
[important] Greedy Similarity Search
================================================================================

from scipy.spatial.distance import cdist
import numpy as np

D = cdist(pred, text_canditates_tf_svd, metric='euclidean')
n_pred, n_cand = D.shape
pred = np.full(n_pred, -1, dtype=int)
used = set()

for i in range(n_pred):
    sorted_idx = np.argsort(D[i])
    for idx in sorted_idx:
        if idx not in used:
            pred[i] = idx
            used.add(idx)
            break


================================================================================
[important] Hungarian Search
================================================================================

from scipy.optimize import linear_sum_assignment

cost_matrix = -sim_matrix
row_ind, col_ind = linear_sum_assignment(cost_matrix)
answers = [""] * len(en_words)
for i, j in zip(row_ind, col_ind):
    answers[i] = ro_words[j]


================================================================================
[nlp] Preprocess Text Deep
================================================================================

import re
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
lemmatizer = WordNetLemmatizer()

def preprocess_text(text):
    text = text.lower()
    text = re.sub("nevergiveup"," ",text)
    text = re.sub(r"\s+", " ", text).strip()
    text= re.sub(r"https?://\S+", "", text)
    tokens = word_tokenize(text)
    tokens = [lemmatizer.lemmatize(w) for w in tokens]
    return " ".join(tokens)


================================================================================
[nlp] Simplu toate gensim
================================================================================

from gensim.models import KeyedVectors
wv = KeyedVectors.load_word2vec_format("word2vec.txt", binary=False)
vec = wv["cat"]
---------------------------
from gensim.models import KeyedVectors
glove = KeyedVectors.load_word2vec_format("glove.txt", binary=False, no_header=True)
vec = glove["cat"]
--------------------------
from gensim.models import Word2Vec
model = Word2Vec(sentences=[["hello", "world"], ["cat", "dog"]], vector_size=100, window=5, min_count=1)
vec = model.wv["cat"]
Word2Vec(
    sentences=...,
    vector_size=100,   # dimensiunea embeddingului
    window=5,          # context (cate cuvinte in jur)
    min_count=1,       # ignora cuvinte rare
    workers=4,         # threads
    sg=0,              # 0 = CBOW, 1 = Skip-gram
    epochs=5           # IMPORTANT
)
--------------------------
from gensim.models import FastText
model = FastText(sentences=[["hello","world"],["cat","dog"]], vector_size=100)
vec = model.wv["cat"]
FastText(
    sentences=...,
    vector_size=100,
    window=5,
    min_count=1,
    sg=1,
    epochs=5,
    min_n=3,   # subword
    max_n=6
)


================================================================================
[vision] Ssim
================================================================================

def get_lowest_ssim(row_tst):
    img_tst_1 = np.array(Image.open(row_tst.img1).convert("L"))
    img_tst_2 = np.array(Image.open(row_tst.img2).convert("L"))
    img_tst_3 = np.array(Image.open(row_tst.img3).convert("L"))
    idk_bro = np.array([ssim(img_tst_1, img_tst_2), ssim(img_tst_1, img_tst_3), ssim(img_tst_2, img_tst_3)])
    if np.argmax(idk_bro) +1 == 1:
        return 3
    elif np.argmax(idk_bro) +1 == 2:
        return 2
    else:
        return 1


================================================================================
[cnn] MultiHead task(Also has how to treat input if 2 imgs)
================================================================================

class Modelu(nn.Module):
    def __init__(self, num_classes1, num_classes2, num_classes3):
        super().__init__()
        backbone = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1)
        backbone.fc = nn.Identity()
        self.encoder = backbone

        self.shared = nn.Sequential(
            nn.Linear(512 * 3, 256),
            nn.ReLU()
        )

        self.head1 = nn.Linear(256, num_classes1)
        self.head2 = nn.Linear(256, num_classes2)
        self.head3 = nn.Linear(256, num_classes3)

    def forward(self, x1, x2):
        e1 = self.encoder(x1)
        e2 = self.encoder(x2)

        x = torch.cat([e1, e2, torch.abs(e1 - e2)], dim=1)
        x = self.shared(x)

        out1 = self.head1(x)
        out2 = self.head2(x)
        out3 = self.head3(x)

        return out1, out2, out3
for img1, img2, y1, y2, y3 in loader:
    img1, img2 = img1.to(device), img2.to(device)
    y1, y2, y3 = y1.to(device), y2.to(device), y3.to(device)

    out1, out2, out3 = model(img1, img2)

    loss1 = criterion1(out1, y1)
    loss2 = criterion2(out2, y2)
    loss3 = criterion3(out3, y3)

    loss = 0.5 * loss1 + 1.0 * loss2 + 1.5 * loss3 (in functie de cine e cel mai important)


================================================================================
[nlp] Hstack Tfidf
================================================================================

from scipy.sparse import hstack

text_char = tfidf_char.fit_transform(text)
text_word = tfidf_word.fit_transform(text)

text_final = hstack([text_word, text_char])
