

# # FAST TEXT
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
stopwords = set(stopwords.words("english"))
from gensim.models import FastText
lemmatizer= WordNetLemmatizer()
def preprocess_text(text):
    tokens = word_tokenize(text)
    tokens = [lemmatizer.lemmatize(w) for w in tokens if w not in stopwords]
    return " ".join(tokens)
def simple_spliter(text):
    return text.split()
text = text.apply(preprocess_text)
text = text.applt(simple_spliter)
ft=FastText(sentences=text, vector_size=300, window=5)
import torch
def get_embeddings(text):
    embeddings = []
    for x in text:
        if x in ft.wv:
            embeddings.append(ft.wv[x])
    return torch.tensor(embeddings)
embeddings = [get_embeddings(token) for token in text]
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
stopwords = set(stopwords.words('english'))
def preprocess_text(text):
    tokens = word_tokenize(text)
    tokens = [lemmatizer.lemmatize(w) for w in tokens if w not in stopwords]
    return " ".join(tokens)
def simple_tokenizer(text):
    return text.split()
text.apply(preprocess_text)
text.apply(simple_tokenizer)
from gensim.models import FastText
ft = FastTest(sencentes = text, vector_size =300, window =5)
import torch
def get_embeddings(text):
    for x in text:
        if x in ft.wv:
            embeddings.append(ft.wv[x])
    return torch.tensor(embeddings)    
embs = [get_embeddings(token) for token in text]
from torch.nn.utils.rnn import pad_sequence
def collate_fn(batch):
    X,y = zip(*batch)
    X = pad_sequence(X, padding_value = 0.0, batch_first=True)
    return X, torch.stack(y)
def collate_fn_test(batch):
    X = pad_sequence(batch, padding_value = 0.0, batch_first=True)
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
lemmatizer=WordNetLemmatizer()
stopwords = set(stopwords.words("english"))
def preprocess_text(text):
    tokens = word_tokenize(text)
    tokens = [lemmatizer.lemmatize(x) for x in tokens if x not in stopwords]
    return " ".join(tokens)
def simple_tokenizer(text):
    return text.list()
text = text.apply(preprocess_text)
text = text.apply(simple_tokenizer)
from gensim.models import FastText
ft = FastText(sentences = text, vector_size =300, window = 5)
import torch
embeddings = []
def get_emb(text):
    for x in text:
        if x in ft.wv:
            embeddings.appen(ft.wv[x])
    return torch.tensor(embeddings)
emb = [get_emb(tokens) for tokens in text]
def collate_fn(batch):
    X,y = zip(*batch)
    X = pad_sequence(X, padding_value = 0.0, batch_first = True)
    return X, torch.stack(y)
def collate_fn_test(batch):
    return pad_sequence(batch, padding_value = 0.0, batch_first= True)
all_probs = []
all_labels = []
with torch.no_grad():
    for img, label in valid_loader:
        img = img.to(device)
        output = model(img)
        probs = torch.sigmoid(output)
        all_probs.append(probs.cpu())
        all_labels.append(label.cpu())
all_probs = torch.cat(all_probs).numpy()
all_labels = torch.cat(all_labels).numpy()
best_thresholds = [0.0]*4
for cls in range(4):
    best_cls_f1 = -1
    best_cls_threshold = 0.5
    for k in range(11):
        threshold = k/10
        preds_cls = (all_probs[:, cls] > threshold).astype(int)
        f1 = f1_score(all_labels[:, cls], preds_cls)
        if f1>best_cls_f1:
            best_cls_f1 = f1
            best_cls_threshold = threshold
    best_thresholds[cls] = best_thresholds
final_preds = np.zeros_like(all_probs, dtype=int)
for cls in range(4):
    final_preds[:, cls] = (all_probs[:, cls]>best_thresholds[cls]).astype(int)
f1 = f1_score(all_labels, final_preds, average = 'macro')
all_probs =[]
all_labels = []
with torch.no_grad():
    for img, label in valid_loader:
        img = img.to(device)
        output = model(img)
        probs = torch.sigmoid(output)
        all_probs.append(probs.cpu())
        all_labels.append(label.cpu())
all_probs = torch.cat(all_probs).numpy()
all_labels = torch.cat(all_labels).numpy()
best_thresholds = [0.0]*4
for cls in range(4):
    best_cls_f1 = 0
    best_cls_threshold = 0.0
    for k in range(11):
        threshold=k/10
        preds_cls = (all_probs[:, cls]>threshold).astype(int)
        f1 = f1_score(all_labels[:,cls], preds_cls)
        if f1 >best_cls_f1:
            best_cls_f1 = f1
            best_cls_threshold = threshold
    best_cls_threshold[cls] = best_cls_threshold
final_preds = np.zeros_list(all_probs,dtype =int)
for cls in range(4):
    final_preds[:, cls] =(all_probs[:,cls]> best_thresholds[cls]).astype(int)
f1 = f1_score(all_labels, final_preds,average='macro')
import numpy as np
all_probs =[]
all_labels =[]
with torch.no_grad():
    for img, label in valid_loader:
        img = img.to(device)
        output = model(img)
        probs = torch.sigmoid(output)
        all_probs.append(probs.cpu())
        all_labels.append(label.cpu())
all_probs = torch.cat(all_probs).numpy()
all_labels = torch.cat(all_labels).numpy()
best_thresholds = [0.0]*4
for cls in range(4):
    best_cls_f1 = 0
    best_cls_threshold = 0.0
    for k in range(11):
        threshold = k/10
        probs = (all_probs[:,cls] > threshold).astype(int)
        f1 = f1_score(all_labels[:,cls], probs)
        if f1>best_cls_f1:
            best_cls_f1 = f1
            best_cls_threshold = threshold
    best_cls_threshold[cls] = best_cls_threshold
final_preds= np.zeros_like(all_probs,dtype=int)
for cls in range(4):
    final_preds[:,cls] = (all_probs[:,cls]>best_thresholds[cls]).astyype(int)
f1 = f1_score(all_labels,final_preds,average='macro')
# # GLOVE
embeddings_index ={}
with open("glove.6b.200d.txt", encoding='utf-8') as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:],dtype=np.float32)
        embeddings_index[word] = vector
embeddings_index = {}
with open("") as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype = np.float32)
        embeddings_index[word] = vector
embeddings_index = {}
with open("") as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype=np.float32)
def sentence_embeddings(tokens, embeddings_index):
    vector = [embeddings_index[word] for word in tokens]
    if len(vector) == 0:
        return np.array(200)
    return np.mean(vector, axis=0)
embeddings = np.array[[sentence_embeddings(tokens, embeddings_index) for tokens in text_tokenized]]
embeddings_index = {}
with open("") as f:
    for line in f:
        values = line.split()
        word = values[0]
        vector = np.asarray(values[1:], dtype=np.float32)
        embeddings_index[word] = vector
def sentence_embeddings(token, embeddings_index):
    vector = [embeddings_index[word] for word in token if word in embeddings_index]
    if len(vector) == 0:
        return np.zeros(200)
    return np.mean(vector,axis=0)
def sentence_embeddings(tokens,embeddings_index):
    vectors = [embeddings_index(word) for word in tokens if word in embeddings_index]
    if len(vectors) == 0:
        return np.zeros(200)
    return np.mean(vectors, axis=0)
def sentence_embeddings(tokens,embeddings_index):
    vectors = [embeddings_index[word] for word in tokens if word in embeddings_index]
    if len(vectors) ==0:
        return np.zeros(200)
    return np.mean(vectors, axis=0)
from nltk.tokenize import word_tokenize
text_tokenized = word_tokenize(text)
vectors = np.array[[sentence_embeddings(token, embeddings_index) for token in text_tokenized]]
vectors = np.array([sentence_embeddings(tokens,embeddings_index) for tokens in text_tokenized])
vectors = np.array([sentence_embeddings(tokens, embeddings_index) for tokens in text_tokenized])
vectors = np.array([sentence_embeddings(tokens, embeddings_index) for tokens in text_tokenized])
# #### Cosine Similarity
vector1 = vector1 / np.linalg.norm(vector1)
vector2 = vector2 / np.linalg.norm(vector2)
sim = vector1 @ vector2
pred_indices = np.argmax(sim, axis=1)
pred_labels = [labels[i] for i in pred_indices]
# # WORD 2 INDEX + LSTM
# #### Total count
word2idx = {"<PAD>":0, "<UNK>":1}
for text in all_texts:
    for word in text.split():
        if word not in word2idx:
            word2idx[word] = len(word2idx)
index_texts = []
for element in all_texts:
    index_texts.append([word2idx.get(word,1) for word in element.split()])
word2idx ={"<PAD>":0, "<UNK>":1}
for text in all_texts:
    for word in text.split():
        word = word.lower()
        if word not in word2idx:
            word2idx[word] = len(word2idx)
indixs = []
for element in all_texts:
    indixs.append([word2idx.get(word,1) for word in element.split()])
word2idx = {"<PAD>":0, "<UNK>":1}
for text in all_texts:
    for word in text.split():
        w = word.lower()
        if w not in word2idx:
            word2idx[w] = len(word2idx)
index_texts = []
for element in all_texts:
    index_texts.append([word2idx.get(word,1) for word in element.split()])
embedding = torch.nn.Embedding(
    num_embeddings=len(word2idx),
    embedding_dim=8,
    padding_idx=0  # IMPORTANT
)
word2idx = {"<PAD>":0, "<UNK>":1}
for text in all_texts:
    for word in text.split():
        word = word.lower()
        if word not in word2idx:
            word2idx[word] = len(word2idx)
index_texts = []
for element in all_texts:
    index_texts.append([word2idx.get(word,1) for word in element.split()])
word2idx = {"<PAD>":0, "<UNK>":1}
for text in all_texts:
    for word in text.split():
        word = word.lower()
        if word not in word2idx:
            word2idx[word] = len(word2idx)
text_index = []
for element in all_texts:
    text_index.append([word2idx.get(word) for word in element.split()])
word2idx = {'PAD':0, "UNK":1}
for text in all_texts:
    for word in text.split():
        word = word.lower()
        if word not in word2idx:
            word2idx[word] = len(word2idx)
indici = []
for element in all_texts:
    indici.append([word2idx[word] for word in element.split()])
