# -- Code Cell --
import pandas as pd
train = pd.read_csv("./train.csv")

# -- Code Cell --
train['text'].unique()

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(ngram_range=(1,2), max_features=50000)
text = train['text'].unique()
text_tf = tfidf.fit_transform(text)
text_tf

# -- Code Cell --
groups = train.groupby('id').agg(list)
groups['coords'][1][0]

# -- Code Cell --
import numpy as np

# -- Code Cell --
np.unique(groups['coords'][1][0].split("|"))

# -- Code Cell --
values = []
for idx,t in enumerate(groups['coords'][1][0].split("|")):
    if t != '':
        values.append(float(t))
values

# -- Code Cell --
values_id_1 = []
for z in range(len(groups['coords'][1])):
    for idx,t in enumerate(groups['coords'][1][0].split("|")):
        if t != '':
            values_id_1.append(float(t))
values_id_1

# -- Code Cell --
values_real = []
for tt in range(len(groups['coords'])): 
        temp_values = []
        values_id_all = {}
        c=0
        for ss in range(len(groups['coords'][tt])):
                for t in groups['coords'][tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = np.mean(temp_values)
                c+=1
        values_real.append(np.mean(list(values_id_all.values())))
values_real

# -- Code Cell --
len(values_real)

# -- Code Cell --
text_tf = text_tf.toarray()

# -- Code Cell --
from sklearn.linear_model import Ridge
model = Ridge(alpha=1.0)
values_real = np.array(values_real)
values_real = values_real.reshape(-1, 1)
model.fit(values_real, text_tf)

# -- Code Cell --
candidates = pd.read_csv("./candidates.csv")
test = pd.read_csv("./test.csv")

# -- Code Cell --
test

# -- Code Cell --
groups2 = test.groupby('datapointID').agg(list)

# -- Code Cell --
groups2["coords"][10][0]

# -- Code Cell --
values_real = []
for tt in range(len(groups2['coords'])): 
        temp_values = []
        values_id_all = {}
        c=0
        for ss in range(len(groups2['coords'].iloc[tt])):
                for t in groups2['coords'].iloc[tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = np.mean(temp_values)
                c+=1
        values_real.append(np.mean(list(values_id_all.values())))
values_real

# -- Code Cell --
candidates_tf = tfidf.transform(candidates['text'])

# -- Code Cell --
candidates_tf = candidates_tf.toarray()

# -- Code Cell --
len(candidates_tf)

# -- Code Cell --
values_real = np.array(values_real)
values_real = values_real.reshape(-1,1)
preds = model.predict(values_real)

# -- Code Cell --
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.metrics import f1_score

# -- Code Cell --
labels = [0] *50
for idx, pred in enumerate(preds):
    best_corr = -1
    for index, text in enumerate(candidates_tf):
        pred = np.array(pred)
        pred = pred.reshape(1, -1)
        text = np.array(text)
        text = text.reshape(1, -1)
        if cosine_similarity(pred,text)[0][0]>best_corr:
            best_corr = cosine_similarity(pred,text)
            labels[idx]=index

# -- Code Cell --
datapoint = [i for i in range(1,51)]
datapoint

# -- Code Cell --
sub1 = pd.DataFrame({
    'subtaskID':1,
    'datapointID':datapoint,
    'answer':labels
}).to_csv("subs.csv",index=False)

# -- Code Cell --
train['text'].unique()

# -- Code Cell --
candidates['text'].unique()

# -- Code Cell --
texts = train['text'].unique()
texts[0]

# -- Code Cell --
values_real_train = []
for tt in range(len(groups['coords'])): 
        values_id_all = {}
        c=0
        for ss in range(len(groups['coords'][tt])):
                temp_values = []
                for t in groups['coords'][tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = temp_values
                c+=1
        values_real_train.append(np.mean(list(values_id_all.values()),axis=0))

# -- Code Cell --
values_real = []
for tt in range(len(groups2['coords'])): 
        values_id_all = {}
        c=0
        for ss in range(len(groups2['coords'].iloc[tt])):
                temp_values = []
                for t in groups2['coords'].iloc[tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = temp_values   
                c+=1
        values_real.append(np.mean(list(values_id_all.values()), axis=0))  

# -- Code Cell --
arr = np.array(values_real)
arr.shape

# -- Code Cell --
train_ids = groups.index.to_numpy()
train_ids

# -- Code Cell --
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(random_state=42)
model.fit(values_real_train,train_ids)
pred = model.predict(values_real)

# -- Code Cell --
X_train = []
y_train = []
X_test =[]
for i in range(len(train)):
    temp_values = []
    for t in train['coords'][i].split("|"):
        if t != '':
            temp_values.append(float(t))
    
    X_train.append(temp_values)
    y_train.append(train['id'][i])
for i in range(len(test)):
    temp_values = []
    for t in test['coords'][i].split("|"):
        if t != '':
            temp_values.append(float(t))
    
    X_test.append(temp_values)

# -- Code Cell --
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)
model = LogisticRegression(max_iter=1000)
model.fit(X_train,y_train)
preds = model.predict(X_test)

# -- Code Cell --
from collections import Counter

final_preds = []

for did in sorted(test['datapointID'].unique()):
    pred_for_did = preds[test['datapointID'] == did]
    most_common_pred = Counter(pred_for_did).most_common(1)[0][0]
    final_preds.append(most_common_pred)

# -- Code Cell --
texts_candidate = candidates['text']
id_to_text = train.groupby('id')['text'].first()
answer = []
pred_to_texts = []
for value in final_preds:
    pred_to_texts.append(id_to_text[value])
for text_from_pred in pred_to_texts:
    for idx,text in enumerate(texts_candidate):
        if text_from_pred == text:
            answer.append(idx)

# -- Code Cell --
len(answer)

# -- Code Cell --
sub1 = pd.DataFrame({
    'subtaskID':1,
    'datapointID':datapoint,
    'answer':answer
}).to_csv("subs.csv",index=False)

# -- Markdown Cell --
# # STRATEGII ABORDATE
# ### 1. Mean la fiecare lista de coords, apoi mean la fiecare cluster de 45-65 de coords -> Score 0
# ### 2. Vector mean la fiecare cluster de 45-65 -> Score 0
# ### 3. Full lista de coords + LR pentru a prezice datapointID, apoi aggregate la predictii cu textul din train si match exact cu textele din candidates -> Score 0

# -- Code Cell --


# -- Code Cell --


# -- Code Cell --


# -- Code Cell --


# -- Code Cell --


# -- Code Cell --
import pandas as pd
import numpy as np
train = pd.read_csv('train.csv')
test = pd.read_csv("test.csv")

# -- Code Cell --
text = train['text'].unique()
text

# -- Code Cell --
groups = train.groupby('id').agg(list)

groups['coords'][1][0]

# -- Code Cell --
values_real = []
for tt in range(len(groups['coords'])): 
        values_id_all = {}
        c=0
        for ss in range(len(groups['coords'][tt])):
                temp_values = []
                for t in groups['coords'][tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = np.mean(temp_values)
                c+=1
        values_real.append(np.mean(list(values_id_all.values())))
values_real

# -- Code Cell --
values_real = np.array(values_real).reshape(-1, 1)

# -- Code Cell --
from nltk.stem import PorterStemmer,WordNetLemmatizer
from nltk.tokenize import TweetTokenizer
import nltk
nltk.download('averaged_perceptron_tagger_eng')
from nltk.corpus import stopwords,wordnet
def wordnet_tag(word):
    tag = nltk.pos_tag([word.lower()])[0][1][0].lower()
 
    tag_dict = {
        'n': wordnet.NOUN,
        'r' : wordnet.ADV,
        'v' : wordnet.VERB,
        'j': wordnet.ADJ,
 
    }
 
    return tag_dict.get(tag,wordnet.NOUN)
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()
tokenizer = TweetTokenizer(preserve_case=False,reduce_len=True)
stop_words = stopwords.words('english')
 
def process_text(text):
 
    tokens = tokenizer.tokenize(text.lower())
    tokens = [token for token in tokens if token.isalpha() and token not in stop_words]
 
    tokens = [lemmatizer.lemmatize(word,wordnet_tag(word)) for word in tokens]
 
    return ' '.join(tokens)

# -- Code Cell --
from sklearn.decomposition import TruncatedSVD
svd= TruncatedSVD(n_components=300)
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer(ngram_range=(1,3), max_features=300)
processed_text = [process_text(t) for t in text]
text_tf = tfidf.fit_transform(processed_text)
text_svd = svd.fit_transform(text_tf)

# -- Code Cell --
train_avg = train.groupby('id')['coords'].apply(lambda x: np.mean(np.vstack([[float(v) for v in coord.split('|') if v != '']for coord in x]),axis=0))

# -- Code Cell --
from sklearn.linear_model import Ridge
model = Ridge(alpha=1)
X_train = np.vstack(train_avg.values)
model.fit(X_train, text_svd)

# -- Code Cell --
groups2 = test.groupby('datapointID').agg(list)
groups2['coords'][1][0]

# -- Code Cell --
values_real_test = []
for tt in range(len(groups2['coords'])): 
        values_id_all = {}
        c=0
        for ss in range(len(groups2['coords'].iloc[tt])):
                temp_values = []
                for t in groups2['coords'].iloc[tt][ss].split("|"):
                        if t != '':
                                temp_values.append(float(t))
                values_id_all[c] = np.mean(temp_values)
                c+=1
        values_real_test.append(np.mean(list(values_id_all.values())))
values_real_test

# -- Code Cell --
values_real_test = np.array(values_real_test).reshape(-1, 1)

# -- Code Cell --
test_avg = test.groupby('datapointID')['coords'].apply(lambda x: np.mean(np.vstack([[float(v) for v in coord.split('|') if v != '']for coord in x]),axis=0))
X_test = np.vstack(test_avg.values)

# -- Code Cell --
preds = model.predict(X_test)

# -- Code Cell --
candidates = pd.read_csv('candidates.csv')

# -- Code Cell --
from sklearn.linear_model import Ridge
model = Ridge(alpha=1)
X_train = np.vstack(train_avg.values)
model.fit(X_train, text_svd)

# -- Code Cell --
texts_candidate = candidates["text"]

# -- Code Cell --
processed_candidates = [process_text(t) for t in texts_candidate]
texts_candidate_tf = tfidf.transform(processed_candidates)

# -- Code Cell --
texts_candidate_svd = svd.transform(texts_candidate_tf)

# -- Code Cell --
texts_candidate_svd

# -- Code Cell --
from sklearn.metrics.pairwise import cosine_similarity

# -- Code Cell --
labels = [0] * 50
for idx, test_pred in enumerate(preds):
    best_similarity_for_this_pred = -67

    for index, choose_text in enumerate(texts_candidate_svd):
        similarity = cosine_similarity(test_pred.reshape(1, -1),choose_text.reshape(1, -1))[0][0]
        if similarity > best_similarity_for_this_pred:
            best_similarity_for_this_pred = similarity
            labels[idx] = index

# -- Code Cell --
labels

# -- Code Cell --
from sklearn.metrics.pairwise import euclidean_distances

dist = euclidean_distances(preds, texts_candidate_svd)
labels = dist.argmin(axis=1)

# -- Code Cell --
from scipy.spatial.distance import cdist
import numpy as np
 
D = cdist(preds, texts_candidate_svd, metric='euclidean')
 
n_pred, n_cand = D.shape
pred = np.full(n_pred, -1, dtype=int)
used = set()
 
for i in range(n_pred):
    sorted_idx = np.argsort(D[i])
    
    for idx in sorted_idx:
        if idx not in used:
            pred[i] = idx
            used.add(idx)
            break

# -- Code Cell --
pred

# -- Code Cell --
sub = pd.DataFrame({
    "subtaskID":1,
    "datapointID":[i + 1 for i in range(50)],
    'answer':pred
}).to_csv("subs.csv",index=False)

# -- Code Cell --
