# -- Code Cell --
from nltk.stem import PorterStemmer,WordNetLemmatizer
from nltk.tokenize import TweetTokenizer
import nltk
nltk.download('averaged_perceptron_tagger_eng')
from nltk.corpus import stopwords,wordnet
def wordnet_tag(word):
    tag = nltk.pos_tag([word.lower()])[0][1][0].lower()
 
    tag_dict = {
        'n': wordnet.NOUN,
        'r' : wordnet.ADV,
        'v' : wordnet.VERB,
        'j': wordnet.ADJ,
 
    }
    return tag_dict.get(tag,wordnet.NOUN)
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()
tokenizer = TweetTokenizer(preserve_case=False,reduce_len=True)
stop_words = stopwords.words('english')
 
def process_text(text):
 
    tokens = tokenizer.tokenize(text.lower())
    tokens = [token for token in tokens if token.isalpha() and token not in stop_words]
 
    tokens = [lemmatizer.lemmatize(word,wordnet_tag(word)) for word in tokens]
 
    return ' '.join(tokens)

# -- Code Cell --
import pandas as pd
import numpy as np
train = pd.read_csv('train.csv')
test = pd.read_csv("test.csv")
candidates = pd.read_csv("candidates.csv")

# -- Code Cell --
groups = train.groupby('id').agg(list)

# -- Code Cell --
mean_per_point_to_keep_shape = []
for group_idx in range(len(groups['coords'])):
    temp_values = {j: [] for j in range(728)}
    for t in range(len(groups['coords'][group_idx])):
        c = 0
        for value in groups['coords'][group_idx][t].split("|"):
            if value != "":
                temp_values[c].append(float(value)) 
                c += 1 
    means = []
    for j in range(728):
        means.append(np.mean(temp_values[j]))
    mean_per_point_to_keep_shape.append(means)  

# -- Code Cell --
text = train['text'].unique()
preprocess_text = [process_text(t) for t in text]

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
tfidf = TfidfVectorizer(ngram_range=(1,3))
preprocess_text_tf=tfidf.fit_transform(preprocess_text)
svd = TruncatedSVD(n_components=300)
preprocess_text_tf_svd = svd.fit_transform(preprocess_text_tf)

# -- Code Cell --
from sklearn.linear_model import Ridge
model = Ridge(alpha = 1.0)
model.fit(mean_per_point_to_keep_shape,preprocess_text_tf_svd)

# -- Code Cell --
groups2 = test.groupby("datapointID").agg(list)

# -- Code Cell --
mean_per_point_to_keep_shape_test = []
for group_idx in range(len(groups2['coords'])):
    temp_values = {j: [] for j in range(728)}
    for t in range(len(groups2['coords'].iloc[group_idx])):
        c = 0
        for value in groups2['coords'].iloc[group_idx][t].split("|"):
            if value != "":
                temp_values[c].append(float(value)) 
                c += 1 
    means = []
    for j in range(728):
        means.append(np.mean(temp_values[j]))
    mean_per_point_to_keep_shape_test.append(means)  

# -- Code Cell --
pred = model.predict(mean_per_point_to_keep_shape_test)

# -- Code Cell --
text_canditates = candidates['text']
text_canditates_tf = tfidf.transform(text_canditates)
text_canditates_tf_svd = svd.transform(text_canditates_tf)

# -- Code Cell --
from scipy.spatial.distance import cdist
import numpy as np
 
D = cdist(pred, text_canditates_tf_svd, metric='euclidean')
 
n_pred, n_cand = D.shape
pred = np.full(n_pred, -1, dtype=int)
used = set()
 
for i in range(n_pred):
    sorted_idx = np.argsort(D[i])
    
    for idx in sorted_idx:
        if idx not in used:
            pred[i] = idx
            used.add(idx)
            break

# -- Code Cell --
pred

# -- Code Cell --
