# -- Code Cell --
import pandas as pd

# -- Code Cell --
train = pd.read_csv('train_data.csv')
train.head()

# -- Code Cell --
#FASTTEXT

# -- Code Cell --
X = pd.concat([train[train['is_food']==1]['text_en'],train[train['is_food']==0]['text_en']]).reset_index(drop=True)
y = pd.concat([train[train['is_food']==1]['is_food'],train[train['is_food']==0]['is_food']]).reset_index(drop=True)
y.unique()

# -- Code Cell --


# -- Code Cell --
X

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from catboost import CatBoostClassifier
from sklearn.ensemble import RandomForestClassifier
tfidf = TfidfVectorizer(analyzer='word', ngram_range=(3,5), max_features=1200000)
X_tf = tfidf.fit_transform(X)
X_train, X_test, y_train, y_test =train_test_split(X_tf,y, test_size=0.2, random_state=42)
model= CatBoostClassifier(random_state=42)
model.fit(X_train,y_train)
pred = model.predict(X_test)
acc = f1_score(y_test,pred, average='macro')
acc

# -- Code Cell --
test = pd.read_csv('test_data.csv')
text_test = test['text_en']
text_test = tfidf.transform(text_test)

# -- Code Cell --
model.fit(X_tf,y)
pred_final_task1 = model.predict(text_test)

# -- Code Cell --


# -- Code Cell --
train = pd.read_csv("./nlp_train.csv")
queries = pd.read_csv("nlp_test_queries.csv")
gallery = pd.read_csv("nlp_test_gallery.csv")

# -- Code Cell --
parallel = train[train["is_food"] == -1].reset_index(drop=True)
train_es = parallel["text_es"].fillna("").astype(str)
train_en = parallel["text_en"].fillna("").astype(str)

query_es = queries["query_text_es"].fillna("").astype(str)
gallery_en = gallery["text_en"].fillna("").astype(str)

# -- Code Cell --
tfidf_es = TfidfVectorizer(lowercase=True)
tfidf_en = TfidfVectorizer(lowercase=True)

X_train_es = tfidf_es.fit_transform(train_es)
X_train_en = tfidf_en.fit_transform(train_en)

X_query_es = tfidf_es.transform(query_es)
X_gallery_en = tfidf_en.transform(gallery_en)

# -- Code Cell --
from sklearn.decomposition import TruncatedSVD

k_es = min(300, X_train_es.shape[0] - 1, X_train_es.shape[1] - 1)
k_en = min(300, X_train_en.shape[0] - 1, X_train_en.shape[1] - 1)

k_es = max(k_es, 2)
k_en = max(k_en, 2)
svd_es = TruncatedSVD(n_components=k_es, random_state=42)
svd_en = TruncatedSVD(n_components=k_en, random_state=42)

Z_train_es = svd_es.fit_transform(X_train_es)
Z_train_en = svd_en.fit_transform(X_train_en)

Z_query_es = svd_es.transform(X_query_es)
Z_gallery_en = svd_en.transform(X_gallery_en)

# -- Code Cell --
from sklearn.linear_model import Ridge
mapper = Ridge(alpha=1.0)
mapper.fit(Z_train_es, Z_train_en)

Z_query_proj = mapper.predict(Z_query_es)

# -- Code Cell --
from sklearn.metrics.pairwise import cosine_similarity
sim = cosine_similarity(Z_query_proj, Z_gallery_en)
best_idx = sim.argmax(axis=1)

pred_candidate_ids = gallery.iloc[best_idx]["candidate_id"].values

# -- Code Cell --
len(pred_candidate_ids)

# -- Code Cell --
rows =[]
for idx in range(2000):
    if idx<400:
        rows.append({"subtaskID":1,"datapointID":test['datapoint_id'][idx], "answer":pred_final_task1[idx]})
        rows.append({"subtaskID":2,"datapointID":queries['query_id'][idx], "answer":pred_candidate_ids[idx]})
    else:
        rows.append({"subtaskID":2,"datapointID":queries['query_id'][idx], "answer":pred_candidate_ids[idx]})

# -- Code Cell --
df = pd.DataFrame(rows)
df.to_csv('subs.csv',index=False)

# -- Code Cell --
