# -- Code Cell --
import pandas as pd
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# -- Code Cell --
train.head()

# -- Code Cell --
train['Text'] = train['Text'].str.lower()

# -- Code Cell --
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

stop = set(stopwords.words('english'))
lem = WordNetLemmatizer()

def clean_tokens(text):
    tokens = word_tokenize(text)
    tokens = [w for w in tokens if w not in stop]
    tokens = [lem.lemmatize(w) for w in tokens]
    return tokens

train['tokens'] = train['Text'].apply(clean_tokens)
train['text_clean'] = train['tokens'].str.join(" ")

# -- Code Cell --
train

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC

X = train['Text']
y = train['language']

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

print("TF-IDF...")
tfidf = TfidfVectorizer(analyzer='char_wb', ngram_range=(3,5), max_features=50000)

X_train_tf = tfidf.fit_transform(X_train)
X_test_tf = tfidf.transform(X_test)

print("Training...")
model = LinearSVC()

model.fit(X_train_tf, y_train)

pred = model.predict(X_test_tf)
acc = accuracy_score(y_test, pred)
print("Accuracy:", acc)

# -- Code Cell --
X_tf = tfidf.fit_transform(X)
model.fit(X_tf,y)
X_test_tf  = tfidf.transform(test['Text'])
preds = model.predict(X_test_tf)

# -- Code Cell --
test.head()

# -- Code Cell --
rows = []
for idx, row in test.iterrows():
    rows.append({"SampleID":row['SampleID'],'language':preds[idx]})
pd.DataFrame(rows).to_csv('subs.csv',index=False)