

import pandas as pd
train = pd.read_csv('train.csv')
test  = pd.read_csv('test.csv')
train.head()
train['label'].value_counts()
train['text_len']  = train['text'].str.len()
test['text_len']  = test['text'].str.len()
train['text'] = train['text'].str.lower()
test['text'] = test['text'].str.lower()
train['text'] = train['text'].str.replace(r'[^a-zA-Z\s]', '', regex=True)
test['text'] = test['text'].str.replace(r'[^a-zA-Z\s]', '', regex=True)
from nltk.tokenize import word_tokenize
train['tokens'] = train['text'].apply(word_tokenize)
test['tokens'] = test['text'].apply(word_tokenize)
from nltk.corpus import stopwords
stop = set(stopwords.words('english'))
train['tokens'] = train['tokens'].apply(lambda tokens: [w for w in tokens if w not in stop])
test['tokens'] = test['tokens'].apply(lambda tokens: [w for w in tokens if w not in stop])
from nltk.stem import WordNetLemmatizer
lem = WordNetLemmatizer()
train['tokens'] = train['tokens'].apply(lambda tokens: [lem.lemmatize(w) for w in tokens])
test['tokens'] = test['tokens'].apply(lambda tokens: [lem.lemmatize(w) for w in tokens])
train['text_clean'] = train['tokens'].apply(lambda tokens: ' '.join(tokens))
test['text_clean'] = test['tokens'].apply(lambda tokens: ' '.join(tokens))
from sklearn.model_selection import train_test_split
X=train.drop(columns=['label']).copy()
y= train['label'].copy()
X_train, X_test, y_train ,y_test =train_test_split(X,y,test_size=0.2,random_state=42)
from sklearn.feature_extraction.text import TfidfVectorizer
import scipy.sparse as sp
vec = TfidfVectorizer(max_features=20000, ngram_range=(1,2), sublinear_tf=True)
X_train_tfidf = vec.fit_transform(X_train['text_clean'])
X_test_tfidf  = vec.transform(X_test['text_clean'])
X_tfidf = vec.transform(X['text_clean'])
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
model = LinearSVC(random_state=42)
model.fit(X_train_tfidf, y_train)
proba = model.predict(X_test_tfidf)
from sklearn.metrics import accuracy_score
acc = accuracy_score(y_test, proba)
acc
test_tfidf = vec.transform(test['text_clean'])
model.fit(X_tfidf,y)
proba2 = model.predict(test_tfidf)
test.head()
rows = []
for i, row in test.iterrows():
    rows.append({'id':row['id'],'label':proba2[i]})
pd.DataFrame(rows).to_csv('submission.csv',index=False)
