# -- Code Cell --
import pandas as pd
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# -- Code Cell --
train.head()

# -- Code Cell --
#TASK 1
train['chirp_len']  = train['chirp'].str.len()
test['chirp_len']  = test['chirp'].str.len()

# -- Code Cell --
train['chirp_len'] 

# -- Code Cell --
#TASK 2
train['hastags'] = train['chirp'].str.count('#')
test['hastags'] = test['chirp'].str.count('#')

# -- Code Cell --
train.head()

# -- Code Cell --
train['chirp'] = train['chirp'].str.lower()
test['chirp'] = test['chirp'].str.lower()

# -- Code Cell --
train['chirp'] = train['chirp'].str.replace(r'[^a-zA-Z\s]', '', regex=True)
test['chirp'] = test['chirp'].str.replace(r'[^a-zA-Z\s]', '', regex=True)

# -- Code Cell --
from nltk.tokenize import word_tokenize
train['tokens'] = train['chirp'].apply(word_tokenize)
test['tokens'] = test['chirp'].apply(word_tokenize)

# -- Code Cell --
from nltk.corpus import stopwords
stop = set(stopwords.words('english'))
train['tokens'] = train['tokens'].apply(lambda tokens: [w for w in tokens if w not in stop])
test['tokens'] = test['tokens'].apply(lambda tokens: [w for w in tokens if w not in stop])

# -- Code Cell --
# from nltk.stem import SnowballStemmer
# stemmer = SnowballStemmer('english')
# train['tokens'] = train['tokens'].apply(lambda tokens: [stemmer.stem(w) for w in tokens])
# test['tokens'] = test['tokens'].apply(lambda tokens: [stemmer.stem(w) for w in tokens])

# -- Code Cell --
from nltk.stem import WordNetLemmatizer
lem = WordNetLemmatizer()
train['tokens'] = train['tokens'].apply(lambda tokens: [lem.lemmatize(w) for w in tokens])
test['tokens'] = test['tokens'].apply(lambda tokens: [lem.lemmatize(w) for w in tokens])

# -- Code Cell --
train['text_clean'] = train['tokens'].apply(lambda tokens: ' '.join(tokens))
test['text_clean'] = test['tokens'].apply(lambda tokens: ' '.join(tokens))

# -- Code Cell --
from sklearn.model_selection import train_test_split
X=train.drop(columns=['label']).copy()
y= train['label'].copy()
X_train, X_test, y_train ,y_test =train_test_split(X,y,test_size=0.2,random_state=42)

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
import scipy.sparse as sp
vec = TfidfVectorizer(max_features=20000, ngram_range=(1,2), sublinear_tf=True)
char_vec  = TfidfVectorizer(analyzer='char_wb', ngram_range=(2,4), max_features=20000)
X_train_c = char_vec.fit_transform(X_train['text_clean'])
X_train_tfidf = vec.fit_transform(X_train['text_clean'])
X_test_tfidf  = vec.transform(X_test['text_clean'])
X_test_c = char_vec.transform(X_test['text_clean'])
X_test_final = sp.hstack([X_test_tfidf, X_test_c])
X_train_final = sp.hstack([X_train_tfidf, X_train_c])

# -- Code Cell --
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(random_state=42)
model.fit(X_train_final, y_train)
proba = model.predict_proba(X_test_final)[:, 1]

# -- Code Cell --
from sklearn.metrics import roc_auc_score
acc = roc_auc_score(y_test, proba)
acc

# -- Code Cell --
X_tfidf = vec.transform(X['text_clean'])
X_c = char_vec.transform(X['text_clean'])
test_tfidf = vec.transform(test['text_clean'])
test_c = char_vec.transform(test['text_clean'])
test_final = sp.hstack([test_tfidf, test_c])
X_final = sp.hstack([X_tfidf,X_c])

# -- Code Cell --
test

# -- Code Cell --
model.fit(X_final,y)
proba2 = model.predict_proba(test_final)[:, 1]

# -- Code Cell --
rows = []
for i, row in test.iterrows():
    rows.append({'subtaskID':1, 'datapointID':row['id'], 'answer':row['chirp_len']})
    rows.append({'subtaskID':2, 'datapointID':row['id'], 'answer':row['hastags']})
    rows.append({'subtaskID':3, 'datapointID':row['id'], 'answer':proba2[i]})

pd.DataFrame(rows).to_csv('submission.csv',index=False)