# -- Code Cell --
import pandas as pd
train = pd.read_csv('train.csv')
test= pd.read_csv('test.csv')

# -- Code Cell --
train.head()

# -- Code Cell --
train['label_name'].value_counts()

# -- Code Cell --
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import re
stop = set(stopwords.words('english'))
lem = WordNetLemmatizer()
train['text'] = train['text'].str.lower()
train['text'] = train['text'].str.replace(r'[^a-z\s]',"",regex=True)
def clean(text):
    text = str(text).lower()
    text = re.sub(r'\{@(.*?)@\}', r'\1', text)
    text = re.sub(r'\{\{USERNAME\}\}', ' username ', text)
    text = re.sub(r'\{\{URL\}\}', ' url ', text)
    text = re.sub(r'[^a-z0-9\s#]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text
train['tokens'] = train['text'].apply(clean)
train['clean_text'] = train['tokens'].str.join(" ")

# -- Code Cell --
train['clean_text']

# -- Code Cell --


# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score
from sklearn.ensemble import GradientBoostingClassifier
tfidf = TfidfVectorizer(ngram_range=(1,2), max_features=50000, sublinear_tf=True, min_df=2)
X = train.drop(columns=['ID','label_name'])
y = train['label_name']
X_train, X_test, y_trian, y_test = train_test_split(X,y,test_size=0.2,random_state=42)
X_train_tfidf = tfidf.fit_transform(X_train['clean_text'])
X_test_tfidf = tfidf.transform(X_test['clean_text'])
model = GradientBoostingClassifier(n_estimators=100,random_state=42)
model.fit(X_train_tfidf,y_trian)
pred = model.predict(X_test_tfidf)
acc = f1_score(y_test,pred,average='weighted')
acc

# -- Code Cell --
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.utils.class_weight import compute_class_weight

# -- Code Cell --
classes = np.unique(y)
weights = compute_class_weight('balanced', classes=classes, y=y)
class_weight = dict(zip(classes, weights))

model = LogisticRegression(C=5, max_iter=1000, class_weight=class_weight, solver='lbfgs')

scores = cross_val_score(model, X, y, cv=5, scoring='f1_weighted')
print(f"CV F1: {scores.mean():.4f} ± {scores.std():.4f}")

# -- Code Cell --
import pandas as pd
import numpy as np
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.utils.class_weight import compute_class_weight
from scipy.sparse import hstack

train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

def clean(text):
    text = str(text).lower()
    text = re.sub(r'\{@(.*?)@\}', r'\1', text)
    text = re.sub(r'\{\{USERNAME\}\}', ' username ', text)
    text = re.sub(r'\{\{URL\}\}', ' url ', text)
    text = re.sub(r'[^a-z0-9\s#]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text

train['clean'] = train['text'].apply(clean)
test['clean'] = test['text'].apply(clean)

tfidf_word = TfidfVectorizer(ngram_range=(1,2), max_features=50000, sublinear_tf=True, min_df=2)
tfidf_char = TfidfVectorizer(analyzer='char_wb', ngram_range=(3,5), max_features=30000, sublinear_tf=True, min_df=3)

X_word = tfidf_word.fit_transform(train['clean'])
X_char = tfidf_char.fit_transform(train['clean'])
X = hstack([X_word, X_char])

Xt_word = tfidf_word.transform(test['clean'])
Xt_char = tfidf_char.transform(test['clean'])
Xt = hstack([Xt_word, Xt_char])

y = train['label_name']
model = LogisticRegression(C=5, max_iter=1000, class_weight='balanced', solver='lbfgs')

scores = cross_val_score(model, X, y, cv=5, scoring='f1_weighted')
print(f"CV F1: {scores.mean():.4f} ± {scores.std():.4f}")

model.fit(X, y)
preds = model.predict(Xt)

out = pd.DataFrame({
    'subtaskID': 1,
    'datapointID': test['ID'],
    'answer': preds
})
out.to_csv('submission.csv', index=False)

# -- Code Cell --

