

import pandas as pd
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
train.head()
import re
import string
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer
nltk.download('punkt')
nltk.download('punkt_tab')
nltk.download('stopwords')
nltk.download('wordnet')

lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))
def preprocess_text(text):
    text = text.lower()
    text = re.sub(r'\d+', '', text)
    text = re.sub(r'[^\w\s]', '', text)
    text = re.sub(r'[^\x00-\x7F]+', '', text)
    tokens = nltk.word_tokenize(text)
    tokens = [w for w in tokens if w not in stop_words]
    lemmas = [lemmatizer.lemmatize(w) for w in tokens]
    return " ".join(lemmas)
train['text_clean'] = train['text'].apply(preprocess_text)
train.head()
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score
from imblearn.over_sampling import RandomOverSampler
from catboost import CatBoostClassifier

# 0) Encode labels (safe pentru CatBoost + metrici)
le = LabelEncoder()
y = le.fit_transform(train["label_name"])

# 1) Split pe text brut
X_train_text, X_test_text, y_train, y_test = train_test_split(
    train["text_clean"],
    y,
    test_size=0.2,
    random_state=42,
    stratify=y
)

# 2) TF-IDF (fit doar pe train)
tfidf = TfidfVectorizer(ngram_range=(1,2), min_df=2, max_df=0.95)
X_train = tfidf.fit_transform(X_train_text)
X_test  = tfidf.transform(X_test_text)

# 3) Oversampling DOAR pe train (pe TF-IDF)
ros = RandomOverSampler(random_state=42)
X_train_os, y_train_os = ros.fit_resample(X_train, y_train)

# 4) CatBoost (Multiclass) - de obicei vrea dense
X_train_os_dense = X_train_os.toarray()
X_test_dense     = X_test.toarray()

model_cb = CatBoostClassifier(
    loss_function="MultiClass",
    eval_metric="TotalF1",
    iterations=1000,
    learning_rate=0.1,
    depth=6,
    random_seed=42,
    verbose=200
)

model_cb.fit(
    X_train_os_dense, y_train_os,
    eval_set=(X_test_dense, y_test),
    use_best_model=True
)

y_pred = model_cb.predict(X_test_dense)
y_pred = np.array(y_pred).ravel().astype(int)

print("CatBoost F1 macro:", f1_score(y_test, y_pred, average="macro"))

train['label_name'].value_counts()
