# -- Code Cell --
import pandas as pd
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

# -- Code Cell --
train.head()

# -- Code Cell --


# -- Code Cell --
train['MSZoning'].value_counts()

# -- Code Cell --
train.info()

# -- Code Cell --
cat_cols = train.select_dtypes(include=object).columns
num_cols =[c for c in train if c not in cat_cols]
cat_cols = [c for c in cat_cols if c!='text']
num_cols =[c for c in num_cols if c!='SalePrice']

# -- Code Cell --
cat_cols

# -- Code Cell --
train['text'][98]

# -- Code Cell --
train['SalePrice']

# -- Code Cell --
from sklearn.impute import SimpleImputer
cat_imp = SimpleImputer(strategy='most_frequent')
num_imp = SimpleImputer(strategy='median')
train[cat_cols] = cat_imp.fit_transform(train[cat_cols])
test[cat_cols]=cat_imp.transform(test[cat_cols])
train[num_cols] = num_imp.fit_transform(train[num_cols])
test[num_cols] = num_imp.transform(test[num_cols])

# -- Code Cell --
test.info()

# -- Code Cell --
!pip install kmodes

# -- Code Cell --
X2 = train.drop(columns=['SalePrice', "text"])
cat_indices = [X2.columns.get_loc(col) for col in cat_cols]

# -- Code Cell --
X2.columns

# -- Code Cell --
test.columns

# -- Code Cell --
# from kmodes.kprototypes import KPrototypes
# kproto = KPrototypes(n_clusters=6, random_state=42)

# clusters = kproto.fit_predict(X2,categorical=cat_indices)

# -- Code Cell --
# test_for_cluster= test.drop(columns=['text','ID'])

# -- Code Cell --
# clusters_test = kproto.predict(test_for_cluster, categorical=cat_indices)

# -- Code Cell --
# clusters_test

# -- Code Cell --
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
tfidf = TfidfVectorizer()
X_text = tfidf.fit_transform(train['text'])
test_mod = tfidf.transform(test['text'])
kmeans = KMeans(n_clusters=5, random_state=42)
train['cluster_label'] = kmeans.fit_predict(X_text)
test['cluster_label'] = kmeans.predict(test_mod)

# -- Code Cell --
import matplotlib.pyplot as plt
feature = "SalePrice"   
plt.figure()
plt.hist(train[feature], bins=30)
plt.show()

# -- Code Cell --
import numpy as np
train["SalePrice_log"] = np.log1p(train["SalePrice"])

# -- Code Cell --
from catboost import CatBoostRegressor
from sklearn.metrics import r2_score
from sklearn.model_selection import train_test_split
train["text"] = train["text"].astype(str).str.lower().str.replace("\n", " ", regex=False)

X = train.drop(columns=['SalePrice','SalePrice_log'])
y= train['SalePrice_log']

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=42)
model = CatBoostRegressor(random_state=42,cat_features=cat_cols,text_features=["text"])
model.fit(X_train, y_train)
pred = model.predict(X_test)
# pred_real = np.expm1(pred)
acc = r2_score(y_test,pred)
acc

# -- Code Cell --
model.fit(X,y)
pred_final = model.predict(test)

# -- Code Cell --
pred_final_fmm = np.expm1(pred_final)

# -- Code Cell --
rows =[]
for idx,row in test.iterrows():
    rows.append({'subtaskID':1, 'datapointID':row['ID'],'answer':row['cluster_label']})
    rows.append({'subtaskID':2, 'datapointID':row['ID'],'answer':pred_final_fmm[idx]})
pd.DataFrame(rows).to_csv('Submission.csv',index=False)