import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

import re
import nltk

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans

from wordcloud import WordCloud

nltk.download('stopwords')
from nltk.corpus import stopwords
structured_data = pd.DataFrame({
    'PatientID': [1, 2, 3, 4, 5],
    'Age': [34, 45, 23, 50, 40],
    'Gender': ['M', 'F','F','M','M'],
    'Diagnosis': ['Diabetes', 'Hypertension', 'Asthma', 'Diabetes','Hypertension']
})
structured_data.head()
unstructured_data = [
    "Patient has a history of hypertension and diabetes. Prescribed medication X.",
    "Asthma diagnosis confirmed. Patient advised to use inhaler daily.",
    "Hypertension patient. Needs regular monitoring of blood pressure.",
    "Diabetes patient. Recommended diet and exercise.",
    "Patient diagnosed with hypertension. Medication Y prescribed."
]
structured_data.isnull().sum()
structured_data['Gender']=structured_data['Gender'].map({'M':0,'F':1})
structured_data.head()
structured_data=pd.get_dummies(structured_data,columns=['Diagnosis'])
structured_data.head()
def preprocess_text(text):
    text=text.lower()
    text=re.sub(r'\d+','',text)
    text=re.sub(r'\W+',' ',text)
    text=' '.join([word for word in text.split() if word not in stopwords.words('english')])
    return text
cleaned_notes=[preprocess_text(note) for note in unstructured_data]
vectorizer=TfidfVectorizer(max_features=10)
X_tfidf=vectorizer.fit_transform(cleaned_notes).toarray()

pd.DataFrame(X_tfidf,columns=vectorizer.get_feature_names_out()).head()
pca=PCA(n_components=2)
structured_data_pca=pca.fit_transform(structured_data.drop('PatientID',axis=1))
plt.scatter(structured_data_pca[:,0],structured_data_pca[:,1],c='blue',marker='o')
plt.title('PCA')
plt.xlabel('Principal component 1')
plt.ylabel('Principal component 2')
plt.show()
kmeans = KMeans(n_clusters=2, random_state=0).fit(X_tfidf)
for i in range(2):
    cluster_words=' '.join([cleaned_notes[j]for j in range(len(cleaned_notes)) if kmeans.labels_[j]==i])
    wordcloud=WordCloud(width=800,height=400,background_color='white').generate(cluster_words)
    plt.imshow(wordcloud,interpolation='bilinear')
    plt.axis('off')
    plt.title(f'Word cloud for cluster{i}')
    plt.show()
