# AUTOGENERATED! DO NOT EDIT! File to edit: 0_Short-Text-Analyzer_core.ipynb (unless otherwise specified).

__all__ = ['logger', 'shortTextAnalyzer', 'dataDetectLang', 'dataTranslateToEng', 'embeddingComments',
           'analyzePolarity', 'analyzeSubjectivity', 'clusterByHBDSCAN', 'clusterByKMeans']

# Cell
import os
import logging
import pandas as pd
import re
import matplotlib.pyplot as plt
import numpy as np

logger = logging.getLogger(__name__)

# Cell
class shortTextAnalyzer:

    def __init__(self, comments, num_topics):
        """
        Initialize the analyzer
        : param comments: (pd.Series) A Series of comment with at least 100 comments/elements.
        : param num_topics: (int) number of topics (clusters) that will be used via KMeans clustering
        : return: None
        """
        self.comments = comments
        self.numTopic = num_topics
        return None



    def validateInputs(self):
        """
        Validating the input
        : is it a Series?
        : is all element a string?
        : any empty string?
        """

        logger.info("Validating the inputs...")

        # is it a pd.Series object?
        if (type(self.comments) == pd.core.series.Series):
            for text in self.comments:

                # is it a string?
                if (type(text) != str):
                    raise ValueError('Incorrect input type: at least one element is not string')

                # is it an empty-string?
                if (len(text) <= 1):
                    raise ValueError('Incorrect input: at least one element is an empty string')

            return True

        else:
            raise ValueError('Incorrect parameter type: the input must be str or pandas Series')
            return True




    def preProcessInputs(self):
        """
        Preprocessing the input.
        : return: a dataframe of clean text
        """

        logger.info("Preprocessing the inputs...")

        if (self.validateInputs()):

            # Covert to pandas object
            comments_df = pd.DataFrame(self.comments)
            comments_df.columns = ['comments']

            # what is the language of the input?
            comments_df['comment_lang'] = comments_df.comments.map(lambda x: dataDetectLang(x))

            # translate to english
            translator = Translator()
            comments_df['comments_clean'] = comments_df.apply(lambda x: dataTranslateToEng(x['comments']) \
                                                            if x['comment_lang'] != 'en' \
                                                            else x['comments'], axis=1)


            # With BERT we don't process the texts;
            # otherwise, we lose the context (stemming, lemmatization)
            # or change the texts outright (stop words removal).

            self.embeddingRaw, self.embeddingForVisualize, self.embeddingForCluster = embeddingComments(comments_df.comments_clean)

            self.comments_df = comments_df
            return self.comments_df


    def analyze_getResult(self):
        """
        Perform the following analysis:
        * Sentiment Analysis (Polarity and subjectivity)
        * Assigning the cluster number to each row.

        return: a dataframe with the original text and the output from the analysis, including
            - Polarity score: (float) [-1,1] where 1 means positive statement and -1 means a negative statement.
            - Subjectivity score: (float) [0,1] where 1 means personal opinion and 0 means refer to factual information.
            -
        """

        logger.info('Analyzing the input...')
        if (self.validateInputs()):

            comments_df = self.preProcessInputs()

            comments_df['sentimentScore'] = comments_df.comments_clean.apply(lambda x: analyzePolarity(x))
            comments_df['subjectiveScore'] = comments_df.comments_clean.apply(lambda x: analyzeSubjectivity(x))

            comments_df['clusterByKMeans'] = clusterByKMeans(self.embeddingForCluster, self.numTopic)
            clusterByHDBSCAN, probHDBSCAN, paramHDBSCAN, allResultHDBSCAN = clusterByHBDSCAN(self.embeddingForCluster)
            comments_df['clusterByHDBSCAN'] = clusterByHDBSCAN


            self.paramHDBSCAN = paramHDBSCAN
            self.allResultHDBSCAN = allResultHDBSCAN
            self.comments_df = comments_df

            logger.info('Done...')
            print ("Number of clusters via HDBSCAN is: ", self.paramHDBSCAN.iloc[0].number_cluster)
            print ("Number of clusters via KMeans is:  ", self.numTopic)

            return self.comments_df


    def plot_output(self):
        """
        Visualize the output.
        """

        embedding2D = pd.DataFrame(self.embeddingForVisualize)
        comments_df = self.comments_df

        ### Plotting KMeans
        plt.figure(figsize=(8, 8))
        plt.scatter(embedding2D[0], embedding2D[1], c=comments_df['clusterByKMeans'])
        plt.title("2D Projection of the data coloured by cluster from KMeans")

        ### Plotting HDBSCAN
        plt.figure(figsize=(8, 8))
        plt.scatter(embedding2D[0], embedding2D[1],  c=comments_df['clusterByHDBSCAN'])
        plt.title("2D Projection of the data coloured by cluster from HDBSCAN")

# Cell
from langdetect import detect

def dataDetectLang(text):
    '''
    Determine the language of text.
    : param text: (str) input text
    '''
    try:
        return detect(text)
    except Exception as ex:
        print (ex)
        print ("error: ", text)
        return "N/A"

# Cell
from googletrans import Translator

def dataTranslateToEng(text):
    '''
    Translate the non-English to English.
    **To do: config the connection (error with `googletrans`) . See https://github.com/ssut/py-googletrans/issues/234
    '''
    try:
        translator = Translator()
        translated=translator.translate(text)
        return translated.text
    except Exception as ex:
        print (ex)
        return text
#     return text

# Cell
from sentence_transformers import SentenceTransformer
import umap.umap_ as umap

def embeddingComments(comments):
    """
    Getting a vector representation of the comments via BERT
    : return
        - embeddingRaw :
        - embeddingForVisualize :
        - embeddingForCluster :
    """
    model = SentenceTransformer('distilbert-base-nli-mean-tokens')
    embeddingRaw = model.encode(comments.tolist(), show_progress_bar=True)


    ### Dimension Reduction with UMAP
    # The embedding out has 768 dimensions. We will use UMAP to reduce the number of dimension.
    # see UMAP's parameter for clustering: https://umap-learn.readthedocs.io/en/latest/clustering.html#traditional-clustering

    n_neighbors = 100  # Need large n_neighbors to capture more global view
    min_dist = 0    # how tightly points are packed together? => 0 since we want it to pack densely because we will use the density for clustering

    # Embedding for the visualization pupose, i.e. 2-dimensions data
    embeddingForVisualize = umap.UMAP(n_neighbors=n_neighbors
                                   , n_components=2
                                   , min_dist=min_dist
                                   , metric='cosine').fit_transform(embeddingRaw)

    # For the clustering purpose, we allow a few more dimensions so that it doesn't compress too much (i.e. 5-dimensions is reasonable for HDBSCAN or KNN)
    embeddingForCluster = umap.UMAP(n_neighbors=n_neighbors
                                   , n_components=5
                                   , min_dist=min_dist).fit_transform(embeddingRaw)

    return embeddingRaw, embeddingForVisualize, embeddingForCluster

# Cell
from textblob import TextBlob

def analyzePolarity(text):
    """
    Polarity score ranges from [-1,1] where 1 means positive statement and -1 means a negative statement.
    """
    testimonial = TextBlob(text)
    return testimonial.sentiment.polarity


def analyzeSubjectivity(text):
    """
    Subjective score ranges from [0,1] where 1 refer to personal opinion, emotion or judgment and 0 means it is factual information.
    """
    testimonial = TextBlob(text)
    return testimonial.sentiment.subjectivity

# Cell
import hdbscan
from sklearn.cluster import KMeans

def clusterByHBDSCAN(X_vectors):

    score_df = pd.DataFrame()  # score for each parameter
    use_range = [3, 5, 10, 20, 30, 50, 75, 100, 150, 200, 300, 500, 800, 1000]
    result_df = pd.DataFrame(X_vectors)
    for i in use_range:

        clusterer = hdbscan.HDBSCAN(min_cluster_size= i , gen_min_span_tree=True).fit(X_vectors)

        number_outlier = len(np.where( clusterer.labels_ == -1 )[0])  /   len( clusterer.labels_ )
        number_cluster   = len(pd.Series(clusterer.labels_).unique())
        tempdf = pd.DataFrame({'min_cluster_size': i
                               , 'number_outlier': round(number_outlier,2)
                               , 'number_cluster': number_cluster}, index = [i])

        score_df = pd.concat([score_df, tempdf])
        result_df['min_cluster_size'+str(i)] = clusterer.labels_

    ### Build the best clustering model
    # Select the min_cluster_size with the lowest number of outlier, and the highest number of cluster
    score_df.sort_values(['number_outlier', 'number_cluster'], ascending=[True, False], inplace=True)
    best_min_cluster_size = int(score_df.iloc[0].min_cluster_size)

    # Final HDBSCAN model
    hdbscan_final = hdbscan.HDBSCAN(min_cluster_size= best_min_cluster_size
                                    , gen_min_span_tree=True).fit(X_vectors)

    return hdbscan_final.labels_, hdbscan_final.probabilities_, score_df, result_df


def clusterByKMeans(X_vectors, n_clusters):

    kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(X_vectors)
    return kmeans.labels_
