import pandas as pd
#import ssl
import matplotlib.pyplot as plt

'''
Exercise 2
    2) Create a Python package (installable with pip) that downloads the data from a public
    database and makes some statistics with that data. The type of statistics applied is left to your
    choice. Using pandas, matplotlib and numpy to deal with the data will be considered a plus.
    As an example, you can use one of these public databases or any other of your preference:
    a) https://data.nasa.gov/Applied-Science/Outgassing-Db/r588-f7pr
    b) https://data.nasa.gov/Space-Science/Fireball-And-Bolide-Reports/mc52-syum
'''
'''
A quick review of the selected data set:
    https://data.nasa.gov/Applied-Science/Outgassing-Db/r588-f7pr
    This tenth compilation of outgassing data of materials intended for spacecraft use supersedes 
    Reference Publication 1124, Revision 3, September 1993. 
    The data were obtained at the Goddard Space Flight Center (GSFC), 
    utilizing equipment developed at Stanford Research Institute (SRI) under contract 
    to the Jet Propulsion Laboratory (JPL).
    The data set contains 
        sample_material:  name of each material
        id: material identifier
        mfr: manufacturer of the material
        tml:  total mass loss
        category: *
        cvcm: collected volatile condensable material
        space_code: * 
        wvr: water vapor regained
        material_usage: uses (adhesive, greases, paints, etc).
'''
'''
Abstract of the procedure
    Using pandas, the code applies to the dataset the split-apply-combine strategy for data analysis.
        split: split the data into groups. groupby om tje original pandas dataframe;
        apply: apply a function that computes any statistics
        combine: combine results to create a new DataFrame  
'''


def f_getdataframefromwebsite(str_website):
    """
        f_getdataframefromwebsite: Downlaod a json file from internet and creates the pandas DataFrame

        :param str_website: string to json file in internet
        :return: DataFrame
    """
    # Need to do this in order to download the file without SSL problems
    #ssl._create_default_https_context = ssl._create_unverified_context
    df_complete = pd.read_json(str_website)
    # return the complete DataFrame
    return df_complete


def f_filter(df_complete):
    """
        f_filter: Deletes the rows with NaN values in the DataFrame and also the columns without usefull information
        for this example

        :param df_complete: pandas DataFrame to be filtered
        :return: pandas DataFrame filtered
    """
    # simple sanitize database. delete registers with NaN
    df_filtered = df_complete.dropna()
    # drop columns which have no meaning in this analysis. in this case id, sample_material, category, space_code and
    df_filtered = df_filtered.drop(columns=['id', 'sample_material', 'category', 'space_code', 'cure'])

    return df_filtered


def f_split(df_filtered):
    """
         f_split: Generates different groups inside the dataset according the material usage. Shows also some
         information as: number of groups, name of the first groups created and autogenerated statistics

         :param df_complete: pandas DataFrame to be filtered
         :return: pandas DataFrameGroupBy
    """
    dfgb_splitted_by_material = df_filtered.groupby('material_usage')

    # Number of material types
    print("There are ", len(dfgb_splitted_by_material), " different material's types in the downloaded dataset")

    # Print first elements, by default the first 5 elements
    #print(dfgb_splitted_by_material.head())

    # Here are some statistics automatically made by pandas groupby function.
    # count, mean, min, std, 25%, 50%, 75%, max. Useful to compare with the result.
    #print(dfgb_splitted_by_material.describe())

    return dfgb_splitted_by_material


def f_apply_count(df_filtered):
    """
         f_apply_count: Counts the number of elements of each material_usage of the DataFrame

         :param df_filtered: pandas DataFrame filtered
         :return: pandas DataFrameGroupBy with count column
    """
    dfgb_material_count = df_filtered.groupby('material_usage').count()
    dfgb_material_count = dfgb_material_count.drop(columns=['tml', 'cvcm', 'wvr'])
    dfgb_material_count.rename(columns={'mfr': 'count'},
                               inplace=True)

    return dfgb_material_count


def f_apply_mean(dfgb_splitted_by_material):
    """
         f_apply_mean: Calculate the mean value for each meaningful numeric value of the DataFrameGroupBy

         :param dfgb_splitted_by_material: pandas DataFrameGroupBy to obtain the mean value
         :return: pandas DataFrame with mean values
    """
    df_material_statistics_mean = dfgb_splitted_by_material.mean()
    df_material_statistics_mean.rename(columns={'tml': 'tml_mean', 'cvcm': 'cvcm_mean', 'wvr': 'wvr_mean'},
                                         inplace=True)
    return df_material_statistics_mean


def f_apply_min(dfgb_splitted_by_material):
    """
         f_apply_min: Calculate the min value for each meaningful numeric value of the DataFrameGroupBy

         :param dfgb_splitted_by_material: pandas DataFrameGroupBy to obtain the minimum value
         :return: pandas DataFrame with min values
    """
    df_material_statistics_min = dfgb_splitted_by_material.min()
    df_material_statistics_min.rename(columns={'tml': 'tml_min', 'cvcm': 'cvcm_min', 'wvr': 'wvr_min'},
                                      inplace=True)
    return df_material_statistics_min


def f_apply_max(dfgb_splitted_by_material):
    """
         f_apply_max: Calculate the max value for each meaningful numeric value of the DataFrameGroupBy

         :param dfgb_splitted_by_material: pandas DataFrameGroupBy to obtain the maximum value
         :return: pandas DataFrame with min values
    """
    df_material_statistics_max = dfgb_splitted_by_material.max()
    df_material_statistics_max.rename(columns={'tml': 'tml_max', 'cvcm': 'cvcm_max', 'wvr': 'wvr_max'},
                                      inplace=True)
    return df_material_statistics_max


def f_combine(df_material_mean, df_material_min, df_material_max, df_material_count):
    """
         f_combine: Combines the useful information from several DataFrameGroupBy calculated previously.
         This information is: mean, min, max and count (quantity)

         :param df_material_mean: pandas DataFrameGroupBy with mean values
                df_material_min: pandas DataFrameGroupBy with min values
                df_material_max: pandas DataFrameGroupBy with max values
                df_material_count: pandas DataFrameGroupBy with count values
         :return: pandas DataFrame with all data joined.
    """
    # Extract columns needed
    extracted_cols_min = df_material_min[["tml_min", "cvcm_min", "wvr_min"]]
    # merge with previous DataFrame
    df_material_statistics = df_material_mean.join(extracted_cols_min)

    # Extract columns needed
    extracted_cols_max = df_material_max[["tml_max", "cvcm_max", "wvr_max"]]
    # merge with previous DataFrame
    df_material_statistics = df_material_statistics.join(extracted_cols_max)

    # Extract columns needed
    extracted_cols_count = df_material_count["count"]
    # merge with previous DataFrame
    df_material_statistics = df_material_statistics.join(extracted_cols_count)

    return df_material_statistics


def f_printtablewithstatistics(df_material_statistics):
    """
         f_printtablewithstatistics: order by count and print in console the first results of
         the table created by combine method

         :param df_material_statistics: pandas DataFrame with statistical values
         :return: nothing.
    """
    # Add sort to get the material_usages with more information
    df_material_statistics = df_material_statistics.sort_values(by=['count'], ascending=False)
    # Print the first 5 rows
    print(df_material_statistics.head())
    # Extract columns needed to check vs plot
    df_tml_statistics = df_material_statistics[["tml_mean", "tml_min", "tml_max", "count"]]
    print(df_tml_statistics.head())



def f_plotexampledata(df_material_statistics, df_filtered):
    """
         f_plotexampledata: plot the first two results of the analysis for one column.
         In this case is tml (total mass loss)

         :param df_material_statistics: pandas DataFrame with statistical values
                df_filtered: pandas DataFrame with raw values
         :return: nothing.
    """
    #Reduce only to show a few values
    df_material_statistics = df_material_statistics.sort_values(by=['count'], ascending=False)
    df_material_statistics_5 = df_material_statistics.iloc[:5].reset_index()
    # Get the names of the first materials
    str_plottitle_1 = df_material_statistics_5.iloc[0]['material_usage']
    str_plottitle_2 = df_material_statistics_5.iloc[1]['material_usage']
    # Apply simple filter to get the raw values for that material
    df_first_for_plot = df_filtered[df_filtered["material_usage"] == str_plottitle_1]
    df_second_for_plot = df_filtered[df_filtered["material_usage"] == str_plottitle_2]

    # Plot using the two new dataframes
    # First subplot
    plt.figure(1)
    plt.subplot(1, 2, 1)
    ax0 = df_first_for_plot.boxplot(column=['tml'])  # no need to specify for first axis
    ax0.set_title(str_plottitle_1)
    ax0.set_xlabel("total mass loss")
    # Second subplot
    plt.subplot(1, 2, 2)
    ax1 = df_second_for_plot.boxplot(column=['tml'])
    ax1.set_title(str_plottitle_2)
    ax1.set_xlabel("total mass loss")
    plt.savefig("tml_boxplot.png")


#if __name__ == '__main__':
def getdata_tml():
    """
         main: Dowload a fixed json from the website and applies the method split-apply-combine to get some
         information regarding the statistics inside the database
     """
    # Get data
    df_complete = f_getdataframefromwebsite("https://data.nasa.gov/resource/r588-f7pr.json")
    # Filter dataframe
    df_filtered = f_filter(df_complete)

    # Split DataFrame according the materials
    dfgb_splitted_by_material = f_split(df_filtered)

    # Apply some functions to the dataset
    df_material_count = f_apply_count(df_filtered)
    df_material_mean = f_apply_mean(dfgb_splitted_by_material)
    df_material_min = f_apply_min(dfgb_splitted_by_material)
    df_material_max = f_apply_max(dfgb_splitted_by_material)

    # Combine the results into a new DataFrame
    df_material_statistics = f_combine(df_material_mean, df_material_min, df_material_max, df_material_count)

    # Print in console some results in table format
    f_printtablewithstatistics(df_material_statistics)

    # Plot using pandas and matplotlib
    f_plotexampledata(df_material_statistics, df_filtered)


