# AUTOGENERATED! DO NOT EDIT! File to edit: 00_core.ipynb (unless otherwise specified).

__all__ = ['load_public_data', 'prepare_baseline_and_intervention_usable_data', 'get_adherent_column',
           'most_active_user', 'convert_loggings', 'get_certain_types', 'breakfast_analysis_summary',
           'breakfast_analysis_variability', 'dinner_analysis_summary', 'dinner_analysis_variability',
           'filtering_usable_data']

# Cell
from .food_parser import *
import warnings
warnings.filterwarnings('ignore')
import pandas as pd
import numpy as np
from scipy import stats
import seaborn as sns
import os
import matplotlib.pyplot as plt
import pickle
from datetime import date
from datetime import datetime
from collections import defaultdict

# Cell
def load_public_data(in_path):
    """
    Description:\n
        Load original public data and output processed data in pickle format.\n

        Process includes:\n
        1. Dropping 'foodimage_file_name' column.\n
        2. Handling the format of time by generating a new column, 'original_logtime_notz'\n
        3. Generating the date column, 'date'\n
        4. Converting time into float number into a new column, 'local_time'\n
        5. Converting time in the 'local_time' column so that day starts at 4 am.\n
        6. Converting time to a format of HH:MM:SS, 'time'\n
        7. Generating the column 'week_from_start' that contains the week number that the participants input the food item.\n
        8. Generating 'year' column based on the input data.\n
        9. Outputing the data into a pickle format file.\n

    Input:\n
        - in_path : input path, csv file\n

    Output:\n
        - public_all: the processed dataframe\n

    Requirements:\n
        csv file must have the following columns:\n
            - foodimage_file_name\n
            - original_logtime\n
            - date\n
            - unique_code\n
    """
    public_all = universal_key(in_path).drop(columns = ['foodimage_file_name'])

    def handle_public_time(s):
        tmp_s = s.replace('p.m.', '').replace('a.m.', '')
        try:
            return pd.to_datetime(' '.join(tmp_s.split()[:2]) )
        except:
            try:
                if int(tmp_s.split()[1][:2]) > 12:
                    tmp_s = s.replace('p.m.', '').replace('a.m.', '').replace('PM', '').replace('pm', '')
                return pd.to_datetime(' '.join(tmp_s.split()[:2]) )
            except:
                return np.nan

    original_logtime_notz_lst = []
    for t in (public_all.original_logtime.values):
        original_logtime_notz_lst.append(handle_public_time(t))
    public_all['original_logtime_notz'] = original_logtime_notz_lst

    public_all = public_all.dropna().reset_index(drop = True)

    def find_date(d):
        if d.hour < 4:
            return d.date() - pd.Timedelta('1 day')
        else:
            return d.date()
    public_all['date'] = public_all['original_logtime_notz'].apply(find_date)


    # Handle the time - Time in floating point format
    public_all['local_time'] = public_all.original_logtime_notz.apply(lambda x: pd.Timedelta(x.time().isoformat()).total_seconds() /3600.).values
    day_begins_at = 4
    public_all.loc[(public_all['local_time'] < day_begins_at), 'local_time'] = 24.0 + public_all.loc[(public_all['local_time'] < day_begins_at), 'local_time']

    # Handle the time - Time in Datetime object format
    public_all['time'] = pd.DatetimeIndex(public_all.original_logtime_notz).time

    # Handle week from start
    public_start_time_dic = dict(public_all.groupby('unique_code').agg(np.min)['date'])
    def count_week_public(s):
        return (s.date - public_start_time_dic[s.unique_code]).days // 7 + 1
    public_all['week_from_start'] = public_all.apply(count_week_public, axis = 1)

    public_all['year'] = public_all.date.apply(lambda d: d.year)

    return public_all

# Cell
def prepare_baseline_and_intervention_usable_data(in_path):
    """
    Description:\n
        Filter and create baseline_expanded and intervention groups based on in_path pickle file.\n

    Input:\n
        - in_path : input path, file in pickle, csv or pandas dataframe format\n

    Return:\n
        - baseline expanded and intervention dataframes in a list format where index 0 is the baseline dataframe and 1 is the intervention dataframe.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - week_from_start\n
            - unique_code\n
            - desc_text\n
            - date\n

    """


    public_all = universal_key(in_path)

    # create baseline data
    df_public_baseline = public_all.query('week_from_start <= 2')
    df_public_baseline_usable, public_baseline_usable_id_set = \
    filtering_usable_data(df_public_baseline, num_items = 40, num_days = 12)

    # create intervention data
    df_public_intervention = public_all.query('week_from_start in [13, 14]')
    df_public_intervention_usable, public_intervention_usable_id_set = \
    filtering_usable_data(df_public_intervention, num_items = 20, num_days = 8)

    # create df that contains both baseline and intervention id_set that contains data for the first two weeks
    expanded_baseline_usable_id_set = set(list(public_baseline_usable_id_set) + list(public_intervention_usable_id_set))
    df_public_basline_usable_expanded = public_all.loc[public_all.apply(lambda s: s.week_from_start <= 2 \
                                                    and s.unique_code in expanded_baseline_usable_id_set, axis = 1)]

    return [df_public_basline_usable_expanded, df_public_intervention_usable]

# Cell
def get_adherent_column(in_path):
    """
    Description:\n
        A logging is true if the there are more than 2 loggings in one day w/ more than 4hrs apart from the earliest logging and the latest logging and False otherwise.\n

    Input:\n
        - in_path : input path, file in pickle, csv or pandas dataframe format.\n

    Return:\n
        - A dataframe with a boolean column called adherence. Details on the criteria is in the description.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - unique_code\n
            - local_time\n
            - date\n

    """
    def adherent(s):
        if len(s.values) >= 2 and (max(s.values) - min(s.values)) >= 4:
            return True
        else:
            return False

    df = universal_key(in_path)

    adherent_dict = dict(df.groupby(['unique_code', 'date'])['local_time'].agg(adherent))
    df['adherence'] = df.apply(lambda x: adherent_dict[(x.unique_code, x.date)], axis = 1)

    return df

# Cell
def most_active_user(in_path, food_type = ["f", "b", "m", "w"]):
    """
    Description:\n
        This function returns a dataframe reports the number of adherent logging days for each user in the in_path file. The default order is descending.\n

    Input:\n
        - in_path : input path, file in pickle, csv or pandas dataframe format.\n
        - food_type : food types to filter in a list format. Default: ["f", "b", "m", "w"]. Available food types:\n
            1. 'w' : water \n
            2. 'b' : beverage \n
            3. 'f' : food \n
            4. 'm' : medicine \n

    Return:\n
        - A dataframe contains the number of adherent logging days for each user.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - food_type\n
            - unique_code\n
            - date\n

    """

    public_all = universal_key(in_path)

    # filter the dataframe so it only contains input food type

    filtered_users = public_all.query('food_type in @food_type')

    filtered_users_w_adherence = get_adherent_column(filtered_users)

    public_top_users_day_counts = pd.DataFrame(filtered_users_w_adherence.query('adherence == True')\
                            [['date', 'unique_code']].groupby('unique_code')['date'].nunique())\
                            .sort_values(by = 'date', ascending = False).rename(columns = {'date': 'day_count'})


    return public_top_users_day_counts

# Cell
def convert_loggings(in_path):
    """
    Description:\n
       This function convert all the loggings in the in_path file into a list of individual items based on the desc_text column.\n
    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n

    Return:\n
        - A dataframe contains the cleaned version of the desc_text.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - food_type\n
            - desc_text\n

    """

    public_all = universal_key(in_path)

    # initialize food parser instance
    fp = FoodParser()
    fp.initialization()

    # parse food
    parsed = [fp.parse_food(i, return_sentence_tag = True) for i in public_all.desc_text.values]

    public_all_parsed = pd.DataFrame({
    'ID': public_all.unique_code,
    'food_type': public_all.food_type,
    'desc_text': public_all.desc_text,
    'cleaned': parsed
    })

    public_all_parsed['cleaned'] = public_all_parsed['cleaned'].apply(lambda x: x[0])


    return public_all_parsed

# Cell
def get_certain_types(in_path, food_type):
    """
    Description:\n
       This function filters with the expected food types and return a cleaner version of in_path file.\n
    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n
        - food_type : expected types of the loggings for filtering, in format of list. Available types:  \n
            1. 'w' : water \n
            2. 'b' : beverage \n
            3. 'f' : food \n
            4. 'm' : medicine \n

    Return:\n
        - A filtered dataframe with expected food type/types with five columns: 'unique_code','food_type', 'desc_text', 'date', 'local_time'.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - food_type\n
            - desc_text\n
            - unique_code\n
            - desc_text\n
            - date\n
            - local_time\n

    """

    df = universal_key(in_path)

    if len(food_type) == 0:
        return df[['unique_code','food_type', 'desc_text', 'date', 'local_time']]

    if len(food_type) == 1:
        if food_type[0] not in ['w', 'b', 'f', 'm']:
            raise Exception("not a valid logging type")
        filtered = df[df['food_type']==food_type[0]][['unique_code','food_type', 'desc_text', 'date', 'local_time']].reset_index(drop = True)
    else:
        filtered = df[df['food_type'].isin(food_type)][['unique_code','food_type', 'desc_text', 'date', 'local_time']].reset_index(drop = True)


    return filtered


# Cell
def breakfast_analysis_summary(in_path):
    """
    Description:\n
       This function takes the adherent loggings and calculate the 5%,10%,25%,50%,75%,90%,95% quantile of breakfast time for each user.\n

    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n

    Return:\n
        - A summary table with 5%,10%,25%,50%,75%,90%,95% quantile of breakfast time for all subjects from the in_path file.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - unique_code\n
            - date\n
            - local_time\n

    """

    df = universal_key(in_path)

    # leave only the adherent loggings
    df = get_adherent_column(df)
    df = df[df['adherence']==True]

    breakfast_series = df.groupby(['unique_code', 'date'])['local_time'].min().groupby('unique_code').quantile([0.05, 0.10, 0.25, 0.5, 0.75, 0.90, 0.95])
    breakfast_df = pd.DataFrame(breakfast_series)
    all_rows = []
    for index in breakfast_df.index:
        tmp_dict = dict(breakfast_series[index[0]])
        tmp_dict['id'] = index[0]
        all_rows.append(tmp_dict)
    breakfast_summary_df = pd.DataFrame(all_rows, columns = ['id', 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95])\
        .rename(columns = {0.05: '5%', 0.1: '10%', 0.25: '25%', 0.5: '50%', 0.75: '75%', 0.9: '90%', 0.95: '95%'})\
        .drop_duplicates().reset_index(drop = True)

    return breakfast_summary_df

# Cell
def breakfast_analysis_variability(in_path):
    """
    Description:\n
       This function calculates the variability by subtracting 5%,10%,25%,50%,75%,90%,95% quantile of breakfast time from the 50% breakfast time. It also make a histogram that represents the 90%-10% interval for all subjects.\n

    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n

    Return:\n
        - A dataframe that contains 5%,10%,25%,50%,75%,90%,95% quantile of breakfast time minus 50% time for each subjects from the in_path file.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - unique_code\n
            - date\n
            - local_time\n
    """


    df = universal_key(in_path)

    # leave only the adherent loggings
    df = get_adherent_column(df)
    df = df[df['adherence']==True]

    breakfast_series = df.groupby(['unique_code', 'date'])['local_time'].min().groupby('unique_code').quantile([0.05, 0.10, 0.25, 0.5, 0.75, 0.90, 0.95])
    breakfast_df = pd.DataFrame(breakfast_series)
    all_rows = []
    for index in breakfast_df.index:
        tmp_dict = dict(breakfast_series[index[0]])
        tmp_dict['id'] = index[0]
        all_rows.append(tmp_dict)
    breakfast_summary_df = pd.DataFrame(all_rows, columns = ['id', 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95])\
        .rename(columns = {0.05: '5%', 0.1: '10%', 0.25: '25%', 0.5: '50%', 0.75: '75%', 0.9: '90%', 0.95: '95%'})\
        .drop_duplicates().reset_index(drop = True)
    breakfast_variability_df = breakfast_summary_df.copy()

    for col in breakfast_variability_df.columns:
        if col == 'id' or col == '50%':
            continue
        breakfast_variability_df[col] = breakfast_variability_df[col] - breakfast_variability_df['50%']
    breakfast_variability_df['50%'] = breakfast_variability_df['50%'] - breakfast_variability_df['50%']

    fig, ax = plt.subplots(1, 1, figsize = (10, 10), dpi=80)
    sns_plot = sns.distplot( breakfast_variability_df['90%'] - breakfast_variability_df['10%'] )
    ax.set(xlabel='Variation Distribution for Breakfast (90% - 10%)', ylabel='Kernel Density Estimation')

    return breakfast_variability_df

# Cell
def dinner_analysis_summary(in_path, out_path=None, export = False):
    """
    Description:\n
       This function takes the adherent loggings and calculate the 5%,10%,25%,50%,75%,90%,95% quantile of dinner time for each user.\n

    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n

    Return:\n
        - A summary table with 5%,10%,25%,50%,75%,90%,95% quantile of dinner time for all subjects from the in_path file.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - unique_code\n
            - date\n
            - local_time\n
    """

    df = universal_key(in_path)

    # leave only the adherent loggings
    df = get_adherent_column(df)
    df = df[df['adherence']==True]

    dinner_series = df.groupby(['unique_code', 'date'])['local_time'].max().groupby('unique_code').quantile([0.05, 0.10, 0.25, 0.5, 0.75, 0.90, 0.95])
    dinner_df = pd.DataFrame(dinner_series)
    all_rows = []
    for index in dinner_df.index:
        tmp_dict = dict(dinner_series[index[0]])
        tmp_dict['id'] = index[0]
        all_rows.append(tmp_dict)
    dinner_summary_df = pd.DataFrame(all_rows, columns = ['id', 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95])\
        .rename(columns = {0.05: '5%', 0.1: '10%', 0.25: '25%', 0.5: '50%', 0.75: '75%', 0.9: '90%', 0.95: '95%'})\
        .drop_duplicates().reset_index(drop = True)


    return dinner_summary_df

# Cell
def dinner_analysis_variability(in_path, out_path=None, export = False):
    """
     Description:\n
       This function calculates the variability by subtracting 5%,10%,25%,50%,75%,90%,95% quantile of dinner time from the 50% dinner time. It also make a histogram that represents the 90%-10% interval for all subjects.\n

    Input:\n
        - in_path : input path, file in pickle, csv or panda dataframe format.\n

    Return:\n
        - A dataframe that contains 5%,10%,25%,50%,75%,90%,95% quantile of dinner time minus 50% time for each subjects from the in_path file.\n

    Requirements:\n
        in_path file must have the following columns:\n
            - unique_code\n
            - date\n
            - local_time\n
    """

    df = universal_key(in_path)

    # leave only the adherent loggings
    df = get_adherent_column(df)
    df = df[df['adherence']==True]

    dinner_series = df.groupby(['unique_code', 'date'])['local_time'].max().groupby('unique_code').quantile([0.05, 0.10, 0.25, 0.5, 0.75, 0.90, 0.95])
    dinner_df = pd.DataFrame(dinner_series)
    all_rows = []
    for index in dinner_df.index:
        tmp_dict = dict(dinner_series[index[0]])
        tmp_dict['id'] = index[0]
        all_rows.append(tmp_dict)
    dinner_summary_df = pd.DataFrame(all_rows, columns = ['id', 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95])\
        .rename(columns = {0.05: '5%', 0.1: '10%', 0.25: '25%', 0.5: '50%', 0.75: '75%', 0.9: '90%', 0.95: '95%'})\
        .drop_duplicates().reset_index(drop = True)
    dinner_variability_df = dinner_summary_df.copy()

    for col in dinner_variability_df.columns:
        if col == 'id' or col == '50%':
            continue
        dinner_variability_df[col] = dinner_variability_df[col] - dinner_variability_df['50%']
    dinner_variability_df['50%'] = dinner_variability_df['50%'] - dinner_variability_df['50%']

    fig, ax = plt.subplots(1, 1, figsize = (10, 10), dpi=80)
    sns_plot = sns.distplot( dinner_variability_df['90%'] - dinner_variability_df['10%'] )
    ax.set(xlabel='Variation Distribution for Dinner (90% - 10%)', ylabel='Kernel Density Estimation')

    return dinner_variability_df

# Cell
def filtering_usable_data(df, num_items, num_days):
    '''
    Description:\n
        This function filters the cleaned app data given certain criteria\n
    Input:\n
        - df (pd.DataFrame): the dataframe to be filtered\n
        - num_items (int):   number of items to be used as cut-off\n
        - num_days (int):    number of days to be used as cut-off\n
    Output:\n
        - df_usable:         a panda DataFrame with filtered rows\n
        - set_usable:        a set of unique_code to be included as "usable"\n
    Side Effects:\n
        None\n
    Requirements:\n
        df should have the following columns:\n
            - unique_code\n
            - desc_text\n
            - date\n
    Used in:\n
        Analysis pipeline\n
    '''
    print(' => filtering_usable_data()')
    print('  => using the following criteria:', num_items, 'items and', num_days, 'days logged in two weeks.')

    # Item logged
    log_item_count = df.groupby('unique_code').agg('count')[['desc_text']].rename(columns = {'desc_text': 'Total Logged'})

    # Day counts
    log_days_count = df[['unique_code', 'date']]\
        .drop_duplicates().groupby('unique_code').agg('count').rename(columns = {'date': 'Day Count'})

    item_count_passed = set(log_item_count[log_item_count['Total Logged'] >= num_items].index)
    day_count_passed = set(log_days_count[log_days_count['Day Count'] >= num_days].index)

    print('  => # of public users pass the criteria:', end = ' ')
    print(len(item_count_passed & day_count_passed))
    passed_participant_set = item_count_passed & day_count_passed
    df_usable = df.loc[df.unique_code.apply(lambda c: c in passed_participant_set)]\
        .dropna().copy().reset_index(drop = True)
    # print('  => Now returning the pd.DataFrame object with the head like the following.')
    # display(df_usable.head(5))
    return df_usable, set(df_usable.unique_code.unique())