{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp rl.policies.bandit_policies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bandit Policies\n",
    "> Implementation of bandit policies including UCB and TS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import norm, beta\n",
    "from abc import ABC, abstractmethod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ABPolicy(ABC):\n",
    "    def __init__(self, bandit, slate_size, scores_logging):\n",
    "        self.name = None\n",
    "        self.slate_size = slate_size\n",
    "        self.history = pd.DataFrame(data=None, columns=['item_id', 'reward'])\n",
    "        if scores_logging is True:\n",
    "            self.scores_log = pd.DataFrame(data=None, columns=bandit.actions)\n",
    "        else:\n",
    "            self.scores_log = None\n",
    "\n",
    "    @abstractmethod\n",
    "    def get_recommendations(self):\n",
    "        ...\n",
    "\n",
    "    def update(self, rewards):\n",
    "        # append new events to history\n",
    "        self.history = self.history.append(rewards, ignore_index=True)\n",
    "    \n",
    "    def _sort_actions(self, scores):\n",
    "        \"\"\" Sort actions by score and shuffle actions with same score\n",
    "            Inputs:\n",
    "                scores: pandas.Series with actions as index \"\"\"\n",
    "        sorted_actions = sorted(\n",
    "            scores.sample(frac=1).index,\n",
    "            key=lambda idx: scores.loc[idx],\n",
    "            reverse=True)\n",
    "        return sorted_actions\n",
    "\n",
    "    def _update_scores_history(self, scores):\n",
    "        if self.scores_log is not None:\n",
    "            self.scores_log = self.scores_log.append(\n",
    "                pd.DataFrame(\n",
    "                    data=scores.to_numpy().reshape((1,-1)),\n",
    "                    columns=self.scores_log.columns),\n",
    "                ignore_index=True)\n",
    "            self.scores_log = self.scores_log.astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class EpsilonGreedy(ABPolicy):\n",
    "    def __init__(self, bandit, epsilon, slate_size=1, scores_logging=False):\n",
    "        super(EpsilonGreedy, self).__init__(bandit, slate_size, scores_logging)\n",
    "        self.name = '{}-Greedy'.format(epsilon)\n",
    "        self.epsilon = epsilon\n",
    "        self.action_values = pd.DataFrame(data=0, columns=['value', 'count'],\n",
    "                                          index=bandit.actions)\n",
    "\n",
    "    def get_recommendations(self):\n",
    "        # sort actions by value and shuffle actions with same value\n",
    "        sorted_actions = self._sort_actions(self.action_values['value'])\n",
    "        # choose recommendations\n",
    "        if np.random.random() < self.epsilon:\n",
    "            recs = np.random.choice(sorted_actions[self.slate_size:],\n",
    "                                    size=self.slate_size, replace=False)\n",
    "        else:\n",
    "            recs = sorted_actions[:self.slate_size]\n",
    "        # update history of action scores\n",
    "        self._update_scores_history(self.action_values['value'])\n",
    "        return recs\n",
    "\n",
    "    def update(self, rewards):\n",
    "        super(EpsilonGreedy, self).update(rewards)\n",
    "        # update action values\n",
    "        for _, (item_id, reward) in rewards.iterrows():\n",
    "            value = self.action_values.loc[item_id, 'value']\n",
    "            N = self.action_values.loc[item_id, 'count']\n",
    "            self.action_values.loc[item_id, 'value'] = (value * N + reward) / (N + 1)\n",
    "            self.action_values.loc[item_id, 'count'] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class UCB(ABPolicy):\n",
    "    def __init__(self, bandit, slate_size=1, scores_logging=False):\n",
    "        super(UCB, self).__init__(bandit, slate_size, scores_logging)\n",
    "        self.name = 'UCB'\n",
    "        self.action_values = pd.DataFrame(data=0, columns=['value', 'count'],\n",
    "                                          index=bandit.actions)\n",
    "\n",
    "    def get_recommendations(self):\n",
    "        # compute UCB for each action\n",
    "        current_step = len(self.history)\n",
    "        if current_step > 0:\n",
    "            scores = self.action_values['count'].apply(\n",
    "                lambda N: np.sqrt(2*np.log(current_step) / N) if N > 0 else np.Inf)\n",
    "            scores = scores + self.action_values['value']\n",
    "        else:\n",
    "            scores = pd.Series(data=np.Inf, index=self.action_values.index)\n",
    "        # sort actions by score and shuffle actions with same score\n",
    "        sorted_actions = self._sort_actions(scores)\n",
    "        # choose recommendations\n",
    "        recs = sorted_actions[:self.slate_size]\n",
    "        # update history of action scores\n",
    "        self._update_scores_history(scores)\n",
    "        return recs\n",
    "\n",
    "    def update(self, rewards):\n",
    "        super(UCB, self).update(rewards)\n",
    "        # update action values\n",
    "        for _, (item_id, reward) in rewards.iterrows():\n",
    "            value = self.action_values.loc[item_id, 'value']\n",
    "            N = self.action_values.loc[item_id, 'count']\n",
    "            self.action_values.loc[item_id, 'value'] = (value * N + reward) / (N + 1)\n",
    "            self.action_values.loc[item_id, 'count'] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class TS(ABPolicy):\n",
    "    def __init__(self, bandit, slate_size=1, scores_logging=False):\n",
    "        super(TS, self).__init__(bandit, slate_size, scores_logging)\n",
    "        self.name = 'Thompson Sampling'\n",
    "        self.beta_params = pd.DataFrame(data=1, columns=['alpha', 'beta'],\n",
    "                                        index=bandit.actions)\n",
    "    \n",
    "    def get_recommendations(self):\n",
    "        # sample expected value for each action\n",
    "        expected_values = pd.Series(\n",
    "            data=4.5 * beta.rvs(self.beta_params['alpha'], self.beta_params['beta']) + 0.5,\n",
    "            index=self.beta_params.index)\n",
    "        # sort actions by value and shuffle actions with same value\n",
    "        sorted_actions = self._sort_actions(expected_values)\n",
    "        # choose recommendations\n",
    "        recs = sorted_actions[:self.slate_size]\n",
    "        # update history of action scores\n",
    "        self._update_scores_history(expected_values)\n",
    "        return recs\n",
    "\n",
    "    def update(self, rewards):\n",
    "        super(TS, self).update(rewards)\n",
    "        # update action value distribution prior\n",
    "        for _, (item_id, reward) in rewards.iterrows():\n",
    "            self.beta_params.loc[item_id, 'alpha'] += (reward - 0.5) / 4.5\n",
    "            self.beta_params.loc[item_id, 'beta'] += (5.0 - reward) / 4.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating bandits offline with replay method on movielens dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ml-100k.zip         100%[===================>]   4.70M  5.76MB/s    in 0.8s    \n"
     ]
    }
   ],
   "source": [
    "!wget -q --show-progress http://files.grouplens.org/datasets/movielens/ml-100k.zip\n",
    "!unzip -q ml-100k.zip\n",
    "\n",
    "rating_df = pd.read_csv('ml-100k/u.data', sep='\\t', header=None, names=['user_id','movie_id','rating','timestamp'], usecols=['movie_id', 'rating'])\n",
    "rating_df.columns = ['movieId', 'rating']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(data, num_ratings, num_movies):\n",
    "    \"\"\" Make each movieId/action uniformly distributed \"\"\"\n",
    "    # filters out movies with less than `num_ratings` ratings\n",
    "    movies = data.groupby('movieId').agg({'rating': 'count'})\n",
    "    if num_movies is not None:\n",
    "        movies_to_keep = movies[(movies['rating'] >= num_ratings)].sample(\n",
    "            n=num_movies, random_state=12).index\n",
    "    else:\n",
    "        movies_to_keep = movies[(movies['rating'] >= num_ratings)].index\n",
    "    data = data[data['movieId'].isin(movies_to_keep)]\n",
    "    # take a random sample of size `num_ratings` for each movie\n",
    "    data = data.groupby('movieId').sample(n=num_ratings, random_state=42)\n",
    "    # shuffle rows to randomize data stream\n",
    "    data = data.sample(frac=1, random_state=42)\n",
    "    # reset index to create pseudo-timestamp index\n",
    "    data = data.reset_index(drop=True)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_RATINGS = 30        # with full dataset  -> 10000\n",
    "                        # with small dataset -> 30\n",
    "NUM_MOVIES = None\n",
    "SLATE_SIZE = 5\n",
    "BATCH_SIZE = 100        # with replay eval   -> 100\n",
    "                        # with simulated env -> 1\n",
    "STREAM_LENGTH = 150     # with full dataset  -> 50000\n",
    "                        # with small dataset -> 150\n",
    "MODE = 'replay'         # 'replay' or 'sim'\n",
    "SCORES_LOG = False      # logging movie scores or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data\n",
    "logged_events = get_data(rating_df, NUM_RATINGS, NUM_MOVIES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBandit():\n",
    "    \"\"\" Implementation of a bandit problem with replay evaluation \"\"\"\n",
    "    def __init__(self, logged_events, batch_size=1):\n",
    "        self.events = logged_events.rename(columns={'rating': 'reward'})\n",
    "        self.actions = np.sort(logged_events['movieId'].unique())\n",
    "        self.batch_size = batch_size\n",
    "        self.stream_length = len(self.events) // batch_size\n",
    "    \n",
    "    def get_rewards(self, recommendations, n_event):\n",
    "        # generate events\n",
    "        idx = n_event * self.batch_size\n",
    "        events =  self.events.iloc[idx:idx+self.batch_size]\n",
    "        # keep only events that match with the recommendation slate\n",
    "        rewards = events[events['movieId'].isin(recommendations)]\n",
    "        return rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMBER OF MOVIES/ACTIONS: 806\n",
      "POLICY: 0.1-Greedy\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6754f1e878e426f8e179ac4650b3f93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/241 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HISTORY LENGTH: 147\n",
      "\n",
      "POLICY: UCB\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78313e1a59a34330a1f55f1200e992e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/241 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HISTORY LENGTH: 157\n",
      "\n",
      "POLICY: Thompson Sampling\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d2448178429460e8043c2119ff413f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/241 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HISTORY LENGTH: 150\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bandit = ReplayBandit(logged_events, BATCH_SIZE)\n",
    "STREAM_LENGTH = bandit.stream_length\n",
    "\n",
    "print(\"NUMBER OF MOVIES/ACTIONS: {}\".format(len(bandit.actions)))\n",
    "\n",
    "# instantiate policies\n",
    "policies = [\n",
    "    EpsilonGreedy(bandit, epsilon=0.1, slate_size=SLATE_SIZE, scores_logging=SCORES_LOG),\n",
    "    UCB(bandit, slate_size=SLATE_SIZE, scores_logging=SCORES_LOG),\n",
    "    TS(bandit, slate_size=SLATE_SIZE, scores_logging=SCORES_LOG),\n",
    "    ]\n",
    "\n",
    "# evaluate policies\n",
    "for policy in policies:\n",
    "    print(\"POLICY: {}\".format(policy.name))\n",
    "    from tqdm.notebook import tqdm\n",
    "    for i in tqdm(range(STREAM_LENGTH), ascii=True):\n",
    "        recs = policy.get_recommendations()\n",
    "        rewards = bandit.get_rewards(recs, i)\n",
    "        policy.update(rewards)\n",
    "    print(\"HISTORY LENGTH: {}\".format(len(policy.history)))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "def plot_rewards(*policies, title=None):\n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(11,5))\n",
    "    fig.suptitle(title)\n",
    "    for policy in policies:\n",
    "        # get cumulative rewards\n",
    "        cumsum_rewards = policy.history.reward.cumsum()\n",
    "        # get average rewards\n",
    "        timesteps = np.arange(len(cumsum_rewards)) + 1\n",
    "        avg_rewards = cumsum_rewards / timesteps\n",
    "        # plots\n",
    "        ax1.plot(timesteps, avg_rewards, label=policy.name)\n",
    "        ax2.plot(timesteps, cumsum_rewards, label=policy.name)\n",
    "    #\n",
    "    ax1.yaxis.set_major_formatter(FormatStrFormatter('%.1f'))\n",
    "    ax1.set_xlabel('time step')\n",
    "    ax1.set_ylabel('average reward')\n",
    "    ax1.legend(loc='lower right')\n",
    "    #\n",
    "    ax2.yaxis.set_major_formatter(FormatStrFormatter('%d'))\n",
    "    ax2.set_xlabel('time step')\n",
    "    ax2.set_ylabel('cumulative reward')\n",
    "    ax2.legend(loc='lower right')\n",
    "    #\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def plot_action_values(*policies):\n",
    "    fig, axs = plt.subplots(nrows=1, ncols=len(policies), figsize=(15,5), squeeze=False)\n",
    "    fig.suptitle(\"Action scores\")\n",
    "    axs = axs.ravel()\n",
    "    for i, policy in enumerate(policies):\n",
    "        cbar = True if i == len(axs)-1 else False\n",
    "        sns.heatmap(policy.scores_log.T, ax=axs[i], vmin=2.5, vmax=5, cmap='hot',\n",
    "                    cbar=cbar, xticklabels=1000, yticklabels=False)\n",
    "        axs[i].set_xlabel('time step')\n",
    "        axs[i].title.set_text(policy.name)\n",
    "    axs[0].set_ylabel('movieId')\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59020 characters]\n",
      "text/plain": [
       "<Figure size 792x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot results\n",
    "plot_rewards(*policies)\n",
    "if SCORES_LOG is True:\n",
    "    plot_action_values(*policies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-26 05:47:55\n",
      "\n",
      "recohut: 0.0.7\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas    : 1.1.5\n",
      "seaborn   : 0.11.2\n",
      "torch     : 1.10.0+cu111\n",
      "IPython   : 5.5.0\n",
      "numpy     : 1.19.5\n",
      "matplotlib: 3.2.2\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
