{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp rl.agents.replay_agents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replay Agents\n",
    "> Replay Agents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ABTestReplayer():\n",
    "    '''\n",
    "    A class to provide functionality for simulating the method on an A/B test.\n",
    "    '''\n",
    "    \n",
    "    def __init__(self, n_visits, n_test_visits, reward_history, item_col_name, visitor_col_name, reward_col_name, n_iterations=1):\n",
    "        \n",
    "        self.reward_history = reward_history\n",
    "        self.item_col_name = item_col_name\n",
    "        self.visitor_col_name = visitor_col_name\n",
    "        self.reward_col_name = reward_col_name\n",
    "\n",
    "        # number of runs to average over\n",
    "        self.n_iterations = n_iterations\n",
    "\n",
    "        \n",
    "        # TODO: validate that n_test_visits <= n_visits\n",
    "    \n",
    "        self.n_test_visits = n_test_visits\n",
    "\n",
    "        # number of visits to replay/simulate\n",
    "        self.n_visits = n_visits\n",
    "\n",
    "        # items under test\n",
    "        self.items = self.reward_history[self.item_col_name].unique()\n",
    "        self.n_items = len(self.items)\n",
    "        \n",
    "        # visitors in the historical reward_history (e.g., from ratings df)\n",
    "        self.visitors = self.reward_history[self.visitor_col_name].unique()\n",
    "        self.n_visitors = len(self.visitors)\n",
    "        \n",
    "        self.is_testing = True\n",
    "        self.best_item_id = None\n",
    "        \n",
    "    def reset(self):\n",
    "        # number of times each item has been sampled (previously n_sampled)\n",
    "        self.n_item_samples = np.zeros(self.n_items)\n",
    "        \n",
    "        # fraction of time each item has resulted in a reward (previously movie_clicks)\n",
    "        self.n_item_rewards = np.zeros(self.n_items)\n",
    "        \n",
    "        self.is_testing = True\n",
    "        self.best_item_idx = None\n",
    "    \n",
    "    def select_item(self):\n",
    "        if self.is_testing:\n",
    "            return np.random.randint(self.n_items)\n",
    "        else:\n",
    "            return self.best_item_idx\n",
    "            \n",
    "    def record_result(self, visit, item_idx, reward):\n",
    "    \n",
    "        self.n_item_samples[item_idx] += 1\n",
    "        \n",
    "        alpha = 1./self.n_item_samples[item_idx]\n",
    "\n",
    "        self.n_item_rewards[item_idx] += alpha * (reward - self.n_item_rewards[item_idx])\n",
    "        \n",
    "        if (visit == self.n_test_visits - 1): # this was the last visit during the testing phase\n",
    "            \n",
    "            self.is_testing = False\n",
    "            self.best_item_idx = np.argmax(self.n_item_rewards)\n",
    "\n",
    "    def simulator(self):\n",
    "    \n",
    "        results = []\n",
    "\n",
    "        for iteration in tqdm(range(0, self.n_iterations)):\n",
    "        \n",
    "            self.reset()\n",
    "            \n",
    "            total_rewards = 0\n",
    "            fraction_relevant = np.zeros(self.n_visits)\n",
    "\n",
    "            for visit in range(0, self.n_visits):\n",
    "            \n",
    "                found_match = False\n",
    "                while not found_match:\n",
    "                \n",
    "                    # choose a random visitor\n",
    "                    visitor_idx = np.random.randint(self.n_visitors)\n",
    "                    visitor_id = self.visitors[visitor_idx]\n",
    "\n",
    "                    # select an item to offer the visitor\n",
    "                    item_idx = self.select_item()\n",
    "                    item_id = self.items[item_idx]\n",
    "                    \n",
    "                    # if this interaction exists in the history, count it\n",
    "                    reward = self.reward_history.query(\n",
    "                        '{} == @item_id and {} == @visitor_id'.format(self.item_col_name, self.visitor_col_name))[self.reward_col_name]\n",
    "                    \n",
    "                    found_match = reward.shape[0] > 0\n",
    "                \n",
    "                reward_value = reward.iloc[0]\n",
    "                \n",
    "                self.record_result(visit, item_idx, reward_value)\n",
    "                \n",
    "                # record metrics\n",
    "                total_rewards += reward_value\n",
    "                fraction_relevant[visit] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                result = {}\n",
    "                result['iteration'] = iteration\n",
    "                result['visit'] = visit\n",
    "                result['item_id'] = item_id\n",
    "                result['visitor_id'] = visitor_id\n",
    "                result['reward'] = reward_value\n",
    "                result['total_reward'] = total_rewards\n",
    "                result['fraction_relevant'] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                results.append(result)\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class EpsilonGreedyReplayer():\n",
    "    '''\n",
    "    A class to provide functionality for simulating the replayer method on an epsilon-Greedy bandit algorithm.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, epsilon, n_visits, reward_history, item_col_name, visitor_col_name, reward_col_name, n_iterations=1):\n",
    "        \n",
    "        self.reward_history = reward_history\n",
    "        self.item_col_name = item_col_name\n",
    "        self.visitor_col_name = visitor_col_name\n",
    "        self.reward_col_name = reward_col_name\n",
    "\n",
    "        # number of runs to average over\n",
    "        self.n_iterations = n_iterations\n",
    "    \n",
    "        # parameter to control exploration vs exploitation\n",
    "        self.epsilon = epsilon\n",
    "\n",
    "        # number of visits to replay/simulate\n",
    "        self.n_visits = n_visits\n",
    "\n",
    "        # items under test\n",
    "        self.items = self.reward_history[self.item_col_name].unique()\n",
    "        self.n_items = len(self.items)\n",
    "        \n",
    "        # visitors in the historical reward_history (e.g., from ratings df)\n",
    "        self.visitors = self.reward_history[self.visitor_col_name].unique()\n",
    "        self.n_visitors = len(self.visitors)\n",
    "        \n",
    "        self.is_testing = True\n",
    "        self.best_item_id = None\n",
    "    \n",
    "    def select_item(self):\n",
    "        \n",
    "        # decide to explore or exploit\n",
    "        if np.random.uniform() < self.epsilon: # explore\n",
    "            item_id = np.random.randint(self.n_items)\n",
    "            \n",
    "        else: # exploit\n",
    "            item_id = np.argmax(self.n_item_rewards)\n",
    "            \n",
    "        return item_id\n",
    "    \n",
    "    def reset(self):\n",
    "        # number of times each item has been sampled (previously n_sampled)\n",
    "        self.n_item_samples = np.zeros(self.n_items)\n",
    "        \n",
    "        # fraction of time each item has resulted in a reward (previously movie_clicks)\n",
    "        self.n_item_rewards = np.zeros(self.n_items)\n",
    "\n",
    "    def record_result(self, visit, item_idx, reward):\n",
    "    \n",
    "        self.n_item_samples[item_idx] += 1\n",
    "        \n",
    "        alpha = 1./self.n_item_samples[item_idx]\n",
    "        self.n_item_rewards[item_idx] += alpha * (reward - self.n_item_rewards[item_idx])\n",
    "\n",
    "    \n",
    "    def simulator(self):\n",
    "\n",
    "        results = []\n",
    "\n",
    "        for iteration in tqdm(range(0, self.n_iterations)):\n",
    "        \n",
    "            self.reset()\n",
    "            \n",
    "            total_rewards = 0\n",
    "            fraction_relevant = np.zeros(self.n_visits)\n",
    "\n",
    "            for visit in range(0, self.n_visits):\n",
    "            \n",
    "                found_match = False\n",
    "                while not found_match:\n",
    "                \n",
    "                    # choose a random visitor\n",
    "                    visitor_idx = np.random.randint(self.n_visitors)\n",
    "                    visitor_id = self.visitors[visitor_idx]\n",
    "\n",
    "                    # select an item to offer the visitor\n",
    "                    item_idx = self.select_item()\n",
    "                    item_id = self.items[item_idx]\n",
    "                    \n",
    "                    # if this interaction exists in the history, count it\n",
    "                    reward = self.reward_history.query(\n",
    "                        '{} == @item_id and {} == @visitor_id'.format(self.item_col_name, self.visitor_col_name))[self.reward_col_name]\n",
    "                    \n",
    "                    found_match = reward.shape[0] > 0\n",
    "                \n",
    "                reward_value = reward.iloc[0]\n",
    "                \n",
    "                self.record_result(visit, item_idx, reward_value)\n",
    "                \n",
    "                # record metrics\n",
    "                total_rewards += reward_value\n",
    "                fraction_relevant[visit] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                result = {}\n",
    "                result['iteration'] = iteration\n",
    "                result['visit'] = visit\n",
    "                result['item_id'] = item_id\n",
    "                result['visitor_id'] = visitor_id\n",
    "                result['reward'] = reward_value\n",
    "                result['total_reward'] = total_rewards\n",
    "                result['fraction_relevant'] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                results.append(result)\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ThompsonSamplingReplayer():\n",
    "    '''\n",
    "    A class to provide functionality for simulating the replayer method on a Thompson Sampling bandit algorithm\n",
    "\n",
    "    '''\n",
    "\n",
    "    def __init__(self, n_visits, reward_history, item_col_name, visitor_col_name, reward_col_name, n_iterations=1):\n",
    "        \n",
    "        self.reward_history = reward_history\n",
    "        self.item_col_name = item_col_name\n",
    "        self.visitor_col_name = visitor_col_name\n",
    "        self.reward_col_name = reward_col_name\n",
    "\n",
    "        # number of runs to average over\n",
    "        self.n_iterations = n_iterations\n",
    "    \n",
    "\n",
    "\n",
    "        # number of visits to replay/simulate\n",
    "        self.n_visits = n_visits\n",
    "\n",
    "        # items under test\n",
    "        self.items = self.reward_history[self.item_col_name].unique()\n",
    "        self.n_items = len(self.items)\n",
    "        \n",
    "        # visitors in the historical reward_history (e.g., from ratings df)\n",
    "        self.visitors = self.reward_history[self.visitor_col_name].unique()\n",
    "        self.n_visitors = len(self.visitors)\n",
    "        \n",
    "        self.is_testing = True\n",
    "        self.best_item_id = None\n",
    "\n",
    "    def reset(self):\n",
    "        self.alphas = np.ones(self.n_items)\n",
    "        self.betas = np.ones(self.n_items)\n",
    "\n",
    "    def select_item(self):\n",
    "    \n",
    "        samples = [np.random.beta(a,b) for a,b in zip(self.alphas, self.betas)]\n",
    "        \n",
    "        return np.argmax(samples)\n",
    "\n",
    "    def record_result(self, visit, item_idx, reward):\n",
    "        \n",
    "        # update value estimate\n",
    "        if reward == 1:\n",
    "            self.alphas[item_idx] += 1\n",
    "        else:\n",
    "            self.betas[item_idx] += 1\n",
    "\n",
    "\n",
    "\n",
    "    def simulator(self):\n",
    "\n",
    "        results = []\n",
    "\n",
    "        for iteration in tqdm(range(0, self.n_iterations)):\n",
    "        \n",
    "            self.reset()\n",
    "            \n",
    "            total_rewards = 0\n",
    "            fraction_relevant = np.zeros(self.n_visits)\n",
    "\n",
    "            for visit in range(0, self.n_visits):\n",
    "            \n",
    "                found_match = False\n",
    "                while not found_match:\n",
    "                \n",
    "                    # choose a random visitor\n",
    "                    visitor_idx = np.random.randint(self.n_visitors)\n",
    "                    visitor_id = self.visitors[visitor_idx]\n",
    "\n",
    "                    # select an item to offer the visitor\n",
    "                    item_idx = self.select_item()\n",
    "                    item_id = self.items[item_idx]\n",
    "                    \n",
    "                    # if this interaction exists in the history, count it\n",
    "                    reward = self.reward_history.query(\n",
    "                        '{} == @item_id and {} == @visitor_id'.format(self.item_col_name, self.visitor_col_name))[self.reward_col_name]\n",
    "                    \n",
    "                    found_match = reward.shape[0] > 0\n",
    "                \n",
    "                reward_value = reward.iloc[0]\n",
    "                \n",
    "                self.record_result(visit, item_idx, reward_value)\n",
    "                \n",
    "                # record metrics\n",
    "                total_rewards += reward_value\n",
    "                fraction_relevant[visit] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                result = {}\n",
    "                result['iteration'] = iteration\n",
    "                result['visit'] = visit\n",
    "                result['item_id'] = item_id\n",
    "                result['visitor_id'] = visitor_id\n",
    "                result['reward'] = reward_value\n",
    "                result['total_reward'] = total_rewards\n",
    "                result['fraction_relevant'] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                results.append(result)\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class ReplaySimulator(object):\n",
    "    '''\n",
    "    A class to provide base functionality for simulating the replayer method for online algorithms.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, n_visits, reward_history, item_col_name, visitor_col_name, reward_col_name, n_iterations=1, random_seed=1):\n",
    "\n",
    "        np.random.seed(random_seed)\n",
    "    \n",
    "        self.reward_history = reward_history\n",
    "        self.item_col_name = item_col_name\n",
    "        self.visitor_col_name = visitor_col_name\n",
    "        self.reward_col_name = reward_col_name\n",
    "\n",
    "        # number of visits to replay/simulate\n",
    "        self.n_visits = n_visits\n",
    "        \n",
    "        # number of runs to average over\n",
    "        self.n_iterations = n_iterations\n",
    "    \n",
    "        # items under test\n",
    "        self.items = self.reward_history[self.item_col_name].unique()\n",
    "        self.n_items = len(self.items)\n",
    "        \n",
    "        # visitors in the historical reward_history (e.g., from ratings df)\n",
    "        self.visitors = self.reward_history[self.visitor_col_name].unique()\n",
    "        self.n_visitors = len(self.visitors)\n",
    "        \n",
    "\n",
    "    def reset(self):\n",
    "        # number of times each item has been sampled (previously n_sampled)\n",
    "        self.n_item_samples = np.zeros(self.n_items)\n",
    "        \n",
    "        # fraction of time each item has resulted in a reward (previously movie_clicks)\n",
    "        self.n_item_rewards = np.zeros(self.n_items)\n",
    "        \n",
    "    \n",
    "    def replay(self):\n",
    "        \n",
    "        results = []\n",
    "\n",
    "        for iteration in tqdm(range(0, self.n_iterations)):\n",
    "        \n",
    "            self.reset()\n",
    "            \n",
    "            total_rewards = 0\n",
    "            fraction_relevant = np.zeros(self.n_visits)\n",
    "\n",
    "            for visit in range(0, self.n_visits):\n",
    "            \n",
    "                found_match = False\n",
    "                while not found_match:\n",
    "                \n",
    "                    # choose a random visitor\n",
    "                    visitor_idx = np.random.randint(self.n_visitors)\n",
    "                    visitor_id = self.visitors[visitor_idx]\n",
    "\n",
    "                    # select an item to offer the visitor\n",
    "                    item_idx = self.select_item()\n",
    "                    item_id = self.items[item_idx]\n",
    "                    \n",
    "                    # if this interaction exists in the history, count it\n",
    "                    reward = self.reward_history.query(\n",
    "                        '{} == @item_id and {} == @visitor_id'.format(self.item_col_name, self.visitor_col_name))[self.reward_col_name]\n",
    "                    \n",
    "                    found_match = reward.shape[0] > 0\n",
    "                \n",
    "                reward_value = reward.iloc[0]\n",
    "                \n",
    "                self.record_result(visit, item_idx, reward_value)\n",
    "                \n",
    "                ## record metrics\n",
    "                total_rewards += reward_value\n",
    "                fraction_relevant[visit] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                result = {}\n",
    "                result['iteration'] = iteration\n",
    "                result['visit'] = visit\n",
    "                result['item_id'] = item_id\n",
    "                result['visitor_id'] = visitor_id\n",
    "                result['reward'] = reward_value\n",
    "                result['total_reward'] = total_rewards\n",
    "                result['fraction_relevant'] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                results.append(result)\n",
    "        \n",
    "        return results\n",
    "        \n",
    "    def select_item(self):\n",
    "        return np.random.randint(self.n_items)\n",
    "        \n",
    "    def record_result(self, visit, item_idx, reward):\n",
    "    \n",
    "        self.n_item_samples[item_idx] += 1\n",
    "        \n",
    "        alpha = 1./self.n_item_samples[item_idx]\n",
    "        self.n_item_rewards[item_idx] += alpha * (reward - self.n_item_rewards[item_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class UCBSamplingReplayer():\n",
    "    '''\n",
    "    A class to provide functionality for simulating the replayer method on a Thompson Sampling bandit algorithm\n",
    "\n",
    "    '''\n",
    "\n",
    "    def __init__(self,ucb_c, n_visits, reward_history, item_col_name, visitor_col_name, reward_col_name, n_iterations=1):\n",
    "        \n",
    "        self.reward_history = reward_history\n",
    "        self.item_col_name = item_col_name\n",
    "        self.visitor_col_name = visitor_col_name\n",
    "        self.reward_col_name = reward_col_name\n",
    "        self.ucb_c = ucb_c\n",
    "\n",
    "        # number of runs to average over\n",
    "        self.n_iterations = n_iterations\n",
    "    \n",
    "\n",
    "\n",
    "        # number of visits to replay/simulate\n",
    "        self.n_visits = n_visits\n",
    "\n",
    "        # items under test\n",
    "        self.items = self.reward_history[self.item_col_name].unique()\n",
    "        self.n_items = len(self.items)\n",
    "        \n",
    "        # visitors in the historical reward_history (e.g., from ratings df)\n",
    "        self.visitors = self.reward_history[self.visitor_col_name].unique()\n",
    "        self.n_visitors = len(self.visitors)\n",
    "        \n",
    "        self.is_testing = True\n",
    "        self.best_item_id = None\n",
    "\n",
    "    def reset(self):\n",
    "        self.Q = np.zeros(self.n_items) # q-value of actions\n",
    "        self.N = np.zeros(self.n_items) + 0.0001 # action count\n",
    "        self.timestep = 1\n",
    "\n",
    "\n",
    "    def select_item(self):\n",
    "\n",
    "        ln_timestep = np.log(np.full(self.n_items, self.timestep))\n",
    "        confidence = self.ucb_c * np.sqrt(ln_timestep/self.N)\n",
    "\n",
    "        action = np.argmax(self.Q + confidence)\n",
    "        self.timestep += 1\n",
    "        \n",
    "        return action\n",
    "\n",
    "    def record_result(self, visit, item_idx, reward):\n",
    "        \n",
    "        # update value estimate\n",
    "        self.N[item_idx] += 1 # increment action count\n",
    "        self.Q[item_idx] += 1/self.N[item_idx] * (reward - self.Q[item_idx]) # inc. update rule\n",
    "\n",
    "\n",
    "    def simulator(self):\n",
    "\n",
    "        results = []\n",
    "\n",
    "        for iteration in tqdm(range(0, self.n_iterations)):\n",
    "        \n",
    "            self.reset()\n",
    "            \n",
    "            total_rewards = 0\n",
    "            fraction_relevant = np.zeros(self.n_visits)\n",
    "\n",
    "            for visit in range(0, self.n_visits):\n",
    "            \n",
    "                found_match = False\n",
    "                while not found_match:\n",
    "                \n",
    "                    # choose a random visitor\n",
    "                    visitor_idx = np.random.randint(self.n_visitors)\n",
    "                    visitor_id = self.visitors[visitor_idx]\n",
    "\n",
    "                    # select an item to offer the visitor\n",
    "                    item_idx = self.select_item()\n",
    "                    item_id = self.items[item_idx]\n",
    "                    \n",
    "                    # if this interaction exists in the history, count it\n",
    "                    reward = self.reward_history.query(\n",
    "                        '{} == @item_id and {} == @visitor_id'.format(self.item_col_name, self.visitor_col_name))[self.reward_col_name]\n",
    "                    \n",
    "                    found_match = reward.shape[0] > 0\n",
    "                \n",
    "                reward_value = reward.iloc[0]\n",
    "                \n",
    "                self.record_result(visit, item_idx, reward_value)\n",
    "                \n",
    "                # record metrics\n",
    "                total_rewards += reward_value\n",
    "                fraction_relevant[visit] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                result = {}\n",
    "                result['iteration'] = iteration\n",
    "                result['visit'] = visit\n",
    "                result['item_id'] = item_id\n",
    "                result['visitor_id'] = visitor_id\n",
    "                result['reward'] = reward_value\n",
    "                result['total_reward'] = total_rewards\n",
    "                result['fraction_relevant'] = total_rewards * 1. / (visit + 1)\n",
    "                \n",
    "                results.append(result)\n",
    "        \n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting kaggle\n",
      "  Downloading kaggle-1.5.12.tar.gz (58 kB)\n",
      "\u001b[?25l\r\u001b[K     |█████▋                          | 10 kB 16.6 MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 20 kB 15.0 MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 30 kB 10.6 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 40 kB 9.3 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 51 kB 5.7 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 58 kB 2.7 MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: kaggle\n",
      "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for kaggle: filename=kaggle-1.5.12-py3-none-any.whl size=73051 sha256=af09991cd9752149427a8c7425a45ec5996a6496469a41710e1d6a123ccd647e\n",
      "  Stored in directory: /root/.cache/pip/wheels/62/d6/58/5853130f941e75b2177d281eb7e44b4a98ed46dd155f556dc5\n",
      "Successfully built kaggle\n",
      "Installing collected packages: kaggle\n",
      "  Attempting uninstall: kaggle\n",
      "    Found existing installation: kaggle 1.5.12\n",
      "    Uninstalling kaggle-1.5.12:\n",
      "      Successfully uninstalled kaggle-1.5.12\n",
      "Successfully installed kaggle-1.5.12\n",
      "Downloading amazon-product-reviews.zip to /content/recohut\n",
      " 85% 93.0M/109M [00:00<00:00, 117MB/s]\n",
      "100% 109M/109M [00:00<00:00, 115MB/s] \n",
      "Archive:  amazon-product-reviews.zip\n",
      "  inflating: ratings_Electronics (1).csv  \n"
     ]
    }
   ],
   "source": [
    "# !pip install -q -U kaggle\n",
    "# !pip install --upgrade --force-reinstall --no-deps kaggle\n",
    "# !mkdir ~/.kaggle\n",
    "# !cp /content/drive/MyDrive/kaggle.json ~/.kaggle/\n",
    "# !chmod 600 ~/.kaggle/kaggle.json\n",
    "\n",
    "!kaggle datasets download -d saurav9786/amazon-product-reviews\n",
    "!unzip amazon-product-reviews.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "header_list = [\"User_ID\", \"Product_ID\", \"Rating\", \"Time_Stamp\"]\n",
    "rating_df = pd.read_csv('ratings_Electronics (1).csv', names=header_list)\n",
    "\n",
    "reward_threshold = 4\n",
    "rating_df['reward'] = rating_df.eval('Rating > @reward_threshold').astype(int)\n",
    "\n",
    "n_visits = 500\n",
    "n_iterations = 1\n",
    "n_test_visits = 100\n",
    "\n",
    "reward_history = rating_df[:1000]\n",
    "item_col_name = 'Product_ID'\n",
    "visitor_col_name = 'User_ID'\n",
    "reward_col_name = 'reward'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A/B Test Simulations...starts...!!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [01:09<00:00, 69.08s/it]\n"
     ]
    }
   ],
   "source": [
    "#################### A/B testing ###############\n",
    "\n",
    "print(\"A/B Test Simulations...starts...!!!\")\n",
    "\n",
    "ab_results = ABTestReplayer(n_visits, n_test_visits, reward_history,\n",
    "                            item_col_name, visitor_col_name, reward_col_name,\n",
    "                            n_iterations=n_iterations).simulator()\n",
    "\n",
    "ab_results_df = pd.DataFrame(ab_results)\n",
    "ab_results_df.to_csv('ab_results_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epsilon - Greedy Simulations...starts...!!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:02<00:00,  2.76s/it]\n"
     ]
    }
   ],
   "source": [
    "################# Epsilon - Greedy Simulations ##############\n",
    "\n",
    "print(\"Epsilon - Greedy Simulations...starts...!!!\")\n",
    "\n",
    "epsilon = 0.05\n",
    "epsilon_results = EpsilonGreedyReplayer(epsilon, n_visits, reward_history,\n",
    "                                        item_col_name, visitor_col_name, reward_col_name,\n",
    "                                        n_iterations=n_iterations).simulator()\n",
    "\n",
    "epsilon_results_df = pd.DataFrame(epsilon_results)\n",
    "epsilon_results_df.to_csv('epsilon_results_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thompson Sampling Simulations...starts...!!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [15:27<00:00, 927.84s/it]\n"
     ]
    }
   ],
   "source": [
    "################### Thompson Sampling Simulations #######################\n",
    "\n",
    "print(\"Thompson Sampling Simulations...starts...!!!\")\n",
    "\n",
    "thompson_results = ThompsonSamplingReplayer(n_visits, reward_history,\n",
    "                                            item_col_name, visitor_col_name, reward_col_name,\n",
    "                                            n_iterations=n_iterations).simulator()\n",
    "\n",
    "thompson_results_df = pd.DataFrame(thompson_results)\n",
    "thompson_results_df.to_csv('thompson_results_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Upper Confidence Bounds Simulations...starts...!!!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [15:58<00:00, 958.75s/it]\n"
     ]
    }
   ],
   "source": [
    "####################  Upper Confidence Bounds #########################\n",
    "\n",
    "print(\"Upper Confidence Bounds Simulations...starts...!!!\")\n",
    "\n",
    "ucb = 2\n",
    "ucb_results = UCBSamplingReplayer(ucb, n_visits, reward_history,\n",
    "                                  item_col_name, visitor_col_name, reward_col_name,\n",
    "                                  n_iterations=n_iterations).simulator()\n",
    "\n",
    "ucb_results_df = pd.DataFrame(ucb_results)\n",
    "ucb_results_df.to_csv('ucb_results_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 105936 characters]\n",
      "text/plain": [
       "<Figure size 864x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ucb_results_df = pd.read_csv('ucb_results_df.csv').drop('Unnamed: 0', axis=1)\n",
    "thompson_results_df = pd.read_csv('thompson_results_df.csv').drop('Unnamed: 0', axis=1)\n",
    "epsilon_results_df = pd.read_csv('epsilon_results_df.csv').drop('Unnamed: 0', axis=1)\n",
    "ab_results_df = pd.read_csv('ab_results_df.csv').drop('Unnamed: 0', axis=1)\n",
    "\n",
    "#Grouping the each data frame with visit with mean\n",
    "ucb_avg_results_df = ucb_results_df.groupby('visit', as_index=False).mean()\n",
    "thompson_avg_results_df = thompson_results_df.groupby('visit', as_index=False).mean()\n",
    "epsilon_avg_results_df = epsilon_results_df.groupby('visit', as_index=False).mean()\n",
    "ab_avg_results_df = ab_results_df.groupby('visit', as_index=False).mean()\n",
    "\n",
    "# using a color-blind friendly palette with 10 colors\n",
    "color_blind_palette_10 = ['#cfcfcf', '#ffbc79', '#a2c8ec', '#898989', '#c85200',\n",
    "                          '#5f9ed1', '#595959', '#ababab', '#ff800e', '#006ba4']\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(12,10))\n",
    "\n",
    "for (avg_results_df, style) in [(ucb_avg_results_df, 'r-'),\n",
    "                                (thompson_avg_results_df, 'g--'),\n",
    "                                (epsilon_avg_results_df, 'b-'),\n",
    "                                (ab_avg_results_df, 'y--')]:\n",
    "    \n",
    "    ax.plot(avg_results_df.visit, avg_results_df.fraction_relevant, style, linewidth=3.5)\n",
    "\n",
    "\n",
    "ax.set_title('Percentage of Liked Recommendations')\n",
    "ax.set_xlabel('Recommendation #')\n",
    "ax.set_ylabel('% of Recs Clicked')\n",
    "\n",
    "#ax.set_xticks(range(0,22000,5000))\n",
    "#ax.set_ylim(0.2, 0.6)\n",
    "#ax.set_yticks(np.arange(0.2, 0.7, 0.1))\n",
    "\n",
    "#rescale the y-axis tick labels to show them as a percentage\n",
    "ax.set_yticklabels((ax.get_yticks()*100).astype(int))\n",
    "\n",
    "ax.legend(['UCB ',\n",
    "           'Thompson Sampling',\n",
    "           '$\\epsilon$ Greedy',\n",
    "           'A/B Test'\n",
    "          ],\n",
    "          loc='lower right'\n",
    "         )\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above it is clear that Thompson smapling of multi arm bandit outperforms A/B testing. In the lower samples the A/B test perform better than all other algorithim, but as in when the number samples increase the thompson sampling starts performing beter and better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-26 05:02:48\n",
      "\n",
      "recohut: 0.0.7\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.144+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "numpy     : 1.19.5\n",
      "matplotlib: 3.2.2\n",
      "IPython   : 5.5.0\n",
      "pandas    : 1.1.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d -p recohut"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
