{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.epsilon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Epsilon\n",
    "> ϵ-greedy algorithmic modules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from typing import List, Tuple\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from recohut.models.bandits import EpsilonBandit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class EpsilonGreedy:\n",
    "    \"\"\"\n",
    "    Class that is used to run a Epsilon-greedy multi-armed bandits test.\n",
    "    Attributes:\n",
    "        epsilon: Percentage of exploration.\n",
    "        batch_size: Number of examples per batch.\n",
    "    \n",
    "    Methods:\n",
    "        add_bandit: Adds a new BetaBandit to the test.\n",
    "        update_bandit: Updates the priors of the BetaBandit at index idx.\n",
    "        add_best_bandit: Adds the best bandit of the current batch.\n",
    "        bandit_batch: Determines how many times each bandit gets used in the running batch.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, epsilon: float=0.2, batch_size: int=1000):\n",
    "        \"\"\"\n",
    "        Initializes a new instance of EpsilonGreedy with the passed parmeters\n",
    "        Args:\n",
    "            epsilon: Percentage of exploration.\n",
    "            batch_size: Number of examples per batch.\n",
    "        \"\"\"\n",
    "    \n",
    "        self.batch_size = batch_size\n",
    "        self.epsilon = epsilon\n",
    "        self.bandits = list()\n",
    "        self.best_bandits = list()\n",
    "    \n",
    "    def add_bandit(self, positive_examples: int=0):\n",
    "        \"\"\"\n",
    "        Adds a new EpsilonBandit to the test.\n",
    "        Args:\n",
    "            positive_examples: Number of positive examples.\n",
    "        \"\"\"\n",
    "        self.bandits.append(EpsilonBandit(positive_examples))\n",
    "\n",
    "    def update_bandit(self, idx: int, positive_examples: int=0):\n",
    "        \"\"\"\n",
    "        Updates the priors of the BetaBandit at index idx.\n",
    "        Args:\n",
    "            idx: Index of the bandit to be updated.\n",
    "            positive_examples: Number of positive examples.\n",
    "        \"\"\"\n",
    "\n",
    "        self.bandits[idx].update(positive_examples)\n",
    "\n",
    "    def add_best_bandit(self):\n",
    "        \"\"\"\n",
    "        Adds the best bandit of the current batch.\n",
    "        \"\"\"\n",
    "\n",
    "        idx = 0\n",
    "        maxVal = 0\n",
    "        for i in range(len(self.bandits)):\n",
    "            val = self.bandits[i].get_value()\n",
    "            if val > maxVal:\n",
    "                maxVal = val\n",
    "                idx = i\n",
    "        self.best_bandits.append(idx)\n",
    "\n",
    "    def bandit_batch(self) -> Tuple[int, int]:\n",
    "        \"\"\"\n",
    "        Determines which bandit is the best and how many times examples\n",
    "        get used for exploration.\n",
    "        Returns:\n",
    "            The number of exploratory examples and the best bandit's index.\n",
    "        \"\"\"\n",
    "\n",
    "        self.add_best_bandit()\n",
    "        n_bandits = len(self.bandits)\n",
    "        exploration_total = self.batch_size * self.epsilon\n",
    "        exploration = int(exploration_total / n_bandits)\n",
    "\n",
    "        return exploration, self.best_bandits[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class EpsilonGreedyRunner:\n",
    "    \"\"\"\n",
    "    Class that is used to run simulations of Thompson sampling tests.\n",
    "    Attributes:\n",
    "        bandit_returns: List of average returns per bandit.\n",
    "        epsilon: Percentage of exploration.\n",
    "        batch_size: Number of examples per batch.\n",
    "        batches: Number of batches.\n",
    "        simulations: Number of simulations.\n",
    "    \n",
    "    Methods:\n",
    "        init_bandits: Prepares everything for new simulation.\n",
    "        run: Runs the simulations and tracks performance.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, bandit_returns: List[float], epsilon: float=0.2, batch_size: int=10000, batches: int=10, simulations: int=100):\n",
    "        \"\"\"\n",
    "        Initializes a new instance of RunEpsilonGreedy with the passed parameters.\n",
    "        Attributes:\n",
    "            bandit_returns: List of average returns per bandit.\n",
    "            epsilon: Percentage of exploration.\n",
    "            batch_size: Number of examples per batch.\n",
    "            batches: Number of batches.\n",
    "            simulations: Number of simulations.\n",
    "        \"\"\"\n",
    "\n",
    "        self.bandit_returns = bandit_returns\n",
    "        self.n_bandits = len(bandit_returns)\n",
    "        self.bandits = list(range(self.n_bandits))\n",
    "\n",
    "        self.epsilon = epsilon\n",
    "        self.batch_size = batch_size\n",
    "        self.batches = batches\n",
    "        self.simulations = simulations\n",
    "\n",
    "        self.df_bids = pd.DataFrame(columns=self.bandit_returns)\n",
    "        self.df_clicks = pd.DataFrame(columns=self.bandit_returns)\n",
    "\n",
    "    def init_bandits(self):\n",
    "        \"\"\"\n",
    "        Prepares everything for new simulation.\n",
    "        \"\"\"\n",
    "\n",
    "        self.first_batch = True\n",
    "        self.bandit_positive_examples = [0] * self.n_bandits\n",
    "        self.bandit_total_examples = [0] * self.n_bandits\n",
    "        self.eps = EpsilonGreedy(self.epsilon, self.batch_size)\n",
    "        for i in self.bandits:\n",
    "            self.eps.add_bandit()\n",
    "    \n",
    "    def run(self):\n",
    "        \"\"\"\n",
    "        Runs the simulations and tracks performance.\n",
    "        \"\"\"\n",
    "\n",
    "        for j in range(self.simulations):\n",
    "            self.init_bandits()\n",
    "            for i in range(self.batches):\n",
    "                exploration_examples, best_bandit = self.eps.bandit_batch()\n",
    "                if self.first_batch:\n",
    "                    self.first_batch = False\n",
    "                    exploration_examples = self.batch_size // self.n_bandits\n",
    "                for idx in self.bandits:\n",
    "                    self.bandit_total_examples[idx] += exploration_examples\n",
    "                    positive_examples = np.random.binomial(exploration_examples, self.bandit_returns[idx])\n",
    "                    self.bandit_positive_examples[idx] += positive_examples\n",
    "                    self.eps.update_bandit(idx, positive_examples)\n",
    "                \n",
    "                exploitation_examples = self.batch_size - exploration_examples * self.n_bandits\n",
    "                self.bandit_total_examples[best_bandit] += exploitation_examples\n",
    "                self.bandit_positive_examples[best_bandit] += np.random.binomial(exploitation_examples, self.bandit_returns[best_bandit])\n",
    "\n",
    "                if self.df_bids.shape[0] < self.batches:\n",
    "                    self.df_bids.loc[i] = self.bandit_total_examples\n",
    "                    self.df_clicks.loc[i] = self.bandit_positive_examples\n",
    "                else:\n",
    "                    self.df_bids.loc[i] += self.bandit_total_examples\n",
    "                    self.df_clicks.loc[i] += self.bandit_positive_examples\n",
    "        self.df_bids /= self.simulations\n",
    "        self.df_clicks /= self.simulations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-18 05:32:15\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas : 1.1.5\n",
      "google : 2.0.3\n",
      "numpy  : 1.19.5\n",
      "IPython: 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
