{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp metrics.utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metric utils\n",
    "> Model evaluation utilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "import torch\n",
    "from sklearn.metrics import precision_score, recall_score, ndcg_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def calculate_precision_recall(X, y_true, y_pred, N, threshold):\n",
    "    \"\"\"Calculate the precision and recall scores.\n",
    "\n",
    "    Args:\n",
    "        X\n",
    "        y_true\n",
    "        y_pred\n",
    "        N\n",
    "        threshold\n",
    "    \n",
    "    Returns:\n",
    "        precision_score (float)\n",
    "        recall_score (float)\n",
    "    \"\"\"\n",
    "    precision = 0\n",
    "    recall = 0\n",
    "    count = 0\n",
    "    \n",
    "    rec_true = np.array([1 if rating >= threshold else 0 for rating in y_true])\n",
    "    rec_pred = np.zeros(y_pred.size)\n",
    "    \n",
    "    for user_id in np.unique(X[:,0]):\n",
    "        indices = np.where(X[:,0] == user_id)[0]\n",
    "        \n",
    "        rec_true = np.array([1 if y_true[i] >= threshold else 0 for i in indices])\n",
    "\n",
    "        if (np.count_nonzero(rec_true) > 0): # ignore test users without relevant ratings\n",
    "        \n",
    "            user_pred = np.array([y_pred[i] for i in indices])\n",
    "            rec_pred = np.zeros(indices.size)\n",
    "\n",
    "            for pos in np.argsort(user_pred)[-N:]:\n",
    "                if user_pred[pos] >= threshold:\n",
    "                    rec_pred[pos] = 1\n",
    "            \n",
    "            precision += precision_score(rec_true, rec_pred, zero_division=0)\n",
    "            recall += recall_score(rec_true, rec_pred)\n",
    "            count += 1\n",
    "        \n",
    "    return precision/count, recall/count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def calculate_ndcg(X, y_true, y_pred, N):\n",
    "    \"\"\"Calculate the NDCG score.\n",
    "\n",
    "    Args:\n",
    "        X\n",
    "        y_true\n",
    "        y_pred\n",
    "        N\n",
    "    \n",
    "    Returns:\n",
    "        ndcg_score (float)\n",
    "    \"\"\"\n",
    "    ndcg = 0\n",
    "    count = 0\n",
    "    \n",
    "    for user_id in np.unique(X[:,0]):\n",
    "        indices = np.where(X[:,0] == user_id)[0]\n",
    "        \n",
    "        user_true = np.array([y_true[i] for i in indices])\n",
    "        user_pred = np.array([y_pred[i] for i in indices])  \n",
    "        \n",
    "        user_true = np.expand_dims(user_true, axis=0)\n",
    "        user_pred = np.expand_dims(user_pred, axis=0)\n",
    "                \n",
    "        if user_true.size > 1:\n",
    "            ndcg += ndcg_score(user_true, user_pred, k=N, ignore_ties=False)\n",
    "            count += 1\n",
    "    \n",
    "    return ndcg / count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def recall(scores, labels, k):\n",
    "    scores = scores.cpu()\n",
    "    labels = labels.cpu()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank[:, :k]\n",
    "    hit = labels.gather(1, cut)\n",
    "    return (hit.sum(1).float() / torch.min(torch.Tensor([k]).to(hit.device), labels.sum(1).float())).mean().cpu().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def ndcg(scores, labels, k):\n",
    "    scores = scores.cpu()\n",
    "    labels = labels.cpu()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank[:, :k]\n",
    "    hits = labels.gather(1, cut)\n",
    "    position = torch.arange(2, 2+k)\n",
    "    weights = 1 / torch.log2(position.float())\n",
    "    dcg = (hits.float() * weights).sum(1)\n",
    "    idcg = torch.Tensor([weights[:min(int(n), k)].sum() for n in labels.sum(1)])\n",
    "    ndcg = dcg / idcg\n",
    "    return ndcg.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def recalls_and_ndcgs_for_ks(scores, labels, ks):\n",
    "    metrics = {}\n",
    "\n",
    "    scores = scores\n",
    "    labels = labels\n",
    "    answer_count = labels.sum(1)\n",
    "\n",
    "    labels_float = labels.float()\n",
    "    rank = (-scores).argsort(dim=1)\n",
    "    cut = rank\n",
    "    for k in sorted(ks, reverse=True):\n",
    "       cut = cut[:, :k]\n",
    "       hits = labels_float.gather(1, cut)\n",
    "       metrics['Recall@%d' % k] = \\\n",
    "           (hits.sum(1) / torch.min(torch.Tensor([k]).to(labels.device), labels.sum(1).float())).mean().cpu().item()\n",
    "\n",
    "       position = torch.arange(2, 2+k)\n",
    "       weights = 1 / torch.log2(position.float())\n",
    "       dcg = (hits * weights.to(hits.device)).sum(1)\n",
    "       idcg = torch.Tensor([weights[:min(int(n), k)].sum() for n in answer_count]).to(dcg.device)\n",
    "       ndcg = (dcg / idcg).mean()\n",
    "       metrics['NDCG@%d' % k] = ndcg.cpu().item()\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-dc7edf9e-9c77-4a3f-b173-0a23098df39e\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deepmf</th>\n",
       "      <th>item</th>\n",
       "      <th>ncf</th>\n",
       "      <th>user</th>\n",
       "      <th>vdeepmf</th>\n",
       "      <th>vncf</th>\n",
       "      <th>y_test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.726801</td>\n",
       "      <td>496</td>\n",
       "      <td>2.854854</td>\n",
       "      <td>68</td>\n",
       "      <td>2.472237</td>\n",
       "      <td>2.620151</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.349192</td>\n",
       "      <td>473</td>\n",
       "      <td>3.002311</td>\n",
       "      <td>633</td>\n",
       "      <td>2.847424</td>\n",
       "      <td>2.690057</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.726862</td>\n",
       "      <td>329</td>\n",
       "      <td>3.605561</td>\n",
       "      <td>1405</td>\n",
       "      <td>3.810497</td>\n",
       "      <td>3.466036</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.467009</td>\n",
       "      <td>328</td>\n",
       "      <td>3.389759</td>\n",
       "      <td>1240</td>\n",
       "      <td>3.639901</td>\n",
       "      <td>3.205043</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.140076</td>\n",
       "      <td>54</td>\n",
       "      <td>3.194410</td>\n",
       "      <td>841</td>\n",
       "      <td>2.887761</td>\n",
       "      <td>2.848487</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dc7edf9e-9c77-4a3f-b173-0a23098df39e')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-dc7edf9e-9c77-4a3f-b173-0a23098df39e button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-dc7edf9e-9c77-4a3f-b173-0a23098df39e');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "     deepmf  item       ncf  user   vdeepmf      vncf  y_test\n",
       "0  2.726801   496  2.854854    68  2.472237  2.620151     3.0\n",
       "1  3.349192   473  3.002311   633  2.847424  2.690057     3.5\n",
       "2  3.726862   329  3.605561  1405  3.810497  3.466036     4.0\n",
       "3  3.467009   328  3.389759  1240  3.639901  3.205043     0.5\n",
       "4  3.140076    54  3.194410   841  2.887761  2.848487     3.0"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "sample_data = pd.DataFrame.from_records(\n",
    "    [{'deepmf': 2.7268011569976807,\n",
    "  'item': 496,\n",
    "  'ncf': 2.854853630065918,\n",
    "  'user': 68,\n",
    "  'vdeepmf': 2.4722371101379395,\n",
    "  'vncf': 2.620150566101074,\n",
    "  'y_test': 3.0},\n",
    " {'deepmf': 3.3491923809051514,\n",
    "  'item': 473,\n",
    "  'ncf': 3.0023105144500732,\n",
    "  'user': 633,\n",
    "  'vdeepmf': 2.847424030303955,\n",
    "  'vncf': 2.6900570392608643,\n",
    "  'y_test': 3.5},\n",
    " {'deepmf': 3.7268624305725098,\n",
    "  'item': 329,\n",
    "  'ncf': 3.605560779571533,\n",
    "  'user': 1405,\n",
    "  'vdeepmf': 3.810497283935547,\n",
    "  'vncf': 3.466035842895508,\n",
    "  'y_test': 4.0},\n",
    " {'deepmf': 3.4670088291168213,\n",
    "  'item': 328,\n",
    "  'ncf': 3.389759063720703,\n",
    "  'user': 1240,\n",
    "  'vdeepmf': 3.6399013996124268,\n",
    "  'vncf': 3.205043315887451,\n",
    "  'y_test': 0.5},\n",
    " {'deepmf': 3.140076160430908,\n",
    "  'item': 54,\n",
    "  'ncf': 3.1944096088409424,\n",
    "  'user': 841,\n",
    "  'vdeepmf': 2.887760877609253,\n",
    "  'vncf': 2.848487138748169,\n",
    "  'y_test': 3.0}]\n",
    ")\n",
    "sample_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample data of 5 users/items\n",
    "sample_data_2 = {\n",
    "    'y_true':np.array([1,2,3,4,5]),\n",
    "    'y_pred':np.array([1,3,3,2,4]),\n",
    "    'ids':np.array([[1,2],[1,3],[2,4],[2,5],[3,2]]),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unittest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unittest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestMetricUtils(unittest.TestCase):\n",
    "    def setUp(self):\n",
    "        self.sample_data = sample_data\n",
    "        self.sample_data_2 = sample_data_2\n",
    "        self.method = 'ncf'\n",
    "        self.like_threshold = 3\n",
    "            \n",
    "    def testPrecisionRecall(self):\n",
    "        num_recommendations = 2\n",
    "        ids = self.sample_data[['user', 'item']].to_numpy()\n",
    "        y_true = self.sample_data['y_test'].to_numpy()\n",
    "        y_pred = self.sample_data[self.method].to_numpy()\n",
    "        precision, recall = calculate_precision_recall(ids, y_true, y_pred, num_recommendations, self.like_threshold)\n",
    "        self.assertEqual(precision, 0.75)\n",
    "        self.assertEqual(recall, 0.75)\n",
    "\n",
    "    def testNDCG(self):\n",
    "        num_recommendations = 2\n",
    "        ids = self.sample_data_2['ids']\n",
    "        y_true = self.sample_data_2['y_true']\n",
    "        y_pred = self.sample_data_2['y_pred']\n",
    "        ndcg = calculate_ndcg(ids, y_true, y_pred, num_recommendations)\n",
    "        self.assertAlmostEqual(ndcg, 0.9686, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "testNDCG (__main__.TestMetricUtils) ... ok\n",
      "testPrecisionRecall (__main__.TestMetricUtils) ... ok\n",
      "\n",
      "----------------------------------------------------------------------\n",
      "Ran 2 tests in 0.017s\n",
      "\n",
      "OK\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<unittest.main.TestProgram at 0x7f1413e21690>"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unittest.main(argv=[''], verbosity=2, exit=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-18 10:07:34\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "pandas : 1.1.5\n",
      "torch  : 1.10.0+cu111\n",
      "IPython: 5.5.0\n",
      "numpy  : 1.19.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
