{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/RecoHut-Projects/recohut/blob/master/tutorials/modeling/T541654_group_rec_ddpg_ml1m_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Group Recommendations with Actor-critic RL Agent in MDP Environment on ML-1m Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='https://github.com/RecoHut-Stanzas/S758139/raw/main/images/group_recommender_actorcritic_1.svg'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Step 1 - Setup the environment**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **1.1 Install libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Building wheel for recohut (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
     ]
    }
   ],
   "source": [
    "!pip install -q -U git+https://github.com/RecoHut-Projects/recohut.git -b v0.0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **1.2 Download datasets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ml-1m.zip           100%[===================>]   5.64M  18.3MB/s    in 0.3s    \n"
     ]
    }
   ],
   "source": [
    "!wget -q --show-progress https://files.grouplens.org/datasets/movielens/ml-1m.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **1.3 Import libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Tuple, List, Dict\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "from collections import deque, defaultdict\n",
    "import shutil\n",
    "import zipfile\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from scipy.sparse import coo_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utils\n",
    "from recohut.transforms.user_grouping import GroupGenerator\n",
    "from recohut.layers.ou_noise import OUNoise\n",
    "\n",
    "# Models\n",
    "from recohut.models.actor_critic import Actor, Critic\n",
    "from recohut.models.embedding import GroupEmbedding\n",
    "\n",
    "# RL\n",
    "from recohut.rl.memory import ReplayMemory\n",
    "from recohut.rl.agents.ddpg import DDPGAgent\n",
    "from recohut.rl.envs.recsys import Env"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **1.4 Set params**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config(object):\n",
    "    \"\"\"\n",
    "    Configurations\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        # Data\n",
    "        self.data_folder_path = './data/silver'\n",
    "        self.item_path = os.path.join(self.data_folder_path, 'movies.dat')\n",
    "        self.user_path = os.path.join(self.data_folder_path, 'users.dat')\n",
    "        self.group_path = os.path.join(self.data_folder_path, 'groupMember.dat')\n",
    "        self.saves_folder_path = os.path.join('saves')\n",
    "\n",
    "        # Recommendation system\n",
    "        self.history_length = 5\n",
    "        self.top_K_list = [5, 10, 20]\n",
    "        self.rewards = [0, 1]\n",
    "\n",
    "        # Reinforcement learning\n",
    "        self.embedding_size = 32\n",
    "        self.state_size = self.history_length + 1\n",
    "        self.action_size = 1\n",
    "        self.embedded_state_size = self.state_size * self.embedding_size\n",
    "        self.embedded_action_size = self.action_size * self.embedding_size\n",
    "\n",
    "        # Numbers\n",
    "        self.item_num = None\n",
    "        self.user_num = None\n",
    "        self.group_num = None\n",
    "        self.total_group_num = None\n",
    "\n",
    "        # Environment\n",
    "        self.env_n_components = self.embedding_size\n",
    "        self.env_tol = 1e-4\n",
    "        self.env_max_iter = 1000\n",
    "        self.env_alpha = 0.001\n",
    "\n",
    "        # Actor-Critic network\n",
    "        self.actor_hidden_sizes = (128, 64)\n",
    "        self.critic_hidden_sizes = (32, 16)\n",
    "\n",
    "        # DDPG algorithm\n",
    "        self.tau = 1e-3\n",
    "        self.gamma = 0.9\n",
    "\n",
    "        # Optimizer\n",
    "        self.batch_size = 64\n",
    "        self.buffer_size = 100000\n",
    "        self.num_episodes = 10 # recommended = 1000\n",
    "        self.num_steps = 5 # recommended = 100\n",
    "        self.embedding_weight_decay = 1e-6\n",
    "        self.actor_weight_decay = 1e-6\n",
    "        self.critic_weight_decay = 1e-6\n",
    "        self.embedding_learning_rate = 1e-4\n",
    "        self.actor_learning_rate = 1e-4\n",
    "        self.critic_learning_rate = 1e-4\n",
    "        self.eval_per_iter = 10\n",
    "\n",
    "        # OU noise\n",
    "        self.ou_mu = 0.0\n",
    "        self.ou_theta = 0.15\n",
    "        self.ou_sigma = 0.2\n",
    "        self.ou_epsilon = 1.0\n",
    "\n",
    "        # GPU\n",
    "        if torch.cuda.is_available():\n",
    "            self.device = torch.device(\"cuda:0\")\n",
    "        else:\n",
    "            self.device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Step 2 - Data preparation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './ml-1m'\n",
    "output_path = './data/silver'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save data: ./data/silver\n",
      "# Users: 1636\n",
      "# Items: 2010\n",
      "# Groups: 1000\n",
      "# U-I ratings: 437654\n",
      "# G-I ratings: 50943\n",
      "Avg. # ratings / user: 267.51\n",
      "Avg. # ratings / group: 50.94\n",
      "Avg. group size: 2.20\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'./data/silver/users.dat'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = pd.read_csv(os.path.join(data_path,'ratings.dat'), sep='::', engine='python', header=None)\n",
    "\n",
    "group_generator = GroupGenerator(\n",
    "                    user_ids=np.arange(ratings[0].max()+1),\n",
    "                    item_ids=np.arange(ratings[1].max()+1),\n",
    "                    ratings=ratings,\n",
    "                    output_path=output_path,\n",
    "                    rating_threshold=4,\n",
    "                    num_groups=1000,\n",
    "                    group_sizes=[2, 3, 4, 5],\n",
    "                    min_num_ratings=20,\n",
    "                    train_ratio=0.7,\n",
    "                    val_ratio=0.1,\n",
    "                    negative_sample_size=100,\n",
    "                    verbose=True)\n",
    "\n",
    "shutil.copyfile(src=os.path.join(data_path, 'movies.dat'), dst=os.path.join(output_path, 'movies.dat'))\n",
    "shutil.copyfile(src=os.path.join(data_path, 'users.dat'), dst=os.path.join(output_path, 'users.dat'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['userRatingTrain.dat',\n",
       " 'groupMember.dat',\n",
       " 'groupRatingTestNegative.dat',\n",
       " 'userRatingValNegative.dat',\n",
       " 'userRatingVal.dat',\n",
       " 'userRatingTestNegative.dat',\n",
       " 'users.dat',\n",
       " 'userRatingTest.dat',\n",
       " 'groupRatingTrain.dat',\n",
       " 'groupRatingTest.dat',\n",
       " 'movies.dat',\n",
       " 'groupRatingVal.dat',\n",
       " 'groupRatingValNegative.dat']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataLoader(object):\n",
    "    \"\"\"\n",
    "    Data Loader\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config: Config):\n",
    "        \"\"\"\n",
    "        Initialize DataLoader\n",
    "        :param config: configurations\n",
    "        \"\"\"\n",
    "        self.config = config\n",
    "        self.history_length = config.history_length\n",
    "        self.item_num = self.get_item_num()\n",
    "        self.user_num = self.get_user_num()\n",
    "        self.group_num, self.total_group_num, self.group2members_dict, self.user2group_dict = self.get_groups()\n",
    "\n",
    "        if not os.path.exists(self.config.saves_folder_path):\n",
    "            os.mkdir(self.config.saves_folder_path)\n",
    "\n",
    "    def get_item_num(self) -> int:\n",
    "        \"\"\"\n",
    "        Get number of items\n",
    "        :return: number of items\n",
    "        \"\"\"\n",
    "        df_item = pd.read_csv(self.config.item_path, sep='::', index_col=0, engine='python')\n",
    "        self.config.item_num = df_item.index.max()\n",
    "        return self.config.item_num\n",
    "\n",
    "    def get_user_num(self) -> int:\n",
    "        \"\"\"\n",
    "        Get number of users\n",
    "        :return: number of users\n",
    "        \"\"\"\n",
    "        df_user = pd.read_csv(self.config.user_path, sep='::', index_col=0, engine='python')\n",
    "        self.config.user_num = df_user.index.max()\n",
    "        return self.config.user_num\n",
    "\n",
    "    def get_groups(self):\n",
    "        \"\"\"\n",
    "        Get number of groups and group members\n",
    "        :return: group_num, total_group_num, group2members_dict, user2group_dict\n",
    "        \"\"\"\n",
    "        df_group = pd.read_csv(self.config.group_path, sep=' ', header=None, index_col=None,\n",
    "                               names=['GroupID', 'Members'])\n",
    "        df_group['Members'] = df_group['Members']. \\\n",
    "            apply(lambda group_members: tuple(map(int, group_members.split(','))))\n",
    "        group_num = df_group['GroupID'].max()\n",
    "\n",
    "        users = set()\n",
    "        for members in df_group['Members']:\n",
    "            users.update(members)\n",
    "        users = sorted(users)\n",
    "        total_group_num = group_num + len(users)\n",
    "\n",
    "        df_user_group = pd.DataFrame()\n",
    "        df_user_group['GroupID'] = list(range(group_num + 1, total_group_num + 1))\n",
    "        df_user_group['Members'] = [(user,) for user in users]\n",
    "        df_group = df_group.append(df_user_group, ignore_index=True)\n",
    "        group2members_dict = {row['GroupID']: row['Members'] for _, row in df_group.iterrows()}\n",
    "        user2group_dict = {user: group_num + user_index + 1 for user_index, user in enumerate(users)}\n",
    "\n",
    "        self.config.group_num = group_num\n",
    "        self.config.total_group_num = total_group_num\n",
    "        return group_num, total_group_num, group2members_dict, user2group_dict\n",
    "\n",
    "    def load_rating_data(self, mode: str, dataset_name: str, is_appended=True) -> pd.DataFrame():\n",
    "        \"\"\"\n",
    "        Load rating data\n",
    "        :param mode: in ['user', 'group']\n",
    "        :param dataset_name: name of the dataset in ['train', 'val', 'test']\n",
    "        :param is_appended: True to append all datasets before this dataset\n",
    "        :return: df_rating\n",
    "        \"\"\"\n",
    "        assert (mode in ['user', 'group']) and (dataset_name in ['train', 'val', 'test'])\n",
    "        rating_path = os.path.join(self.config.data_folder_path, mode + 'Rating' + dataset_name.capitalize() + '.dat')\n",
    "        df_rating_append = pd.read_csv(rating_path, sep=' ', header=None, index_col=None,\n",
    "                                       names=['GroupID', 'MovieID', 'Rating', 'Timestamp'])\n",
    "        print('Read data:', rating_path)\n",
    "\n",
    "        if is_appended:\n",
    "            if dataset_name == 'train':\n",
    "                df_rating = df_rating_append\n",
    "            elif dataset_name == 'val':\n",
    "                df_rating = self.load_rating_data(mode=mode, dataset_name='train')\n",
    "                df_rating = df_rating.append(df_rating_append, ignore_index=True)\n",
    "            else:\n",
    "                df_rating = self.load_rating_data(mode=mode, dataset_name='val')\n",
    "                df_rating = df_rating.append(df_rating_append, ignore_index=True)\n",
    "        else:\n",
    "            df_rating = df_rating_append\n",
    "\n",
    "        return df_rating\n",
    "\n",
    "    def _load_rating_matrix(self, df_rating: pd.DataFrame()):\n",
    "        \"\"\"\n",
    "        Load rating matrix\n",
    "        :param df_rating: rating data\n",
    "        :return: rating_matrix\n",
    "        \"\"\"\n",
    "        group_ids = df_rating['GroupID']\n",
    "        item_ids = df_rating['MovieID']\n",
    "        ratings = df_rating['Rating']\n",
    "        rating_matrix = coo_matrix((ratings, (group_ids, item_ids)),\n",
    "                                   shape=(self.total_group_num + 1, self.config.item_num + 1)).tocsr()\n",
    "        return rating_matrix\n",
    "\n",
    "    def load_rating_matrix(self, dataset_name: str):\n",
    "        \"\"\"\n",
    "        Load group rating matrix\n",
    "        :param dataset_name: name of the dataset in ['train', 'val', 'test']\n",
    "        :return: rating_matrix\n",
    "        \"\"\"\n",
    "        assert dataset_name in ['train', 'val', 'test']\n",
    "\n",
    "        df_user_rating = self.user2group(self.load_rating_data(mode='user', dataset_name=dataset_name))\n",
    "        df_group_rating = self.load_rating_data(mode='group', dataset_name=dataset_name)\n",
    "        df_group_rating = df_group_rating.append(df_user_rating, ignore_index=True)\n",
    "        rating_matrix = self._load_rating_matrix(df_group_rating)\n",
    "\n",
    "        return rating_matrix\n",
    "\n",
    "    def user2group(self, df_user_rating):\n",
    "        \"\"\"\n",
    "        Change user ids to group ids\n",
    "        :param df_user_rating: user rating\n",
    "        :return: df_user_rating\n",
    "        \"\"\"\n",
    "        df_user_rating['GroupID'] = df_user_rating['GroupID'].apply(lambda user_id: self.user2group_dict[user_id])\n",
    "        return df_user_rating\n",
    "\n",
    "    def _load_eval_data(self, df_data_train: pd.DataFrame(), df_data_eval: pd.DataFrame(),\n",
    "                        negative_samples_dict: Dict[tuple, list]) -> pd.DataFrame():\n",
    "        \"\"\"\n",
    "        Write evaluation data\n",
    "        :param df_data_train: train data\n",
    "        :param df_data_eval: evaluation data\n",
    "        :param negative_samples_dict: one dictionary mapping (group_id, item_id) to negative samples\n",
    "        :return: data for evaluation\n",
    "        \"\"\"\n",
    "        df_eval = pd.DataFrame()\n",
    "        last_state_dict = defaultdict(list)\n",
    "        groups = []\n",
    "        histories = []\n",
    "        actions = []\n",
    "        negative_samples = []\n",
    "\n",
    "        for group_id, rating_group in df_data_train.groupby(['GroupID']):\n",
    "            rating_group.sort_values(by=['Timestamp'], ascending=True, ignore_index=True, inplace=True)\n",
    "            state = rating_group[rating_group['Rating'] == 1]['MovieID'].values.tolist()\n",
    "            last_state_dict[group_id] = state[-self.config.history_length:]\n",
    "\n",
    "        for group_id, rating_group in df_data_eval.groupby(['GroupID']):\n",
    "            rating_group.sort_values(by=['Timestamp'], ascending=True, ignore_index=True, inplace=True)\n",
    "            action = rating_group[rating_group['Rating'] == 1]['MovieID'].values.tolist()\n",
    "            state = deque(maxlen=self.history_length)\n",
    "            state.extend(last_state_dict[group_id])\n",
    "            for item_id in action:\n",
    "                if len(state) == self.config.history_length:\n",
    "                    groups.append(group_id)\n",
    "                    histories.append(list(state))\n",
    "                    actions.append(item_id)\n",
    "                    negative_samples.append(negative_samples_dict[(group_id, item_id)])\n",
    "                state.append(item_id)\n",
    "\n",
    "        df_eval['group'] = groups\n",
    "        df_eval['history'] = histories\n",
    "        df_eval['action'] = actions\n",
    "        df_eval['negative samples'] = negative_samples\n",
    "\n",
    "        return df_eval\n",
    "\n",
    "    def load_negative_samples(self, mode: str, dataset_name: str):\n",
    "        \"\"\"\n",
    "        Load negative samples\n",
    "        :param mode: in ['user', 'group']\n",
    "        :param dataset_name: name of the dataset in ['val', 'test']\n",
    "        :return: negative_samples_dict\n",
    "        \"\"\"\n",
    "        assert (mode in ['user', 'group']) and (dataset_name in ['val', 'test'])\n",
    "        negative_samples_path = os.path.join(self.config.data_folder_path, mode + 'Rating'\n",
    "                                             + dataset_name.capitalize() + 'Negative.dat')\n",
    "        negative_samples_dict = {}\n",
    "\n",
    "        with open(negative_samples_path, 'r') as negative_samples_file:\n",
    "            for line in negative_samples_file.readlines():\n",
    "                negative_samples = line.split()\n",
    "                ids = negative_samples[0][1:-1].split(',')\n",
    "                group_id = int(ids[0])\n",
    "                if mode == 'user':\n",
    "                    group_id = self.user2group_dict[group_id]\n",
    "                item_id = int(ids[1])\n",
    "                negative_samples = list(map(int, negative_samples[1:]))\n",
    "                negative_samples_dict[(group_id, item_id)] = negative_samples\n",
    "\n",
    "        return negative_samples_dict\n",
    "\n",
    "    def load_eval_data(self, mode: str, dataset_name: str, reload=False):\n",
    "        \"\"\"\n",
    "        Load evaluation data\n",
    "        :param mode: in ['user', 'group']\n",
    "        :param dataset_name: in ['val', 'test']\n",
    "        :param reload: True to reload the dataset file\n",
    "        :return: data for evaluation\n",
    "        \"\"\"\n",
    "        assert (mode in ['user', 'group']) and (dataset_name in ['val', 'test'])\n",
    "        exp_eval_path = os.path.join(self.config.saves_folder_path, 'eval_' + mode + '_' + dataset_name + '_'\n",
    "                                     + str(self.config.history_length) + '.pkl')\n",
    "\n",
    "        if reload or not os.path.exists(exp_eval_path):\n",
    "            if dataset_name == 'val':\n",
    "                df_rating_train = self.load_rating_data(mode=mode, dataset_name='train')\n",
    "            else:\n",
    "                df_rating_train = self.load_rating_data(mode=mode, dataset_name='val')\n",
    "            df_rating_eval = self.load_rating_data(mode=mode, dataset_name=dataset_name, is_appended=False)\n",
    "\n",
    "            if mode == 'user':\n",
    "                df_rating_train = self.user2group(df_rating_train)\n",
    "                df_rating_eval = self.user2group(df_rating_eval)\n",
    "\n",
    "            negative_samples_dict = self.load_negative_samples(mode=mode, dataset_name=dataset_name)\n",
    "            df_eval = self._load_eval_data(df_rating_train, df_rating_eval, negative_samples_dict)\n",
    "            df_eval.to_pickle(exp_eval_path)\n",
    "            print('Save data:', exp_eval_path)\n",
    "        else:\n",
    "            df_eval = pd.read_pickle(exp_eval_path)\n",
    "            print('Load data:', exp_eval_path)\n",
    "\n",
    "        return df_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Step 3 - Training & Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Evaluator(object):\n",
    "    \"\"\"\n",
    "    Evaluator\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, config: Config):\n",
    "        \"\"\"\n",
    "        Initialize Evaluator\n",
    "        :param config: configurations\n",
    "        \"\"\"\n",
    "        self.config = config\n",
    "\n",
    "    def evaluate(self, agent: DDPGAgent, df_eval: pd.DataFrame(), mode: str, top_K=5):\n",
    "        \"\"\"\n",
    "        Evaluate the agent\n",
    "        :param agent: agent\n",
    "        :param df_eval: evaluation data\n",
    "        :param mode: in ['user', 'group']\n",
    "        :param top_K: length of the recommendation list\n",
    "        :return: avg_recall_score, avg_ndcg_score\n",
    "        \"\"\"\n",
    "        recall_scores = []\n",
    "        ndcg_scores = []\n",
    "\n",
    "        for _, row in df_eval.iterrows():\n",
    "            group = row['group']\n",
    "            history = row['history']\n",
    "            item_true = row['action']\n",
    "            item_candidates = row['negative samples'] + [item_true]\n",
    "            np.random.shuffle(item_candidates)\n",
    "\n",
    "            state = [group] + history\n",
    "            items_pred = agent.get_action(state=state, item_candidates=item_candidates, top_K=top_K)\n",
    "\n",
    "            recall_score = 0\n",
    "            ndcg_score = 0\n",
    "\n",
    "            for k, item in enumerate(items_pred):\n",
    "                if item == item_true:\n",
    "                    recall_score = 1\n",
    "                    ndcg_score = np.log2(2) / np.log2(k + 2)\n",
    "                    break\n",
    "\n",
    "            recall_scores.append(recall_score)\n",
    "            ndcg_scores.append(ndcg_score)\n",
    "\n",
    "        avg_recall_score = float(np.mean(recall_scores))\n",
    "        avg_ndcg_score = float(np.mean(ndcg_scores))\n",
    "        print('%s: Recall@%d = %.4f, NDCG@%d = %.4f' % (mode.capitalize(), top_K, avg_recall_score,\n",
    "                                                        top_K, avg_ndcg_score))\n",
    "        return avg_recall_score, avg_ndcg_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(config: Config, env: Env, agent: DDPGAgent, evaluator: Evaluator,\n",
    "          df_eval_user: pd.DataFrame(), df_eval_group: pd.DataFrame()):\n",
    "    \"\"\"\n",
    "    Train the agent with the environment\n",
    "    :param config: configurations\n",
    "    :param env: environment\n",
    "    :param agent: agent\n",
    "    :param evaluator: evaluator\n",
    "    :param df_eval_user: user evaluation data\n",
    "    :param df_eval_group: group evaluation data\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    rewards = []\n",
    "    for episode in range(config.num_episodes):\n",
    "        state = env.reset()\n",
    "        agent.noise.reset()\n",
    "        episode_reward = 0\n",
    "\n",
    "        for step in range(config.num_steps):\n",
    "            action = agent.get_action(state)\n",
    "            new_state, reward, _, _ = env.step(action)\n",
    "            agent.replay_memory.push((state, action, reward, new_state))\n",
    "            state = new_state\n",
    "            episode_reward += reward\n",
    "\n",
    "            if len(agent.replay_memory) >= config.batch_size:\n",
    "                agent.update()\n",
    "\n",
    "        rewards.append(episode_reward / config.num_steps)\n",
    "        print('Episode = %d, average reward = %.4f' % (episode, episode_reward / config.num_steps))\n",
    "        if (episode + 1) % config.eval_per_iter == 0:\n",
    "            for top_K in config.top_K_list:\n",
    "                evaluator.evaluate(agent=agent, df_eval=df_eval_user, mode='user', top_K=top_K)\n",
    "            for top_K in config.top_K_list:\n",
    "                evaluator.evaluate(agent=agent, df_eval=df_eval_group, mode='group', top_K=top_K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GroupEmbedding(\n",
      "  (user_embedding): Embedding(6041, 32)\n",
      "  (item_embedding): Embedding(3953, 32)\n",
      "  (user_attention): Sequential(\n",
      "    (0): Linear(in_features=32, out_features=32, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=32, out_features=1, bias=True)\n",
      "  )\n",
      "  (user_softmax): Softmax(dim=-1)\n",
      ")\n",
      "Actor(\n",
      "  (net): Sequential(\n",
      "    (0): Linear(in_features=192, out_features=128, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=128, out_features=64, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=64, out_features=32, bias=True)\n",
      "  )\n",
      ")\n",
      "Critic(\n",
      "  (net): Sequential(\n",
      "    (0): Linear(in_features=224, out_features=32, bias=True)\n",
      "    (1): ReLU()\n",
      "    (2): Linear(in_features=32, out_features=16, bias=True)\n",
      "    (3): ReLU()\n",
      "    (4): Linear(in_features=16, out_features=1, bias=True)\n",
      "  )\n",
      ")\n",
      "Episode = 0, average reward = 0.0000\n",
      "Episode = 1, average reward = 0.0000\n",
      "Episode = 2, average reward = 0.4000\n",
      "Episode = 3, average reward = 0.0000\n",
      "Episode = 4, average reward = 0.0000\n",
      "Episode = 5, average reward = 0.0000\n",
      "Episode = 6, average reward = 0.0000\n",
      "Episode = 7, average reward = 0.0000\n",
      "Episode = 8, average reward = 0.0000\n",
      "Episode = 9, average reward = 0.0000\n",
      "User: Recall@5 = 0.0486, NDCG@5 = 0.0280\n",
      "User: Recall@10 = 0.0980, NDCG@10 = 0.0437\n",
      "User: Recall@20 = 0.1924, NDCG@20 = 0.0673\n",
      "Group: Recall@5 = 0.0426, NDCG@5 = 0.0235\n",
      "Group: Recall@10 = 0.0915, NDCG@10 = 0.0390\n",
      "Group: Recall@20 = 0.1956, NDCG@20 = 0.0651\n"
     ]
    }
   ],
   "source": [
    "config = Config()\n",
    "dataloader = DataLoader(config)\n",
    "rating_matrix_train = dataloader.load_rating_matrix(dataset_name='val')\n",
    "df_eval_user_test = dataloader.load_eval_data(mode='user', dataset_name='test')\n",
    "df_eval_group_test = dataloader.load_eval_data(mode='group', dataset_name='test')\n",
    "env = Env(config=config, rating_matrix=rating_matrix_train, dataset_name='val')\n",
    "noise = OUNoise(embedded_action_size=config.embedded_action_size, ou_mu=config.ou_mu,\n",
    "        ou_theta=config.ou_theta, ou_sigma=config.ou_sigma, ou_epsilon=config.ou_epsilon)\n",
    "agent = DDPGAgent(config=config, noise=noise, group2members_dict=dataloader.group2members_dict, verbose=True)\n",
    "evaluator = Evaluator(config=config)\n",
    "train(config=config, env=env, agent=agent, evaluator=evaluator,\n",
    "        df_eval_user=df_eval_user_test, df_eval_group=df_eval_group_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Closure**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more details, you can refer to https://github.com/RecoHut-Stanzas/S758139."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://github.com/RecoHut-Stanzas/S758139/blob/main/reports/S758139_Report.ipynb\" alt=\"S758139_Report\"> <img src=\"https://img.shields.io/static/v1?label=report&message=active&color=green\" /></a> <a href=\"https://github.com/RecoHut-Stanzas/S758139\" alt=\"S758139\"> <img src=\"https://img.shields.io/static/v1?label=code&message=github&color=blue\" /></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-19 15:04:51\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "IPython: 5.5.0\n",
      "torch  : 1.10.0+cu111\n",
      "numpy  : 1.19.5\n",
      "pandas : 1.1.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**END**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
