{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp datasets.synthetic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synthetic Datasets\n",
    "> Methods to generate synthetic sample datasets for concept testing and validations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Synthetic():\n",
    "  def __init__(self, version='v1'):\n",
    "    self.version = version\n",
    "\n",
    "  def explicit(self):\n",
    "    \"\"\"\n",
    "    In the \"explicit feedback\" scenario, interactions between users and items\n",
    "    are numerical / ordinal ratings or binary preferences such as like or\n",
    "    dislike. These types of interactions are termed as explicit feedback.\n",
    "    \"\"\"\n",
    "    if self.version=='v1':\n",
    "      \"\"\"\n",
    "      The following shows a dummy data for the explicit rating type of feedback. In the data,\n",
    "        - There are 3 users whose IDs are 1, 2, 3.\n",
    "        - There are 3 items whose IDs are 1, 2, 3.\n",
    "        - Items are rated by users only once. So even when users interact with items at different timestamps, the ratings are kept the same. This is seen in some use cases such as movie recommendations, where users' ratings do not change dramatically over a short period of time.\n",
    "        - Timestamps of when the ratings are given are also recorded.\n",
    "      \"\"\"\n",
    "      data = pd.DataFrame({\n",
    "          \"USERID\": [1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3],\n",
    "          \"ITEMID\": [1, 1, 2, 2, 2, 1, 2, 1, 2, 3, 3, 3, 3, 3, 1],\n",
    "          \"RATING\": [4, 4, 3, 3, 3, 4, 5, 4, 5, 5, 5, 5, 5, 5, 4],\n",
    "          \"TIMESTAMP\": [\n",
    "              '2000-01-01', '2000-01-01', '2000-01-02', '2000-01-02', '2000-01-02',\n",
    "              '2000-01-01', '2000-01-01', '2000-01-03', '2000-01-03', '2000-01-03',\n",
    "              '2000-01-01', '2000-01-03', '2000-01-03', '2000-01-03', '2000-01-04'\n",
    "          ]\n",
    "      })\n",
    "\n",
    "      return data\n",
    "\n",
    "  def implicit(self):\n",
    "    \"\"\"\n",
    "    Many times there are no explicit ratings or preferences given by users,\n",
    "    that is, the interactions are usually implicit. For example, a user may\n",
    "    puchase something on a website, click an item on a mobile app, or order\n",
    "    food from a restaurant. This information may reflect users' preference\n",
    "    towards the items in an implicit manner.\n",
    "    \"\"\"\n",
    "    if self.version=='v1':\n",
    "      \"\"\"\n",
    "      The following shows a dummy data for the implicit rating type of feedback. In the data,\n",
    "        - There are 3 users whose IDs are 1, 2, 3.\n",
    "        - There are 3 items whose IDs are 1, 2, 3.\n",
    "        - There are no ratings or explicit feedback given by the users. Sometimes there are the types of events. In this dummy dataset, for illustration purposes, there are three types for the interactions between users and items, that is, click, add and purchase, meaning \"click on the item\", \"add the item into cart\" and \"purchase the item\", respectively.\n",
    "        - Sometimes there is other contextual or associative information available for the types of interactions. E.g., \"time-spent on visiting a site before clicking\" etc. For simplicity, only the type of interactions is considered in this notebook.\n",
    "        - The timestamp of each interaction is also given.\n",
    "      \"\"\"\n",
    "      data = pd.DataFrame({\n",
    "          \"USERID\": [1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3],\n",
    "          \"ITEMID\": [1, 1, 2, 2, 2, 1, 2, 1, 2, 3, 3, 3, 3, 3, 1],\n",
    "          \"EVENT\": [\n",
    "              'click', 'click', 'click', 'click', 'purchase',\n",
    "              'click', 'purchase', 'add', 'purchase', 'purchase',\n",
    "              'click', 'click', 'add', 'purchase', 'click'\n",
    "          ],\n",
    "          \"TIMESTAMP\": [\n",
    "              '2000-01-01', '2000-01-01', '2000-01-02', '2000-01-02', '2000-01-02',\n",
    "              '2000-01-01', '2000-01-01', '2000-01-03', '2000-01-03', '2000-01-03',\n",
    "              '2000-01-01', '2000-01-03', '2000-01-03', '2000-01-03', '2000-01-04'\n",
    "          ]\n",
    "      })\n",
    "\n",
    "      return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Session():\n",
    "  def __init__(self, version='trivago'):\n",
    "    self.version = version\n",
    "\n",
    "  def train(self):\n",
    "    if self.version=='trivago':\n",
    "      d_train = {\n",
    "          \"user_id\": [\n",
    "              \"64BL89\", \"64BL89\", \"64BL89\", \"64BL89\",\n",
    "              \"64BLF\", \"64BLF\",\n",
    "              \"64BL89\", \"64BL89\", \"64BL89\", \"64BL89\"\n",
    "          ],\n",
    "          \"session_id\": [\n",
    "              \"3579f89\", \"3579f89\", \"3579f89\", \"3579f89\",\n",
    "              \"4504h9\", \"4504h9\",\n",
    "              \"5504hFL\", \"5504hFL\", \"5504hFL\", \"5504hFL\"\n",
    "          ],\n",
    "          \"timestamp\": [\n",
    "              1, 2, 3, 4,\n",
    "              2, 4,\n",
    "              7, 8, 9, 10\n",
    "          ],\n",
    "          \"step\": [\n",
    "              1, 2, 3, 4,\n",
    "              1, 2,\n",
    "              1, 2, 3, 4\n",
    "          ],\n",
    "          \"action_type\": [\n",
    "              \"interaction item image\", \"clickout item\", \n",
    "                  \"interaction item info\", \"filter selection\",\n",
    "              \"interaction item image\", \"clickout item\",\n",
    "                  \"filter selection\", \"clickout item\", \n",
    "              \"interaction item image\", \"clickout item\"\n",
    "          ],\n",
    "          \"reference\": [\n",
    "              \"5001\", \"5002\", \"5003\", \"unknown\",\n",
    "              \"5010\", \"5001\",\n",
    "              \"unknown\", \"5004\", \"5001\", \"5001\"\n",
    "          ],\n",
    "          \"impressions\": [\n",
    "              np.NaN, \"5014|5002|5010\", np.NaN, np.NaN,\n",
    "              np.NaN, \"5001|5023|5040|5005\",\n",
    "              np.NaN, \"5010|5001|5023|5004|5002|5008\", \n",
    "                  np.NaN, \"5010|5001|5023|5004|5002|5008\"\n",
    "          ],\n",
    "          \"prices\": [\n",
    "              np.NaN, \"100|125|120\", np.NaN, np.NaN,\n",
    "              np.NaN, \"75|110|65|210\",\n",
    "              np.NaN, \"120|89|140|126|86|110\", np.NaN, \"120|89|140|126|86|110\"\n",
    "          ]\n",
    "      }\n",
    "\n",
    "      df_train = pd.DataFrame(d_train)\n",
    "\n",
    "      return df_train\n",
    "\n",
    "\n",
    "  def test(self):\n",
    "    if self.version=='trivago':\n",
    "      d_test = {\n",
    "          \"user_id\": [\n",
    "              \"64BL89\", \"64BL89\",\n",
    "              \"64BL91F2\", \"64BL91F2\", \"64BL91F2\"\n",
    "          ],\n",
    "          \"session_id\": [\n",
    "              \"3579f90\", \"3579f90\",\n",
    "              \"3779f92\", \"3779f92\", \"3779f92\"\n",
    "          ],\n",
    "          \"timestamp\": [\n",
    "              5, 6,\n",
    "              9, 10, 11\n",
    "          ],\n",
    "          \"step\": [\n",
    "              1, 2,\n",
    "              1, 2, 3\n",
    "          ],\n",
    "          \"action_type\": [\n",
    "              \"interaction item image\", \"clickout item\",\n",
    "              \"interaction item info\", \"clickout item\", \"filter selection\"\n",
    "          ],\n",
    "          \"reference\": [\n",
    "              \"5023\", np.NaN,\n",
    "              \"5010\", np.NaN, \"unknown\"\n",
    "          ],\n",
    "          \"impressions\": [\n",
    "              np.NaN, \"5002|5003|5010|5004|5001|5023\",\n",
    "              np.NaN, \"5001|5004|5010|5014\", np.NaN\n",
    "          ],\n",
    "          \"prices\": [\n",
    "              np.NaN, \"120|75|110|105|89|99\",\n",
    "              np.NaN, \"76|102|115|124\", np.NaN\n",
    "          ]\n",
    "      }\n",
    "      df_test = pd.DataFrame(d_test)\n",
    "\n",
    "      return df_test\n",
    "\n",
    "\n",
    "  def items(self):\n",
    "    if self.version=='trivago':\n",
    "      d_item_meta = {\n",
    "          \"item_id\": [\"5001\", \"5002\", \"5003\", \"5004\"],\n",
    "          \"properties\": [\n",
    "              \"Wifi|Croissant|TV\",\n",
    "              \"Wifi|TV\",\n",
    "              \"Croissant\",\n",
    "              \"Shoe dryer\"\n",
    "          ]\n",
    "      }\n",
    "      df_item_meta =  pd.DataFrame(d_item_meta)\n",
    "\n",
    "      return df_item_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class SequentialMarkov():\n",
    "\n",
    "  \"\"\"Originally published here: https://github.com/maciejkula/spotlight/blob/master/spotlight/datasets/synthetic.py\"\"\"\n",
    "\n",
    "  @staticmethod\n",
    "  def _build_transition_matrix(num_items,\n",
    "                              concentration_parameter,\n",
    "                              random_state,\n",
    "                              atol=0.001):\n",
    "\n",
    "      def _is_doubly_stochastic(matrix, atol):\n",
    "\n",
    "          return (np.all(np.abs(1.0 - matrix.sum(axis=0)) < atol) and\n",
    "                  np.all(np.abs(1.0 - matrix.sum(axis=1)) < atol))\n",
    "\n",
    "      transition_matrix = random_state.dirichlet(\n",
    "          np.repeat(concentration_parameter, num_items),\n",
    "          num_items)\n",
    "\n",
    "      for _ in range(100):\n",
    "\n",
    "          if _is_doubly_stochastic(transition_matrix, atol):\n",
    "              break\n",
    "\n",
    "          transition_matrix /= transition_matrix.sum(axis=0)\n",
    "          transition_matrix /= transition_matrix.sum(1)[:, np.newaxis]\n",
    "\n",
    "      return transition_matrix\n",
    "\n",
    "  @staticmethod\n",
    "  def _generate_sequences(num_steps,\n",
    "                          transition_matrix,\n",
    "                          order,\n",
    "                          random_state):\n",
    "\n",
    "      elements = []\n",
    "\n",
    "      num_states = transition_matrix.shape[0]\n",
    "\n",
    "      transition_matrix = np.cumsum(transition_matrix,\n",
    "                                    axis=1)\n",
    "\n",
    "      rvs = random_state.rand(num_steps)\n",
    "      state = random_state.randint(transition_matrix.shape[0], size=order,\n",
    "                                  dtype=np.int64)\n",
    "\n",
    "      for rv in rvs:\n",
    "\n",
    "          row = transition_matrix[state].mean(axis=0)\n",
    "          new_state = min(num_states - 1,\n",
    "                          np.searchsorted(row, rv))\n",
    "\n",
    "          state[:-1] = state[1:]\n",
    "          state[-1] = new_state\n",
    "\n",
    "          elements.append(new_state)\n",
    "\n",
    "      return np.array(elements, dtype=np.int32)\n",
    "\n",
    "\n",
    "  def generate_sequential(self,\n",
    "                          num_users=100,\n",
    "                          num_items=1000,\n",
    "                          num_interactions=10000,\n",
    "                          concentration_parameter=0.1,\n",
    "                          order=3,\n",
    "                          random_state=None):\n",
    "      \"\"\"\n",
    "      Generate a dataset of user-item interactions where sequential\n",
    "      information matters.\n",
    "      The interactions are generated by a n-th order Markov chain with\n",
    "      a uniform stationary distribution, where transition probabilities\n",
    "      are given by doubly-stochastic transition matrix. For n-th order chains,\n",
    "      transition probabilities are a convex combination of the transition\n",
    "      probabilities of the last n states in the chain.\n",
    "      The transition matrix is sampled from a Dirichlet distribution described\n",
    "      by a constant concentration parameter. Concentration parameters closer\n",
    "      to zero generate more predictable sequences.\n",
    "      Parameters\n",
    "      ----------\n",
    "      num_users: int, optional\n",
    "          number of users in the dataset\n",
    "      num_items: int, optional\n",
    "          number of items (Markov states) in the dataset\n",
    "      num_interactions: int, optional\n",
    "          number of interactions to generate\n",
    "      concentration_parameter: float, optional\n",
    "          Controls how predictable the sequence is. Values\n",
    "          closer to zero give more predictable sequences.\n",
    "      order: int, optional\n",
    "          order of the Markov chain\n",
    "      random_state: numpy.random.RandomState, optional\n",
    "          random state used to generate the data\n",
    "      \"\"\"\n",
    "\n",
    "      if random_state is None:\n",
    "          random_state = np.random.RandomState()\n",
    "\n",
    "      transition_matrix = self._build_transition_matrix(\n",
    "          num_items - 1,\n",
    "          concentration_parameter,\n",
    "          random_state)\n",
    "\n",
    "      user_ids = np.sort(random_state.randint(0,\n",
    "                                              num_users,\n",
    "                                              num_interactions,\n",
    "                                              dtype=np.int32))\n",
    "      item_ids = self._generate_sequences(num_interactions,\n",
    "                                    transition_matrix,\n",
    "                                    order,\n",
    "                                    random_state) + 1\n",
    "      timestamps = np.arange(len(user_ids), dtype=np.int32)\n",
    "      ratings = np.ones(len(user_ids), dtype=np.float32)\n",
    "\n",
    "      _df = pd.DataFrame(list(zip(user_ids, item_ids, ratings, timestamps)),\n",
    "                        columns=['USERIDS','ITEMIDS','RATINGS','TIMESTAMP'])\n",
    "      _df = _df.drop_duplicates()\n",
    "\n",
    "      return _df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-18 08:48:45\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "numpy  : 1.19.5\n",
      "pandas : 1.1.5\n",
      "IPython: 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
