{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp transforms.datasets.movielens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MovieLens Dataset Transformation\n",
    "> Implementation of transformation functions specific to movielens datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def sparseFeature(feat, feat_num, embed_dim=4):\n",
    "    \"\"\"\n",
    "    create dictionary for sparse feature\n",
    "    :param feat: feature name\n",
    "    :param feat_num: the total number of sparse features that do not repeat\n",
    "    :param embed_dim: embedding dimension\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    return {'feat': feat, 'feat_num': feat_num, 'embed_dim': embed_dim}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def create_ml_1m_dataset(file, trans_score=2, embed_dim=8, test_neg_num=100):\n",
    "    \"\"\"\n",
    "    :param file: A string. dataset path.\n",
    "    :param trans_score: A scalar. Greater than it is 1, and less than it is 0.\n",
    "    :param embed_dim: A scalar. latent factor.\n",
    "    :param test_neg_num: A scalar. The number of test negative samples\n",
    "    :return: user_num, item_num, train_df, test_df\n",
    "    \"\"\"\n",
    "    print('==========Data Preprocess Start=============')\n",
    "    data_df = pd.read_csv(file, sep=\"::\", engine='python',\n",
    "                          names=['user_id', 'item_id', 'label', 'Timestamp'])\n",
    "    # filtering\n",
    "    data_df['item_count'] = data_df.groupby('item_id')['item_id'].transform('count')\n",
    "    data_df = data_df[data_df.item_count >= 5]\n",
    "    # trans score\n",
    "    data_df = data_df[data_df.label >= trans_score]\n",
    "    # sort\n",
    "    data_df = data_df.sort_values(by=['user_id', 'Timestamp'])\n",
    "    # split dataset and negative sampling\n",
    "    print('============Negative Sampling===============')\n",
    "    train_data, val_data, test_data = defaultdict(list), defaultdict(list), defaultdict(list)\n",
    "    item_id_max = data_df['item_id'].max()\n",
    "    for user_id, df in tqdm(data_df[['user_id', 'item_id']].groupby('user_id')):\n",
    "        pos_list = df['item_id'].tolist()\n",
    "\n",
    "        def gen_neg():\n",
    "            neg = pos_list[0]\n",
    "            while neg in set(pos_list):\n",
    "                neg = random.randint(1, item_id_max)\n",
    "            return neg\n",
    "\n",
    "        neg_list = [gen_neg() for i in range(len(pos_list) + test_neg_num)]\n",
    "        for i in range(1, len(pos_list)):\n",
    "            hist_i = pos_list[:i]\n",
    "            if i == len(pos_list) - 1:\n",
    "                test_data['user_id'].append(user_id)\n",
    "                test_data['pos_id'].append(pos_list[i])\n",
    "                test_data['neg_id'].append(neg_list[i:])\n",
    "            elif i == len(pos_list) - 2:\n",
    "                val_data['user_id'].append(user_id)\n",
    "                val_data['pos_id'].append(pos_list[i])\n",
    "                val_data['neg_id'].append(neg_list[i])\n",
    "            else:\n",
    "                train_data['user_id'].append(user_id)\n",
    "                train_data['pos_id'].append(pos_list[i])\n",
    "                train_data['neg_id'].append(neg_list[i])\n",
    "    # feature columns\n",
    "    user_num, item_num = data_df['user_id'].max() + 1, data_df['item_id'].max() + 1\n",
    "    feat_col = [sparseFeature('user_id', user_num, embed_dim),\n",
    "                sparseFeature('item_id', item_num, embed_dim)]\n",
    "    # shuffle\n",
    "    random.shuffle(train_data)\n",
    "    random.shuffle(val_data)\n",
    "    train = [np.array(train_data['user_id']), np.array(train_data['pos_id']),\n",
    "               np.array(train_data['neg_id'])]\n",
    "    val = [np.array(val_data['user_id']), np.array(val_data['pos_id']),\n",
    "             np.array(val_data['neg_id'])]\n",
    "    test = [np.array(test_data['user_id']), np.array(test_data['pos_id']),\n",
    "              np.array(test_data['neg_id'])]\n",
    "    print('============Data Preprocess End=============')\n",
    "    return feat_col, train, val, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def create_implicit_ml_1m_dataset(file, trans_score=2, embed_dim=8, maxlen=40):\n",
    "    \"\"\"\n",
    "    :param file: A string. dataset path.\n",
    "    :param trans_score: A scalar. Greater than it is 1, and less than it is 0.\n",
    "    :param embed_dim: A scalar. latent factor.\n",
    "    :param maxlen: A scalar. maxlen.\n",
    "    :return: user_num, item_num, train_df, test_df\n",
    "    \"\"\"\n",
    "    print('==========Data Preprocess Start=============')\n",
    "    data_df = pd.read_csv(file, sep=\"::\", engine='python',\n",
    "                          names=['user_id', 'item_id', 'label', 'Timestamp'])\n",
    "    # implicit dataset\n",
    "    data_df = data_df[data_df.label >= trans_score]\n",
    "\n",
    "    # sort\n",
    "    data_df = data_df.sort_values(by=['user_id', 'Timestamp'])\n",
    "\n",
    "    train_data, val_data, test_data = [], [], []\n",
    "\n",
    "    item_id_max = data_df['item_id'].max()\n",
    "    for user_id, df in tqdm(data_df[['user_id', 'item_id']].groupby('user_id')):\n",
    "        pos_list = df['item_id'].tolist()\n",
    "\n",
    "        def gen_neg():\n",
    "            neg = pos_list[0]\n",
    "            while neg in pos_list:\n",
    "                neg = random.randint(1, item_id_max)\n",
    "            return neg\n",
    "\n",
    "        neg_list = [gen_neg() for i in range(len(pos_list) + 100)]\n",
    "        for i in range(1, len(pos_list)):\n",
    "            hist_i = pos_list[:i]\n",
    "            if i == len(pos_list) - 1:\n",
    "                test_data.append([user_id, hist_i, pos_list[i], 1])\n",
    "                for neg in neg_list[i:]:\n",
    "                    test_data.append([user_id, hist_i, neg, 0])\n",
    "            elif i == len(pos_list) - 2:\n",
    "                val_data.append([user_id, hist_i, pos_list[i], 1])\n",
    "                val_data.append([user_id, hist_i, neg_list[i], 0])\n",
    "            else:\n",
    "                train_data.append([user_id, hist_i, pos_list[i], 1])\n",
    "                train_data.append([user_id, hist_i, neg_list[i], 0])\n",
    "    # item feature columns\n",
    "    user_num, item_num = data_df['user_id'].max() + 1, data_df['item_id'].max() + 1\n",
    "    feature_columns = [sparseFeature('user_id', user_num, embed_dim),\n",
    "                       sparseFeature('item_id', item_num, embed_dim)]\n",
    "\n",
    "    # shuffle\n",
    "    random.shuffle(train_data)\n",
    "    random.shuffle(val_data)\n",
    "    # random.shuffle(test_data)\n",
    "\n",
    "    # create dataframe\n",
    "    train = pd.DataFrame(train_data, columns=['user_id', 'hist', 'target_item', 'label'])\n",
    "    val = pd.DataFrame(val_data, columns=['user_id', 'hist', 'target_item', 'label'])\n",
    "    test = pd.DataFrame(test_data, columns=['user_id', 'hist', 'target_item', 'label'])\n",
    "\n",
    "    print('==================Padding===================')\n",
    "    from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "    train_X = [train['user_id'].values, pad_sequences(train['hist'], maxlen=maxlen), train['target_item'].values]\n",
    "    train_y = train['label'].values\n",
    "    val_X = [val['user_id'].values, pad_sequences(val['hist'], maxlen=maxlen), val['target_item'].values]\n",
    "    val_y = val['label'].values\n",
    "    test_X = [test['user_id'].values, pad_sequences(test['hist'], maxlen=maxlen), test['target_item'].values]\n",
    "    test_y = test['label'].values.tolist()\n",
    "    print('============Data Preprocess End=============')\n",
    "    return feature_columns, (train_X, train_y), (val_X, val_y), (test_X, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ml-1m.zip           100%[===================>]   5.64M  3.71MB/s    in 1.5s    \n",
      "Archive:  ml-1m.zip\n",
      "   creating: ml-1m/\n",
      "  inflating: ml-1m/movies.dat        \n",
      "  inflating: ml-1m/ratings.dat       \n",
      "  inflating: ml-1m/README            \n",
      "  inflating: ml-1m/users.dat         \n"
     ]
    }
   ],
   "source": [
    "!wget -q --show-progress https://files.grouplens.org/datasets/movielens/ml-1m.zip\n",
    "!unzip ml-1m.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'ml-1m/ratings.dat'\n",
    "test_neg_num = 100\n",
    "embed_dim = 64\n",
    "trans_score = 1\n",
    "maxlen = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========Data Preprocess Start=============\n",
      "============Negative Sampling===============\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6040/6040 [00:44<00:00, 134.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============Data Preprocess End=============\n"
     ]
    }
   ],
   "source": [
    "feature_columns, train, val, test = create_ml_1m_dataset(file, trans_score, embed_dim, test_neg_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'embed_dim': 64, 'feat': 'user_id', 'feat_num': 6041},\n",
       " {'embed_dim': 64, 'feat': 'item_id', 'feat_num': 3953}]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([   1,    1,    1, ..., 6040, 6040, 6040]),\n",
       " array([1270, 1721, 1022, ..., 2917, 1921, 1784]),\n",
       " array([2152, 1229, 3617, ..., 2960, 3686, 1569])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([   1,    2,    3, ..., 6038, 6039, 6040]),\n",
       " array([1907, 1544, 3868, ..., 2700, 1204,  161]),\n",
       " array([3132, 1812,  271, ..., 1697, 2718, 3572])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([   1,    2,    3, ..., 6038, 6039, 6040]),\n",
       " array([  48, 1917, 2081, ..., 1183, 1254, 1221]),\n",
       " array([[ 426, 1915, 2201, ..., 1687, 2916, 1266],\n",
       "        [3294, 2362,  167, ..., 1322, 2715, 3013],\n",
       "        [2973, 3000, 1832, ...,  514, 2845, 1901],\n",
       "        ...,\n",
       "        [1258,  335, 3638, ..., 3582, 2221,  763],\n",
       "        [1767, 2924,  691, ..., 1624, 2493,  371],\n",
       "        [1106, 3048, 1940, ..., 3520, 2102, 2275]])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========Data Preprocess Start=============\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6040/6040 [00:35<00:00, 170.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================Padding===================\n",
      "============Data Preprocess End=============\n"
     ]
    }
   ],
   "source": [
    "feature_columns, train, val, test = create_implicit_ml_1m_dataset(file, trans_score, embed_dim, maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'embed_dim': 64, 'feat': 'user_id', 'feat_num': 6041},\n",
       " {'embed_dim': 64, 'feat': 'item_id', 'feat_num': 3953}]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([array([5534, 3031, 1764, ..., 3159, 2137, 3129]),\n",
       "  array([[   0,    0,    0, ...,  349, 1356, 1580],\n",
       "         [   0,    0,    0, ..., 3255, 2108,  507],\n",
       "         [2322, 3316,    9, ..., 2502, 1476, 2759],\n",
       "         ...,\n",
       "         [   0,    0,    0, ..., 1258, 1240, 1270],\n",
       "         [   0,    0,    0, ..., 2038, 1831,   24],\n",
       "         [2379, 3846, 3041, ..., 2391,  866, 3476]], dtype=int32),\n",
       "  array([1372, 2309, 3052, ..., 1285, 2668, 2143])],\n",
       " array([1, 0, 1, ..., 1, 1, 0]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([array([3468, 1903, 1902, ..., 5215, 2977, 3597]),\n",
       "  array([[   0,    0,    0, ..., 3114,  593, 2345],\n",
       "         [   0,    0,    0, ..., 1201, 3671, 3681],\n",
       "         [1754,   44,  247, ..., 1092, 3005, 2605],\n",
       "         ...,\n",
       "         [   0,    0,    0, ..., 1252,  720,  745],\n",
       "         [   0,    0,    0, ..., 2581, 2724, 2763],\n",
       "         [   0,    0,    0, ..., 2096, 2137, 1032]], dtype=int32),\n",
       "  array([3951, 1202,  832, ..., 3177,  476, 1029])],\n",
       " array([0, 0, 1, ..., 0, 0, 1]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-20 12:31:12\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "numpy     : 1.19.5\n",
      "tensorflow: 2.7.0\n",
      "pandas    : 1.1.5\n",
      "IPython   : 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
