{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.tf.pnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Product-based Neural Network (PNN)\n",
    "\n",
    "PNN uses an embedding layer to learn a distributed representation of the categorical data, a product layer to capture interactive patterns between inter-field categories, and further fully connected layers to explore high-order feature interactions.\n",
    "\n",
    "![Untitled](https://github.com/RecoHut-Stanzas/S021355/raw/main/images/img8.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Layer, Input, ReLU\n",
    "from tensorflow.keras.layers import Dense, Embedding, Dropout\n",
    "from tensorflow.keras.regularizers import l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DNN(Layer):\n",
    "    def __init__(self, hidden_units, activation='relu', dropout=0.):\n",
    "        \"\"\"Deep Neural Network\n",
    "\t\t:param hidden_units: A list. Neural network hidden units.\n",
    "\t\t:param activation: A string. Activation function of dnn.\n",
    "\t\t:param dropout: A scalar. Dropout number.\n",
    "\t\t\"\"\"\n",
    "        super(DNN, self).__init__()\n",
    "        self.dnn_network = [Dense(units=unit, activation=activation) for unit in hidden_units]\n",
    "        self.dropout = Dropout(dropout)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        x = inputs\n",
    "        for dnn in self.dnn_network:\n",
    "            x = dnn(x)\n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class PNN(Model):\n",
    "    def __init__(self, feature_columns, hidden_units, mode='in', dnn_dropout=0.,\n",
    "                 activation='relu', embed_reg=1e-6, w_z_reg=1e-6, w_p_reg=1e-6, l_b_reg=1e-6):\n",
    "        \"\"\"\n",
    "        Product-based Neural Networks\n",
    "        :param feature_columns: A list. sparse column feature information.\n",
    "        :param hidden_units: A list. Neural network hidden units.\n",
    "        :param mode: A string. 'in' IPNN or 'out'OPNN.\n",
    "        :param activation: A string. Activation function of dnn.\n",
    "        :param dnn_dropout: A scalar. Dropout of dnn.\n",
    "        :param embed_reg: A scalar. The regularizer of embedding.\n",
    "        :param w_z_reg: A scalar. The regularizer of w_z_ in product layer\n",
    "        :param w_p_reg: A scalar. The regularizer of w_p in product layer\n",
    "        :param l_b_reg: A scalar. The regularizer of l_b in product layer\n",
    "        \"\"\"\n",
    "        super(PNN, self).__init__()\n",
    "        # inner product or outer product\n",
    "        self.mode = mode\n",
    "        self.sparse_feature_columns = feature_columns\n",
    "        # the number of feature fields\n",
    "        self.field_num = len(self.sparse_feature_columns)\n",
    "        self.embed_dim = self.sparse_feature_columns[0]['embed_dim']\n",
    "        # The embedding dimension of each feature field must be the same\n",
    "        self.embed_layers = {\n",
    "            'embed_' + str(i): Embedding(input_dim=feat['feat_num'],\n",
    "                                         input_length=1,\n",
    "                                         output_dim=feat['embed_dim'],\n",
    "                                         embeddings_initializer='random_uniform',\n",
    "                                         embeddings_regularizer=l2(embed_reg))\n",
    "            for i, feat in enumerate(self.sparse_feature_columns)\n",
    "        }\n",
    "        # parameters\n",
    "        self.w_z = self.add_weight(name='w_z',\n",
    "                                   shape=(self.field_num, self.embed_dim, hidden_units[0]),\n",
    "                                   initializer='random_uniform',\n",
    "                                   regularizer=l2(w_z_reg),\n",
    "                                   trainable=True\n",
    "                                   )\n",
    "        if mode == 'in':\n",
    "            self.w_p = self.add_weight(name='w_p',\n",
    "                                       shape=(self.field_num * (self.field_num - 1) // 2, self.embed_dim,\n",
    "                                              hidden_units[0]),\n",
    "                                       initializer='random_uniform',\n",
    "                                       regularizer=l2(w_p_reg),\n",
    "                                       trainable=True)\n",
    "        # out\n",
    "        else:\n",
    "            self.w_p = self.add_weight(name='w_p',\n",
    "                                       shape=(self.field_num * (self.field_num - 1) // 2, self.embed_dim,\n",
    "                                              self.embed_dim, hidden_units[0]),\n",
    "                                       initializer='random_uniform',\n",
    "                                       regularizer=l2(w_p_reg),\n",
    "                                       trainable=True)\n",
    "        self.l_b = self.add_weight(name='l_b', shape=(hidden_units[0], ),\n",
    "                                   initializer='random_uniform',\n",
    "                                   regularizer=l2(l_b_reg),\n",
    "                                   trainable=True)\n",
    "        # dnn\n",
    "        self.dnn_network = DNN(hidden_units[1:], activation, dnn_dropout)\n",
    "        self.dense_final = Dense(1)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        sparse_inputs = inputs\n",
    "        sparse_embed = [self.embed_layers['embed_{}'.format(i)](sparse_inputs[:, i])\n",
    "                 for i in range(sparse_inputs.shape[1])]\n",
    "        sparse_embed = tf.transpose(tf.convert_to_tensor(sparse_embed), [1, 0, 2])  # (None, field_num, embed_dim)\n",
    "        # product layer\n",
    "        row = []\n",
    "        col = []\n",
    "        for i in range(len(self.sparse_feature_columns) - 1):\n",
    "            for j in range(i + 1, len(self.sparse_feature_columns)):\n",
    "                row.append(i)\n",
    "                col.append(j)\n",
    "        p = tf.gather(sparse_embed, row, axis=1)\n",
    "        q = tf.gather(sparse_embed, col, axis=1)\n",
    "        if self.mode == 'in':\n",
    "            l_p = tf.tensordot(p*q, self.w_p, axes=2)  # (None, hidden[0])\n",
    "        else:  # out\n",
    "            u = tf.expand_dims(q, 2)  # (None, field_num(field_num-1)/2, 1, emb_dim)\n",
    "            v = tf.expand_dims(p, 2)  # (None, field_num(field_num-1)/2, 1, emb_dim)\n",
    "            l_p = tf.tensordot(tf.matmul(tf.transpose(u, [0, 1, 3, 2]), v), self.w_p, axes=3)  # (None, hidden[0])\n",
    "\n",
    "        l_z = tf.tensordot(sparse_embed, self.w_z, axes=2)  # (None, hidden[0])\n",
    "        l_1 = tf.nn.relu(tf.concat([l_z + l_p + self.l_b], axis=-1))\n",
    "        # dnn layer\n",
    "        dnn_x = self.dnn_network(l_1)\n",
    "        outputs = tf.nn.sigmoid(self.dense_final(dnn_x))\n",
    "        return outputs\n",
    "\n",
    "    def summary(self):\n",
    "        sparse_inputs = Input(shape=(len(self.sparse_feature_columns),), dtype=tf.int32)\n",
    "        Model(inputs=sparse_inputs, outputs=self.call(sparse_inputs)).summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model():\n",
    "    user_features = {'feat': 'user_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    seq_features = {'feat': 'item_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    features = [user_features, seq_features]\n",
    "    model = PNN(features, hidden_units=[8, 4, 2], dnn_dropout=0.5)\n",
    "    model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The following Variables were used in a Lambda layer's call (tf.tensordot), but are not present in its tracked objects:   <tf.Variable 'w_p:0' shape=(1, 8, 8) dtype=float32>. This is a strong indication that the Lambda layer should be rewritten as a subclassed Layer.\n",
      "WARNING:tensorflow:The following Variables were used in a Lambda layer's call (tf.tensordot_1), but are not present in its tracked objects:   <tf.Variable 'w_z:0' shape=(2, 8, 8) dtype=float32>. This is a strong indication that the Lambda layer should be rewritten as a subclassed Layer.\n",
      "WARNING:tensorflow:The following Variables were used in a Lambda layer's call (tf.__operators__.add_1), but are not present in its tracked objects:   <tf.Variable 'l_b:0' shape=(8,) dtype=float32>. This is a strong indication that the Lambda layer should be rewritten as a subclassed Layer.\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_2 (InputLayer)           [(None, 2)]          0           []                               \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_2 (Sl  (None,)             0           ['input_2[0][0]']                \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_3 (Sl  (None,)             0           ['input_2[0][0]']                \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " embedding_6 (Embedding)        (None, 8)            800         ['tf.__operators__.getitem_2[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " embedding_7 (Embedding)        (None, 8)            800         ['tf.__operators__.getitem_3[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " tf.convert_to_tensor (TFOpLamb  (2, None, 8)        0           ['embedding_6[0][0]',            \n",
      " da)                                                              'embedding_7[0][0]']            \n",
      "                                                                                                  \n",
      " tf.compat.v1.transpose (TFOpLa  (None, 2, 8)        0           ['tf.convert_to_tensor[0][0]']   \n",
      " mbda)                                                                                            \n",
      "                                                                                                  \n",
      " tf.compat.v1.gather (TFOpLambd  (None, 1, 8)        0           ['tf.compat.v1.transpose[0][0]'] \n",
      " a)                                                                                               \n",
      "                                                                                                  \n",
      " tf.compat.v1.gather_1 (TFOpLam  (None, 1, 8)        0           ['tf.compat.v1.transpose[0][0]'] \n",
      " bda)                                                                                             \n",
      "                                                                                                  \n",
      " tf.math.multiply (TFOpLambda)  (None, 1, 8)         0           ['tf.compat.v1.gather[0][0]',    \n",
      "                                                                  'tf.compat.v1.gather_1[0][0]']  \n",
      "                                                                                                  \n",
      " tf.tensordot_1 (TFOpLambda)    (None, 8)            0           ['tf.compat.v1.transpose[0][0]'] \n",
      "                                                                                                  \n",
      " tf.tensordot (TFOpLambda)      (None, 8)            0           ['tf.math.multiply[0][0]']       \n",
      "                                                                                                  \n",
      " tf.__operators__.add (TFOpLamb  (None, 8)           0           ['tf.tensordot_1[0][0]',         \n",
      " da)                                                              'tf.tensordot[0][0]']           \n",
      "                                                                                                  \n",
      " tf.__operators__.add_1 (TFOpLa  (None, 8)           0           ['tf.__operators__.add[0][0]']   \n",
      " mbda)                                                                                            \n",
      "                                                                                                  \n",
      " tf.identity (TFOpLambda)       (None, 8)            0           ['tf.__operators__.add_1[0][0]'] \n",
      "                                                                                                  \n",
      " tf.nn.relu (TFOpLambda)        (None, 8)            0           ['tf.identity[0][0]']            \n",
      "                                                                                                  \n",
      " dnn (DNN)                      (None, 2)            46          ['tf.nn.relu[0][0]']             \n",
      "                                                                                                  \n",
      " dense_9 (Dense)                (None, 1)            3           ['dnn[0][0]']                    \n",
      "                                                                                                  \n",
      " tf.math.sigmoid_1 (TFOpLambda)  (None, 1)           0           ['dense_9[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 1,649\n",
      "Trainable params: 1,649\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "test_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-20 10:57:24\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "tensorflow: 2.7.0\n",
      "IPython   : 5.5.0\n",
      "numpy     : 1.19.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
