{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.tf.dcn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep and Cross Network (DCN)\n",
    "\n",
    "In real world recommendation systems, we often have large and sparse feature space. So identifying effective feature processes in this setting would often require manual feature engineering or exhaustive search, which is highly inefficient. To tackle this issue, Google Research team has proposed Deep and Cross Network, DCN**.**\n",
    "\n",
    "It starts with an input layer, typically an embedding layer, followed by a cross network containing multiple cross layers that models explicitly feature interactions, and then combines with a deep network that models implicit feature interactions. The deep network is just a traditional multilayer construction. But the core of DCN is really the cross network. It explicitly applies feature crossing at each layer. And the highest polynomial degree increases with layer depth. The figure here shows the deep and cross layer in the mathematical form.\n",
    "\n",
    "There are a couple of ways to combine the cross network and the deep network:\n",
    "\n",
    "- Stack the deep network on top of the cross network.\n",
    "- Place deep & cross networks in parallel.\n",
    "\n",
    "![Untitled](https://github.com/RecoHut-Stanzas/S021355/raw/main/images/img9.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Layer, Input, ReLU\n",
    "from tensorflow.keras.layers import Dense, Embedding, Dropout\n",
    "from tensorflow.keras.regularizers import l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class CrossNetwork(Layer):\n",
    "    def __init__(self, layer_num, reg_w=1e-6, reg_b=1e-6):\n",
    "        \"\"\"CrossNetwork\n",
    "        :param layer_num: A scalar. The depth of cross network\n",
    "        :param reg_w: A scalar. The regularizer of w\n",
    "        :param reg_b: A scalar. The regularizer of b\n",
    "        \"\"\"\n",
    "        super(CrossNetwork, self).__init__()\n",
    "        self.layer_num = layer_num\n",
    "        self.reg_w = reg_w\n",
    "        self.reg_b = reg_b\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        dim = int(input_shape[-1])\n",
    "        self.cross_weights = [\n",
    "            self.add_weight(name='w_' + str(i),\n",
    "                            shape=(dim, 1),\n",
    "                            initializer='random_normal',\n",
    "                            regularizer=l2(self.reg_w),\n",
    "                            trainable=True\n",
    "                            )\n",
    "            for i in range(self.layer_num)]\n",
    "        self.cross_bias = [\n",
    "            self.add_weight(name='b_' + str(i),\n",
    "                            shape=(dim, 1),\n",
    "                            initializer='random_normal',\n",
    "                            regularizer=l2(self.reg_b),\n",
    "                            trainable=True\n",
    "                            )\n",
    "            for i in range(self.layer_num)]\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        x_0 = tf.expand_dims(inputs, axis=2)  # (batch_size, dim, 1)\n",
    "        x_l = x_0  # (None, dim, 1)\n",
    "        for i in range(self.layer_num):\n",
    "            x_l1 = tf.tensordot(x_l, self.cross_weights[i], axes=[1, 0])  # (batch_size, dim, dim)\n",
    "            x_l = tf.matmul(x_0, x_l1) + self.cross_bias[i] + x_l  # (batch_size, dim, 1)\n",
    "        x_l = tf.squeeze(x_l, axis=2)  # (batch_size, dim)\n",
    "        return x_l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DNN(Layer):\n",
    "    def __init__(self, hidden_units, activation='relu', dropout=0.):\n",
    "        \"\"\"Deep Neural Network\n",
    "\t\t:param hidden_units: A list. Neural network hidden units.\n",
    "\t\t:param activation: A string. Activation function of dnn.\n",
    "\t\t:param dropout: A scalar. Dropout number.\n",
    "\t\t\"\"\"\n",
    "        super(DNN, self).__init__()\n",
    "        self.dnn_network = [Dense(units=unit, activation=activation) for unit in hidden_units]\n",
    "        self.dropout = Dropout(dropout)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        x = inputs\n",
    "        for dnn in self.dnn_network:\n",
    "            x = dnn(x)\n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DCN(Model):\n",
    "    def __init__(self, feature_columns, hidden_units, activation='relu',\n",
    "                 dnn_dropout=0., embed_reg=1e-6, cross_w_reg=1e-6, cross_b_reg=1e-6):\n",
    "        \"\"\"\n",
    "        Deep&Cross Network\n",
    "        :param feature_columns: A list. sparse column feature information.\n",
    "        :param hidden_units: A list. Neural network hidden units.\n",
    "        :param activation: A string. Activation function of dnn.\n",
    "        :param dnn_dropout: A scalar. Dropout of dnn.\n",
    "        :param embed_reg: A scalar. The regularizer of embedding.\n",
    "        :param cross_w_reg: A scalar. The regularizer of cross network.\n",
    "        :param cross_b_reg: A scalar. The regularizer of cross network.\n",
    "        \"\"\"\n",
    "        super(DCN, self).__init__()\n",
    "        self.sparse_feature_columns = feature_columns\n",
    "        self.layer_num = len(hidden_units)\n",
    "        self.embed_layers = {\n",
    "            'embed_' + str(i): Embedding(input_dim=feat['feat_num'],\n",
    "                                         input_length=1,\n",
    "                                         output_dim=feat['embed_dim'],\n",
    "                                         embeddings_initializer='random_uniform',\n",
    "                                         embeddings_regularizer=l2(embed_reg))\n",
    "            for i, feat in enumerate(self.sparse_feature_columns)\n",
    "        }\n",
    "        self.cross_network = CrossNetwork(self.layer_num, cross_w_reg, cross_b_reg)\n",
    "        self.dnn_network = DNN(hidden_units, activation, dnn_dropout)\n",
    "        self.dense_final = Dense(1, activation=None)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        sparse_inputs = inputs\n",
    "        sparse_embed = tf.concat([self.embed_layers['embed_{}'.format(i)](sparse_inputs[:, i])\n",
    "                                  for i in range(sparse_inputs.shape[1])], axis=-1)\n",
    "        x = sparse_embed\n",
    "        # Cross Network\n",
    "        cross_x = self.cross_network(x)\n",
    "        # DNN\n",
    "        dnn_x = self.dnn_network(x)\n",
    "        # Concatenate\n",
    "        total_x = tf.concat([cross_x, dnn_x], axis=-1)\n",
    "        outputs = tf.nn.sigmoid(self.dense_final(total_x))\n",
    "        return outputs\n",
    "\n",
    "    def summary(self):\n",
    "        sparse_inputs = Input(shape=(len(self.sparse_feature_columns),), dtype=tf.int32)\n",
    "        Model(inputs=sparse_inputs, outputs=self.call(sparse_inputs)).summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model():\n",
    "    user_features = {'feat': 'user_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    seq_features = {'feat': 'item_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    features = [user_features, seq_features]\n",
    "    model = DCN(features, hidden_units=[8, 4, 2], dnn_dropout=0.5)\n",
    "    model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_3 (InputLayer)           [(None, 2)]          0           []                               \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_4 (Sl  (None,)             0           ['input_3[0][0]']                \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_5 (Sl  (None,)             0           ['input_3[0][0]']                \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " embedding_8 (Embedding)        (None, 8)            800         ['tf.__operators__.getitem_4[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " embedding_9 (Embedding)        (None, 8)            800         ['tf.__operators__.getitem_5[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " tf.concat_1 (TFOpLambda)       (None, 16)           0           ['embedding_8[0][0]',            \n",
      "                                                                  'embedding_9[0][0]']            \n",
      "                                                                                                  \n",
      " cross_network (CrossNetwork)   (None, 16)           96          ['tf.concat_1[0][0]']            \n",
      "                                                                                                  \n",
      " dnn_1 (DNN)                    (None, 2)            182         ['tf.concat_1[0][0]']            \n",
      "                                                                                                  \n",
      " tf.concat_2 (TFOpLambda)       (None, 18)           0           ['cross_network[0][0]',          \n",
      "                                                                  'dnn_1[0][0]']                  \n",
      "                                                                                                  \n",
      " dense_13 (Dense)               (None, 1)            19          ['tf.concat_2[0][0]']            \n",
      "                                                                                                  \n",
      " tf.math.sigmoid_2 (TFOpLambda)  (None, 1)           0           ['dense_13[0][0]']               \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 1,897\n",
      "Trainable params: 1,897\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "test_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-20 11:01:24\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "tensorflow: 2.7.0\n",
      "IPython   : 5.5.0\n",
      "numpy     : 1.19.5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
