{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.tf.deepcross"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Crossing\n",
    "\n",
    "The input of Deep Crossing is a set of individual features that can be either dense or sparse. The important crossing features are discovered implicitly by the networks, which are comprised of an embedding and stacking layer, as well as a cascade of Residual Units.\n",
    "\n",
    "![Untitled](https://github.com/RecoHut-Stanzas/S021355/raw/main/images/img7.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Layer, Input, ReLU\n",
    "from tensorflow.keras.layers import Dense, Embedding, Dropout\n",
    "from tensorflow.keras.regularizers import l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Residual_Units(Layer):\n",
    "    \"\"\"\n",
    "    Residual Units\n",
    "    \"\"\"\n",
    "    def __init__(self, hidden_unit, dim_stack):\n",
    "        \"\"\"\n",
    "        :param hidden_unit: A list. Neural network hidden units.\n",
    "        :param dim_stack: A scalar. The dimension of inputs unit.\n",
    "        \"\"\"\n",
    "        super(Residual_Units, self).__init__()\n",
    "        self.layer1 = Dense(units=hidden_unit, activation='relu')\n",
    "        self.layer2 = Dense(units=dim_stack, activation=None)\n",
    "        self.relu = ReLU()\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        x = inputs\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        outputs = self.relu(x + inputs)\n",
    "        return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class DeepCross(Model):\n",
    "    def __init__(self, feature_columns, hidden_units, res_dropout=0., embed_reg=1e-6):\n",
    "        \"\"\"\n",
    "        Deep&Crossing\n",
    "        :param feature_columns: A list. sparse column feature information.\n",
    "        :param hidden_units: A list. Neural network hidden units.\n",
    "        :param res_dropout: A scalar. Dropout of resnet.\n",
    "        :param embed_reg: A scalar. The regularizer of embedding.\n",
    "        \"\"\"\n",
    "        super(DeepCross, self).__init__()\n",
    "        self.sparse_feature_columns = feature_columns\n",
    "        self.embed_layers = {\n",
    "            'embed_' + str(i): Embedding(input_dim=feat['feat_num'],\n",
    "                                         input_length=1,\n",
    "                                         output_dim=feat['embed_dim'],\n",
    "                                         embeddings_initializer='random_uniform',\n",
    "                                         embeddings_regularizer=l2(embed_reg))\n",
    "            for i, feat in enumerate(self.sparse_feature_columns)\n",
    "        }\n",
    "        # the total length of embedding layers\n",
    "        embed_layers_len = sum([feat['embed_dim'] for feat in self.sparse_feature_columns])\n",
    "        self.res_network = [Residual_Units(unit, embed_layers_len) for unit in hidden_units]\n",
    "        self.res_dropout = Dropout(res_dropout)\n",
    "        self.dense = Dense(1, activation=None)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        sparse_inputs = inputs\n",
    "        sparse_embed = tf.concat([self.embed_layers['embed_{}'.format(i)](sparse_inputs[:, i])\n",
    "                                  for i in range(sparse_inputs.shape[1])], axis=-1)\n",
    "        r = sparse_embed\n",
    "        for res in self.res_network:\n",
    "            r = res(r)\n",
    "        r = self.res_dropout(r)\n",
    "        outputs = tf.nn.sigmoid(self.dense(r))\n",
    "        return outputs\n",
    "\n",
    "    def summary(self):\n",
    "        sparse_inputs = Input(shape=(len(self.sparse_feature_columns),), dtype=tf.int32)\n",
    "        Model(inputs=sparse_inputs, outputs=self.call(sparse_inputs)).summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model():\n",
    "    user_features = {'feat': 'user_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    seq_features = {'feat': 'item_id', 'feat_num': 100, 'embed_dim': 8}\n",
    "    features = [user_features, seq_features]\n",
    "    model = DeepCross(features, hidden_units=[8, 4, 2])\n",
    "    model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, 2)]          0           []                               \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem (Slic  (None,)             0           ['input_1[0][0]']                \n",
      " ingOpLambda)                                                                                     \n",
      "                                                                                                  \n",
      " tf.__operators__.getitem_1 (Sl  (None,)             0           ['input_1[0][0]']                \n",
      " icingOpLambda)                                                                                   \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, 8)            800         ['tf.__operators__.getitem[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " embedding_1 (Embedding)        (None, 8)            800         ['tf.__operators__.getitem_1[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " tf.concat (TFOpLambda)         (None, 16)           0           ['embedding[0][0]',              \n",
      "                                                                  'embedding_1[0][0]']            \n",
      "                                                                                                  \n",
      " residual__units (Residual_Unit  (None, 16)          280         ['tf.concat[0][0]']              \n",
      " s)                                                                                               \n",
      "                                                                                                  \n",
      " residual__units_1 (Residual_Un  (None, 16)          148         ['residual__units[0][0]']        \n",
      " its)                                                                                             \n",
      "                                                                                                  \n",
      " residual__units_2 (Residual_Un  (None, 16)          82          ['residual__units_1[0][0]']      \n",
      " its)                                                                                             \n",
      "                                                                                                  \n",
      " dropout (Dropout)              (None, 16)           0           ['residual__units_2[0][0]']      \n",
      "                                                                                                  \n",
      " dense_6 (Dense)                (None, 1)            17          ['dropout[0][0]']                \n",
      "                                                                                                  \n",
      " tf.math.sigmoid (TFOpLambda)   (None, 1)            0           ['dense_6[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 2,127\n",
      "Trainable params: 2,127\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "test_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-20 10:45:29\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "tensorflow: 2.7.0\n",
      "numpy     : 1.19.5\n",
      "IPython   : 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
