{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# QUICK MODEL BUILD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMPORT LIB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from slik_wrangler import pipeline as ppl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from slik_wrangler import preprocessing as pp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlflow\n",
    "import catboost\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from slik_wrangler import loadfile as lf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from slik_wrangler import utils as ut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CSV file read sucessfully\n",
      "\n",
      "Data has 27564 rows and 23 columns\n"
     ]
    }
   ],
   "source": [
    "train = lf.read_file('Installs and Download.csv',nrows=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = train[train.event_type!='af_complete_registration']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Selection\n",
    "Choose an Algorithm of your choice.\n",
    "\n",
    "Pass a dict of params if you want. The dictionary keys should either be model, fit or hyperparameters.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'model':{\n",
    "        'random_state':0, \n",
    "        'max_iter':1000 \n",
    "    }}\n",
    "alg = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "params = {\n",
    "    'model':{\n",
    "        'random_state':42, \n",
    "        'n_estimators':200 \n",
    "    }}\n",
    "alg = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "      'model': {\n",
    "          'iterations':2000, \n",
    "          \"depth\" :6, \n",
    "          \"learning_rate\":0.1, \n",
    "          \"l2_leaf_reg\": 10, \n",
    "          \"loss_function\":'Logloss',\n",
    "          \"eval_metric\":'AUC'\n",
    "      },\n",
    "\n",
    "      'fit': {\n",
    "        'early_stopping_rounds': 6,\n",
    "        'logging_level': 'Silent',         \n",
    "      },\n",
    "    \n",
    "        'hyperparameters':{'C': [0.1, 1.0, 10, 100],\n",
    "               'solver': ['liblinear','newton-cg']\n",
    "                          }\n",
    "    }\n",
    "\n",
    "model_params = params['model']\n",
    "# fit_params = params['fit']\n",
    "\n",
    "alg = catboost.CatBoostClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "param= {'C': [0.1, 1.0, 10, 100],\n",
    "               'solver': ['liblinear','newton-cg']}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BUILDING PIPELINE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basic understanding of pipelines\n",
    "\n",
    "Build steps for numerical and categorical transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "# Applying SimpleImputer and will search for different scalers using GridSearchCV\n",
    "# numerical_transformer = Pipeline(steps=[\n",
    "#     ('imputer', SimpleImputer(strategy='median')),\n",
    "#     ('scaler', StandardScaler())])\n",
    "numerical_transformer = Pipeline(steps=[('scaler', StandardScaler())])\n",
    "# categorical_transformer= Pipeline(steps=[('imputer', SimpleImputer(strategy='constant', fill_value='Missing'))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying SimpleImputer and then OneHotEncoder\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='constant', fill_value='Missing')),\n",
    "    ('onehot', OneHotEncoder(handle_unknown='ignore',sparse = False))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------- Loading clean data ---------------\n",
      "\n",
      "\n",
      "Clean data exists in ./InstallDownload/data/train_data.pkl\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use clean data:(y/n) \n",
      " n\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------- Identifying columns present in the data ---------------\n",
      "\n",
      "Target column is transformed_event_type. Attribute in target column incldes:\n",
      "[0, 1]\n",
      "\n",
      "Features with high cardinality:['device_manufacturer', 'device_model']\n",
      "\n",
      "\n",
      "--------------- Saving Attributes in Project path ---------------\n",
      "\n",
      "\n",
      "Data columns successfully identified and attributes are stored in InstallDownload/data/metadata\n",
      "\n",
      "\n",
      "--------------- Metric Performance ---------------\n",
      "\n",
      "\n",
      "Metric performance on test data\n",
      "{'accuracy': 0.8269689737470167, 'precision': 0.125, 'recall': 0.012345679012345678, 'f1': 0.02247191011235955}\n",
      "\n",
      "\n",
      "\n",
      "confusion matrix\n",
      "[[2074   35]\n",
      " [ 400    5]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>div.sk-top-container {color: black;background-color: white;}div.sk-toggleable {background-color: white;}label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}div.sk-estimator:hover {background-color: #d4ebff;}div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}div.sk-item {z-index: 1;}div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}div.sk-parallel-item:only-child::after {width: 0;}div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}div.sk-label-container {position: relative;z-index: 2;text-align: center;}div.sk-container {display: inline-block;position: relative;}</style><div class=\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5a456797-6b3d-4483-a993-4b821f664936\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5a456797-6b3d-4483-a993-4b821f664936\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('preprocessor',\n",
       "                 Pipeline(steps=[('data_transformer',\n",
       "                                  ColumnTransformer(transformers=[('numerical',\n",
       "                                                                   Pipeline(steps=[('scaler',\n",
       "                                                                                    StandardScaler())]),\n",
       "                                                                   ['sms_cost',\n",
       "                                                                    'spend_voice',\n",
       "                                                                    'spend_data',\n",
       "                                                                    'customer_class',\n",
       "                                                                    'age',\n",
       "                                                                    'transformed_gender']),\n",
       "                                                                  ('categorical',\n",
       "                                                                   Pipeline(steps=[('imputer',\n",
       "                                                                                    SimpleImputer(fill_value='Missing',\n",
       "                                                                                                  strategy='constant')),\n",
       "                                                                                   ('onehot',\n",
       "                                                                                    OneHotEncoder(handle_unknown='ignore',\n",
       "                                                                                                  sparse=False))]),\n",
       "                                                                   ['location_region',\n",
       "                                                                    'location_state',\n",
       "                                                                    'device_manufacturer',\n",
       "                                                                    'device_model',\n",
       "                                                                    'binned_age'])]))])),\n",
       "                ('model',\n",
       "                 <catboost.core.CatBoostClassifier object at 0x7ff249a6d9d0>)])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6d0a6dd5-3085-4b62-9fc2-54a8c91400f6\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6d0a6dd5-3085-4b62-9fc2-54a8c91400f6\">preprocessor: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('data_transformer',\n",
       "                 ColumnTransformer(transformers=[('numerical',\n",
       "                                                  Pipeline(steps=[('scaler',\n",
       "                                                                   StandardScaler())]),\n",
       "                                                  ['sms_cost', 'spend_voice',\n",
       "                                                   'spend_data',\n",
       "                                                   'customer_class', 'age',\n",
       "                                                   'transformed_gender']),\n",
       "                                                 ('categorical',\n",
       "                                                  Pipeline(steps=[('imputer',\n",
       "                                                                   SimpleImputer(fill_value='Missing',\n",
       "                                                                                 strategy='constant')),\n",
       "                                                                  ('onehot',\n",
       "                                                                   OneHotEncoder(handle_unknown='ignore',\n",
       "                                                                                 sparse=False))]),\n",
       "                                                  ['location_region',\n",
       "                                                   'location_state',\n",
       "                                                   'device_manufacturer',\n",
       "                                                   'device_model',\n",
       "                                                   'binned_age'])]))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"0f38dfac-8b14-47a2-8cfb-b3e4db68e9dd\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"0f38dfac-8b14-47a2-8cfb-b3e4db68e9dd\">data_transformer: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('numerical',\n",
       "                                 Pipeline(steps=[('scaler', StandardScaler())]),\n",
       "                                 ['sms_cost', 'spend_voice', 'spend_data',\n",
       "                                  'customer_class', 'age',\n",
       "                                  'transformed_gender']),\n",
       "                                ('categorical',\n",
       "                                 Pipeline(steps=[('imputer',\n",
       "                                                  SimpleImputer(fill_value='Missing',\n",
       "                                                                strategy='constant')),\n",
       "                                                 ('onehot',\n",
       "                                                  OneHotEncoder(handle_unknown='ignore',\n",
       "                                                                sparse=False))]),\n",
       "                                 ['location_region', 'location_state',\n",
       "                                  'device_manufacturer', 'device_model',\n",
       "                                  'binned_age'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"16cd6bac-fd95-423b-b9ef-fd5776fee76f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"16cd6bac-fd95-423b-b9ef-fd5776fee76f\">numerical</label><div class=\"sk-toggleable__content\"><pre>['sms_cost', 'spend_voice', 'spend_data', 'customer_class', 'age', 'transformed_gender']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"bd34160a-ee77-460a-90e1-bd4eb65d5d88\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"bd34160a-ee77-460a-90e1-bd4eb65d5d88\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"44d45f13-9637-4c90-9113-6df93be05310\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"44d45f13-9637-4c90-9113-6df93be05310\">categorical</label><div class=\"sk-toggleable__content\"><pre>['location_region', 'location_state', 'device_manufacturer', 'device_model', 'binned_age']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"6dc1c054-a8df-4d2b-9901-bcd683f1196b\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"6dc1c054-a8df-4d2b-9901-bcd683f1196b\">SimpleImputer</label><div class=\"sk-toggleable__content\"><pre>SimpleImputer(fill_value='Missing', strategy='constant')</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"de6f5c4d-84e1-4220-a171-52052418bd5a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"de6f5c4d-84e1-4220-a171-52052418bd5a\">OneHotEncoder</label><div class=\"sk-toggleable__content\"><pre>OneHotEncoder(handle_unknown='ignore', sparse=False)</pre></div></div></div></div></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3daaa792-41d1-496f-9150-82c856382b9e\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3daaa792-41d1-496f-9150-82c856382b9e\">CatBoostClassifier</label><div class=\"sk-toggleable__content\"><pre><catboost.core.CatBoostClassifier object at 0x7ff249a6d9d0></pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('preprocessor',\n",
       "                 Pipeline(steps=[('data_transformer',\n",
       "                                  ColumnTransformer(transformers=[('numerical',\n",
       "                                                                   Pipeline(steps=[('scaler',\n",
       "                                                                                    StandardScaler())]),\n",
       "                                                                   ['sms_cost',\n",
       "                                                                    'spend_voice',\n",
       "                                                                    'spend_data',\n",
       "                                                                    'customer_class',\n",
       "                                                                    'age',\n",
       "                                                                    'transformed_gender']),\n",
       "                                                                  ('categorical',\n",
       "                                                                   Pipeline(steps=[('imputer',\n",
       "                                                                                    SimpleImputer(fill_value='Missing',\n",
       "                                                                                                  strategy='constant')),\n",
       "                                                                                   ('onehot',\n",
       "                                                                                    OneHotEncoder(handle_unknown='ignore',\n",
       "                                                                                                  sparse=False))]),\n",
       "                                                                   ['location_region',\n",
       "                                                                    'location_state',\n",
       "                                                                    'device_manufacturer',\n",
       "                                                                    'device_model',\n",
       "                                                                    'binned_age'])]))])),\n",
       "                ('model',\n",
       "                 <catboost.core.CatBoostClassifier object at 0x7ff249a6d9d0>)])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppl.build_data_pipeline(new_df,clean_data=True,target_column='event_type',id_column='msisdn',display_inline=False,project_path='./InstallDownload/',\\\n",
    "                        numerical_transformer = numerical_transformer, categorical_transformer = categorical_transformer,\\\n",
    "                        pca=False,algorithm=alg,select_columns=['msisdn','sms_cost', 'spend_voice', 'spend_data', 'customer_class', \\\n",
    "                                                                'age', 'gender','location_region','location_state','device_manufacturer',\\\n",
    "                                                                'device_model','event_type'],\\\n",
    "                              params=params,hashing=False,nrows=7000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msisdn</th>\n",
       "      <th>timeline</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>event_type</th>\n",
       "      <th>ad_text</th>\n",
       "      <th>spend_total</th>\n",
       "      <th>spend_vas</th>\n",
       "      <th>sms_cost</th>\n",
       "      <th>spend_voice</th>\n",
       "      <th>spend_data</th>\n",
       "      <th>...</th>\n",
       "      <th>timestampIs_month_end</th>\n",
       "      <th>timestampIs_month_start</th>\n",
       "      <th>timestampIs_quarter_end</th>\n",
       "      <th>timestampHour</th>\n",
       "      <th>timestampMinute</th>\n",
       "      <th>timestampIs_quarter_start</th>\n",
       "      <th>timestampIs_year_end</th>\n",
       "      <th>timestampIs_year_start</th>\n",
       "      <th>timestampElapsed</th>\n",
       "      <th>transformed_gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>af81bb96f9ccd7ecc223495ba44efc70</td>\n",
       "      <td>1.561736e+12</td>\n",
       "      <td>2019-07-07 07:59:40.000</td>\n",
       "      <td>install</td>\n",
       "      <td>All your favorite songs in just one place! Dow...</td>\n",
       "      <td>7366.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1562486380</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fa4bcfafd145868f0119bb89bd1b17db</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2019-09-20 07:09:18.408</td>\n",
       "      <td>install</td>\n",
       "      <td>Stop running helter-skelter when you need mone...</td>\n",
       "      <td>3224.950000</td>\n",
       "      <td>315.000000</td>\n",
       "      <td>88.000000</td>\n",
       "      <td>2617.230000</td>\n",
       "      <td>60.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1568963358</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>da2ae0a034979823b27139bb9479a6d9</td>\n",
       "      <td>1.562234e+12</td>\n",
       "      <td>2019-07-07 10:46:07.000</td>\n",
       "      <td>install</td>\n",
       "      <td>Listen to the latest hit songs from your favor...</td>\n",
       "      <td>5360.216671</td>\n",
       "      <td>459.040738</td>\n",
       "      <td>86.083116</td>\n",
       "      <td>2576.900000</td>\n",
       "      <td>636.508923</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "      <td>46</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1562496367</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f41536082453e49ad8db7e2eb7365112</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2019-09-17 13:36:03.466</td>\n",
       "      <td>install</td>\n",
       "      <td>Stop running helter-skelter when you need mone...</td>\n",
       "      <td>5360.216671</td>\n",
       "      <td>459.040738</td>\n",
       "      <td>192.000000</td>\n",
       "      <td>1636.140000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>13</td>\n",
       "      <td>36</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1568727363</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e7fd275d9958fa5d0219f5f0c21104d6</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2020-05-16 05:55:33.454</td>\n",
       "      <td>af_app_opened</td>\n",
       "      <td>Hurry and win 15GB of data. Download the ALAT ...</td>\n",
       "      <td>299.816720</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1589608533</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>cd7486a23904e5c93bb7691a21905e57</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2020-06-29 12:07:11.990</td>\n",
       "      <td>install</td>\n",
       "      <td>Need to Invest, But No capital? Install the AL...</td>\n",
       "      <td>5360.216671</td>\n",
       "      <td>213.097732</td>\n",
       "      <td>43.897461</td>\n",
       "      <td>1611.332573</td>\n",
       "      <td>445.096367</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1593432431</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>6b2a1050505122ed8d8d1405263a2586</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2019-08-26 17:11:17.067</td>\n",
       "      <td>install</td>\n",
       "      <td>Claim your daily BONUS to get as high as 70K! ...</td>\n",
       "      <td>3817.303809</td>\n",
       "      <td>213.097732</td>\n",
       "      <td>43.897461</td>\n",
       "      <td>1611.332573</td>\n",
       "      <td>445.096367</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1566839477</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>d5dd1b830c3144e73a3c80ec36565f41</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2019-09-14 08:49:53.302</td>\n",
       "      <td>install</td>\n",
       "      <td>Stop running helter-skelter when you need mone...</td>\n",
       "      <td>5461.595000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>86.083116</td>\n",
       "      <td>3094.650000</td>\n",
       "      <td>145.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1568450993</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>f0b590252c708f849d085342da4cf858</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2019-09-14 11:25:14.821</td>\n",
       "      <td>install</td>\n",
       "      <td>Stop running helter-skelter when you need mone...</td>\n",
       "      <td>5360.216671</td>\n",
       "      <td>459.040738</td>\n",
       "      <td>86.083116</td>\n",
       "      <td>4867.950000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1568460314</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>70e55e080d130856cf0f263556442fd3</td>\n",
       "      <td>1.564159e+12</td>\n",
       "      <td>2020-04-08 11:24:06.386</td>\n",
       "      <td>af_app_opened</td>\n",
       "      <td>Get a payday before your Payday. Loans of up t...</td>\n",
       "      <td>6764.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>11</td>\n",
       "      <td>24</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1586345046</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8720 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                msisdn      timeline               timestamp  \\\n",
       "0     af81bb96f9ccd7ecc223495ba44efc70  1.561736e+12 2019-07-07 07:59:40.000   \n",
       "2     fa4bcfafd145868f0119bb89bd1b17db  1.564159e+12 2019-09-20 07:09:18.408   \n",
       "3     da2ae0a034979823b27139bb9479a6d9  1.562234e+12 2019-07-07 10:46:07.000   \n",
       "4     f41536082453e49ad8db7e2eb7365112  1.564159e+12 2019-09-17 13:36:03.466   \n",
       "5     e7fd275d9958fa5d0219f5f0c21104d6  1.564159e+12 2020-05-16 05:55:33.454   \n",
       "...                                ...           ...                     ...   \n",
       "9995  cd7486a23904e5c93bb7691a21905e57  1.564159e+12 2020-06-29 12:07:11.990   \n",
       "9996  6b2a1050505122ed8d8d1405263a2586  1.564159e+12 2019-08-26 17:11:17.067   \n",
       "9997  d5dd1b830c3144e73a3c80ec36565f41  1.564159e+12 2019-09-14 08:49:53.302   \n",
       "9998  f0b590252c708f849d085342da4cf858  1.564159e+12 2019-09-14 11:25:14.821   \n",
       "9999  70e55e080d130856cf0f263556442fd3  1.564159e+12 2020-04-08 11:24:06.386   \n",
       "\n",
       "         event_type                                            ad_text  \\\n",
       "0           install  All your favorite songs in just one place! Dow...   \n",
       "2           install  Stop running helter-skelter when you need mone...   \n",
       "3           install  Listen to the latest hit songs from your favor...   \n",
       "4           install  Stop running helter-skelter when you need mone...   \n",
       "5     af_app_opened  Hurry and win 15GB of data. Download the ALAT ...   \n",
       "...             ...                                                ...   \n",
       "9995        install  Need to Invest, But No capital? Install the AL...   \n",
       "9996        install  Claim your daily BONUS to get as high as 70K! ...   \n",
       "9997        install  Stop running helter-skelter when you need mone...   \n",
       "9998        install  Stop running helter-skelter when you need mone...   \n",
       "9999  af_app_opened  Get a payday before your Payday. Loans of up t...   \n",
       "\n",
       "      spend_total   spend_vas    sms_cost  spend_voice   spend_data  ...  \\\n",
       "0     7366.400000    0.000000    0.000000     0.000000     0.000000  ...   \n",
       "2     3224.950000  315.000000   88.000000  2617.230000    60.000000  ...   \n",
       "3     5360.216671  459.040738   86.083116  2576.900000   636.508923  ...   \n",
       "4     5360.216671  459.040738  192.000000  1636.140000  1000.000000  ...   \n",
       "5      299.816720    0.000000    0.000000     0.000000     0.000000  ...   \n",
       "...           ...         ...         ...          ...          ...  ...   \n",
       "9995  5360.216671  213.097732   43.897461  1611.332573   445.096367  ...   \n",
       "9996  3817.303809  213.097732   43.897461  1611.332573   445.096367  ...   \n",
       "9997  5461.595000    0.000000   86.083116  3094.650000   145.000000  ...   \n",
       "9998  5360.216671  459.040738   86.083116  4867.950000     0.000000  ...   \n",
       "9999  6764.000000    0.000000    0.000000     0.000000     0.000000  ...   \n",
       "\n",
       "      timestampIs_month_end timestampIs_month_start  timestampIs_quarter_end  \\\n",
       "0                     False                   False                    False   \n",
       "2                     False                   False                    False   \n",
       "3                     False                   False                    False   \n",
       "4                     False                   False                    False   \n",
       "5                     False                   False                    False   \n",
       "...                     ...                     ...                      ...   \n",
       "9995                  False                   False                    False   \n",
       "9996                  False                   False                    False   \n",
       "9997                  False                   False                    False   \n",
       "9998                  False                   False                    False   \n",
       "9999                  False                   False                    False   \n",
       "\n",
       "     timestampHour timestampMinute timestampIs_quarter_start  \\\n",
       "0                7              59                     False   \n",
       "2                7               9                     False   \n",
       "3               10              46                     False   \n",
       "4               13              36                     False   \n",
       "5                5              55                     False   \n",
       "...            ...             ...                       ...   \n",
       "9995            12               7                     False   \n",
       "9996            17              11                     False   \n",
       "9997             8              49                     False   \n",
       "9998            11              25                     False   \n",
       "9999            11              24                     False   \n",
       "\n",
       "     timestampIs_year_end timestampIs_year_start timestampElapsed  \\\n",
       "0                   False                  False       1562486380   \n",
       "2                   False                  False       1568963358   \n",
       "3                   False                  False       1562496367   \n",
       "4                   False                  False       1568727363   \n",
       "5                   False                  False       1589608533   \n",
       "...                   ...                    ...              ...   \n",
       "9995                False                  False       1593432431   \n",
       "9996                False                  False       1566839477   \n",
       "9997                False                  False       1568450993   \n",
       "9998                False                  False       1568460314   \n",
       "9999                False                  False       1586345046   \n",
       "\n",
       "     transformed_gender  \n",
       "0                     0  \n",
       "2                     0  \n",
       "3                     0  \n",
       "4                     0  \n",
       "5                     0  \n",
       "...                 ...  \n",
       "9995                  0  \n",
       "9996                  0  \n",
       "9997                  0  \n",
       "9998                  0  \n",
       "9999                  0  \n",
       "\n",
       "[8720 rows x 40 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from slik import utils as ut\n",
    "ut.load_pickle('./InstallDownload/data/validation_data.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PIPELINE TRANSFORM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ppl.pipeline_transform_predict(data='/Users/terra-016/Downloads/Installs and Download.csv',\n",
    "                  project_path='./InstallDownload/',model_path='./InstallDownload/model/CatBoostClassifier.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = dataframe['transformed_event_type'].loc[list(train_df.index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------- Metric Performance ---------------\n",
      "\n",
      "{'accuracy': 0.7613532110091743, 'precision': 0.6614626129827444, 'recall': 0.3253839935327405, 'f1': 0.4361961528041181}\n",
      "\n",
      "--------------- Saving Plots ---------------\n",
      "\n",
      "Logged ./InstallDownload/plots/scores.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/terra-016/Downloads/slik_python_package/slik/pipeline.py:555: UserWarning: Transformer scaler (type StandardScaler) does not provide get_feature_names. Will return input column names if available\n",
      "  warnings.warn(\"Transformer %s (type %s) does not \"\n",
      "/Users/terra-016/Downloads/slik_python_package/slik/pipeline.py:555: UserWarning: Transformer imputer (type SimpleImputer) does not provide get_feature_names. Will return input column names if available\n",
      "  warnings.warn(\"Transformer %s (type %s) does not \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logged ./InstallDownload/plots/feature_importance.png\n",
      "Logged ./InstallDownload/plots/confusion_matrix.png\n",
      "Logged ./InstallDownload/plots/roc_curve.png\n",
      "Logged ./InstallDownload/plots/pr_curve.png\n"
     ]
    }
   ],
   "source": [
    "ppl.evaluate_model('./InstallDownload/model/CatBoostClassifier.pkl',new_df,project_path='./InstallDownload/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
