{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp utils.common_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Common utils\n",
    "> A collection of utilities often used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import sys\n",
    "import os\n",
    "import ssl\n",
    "import os.path as osp\n",
    "from six.moves import urllib\n",
    "import errno\n",
    "import tarfile\n",
    "import zipfile\n",
    "import bz2\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporti\n",
    "def makedirs(path):\n",
    "    try:\n",
    "        os.makedirs(osp.expanduser(osp.normpath(path)))\n",
    "    except OSError as e:\n",
    "        if e.errno != errno.EEXIST and osp.isdir(path):\n",
    "            raise e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def wget_download(url, savepath):\n",
    "    import wget\n",
    "    wget.download(url, str(savepath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def download_url(url: str, folder: str, log: bool = True):\n",
    "    r\"\"\"Downloads the content of an URL to a specific folder.\n",
    "    Args:\n",
    "        url (string): The url.\n",
    "        folder (string): The folder.\n",
    "        log (bool, optional): If :obj:`False`, will not print anything to the\n",
    "            console. (default: :obj:`True`)\n",
    "    \"\"\"\n",
    "\n",
    "    filename = url.rpartition('/')[2]\n",
    "    filename = filename if filename[0] == '?' else filename.split('?')[0]\n",
    "    path = osp.join(folder, filename)\n",
    "\n",
    "    if osp.exists(path):  # pragma: no cover\n",
    "        if log:\n",
    "            print(f'Using existing file {filename}', file=sys.stderr)\n",
    "        return path\n",
    "\n",
    "    if log:\n",
    "        print(f'Downloading {url}', file=sys.stderr)\n",
    "\n",
    "    makedirs(folder)\n",
    "\n",
    "    context = ssl._create_unverified_context()\n",
    "    data = urllib.request.urlopen(url, context=context)\n",
    "\n",
    "    with open(path, 'wb') as f:\n",
    "        f.write(data.read())\n",
    "\n",
    "    return path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading https://files.grouplens.org/datasets/movielens/ml-1m.zip\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'./data/bronze/ml-1m.zip'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "download_url('https://files.grouplens.org/datasets/movielens/ml-1m.zip',\n",
    "             './data/bronze')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data\n",
      "└── bronze\n",
      "    └── ml-1m.zip\n",
      "\n",
      "1 directory, 1 file\n"
     ]
    }
   ],
   "source": [
    "!tree ./data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34m./data\u001b[00m\n",
      "├── [ 24M]  \u001b[01;34mbronze\u001b[00m\n",
      "│   └── [ 24M]  \u001b[01;34mml-1m\u001b[00m\n",
      "│       ├── [167K]  movies.dat\n",
      "│       ├── [ 23M]  ratings.dat\n",
      "│       ├── [5.4K]  README\n",
      "│       └── [131K]  users.dat\n",
      "└── [3.0M]  \u001b[01;34msilver\u001b[00m\n",
      "    └── [3.0M]  \u001b[01;34mml-1m_min_rating0-min_uc5-min_sc5-splitleave_one_out\u001b[00m\n",
      "        └── [3.0M]  dataset.pkl\n",
      "\n",
      "  27M used in 4 directories, 5 files\n"
     ]
    }
   ],
   "source": [
    "!tree --du -h -C ./data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#exporti\n",
    "def maybe_log(path, log=True):\n",
    "    if log:\n",
    "        print(f'Extracting {path}', file=sys.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def extract_tar(path: str, folder: str, mode: str = 'r:gz', log: bool = True):\n",
    "    r\"\"\"Extracts a tar archive to a specific folder.\n",
    "    Args:\n",
    "        path (string): The path to the tar archive.\n",
    "        folder (string): The folder.\n",
    "        mode (string, optional): The compression mode. (default: :obj:`\"r:gz\"`)\n",
    "        log (bool, optional): If :obj:`False`, will not print anything to the\n",
    "            console. (default: :obj:`True`)\n",
    "    \"\"\"\n",
    "    maybe_log(path, log)\n",
    "    with tarfile.open(path, mode) as f:\n",
    "        f.extractall(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def extract_zip(path: str, folder: str, log: bool = True):\n",
    "    r\"\"\"Extracts a zip archive to a specific folder.\n",
    "    Args:\n",
    "        path (string): The path to the tar archive.\n",
    "        folder (string): The folder.\n",
    "        log (bool, optional): If :obj:`False`, will not print anything to the\n",
    "            console. (default: :obj:`True`)\n",
    "    \"\"\"\n",
    "    maybe_log(path, log)\n",
    "    with zipfile.ZipFile(path, 'r') as f:\n",
    "        f.extractall(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def extract_bz2(path: str, folder: str, log: bool = True):\n",
    "    r\"\"\"Extracts a bz2 archive to a specific folder.\n",
    "    Args:\n",
    "        path (string): The path to the tar archive.\n",
    "        folder (string): The folder.\n",
    "        log (bool, optional): If :obj:`False`, will not print anything to the\n",
    "            console. (default: :obj:`True`)\n",
    "    \"\"\"\n",
    "    maybe_log(path, log)\n",
    "    path = osp.abspath(path)\n",
    "    with bz2.open(path, 'r') as r:\n",
    "        with open(osp.join(folder, '.'.join(path.split('.')[:-1])), 'wb') as w:\n",
    "            w.write(r.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def extract_gz(path: str, folder: str, log: bool = True):\n",
    "    r\"\"\"Extracts a gz archive to a specific folder.\n",
    "    Args:\n",
    "        path (string): The path to the tar archive.\n",
    "        folder (string): The folder.\n",
    "        log (bool, optional): If :obj:`False`, will not print anything to the\n",
    "            console. (default: :obj:`True`)\n",
    "    \"\"\"\n",
    "    maybe_log(path, log)\n",
    "    path = osp.abspath(path)\n",
    "    with gzip.open(path, 'r') as r:\n",
    "        with open(osp.join(folder, '.'.join(path.split('.')[:-1])), 'wb') as w:\n",
    "            w.write(r.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-21 12:52:30\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "sys    : 3.7.12 (default, Sep 10 2021, 00:21:48) \n",
      "[GCC 7.5.0]\n",
      "google : 2.0.3\n",
      "numpy  : 1.19.5\n",
      "pandas : 1.1.5\n",
      "tarfile: 0.9.0\n",
      "six    : 1.15.0\n",
      "IPython: 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
