Cifar_10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T16:57:31.564216Z",
     "start_time": "2017-10-27T16:57:31.546169Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 如果训练下载的Kaggle的完整数据集，把下面改False\n",
    "demo = False\n",
    "if demo:\n",
    "    import zipfile\n",
    "    for fin in ['train_tiny.zip', 'test_tiny.zip', 'trainLabels.csv.zip']:\n",
    "        with zipfile.ZipFile('../data/kaggle_cifar10/' + fin, 'r') as zin:\n",
    "            zin.extractall('../data/kaggle_cifar10/')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "函数中的参数如data_dir、train_dir和test_dir对应上述数据存放路径及训练和测试的图片集文件夹名称。参数label_file为训练数据标签的文件名称。参数input_dir时整理后数据集文件夹名称。参数valid_ratio是验证集占原始训练集的比重。以valid_ratio=0.1为例，由于原始训练数据有5万张图片，调参时将有4万5千张图片用于训练（整理后存放在input_dir/train）而另外5千张图片为验证集（整理后存放在input_dir/valid）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T16:57:33.161801Z",
     "start_time": "2017-10-27T16:57:33.090089Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "def reorg_cifar10_data(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio):\n",
    "    # 读取训练数据标签。\n",
    "    with open(os.path.join(data_dir, label_file), 'r') as f:\n",
    "        # 跳过文件头行（栏名称）。\n",
    "        lines = f.readlines()[1:]\n",
    "        tokens = [l.rstrip().split(',') for l in lines]\n",
    "        idx_label = dict(((int(idx), label) for idx, label in tokens))\n",
    "    labels = set(idx_label.values())\n",
    "\n",
    "    num_train = len(os.listdir(os.path.join(data_dir, train_dir)))\n",
    "    num_train_tuning = int(num_train * (1 - valid_ratio))\n",
    "    assert 0 < num_train_tuning < num_train\n",
    "    num_train_tuning_per_label = num_train_tuning // len(labels)\n",
    "    label_count = dict()\n",
    "\n",
    "    def mkdir_if_not_exist(path):\n",
    "        if not os.path.exists(os.path.join(*path)):\n",
    "            os.makedirs(os.path.join(*path))\n",
    "\n",
    "    # 整理训练和验证集。\n",
    "    for train_file in os.listdir(os.path.join(data_dir, train_dir)):\n",
    "        idx = int(train_file.split('.')[0])\n",
    "        label = idx_label[idx]\n",
    "        mkdir_if_not_exist([data_dir, input_dir, 'train_valid', label])\n",
    "        shutil.copy(os.path.join(data_dir, train_dir, train_file),\n",
    "                    os.path.join(data_dir, input_dir, 'train_valid', label))\n",
    "        if label not in label_count or label_count[label] < num_train_tuning_per_label:\n",
    "            mkdir_if_not_exist([data_dir, input_dir, 'train', label])\n",
    "            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n",
    "                        os.path.join(data_dir, input_dir, 'train', label))\n",
    "            label_count[label] = label_count.get(label, 0) + 1\n",
    "        else:\n",
    "            mkdir_if_not_exist([data_dir, input_dir, 'valid', label])\n",
    "            shutil.copy(os.path.join(data_dir, train_dir, train_file),\n",
    "                        os.path.join(data_dir, input_dir, 'valid', label))\n",
    "\n",
    "    # 整理测试集。\n",
    "    mkdir_if_not_exist([data_dir, input_dir, 'test', 'unknown'])\n",
    "    for test_file in os.listdir(os.path.join(data_dir, test_dir)):\n",
    "        shutil.copy(os.path.join(data_dir, test_dir, test_file),\n",
    "                    os.path.join(data_dir, input_dir, 'test', 'unknown'))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "将10%的训练样本作为调参时的验证集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-25T04:52:20.356214Z",
     "start_time": "2017-10-25T04:52:18.560639Z"
    }
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-123a6445f0f9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[0minput_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'train_valid_test'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[0mvalid_ratio\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[0mreorg_cifar10_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_ratio\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-3-ce238aaaa2a2>\u001b[0m in \u001b[0;36mreorg_cifar10_data\u001b[1;34m(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio)\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[0mmkdir_if_not_exist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train_valid'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m         shutil.copy(os.path.join(data_dir, train_dir, train_file),\n\u001b[1;32m---> 29\u001b[1;33m                     os.path.join(data_dir, input_dir, 'train_valid', label))\n\u001b[0m\u001b[0;32m     30\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlabel\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mlabel_count\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mlabel_count\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mnum_train_tuning_per_label\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m             \u001b[0mmkdir_if_not_exist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\shutil.py\u001b[0m in \u001b[0;36mcopy\u001b[1;34m(src, dst, follow_symlinks)\u001b[0m\n\u001b[0;32m    239\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdst\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m         \u001b[0mdst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbasename\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 241\u001b[1;33m     \u001b[0mcopyfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfollow_symlinks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfollow_symlinks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    242\u001b[0m     \u001b[0mcopymode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfollow_symlinks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfollow_symlinks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    243\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mdst\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\shutil.py\u001b[0m in \u001b[0;36mcopyfile\u001b[1;34m(src, dst, follow_symlinks)\u001b[0m\n\u001b[0;32m    118\u001b[0m         \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msymlink\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadlink\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdst\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    119\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 120\u001b[1;33m         \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfsrc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    121\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdst\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfdst\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    122\u001b[0m                 \u001b[0mcopyfileobj\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfsrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfdst\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if demo:\n",
    "    # 注意：此处使用小训练集为便于网页编译。Kaggle的完整数据集应包括5万训练样本。\n",
    "    train_dir = 'train_tiny'\n",
    "    # 注意：此处使用小测试集为便于网页编译。Kaggle的完整数据集应包括30万测试样本。\n",
    "    test_dir = 'test_tiny'\n",
    "    # 注意：此处相应使用小批量。对Kaggle的完整数据集可设较大的整数，例如128。\n",
    "    batch_size = 1\n",
    "else:\n",
    "    train_dir = 'train'\n",
    "    test_dir = 'test'\n",
    "    batch_size = 128\n",
    "\n",
    "data_dir = 'CIFAR_10/'\n",
    "label_file = 'trainLabels.csv'\n",
    "input_dir = 'train_valid_test'\n",
    "valid_ratio = 0.1\n",
    "reorg_cifar10_data(data_dir, label_file, train_dir, test_dir, input_dir, valid_ratio)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "为避免过拟合，我们在这里使用image.CreateAugmenter来加强数据集。例如我们设rand_mirror=True即可随机对每张图片做镜面反转。我们也通过mean和std对彩色图像RGB三个通道分别做标准化。以下我们列举了该函数里的所有参数，这些参数都是可以调的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T16:59:17.799400Z",
     "start_time": "2017-10-27T16:59:14.659612Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\urllib3\\contrib\\pyopenssl.py:46: DeprecationWarning: OpenSSL.rand is deprecated - you should use os.urandom instead\n",
      "  import OpenSSL.SSL\n"
     ]
    }
   ],
   "source": [
    "from mxnet import autograd\n",
    "from mxnet import gluon\n",
    "from mxnet import image\n",
    "from mxnet import init\n",
    "from mxnet import nd\n",
    "from mxnet.gluon.data import vision\n",
    "import numpy as np\n",
    "\n",
    "def transform_train(data, label):\n",
    "    im = data.asnumpy()\n",
    "    im = np.pad(im, ((4, 4), (4, 4), (0, 0)), mode='constant', constant_values=0)\n",
    "    im = data.astype('float32') / 255\n",
    "    auglist = image.CreateAugmenter(data_shape=(3, 32, 32), resize=0,\n",
    "                        rand_crop=True, rand_mirror=True, rand_resize=True, \n",
    "                        mean=np.array([0.4914, 0.4822, 0.4465]),\n",
    "                        std=np.array([0.2023, 0.1994, 0.2010]),\n",
    "                        brightness=0, contrast=0,\n",
    "                        saturation=0, hue=0,\n",
    "                        pca_noise=0.01, rand_gray=0, inter_method=2)\n",
    "    for aug in auglist:\n",
    "        im = aug(im)\n",
    "    # 将数据格式从\"高*宽*通道\"改为\"通道*高*宽\"。\n",
    "    im = nd.transpose(im, (2,0,1))\n",
    "    return (im, nd.array([label]).asscalar().astype('float32'))\n",
    "\n",
    "# 测试时，无需对图像做标准化以外的增强数据处理。\n",
    "def transform_test(data, label):\n",
    "    im = data.astype('float32') / 255\n",
    "    auglist = image.CreateAugmenter(data_shape=(3, 32, 32),\n",
    "                        mean=np.array([0.4914, 0.4822, 0.4465]),\n",
    "                        std=np.array([0.2023, 0.1994, 0.2010]))\n",
    "    for aug in auglist:\n",
    "        im = aug(im)\n",
    "    im = nd.transpose(im, (2,0,1))\n",
    "    return (im, nd.array([label]).asscalar().astype('float32'))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "使用Gluon中的ImageFolderDataset类来读取整理后的数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T16:59:23.588223Z",
     "start_time": "2017-10-27T16:59:21.601908Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = 'CIFAR_10/'\n",
    "label_file = 'trainLabels.csv'\n",
    "input_dir = 'train_valid_test'\n",
    "input_str = data_dir + '/' + input_dir + '/'\n",
    "#batch_size = 128\n",
    "batch_size = 64\n",
    "\n",
    "# 读取原始图像文件。flag=1说明输入图像有三个通道（彩色）。\n",
    "train_ds = vision.ImageFolderDataset(input_str + 'train', flag=1,\n",
    "                                     transform=transform_train)\n",
    "valid_ds = vision.ImageFolderDataset(input_str + 'valid', flag=1,\n",
    "                                     transform=transform_test)\n",
    "train_valid_ds = vision.ImageFolderDataset(input_str + 'train_valid',\n",
    "                                           flag=1, transform=transform_train)\n",
    "test_ds = vision.ImageFolderDataset(input_str + 'test', flag=1,\n",
    "                                     transform=transform_test)\n",
    "\n",
    "loader = gluon.data.DataLoader\n",
    "train_data = loader(train_ds, batch_size, shuffle=True, last_batch='keep')\n",
    "valid_data = loader(valid_ds, batch_size, shuffle=True, last_batch='keep')\n",
    "train_valid_data = loader(train_valid_ds, batch_size, shuffle=True, last_batch='keep')\n",
    "test_data = loader(test_ds, batch_size, shuffle=False, last_batch='keep')\n",
    "\n",
    "# 交叉熵损失函数。\n",
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "使用了ResNet-18模型。我们使用hybridizing来提升执行效率。\n",
    "注意：模型可以重新设计，参数也可以重新调整。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-28T03:36:57.734945Z",
     "start_time": "2017-10-28T03:36:57.575991Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Residual_v2_bottleneck(nn.HybridBlock):\n",
    "    def __init__(self, channels, same_shape=True):\n",
    "        super(Residual_v2_bottleneck, self).__init__()\n",
    "        self.same_shape = same_shape\n",
    "        with self.name_scope():\n",
    "            strides = 1 if same_shape else 2\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(channels // 4, 1, use_bias=False)\n",
    "            self.bn2 = nn.BatchNorm()\n",
    "            self.conv2 = nn.Conv2D(\n",
    "                channels // 4, 3, padding=1, strides=strides, use_bias=False)\n",
    "            self.bn3 = nn.BatchNorm()\n",
    "            self.conv3 = nn.Conv2D(channels, 1, use_bias=False)\n",
    "            self.bn4 = nn.BatchNorm()\n",
    "\n",
    "            if not same_shape:\n",
    "                self.conv4 = nn.Conv2D(\n",
    "                    channels, 1, strides=strides, use_bias=False)\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = self.conv1(self.bn1(x))\n",
    "        out = F.relu(self.bn2(out))\n",
    "        out = F.relu(self.bn3(self.conv2(out)))\n",
    "        out = self.bn4(self.conv3(out))\n",
    "        if not self.same_shape:\n",
    "            x = self.conv4(x)\n",
    "        return out + x\n",
    "\n",
    "\n",
    "class ResNet164_v2(nn.HybridBlock):\n",
    "    def __init__(self, num_classes, verbose=False):\n",
    "        super(ResNet164_v2, self).__init__()\n",
    "        self.verbose = verbose\n",
    "        with self.name_scope():\n",
    "            net = self.net = nn.HybridSequential()\n",
    "            # block 1\n",
    "            net.add(nn.Conv2D(64, 3, 1, 1, use_bias=False))\n",
    "            # block 2\n",
    "            for _ in range(27):\n",
    "                net.add(Residual_v2_bottleneck(64))\n",
    "            # block 3\n",
    "            net.add(Residual_v2_bottleneck(128, same_shape=False))\n",
    "            for _ in range(26):\n",
    "                net.add(Residual_v2_bottleneck(128))\n",
    "            # block 4\n",
    "            net.add(Residual_v2_bottleneck(256, same_shape=False))\n",
    "            for _ in range(26):\n",
    "                net.add(Residual_v2_bottleneck(256))\n",
    "            # block 5\n",
    "            net.add(nn.BatchNorm())\n",
    "            net.add(nn.Activation('relu'))\n",
    "            net.add(nn.AvgPool2D(8))\n",
    "            net.add(nn.Dense(num_classes))\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = x\n",
    "        for i, b in enumerate(self.net):\n",
    "            out = b(out)\n",
    "            if self.verbose:\n",
    "                print('Block %d output: %s' % (i + 1, out.shape))\n",
    "        return out\n",
    "\n",
    "def get_net(ctx):\n",
    "    num_outputs = 10\n",
    "    net = ResNet164_v2(num_outputs)\n",
    "    net.initialize(ctx=ctx, init=mx.init.Xavier())\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-28T03:37:07.629657Z",
     "start_time": "2017-10-28T03:37:07.525352Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet164_v2(\n",
      "  (net): HybridSequential(\n",
      "    (0): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (1): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (2): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (3): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (4): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (5): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (6): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (7): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (8): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (9): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (10): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (11): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (12): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (13): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (14): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (15): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (16): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (17): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (18): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (19): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (20): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (21): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (22): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (23): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (24): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (25): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (26): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (27): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (28): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv4): Conv2D(128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "    )\n",
      "    (29): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (30): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (31): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (32): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (33): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (34): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (35): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (36): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (37): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (38): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (39): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (40): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (41): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (42): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (43): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (44): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (45): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (46): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (47): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (48): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (49): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (50): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (51): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (52): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (53): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (54): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (55): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv4): Conv2D(256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "    )\n",
      "    (56): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (57): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (58): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (59): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (60): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (61): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (62): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (63): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (64): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (65): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (66): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (67): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (68): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (69): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (70): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (71): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (72): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (73): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (74): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (75): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (76): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (77): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (78): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (79): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (80): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (81): Residual_v2_bottleneck(\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv1): Conv2D(64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn3): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn4): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (82): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    (83): Activation(relu)\n",
      "    (84): AvgPool2D(size=(8, 8), stride=(8, 8), padding=(0, 0), ceil_mode=False)\n",
      "    (85): Dense(10, linear)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "ctx = utils.try_gpu()\n",
    "net = get_net(ctx)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-25T17:21:25.456405Z",
     "start_time": "2017-10-25T17:21:25.324824Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mxnet.gluon import nn\n",
    "from mxnet import nd\n",
    "\n",
    "class Residual(nn.HybridBlock):\n",
    "    def __init__(self, channels, same_shape=True, **kwargs):\n",
    "        super(Residual, self).__init__(**kwargs)\n",
    "        self.same_shape = same_shape\n",
    "        with self.name_scope():\n",
    "            strides = 1 if same_shape else 2\n",
    "            self.conv1 = nn.Conv2D(channels, kernel_size=3, padding=1,\n",
    "                                  strides=strides)\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv2 = nn.Conv2D(channels, kernel_size=3, padding=1)\n",
    "            self.bn2 = nn.BatchNorm()\n",
    "            if not same_shape:\n",
    "                self.conv3 = nn.Conv2D(channels, kernel_size=1,\n",
    "                                      strides=strides)\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if not self.same_shape:\n",
    "            x = self.conv3(x)\n",
    "        return F.relu(out + x)\n",
    "\n",
    "\n",
    "class ResNet(nn.HybridBlock):\n",
    "    def __init__(self, num_classes, verbose=False, **kwargs):\n",
    "        super(ResNet, self).__init__(**kwargs)\n",
    "        self.verbose = verbose\n",
    "        with self.name_scope():\n",
    "            net = self.net = nn.HybridSequential()\n",
    "            # block 1\n",
    "            net.add(nn.Conv2D(channels=32, kernel_size=3, strides=1, padding=1))\n",
    "            net.add(nn.BatchNorm())\n",
    "            net.add(nn.Activation(activation='relu'))\n",
    "            # block 2\n",
    "            for _ in range(3):\n",
    "                net.add(Residual(channels=32))\n",
    "            # block 3\n",
    "            net.add(Residual(channels=64, same_shape=False))\n",
    "            for _ in range(2):\n",
    "                net.add(Residual(channels=64))\n",
    "            # block 4\n",
    "            net.add(Residual(channels=128, same_shape=False))\n",
    "            for _ in range(2):\n",
    "                net.add(Residual(channels=128))\n",
    "            # block 5\n",
    "            net.add(nn.AvgPool2D(pool_size=8))\n",
    "            net.add(nn.Flatten())\n",
    "            net.add(nn.Dense(num_classes))\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = x\n",
    "        for i, b in enumerate(self.net):\n",
    "            out = b(out)\n",
    "            if self.verbose:\n",
    "                print('Block %d output: %s'%(i+1, out.shape))\n",
    "        return out\n",
    "\n",
    "\n",
    "def get_net(ctx):\n",
    "    num_outputs = 10\n",
    "    net = ResNet(num_outputs)\n",
    "    net.initialize(ctx=ctx, init=init.Xavier())\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-23T19:03:44.798092Z",
     "start_time": "2017-10-23T19:03:44.781047Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet(\n",
      "  (net): HybridSequential(\n",
      "    (0): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    (2): Activation(relu)\n",
      "    (3): Residual(\n",
      "      (conv1): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (4): Residual(\n",
      "      (conv1): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (5): Residual(\n",
      "      (conv1): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (6): Residual(\n",
      "      (conv1): Conv2D(64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(64, kernel_size=(1, 1), stride=(2, 2))\n",
      "    )\n",
      "    (7): Residual(\n",
      "      (conv1): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (8): Residual(\n",
      "      (conv1): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (9): Residual(\n",
      "      (conv1): Conv2D(128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv3): Conv2D(128, kernel_size=(1, 1), stride=(2, 2))\n",
      "    )\n",
      "    (10): Residual(\n",
      "      (conv1): Conv2D(128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (11): Residual(\n",
      "      (conv1): Conv2D(128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (conv2): Conv2D(128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (bn2): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    )\n",
      "    (12): AvgPool2D(size=(8, 8), stride=(8, 8), padding=(0, 0), ceil_mode=False)\n",
      "    (13): Flatten\n",
      "    (14): Dense(10, linear)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "ctx = utils.try_gpu()\n",
    "net = get_net(ctx)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T21:39:59.839609Z",
     "start_time": "2017-10-26T21:39:59.689639Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from mxnet import nd\n",
    "from mxnet.gluon import nn\n",
    "import mxnet as mx\n",
    "\n",
    "def conv_block(channels):\n",
    "    out = nn.HybridSequential()\n",
    "    out.add(\n",
    "        nn.BatchNorm(),\n",
    "        nn.Activation('relu'),\n",
    "        nn.Conv2D(channels, kernel_size=3, padding=1)\n",
    "    )\n",
    "    return out\n",
    "\n",
    "class DenseBlock(nn.HybridBlock):\n",
    "    def __init__(self, layers, growth_rate, **kwargs):\n",
    "        super(DenseBlock, self).__init__(**kwargs)\n",
    "        self.net = nn.HybridSequential()\n",
    "        for i in range(layers):\n",
    "            self.net.add(conv_block(growth_rate))\n",
    "\n",
    "    def forward(self, x):\n",
    "        for layer in self.net:\n",
    "            out = layer(x)\n",
    "            #x = nd.concat(x, out, dim=1)\n",
    "            x = mx.symbol.Concat(x, out)\n",
    "        return x\n",
    "    \n",
    "def transition_block(channels):\n",
    "    out = nn.HybridSequential()\n",
    "    out.add(\n",
    "        nn.BatchNorm(),\n",
    "        nn.Activation('relu'),\n",
    "        nn.Conv2D(channels, kernel_size=1),\n",
    "        nn.AvgPool2D(pool_size=2, strides=2)\n",
    "    )\n",
    "    return out\n",
    "\n",
    "init_channels = 64\n",
    "growth_rate = 32\n",
    "block_layers = [6, 12, 48, 32]#denseNet_201\n",
    "'''\n",
    "def dense_net(num_classes):\n",
    "    net = nn.HybridSequential()\n",
    "    # add name_scope on the outermost Sequential\n",
    "    with net.name_scope():\n",
    "        # first block\n",
    "        net.add(\n",
    "            nn.Conv2D(init_channels, kernel_size=7,\n",
    "                      strides=2, padding=3),\n",
    "            nn.BatchNorm(),\n",
    "            nn.Activation('relu'),\n",
    "            nn.MaxPool2D(pool_size=3, strides=2, padding=1)\n",
    "        )\n",
    "        # dense blocks\n",
    "        channels = init_channels\n",
    "        for i, layers in enumerate(block_layers):\n",
    "            net.add(DenseBlock(layers, growth_rate))\n",
    "            channels += layers * growth_rate\n",
    "            if i != len(block_layers)-1:\n",
    "                net.add(transition_block(channels//2))\n",
    "        # last block\n",
    "        net.add(\n",
    "            nn.BatchNorm(),\n",
    "            nn.Activation('relu'),\n",
    "            nn.AvgPool2D(pool_size=1),\n",
    "            nn.Flatten(),\n",
    "            nn.Dense(num_classes)\n",
    "        )\n",
    "    return net'''\n",
    "\n",
    "class DenseNet(nn.HybridBlock):\n",
    "    def __init__(self, num_classes, verbose=False, **kwargs):\n",
    "        super(DenseNet, self).__init__(**kwargs)\n",
    "        self.verbose = verbose\n",
    "        with self.name_scope():\n",
    "            net = self.net = nn.HybridSequential()\n",
    "            # first block\n",
    "            net.add(\n",
    "                nn.Conv2D(init_channels, kernel_size=7,\n",
    "                          strides=2, padding=3),\n",
    "                nn.BatchNorm(),\n",
    "                nn.Activation('relu'),\n",
    "                nn.MaxPool2D(pool_size=3, strides=2, padding=1)\n",
    "            )\n",
    "            # dense blocks\n",
    "            channels = init_channels\n",
    "            for i, layers in enumerate(block_layers):\n",
    "                net.add(DenseBlock(layers, growth_rate))\n",
    "                channels += layers * growth_rate\n",
    "                if i != len(block_layers)-1:\n",
    "                    net.add(transition_block(channels//2))\n",
    "            # last block\n",
    "            net.add(\n",
    "                nn.BatchNorm(),\n",
    "                nn.Activation('relu'),\n",
    "                nn.AvgPool2D(pool_size=1),\n",
    "                nn.Flatten(),\n",
    "                nn.Dense(num_classes)\n",
    "                )\n",
    "            \n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = x\n",
    "        for i, b in enumerate(self.net):\n",
    "            out = b(out)\n",
    "            if self.verbose:\n",
    "                print('Block %d output: %s'%(i+1, out.shape))\n",
    "        return out\n",
    "            \n",
    "\n",
    "def get_dense_net(ctx):\n",
    "    num_classes = 10\n",
    "    net = DenseNet(num_classes)\n",
    "    net.initialize(ctx=ctx, init=init.Xavier())\n",
    "    return net\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-25T17:52:34.811201Z",
     "start_time": "2017-10-25T17:52:34.757626Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DenseNet(\n",
      "  (net): HybridSequential(\n",
      "    (0): Conv2D(64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))\n",
      "    (1): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    (2): Activation(relu)\n",
      "    (3): MaxPool2D(size=(3, 3), stride=(2, 2), padding=(1, 1), ceil_mode=False)\n",
      "    (4): DenseBlock(\n",
      "      (net): HybridSequential(\n",
      "        (0): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (1): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (2): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (3): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (4): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (5): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (5): HybridSequential(\n",
      "      (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (1): Activation(relu)\n",
      "      (2): Conv2D(128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (3): AvgPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "    )\n",
      "    (6): DenseBlock(\n",
      "      (net): HybridSequential(\n",
      "        (0): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (1): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (2): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (3): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (4): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (5): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (6): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (7): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (8): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (9): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (10): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (11): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (7): HybridSequential(\n",
      "      (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (1): Activation(relu)\n",
      "      (2): Conv2D(320, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (3): AvgPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "    )\n",
      "    (8): DenseBlock(\n",
      "      (net): HybridSequential(\n",
      "        (0): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (1): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (2): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (3): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (4): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (5): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (6): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (7): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (8): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (9): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (10): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (11): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (12): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (13): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (14): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (15): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (16): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (17): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (18): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (19): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (20): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (21): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (22): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (23): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (9): HybridSequential(\n",
      "      (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "      (1): Activation(relu)\n",
      "      (2): Conv2D(704, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (3): AvgPool2D(size=(2, 2), stride=(2, 2), padding=(0, 0), ceil_mode=False)\n",
      "    )\n",
      "    (10): DenseBlock(\n",
      "      (net): HybridSequential(\n",
      "        (0): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (1): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (2): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (3): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (4): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (5): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (6): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (7): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (8): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (9): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (10): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (11): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (12): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (13): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (14): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "        (15): HybridSequential(\n",
      "          (0): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "          (1): Activation(relu)\n",
      "          (2): Conv2D(32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (11): BatchNorm(axis=1, eps=1e-05, momentum=0.9, fix_gamma=False)\n",
      "    (12): Activation(relu)\n",
      "    (13): AvgPool2D(size=(1, 1), stride=(1, 1), padding=(0, 0), ceil_mode=False)\n",
      "    (14): Flatten\n",
      "    (15): Dense(10, linear)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "ctx = utils.try_gpu()\n",
    "net = get_dense_net(ctx)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T17:00:01.271250Z",
     "start_time": "2017-10-27T17:00:01.265743Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T17:00:26.705522Z",
     "start_time": "2017-10-27T17:00:26.377649Z"
    }
   },
   "outputs": [],
   "source": [
    "import math\n",
    "from mxnet import nd\n",
    "from mxnet.gluon import nn\n",
    "class Bottleneck(nn.HybridBlock):\n",
    "    def __init__(self, growthRate):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        interChannels = 4 * growthRate\n",
    "        with self.name_scope():\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(\n",
    "                interChannels,\n",
    "                kernel_size=1,\n",
    "                use_bias=False,\n",
    "                weight_initializer=init.Normal(math.sqrt(2. / interChannels)))\n",
    "            self.bn2 = nn.BatchNorm()\n",
    "            self.conv2 = nn.Conv2D(\n",
    "                growthRate,\n",
    "                kernel_size=3,\n",
    "                padding=1,\n",
    "                use_bias=False,\n",
    "                weight_initializer=init.Normal(\n",
    "                    math.sqrt(2. / (9 * growthRate))))\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = self.conv1(F.relu(self.bn1(x)))\n",
    "        out = self.conv2(F.relu(self.bn2(out)))\n",
    "        out = F.concat(* [x, out], dim=1)\n",
    "        return out\n",
    "\n",
    "\n",
    "class SingleLayer(nn.HybridBlock):\n",
    "    def __init__(self, growthRate):\n",
    "        super(SingleLayer, self).__init__()\n",
    "        with self.name_scope():\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(\n",
    "                growthRate,\n",
    "                kernel_size=3,\n",
    "                padding=1,\n",
    "                use_bias=False,\n",
    "                weight_initializer=init.Normal(\n",
    "                    math.sqrt(2. / (9 * growthRate))))\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = self.conv1(F.relu(self.bn1(x)))\n",
    "        out = F.concat(* [x, out], 1)\n",
    "        return out\n",
    "\n",
    "\n",
    "class Transition(nn.HybridBlock):\n",
    "    def __init__(self, nOutChannels):\n",
    "        super(Transition, self).__init__()\n",
    "        with self.name_scope():\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.conv1 = nn.Conv2D(\n",
    "                nOutChannels,\n",
    "                kernel_size=1,\n",
    "                use_bias=False,\n",
    "                weight_initializer=init.Normal(math.sqrt(2. / nOutChannels)))\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = self.conv1(F.relu(self.bn1(x)))\n",
    "        out = F.Pooling(out, kernel=(2, 2), stride=(2, 2), pool_type='avg')\n",
    "        return out\n",
    "\n",
    "\n",
    "class DenseNet(nn.HybridBlock):\n",
    "    def __init__(self, growthRate, depth, reduction, nClasses, bottleneck):\n",
    "        super(DenseNet, self).__init__()\n",
    "\n",
    "        nDenseBlocks = (depth - 4) // 3\n",
    "        if bottleneck:\n",
    "            nDenseBlocks //= 2\n",
    "\n",
    "        nChannels = 2 * growthRate\n",
    "        with self.name_scope():\n",
    "            self.conv1 = nn.Conv2D(\n",
    "                nChannels,\n",
    "                kernel_size=3,\n",
    "                padding=1,\n",
    "                use_bias=False,\n",
    "                weight_initializer=init.Normal(math.sqrt(2. / nChannels)))\n",
    "            self.dense1 = self._make_dense(growthRate, nDenseBlocks,\n",
    "                                           bottleneck)\n",
    "\n",
    "        nChannels += nDenseBlocks * growthRate\n",
    "        nOutChannels = int(math.floor(nChannels * reduction))\n",
    "        with self.name_scope():\n",
    "            self.trans1 = Transition(nOutChannels)\n",
    "\n",
    "        nChannels = nOutChannels\n",
    "        with self.name_scope():\n",
    "            self.dense2 = self._make_dense(growthRate, nDenseBlocks,\n",
    "                                           bottleneck)\n",
    "        nChannels += nDenseBlocks * growthRate\n",
    "        nOutChannels = int(math.floor(nChannels * reduction))\n",
    "        with self.name_scope():\n",
    "            self.trans2 = Transition(nOutChannels)\n",
    "\n",
    "        nChannels = nOutChannels\n",
    "        with self.name_scope():\n",
    "            self.dense3 = self._make_dense(growthRate, nDenseBlocks,\n",
    "                                           bottleneck)\n",
    "        nChannels += nDenseBlocks * growthRate\n",
    "\n",
    "        with self.name_scope():\n",
    "            self.bn1 = nn.BatchNorm()\n",
    "            self.fc = nn.Dense(nClasses)\n",
    "\n",
    "    def _make_dense(self, growthRate, nDenseBlocks, bottleneck):\n",
    "        layers = nn.HybridSequential()\n",
    "        for i in range(int(nDenseBlocks)):\n",
    "            if bottleneck:\n",
    "                layers.add(Bottleneck(growthRate))\n",
    "            else:\n",
    "                layers.add(SingleLayer(growthRate))\n",
    "        return layers\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        out = self.conv1(x)\n",
    "        out = self.trans1(self.dense1(out))\n",
    "        out = self.trans2(self.dense2(out))\n",
    "        out = self.dense3(out)\n",
    "        out = F.Pooling(\n",
    "            F.relu(self.bn1(out)),\n",
    "            global_pool=1,\n",
    "            pool_type='avg',\n",
    "            kernel=(8, 8))\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T17:02:39.386954Z",
     "start_time": "2017-10-27T17:02:39.310245Z"
    }
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "net = DenseNet(growthRate=12, depth=100, reduction=0.5,\n",
    "                            bottleneck=True, nClasses=10)\n",
    "net.hybridize()\n",
    "net.initialize(ctx=mx.gpu(0))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "在过拟合中我们讲过，过度依赖训练数据集的误差来推断测试数据集的误差容易导致过拟合。由于图像分类训练时间可能较长，为了方便，我们这里不再使用K折交叉验证，而是依赖验证集的结果来调参。\n",
    "定义模型训练函数。这里我们记录每个epoch的训练时间。这有助于我们比较不同模型设计的时间成本。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-28T03:37:31.751934Z",
     "start_time": "2017-10-28T03:37:31.709321Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import utils\n",
    "\n",
    "def train(net, train_data, valid_data, num_epochs, lr, wd, ctx, lr_period, lr_decay):\n",
    "    trainer = gluon.Trainer(\n",
    "        net.collect_params(), 'sgd', {'learning_rate': lr, 'momentum': 0.9, 'wd': wd})\n",
    "\n",
    "    prev_time = datetime.datetime.now()\n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss = 0.0\n",
    "        train_acc = 0.0\n",
    "        if epoch > 0 and (epoch == 89 or epoch == 139):\n",
    "            trainer.set_learning_rate(trainer.learning_rate * lr_decay)\n",
    "        for data, label in train_data:\n",
    "            label = label.as_in_context(ctx)\n",
    "            with autograd.record():\n",
    "                output = net(data.as_in_context(ctx))\n",
    "                loss = softmax_cross_entropy(output, label)\n",
    "            loss.backward()\n",
    "            trainer.step(batch_size)\n",
    "            train_loss += nd.mean(loss).asscalar()\n",
    "            train_acc += utils.accuracy(output, label)\n",
    "        cur_time = datetime.datetime.now()\n",
    "        h, remainder = divmod((cur_time - prev_time).seconds, 3600)\n",
    "        m, s = divmod(remainder, 60)\n",
    "        time_str = \"Time %02d:%02d:%02d\" % (h, m, s)\n",
    "        if valid_data is not None:\n",
    "            valid_acc = utils.evaluate_accuracy(valid_data, net, ctx)\n",
    "            epoch_str = (\"Epoch %d. Loss: %f, Train acc %f, Valid acc %f, \"\n",
    "                         % (epoch, train_loss / len(train_data),\n",
    "                            train_acc / len(train_data), valid_acc))\n",
    "        else:\n",
    "            epoch_str = (\"Epoch %d. Loss: %f, Train acc %f, \"\n",
    "                         % (epoch, train_loss / len(train_data),\n",
    "                            train_acc / len(train_data)))\n",
    "        prev_time = cur_time\n",
    "        print(epoch_str + time_str + ', lr ' + str(trainer.learning_rate))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "将依据验证集的结果不断优化模型设计和调整参数。依据下面的参数设置，优化算法的学习率将在每80个epoch自乘0.1。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-27T17:04:25.266220Z",
     "start_time": "2017-10-27T17:04:19.090507Z"
    }
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-fe752dc00ce1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;31m#et.hybridize()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m train(net, train_data, valid_data, num_epochs, learning_rate,\n\u001b[1;32m---> 15\u001b[1;33m       weight_decay, ctx, lr_period, lr_decay)\n\u001b[0m",
      "\u001b[1;32m<ipython-input-12-9335651a7cbc>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(net, train_data, valid_data, num_epochs, lr, wd, ctx, lr_period, lr_decay)\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m89\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m139\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m             \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_learning_rate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearning_rate\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlr_decay\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtrain_data\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m             \u001b[0mlabel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_in_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecord\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\gluon\\data\\dataloader.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     91\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_batch_sampler\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m             \u001b[1;32myield\u001b[0m \u001b[0m_batchify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\gluon\\data\\dataloader.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     91\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mbatch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_batch_sampler\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m             \u001b[1;32myield\u001b[0m \u001b[0m_batchify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\gluon\\data\\vision.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, idx)\u001b[0m\n\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 349\u001b[1;33m         \u001b[0mimg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    350\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\image\\image.py\u001b[0m in \u001b[0;36mimread\u001b[1;34m(filename, *args, **kwargs)\u001b[0m\n\u001b[0;32m     80\u001b[0m     \u001b[1;33m<\u001b[0m\u001b[0mNDArray\u001b[0m \u001b[1;36m224\u001b[0m\u001b[0mx224x3\u001b[0m \u001b[1;33m@\u001b[0m\u001b[0mcpu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     81\u001b[0m     \"\"\"\n\u001b[1;32m---> 82\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_internal\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_cvimread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\ndarray\\register.py\u001b[0m in \u001b[0;36m_cvimread\u001b[1;34m(filename, flag, to_rgb, out, name, **kwargs)\u001b[0m\n",
      "\u001b[1;32mE:\\software\\anaconda\\envs\\gluon\\lib\\site-packages\\mxnet\\_ctypes\\ndarray.py\u001b[0m in \u001b[0;36m_imperative_invoke\u001b[1;34m(handle, ndargs, keys, vals, out)\u001b[0m\n\u001b[0;32m     90\u001b[0m         \u001b[0mc_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_char_p\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mc_str\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[0mc_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mctypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_char_p\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mc_str\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mval\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mvals\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m         ctypes.byref(out_stypes)))\n\u001b[0m\u001b[0;32m     93\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0moriginal_output\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "ctx = utils.try_gpu()\n",
    "num_epochs = 200\n",
    "\n",
    "learning_rate = 0.1\n",
    "#weight_decay = 5e-4\n",
    "weight_decay = 1e-4\n",
    "lr_period = 90\n",
    "lr_decay = 0.1\n",
    "\n",
    "#net = get_dense_net(ctx)\n",
    "#net = get_net164_v2(ctx)\n",
    "#net = get_net(ctx)\n",
    "#et.hybridize()\n",
    "train(net, train_data, valid_data, num_epochs, learning_rate,\n",
    "      weight_decay, ctx, lr_period, lr_decay)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "当得到一组满意的模型设计和参数后，我们使用全部训练数据集（含验证集）重新训练模型，并对测试集分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-29T05:35:25.027307Z",
     "start_time": "2017-10-28T05:40:12.562306Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 0.301873, Train acc 0.896160, Time 00:13:10, lr 0.1\n",
      "Epoch 1. Loss: 0.295959, Train acc 0.897578, Time 00:12:37, lr 0.1\n",
      "Epoch 2. Loss: 0.293191, Train acc 0.897978, Time 00:13:04, lr 0.1\n",
      "Epoch 3. Loss: 0.286961, Train acc 0.900296, Time 00:13:07, lr 0.1\n",
      "Epoch 4. Loss: 0.289797, Train acc 0.899397, Time 00:13:06, lr 0.1\n",
      "Epoch 5. Loss: 0.278168, Train acc 0.903413, Time 00:13:10, lr 0.1\n",
      "Epoch 6. Loss: 0.273415, Train acc 0.903892, Time 00:13:17, lr 0.1\n",
      "Epoch 7. Loss: 0.272304, Train acc 0.904632, Time 00:13:09, lr 0.1\n",
      "Epoch 8. Loss: 0.267113, Train acc 0.905990, Time 00:13:08, lr 0.1\n",
      "Epoch 9. Loss: 0.266765, Train acc 0.907469, Time 00:13:09, lr 0.1\n",
      "Epoch 10. Loss: 0.259933, Train acc 0.910866, Time 00:13:10, lr 0.1\n",
      "Epoch 11. Loss: 0.258439, Train acc 0.909127, Time 00:13:11, lr 0.1\n",
      "Epoch 12. Loss: 0.259277, Train acc 0.909247, Time 00:13:10, lr 0.1\n",
      "Epoch 13. Loss: 0.254184, Train acc 0.911965, Time 00:13:10, lr 0.1\n",
      "Epoch 14. Loss: 0.260284, Train acc 0.910606, Time 00:13:12, lr 0.1\n",
      "Epoch 15. Loss: 0.247951, Train acc 0.912824, Time 00:13:12, lr 0.1\n",
      "Epoch 16. Loss: 0.253460, Train acc 0.911645, Time 00:13:10, lr 0.1\n",
      "Epoch 17. Loss: 0.248304, Train acc 0.914322, Time 00:13:14, lr 0.1\n",
      "Epoch 18. Loss: 0.253473, Train acc 0.912104, Time 00:15:50, lr 0.1\n",
      "Epoch 19. Loss: 0.240107, Train acc 0.915941, Time 00:12:36, lr 0.1\n",
      "Epoch 20. Loss: 0.240009, Train acc 0.915861, Time 00:10:47, lr 0.1\n",
      "Epoch 21. Loss: 0.240145, Train acc 0.916340, Time 00:11:36, lr 0.1\n",
      "Epoch 22. Loss: 0.235877, Train acc 0.917060, Time 00:12:53, lr 0.1\n",
      "Epoch 23. Loss: 0.233228, Train acc 0.918938, Time 00:13:06, lr 0.1\n",
      "Epoch 24. Loss: 0.240312, Train acc 0.916640, Time 00:13:14, lr 0.1\n",
      "Epoch 25. Loss: 0.232444, Train acc 0.918978, Time 00:13:02, lr 0.1\n",
      "Epoch 26. Loss: 0.232858, Train acc 0.918498, Time 00:13:06, lr 0.1\n",
      "Epoch 27. Loss: 0.225353, Train acc 0.922794, Time 00:13:08, lr 0.1\n",
      "Epoch 28. Loss: 0.225599, Train acc 0.921156, Time 00:13:12, lr 0.1\n",
      "Epoch 29. Loss: 0.223827, Train acc 0.920996, Time 00:13:08, lr 0.1\n",
      "Epoch 30. Loss: 0.224894, Train acc 0.921336, Time 00:13:08, lr 0.1\n",
      "Epoch 31. Loss: 0.225811, Train acc 0.920676, Time 00:13:10, lr 0.1\n",
      "Epoch 32. Loss: 0.222406, Train acc 0.922754, Time 00:13:10, lr 0.1\n",
      "Epoch 33. Loss: 0.220951, Train acc 0.923693, Time 00:13:11, lr 0.1\n",
      "Epoch 34. Loss: 0.220711, Train acc 0.923434, Time 00:13:17, lr 0.1\n",
      "Epoch 35. Loss: 0.217264, Train acc 0.924453, Time 00:13:47, lr 0.1\n",
      "Epoch 36. Loss: 0.213055, Train acc 0.925432, Time 00:09:01, lr 0.1\n",
      "Epoch 37. Loss: 0.213904, Train acc 0.924632, Time 00:04:28, lr 0.1\n",
      "Epoch 38. Loss: 0.224523, Train acc 0.922015, Time 00:04:28, lr 0.1\n",
      "Epoch 39. Loss: 0.213644, Train acc 0.924972, Time 00:04:28, lr 0.1\n",
      "Epoch 40. Loss: 0.211891, Train acc 0.925711, Time 00:04:28, lr 0.1\n",
      "Epoch 41. Loss: 0.207957, Train acc 0.927550, Time 00:04:28, lr 0.1\n",
      "Epoch 42. Loss: 0.211884, Train acc 0.926570, Time 00:04:28, lr 0.1\n",
      "Epoch 43. Loss: 0.209725, Train acc 0.925991, Time 00:04:28, lr 0.1\n",
      "Epoch 44. Loss: 0.209799, Train acc 0.927170, Time 00:04:28, lr 0.1\n",
      "Epoch 45. Loss: 0.204355, Train acc 0.928249, Time 00:04:28, lr 0.1\n",
      "Epoch 46. Loss: 0.213633, Train acc 0.924932, Time 00:04:28, lr 0.1\n",
      "Epoch 47. Loss: 0.205516, Train acc 0.928768, Time 00:07:31, lr 0.1\n",
      "Epoch 48. Loss: 0.207727, Train acc 0.926131, Time 00:04:39, lr 0.1\n",
      "Epoch 49. Loss: 0.203361, Train acc 0.929228, Time 00:04:29, lr 0.1\n",
      "Epoch 50. Loss: 0.210919, Train acc 0.927430, Time 00:04:33, lr 0.1\n",
      "Epoch 51. Loss: 0.198121, Train acc 0.931466, Time 00:04:30, lr 0.1\n",
      "Epoch 52. Loss: 0.199710, Train acc 0.931805, Time 00:10:31, lr 0.1\n",
      "Epoch 53. Loss: 0.202283, Train acc 0.929747, Time 00:13:54, lr 0.1\n",
      "Epoch 54. Loss: 0.208043, Train acc 0.926930, Time 00:06:59, lr 0.1\n",
      "Epoch 55. Loss: 0.200010, Train acc 0.929967, Time 00:04:29, lr 0.1\n",
      "Epoch 56. Loss: 0.203477, Train acc 0.928109, Time 00:04:30, lr 0.1\n",
      "Epoch 57. Loss: 0.197997, Train acc 0.930766, Time 00:04:31, lr 0.1\n",
      "Epoch 58. Loss: 0.201568, Train acc 0.929208, Time 00:04:30, lr 0.1\n",
      "Epoch 59. Loss: 0.202321, Train acc 0.930027, Time 00:04:31, lr 0.1\n",
      "Epoch 60. Loss: 0.197953, Train acc 0.931186, Time 00:04:31, lr 0.1\n",
      "Epoch 61. Loss: 0.197037, Train acc 0.930746, Time 00:04:31, lr 0.1\n",
      "Epoch 62. Loss: 0.196541, Train acc 0.931726, Time 00:04:32, lr 0.1\n",
      "Epoch 63. Loss: 0.198479, Train acc 0.929847, Time 00:04:33, lr 0.1\n",
      "Epoch 64. Loss: 0.198324, Train acc 0.931766, Time 00:04:29, lr 0.1\n",
      "Epoch 65. Loss: 0.198008, Train acc 0.932165, Time 00:04:29, lr 0.1\n",
      "Epoch 66. Loss: 0.200981, Train acc 0.930786, Time 00:04:30, lr 0.1\n",
      "Epoch 67. Loss: 0.187689, Train acc 0.934143, Time 00:04:30, lr 0.1\n",
      "Epoch 68. Loss: 0.200901, Train acc 0.929568, Time 00:04:30, lr 0.1\n",
      "Epoch 69. Loss: 0.193690, Train acc 0.932645, Time 00:04:29, lr 0.1\n",
      "Epoch 70. Loss: 0.194526, Train acc 0.931426, Time 00:04:28, lr 0.1\n",
      "Epoch 71. Loss: 0.196890, Train acc 0.931526, Time 00:04:28, lr 0.1\n",
      "Epoch 72. Loss: 0.193490, Train acc 0.932225, Time 00:04:28, lr 0.1\n",
      "Epoch 73. Loss: 0.193109, Train acc 0.933144, Time 00:04:28, lr 0.1\n",
      "Epoch 74. Loss: 0.190727, Train acc 0.932685, Time 00:04:29, lr 0.1\n",
      "Epoch 75. Loss: 0.193092, Train acc 0.931865, Time 00:04:28, lr 0.1\n",
      "Epoch 76. Loss: 0.194040, Train acc 0.934523, Time 00:04:29, lr 0.1\n",
      "Epoch 77. Loss: 0.191123, Train acc 0.933244, Time 00:04:29, lr 0.1\n",
      "Epoch 78. Loss: 0.188845, Train acc 0.934982, Time 00:04:29, lr 0.1\n",
      "Epoch 79. Loss: 0.193697, Train acc 0.931606, Time 00:04:30, lr 0.1\n",
      "Epoch 80. Loss: 0.192889, Train acc 0.933424, Time 00:04:29, lr 0.1\n",
      "Epoch 81. Loss: 0.191655, Train acc 0.934143, Time 00:04:29, lr 0.1\n",
      "Epoch 82. Loss: 0.189441, Train acc 0.934083, Time 00:04:30, lr 0.1\n",
      "Epoch 83. Loss: 0.185098, Train acc 0.934863, Time 00:04:31, lr 0.1\n",
      "Epoch 84. Loss: 0.186849, Train acc 0.935102, Time 00:04:28, lr 0.1\n",
      "Epoch 85. Loss: 0.185421, Train acc 0.935662, Time 00:04:32, lr 0.1\n",
      "Epoch 86. Loss: 0.186422, Train acc 0.935082, Time 00:04:32, lr 0.1\n",
      "Epoch 87. Loss: 0.197322, Train acc 0.931366, Time 00:04:33, lr 0.1\n",
      "Epoch 88. Loss: 0.181944, Train acc 0.937320, Time 00:04:28, lr 0.1\n",
      "Epoch 89. Loss: 0.080749, Train acc 0.974704, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 90. Loss: 0.036652, Train acc 0.990229, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 91. Loss: 0.024397, Train acc 0.994985, Time 00:04:31, lr 0.010000000000000002\n",
      "Epoch 92. Loss: 0.017840, Train acc 0.996983, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 93. Loss: 0.014168, Train acc 0.997622, Time 00:04:32, lr 0.010000000000000002\n",
      "Epoch 94. Loss: 0.010743, Train acc 0.998581, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 95. Loss: 0.009614, Train acc 0.998761, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 96. Loss: 0.007827, Train acc 0.999241, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 97. Loss: 0.006582, Train acc 0.999500, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 98. Loss: 0.005795, Train acc 0.999520, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 99. Loss: 0.005074, Train acc 0.999800, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 100. Loss: 0.004825, Train acc 0.999520, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 101. Loss: 0.004274, Train acc 0.999700, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 102. Loss: 0.003903, Train acc 0.999780, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 103. Loss: 0.003755, Train acc 0.999740, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 104. Loss: 0.003413, Train acc 0.999900, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 105. Loss: 0.003297, Train acc 0.999820, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 106. Loss: 0.003171, Train acc 0.999900, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 107. Loss: 0.002860, Train acc 0.999920, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 108. Loss: 0.002486, Train acc 0.999960, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 109. Loss: 0.002532, Train acc 0.999900, Time 00:04:33, lr 0.010000000000000002\n",
      "Epoch 110. Loss: 0.002526, Train acc 0.999900, Time 00:04:31, lr 0.010000000000000002\n",
      "Epoch 111. Loss: 0.002266, Train acc 0.999980, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 112. Loss: 0.002397, Train acc 0.999920, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 113. Loss: 0.002257, Train acc 0.999980, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 114. Loss: 0.002225, Train acc 0.999940, Time 00:04:31, lr 0.010000000000000002\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 115. Loss: 0.002540, Train acc 0.999740, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 116. Loss: 0.001992, Train acc 0.999900, Time 00:04:33, lr 0.010000000000000002\n",
      "Epoch 117. Loss: 0.002054, Train acc 0.999940, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 118. Loss: 0.001883, Train acc 0.999960, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 119. Loss: 0.001771, Train acc 0.999920, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 120. Loss: 0.001882, Train acc 0.999960, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 121. Loss: 0.001791, Train acc 0.999960, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 122. Loss: 0.001978, Train acc 0.999840, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 123. Loss: 0.001871, Train acc 0.999960, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 124. Loss: 0.001775, Train acc 1.000000, Time 00:04:32, lr 0.010000000000000002\n",
      "Epoch 125. Loss: 0.001906, Train acc 0.999980, Time 00:04:32, lr 0.010000000000000002\n",
      "Epoch 126. Loss: 0.001825, Train acc 0.999940, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 127. Loss: 0.001611, Train acc 1.000000, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 128. Loss: 0.001660, Train acc 0.999880, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 129. Loss: 0.001747, Train acc 0.999960, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 130. Loss: 0.001574, Train acc 1.000000, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 131. Loss: 0.001467, Train acc 0.999960, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 132. Loss: 0.001627, Train acc 0.999940, Time 00:04:29, lr 0.010000000000000002\n",
      "Epoch 133. Loss: 0.001467, Train acc 0.999980, Time 00:04:27, lr 0.010000000000000002\n",
      "Epoch 134. Loss: 0.001448, Train acc 1.000000, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 135. Loss: 0.001469, Train acc 0.999960, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 136. Loss: 0.001400, Train acc 0.999980, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 137. Loss: 0.001405, Train acc 0.999980, Time 00:04:28, lr 0.010000000000000002\n",
      "Epoch 138. Loss: 0.001393, Train acc 1.000000, Time 00:04:30, lr 0.010000000000000002\n",
      "Epoch 139. Loss: 0.001303, Train acc 0.999980, Time 00:04:29, lr 0.0010000000000000002\n",
      "Epoch 140. Loss: 0.001302, Train acc 1.000000, Time 00:07:12, lr 0.0010000000000000002\n",
      "Epoch 141. Loss: 0.001239, Train acc 1.000000, Time 00:05:35, lr 0.0010000000000000002\n",
      "Epoch 142. Loss: 0.001220, Train acc 1.000000, Time 00:07:22, lr 0.0010000000000000002\n",
      "Epoch 143. Loss: 0.001241, Train acc 0.999980, Time 00:07:08, lr 0.0010000000000000002\n",
      "Epoch 144. Loss: 0.001264, Train acc 1.000000, Time 00:08:49, lr 0.0010000000000000002\n",
      "Epoch 145. Loss: 0.001246, Train acc 1.000000, Time 00:08:57, lr 0.0010000000000000002\n",
      "Epoch 146. Loss: 0.001256, Train acc 1.000000, Time 00:10:27, lr 0.0010000000000000002\n",
      "Epoch 147. Loss: 0.001251, Train acc 0.999980, Time 00:11:39, lr 0.0010000000000000002\n",
      "Epoch 148. Loss: 0.001293, Train acc 0.999980, Time 00:11:29, lr 0.0010000000000000002\n",
      "Epoch 149. Loss: 0.001252, Train acc 1.000000, Time 00:12:27, lr 0.0010000000000000002\n",
      "Epoch 150. Loss: 0.001185, Train acc 0.999980, Time 00:12:31, lr 0.0010000000000000002\n",
      "Epoch 151. Loss: 0.001226, Train acc 1.000000, Time 00:12:54, lr 0.0010000000000000002\n",
      "Epoch 152. Loss: 0.001190, Train acc 1.000000, Time 00:13:03, lr 0.0010000000000000002\n",
      "Epoch 153. Loss: 0.001227, Train acc 0.999960, Time 00:13:10, lr 0.0010000000000000002\n",
      "Epoch 154. Loss: 0.001281, Train acc 1.000000, Time 00:13:11, lr 0.0010000000000000002\n",
      "Epoch 155. Loss: 0.001506, Train acc 0.999880, Time 00:13:13, lr 0.0010000000000000002\n",
      "Epoch 156. Loss: 0.001186, Train acc 1.000000, Time 00:13:11, lr 0.0010000000000000002\n",
      "Epoch 157. Loss: 0.001175, Train acc 1.000000, Time 00:13:13, lr 0.0010000000000000002\n",
      "Epoch 158. Loss: 0.001205, Train acc 0.999980, Time 00:13:10, lr 0.0010000000000000002\n",
      "Epoch 159. Loss: 0.001127, Train acc 1.000000, Time 00:13:12, lr 0.0010000000000000002\n",
      "Epoch 160. Loss: 0.001153, Train acc 1.000000, Time 00:13:13, lr 0.0010000000000000002\n",
      "Epoch 161. Loss: 0.001102, Train acc 1.000000, Time 00:13:19, lr 0.0010000000000000002\n",
      "Epoch 162. Loss: 0.001217, Train acc 1.000000, Time 00:12:39, lr 0.0010000000000000002\n",
      "Epoch 163. Loss: 0.001151, Train acc 1.000000, Time 00:09:56, lr 0.0010000000000000002\n",
      "Epoch 164. Loss: 0.001167, Train acc 0.999980, Time 00:04:30, lr 0.0010000000000000002\n",
      "Epoch 165. Loss: 0.001139, Train acc 1.000000, Time 00:04:29, lr 0.0010000000000000002\n",
      "Epoch 166. Loss: 0.001152, Train acc 0.999980, Time 00:08:56, lr 0.0010000000000000002\n",
      "Epoch 167. Loss: 0.001258, Train acc 0.999960, Time 00:12:50, lr 0.0010000000000000002\n",
      "Epoch 168. Loss: 0.001164, Train acc 1.000000, Time 00:09:51, lr 0.0010000000000000002\n",
      "Epoch 169. Loss: 0.001207, Train acc 1.000000, Time 00:04:38, lr 0.0010000000000000002\n",
      "Epoch 170. Loss: 0.001252, Train acc 0.999920, Time 00:04:42, lr 0.0010000000000000002\n",
      "Epoch 171. Loss: 0.001175, Train acc 0.999980, Time 00:04:51, lr 0.0010000000000000002\n",
      "Epoch 172. Loss: 0.001166, Train acc 0.999980, Time 00:04:41, lr 0.0010000000000000002\n",
      "Epoch 173. Loss: 0.001295, Train acc 0.999960, Time 00:04:38, lr 0.0010000000000000002\n",
      "Epoch 174. Loss: 0.001124, Train acc 1.000000, Time 00:04:40, lr 0.0010000000000000002\n",
      "Epoch 175. Loss: 0.001190, Train acc 0.999980, Time 00:04:40, lr 0.0010000000000000002\n",
      "Epoch 176. Loss: 0.001131, Train acc 1.000000, Time 00:04:40, lr 0.0010000000000000002\n",
      "Epoch 177. Loss: 0.001133, Train acc 1.000000, Time 00:04:42, lr 0.0010000000000000002\n",
      "Epoch 178. Loss: 0.001200, Train acc 0.999980, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 179. Loss: 0.001108, Train acc 1.000000, Time 00:04:37, lr 0.0010000000000000002\n",
      "Epoch 180. Loss: 0.001101, Train acc 1.000000, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 181. Loss: 0.001134, Train acc 1.000000, Time 00:04:34, lr 0.0010000000000000002\n",
      "Epoch 182. Loss: 0.001061, Train acc 1.000000, Time 00:04:34, lr 0.0010000000000000002\n",
      "Epoch 183. Loss: 0.001131, Train acc 1.000000, Time 00:04:34, lr 0.0010000000000000002\n",
      "Epoch 184. Loss: 0.001083, Train acc 1.000000, Time 00:04:36, lr 0.0010000000000000002\n",
      "Epoch 185. Loss: 0.001229, Train acc 0.999960, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 186. Loss: 0.001103, Train acc 1.000000, Time 00:04:37, lr 0.0010000000000000002\n",
      "Epoch 187. Loss: 0.001066, Train acc 1.000000, Time 00:04:36, lr 0.0010000000000000002\n",
      "Epoch 188. Loss: 0.001143, Train acc 1.000000, Time 00:04:32, lr 0.0010000000000000002\n",
      "Epoch 189. Loss: 0.001103, Train acc 1.000000, Time 00:04:34, lr 0.0010000000000000002\n",
      "Epoch 190. Loss: 0.001094, Train acc 0.999980, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 191. Loss: 0.001109, Train acc 1.000000, Time 00:04:32, lr 0.0010000000000000002\n",
      "Epoch 192. Loss: 0.001141, Train acc 1.000000, Time 00:04:36, lr 0.0010000000000000002\n",
      "Epoch 193. Loss: 0.001229, Train acc 1.000000, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 194. Loss: 0.001086, Train acc 1.000000, Time 00:04:33, lr 0.0010000000000000002\n",
      "Epoch 195. Loss: 0.001079, Train acc 0.999980, Time 00:04:36, lr 0.0010000000000000002\n",
      "Epoch 196. Loss: 0.001187, Train acc 0.999980, Time 00:04:37, lr 0.0010000000000000002\n",
      "Epoch 197. Loss: 0.001067, Train acc 1.000000, Time 00:04:37, lr 0.0010000000000000002\n",
      "Epoch 198. Loss: 0.001107, Train acc 1.000000, Time 00:04:34, lr 0.0010000000000000002\n",
      "Epoch 199. Loss: 0.001413, Train acc 0.999880, Time 00:04:36, lr 0.0010000000000000002\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#net = get_net(ctx)\n",
    "#net = get_dense_net(ctx)\n",
    "net.hybridize()\n",
    "train(net, train_valid_data, None, num_epochs, learning_rate,\n",
    "      weight_decay, ctx, lr_period, lr_decay)\n",
    "\n",
    "preds = []\n",
    "for data, label in test_data:\n",
    "    output = net(data.as_in_context(ctx))\n",
    "    preds.extend(output.argmax(axis=1).astype(int).asnumpy())\n",
    "\n",
    "sorted_ids = list(range(1, len(test_ds) + 1))\n",
    "sorted_ids.sort(key = lambda x:str(x))\n",
    "\n",
    "df = pd.DataFrame({'id': sorted_ids, 'label': preds})\n",
    "df['label'] = df['label'].apply(lambda x: train_valid_ds.synsets[x])\n",
    "df.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}