pytorch-mnist-CVAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prerequisites\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "bs = 100\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./mnist_data/', train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = datasets.MNIST(root='./mnist_data/', train=False, transform=transforms.ToTensor(), download=False)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=bs, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=bs, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CVAE(nn.Module):\n",
    "    def __init__(self, x_dim, h_dim1, h_dim2, z_dim, c_dim):\n",
    "        super(CVAE, self).__init__()\n",
    "        \n",
    "        # encoder part\n",
    "        self.fc1 = nn.Linear(x_dim + c_dim, h_dim1)\n",
    "        self.fc2 = nn.Linear(h_dim1, h_dim2)\n",
    "        self.fc31 = nn.Linear(h_dim2, z_dim)\n",
    "        self.fc32 = nn.Linear(h_dim2, z_dim)\n",
    "        # decoder part\n",
    "        self.fc4 = nn.Linear(z_dim + c_dim, h_dim2)\n",
    "        self.fc5 = nn.Linear(h_dim2, h_dim1)\n",
    "        self.fc6 = nn.Linear(h_dim1, x_dim)\n",
    "    \n",
    "    def encoder(self, x, c):\n",
    "        concat_input = torch.cat([x, c], 1)\n",
    "        h = F.relu(self.fc1(concat_input))\n",
    "        h = F.relu(self.fc2(h))\n",
    "        return self.fc31(h), self.fc32(h)\n",
    "    \n",
    "    def sampling(self, mu, log_var):\n",
    "        std = torch.exp(0.5*log_var)\n",
    "        eps = torch.randn_like(std)\n",
    "        return eps.mul(std).add(mu) # return z sample\n",
    "    \n",
    "    def decoder(self, z, c):\n",
    "        concat_input = torch.cat([z, c], 1)\n",
    "        h = F.relu(self.fc4(concat_input))\n",
    "        h = F.relu(self.fc5(h))\n",
    "        return F.sigmoid(self.fc6(h))\n",
    "    \n",
    "    def forward(self, x, c):\n",
    "        mu, log_var = self.encoder(x.view(-1, 784), c)\n",
    "        z = self.sampling(mu, log_var)\n",
    "        return self.decoder(z, c), mu, log_var\n",
    "\n",
    "# build model\n",
    "cond_dim = train_loader.dataset.train_labels.unique().size(0)\n",
    "cvae = CVAE(x_dim=784, h_dim1=512, h_dim2=256, z_dim=2, c_dim=cond_dim)\n",
    "if torch.cuda.is_available():\n",
    "    cvae.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CVAE(\n",
       "  (fc1): Linear(in_features=794, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc31): Linear(in_features=256, out_features=2, bias=True)\n",
       "  (fc32): Linear(in_features=256, out_features=2, bias=True)\n",
       "  (fc4): Linear(in_features=12, out_features=256, bias=True)\n",
       "  (fc5): Linear(in_features=256, out_features=512, bias=True)\n",
       "  (fc6): Linear(in_features=512, out_features=784, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(cvae.parameters())\n",
    "# return reconstruction error + KL divergence losses\n",
    "def loss_function(recon_x, x, mu, log_var):\n",
    "    BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum')\n",
    "    KLD = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())\n",
    "    return BCE + KLD\n",
    "\n",
    "# one-hot encoding\n",
    "def one_hot(labels, class_size): \n",
    "    targets = torch.zeros(labels.size(0), class_size)\n",
    "    for i, label in enumerate(labels):\n",
    "        targets[i, label] = 1\n",
    "    return Variable(targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    cvae.train()\n",
    "    train_loss = 0\n",
    "    for batch_idx, (data, cond) in enumerate(train_loader):\n",
    "        data, cond = data.cuda(), one_hot(cond, cond_dim).cuda()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        recon_batch, mu, log_var = cvae(data, cond)\n",
    "        loss = loss_function(recon_batch, data, mu, log_var)\n",
    "        \n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item() / len(data)))\n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    cvae.eval()\n",
    "    test_loss= 0\n",
    "    with torch.no_grad():\n",
    "        for data, cond in test_loader:\n",
    "            data, cond = data.cuda(), one_hot(cond, cond_dim).cuda()\n",
    "            recon, mu, log_var = cvae(data, cond)\n",
    "            # sum up batch loss\n",
    "            test_loss += loss_function(recon, data, mu, log_var).item()\n",
    "        \n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('====> Test set loss: {:.4f}'.format(test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lyeoni\\AppData\\Local\\Continuum\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\torch\\nn\\functional.py:1006: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 543.434844\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 171.279551\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 151.962598\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 150.051455\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 138.287529\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 141.765605\n",
      "====> Epoch: 1 Average loss: 161.7477\n",
      "====> Test set loss: 141.0998\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 131.872637\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 133.000068\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 135.350332\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 136.489717\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 138.473506\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 133.425820\n",
      "====> Epoch: 2 Average loss: 138.5404\n",
      "====> Test set loss: 136.8007\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 138.504160\n",
      "Train Epoch: 3 [10000/60000 (17%)]\tLoss: 139.224199\n",
      "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 141.583350\n",
      "Train Epoch: 3 [30000/60000 (50%)]\tLoss: 135.837793\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 135.882217\n",
      "Train Epoch: 3 [50000/60000 (83%)]\tLoss: 138.874668\n",
      "====> Epoch: 3 Average loss: 135.2442\n",
      "====> Test set loss: 134.5329\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 133.129551\n",
      "Train Epoch: 4 [10000/60000 (17%)]\tLoss: 131.289424\n",
      "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 129.505313\n",
      "Train Epoch: 4 [30000/60000 (50%)]\tLoss: 139.929492\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 127.387500\n",
      "Train Epoch: 4 [50000/60000 (83%)]\tLoss: 136.439063\n",
      "====> Epoch: 4 Average loss: 133.5468\n",
      "====> Test set loss: 133.1579\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 126.536963\n",
      "Train Epoch: 5 [10000/60000 (17%)]\tLoss: 134.488359\n",
      "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 136.722021\n",
      "Train Epoch: 5 [30000/60000 (50%)]\tLoss: 132.605830\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 135.631250\n",
      "Train Epoch: 5 [50000/60000 (83%)]\tLoss: 129.641113\n",
      "====> Epoch: 5 Average loss: 132.3955\n",
      "====> Test set loss: 132.3415\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 129.747383\n",
      "Train Epoch: 6 [10000/60000 (17%)]\tLoss: 132.315430\n",
      "Train Epoch: 6 [20000/60000 (33%)]\tLoss: 134.635645\n",
      "Train Epoch: 6 [30000/60000 (50%)]\tLoss: 135.810234\n",
      "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 132.703115\n",
      "Train Epoch: 6 [50000/60000 (83%)]\tLoss: 133.774209\n",
      "====> Epoch: 6 Average loss: 131.5678\n",
      "====> Test set loss: 131.8086\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 129.216807\n",
      "Train Epoch: 7 [10000/60000 (17%)]\tLoss: 123.999443\n",
      "Train Epoch: 7 [20000/60000 (33%)]\tLoss: 136.004893\n",
      "Train Epoch: 7 [30000/60000 (50%)]\tLoss: 120.858408\n",
      "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 125.447529\n",
      "Train Epoch: 7 [50000/60000 (83%)]\tLoss: 127.267295\n",
      "====> Epoch: 7 Average loss: 131.0001\n",
      "====> Test set loss: 131.1121\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 129.502617\n",
      "Train Epoch: 8 [10000/60000 (17%)]\tLoss: 138.006436\n",
      "Train Epoch: 8 [20000/60000 (33%)]\tLoss: 129.243398\n",
      "Train Epoch: 8 [30000/60000 (50%)]\tLoss: 137.556279\n",
      "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 131.780664\n",
      "Train Epoch: 8 [50000/60000 (83%)]\tLoss: 137.294980\n",
      "====> Epoch: 8 Average loss: 130.4619\n",
      "====> Test set loss: 130.8446\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 133.194580\n",
      "Train Epoch: 9 [10000/60000 (17%)]\tLoss: 140.896768\n",
      "Train Epoch: 9 [20000/60000 (33%)]\tLoss: 135.614844\n",
      "Train Epoch: 9 [30000/60000 (50%)]\tLoss: 126.372451\n",
      "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 138.886543\n",
      "Train Epoch: 9 [50000/60000 (83%)]\tLoss: 122.417012\n",
      "====> Epoch: 9 Average loss: 130.0094\n",
      "====> Test set loss: 130.4995\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 130.353721\n",
      "Train Epoch: 10 [10000/60000 (17%)]\tLoss: 127.331006\n",
      "Train Epoch: 10 [20000/60000 (33%)]\tLoss: 124.470029\n",
      "Train Epoch: 10 [30000/60000 (50%)]\tLoss: 130.140059\n",
      "Train Epoch: 10 [40000/60000 (67%)]\tLoss: 131.078369\n",
      "Train Epoch: 10 [50000/60000 (83%)]\tLoss: 125.839062\n",
      "====> Epoch: 10 Average loss: 129.6555\n",
      "====> Test set loss: 130.4557\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 124.079463\n",
      "Train Epoch: 11 [10000/60000 (17%)]\tLoss: 128.753428\n",
      "Train Epoch: 11 [20000/60000 (33%)]\tLoss: 129.707314\n",
      "Train Epoch: 11 [30000/60000 (50%)]\tLoss: 129.212959\n",
      "Train Epoch: 11 [40000/60000 (67%)]\tLoss: 134.728613\n",
      "Train Epoch: 11 [50000/60000 (83%)]\tLoss: 125.235049\n",
      "====> Epoch: 11 Average loss: 129.3203\n",
      "====> Test set loss: 129.9129\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 129.577803\n",
      "Train Epoch: 12 [10000/60000 (17%)]\tLoss: 130.374844\n",
      "Train Epoch: 12 [20000/60000 (33%)]\tLoss: 129.299102\n",
      "Train Epoch: 12 [30000/60000 (50%)]\tLoss: 137.213711\n",
      "Train Epoch: 12 [40000/60000 (67%)]\tLoss: 120.397568\n",
      "Train Epoch: 12 [50000/60000 (83%)]\tLoss: 124.706602\n",
      "====> Epoch: 12 Average loss: 128.9778\n",
      "====> Test set loss: 129.9541\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 130.451846\n",
      "Train Epoch: 13 [10000/60000 (17%)]\tLoss: 131.589805\n",
      "Train Epoch: 13 [20000/60000 (33%)]\tLoss: 125.772773\n",
      "Train Epoch: 13 [30000/60000 (50%)]\tLoss: 122.174785\n",
      "Train Epoch: 13 [40000/60000 (67%)]\tLoss: 129.417246\n",
      "Train Epoch: 13 [50000/60000 (83%)]\tLoss: 132.940732\n",
      "====> Epoch: 13 Average loss: 128.6610\n",
      "====> Test set loss: 129.6289\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 130.284395\n",
      "Train Epoch: 14 [10000/60000 (17%)]\tLoss: 124.329639\n",
      "Train Epoch: 14 [20000/60000 (33%)]\tLoss: 121.214775\n",
      "Train Epoch: 14 [30000/60000 (50%)]\tLoss: 130.364678\n",
      "Train Epoch: 14 [40000/60000 (67%)]\tLoss: 121.170928\n",
      "Train Epoch: 14 [50000/60000 (83%)]\tLoss: 125.343154\n",
      "====> Epoch: 14 Average loss: 128.4466\n",
      "====> Test set loss: 129.5683\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 123.866035\n",
      "Train Epoch: 15 [10000/60000 (17%)]\tLoss: 125.836924\n",
      "Train Epoch: 15 [20000/60000 (33%)]\tLoss: 132.645146\n",
      "Train Epoch: 15 [30000/60000 (50%)]\tLoss: 129.691836\n",
      "Train Epoch: 15 [40000/60000 (67%)]\tLoss: 128.270674\n",
      "Train Epoch: 15 [50000/60000 (83%)]\tLoss: 137.676826\n",
      "====> Epoch: 15 Average loss: 128.1550\n",
      "====> Test set loss: 129.4420\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 120.917705\n",
      "Train Epoch: 16 [10000/60000 (17%)]\tLoss: 122.047549\n",
      "Train Epoch: 16 [20000/60000 (33%)]\tLoss: 131.975264\n",
      "Train Epoch: 16 [30000/60000 (50%)]\tLoss: 123.476270\n",
      "Train Epoch: 16 [40000/60000 (67%)]\tLoss: 128.141191\n",
      "Train Epoch: 16 [50000/60000 (83%)]\tLoss: 134.978047\n",
      "====> Epoch: 16 Average loss: 127.8981\n",
      "====> Test set loss: 129.3484\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 128.595898\n",
      "Train Epoch: 17 [10000/60000 (17%)]\tLoss: 124.345088\n",
      "Train Epoch: 17 [20000/60000 (33%)]\tLoss: 133.704053\n",
      "Train Epoch: 17 [30000/60000 (50%)]\tLoss: 128.807676\n",
      "Train Epoch: 17 [40000/60000 (67%)]\tLoss: 126.502119\n",
      "Train Epoch: 17 [50000/60000 (83%)]\tLoss: 140.141514\n",
      "====> Epoch: 17 Average loss: 127.6752\n",
      "====> Test set loss: 128.9800\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 124.379688\n",
      "Train Epoch: 18 [10000/60000 (17%)]\tLoss: 127.387148\n",
      "Train Epoch: 18 [20000/60000 (33%)]\tLoss: 134.619922\n",
      "Train Epoch: 18 [30000/60000 (50%)]\tLoss: 127.170801\n",
      "Train Epoch: 18 [40000/60000 (67%)]\tLoss: 126.707051\n",
      "Train Epoch: 18 [50000/60000 (83%)]\tLoss: 123.796494\n",
      "====> Epoch: 18 Average loss: 127.4415\n",
      "====> Test set loss: 128.9175\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 133.177246\n",
      "Train Epoch: 19 [10000/60000 (17%)]\tLoss: 127.608613\n",
      "Train Epoch: 19 [20000/60000 (33%)]\tLoss: 121.038291\n",
      "Train Epoch: 19 [30000/60000 (50%)]\tLoss: 123.370674\n",
      "Train Epoch: 19 [40000/60000 (67%)]\tLoss: 123.735811\n",
      "Train Epoch: 19 [50000/60000 (83%)]\tLoss: 124.982715\n",
      "====> Epoch: 19 Average loss: 127.2716\n",
      "====> Test set loss: 128.9586\n",
      "Train Epoch: 20 [0/60000 (0%)]\tLoss: 126.108154\n",
      "Train Epoch: 20 [10000/60000 (17%)]\tLoss: 132.094219\n",
      "Train Epoch: 20 [20000/60000 (33%)]\tLoss: 128.577197\n",
      "Train Epoch: 20 [30000/60000 (50%)]\tLoss: 121.588105\n",
      "Train Epoch: 20 [40000/60000 (67%)]\tLoss: 129.193496\n",
      "Train Epoch: 20 [50000/60000 (83%)]\tLoss: 128.565859\n",
      "====> Epoch: 20 Average loss: 127.0758\n",
      "====> Test set loss: 129.0698\n",
      "Train Epoch: 21 [0/60000 (0%)]\tLoss: 125.990215\n",
      "Train Epoch: 21 [10000/60000 (17%)]\tLoss: 124.129238\n",
      "Train Epoch: 21 [20000/60000 (33%)]\tLoss: 132.637432\n",
      "Train Epoch: 21 [30000/60000 (50%)]\tLoss: 134.627490\n",
      "Train Epoch: 21 [40000/60000 (67%)]\tLoss: 131.386113\n",
      "Train Epoch: 21 [50000/60000 (83%)]\tLoss: 118.270098\n",
      "====> Epoch: 21 Average loss: 126.8905\n",
      "====> Test set loss: 128.7019\n",
      "Train Epoch: 22 [0/60000 (0%)]\tLoss: 122.030664\n",
      "Train Epoch: 22 [10000/60000 (17%)]\tLoss: 127.339102\n",
      "Train Epoch: 22 [20000/60000 (33%)]\tLoss: 122.283379\n",
      "Train Epoch: 22 [30000/60000 (50%)]\tLoss: 128.075762\n",
      "Train Epoch: 22 [40000/60000 (67%)]\tLoss: 121.986211\n",
      "Train Epoch: 22 [50000/60000 (83%)]\tLoss: 127.967393\n",
      "====> Epoch: 22 Average loss: 126.7199\n",
      "====> Test set loss: 128.3939\n",
      "Train Epoch: 23 [0/60000 (0%)]\tLoss: 124.445947\n",
      "Train Epoch: 23 [10000/60000 (17%)]\tLoss: 130.335420\n",
      "Train Epoch: 23 [20000/60000 (33%)]\tLoss: 138.316787\n",
      "Train Epoch: 23 [30000/60000 (50%)]\tLoss: 130.856621\n",
      "Train Epoch: 23 [40000/60000 (67%)]\tLoss: 123.175420\n",
      "Train Epoch: 23 [50000/60000 (83%)]\tLoss: 128.736836\n",
      "====> Epoch: 23 Average loss: 126.5200\n",
      "====> Test set loss: 128.6176\n",
      "Train Epoch: 24 [0/60000 (0%)]\tLoss: 123.077666\n",
      "Train Epoch: 24 [10000/60000 (17%)]\tLoss: 124.032598\n",
      "Train Epoch: 24 [20000/60000 (33%)]\tLoss: 127.668848\n",
      "Train Epoch: 24 [30000/60000 (50%)]\tLoss: 134.161016\n",
      "Train Epoch: 24 [40000/60000 (67%)]\tLoss: 122.512139\n",
      "Train Epoch: 24 [50000/60000 (83%)]\tLoss: 120.910391\n",
      "====> Epoch: 24 Average loss: 126.3727\n",
      "====> Test set loss: 128.5150\n",
      "Train Epoch: 25 [0/60000 (0%)]\tLoss: 126.901182\n",
      "Train Epoch: 25 [10000/60000 (17%)]\tLoss: 128.147129\n",
      "Train Epoch: 25 [20000/60000 (33%)]\tLoss: 119.677139\n",
      "Train Epoch: 25 [30000/60000 (50%)]\tLoss: 131.224746\n",
      "Train Epoch: 25 [40000/60000 (67%)]\tLoss: 127.900459\n",
      "Train Epoch: 25 [50000/60000 (83%)]\tLoss: 121.565029\n",
      "====> Epoch: 25 Average loss: 126.1753\n",
      "====> Test set loss: 128.3140\n",
      "Train Epoch: 26 [0/60000 (0%)]\tLoss: 129.182646\n",
      "Train Epoch: 26 [10000/60000 (17%)]\tLoss: 123.337324\n",
      "Train Epoch: 26 [20000/60000 (33%)]\tLoss: 122.413672\n",
      "Train Epoch: 26 [30000/60000 (50%)]\tLoss: 121.355000\n",
      "Train Epoch: 26 [40000/60000 (67%)]\tLoss: 132.059590\n",
      "Train Epoch: 26 [50000/60000 (83%)]\tLoss: 134.971367\n",
      "====> Epoch: 26 Average loss: 126.0179\n",
      "====> Test set loss: 128.4664\n",
      "Train Epoch: 27 [0/60000 (0%)]\tLoss: 123.304121\n",
      "Train Epoch: 27 [10000/60000 (17%)]\tLoss: 128.168721\n",
      "Train Epoch: 27 [20000/60000 (33%)]\tLoss: 133.523994\n",
      "Train Epoch: 27 [30000/60000 (50%)]\tLoss: 126.522100\n",
      "Train Epoch: 27 [40000/60000 (67%)]\tLoss: 122.112578\n",
      "Train Epoch: 27 [50000/60000 (83%)]\tLoss: 120.407129\n",
      "====> Epoch: 27 Average loss: 125.7919\n",
      "====> Test set loss: 128.2726\n",
      "Train Epoch: 28 [0/60000 (0%)]\tLoss: 126.885420\n",
      "Train Epoch: 28 [10000/60000 (17%)]\tLoss: 116.770684\n",
      "Train Epoch: 28 [20000/60000 (33%)]\tLoss: 129.816104\n",
      "Train Epoch: 28 [30000/60000 (50%)]\tLoss: 130.296367\n",
      "Train Epoch: 28 [40000/60000 (67%)]\tLoss: 123.880352\n",
      "Train Epoch: 28 [50000/60000 (83%)]\tLoss: 123.167656\n",
      "====> Epoch: 28 Average loss: 125.7476\n",
      "====> Test set loss: 128.4135\n",
      "Train Epoch: 29 [0/60000 (0%)]\tLoss: 128.240967\n",
      "Train Epoch: 29 [10000/60000 (17%)]\tLoss: 130.213594\n",
      "Train Epoch: 29 [20000/60000 (33%)]\tLoss: 126.970742\n",
      "Train Epoch: 29 [30000/60000 (50%)]\tLoss: 125.689043\n",
      "Train Epoch: 29 [40000/60000 (67%)]\tLoss: 124.778213\n",
      "Train Epoch: 29 [50000/60000 (83%)]\tLoss: 125.161123\n",
      "====> Epoch: 29 Average loss: 125.5231\n",
      "====> Test set loss: 128.4310\n",
      "Train Epoch: 30 [0/60000 (0%)]\tLoss: 122.761152\n",
      "Train Epoch: 30 [10000/60000 (17%)]\tLoss: 121.126309\n",
      "Train Epoch: 30 [20000/60000 (33%)]\tLoss: 128.538770\n",
      "Train Epoch: 30 [30000/60000 (50%)]\tLoss: 128.564609\n",
      "Train Epoch: 30 [40000/60000 (67%)]\tLoss: 124.153174\n",
      "Train Epoch: 30 [50000/60000 (83%)]\tLoss: 125.070029\n",
      "====> Epoch: 30 Average loss: 125.3449\n",
      "====> Test set loss: 128.2747\n",
      "Train Epoch: 31 [0/60000 (0%)]\tLoss: 134.593965\n",
      "Train Epoch: 31 [10000/60000 (17%)]\tLoss: 129.718887\n",
      "Train Epoch: 31 [20000/60000 (33%)]\tLoss: 124.998877\n",
      "Train Epoch: 31 [30000/60000 (50%)]\tLoss: 122.063672\n",
      "Train Epoch: 31 [40000/60000 (67%)]\tLoss: 124.577256\n",
      "Train Epoch: 31 [50000/60000 (83%)]\tLoss: 120.705264\n",
      "====> Epoch: 31 Average loss: 125.2561\n",
      "====> Test set loss: 128.0725\n",
      "Train Epoch: 32 [0/60000 (0%)]\tLoss: 119.171104\n",
      "Train Epoch: 32 [10000/60000 (17%)]\tLoss: 126.698955\n",
      "Train Epoch: 32 [20000/60000 (33%)]\tLoss: 127.150264\n",
      "Train Epoch: 32 [30000/60000 (50%)]\tLoss: 125.016992\n",
      "Train Epoch: 32 [40000/60000 (67%)]\tLoss: 125.667041\n",
      "Train Epoch: 32 [50000/60000 (83%)]\tLoss: 121.029707\n",
      "====> Epoch: 32 Average loss: 125.0756\n",
      "====> Test set loss: 128.1042\n",
      "Train Epoch: 33 [0/60000 (0%)]\tLoss: 126.942383\n",
      "Train Epoch: 33 [10000/60000 (17%)]\tLoss: 120.612314\n",
      "Train Epoch: 33 [20000/60000 (33%)]\tLoss: 123.660410\n",
      "Train Epoch: 33 [30000/60000 (50%)]\tLoss: 129.489346\n",
      "Train Epoch: 33 [40000/60000 (67%)]\tLoss: 125.785947\n",
      "Train Epoch: 33 [50000/60000 (83%)]\tLoss: 120.628271\n",
      "====> Epoch: 33 Average loss: 125.0094\n",
      "====> Test set loss: 128.0001\n",
      "Train Epoch: 34 [0/60000 (0%)]\tLoss: 129.735371\n",
      "Train Epoch: 34 [10000/60000 (17%)]\tLoss: 129.105596\n",
      "Train Epoch: 34 [20000/60000 (33%)]\tLoss: 133.656943\n",
      "Train Epoch: 34 [30000/60000 (50%)]\tLoss: 113.178818\n",
      "Train Epoch: 34 [40000/60000 (67%)]\tLoss: 121.336064\n",
      "Train Epoch: 34 [50000/60000 (83%)]\tLoss: 120.174883\n",
      "====> Epoch: 34 Average loss: 124.8607\n",
      "====> Test set loss: 128.0390\n",
      "Train Epoch: 35 [0/60000 (0%)]\tLoss: 124.387598\n",
      "Train Epoch: 35 [10000/60000 (17%)]\tLoss: 120.631221\n",
      "Train Epoch: 35 [20000/60000 (33%)]\tLoss: 124.786094\n",
      "Train Epoch: 35 [30000/60000 (50%)]\tLoss: 130.996582\n",
      "Train Epoch: 35 [40000/60000 (67%)]\tLoss: 119.367363\n",
      "Train Epoch: 35 [50000/60000 (83%)]\tLoss: 122.706055\n",
      "====> Epoch: 35 Average loss: 124.7230\n",
      "====> Test set loss: 128.2631\n",
      "Train Epoch: 36 [0/60000 (0%)]\tLoss: 120.660166\n",
      "Train Epoch: 36 [10000/60000 (17%)]\tLoss: 125.011494\n",
      "Train Epoch: 36 [20000/60000 (33%)]\tLoss: 122.200264\n",
      "Train Epoch: 36 [30000/60000 (50%)]\tLoss: 125.277939\n",
      "Train Epoch: 36 [40000/60000 (67%)]\tLoss: 116.444561\n",
      "Train Epoch: 36 [50000/60000 (83%)]\tLoss: 119.972471\n",
      "====> Epoch: 36 Average loss: 124.5897\n",
      "====> Test set loss: 128.1268\n",
      "Train Epoch: 37 [0/60000 (0%)]\tLoss: 125.394326\n",
      "Train Epoch: 37 [10000/60000 (17%)]\tLoss: 116.599443\n",
      "Train Epoch: 37 [20000/60000 (33%)]\tLoss: 126.257939\n",
      "Train Epoch: 37 [30000/60000 (50%)]\tLoss: 120.116367\n",
      "Train Epoch: 37 [40000/60000 (67%)]\tLoss: 125.987617\n",
      "Train Epoch: 37 [50000/60000 (83%)]\tLoss: 120.745146\n",
      "====> Epoch: 37 Average loss: 124.4481\n",
      "====> Test set loss: 128.2004\n",
      "Train Epoch: 38 [0/60000 (0%)]\tLoss: 124.684961\n",
      "Train Epoch: 38 [10000/60000 (17%)]\tLoss: 126.845244\n",
      "Train Epoch: 38 [20000/60000 (33%)]\tLoss: 123.810410\n",
      "Train Epoch: 38 [30000/60000 (50%)]\tLoss: 125.283574\n",
      "Train Epoch: 38 [40000/60000 (67%)]\tLoss: 128.587666\n",
      "Train Epoch: 38 [50000/60000 (83%)]\tLoss: 120.695684\n",
      "====> Epoch: 38 Average loss: 124.3855\n",
      "====> Test set loss: 127.9592\n",
      "Train Epoch: 39 [0/60000 (0%)]\tLoss: 121.638779\n",
      "Train Epoch: 39 [10000/60000 (17%)]\tLoss: 126.289404\n",
      "Train Epoch: 39 [20000/60000 (33%)]\tLoss: 128.402998\n",
      "Train Epoch: 39 [30000/60000 (50%)]\tLoss: 126.957637\n",
      "Train Epoch: 39 [40000/60000 (67%)]\tLoss: 134.657354\n",
      "Train Epoch: 39 [50000/60000 (83%)]\tLoss: 116.111592\n",
      "====> Epoch: 39 Average loss: 124.1941\n",
      "====> Test set loss: 127.9553\n",
      "Train Epoch: 40 [0/60000 (0%)]\tLoss: 120.676680\n",
      "Train Epoch: 40 [10000/60000 (17%)]\tLoss: 121.145176\n",
      "Train Epoch: 40 [20000/60000 (33%)]\tLoss: 128.506357\n",
      "Train Epoch: 40 [30000/60000 (50%)]\tLoss: 125.981484\n",
      "Train Epoch: 40 [40000/60000 (67%)]\tLoss: 127.427490\n",
      "Train Epoch: 40 [50000/60000 (83%)]\tLoss: 118.421523\n",
      "====> Epoch: 40 Average loss: 124.2022\n",
      "====> Test set loss: 127.8238\n",
      "Train Epoch: 41 [0/60000 (0%)]\tLoss: 118.496436\n",
      "Train Epoch: 41 [10000/60000 (17%)]\tLoss: 126.371514\n",
      "Train Epoch: 41 [20000/60000 (33%)]\tLoss: 118.269160\n",
      "Train Epoch: 41 [30000/60000 (50%)]\tLoss: 129.450488\n",
      "Train Epoch: 41 [40000/60000 (67%)]\tLoss: 123.966641\n",
      "Train Epoch: 41 [50000/60000 (83%)]\tLoss: 120.907314\n",
      "====> Epoch: 41 Average loss: 124.0595\n",
      "====> Test set loss: 127.9480\n",
      "Train Epoch: 42 [0/60000 (0%)]\tLoss: 120.748281\n",
      "Train Epoch: 42 [10000/60000 (17%)]\tLoss: 120.720078\n",
      "Train Epoch: 42 [20000/60000 (33%)]\tLoss: 124.286367\n",
      "Train Epoch: 42 [30000/60000 (50%)]\tLoss: 129.405254\n",
      "Train Epoch: 42 [40000/60000 (67%)]\tLoss: 120.885908\n",
      "Train Epoch: 42 [50000/60000 (83%)]\tLoss: 132.045723\n",
      "====> Epoch: 42 Average loss: 123.9214\n",
      "====> Test set loss: 128.2172\n",
      "Train Epoch: 43 [0/60000 (0%)]\tLoss: 126.227949\n",
      "Train Epoch: 43 [10000/60000 (17%)]\tLoss: 123.287031\n",
      "Train Epoch: 43 [20000/60000 (33%)]\tLoss: 118.102373\n",
      "Train Epoch: 43 [30000/60000 (50%)]\tLoss: 123.968652\n",
      "Train Epoch: 43 [40000/60000 (67%)]\tLoss: 117.281416\n",
      "Train Epoch: 43 [50000/60000 (83%)]\tLoss: 116.812783\n",
      "====> Epoch: 43 Average loss: 123.8238\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====> Test set loss: 127.9673\n",
      "Train Epoch: 44 [0/60000 (0%)]\tLoss: 126.146406\n",
      "Train Epoch: 44 [10000/60000 (17%)]\tLoss: 121.270049\n",
      "Train Epoch: 44 [20000/60000 (33%)]\tLoss: 122.125947\n",
      "Train Epoch: 44 [30000/60000 (50%)]\tLoss: 126.316475\n",
      "Train Epoch: 44 [40000/60000 (67%)]\tLoss: 122.943516\n",
      "Train Epoch: 44 [50000/60000 (83%)]\tLoss: 128.219229\n",
      "====> Epoch: 44 Average loss: 123.8189\n",
      "====> Test set loss: 127.9392\n",
      "Train Epoch: 45 [0/60000 (0%)]\tLoss: 124.976865\n",
      "Train Epoch: 45 [10000/60000 (17%)]\tLoss: 121.306445\n",
      "Train Epoch: 45 [20000/60000 (33%)]\tLoss: 117.693623\n",
      "Train Epoch: 45 [30000/60000 (50%)]\tLoss: 117.285332\n",
      "Train Epoch: 45 [40000/60000 (67%)]\tLoss: 120.706074\n",
      "Train Epoch: 45 [50000/60000 (83%)]\tLoss: 120.392090\n",
      "====> Epoch: 45 Average loss: 123.6335\n",
      "====> Test set loss: 128.1894\n",
      "Train Epoch: 46 [0/60000 (0%)]\tLoss: 121.928545\n",
      "Train Epoch: 46 [10000/60000 (17%)]\tLoss: 119.535459\n",
      "Train Epoch: 46 [20000/60000 (33%)]\tLoss: 131.456572\n",
      "Train Epoch: 46 [30000/60000 (50%)]\tLoss: 127.008340\n",
      "Train Epoch: 46 [40000/60000 (67%)]\tLoss: 124.003564\n",
      "Train Epoch: 46 [50000/60000 (83%)]\tLoss: 119.511553\n",
      "====> Epoch: 46 Average loss: 123.7213\n",
      "====> Test set loss: 128.0483\n",
      "Train Epoch: 47 [0/60000 (0%)]\tLoss: 121.481826\n",
      "Train Epoch: 47 [10000/60000 (17%)]\tLoss: 125.165742\n",
      "Train Epoch: 47 [20000/60000 (33%)]\tLoss: 131.840049\n",
      "Train Epoch: 47 [30000/60000 (50%)]\tLoss: 122.547451\n",
      "Train Epoch: 47 [40000/60000 (67%)]\tLoss: 126.694229\n",
      "Train Epoch: 47 [50000/60000 (83%)]\tLoss: 125.942090\n",
      "====> Epoch: 47 Average loss: 123.6721\n",
      "====> Test set loss: 127.9682\n",
      "Train Epoch: 48 [0/60000 (0%)]\tLoss: 131.695859\n",
      "Train Epoch: 48 [10000/60000 (17%)]\tLoss: 113.293994\n",
      "Train Epoch: 48 [20000/60000 (33%)]\tLoss: 125.186035\n",
      "Train Epoch: 48 [30000/60000 (50%)]\tLoss: 123.036543\n",
      "Train Epoch: 48 [40000/60000 (67%)]\tLoss: 117.729854\n",
      "Train Epoch: 48 [50000/60000 (83%)]\tLoss: 117.250947\n",
      "====> Epoch: 48 Average loss: 123.4133\n",
      "====> Test set loss: 127.9083\n",
      "Train Epoch: 49 [0/60000 (0%)]\tLoss: 115.015498\n",
      "Train Epoch: 49 [10000/60000 (17%)]\tLoss: 130.918379\n",
      "Train Epoch: 49 [20000/60000 (33%)]\tLoss: 124.317324\n",
      "Train Epoch: 49 [30000/60000 (50%)]\tLoss: 120.806689\n",
      "Train Epoch: 49 [40000/60000 (67%)]\tLoss: 126.835527\n",
      "Train Epoch: 49 [50000/60000 (83%)]\tLoss: 124.241855\n",
      "====> Epoch: 49 Average loss: 123.3197\n",
      "====> Test set loss: 127.9292\n",
      "Train Epoch: 50 [0/60000 (0%)]\tLoss: 121.166621\n",
      "Train Epoch: 50 [10000/60000 (17%)]\tLoss: 123.946631\n",
      "Train Epoch: 50 [20000/60000 (33%)]\tLoss: 115.903203\n",
      "Train Epoch: 50 [30000/60000 (50%)]\tLoss: 125.992109\n",
      "Train Epoch: 50 [40000/60000 (67%)]\tLoss: 110.466504\n",
      "Train Epoch: 50 [50000/60000 (83%)]\tLoss: 126.132207\n",
      "====> Epoch: 50 Average loss: 123.1466\n",
      "====> Test set loss: 128.0250\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "for epoch in range(1, 51):\n",
    "    train(epoch)\n",
    "    test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lyeoni\\AppData\\Local\\Continuum\\anaconda3\\envs\\tensorflow\\lib\\site-packages\\torch\\nn\\functional.py:1006: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    z = torch.randn(10, 2).cuda()\n",
    "    c = torch.eye(10).cuda()\n",
    "\n",
    "    sample = cvae.decoder(z, c)\n",
    "    save_image(sample.view(10, 1, 28, 28), './samples/sample_' + '.png')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}