116_优化器.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 优化器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "① 损失函数调用backward方法，就可以调用损失函数的反向传播方法，就可以求出我们需要调节的梯度，我们就可以利用我们的优化器就可以根据梯度对参数进行调整，达到整体误差降低的目的。\n",
    "\n",
    "② 梯度要清零，如果梯度不清零会导致梯度累加。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 神经网络优化一轮"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "tensor(2.2978, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2988, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3163, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3253, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2952, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3066, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3085, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3106, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2960, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3053, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2892, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3090, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2956, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3041, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3012, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3043, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2760, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3051, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2951, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3168, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3140, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3096, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2945, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3115, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2987, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3029, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3096, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3064, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3161, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3129, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2903, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3043, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3034, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3169, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3090, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3039, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3019, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3071, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3018, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3083, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2994, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2909, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3130, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2993, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2906, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3084, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3123, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2931, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3059, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3117, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2975, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3109, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3029, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3020, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3022, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3005, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2920, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3016, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3053, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3082, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3011, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3040, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3130, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2981, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2977, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2994, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3075, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3016, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2966, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3015, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3000, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2953, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2958, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2977, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2928, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2989, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2968, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2982, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2912, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3005, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2909, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2940, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2959, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2993, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2933, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2951, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2824, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2987, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2961, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2914, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3025, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2895, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2943, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2974, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2977, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3069, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2972, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2979, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2932, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2940, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3014, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2958, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3013, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2953, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2951, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3116, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2916, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2871, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2975, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2950, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3039, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2901, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2950, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2958, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2893, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2917, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3001, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2988, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3069, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3083, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2841, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2932, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2857, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2971, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2999, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2911, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2977, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3027, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2940, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2939, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2950, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2951, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3000, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2935, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2817, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2977, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3067, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2742, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2964, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2927, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2941, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3003, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2965, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2908, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2885, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2984, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.3009, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2931, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2856, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2907, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2938, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2880, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2975, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2922, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2966, grad_fn=<NllLossBackward0>)\n",
      "tensor(2.2804, grad_fn=<NllLossBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn \n",
    "from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "dataset = torchvision.datasets.CIFAR10(\"./dataset\",train=False,transform=torchvision.transforms.ToTensor(),download=True)       \n",
    "dataloader = DataLoader(dataset, batch_size=64,drop_last=True)\n",
    "\n",
    "class Tudui(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Tudui, self).__init__()        \n",
    "        self.model1 = Sequential(\n",
    "            Conv2d(3,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,64,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Flatten(),\n",
    "            Linear(1024,64),\n",
    "            Linear(64,10)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.model1(x)\n",
    "        return x\n",
    "    \n",
    "loss = nn.CrossEntropyLoss() # 交叉熵    \n",
    "tudui = Tudui()\n",
    "optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器\n",
    "for data in dataloader:\n",
    "    imgs, targets = data\n",
    "    outputs = tudui(imgs)\n",
    "    result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距\n",
    "    optim.zero_grad()  # 梯度清零\n",
    "    result_loss.backward() # 反向传播，计算损失函数的梯度\n",
    "    optim.step()   # 根据梯度，对网络的参数进行调优\n",
    "    print(result_loss) # 对数据只看了一遍，只看了一轮，所以loss下降不大"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 神经网络优化多轮"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "tensor(358.1069, grad_fn=<AddBackward0>)\n",
      "tensor(353.8411, grad_fn=<AddBackward0>)\n",
      "tensor(337.3790, grad_fn=<AddBackward0>)\n",
      "tensor(317.3237, grad_fn=<AddBackward0>)\n",
      "tensor(307.6762, grad_fn=<AddBackward0>)\n",
      "tensor(298.2425, grad_fn=<AddBackward0>)\n",
      "tensor(289.7010, grad_fn=<AddBackward0>)\n",
      "tensor(282.7116, grad_fn=<AddBackward0>)\n",
      "tensor(275.8972, grad_fn=<AddBackward0>)\n",
      "tensor(269.5961, grad_fn=<AddBackward0>)\n",
      "tensor(263.8480, grad_fn=<AddBackward0>)\n",
      "tensor(258.5006, grad_fn=<AddBackward0>)\n",
      "tensor(253.4671, grad_fn=<AddBackward0>)\n",
      "tensor(248.7994, grad_fn=<AddBackward0>)\n",
      "tensor(244.4917, grad_fn=<AddBackward0>)\n",
      "tensor(240.5728, grad_fn=<AddBackward0>)\n",
      "tensor(236.9719, grad_fn=<AddBackward0>)\n",
      "tensor(233.6264, grad_fn=<AddBackward0>)\n",
      "tensor(230.4298, grad_fn=<AddBackward0>)\n",
      "tensor(227.3427, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn \n",
    "from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "dataset = torchvision.datasets.CIFAR10(\"./dataset\",train=False,transform=torchvision.transforms.ToTensor(),download=True)       \n",
    "dataloader = DataLoader(dataset, batch_size=64,drop_last=True)\n",
    "\n",
    "class Tudui(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Tudui, self).__init__()        \n",
    "        self.model1 = Sequential(\n",
    "            Conv2d(3,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,64,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Flatten(),\n",
    "            Linear(1024,64),\n",
    "            Linear(64,10)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.model1(x)\n",
    "        return x\n",
    "    \n",
    "loss = nn.CrossEntropyLoss() # 交叉熵    \n",
    "tudui = Tudui()\n",
    "optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器\n",
    "for epoch in range(20):\n",
    "    running_loss = 0.0\n",
    "    for data in dataloader:\n",
    "        imgs, targets = data\n",
    "        outputs = tudui(imgs)\n",
    "        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距\n",
    "        optim.zero_grad()  # 梯度清零\n",
    "        result_loss.backward() # 反向传播，计算损失函数的梯度\n",
    "        optim.step()   # 根据梯度，对网络的参数进行调优\n",
    "        running_loss = running_loss + result_loss\n",
    "    print(running_loss) # 对这一轮所有误差的总和"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. 神经网络学习率优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "tensor(359.4722, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n",
      "tensor(359.4630, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn \n",
    "from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "dataset = torchvision.datasets.CIFAR10(\"./dataset\",train=False,transform=torchvision.transforms.ToTensor(),download=True)       \n",
    "dataloader = DataLoader(dataset, batch_size=64,drop_last=True)\n",
    "\n",
    "class Tudui(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Tudui, self).__init__()        \n",
    "        self.model1 = Sequential(\n",
    "            Conv2d(3,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,32,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Conv2d(32,64,5,padding=2),\n",
    "            MaxPool2d(2),\n",
    "            Flatten(),\n",
    "            Linear(1024,64),\n",
    "            Linear(64,10)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.model1(x)\n",
    "        return x\n",
    "    \n",
    "loss = nn.CrossEntropyLoss() # 交叉熵    \n",
    "tudui = Tudui()\n",
    "optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optim, step_size=5, gamma=0.1) # 每过 step_size 更新一次优化器，更新是学习率为原来的学习率的的 0.1 倍    \n",
    "for epoch in range(20):\n",
    "    running_loss = 0.0\n",
    "    for data in dataloader:\n",
    "        imgs, targets = data\n",
    "        outputs = tudui(imgs)\n",
    "        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距\n",
    "        optim.zero_grad()  # 梯度清零\n",
    "        result_loss.backward() # 反向传播，计算损失函数的梯度\n",
    "        optim.step()   # 根据梯度，对网络的参数进行调优\n",
    "        scheduler.step() # 学习率太小了，所以20个轮次后，相当于没走多少\n",
    "        running_loss = running_loss + result_loss\n",
    "    print(running_loss) # 对这一轮所有误差的总和"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.3",
   "language": "python",
   "name": "python3.6.3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "350px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}