py-MDNet/modules/models.py 的set_optimizer函数

太过爱你忘了你带给我的痛 2022-11-11 05:48 158阅读 0赞

该函数用了两次，一次是用来初始化优化器，一次是用来更新优化器。

在初始化中，fc4，fc5的lr为0.0005，fc6的lr为0.005

在更新优化器中，fc4，fc5的lr为0.0001，fc6的lr为0.001

import torch
    import torch.nn as nn
    from collections import OrderedDict
    import numpy as np
    import torch.nn.functional as F
    import collections
    import yaml
    def append_params(params, module, prefix):
        for child in module.children():
            for k,p in child._parameters.items():
                if p is None: continue
    
                if isinstance(child, nn.BatchNorm2d):
                    name = prefix + '_bn_' + k
                else:
                    name = prefix + '_' + k
    
                if name not in params:
                    params[name] = p
                else:
                    raise RuntimeError('Duplicated param name: {:s}'.format(name))
    
    
    
    layers = nn.Sequential(OrderedDict([
                    ('conv1', nn.Sequential(nn.Conv2d(3, 96, kernel_size=7, stride=2),
                                            nn.ReLU(inplace=True),#激活函数，同时inplace设置为True，原值将会被覆盖。
                                            nn.LocalResponseNorm(2),#归一化
                                            nn.MaxPool2d(kernel_size=3, stride=2))),
                    ('conv2', nn.Sequential(nn.Conv2d(96, 256, kernel_size=5, stride=2),
                                            nn.ReLU(inplace=True),
                                            nn.LocalResponseNorm(2),
                                            nn.MaxPool2d(kernel_size=3, stride=2))),
                    ('conv3', nn.Sequential(nn.Conv2d(256, 512, kernel_size=3, stride=1),
                                            nn.ReLU(inplace=True))),
                    ('fc4',   nn.Sequential(nn.Linear(512 * 3 * 3, 512),
                                            nn.ReLU(inplace=True))),
                    ('fc5',   nn.Sequential(nn.Dropout(0.5),
                                            nn.Linear(512, 512),
                                            nn.ReLU(inplace=True)))]))
    # for l in layers:
    #     print(l)
    #
    params = collections.OrderedDict()
    for name, module in layers.named_children():
        append_params(params, module, name)
    
    
    #此处为set_optimizer函数：
    lr_base = 0.0005
    lr_mult = {'fc6':10}
    param_list = []
    for k, p in params.items():#原代码中的params.items()为可学习的参数，此处只为了简单说明，用了全部的参数
            lr = lr_base#给每一个可学习的参数设置初始学习率为0.0005.
            for l, m in lr_mult.items():#取出fc6和10.
                if k.startswith(l):#再次判断该层是否是fc6层，如果是，重新设置学习率为0.0005*10
                    lr = lr_base * m  #没看懂啥意思？？？#原来：初始化，fc4，fc5的lr = 0.0005，fc6lr为0.0005*10 = 0.005
                                                        #后来在更新优化器中，将fc4，fc5的lr = 0.0001，fc6lr为0.0001*10 = 0.001
            param_list.append({'params': [p], 'lr':lr})#给每一个参数添加学习率，并且放到param_list中，用于更新参数
    print(param_list)

optimizer = optim.SGD(param\_list, lr = lr, momentum=momentum, weight\_decay=w\_decay)\#用随机梯度下降算法来更新。

输出太长了，只写一个参数的输出：

{'params': [Parameter containing:
    tensor([-9.7996e-04, -1.3808e-02, -4.5937e-03, -6.0048e-03,  1.2511e-02,
            -2.1006e-03, -1.0121e-02, -7.6366e-03,  6.7936e-03, -3.5825e-03,
             4.2030e-03,  2.7011e-04, -4.8736e-04, -1.6295e-03,  1.0839e-04,
            -1.4446e-02, -1.2529e-02, -8.9380e-03,  2.0073e-03, -8.8903e-03,
            -4.6784e-03, -4.3598e-03, -3.1074e-03, -1.1800e-02,  7.5774e-03,
             2.0441e-03, -1.0189e-02,  1.1629e-02, -1.7825e-03, -1.2042e-02,
            -2.8074e-03,  1.3914e-02, -9.7211e-03, -1.1965e-02, -8.9130e-03,
             7.7358e-03, -1.4190e-02,  1.3166e-02,  4.4527e-03, -2.6648e-03,
            -5.4300e-03, -6.2514e-03,  1.1893e-02, -4.0076e-03,  5.4227e-03,
            -9.0793e-05,  1.3886e-02, -1.1545e-04, -9.9586e-03, -4.6667e-03,
            -1.0431e-02, -1.2155e-02,  4.9751e-03,  3.1109e-03, -7.8064e-04,
             2.2946e-03,  1.0914e-02, -1.4820e-03,  1.1918e-02,  1.8418e-03,
             1.0376e-02, -2.9378e-03,  5.3166e-03,  8.3707e-03,  9.0079e-03,
             1.2650e-02, -2.4181e-04, -1.2410e-02, -5.1961e-03, -1.3523e-02,
             8.4351e-03,  7.5505e-03, -1.0436e-02,  1.7089e-03,  2.7658e-03,
            -5.2591e-03,  9.7041e-03, -1.1256e-02,  3.5063e-03, -1.7378e-03,
             1.2455e-02, -1.0269e-03, -7.7194e-03,  1.3258e-02,  2.4350e-03,
            -4.9133e-03,  9.2081e-03,  8.5081e-03,  1.2384e-02, -6.4718e-03,
            -4.8999e-03, -4.0182e-03, -9.2176e-03,  9.7834e-03,  3.7964e-04,
             4.5026e-03,  1.2167e-02, -1.4664e-02,  1.1228e-02,  8.5231e-03,
             1.3651e-02, -5.3227e-03,  5.4230e-03, -8.2249e-04, -1.3989e-02,
             2.1924e-04, -1.0577e-02,  1.1865e-02, -1.1138e-02,  2.6534e-03,
            -1.3639e-02,  1.2414e-02,  8.0182e-03, -8.4310e-03, -4.6266e-04,
             9.6214e-03, -5.0531e-03,  7.6720e-03, -1.0858e-02,  2.1585e-03,
            -1.1626e-02,  2.1764e-03, -1.0342e-02, -7.3508e-03, -7.7608e-03,
             1.3097e-03, -2.5480e-03,  3.2041e-03, -8.7478e-03, -3.9601e-04,
            -1.3795e-03, -9.8961e-03,  1.3321e-02, -5.0472e-03,  2.1772e-03,
             8.8212e-03, -1.3187e-02, -1.9957e-03,  5.6873e-04, -6.9666e-03,
            -8.6587e-03, -9.6083e-03, -1.1854e-03,  1.7915e-04,  6.0027e-03,
            -2.0544e-03, -1.1476e-02, -5.0658e-03, -7.6849e-03, -2.4973e-03,
             2.0419e-03, -4.5506e-03,  2.9620e-03, -3.2309e-03,  1.3539e-02,
            -1.1862e-02, -9.2049e-03,  6.4188e-03,  9.1977e-03, -8.0867e-04,
            -1.1572e-02,  5.5306e-03,  9.5759e-03,  1.2949e-02,  7.0124e-03,
            -5.0841e-03,  1.0426e-02, -1.4442e-02, -1.3870e-02,  1.0678e-02,
            -2.7039e-03,  1.0045e-02, -7.0405e-03,  8.7101e-03,  1.2058e-02,
             7.0557e-03, -2.8028e-03,  1.4522e-02,  5.5917e-03, -1.4681e-02,
            -1.4458e-02, -1.2010e-02, -1.3544e-02,  5.9987e-03, -6.3309e-03,
            -6.5906e-03,  3.2252e-03,  3.5921e-03, -1.1923e-03, -3.7617e-03,
             6.8003e-03,  1.3999e-02,  5.9038e-03,  5.5420e-03,  5.6343e-03,
            -1.0731e-02,  1.1194e-02,  1.6021e-03, -5.9992e-03,  7.3606e-03,
             5.4576e-03, -5.4887e-03, -1.0835e-02, -1.3024e-02, -5.5601e-03,
            -3.8713e-04, -2.5151e-03,  1.0624e-02,  1.1473e-02,  1.3984e-02,
             1.2166e-02,  1.4217e-02, -2.3866e-03, -7.8582e-03, -1.2900e-02,
             9.6388e-03, -6.1611e-03,  6.0932e-03,  3.9141e-03,  1.1766e-02,
             1.2260e-02, -9.8111e-03,  8.2297e-03,  2.4504e-04, -4.9892e-03,
            -4.9711e-03, -3.9541e-03, -9.3591e-03, -3.0520e-04,  1.0177e-02,
             6.5117e-04,  8.6885e-03,  8.4901e-03,  2.4759e-03,  1.3559e-02,
             2.3605e-03, -1.1138e-02,  6.4135e-03,  1.0722e-02,  5.5246e-03,
            -1.3005e-02,  1.4767e-03, -1.4082e-02, -5.9980e-03, -5.3016e-03,
             1.1024e-02, -2.6556e-04,  1.8943e-03, -1.6160e-03,  1.3960e-03,
             7.6261e-03,  4.0395e-03, -9.3436e-03, -8.5269e-03,  4.6853e-03,
             3.7670e-03, -4.5787e-03, -1.3209e-02, -8.1064e-03,  2.3322e-03,
             2.6557e-04,  1.1375e-02,  8.1874e-03,  1.0817e-02,  1.3913e-02,
            -7.1802e-03,  2.8955e-03, -3.1215e-03, -8.6171e-04,  6.0029e-03,
            -1.3699e-02, -4.8303e-03, -3.5216e-03,  3.4359e-04,  1.4860e-03,
            -1.0640e-02, -8.6912e-03, -6.6180e-03, -3.6240e-03, -1.1537e-02,
            -7.2260e-04, -7.3301e-03, -5.9921e-03,  3.4591e-03, -1.4225e-02,
            -8.5539e-03, -6.2051e-03,  1.4569e-02, -2.2009e-03, -2.6747e-03,
            -5.5179e-03,  8.3656e-03,  2.9272e-03,  2.4063e-03, -8.4560e-03,
             6.5624e-03, -1.4684e-03,  5.2038e-03,  7.3709e-03, -1.3974e-02,
             1.0177e-02, -6.8862e-03,  2.3943e-03, -6.8217e-03,  5.6183e-03,
             4.4880e-03,  1.3474e-02, -8.0845e-03, -1.2294e-02,  2.7392e-04,
            -4.8835e-03,  1.2524e-02,  1.1793e-03,  5.1806e-03, -1.4838e-03,
             9.0306e-04, -4.4492e-03,  1.2986e-02, -2.4484e-03,  6.2296e-03,
             1.4317e-03,  7.3858e-03,  3.2452e-03, -8.8778e-03,  6.7631e-03,
             1.0843e-02, -1.2882e-02, -7.8930e-03, -8.5071e-03, -1.2341e-02,
             2.7987e-03, -2.8790e-03, -9.3937e-04, -9.7012e-04, -1.3647e-02,
             6.1630e-04,  3.4063e-03, -1.7538e-03,  3.3001e-04, -1.1046e-02,
             1.3102e-02,  4.0374e-03, -4.2662e-04, -5.3356e-03, -3.3789e-03,
             3.3269e-03, -7.7432e-03,  3.6758e-03, -1.2676e-04,  9.6517e-04,
             9.4532e-03,  1.2901e-02, -5.6450e-03,  9.9988e-03,  1.2033e-02,
             5.8831e-03,  8.7418e-03,  1.1377e-02, -1.2109e-02,  4.8248e-03,
             3.0873e-04, -1.3131e-03,  5.7273e-03,  8.5990e-03, -4.2199e-05,
            -1.1171e-02, -7.7607e-03,  5.5476e-03, -5.1593e-03,  1.7264e-03,
            -1.0673e-02, -8.9686e-03,  8.7543e-03, -1.0857e-02, -2.1810e-03,
             8.1193e-04,  9.4333e-03,  4.9037e-03,  8.9145e-03, -3.7248e-03,
             1.6342e-03, -8.7813e-03,  9.8404e-03, -4.3694e-03, -1.2426e-03,
            -9.7616e-03, -1.2427e-02, -8.6000e-03,  9.4947e-03, -1.4157e-02,
            -2.1250e-03, -1.0904e-02, -8.5022e-03, -7.1042e-03, -6.9400e-03,
            -1.4017e-02,  9.9084e-03,  8.0107e-03, -8.8633e-03,  5.0381e-03,
             4.9674e-03,  3.9428e-03, -4.0924e-03, -6.3208e-03,  1.4038e-02,
             1.1027e-02, -3.7331e-03, -9.0263e-03, -3.5533e-03,  1.3111e-03,
             1.4046e-02, -1.2333e-02,  1.0851e-02,  5.5085e-04,  1.6670e-03,
            -3.7356e-03, -3.5309e-03,  6.6846e-03,  8.6261e-03,  1.0450e-02,
            -7.1360e-03, -6.7073e-03, -6.5171e-03, -3.8617e-03,  8.7826e-04,
            -1.1495e-02,  3.3184e-03, -7.5176e-03,  1.4319e-02, -1.1720e-02,
            -7.2552e-04, -1.3190e-02, -4.3041e-03, -9.3194e-03,  4.1943e-03,
            -1.4046e-02, -3.4918e-03,  5.4038e-03,  1.0315e-02,  1.4209e-03,
            -4.5999e-03, -5.3947e-03,  9.1369e-03, -7.2507e-03, -1.3286e-02,
            -6.9775e-03,  3.7646e-03, -1.1768e-02, -1.0414e-02,  3.2221e-03,
            -7.8921e-03, -3.5480e-03, -1.3368e-03, -8.1434e-03, -1.0687e-02,
            -9.0426e-03, -1.3291e-02,  2.3811e-03, -1.0330e-02, -1.1018e-02,
            -4.9283e-03, -5.0695e-03,  1.2905e-02,  1.4453e-04, -8.7770e-03,
            -5.7908e-03,  5.8476e-03,  1.3474e-02,  1.2490e-02, -2.8245e-03,
             6.9402e-03,  7.4797e-03,  2.1419e-03, -6.8003e-03, -6.3343e-03,
            -4.7033e-04, -6.1954e-04,  1.4372e-02, -9.1525e-04, -4.3067e-03,
             7.1100e-03,  8.7888e-03,  4.6450e-03,  2.4096e-04, -1.8005e-03,
             1.2901e-02, -1.0009e-02,  5.1384e-03,  2.3607e-03, -2.6453e-03,
             1.1616e-02, -1.0240e-02,  1.3125e-04,  4.2553e-03, -6.0711e-04,
            -1.0913e-02, -1.2875e-02,  5.4294e-03,  3.9550e-03,  1.3669e-02,
             5.1295e-03, -1.2370e-02,  1.4325e-02,  1.3003e-02, -1.4419e-02,
            -1.6637e-03, -1.4156e-02, -1.0884e-02,  1.1537e-02,  3.1856e-03,
            -1.1488e-02, -1.2153e-02], requires_grad=True)], 'lr': 0.0005}