Pytorch权重初始化/参数初始化-Toy模板网

这篇具有很好参考价值的文章主要介绍了Pytorch权重初始化/参数初始化。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

refer:

【Pytorch】各网络层的默认初始化方法 https://blog.csdn.net/guofei_fly/article/details/105109883

20230625

其实Pytorch初始化方法就在各自的层的def reset_parameters(self) -> None:方法中。

有人可能会问为什么这个方法和Pytorch直接出来的权重初始值不一样?单步调试会发现其实这个方法运行了至少两次，而每次出来的结果都是不同的，说明初始化结果可能还和函数调用次数有关，但是不论调用多少次还是符合基本的分布情况的。请参考python/pytorch random_seed随机种子 https://blog.csdn.net/qq_43369406/article/details/131342983

"""init param"""
# !!write above on the first line!!
import random, numpy as np, torch

# set random seed
seed = 416
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)

def _weights_init(m):
    """
    intro:
        weights init.
        finish these:
            - torch.nn.Linear
    >>> version 1.0.0
    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.trunc_normal_(m.weight, std=.01)
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            nn.init.zeros_(m.bias)
    elif classname.startswith('Conv'):
        m.weight.data.normal_(0.0, 0.02)
    >>> version 1.0.1
        refer https://blog.csdn.net/guofei_fly/article/details/105109883
        finish nn.Linear, nn.Conv
    
    args:
        :param torch.parameters m: nn.Module
    """
    classname = m.__class__.__name__

    if type(m) == nn.Linear or classname.startswith("Conv"):
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.kaiming_uniform_(m.weight, a=math.sqrt(5), nonlinearity='leaky_relu')
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            fan_in, _ = nn.init._calculate_fan_in_and_fan_out(m.weight)
            bound = 1 / math.sqrt(fan_in)
            nn.init.uniform_(m.bias, -bound, bound)
    elif isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight, mode="fan_out")
        if m.bias is not None:
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.LayerNorm):
        nn.init.zeros_(m.bias)
        nn.init.ones_(m.weight)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)
         
net = nn.Sequential(nn.Conv2d(in_channels=3, out_channels=6, kernel_size=3, stride=1, padding=1), nn.LazyLinear(8), nn.ReLU(), nn.LazyLinear(1))
X = torch.rand(size=(1, 3, 224, 224))   # [ batch_size, channel, height, width ]
Y = net(X)
net.apply(_weights_init)

# check param
print(net[0].weight, '\n', net[0].bias)

20230622

# !!write above on the first line!!
import random, numpy as np, torch
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)

def _weights_init(m):
    """
    intro:
        weights init.
        finish these:
            - torch.nn.Linear
    >>> version 1.0.0
    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.trunc_normal_(m.weight, std=.01)
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            nn.init.zeros_(m.bias)
    
    args:
        :param torch.parameters m: nn.Module
    """
    classname = m.__class__.__name__

    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.trunc_normal_(m.weight, std=.01)
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight, mode="fan_out")
        if m.bias is not None:
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.LayerNorm):
        nn.init.zeros_(m.bias)
        nn.init.ones_(m.weight)
    elif classname.startswith('Conv'):
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)
         
net = nn.Sequential(nn.LazyLinear(8), nn.ReLU(), nn.LazyLinear(1))
X = torch.rand(size=(2, 4))
net.apply(_weights_init)

x.1 指定随机性

正态分布仍然是一个分布，这会使得你的实验结果无法完全复现，要让实验结果完全复现则需要指定seed，如下：

random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)

x.2 参数初始化

参数初始化是很重要的工作，数据决定了梯度下降的山脉长什么样，而参数初始化决定了你从这个山的哪个合适的地方开始出发。Pytorch提供了多种初始化方法，可以搭配nn.Module.apply和你自己写的初始化方法函数，来将nn.Module中所有继承自nn.Module子类中的children对象的参数初始化，如下：

常见的bulit-in初始化方法，

"""

6.3.1. Built-in Initialization
using built-in func to init.

- `nn.init.normal_(module.weight, mean=0, std=0.01)`
- `nn.init.zeros_(module.bias)`
- `nn.init.constant_(module.weight, 1)`
- `nn.init.zeros_(module.bias)`
- `nn.init.xavier_uniform_(module.weight)`
- `nn.init.kaiming_uniform_(module.weight)` # default one for Linear, and the type is Leaky_ReLU
- `nn.init.uniform_(module.weight, -10, 10)`
"""

进行初始化，

def init_normal(module):
    if type(module) == nn.Linear:
        nn.init.normal_(module.weight, mean=0, std=0.01)
        nn.init.zeros_(module.bias)

net.apply(init_normal)
print(net[0].weight.data[0]) 
print(net[0].bias.data[0])

使用自己的初始化方法进行初始化，文章来源地址https://www.toymoban.com/news/detail-510069.html

def _weights_init(m):
    """
    intro:
        weights init.
        finish these:
            - torch.nn.Linear
    >>> version 1.0.0
    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.trunc_normal_(m.weight, std=.01)
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            nn.init.zeros_(m.bias)
    
    args:
        :param torch.parameters m: nn.Module
    """
    classname = m.__class__.__name__

    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape) for name, param in m.named_parameters()][0])    # linear - param - weight
        nn.init.trunc_normal_(m.weight, std=.01)
        if m.bias is not None:
            print("Init", *[(name, param.shape) for name, param in m.named_parameters()][1])    # linear - param - bias
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.Conv2d):
        nn.init.kaiming_normal_(m.weight, mode="fan_out")
        if m.bias is not None:
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.LayerNorm):
        nn.init.zeros_(m.bias)
        nn.init.ones_(m.weight)
    elif classname.startswith('Conv'):
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)
         
net = nn.Sequential(nn.LazyLinear(8), nn.ReLU(), nn.LazyLinear(1))
X = torch.rand(size=(2, 4))
net.apply(_weights_init)