A主机查看当前conda环境(base) ytt@ytt-System-Product-Name:~$ conda env list # conda environments: # base * /home/ytt/anaconda3 DCEvo /home/ytt/anaconda3/envs/DCEvo FAF /home/ytt/anaconda3/envs/FAF # 要迁移的环境 SegMiF /home/ytt/anaconda3/envs/SegMiF TDFusion /home/ytt/anaconda3/envs/TDFusion TIF /home/ytt/anaconda3/envs/TIF habitat /home/ytt/anaconda3/envs/habitat matcha /home/ytt/anaconda3/envs/matcha pytorch20 /home/ytt/anaconda3/envs/pytorch20 tardal /home/ytt/anaconda3/envs/tardal ylsnet /home/ytt/anaconda3/envs/ylsnet yolov5 /home/ytt/anaconda3/envs/yolov5 zjl /home/ytt/anaconda3/envs/zjls打包要迁移的环境(base) ytt@ytt-System-Product-Name:~$ cd /home/ytt/anaconda3/envs # 进入目录 (base) ytt@ytt-System-Product-Name:~$ tar -czvf FAF.tar.gz ./FAF/* # 打包=>使用FTP传到主机BB主机解压到当前目录(base) excelsior@dlu:~/miniconda3/envs$ tar -xzvf FAF.tar.gz # 解压到当前目录查看环境(base) excelsior@dlu:~/miniconda3/envs$ conda env list # conda environments: # base * /home/excelsior/miniconda3 FAF /home/excelsior/miniconda3/envs/FAF # 新解压出来的 INF /home/excelsior/miniconda3/envs/INF IR /home/excelsior/miniconda3/envs/IR MG /home/excelsior/miniconda3/envs/MG MM /home/excelsior/miniconda3/envs/MM MoLA /home/excelsior/miniconda3/envs/MoLA fmujie /home/excelsior/miniconda3/envs/fmujie mld /home/excelsior/miniconda3/envs/mld ylsnet /home/excelsior/miniconda3/envs/ylsnet激活环境并使用(base) excelsior@dlu:~/miniconda3/envs$ conda activate FAF无法使用pip/pip3安装包(FAF) excelsior@dlu:~/miniconda3/envs$ pip list -bash: /home/excelsior/miniconda3/envs/FAF/bin/pip：/home/ytt/anaconda3/envs/FAF/bin/python：解释器错误: 没有那个文件或目录 (FAF) excelsior@dlu:~/miniconda3/envs$ pip3 list -bash: /home/excelsior/miniconda3/envs/FAF/bin/pip3：/root/anaconda3/envs/FAF/bin/python：解释器错误: 权限不够解决方法编辑迁移环境bin目录下的pip文件，pip3同理文件内容#!/home/ytt/anaconda3/envs/FAF/bin/python # -*- coding: utf-8 -*- import re import sys from pip._internal.cli.main import main if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])修改第一行内容至新的环境如下#!/home/excelsior/miniconda3/envs/FAF/bin/python # -*- coding: utf-8 -*- import re import sys from pip._internal.cli.main import main if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(main())(FAF) excelsior@dlu:~/miniconda3/envs$ pip list # 检验能否正常使用 Package Version ------------------------ ----------- absl-py 2.1.0 certifi 2025.4.26 charset-normalizer 3.4.2 contourpy 1.3.2 cycler 0.12.1可以看到A主机是安装的完整的conda，B主机安装的是miniconda，只要是Nvidia显卡驱动支持环境的Cuda版本，就能够通用适配需求遇到了就自然知道了，嘻嘻(●'◡'●)
训练常用命令nohup python train.py > outLog.log &nohup挂后台运行，退出Xshell连接无影响，防止断网> outLog.log将输出输出到outLog.log文件&追加模式一般运行该命令后会提示如下：(base) root@d9dcc1730df7:~/data1/fmujie/dpl/cal_fgvc# nohup python train_distributed.py > outNA2245628.log & [1] 14131 (base) root@d9dcc1730df7:~/data1/fmujie/dpl/cal_fgvc# nohup: ignoring input and redirecting stderr to stdout如果忘记这个进程号也没关系ps -aux | grep train_distributed.py查看(base) root@d9dcc1730df7:~/data1/fmujie/dpl/cal_fgvc# ps -aux | grep train_distributed.py root 14133 104 0.5 34309892 1401112 pts/0 Rl 23:32 0:54 python train_distributed.py --dataset=nabirds --img_size=224 --lr_scale_ratio=4 root 14196 16.3 0.2 16130636 607052 pts/0 Rl 23:32 0:07 python train_distributed.py --dataset=nabirds --img_size=224 --lr_scale_ratio=4 root 14259 15.7 0.2 16130648 607052 pts/0 Sl 23:32 0:07 python train_distributed.py --dataset=nabirds --img_size=224 --lr_scale_ratio=4 root 14322 15.7 0.2 16130660 607116 pts/0 Sl 23:32 0:07 python train_distributed.py --dataset=nabirds --img_size=224 --lr_scale_ratio=4 root 14385 18.5 0.2 16130672 607072 pts/0 Sl 23:32 0:08 python train_distributed.py --dataset=nabirds --img_size=224 --lr_scale_ratio=4 root 14592 0.0 0.0 3296 716 pts/0 S+ 23:33 0:00 grep --color=auto train_distributed.py
R-Drop: Regularized Dropout for Neural NetworksDropout is a powerful and widely used technique to regularize the training of deep neural networks. Dropout在训练和推理时存在不一致的问题（集成学习）R 对每个子模型的分布做一个KL散度import numpy as np def train_r_drop(ratio, x, w1, b1, w2, b2): # 输入复制一份 x = torch.cat([x, x], dim=0) layer1 = np.maximum(0, np.dot(w1, x) + b1) mask1 = np.random.binomial(1, 1-ratio, layer1.shape) layer1 = layer1 * mask1 layer2 = np.maximum(0, np.dot(w2, layer1) + b2) mask2 = np.random.binomial(1, 1-ratio, layer2.shape) layer2 = layer2 * mask2 logits = func(layer2) logits1, logits2 = logits[:bs, :], logits[bs:, :] nll1 = nll(logits1, label) nll2 = nll(logits2, label) kl_los = kl(logits1, logits2) loss = nll1 + nll2 + kl_loss return loss def test(ratio, x, w1, b1, w2, b2): layer1 = np.maximum(0, np.dot(w1, x) + b1) layer1 = layer1 * (1-ratio) layer2 = np.maximum(0, np.dot(w2, layer1) + b2) layer2 = layer2 * (1-ratio) return layer2import torch conv_layer = torch.nn.Conv2d(in_channels=2, out_channels=2, kernel_size=3, padding="same") for i in conv_layer.named_parameters(): print(i) ''' ('weight', Parameter containing: tensor([[[[-0.1885, 0.1133, -0.0103], [-0.0462, -0.1434, -0.0569], [ 0.1969, 0.0827, -0.1245]], [[ 0.1554, 0.1459, 0.0405], [-0.0806, 0.1438, -0.1319], [-0.1495, -0.0960, 0.2193]]], [[[-0.1748, 0.1517, 0.0040], [ 0.0097, 0.1565, 0.1850], [-0.0261, 0.1298, 0.0211]], [[-0.0964, -0.0133, -0.0378], [ 0.0617, -0.2037, 0.0011], [-0.0542, -0.1499, 0.0187]]]], requires_grad=True)) ('bias', Parameter containing: tensor([0.0560, 0.2200], requires_grad=True)) ''' conv_layer.weights.size() ''' 对于输出的第1个通道而言： 对于输入的2个通道都有3×3的卷积核分别在这2个输入通道上做卷积再求和，就得到了，第1个输出通道的特征值。 对于输出的第2个通道而言： 又来了2个卷积核，对输入的2个通道做卷积，得到第2个输出通道的特征值。 torch.Size([2, 2, 3, 3]) 3,3为Kernel_size大小 2,2 是输出×输入 ''' conv_layer.bias.size() ''' torch.Size([2]) 和输出通道一致 ''' conv_layer = torch.nn.Conv2d(in_channels=2, out_channels=4, kernel_size=3, padding="same") print(conv_layer.weight.size()) ''' 4中每个输出通道，都有2套3×3的矩阵对输入进行滑动卷积，再将2个通道的值加起来再加上bias就构成输出通道的特征值 torch.Size([4, 2, 3, 3]) '''Conv2dtorch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros', device=None, dtype=None)stride controls the stride for the cross-correlation, a single number or a tuple.padding controls the amount of padding applied to the input. It can be either a string {‘valid’, ‘same’} or an int / a tuple of ints giving the amount of implicit padding applied on both sides.dilation controls the spacing between the kernel points; also known as the à trous algorithm. It is harder to describe, but this link has a nice visualization of what dilation does.groups controls the connections between inputs and outputs. in_channels and out_channels must both be divisible by groups. For example,At groups=1, all inputs are convolved to all outputs.At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels and producing half the output channels, and both subsequently concatenated.At groups= in_channels, each input channel is convolved with its own set of filters (of size out_channelsin_channelsin_channelsout_channels).The parameters kernel_size, stride, padding, dilation can either be:a single int – in which case the same value is used for the height and width dimensiona tuple of two ints – in which case, the first int is used for the height dimension, and the second int for the width dimension深度和可分离卷积groups > 1；且能够被input_channels和out_channels整除同时将两者分为好多组，单独做卷积。好处：1、并没有将通道完全的mix，假设有8个通道，分为2组，前4个通道后4个通道分别做一个卷积组2、计算量下降resnet的一个变体网络 RepConvpoint-wise convolution1×1的卷积卷积基本假设：局部关联性、平移不变性不考虑周围点，仅考虑自身：只是将每个通道上的值进行加权求和channels mixdepth-wise convolutiongroups > 1 不考虑对所有channels进行mix，而是划分部分miximport torch # 分为2部分：1、输入通道为1，输出通道为2的卷积；2、也是输入通道为1，输出通道为2的卷积，最后再拼起来 计算量下降 conv_layer = torch.nn.Conv2d(in_channels=2, out_channels=4, kernel_size=3, padding="same", groups=2) print(conv_layer.weight.size()) print(conv_layer.bias.size()) ''' torch.Size([4, 1, 3, 3]) torch.Size([4]) ''' conv_layer1 = torch.nn.Conv2d(in_channels=1, out_channels=2, kernel_size=3, padding="same") conv_layer2 = torch.nn.Conv2d(in_channels=1, out_channels=2, kernel_size=3, padding="same") print(conv_layer1.weight.size()) print(conv_layer2.weight.size()) print(conv_layer1.bias.size()) print(conv_layer2.bias.size()) ''' torch.Size([2, 1, 3, 3]) torch.Size([2, 1, 3, 3]) torch.Size([2]) torch.Size([2]) '''如何将3×3卷积、1×1、x融合为1个3×3卷积?import torch import torch.nn as nn import torch.nn.functional as F in_channels = 2 out_channels = 2 kernel_size = 3 w = 9 h = 9 # res_block = 3 * 3 conv + 1 * 1 conv + input # 定义输入图片大小 x = torch.ones(1, in_channels, w, h) # 方法1：原生写法 # 输入与输出通道一致 3 * 3 conv 要做padding conv_2d = nn.Conv2d(in_channels, out_channels, kernel_size, padding="same") # 定义1 * 1 conv只考虑像素点的通道而不考虑周围点的关联性 不需要padding conv_2d_pointwise = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding="same") result1 = conv_2d(x) + conv_2d_pointwise(x) + x # print(result1) # 方法2：算子融合 RepConv # 1) 改造 # 将point-wise卷积和x本身都写为3*3的卷积 # 最终把3个卷积融合为1个卷积 # 扩充point-wise卷积核的weight 1*1 => 3*3 # 1 * 1 => 3 * 3 weight:2(O) 2(I) 1(k) 1(k) => 2(O) 2(I) 3(K1) 3(K2) # 上下、左右各pad 0 pad为从内到外pad 先pad K2上下、左右再pad K1上下、左右 O与I上下、左右不pad pointwise_to_conv_weight = F.pad(conv_2d_pointwise.weight, pad=[1, 1, 1, 1, 0, 0, 0, 0]) conv_2d_for_pointwise = nn.Conv2d(in_channels, out_channels, kernel_size, padding="same") conv_2d_for_pointwise.weight = nn.Parameter(pointwise_to_conv_weight) conv_2d_for_pointwise.bias = conv_2d_pointwise.bias # 将x本身变为1个3*3卷积 x经过卷积后仍为x # 不考虑相邻点融合=>`point-wise convolution` # 不考虑通道间融合=>`depth-wise convolution` # weight:2*2*3*3 # 第1个3*3中间为1, 其余部分为0, 第2个3*3全0; # 第4个3*3中间为1, 其余部分为0, 第3个3*3全0; # 不考虑相邻通道 zeros = torch.unsqueeze(torch.zeros(kernel_size, kernel_size), 0) # 不考虑相邻点 stars = torch.unsqueeze(F.pad(torch.ones(1, 1), pad=[1, 1, 1, 1]), 0) # 第1个channel stars_zeros = torch.unsqueeze(torch.cat([stars, zeros], 0), 0) # 第2个channel zeros_stars = torch.unsqueeze(torch.cat([zeros, stars], 0), 0) identity_to_conv_weight = torch.cat([stars_zeros, zeros_stars], 0) identity_to_conv_bias = torch.zeros([out_channels]) conv_2d_for_identity = nn.Conv2d(in_channels, out_channels, kernel_size, padding="same") conv_2d_for_identity.weight = nn.Parameter(identity_to_conv_weight) conv_2d_for_identity.bias = nn.Parameter(identity_to_conv_bias) result2 = conv_2d(x) + conv_2d_for_pointwise(x) + conv_2d_for_identity(x) # print(result2) print(torch.all(torch.isclose(result1, result2))) # 目前将1*1和x本身都改为3*3 # 2) 融合 conv_2d_for_fussion = nn.Conv2d(in_channels, out_channels, kernel_size, padding="same") conv_2d_for_fussion.weight = nn.Parameter(conv_2d.weight.data + conv_2d_for_pointwise.weight.data + conv_2d_for_identity.weight.data) conv_2d_for_fussion.bias = nn.Parameter(conv_2d.bias.data + conv_2d_for_pointwise.bias.data + conv_2d_for_identity.bias.data) result3 = conv_2d_for_fussion(x) print(torch.all(torch.isclose(result2, result3))) ''' tensor(True) tensor(True) '''
home\.cache\torch\checkpoints有什么用，遇到就知道了，有时候集群网不好，可以本地下载后传到集群上的指定目录，就不用在集群上龟速下载了。
NN.DROPOUTCLASStorch.nn.Dropout(p=0.5, inplace=False)Parametersp (float) – probability of an element to be zeroed. Default: 0.5inplace (bool) – If set to True, will do this operation in-place. Default: FalseShape:Input: (∗)(∗). Input can be of any shapeOutput: (∗)(∗). Output is of the same shape as inputm = nn.Dropout(p=0.2) input = torch.randn(20, 16) output = m(input)如何判断当前是否为Train？ 参数中无training 参数 ==resolution：因为类的父类继承了module类，而module类种有self.training的定义，所以不需要再额外需要training参数了==TORCH.NN.FUNCTIONAL.DROPOUTtorch.nn.functional.dropout(input, p=0.5, training=True, inplace=False)Parametersp (float) – probability of an element to be zeroed. Default: 0.5training (bool) – apply dropout if is True. Default: Trueinplace (bool) – If set to True, will do this operation in-place. Default: FalseReturn typeTensor==函数无继承关系，所以需要额外的training参数，不过该函数还是底层，类内部也需要调用该参数，用的training参数为父父类中定义的==含dropout，训练时相当于集成学习，训练多个网络，而在测试时只是逼近需要在测试期间与训练阶段的期望值相互逼近的话，需要在测试阶段将权重$w$乘以一个$p$。因为在训练时按照$p$的概率保留了一些神经元的连接，所以在测试的时候需要在权重上乘以概率。但是为了在测试时耗时尽可能少，所以在训练时的权重上乘了一个放缩因子（实际运行中）。from .module import Module from .. import functional as F from torch import Tensor __all__ = ['Dropout', 'Dropout1d', 'Dropout2d', 'Dropout3d', 'AlphaDropout', 'FeatureAlphaDropout'] class _DropoutNd(Module): __constants__ = ['p', 'inplace'] p: float inplace: bool def __init__(self, p: float = 0.5, inplace: bool = False) -> None: super().__init__() if p < 0 or p > 1: raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") self.p = p self.inplace = inplace def extra_repr(self) -> str: return f'p={self.p}, inplace={self.inplace}' [docs]class Dropout(_DropoutNd): def forward(self, input: Tensor) -> Tensor: return F.dropout(input, self.p, self.training, self.inplace) [docs]class Dropout1d(_DropoutNd): # 仅有一个Tensor参数，无显式training参数 def forward(self, input: Tensor) -> Tensor: # 调用了函数 self.training哪来的？Class module中 return F.dropout1d(input, self.p, self.training, self.inplace) [docs]class Dropout2d(_DropoutNd): def forward(self, input: Tensor) -> Tensor: return F.dropout2d(input, self.p, self.training, self.inplace) [docs]class Dropout3d(_DropoutNd): def forward(self, input: Tensor) -> Tensor: return F.dropout3d(input, self.p, self.training, self.inplace) [docs]class AlphaDropout(_DropoutNd): def forward(self, input: Tensor) -> Tensor: return F.alpha_dropout(input, self.p, self.training) [docs]class FeatureAlphaDropout(_DropoutNd): def forward(self, input: Tensor) -> Tensor: return F.feature_alpha_dropout(input, self.p, self.training) ... def train(self: T, mode: bool = True) -> T: if not isinstance(mode, bool): raise ValueError("training mode is expected to be boolean") self.training = mode for module in self.children(): module.train(mode) return self def eval(self: T) -> T: return self.train(False) ...caffe2源码GitHub地址：https://github.com/facebookarchive/caffe2if (is_test_) { if (!IsInputOutputAlias(0, 0)) { context_.CopyFromCPU<float>( X.numel(), X.data<float>(), Y->template mutable_data<float>()); } return true; } else { // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) // 算一个放缩因子, ratio为drop的比例(为了使得测试时的时间尽可能少, 将运算放到训练阶段了) float scale = ratio_ >= 1.0 ? 0.0:1. / (1. - ratio_); // mask=true means keep, and mask=false means not keep, so we will // generate probability depending on 1-ratio. // 伯努利分布根据1 - ratio生成 at::bernoulli_distribution<double> dist(1. - ratio_); const float* Xdata = X.data<float>(); float* Ydata = Y->template mutable_data<float>(); auto mask = Output(1, X.sizes(), at::dtype<bool>()); bool* mask_data = mask->template mutable_data<bool>(); // 随机生成器生成一个张量 auto* gen = context_.RandGenerator(); for (int i = 0; i < X.numel(); ++i) { //大于0.5 则为1, 小于则为0 mask_data[i] = dist(gen) > 0.5; // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions,bugprone-narrowing-conversions) Ydata[i] = Xdata[i] * scale * mask_data[i]; } return true; } }numpy两种实现方式import numpy as np def train(ratio, x, w1, b1, w2, b2): layer1 = np.maximum(0, np.dot(w1, x) + b1) mask1 = np.random.binomial(1, 1-ratio, layer1.shape) layer1 = layer1 * mask1 layer2 = np.maximum(0, np.dot(w2, layer1) + b2) mask2 = np.random.binomial(1, 1-ratio, layer2.shape) layer2 = layer2 * mask2 return layer2 def test(ratio, x, w1, b1, w2, b2): layer1 = np.maximum(0, np.dot(w1, x) + b1) layer1 = layer1 * (1-ratio) layer2 = np.maximum(0, np.dot(w2, layer1) + b2) layer2 = layer2 * (1-ratio) return layer2import numpy as np def train(ratio, x, w1, b1, w2, b2): layer1 = np.maximum(0, np.dot(w1, x) + b1) mask1 = np.random.binomial(1, 1-ratio, layer1.shape) layer1 = layer1 * mask1 # 额外/(1-ratio) layer1 = layer1/(1-ratio) layer2 = np.maximum(0, np.dot(w2, layer1) + b2) mask2 = np.random.binomial(1, 1-ratio, layer2.shape) layer2 = layer2 * mask2 layer2 = layer2/(1-ratio) return layer2 def test(x, w1, b1, w2, b2): layer1 = np.maximum(0, np.dot(w1, x) + b1) layer2 = np.maximum(0, np.dot(w2, layer1) + b2) return layer2
Saving and loading models for inference in PyTorchThere are two approaches for saving and loading models for inference in PyTorch.The first is saving and loading the state_dictand the second is saving and loading the entire model. def state_dict(self, *args, destination=None, prefix='', keep_vars=False): r"""Returns a dictionary containing references to the whole state of the module. Both parameters and persistent buffers (e.g. running averages) are included. Keys are corresponding parameter and buffer names. Parameters and buffers set to ``None`` are not included. """ # TODO: Remove `args` and the parsing logic when BC allows. if len(args) > 0: if destination is None: destination = args[0] if len(args) > 1 and prefix == '': prefix = args[1] if len(args) > 2 and keep_vars is False: keep_vars = args[2] # DeprecationWarning is ignored by default warnings.warn( "Positional args are being deprecated, use kwargs instead. Refer to " "https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict" " for details.") if destination is None: destination = OrderedDict() destination._metadata = OrderedDict() local_metadata = dict(version=self._version) if hasattr(destination, "_metadata"): destination._metadata[prefix[:-1]] = local_metadata for hook in self._state_dict_pre_hooks.values(): hook(self, prefix, keep_vars) self._save_to_state_dict(destination, prefix, keep_vars) for name, module in self._modules.items(): if module is not None: module.state_dict(destination=destination, prefix=prefix + name + '.', keep_vars=keep_vars) for hook in self._state_dict_hooks.values(): hook_result = hook(self, destination, prefix, local_metadata) if hook_result is not None: destination = hook_result return destinationSteps构建DatasetDefine and initialize the neural networkclass Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(3, 6, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(6, 16, 5) self.fc1 = nn.Linear(16 * 5 * 5, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(F.relu(self.conv1(x))) x = self.pool(F.relu(self.conv2(x))) x = x.view(-1, 16 * 5 * 5) x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) x = self.fc3(x) return x net = Net() print(net)Initialize the optimizer# 第一个参数传入的是parameters()函数:会自动遍历所有子模块 optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)Save and load the model via state_dictPATH = "state_dict_model.pt" # Save 获取该模型全部的参数和Buffer量，但是没有保存模型的结构 torch.save(net.state_dict(), PATH) # Load 先创建网络结构 model = Net() # 再加载权重 model.load_state_dict(torch.load(PATH)) # 仅针对inference，因为没有保存优化器部分 model.eval()Save and load entire model# Specify a path PATH = "entire_model.pt" # Save torch.save(net, PATH) # Load 不用事先定义结构了 model = torch.load(PATH) model.eval()Saving and loading a general checkpoint in PyTorchSaving and loading a general checkpoint model for ==inference or resuming training== can be helpful for picking up where you last left off. When saving a general checkpoint, you must save more than just the model’s state_dict. ==It is important to also save the optimizer’s state_dict==, as this contains buffers and parameters that are updated as the model trains. Other items that you may want to save are the epoch you left off on, the latest recorded training loss, external torch.nn.Embedding layers, and more, based on your own algorithm.4. Save the general checkpoint# Additional information EPOCH = 5 PATH = "model.pt" LOSS = 0.4 torch.save({ 'epoch': EPOCH, 'model_state_dict': net.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': LOSS, }, PATH)5. Load the general checkpointmodel = Net() optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) checkpoint = torch.load(PATH) # 模型参数 model.load_state_dict(checkpoint['model_state_dict']) # 优化器参数 optimizer.load_state_dict(checkpoint['optimizer_state_dict']) epoch = checkpoint['epoch'] loss = checkpoint['loss'] model.eval() # - or - model.train()Saving and loading multiple models in one file using PyTorch2. Define and initialize the neural networkclass Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(3, 6, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(6, 16, 5) self.fc1 = nn.Linear(16 * 5 * 5, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(F.relu(self.conv1(x))) x = self.pool(F.relu(self.conv2(x))) x = x.view(-1, 16 * 5 * 5) x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) x = self.fc3(x) return x netA = Net() netB = Net()3. Initialize the optimizeroptimizerA = optim.SGD(netA.parameters(), lr=0.001, momentum=0.9) optimizerB = optim.SGD(netB.parameters(), lr=0.001, momentum=0.9)4. Save multiple models# Specify a path to save to PATH = "model.pt" torch.save({ 'modelA_state_dict': netA.state_dict(), 'modelB_state_dict': netB.state_dict(), 'optimizerA_state_dict': optimizerA.state_dict(), 'optimizerB_state_dict': optimizerB.state_dict(), }, PATH)5. Load multiple modelsmodelA = Net() modelB = Net() optimModelA = optim.SGD(modelA.parameters(), lr=0.001, momentum=0.9) optimModelB = optim.SGD(modelB.parameters(), lr=0.001, momentum=0.9) checkpoint = torch.load(PATH) modelA.load_state_dict(checkpoint['modelA_state_dict']) modelB.load_state_dict(checkpoint['modelB_state_dict']) optimizerA.load_state_dict(checkpoint['optimizerA_state_dict']) optimizerB.load_state_dict(checkpoint['optimizerB_state_dict']) modelA.eval() modelB.eval() # - or - modelA.train() modelB.train()
train# 实例化一个模型,在模型后调用.train(True),说明我们将该模型设置为训练模式 def train(self: T, mode: bool = True) -> T: r"""Sets the module in training mode. This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc. Args: mode (bool): whether to set training mode (``True``) or evaluation mode (``False``). Default: ``True``. Returns: Module: self """ # 首先判断mode是否为bool类型 if not isinstance(mode, bool): raise ValueError("training mode is expected to be boolean") self.training = mode # 如果传入了True,则self.training为True,且子模块递归都设置为训练模式 for module in self.children(): module.train(mode) return selfeval()def eval(self: T) -> T: r"""Sets the module in evaluation mode. This has any effect only on certain modules. See documentations of particular modules for details of their behaviors in training/evaluation mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`, etc. This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`. See :ref:`locally-disable-grad-doc` for a comparison between `.eval()` and several similar mechanisms that may be confused with it. Returns: Module: self """ return self.train(False)requires_grad_# 是否需要自动微分 module的函数 def requires_grad_(self: T, requires_grad: bool = True) -> T: r"""Change if autograd should record operations on parameters in this module. This method sets the parameters' :attr:`requires_grad` attributes in-place. This method is helpful for freezing part of the module for finetuning or training parts of a model individually (e.g., GAN training). See :ref:`locally-disable-grad-doc` for a comparison between `.requires_grad_()` and several similar mechanisms that may be confused with it. Args: requires_grad (bool): whether autograd should record operations on parameters in this module. Default: ``True``. Returns: Module: self """ # 对当前模型的所有参数遍历 for p in self.parameters(): # 将每个参数的requires_grad_设置为requires_grad tensor函数 p.requires_grad_(requires_grad) return selfzero_grad在训练的每一步开始之前调用优化器的zero_grad，PyTorch会对每个参数的梯度做一个累积，第一次算梯度正确的，第二次计算前没有调用zero_grad，在第二次计算时梯度变为两倍。对优化器调用就行。def zero_grad(self, set_to_none: bool = True) -> None: r"""Resets gradients of all model parameters. See similar function under :class:`torch.optim.Optimizer` for more context. Args: set_to_none (bool): instead of setting to zero, set the grads to None. See :meth:`torch.optim.Optimizer.zero_grad` for details. """ if getattr(self, '_is_replica', False): warnings.warn( "Calling .zero_grad() from a module created with nn.DataParallel() has no effect. " "The parameters are copied (in a differentiable manner) from the original module. " "This means they are not leaf nodes in autograd and so don't accumulate gradients. " "If you need gradients in your forward method, consider using autograd.grad instead.") for p in self.parameters(): if p.grad is not None: if set_to_none: p.grad = None else: if p.grad.grad_fn is not None: p.grad.detach_() else: p.grad.requires_grad_(False) p.grad.zero_()__repr\_\_魔法函数，对模型进行字符串展示def __repr__(self): # We treat the extra repr like the sub-module, one item per line extra_lines = [] extra_repr = self.extra_repr() # empty string will be split into list [''] if extra_repr: extra_lines = extra_repr.split('\n') child_lines = [] for key, module in self._modules.items(): mod_str = repr(module) mod_str = _addindent(mod_str, 2) child_lines.append('(' + key + '): ' + mod_str) lines = extra_lines + child_lines main_str = self._get_name() + '(' if lines: # simple one-liner info, which most builtin Modules will use if len(extra_lines) == 1 and not child_lines: main_str += extra_lines[0] else: main_str += '\n ' + '\n '.join(lines) + '\n' main_str += ')' return main_strprint(str(test_net)) ''' Net( (linear1): Linear(in_features=2, out_features=3, bias=True) (linear2): Linear(in_features=3, out_features=4, bias=True) (batch_norm): BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) ) '''__dir\_\_def __dir__(self): module_attrs = dir(self.__class__) # 当前类自身的dir attrs = list(self.__dict__.keys()) # 当前字典的所有键值 parameters = list(self._parameters.keys()) # 参数的键值 modules = list(self._modules.keys()) # module的键值 buffers = list(self._buffers.keys()) # buffer的键值 keys = module_attrs + attrs + parameters + modules + buffers # Eliminate attrs that are not legal Python variable names keys = [key for key in keys if not key[0].isdigit()] return sorted(keys)print(dir(test_net)) ''' ['T_destination', '__annotations__', '__call__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_apply', '_backward_hooks', '_backward_pre_hooks', '_buffers', '_call_impl', '_compiled_call_impl', '_forward_hooks', '_forward_hooks_always_called', '_forward_hooks_with_kwargs', '_forward_pre_hooks', '_forward_pre_hooks_with_kwargs', '_get_backward_hooks', '_get_backward_pre_hooks', '_get_name', '_is_full_backward_hook', '_load_from_state_dict', '_load_state_dict_post_hooks', '_load_state_dict_pre_hooks', '_maybe_warn_non_full_backward_hook', '_modules', '_named_members', '_non_persistent_buffers_set', '_parameters', '_register_load_state_dict_pre_hook', '_register_state_dict_hook', '_replicate_for_data_parallel', '_save_to_state_dict', '_slow_forward', '_state_dict_hooks', '_state_dict_pre_hooks', '_version', '_wrapped_call_impl', 'add_module', 'apply', 'batch_norm', 'bfloat16', 'buffers', 'call_super_init', 'children', 'compile', 'cpu', 'cuda', 'double', 'dump_patches', 'eval', 'extra_repr', 'float', 'forward', 'get_buffer', 'get_extra_state', 'get_parameter', 'get_submodule', 'half', 'ipu', 'linear1', 'linear2', 'load_state_dict', 'modules', 'named_buffers', 'named_children', 'named_modules', 'named_parameters', 'parameters', 'register_backward_hook', 'register_buffer', 'register_forward_hook', 'register_forward_pre_hook', 'register_full_backward_hook', 'register_full_backward_pre_hook', 'register_load_state_dict_post_hook', 'register_module', 'register_parameter', 'register_state_dict_pre_hook', 'requires_grad_', 'set_extra_state', 'share_memory', 'state_dict', 'to', 'to_empty', 'train', 'training', 'type', 'xpu', 'zero_grad'] '''Sequentialdef __init__(self, *args): super().__init__() if len(args) == 1 and isinstance(args[0], OrderedDict): # 对字典进行遍历, 向当前Sequential中添加模块,按照给定的Key for key, module in args[0].items(): self.add_module(key, module) else: # 直接传入module实例,也会添加模块,但是模块名称为数字形式,从0开始 for idx, module in enumerate(args): self.add_module(str(idx), module)示例s = nn.Sequential( nn.Linear(2, 3), nn.Linear(3, 4) ) print(s) ''' Sequential( (0): Linear(in_features=2, out_features=3, bias=True) (1): Linear(in_features=3, out_features=4, bias=True) ) ''' print(s._modules) ''' OrderedDict([('0', Linear(in_features=2, out_features=3, bias=True)), ('1', Linear(in_features=3, out_features=4, bias=True))]) '''forwardinput依次的过每个模块def forward(self, input): for module in self: input = module(input) return inputModuleList将很多子module放到列表中Example:: class MyModule(nn.Module): def __init__(self): super().__init__() self.linears = nn.ModuleList([nn.Linear(10, 10) for i in range(10)]) def forward(self, x): # ModuleList can act as an iterable, or be indexed using ints for i, l in enumerate(self.linears): x = self.linears[i // 2](x) + l(x) return x
Save and Load the ModelPyTorch models store the learned parameters in an internal state dictionary, called state_dict. These can be persisted via the torch.save method:model = models.vgg16(weights='IMAGENET1K_V1') # 保存模型的参数、优化器状态、batch_nomalization、drop_out等等一系列的buffer变量 # model.state_dict() 只保留模型的权重 torch.save(model.state_dict(), 'model_weights.pth') # 每个model都会包含一个state_dict()状态字典:包含模型所有的参数和buffer变量To load model weights, you need to create an instance of the same model first, and then load the parameters using load_state_dict() method.model = models.vgg16() # we do not specify ``weights``, i.e. create untrained model model.load_state_dict(torch.load('model_weights.pth')) # 推理模式:会影响到drop_out、batch_nomalization model.eval()同时保存优化器权重的methodimport torch import torch.nn as nn import torch.optim as optim class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = nn.Conv2d(3, 6, 5) self.pool = nn.MaxPool2d(2, 2) self.conv2 = nn.Conv2d(6, 16, 5) self.fc1 = nn.Linear(16 * 5 * 5, 120) self.fc2 = nn.Linear(120, 84) self.fc3 = nn.Linear(84, 10) def forward(self, x): x = self.pool(F.relu(self.conv1(x))) x = self.pool(F.relu(self.conv2(x))) x = x.view(-1, 16 * 5 * 5) x = F.relu(self.fc1(x)) x = F.relu(self.fc2(x)) x = self.fc3(x) return x net = Net() print(net) optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) # Collect all relevant information and build your dictionary. # Additional information EPOCH = 5 PATH = "model.pt" LOSS = 0.4 # 不光要保存模型的权重，还要保存优化器的状态量、Loss、Epoch torch.save({ 'epoch': EPOCH, 'model_state_dict': net.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': LOSS, }, PATH)Remember to first initialize the model and optimizer, then load the dictionary locally.model = Net() optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) checkpoint = torch.load(PATH) model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) epoch = checkpoint['epoch'] loss = checkpoint['loss'] model.eval() # - or - model.train()MODULE SOURCE CODEto def to(self, *args, **kwargs): r"""Moves and/or casts the parameters and buffers. This can be called as .. function:: to(device=None, dtype=None, non_blocking=False) :noindex: .. function:: to(dtype, non_blocking=False) :noindex: .. function:: to(tensor, non_blocking=False) :noindex: .. function:: to(memory_format=torch.channels_last) :noindex: Its signature is similar to :meth:`torch.Tensor.to`, but only accepts floating point or complex :attr:`dtype`\ s. In addition, this method will only cast the floating point or complex parameters and buffers to :attr:`dtype` (if given). The integral parameters and buffers will be moved :attr:`device`, if that is given, but with dtypes unchanged. When :attr:`non_blocking` is set, it tries to convert/move asynchronously with respect to the host if possible, e.g., moving CPU Tensors with pinned memory to CUDA devices. See below for examples. .. note:: This method modifies the module in-place. Args: device (:class:`torch.device`): the desired device of the parameters and buffers in this module dtype (:class:`torch.dtype`): the desired floating point or complex dtype of the parameters and buffers in this module tensor (torch.Tensor): Tensor whose dtype and device are the desired dtype and device for all parameters and buffers in this module memory_format (:class:`torch.memory_format`): the desired memory format for 4D parameters and buffers in this module (keyword only argument) Returns: Module: self Examples:: >>> # xdoctest: +IGNORE_WANT("non-deterministic") >>> linear = nn.Linear(2, 2) >>> linear.weight Parameter containing: tensor([[ 0.1913, -0.3420], [-0.5113, -0.2325]]) >>> linear.to(torch.double) Linear(in_features=2, out_features=2, bias=True) >>> linear.weight Parameter containing: tensor([[ 0.1913, -0.3420], [-0.5113, -0.2325]], dtype=torch.float64) >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA1) >>> gpu1 = torch.device("cuda:1") >>> linear.to(gpu1, dtype=torch.half, non_blocking=True) Linear(in_features=2, out_features=2, bias=True) >>> linear.weight Parameter containing: tensor([[ 0.1914, -0.3420], [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1') >>> cpu = torch.device("cpu") >>> linear.to(cpu) Linear(in_features=2, out_features=2, bias=True) >>> linear.weight Parameter containing: tensor([[ 0.1914, -0.3420], [-0.5112, -0.2324]], dtype=torch.float16) >>> linear = nn.Linear(2, 2, bias=None).to(torch.cdouble) >>> linear.weight Parameter containing: tensor([[ 0.3741+0.j, 0.2382+0.j], [ 0.5593+0.j, -0.4443+0.j]], dtype=torch.complex128) >>> linear(torch.ones(3, 2, dtype=torch.cdouble)) tensor([[0.6122+0.j, 0.1150+0.j], [0.6122+0.j, 0.1150+0.j], [0.6122+0.j, 0.1150+0.j]], dtype=torch.complex128) """ device, dtype, non_blocking, convert_to_format = torch._C._nn._parse_to(*args, **kwargs) if dtype is not None: if not (dtype.is_floating_point or dtype.is_complex): raise TypeError('nn.Module.to only accepts floating point or complex ' f'dtypes, but got desired dtype={dtype}') if dtype.is_complex: warnings.warn( "Complex modules are a new feature under active development whose design may change, " "and some modules might not work as expected when using complex tensors as parameters or buffers. " "Please file an issue at https://github.com/pytorch/pytorch/issues/new?template=bug-report.yml " "if a complex module does not work as expected.") def convert(t): if convert_to_format is not None and t.dim() in (4, 5): return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking, memory_format=convert_to_format) return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking) return self._apply(convert)示例import torch import torch.nn as nn class Net(nn.Module): def __init__(self): # 调用父类=>Module的init函数 super(Net, self).__init__() self.linear1 = nn.Linear(2, 3) self.linear2 = nn.Linear(3, 4) self.batch_norm = nn.BatchNorm2d(4) test_net = Net() linear1 = test_net._modules['linear1'] print(linear1) ''' Linear(in_features=2, out_features=3, bias=True) ''' weight1 = linear1.weight print(weight1, weight1.dtype) ''' Parameter containing: tensor([[ 0.5884, 0.4096], [-0.0927, 0.3592], [ 0.6238, -0.1112]], requires_grad=True) torch.float32 ''' # 现在调用to函数将model中的浮点类型改为双精度类型 test_net.to(torch.double) print(test_net._modules['linear1'].weight.dtype) ''' torch.float64 ''' print(test_net._parameters()) ''' OrderedDict() ''' # _parameters并不会对子模块进行遍历，只是对当前模块自身进行一个搜索:有无实例化一个nn.Parameter的对象,所以此处为空字典 print(test_net._buffers) ''' OrderedDict() ''' print(test_net.state_dict) ''' OrderedDict([('linear1.weight', tensor([[ 0.1710, 0.2324], [ 0.5988, -0.0737], [-0.4426, 0.4938]], dtype=torch.float64)), ('linear1.bias', tensor([-0.6435, 0.2064, 0.4859], dtype=torch.float64)), ('linear2.weight', tensor([[-0.1337, 0.2572, 0.5521], [-0.1224, 0.2688, 0.3103], [-0.0960, -0.2258, -0.0193], [-0.0830, -0.3477, -0.1718]], dtype=torch.float64)), ('linear2.bias', tensor([-0.3606, -0.5329, -0.3330, -0.2828], dtype=torch.float64)), ('batch_norm.weight', tensor([1., 1., 1., 1.], dtype=torch.float64)), ('batch_norm.bias', tensor([0., 0., 0., 0.], dtype=torch.float64)), ('batch_norm.running_mean', tensor([0., 0., 0., 0.], dtype=torch.float64)), ('batch_norm.running_var', tensor([1., 1., 1., 1.], dtype=torch.float64)), ('batch_norm.num_batches_tracked', tensor(0))]) '''__getattr\_\_def __getattr__(self, name: str) -> Any: if '_parameters' in self.__dict__: _parameters = self.__dict__['_parameters'] if name in _parameters: return _parameters[name] if '_buffers' in self.__dict__: _buffers = self.__dict__['_buffers'] if name in _buffers: return _buffers[name] if '_modules' in self.__dict__: modules = self.__dict__['_modules'] if name in modules: return modules[name] raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")_save_to_state_dict将当前model的参数和buffers全部放到一个字典中，前述net.state_dict(),就是调用了module的state_dict，该方法又调用_save_to_state_dictdef _save_to_state_dict(self, destination, prefix, keep_vars): # 俩for循环, O(n)复杂度, 首先对当前model的parameters和buffers进行遍历,然后放到destination字典中 for name, param in self._parameters.items(): if param is not None: destination[prefix + name] = param if keep_vars else param.detach() for name, buf in self._buffers.items(): if buf is not None and name not in self._non_persistent_buffers_set: destination[prefix + name] = buf if keep_vars else buf.detach() extra_state_key = prefix + _EXTRA_STATE_KEY_SUFFIX if getattr(self.__class__, "get_extra_state", Module.get_extra_state) is not Module.get_extra_state: destination[extra_state_key] = self.get_extra_state()def state_dict(self, *args, destination=None, prefix='', keep_vars=False): # TODO: Remove `args` and the parsing logic when BC allows. if len(args) > 0: if destination is None: destination = args[0] if len(args) > 1 and prefix == '': prefix = args[1] if len(args) > 2 and keep_vars is False: keep_vars = args[2] # DeprecationWarning is ignored by default warnings.warn( "Positional args are being deprecated, use kwargs instead. Refer to " "https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict" " for details.") if destination is None: destination = OrderedDict() destination._metadata = OrderedDict() local_metadata = dict(version=self._version) if hasattr(destination, "_metadata"): destination._metadata[prefix[:-1]] = local_metadata for hook in self._state_dict_pre_hooks.values(): hook(self, prefix, keep_vars) # 将当前模块参数和buffers存放在destination字典中 self._save_to_state_dict(destination, prefix, keep_vars) # 对当前模块的子模块进行递归，将每个model的参数buffers都存放在destination字典中 for name, module in self._modules.items(): if module is not None: module.state_dict(destination=destination, prefix=prefix + name + '.', keep_vars=keep_vars) for hook in self._state_dict_hooks.values(): hook_result = hook(self, destination, prefix, local_metadata) if hook_result is not None: destination = hook_result return destination_load_from_state_dict从一个state_stict中得到parameters、buffers def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): for hook in self._load_state_dict_pre_hooks.values(): hook(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) # 先得到键,放在local_state中 persistent_buffers = {k: v for k, v in self._buffers.items() if k not in self._non_persistent_buffers_set} local_name_params = itertools.chain(self._parameters.items(), persistent_buffers.items()) local_state = {k: v for k, v in local_name_params if v is not None} assign_to_params_buffers = local_metadata.get("assign_to_params_buffers", False) for name, param in local_state.items(): key = prefix + name if key in state_dict: input_param = state_dict[key] if not torch.overrides.is_tensor_like(input_param): error_msgs.append(f'While copying the parameter named "{key}", ' 'expected torch.Tensor or Tensor-like object from checkpoint but ' f'received {type(input_param)}' ) continue # This is used to avoid copying uninitialized parameters into # non-lazy modules, since they dont have the hook to do the checks # in such case, it will error when accessing the .shape attribute. is_param_lazy = torch.nn.parameter.is_lazy(param) # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+ if not is_param_lazy and len(param.shape) == 0 and len(input_param.shape) == 1: input_param = input_param[0] if not is_param_lazy and input_param.shape != param.shape: # local shape should match the one in checkpoint error_msgs.append('size mismatch for {}: copying a param with shape {} from checkpoint, ' 'the shape in current model is {}.' .format(key, input_param.shape, param.shape)) continue if param.is_meta and not input_param.is_meta and not assign_to_params_buffers: warnings.warn(f'for {key}: copying from a non-meta parameter in the checkpoint to a meta ' 'parameter in the current model, which is a no-op. (Did you mean to ' 'pass `assign=True` to assign items in the state dictionary to their ' 'corresponding key in the module instead of copying them in place?)') try: with torch.no_grad(): if assign_to_params_buffers: # Shape checks are already done above if (isinstance(param, torch.nn.Parameter) and not isinstance(input_param, torch.nn.Parameter)): setattr(self, name, torch.nn.Parameter(input_param)) else: setattr(self, name, input_param) else: # 外部传来的字典，copy进param param.copy_(input_param) except Exception as ex: error_msgs.append(f'While copying the parameter named "{key}", ' f'whose dimensions in the model are {param.size()} and ' f'whose dimensions in the checkpoint are {input_param.size()}, ' f'an exception occurred : {ex.args}.' ) elif strict: missing_keys.append(key) extra_state_key = prefix + _EXTRA_STATE_KEY_SUFFIX if getattr(self.__class__, "set_extra_state", Module.set_extra_state) is not Module.set_extra_state: if extra_state_key in state_dict: self.set_extra_state(state_dict[extra_state_key]) elif strict: missing_keys.append(extra_state_key) elif strict and (extra_state_key in state_dict): unexpected_keys.append(extra_state_key) if strict: for key in state_dict.keys(): if key.startswith(prefix) and key != extra_state_key: input_name = key[len(prefix):] input_name = input_name.split('.', 1)[0] # get the name of param/buffer/child if input_name not in self._modules and input_name not in local_state: unexpected_keys.append(key)load_state_dict# 递归的将checkpoint中值赋予到模型 def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True, assign: bool = False): if not isinstance(state_dict, Mapping): raise TypeError(f"Expected state_dict to be dict-like, got {type(state_dict)}.") missing_keys: List[str] = [] unexpected_keys: List[str] = [] error_msgs: List[str] = [] # copy state_dict so _load_from_state_dict can modify it metadata = getattr(state_dict, '_metadata', None) state_dict = OrderedDict(state_dict) if metadata is not None: # mypy isn't aware that "_metadata" exists in state_dict state_dict._metadata = metadata # type: ignore[attr-defined] def load(module, local_state_dict, prefix=''): local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) if assign: local_metadata['assign_to_params_buffers'] = assign module._load_from_state_dict( local_state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) for name, child in module._modules.items(): if child is not None: child_prefix = prefix + name + '.' child_state_dict = {k: v for k, v in local_state_dict.items() if k.startswith(child_prefix)} load(child, child_state_dict, child_prefix) # Note that the hook can modify missing_keys and unexpected_keys. incompatible_keys = _IncompatibleKeys(missing_keys, unexpected_keys) for hook in module._load_state_dict_post_hooks.values(): out = hook(module, incompatible_keys) assert out is None, ( "Hooks registered with ``register_load_state_dict_post_hook`` are not" "expected to return new values, if incompatible_keys need to be modified," "it should be done inplace." ) load(self, state_dict) del load if strict: if len(unexpected_keys) > 0: error_msgs.insert( 0, 'Unexpected key(s) in state_dict: {}. '.format( ', '.join(f'"{k}"' for k in unexpected_keys))) if len(missing_keys) > 0: error_msgs.insert( 0, 'Missing key(s) in state_dict: {}. '.format( ', '.join(f'"{k}"' for k in missing_keys))) if len(error_msgs) > 0: raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( self.__class__.__name__, "\n\t".join(error_msgs))) return _IncompatibleKeys(missing_keys, unexpected_keys)_named_members# 通用的查找函数,可查找model的参数buffer本身等等 def _named_members(self, get_members_fn, prefix='', recurse=True, remove_duplicate: bool = True): r"""Helper method for yielding various names + members of modules.""" memo = set() # 有对named_modules的调用，该函数返回所有的模块 modules = self.named_modules(prefix=prefix, remove_duplicate=remove_duplicate) if recurse else [(prefix, self)] # 对所有的模块迭代 for module_prefix, module in modules: members = get_members_fn(module) for k, v in members: if v is None or v in memo: continue if remove_duplicate: memo.add(v) name = module_prefix + ('.' if module_prefix else '') + k yield name, vparameters def parameters(self, recurse: bool = True) -> Iterator[Parameter]: r"""Returns an iterator over module parameters. This is typically passed to an optimizer. Args: recurse (bool): if True, then yields parameters of this module and all submodules. Otherwise, yields only parameters that are direct members of this module. Yields: Parameter: module parameter Example:: >>> # xdoctest: +SKIP("undefined vars") >>> for param in model.parameters(): >>> print(type(param), param.size()) <class 'torch.Tensor'> (20L,) <class 'torch.Tensor'> (20L, 1L, 5L, 5L) """ for name, param in self.named_parameters(recurse=recurse): yield param_parameters只返回当前model的参数不包含子model, parameters()迭代返回所有for p in test_net.parameters(): print(p) ''' Parameter containing: tensor([[ 0.2101, -0.1331], [-0.4235, -0.1817], [ 0.4881, 0.2107]], dtype=torch.float64, requires_grad=True) Parameter containing: tensor([-0.3277, 0.2989, -0.3436], dtype=torch.float64, requires_grad=True) Parameter containing: tensor([[-0.3632, -0.0776, 0.2335], [-0.2908, -0.5474, -0.3428], [-0.1266, -0.5220, 0.2134], [ 0.1425, 0.1375, -0.3167]], dtype=torch.float64, requires_grad=True) Parameter containing: tensor([-0.4652, 0.5581, -0.0013, -0.1475], dtype=torch.float64, requires_grad=True) Parameter containing: tensor([1., 1., 1., 1.], dtype=torch.float64, requires_grad=True) Parameter containing: tensor([0., 0., 0., 0.], dtype=torch.float64, requires_grad=True) '''named_parametersdef named_parameters( self, prefix: str = '', recurse: bool = True, remove_duplicate: bool = True ) -> Iterator[Tuple[str, Parameter]]: r"""Returns an iterator over module parameters, yielding both the name of the parameter as well as the parameter itself. Args: prefix (str): prefix to prepend to all parameter names. recurse (bool): if True, then yields parameters of this module and all submodules. Otherwise, yields only parameters that are direct members of this module. remove_duplicate (bool, optional): whether to remove the duplicated parameters in the result. Defaults to True. Yields: (str, Parameter): Tuple containing the name and parameter Example:: >>> # xdoctest: +SKIP("undefined vars") >>> for name, param in self.named_parameters(): >>> if name in ['bias']: >>> print(param.size()) """ gen = self._named_members( lambda module: module._parameters.items(), prefix=prefix, recurse=recurse, remove_duplicate=remove_duplicate) yield from gen注：parameters()走的named_parameters，named_parameters走的_named_members，传入的是lambda module: module._parameters.items(),返回的是model自身的参数，_named_members又对传入的模块的所有子模块迭代，所以parameters()最终能得到所有模块的参数。下述buffer同理buffersdef buffers(self, recurse: bool = True) -> Iterator[Tensor]: r"""Returns an iterator over module buffers. Args: recurse (bool): if True, then yields buffers of this module and all submodules. Otherwise, yields only buffers that are direct members of this module. Yields: torch.Tensor: module buffer Example:: >>> # xdoctest: +SKIP("undefined vars") >>> for buf in model.buffers(): >>> print(type(buf), buf.size()) <class 'torch.Tensor'> (20L,) <class 'torch.Tensor'> (20L, 1L, 5L, 5L) """ for _, buf in self.named_buffers(recurse=recurse): yield buf def named_buffers(self, prefix: str = '', recurse: bool = True, remove_duplicate: bool = True) -> Iterator[Tuple[str, Tensor]]: r"""Returns an iterator over module buffers, yielding both the name of the buffer as well as the buffer itself. Args: prefix (str): prefix to prepend to all buffer names. recurse (bool, optional): if True, then yields buffers of this module and all submodules. Otherwise, yields only buffers that are direct members of this module. Defaults to True. remove_duplicate (bool, optional): whether to remove the duplicated buffers in the result. Defaults to True. Yields: (str, torch.Tensor): Tuple containing the name and buffer Example:: >>> # xdoctest: +SKIP("undefined vars") >>> for name, buf in self.named_buffers(): >>> if name in ['running_var']: >>> print(buf.size()) """ gen = self._named_members( lambda module: module._buffers.items(), prefix=prefix, recurse=recurse, remove_duplicate=remove_duplicate) yield from genchildrendef children(self) -> Iterator['Module']: r"""Returns an iterator over immediate children modules. Yields: Module: a child module """ for name, module in self.named_children(): yield module def named_children(self) -> Iterator[Tuple[str, 'Module']]: r"""Returns an iterator over immediate children modules, yielding both the name of the module as well as the module itself. Yields: (str, Module): Tuple containing a name and child module Example:: >>> # xdoctest: +SKIP("undefined vars") >>> for name, module in model.named_children(): >>> if name in ['conv4', 'conv5']: >>> print(module) """ memo = set() for name, module in self._modules.items(): if module is not None and module not in memo: memo.add(module) yield name, modulefor p in test_net.named_children(): print(p) ''' ('linear1', Linear(in_features=2, out_features=3, bias=True)) ('linear2', Linear(in_features=3, out_features=4, bias=True)) ('batch_norm', BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) ''' print(test_net._modules) ''' OrderedDict([('linear1', Linear(in_features=2, out_features=3, bias=True)), ('linear2', Linear(in_features=3, out_features=4, bias=True)), ('batch_norm', BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True))]) '''modulesdef modules(self) -> Iterator['Module']: r"""Returns an iterator over all modules in the network. Yields: Module: a module in the network Note: Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. Example:: >>> l = nn.Linear(2, 2) >>> net = nn.Sequential(l, l) >>> for idx, m in enumerate(net.modules()): ... print(idx, '->', m) 0 -> Sequential( (0): Linear(in_features=2, out_features=2, bias=True) (1): Linear(in_features=2, out_features=2, bias=True) ) 1 -> Linear(in_features=2, out_features=2, bias=True) """ for _, module in self.named_modules(): yield module def named_modules(self, memo: Optional[Set['Module']] = None, prefix: str = '', remove_duplicate: bool = True): r"""Returns an iterator over all modules in the network, yielding both the name of the module as well as the module itself. Args: memo: a memo to store the set of modules already added to the result prefix: a prefix that will be added to the name of the module remove_duplicate: whether to remove the duplicated module instances in the result or not Yields: (str, Module): Tuple of name and module Note: Duplicate modules are returned only once. In the following example, ``l`` will be returned only once. Example:: >>> l = nn.Linear(2, 2) >>> net = nn.Sequential(l, l) >>> for idx, m in enumerate(net.named_modules()): ... print(idx, '->', m) 0 -> ('', Sequential( (0): Linear(in_features=2, out_features=2, bias=True) (1): Linear(in_features=2, out_features=2, bias=True) )) 1 -> ('0', Linear(in_features=2, out_features=2, bias=True)) """ if memo is None: memo = set() if self not in memo: if remove_duplicate: memo.add(self) yield prefix, self # 首先返回自身, 再对子模块遍历 for name, module in self._modules.items(): if module is None: continue submodule_prefix = prefix + ('.' if prefix else '') + name yield from module.named_modules(memo, submodule_prefix, remove_duplicate)for p in test_net.named_modules(): print(p) # 返回+自身 print(test_net._modules) # 只是返回子模块 ''' ('', Net( (linear1): Linear(in_features=2, out_features=3, bias=True) (linear2): Linear(in_features=3, out_features=4, bias=True) (batch_norm): BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) )) ('linear1', Linear(in_features=2, out_features=3, bias=True)) ('linear2', Linear(in_features=3, out_features=4, bias=True)) ('batch_norm', BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)) OrderedDict([('linear1', Linear(in_features=2, out_features=3, bias=True)), ('linear2', Linear(in_features=3, out_features=4, bias=True)), ('batch_norm', BatchNorm2d(4, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True))]) '''
24、手写并验证向量内积实现PyTorch二维卷积
该9步过程可以看作是一个矩阵跟另外一个矩阵的矩阵相乘将每步input拉直，例如左上角第一个深蓝色区域拉为332001312行向量，再将Kernel拉为$(012220012)^T$列向量相乘得到第一个最终结果的左上角第一个数值12.0So，上边九步运算可以视为行数为9的矩阵和列数为9的矩阵进行矩阵乘法，再将相乘的结果reshape为欲得到的结果Other Method实现一个长度为25的内积目前Kernel只是3×3的范围大小，若是将Kernel填充一下，eg.左上角第一幅图，Kernel只有9个数，但是我们可以脑补一下浅蓝色部分都填充为0，每一步都将Kernel填充浅蓝色部分为0，那么，原问题就变为25行向量和25列向量相乘。Coding希望把region_vector都放入到region_matrix中，再将region_matrix与Kernel_matrix相乘input = torch.randn(5, 5) # 卷积输入特征图 kernel = torch.randn(3, 3) # 卷积核 bias = torch.randn(1) # 卷积偏置项，默认输出通道数目＝1 # Func2 input flatten 版本 def matrix_mutiplication_for_conv2d_flatten(input, kernel, bias=0, stride=1, padding=0): if padding > 0: input = F.pad(input, (padding, padding, padding, padding)) # 左右上下都pad0 input_h, input_w = input.shape kernel_h, kernel_w = kernel.shape output_w = (math.floor((input_w - kernel_w) / stride) + 1) # 卷积输出的宽度 output_h = (math.floor((input_h - kernel_h) / stride) + 1) # 卷积输出的高度 output = torch.zeros(output_h, output_w) # 初始化输出矩阵 # numel()返回tensor中元素数目 # 上图所示一共9行9列 region_matrix = torch.zeros(output.numel(), kernel.numel()) # 定义region_matrix 存储所有展平后的特征区域 kernel_matrix = kernel.reshape((kernel.numel(), 1)) # 定义kernel_matrix 存储kernel的列向量形式 roll_index = 0 for i in range(0, input_h - kernel_h + 1, stride): # 对高度维度进行遍历 for j in range(0, input_w - kernel_w + 1, stride): # 对宽度维度进行遍历 region = input[i:i+kernel_h, j:j+kernel_w] # 取出被核滑动到的区域 3*3=9 region_vector = torch.flatten(region) # 将该区域展平 3*3=>1*9 region_matrix[roll_index] = region_vector # 每次放入一行1*9，9次将region_matrix全部铺满 roll_index += 1 # 下次再下一行赋值 output_matrix = region_matrix @ kernel_matrix # 两个矩阵相乘，就是对9次的input滑动窗口中的3*3展平后的1*9行向量✖kernel的列向量相乘9次加起来 output = output_matrix.reshape((output_h, output_w)) + bias return output # 矩阵运算实现卷积的结果 flatten版本 mat_mul_conv_flatten_output = matrix_mutiplication_for_conv2d_flatten(input, kernel, bias=bias, padding=1) print(mat_mul_conv_flatten_output.shape, "\n", mat_mul_conv_flatten_output) print(pytorch_api_conv_output.squeeze(0).squeeze(0).shape, "\n", pytorch_api_conv_output.squeeze(0).squeeze(0)) ''' torch.Size([5, 5]) tensor([[-0.2212, -5.2747, -2.2503, -5.2114, -3.0707], [ 0.4797, -0.0270, 4.1195, 4.6948, 0.7474], [ 0.4497, 0.1129, -1.5966, -6.6838, -3.4131], [-1.9987, -1.4590, 1.0481, 1.0733, 2.2200], [ 0.4015, 2.9796, -4.8810, -3.2139, -2.6157]]) torch.Size([5, 5]) tensor([[-0.2212, -5.2747, -2.2503, -5.2114, -3.0707], [ 0.4797, -0.0270, 4.1195, 4.6948, 0.7474], [ 0.4497, 0.1129, -1.5966, -6.6838, -3.4131], [-1.9987, -1.4590, 1.0481, 1.0733, 2.2200], [ 0.4015, 2.9796, -4.8810, -3.2139, -2.6157]]) '''
23、手写并验证滑动相乘实现PyTorch二维卷积
蓝色的input_feature 5*5深蓝色小字部分kernel_size 3*3绿色部分out_feature 3*3stride = 1padding = 0channel = 1padding = 1stride = 2底部input_channels = 2顶端绿色为out_channels = 3kernels = 2*3 = 6（倒数第二行）input = input_feature_map # 卷积输入特征图 kernel = conv_layer.weight.data # 卷积核 input = torch.randn(5, 5) # 卷积输入特征图 kernel = torch.randn(3, 3) # 卷积核 bias = torch.randn(1) # 卷积偏置项，默认输出通道数目＝1 # Func1 用原始的矩阵运算来实现二维卷积,先不考虑batch_size和channel维度 def matrix_mutiplication_for_conv2d(input, kernel, bias=0, stride=1, padding=0): if padding > 0: input = F.pad(input, (padding, padding, padding, padding)) # 左右上下都pad0 input_h, input_w = input.shape kernel_h, kernel_w = kernel.shape output_w = (math.floor((input_w - kernel_w) / stride) + 1) # 卷积输出的宽度 output_h = (math.floor((input_h - kernel_h) / stride) + 1) # 卷积输出的高度 output = torch.zeros(output_h, output_w) # 初始化输出矩阵 for i in range(0, input_h - kernel_h + 1, stride): # 对高度维度进行遍历 for j in range(0, input_w - kernel_w + 1, stride): # 对宽度维度进行遍历 region = input[i:i+kernel_h, j:j+kernel_w] # 取出被核滑动到的区域 output[int(i / stride), int(j / stride)] = torch.sum(region * kernel) + bias # 点乘,并赋值给输出 return output # 矩阵运算实现卷积的结果 mat_mul_conv_output = matrix_mutiplication_for_conv2d(input, kernel, bias=bias, padding=1) print(mat_mul_conv_output.shape, "\n", mat_mul_conv_output) # 调用PyTorch API卷积的结果 pytorch_api_conv_output = F.conv2d(input.reshape((1,1,input.shape[0], input.shape[1])), \ kernel.reshape((1,1,kernel.shape[0], kernel.shape[1])),\ padding=1,\ bias=bias) print(pytorch_api_conv_output.squeeze(0).squeeze(0).shape, "\n", pytorch_api_conv_output.squeeze(0).squeeze(0)) # 验证成功，举证乘法实现的卷积欲pytorch api的结果一致(通道数为1) ''' torch.Size([5, 5]) tensor([[ 0.0254, 2.5571, 0.8080, -2.0241, 3.9600], [ 1.6969, 5.7820, -1.8596, 2.6106, 7.2310], [ 2.8070, 4.2363, -3.0085, 6.3041, 1.2186], [ 5.0747, -1.0790, 0.8183, 3.5965, -2.4651], [ 4.1119, -1.5446, 2.8745, 0.9343, 1.6732]]) torch.Size([5, 5]) tensor([[ 0.0254, 2.5571, 0.8080, -2.0241, 3.9600], [ 1.6969, 5.7820, -1.8596, 2.6106, 7.2310], [ 2.8070, 4.2363, -3.0085, 6.3041, 1.2186], [ 5.0747, -1.0790, 0.8183, 3.5965, -2.4651], [ 4.1119, -1.5446, 2.8745, 0.9343, 1.6732]]) '''# Func3 用原始的矩阵运算来实现二维卷积,考虑batch_size和channel维度 # bias与out_channel的形状一致 def matrix_mutiplication_for_conv2d_full(input, kernel, bias=0, padding=0, stride=2): # input kernel都是4维张量 if padding > 0: input = F.pad(input, (padding, padding, padding, padding, 0, 0, 0, 0)) # 左右上下都pad0,channel和batch_size维度不填充 bs, in_channel, input_h, input_w = input.shape out_channel, in_channel, kernel_h, kernel_w = kernel.shape if bias is None: bias = torch.zeros(out_channel) output_w = (math.floor((input_w - kernel_w) / stride) + 1) # 卷积输出的宽度 output_h = (math.floor((input_h - kernel_h) / stride) + 1) # 卷积输出的高度 output = torch.zeros(bs, out_channel, output_h, output_w) # 初始化输出矩阵 for index in range(bs): # 对batch_size维度考虑(样本层) for oc in range(out_channel): # 对每个输入通道维度合并，作为输出通道维度 eg.每个输出通道都由输入通道的求和得到(输出通道层) for ic in range(in_channel): # 对输入通道维遍历(输入通道层) **每次对输入通道的特征图卷积后加到输出通道** for i in range(0, input_h - kernel_h + 1, stride): # 对高度维度进行遍历(高度) for j in range(0, input_w - kernel_w + 1, stride): # 对宽度维度进行遍历(宽度) region = input[index, ic, i:i+kernel_h, j:j+kernel_w] # 取出被核滑动到的区域 output[index, oc, int(i / stride), int(j / stride)] += torch.sum(region * kernel[oc, ic]) # 点乘,并赋值给输出 output[index, oc] += bias[oc] # 在每个输出batch_size维度的输出通道上添加偏置项 return output input = torch.randn(2, 2, 5, 5) # bs*in_channel*in_h*in_w kernel = torch.randn(3, 2, 3, 3) # out_channel*in_channel*kernel_h*kernel_w bias = torch.randn(3) pytorch_conv2d = F.conv2d(input, kernel, bias=bias, padding=1, stride=2) mm_conv2d_full_output = matrix_mutiplication_for_conv2d_full(input, kernel, bias=bias, padding=1, stride=2) print(pytorch_conv2d.shape, "\n", pytorch_conv2d) print(mm_conv2d_full_output.shape, "\n", mm_conv2d_full_output) print(torch.allclose(pytorch_conv2d, mm_conv2d_full_output)) ''' torch.Size([2, 3, 3, 3]) tensor([[[[ 1.1374, -0.0861, 0.4275], [ 3.6559, 2.5741, 1.3991], [ 0.9067, 9.3524, 1.1320]], [[ 0.8325, -1.5813, -2.5399], [ 0.8679, -8.0568, -4.0715], [-0.2598, -3.6637, -8.9900]], [[-4.8694, -3.0817, -1.4585], [-7.6027, 1.3716, -0.4906], [-1.1015, -7.1992, 0.9741]]], [[[-2.9367, -5.3354, -2.1468], [ 0.8508, 4.3884, -3.4202], [ 0.2540, -2.2766, 1.8590]], [[ 1.4410, 1.1040, -0.2459], [-5.6472, -0.8350, -3.9254], [ 0.5086, -3.6675, 1.0906]], [[-5.2750, -7.2560, -0.5371], [-3.2578, 0.9768, 4.3391], [-3.8552, 0.6670, -2.7320]]]]) torch.Size([2, 3, 3, 3]) tensor([[[[ 1.1374, -0.0861, 0.4275], [ 3.6559, 2.5741, 1.3991], [ 0.9067, 9.3524, 1.1320]], [[ 0.8325, -1.5813, -2.5399], [ 0.8679, -8.0568, -4.0715], [-0.2598, -3.6637, -8.9900]], [[-4.8694, -3.0817, -1.4585], [-7.6027, 1.3716, -0.4906], [-1.1015, -7.1992, 0.9741]]], [[[-2.9367, -5.3354, -2.1468], [ 0.8508, 4.3884, -3.4202], [ 0.2540, -2.2766, 1.8590]], [[ 1.4410, 1.1040, -0.2459], [-5.6472, -0.8350, -3.9254], [ 0.5086, -3.6675, 1.0906]], [[-5.2750, -7.2560, -0.5371], [-3.2578, 0.9768, 4.3391], [-3.8552, 0.6670, -2.7320]]]]) True '''