激活函数Relu对精度和损失的影响研究

￡神魔★判官ぃ 2024-03-22 16:50 61阅读 0赞

**1** **问题**

在学习深度学习的过程中，欲探究激活函数Relu对精度和损失的影响。

**2** **方法**

测试设置激活函数时和没有设置激活函数时网络的性能。

控制其余变量：

*  Beach\_size=128
 *  optimizer = torch.optim.SGD
 *  网络为三层全连接网络（784->512->10）
 *  训练周期=50

测试代码如下

<table> 
 <tbody> 
 <tr> 
 <td>from torchvision import datasets from torchvision.transforms import ToTensor from torch import nn import torch from torch.utils.data import DataLoader import torch.nn.functional as F import matplotlib.pyplot as plt from collections import defaultdict # (5)定义三层全连接网络 # (5.1)创建一个新的类继承nn.Module class MyNet(nn.Module): &nbsp; &nbsp;# (5.2) 定义网络有哪些层，这些层都作为成员变量 &nbsp; &nbsp;def __init__(self) -&gt; None: &nbsp; &nbsp; &nbsp; &nbsp;super().__init__() &nbsp; &nbsp; &nbsp; &nbsp;# 第一层,第一个全连接层Full Connection(FC) &nbsp; &nbsp; &nbsp; &nbsp;# in_features表示该层的前面一层神经元个数 &nbsp; &nbsp; &nbsp; &nbsp;# out_features表示当前这的一层神经元个数 &nbsp; &nbsp; &nbsp; &nbsp;# 对应图里面layer2 &nbsp; &nbsp; &nbsp; &nbsp;self.fc1 = nn.Linear(in_features=784, out_features=512) &nbsp;# in_features就是这一层的输入是多少纬的向量， &nbsp; &nbsp; &nbsp; &nbsp;# 对应layer3 也是就输出层 &nbsp; &nbsp; &nbsp; &nbsp;self.fc2 = nn.Linear(in_features=512, out_features=10) &nbsp;# feature特征 &nbsp; &nbsp;# (5.3) 定义数据在网络中的流动 x就表示输入 &nbsp; &nbsp;# x - 28*28的图像 &nbsp; &nbsp;def forward(self, x): &nbsp; &nbsp; &nbsp; &nbsp;x=torch.flatten(x,1) &nbsp; &nbsp; &nbsp; &nbsp;x = self.fc1(x) &nbsp;# 输出:512,Layer 2 &nbsp; &nbsp; &nbsp; &nbsp;# x=F.relu(x) &nbsp; &nbsp; &nbsp; &nbsp;out = self.fc2(x) &nbsp;# 输出:10,Layer 3 &nbsp; &nbsp; &nbsp; &nbsp;return out #(7)训练网络 #loss_list:记录每一个周期的平均loss数据 def train(dataloader,net,loss_fn,optimizer): &nbsp; &nbsp;size = len((dataloader.dataset)) &nbsp; &nbsp;epoch_loss=0.0 &nbsp; &nbsp;batch_num=len(dataloader)#有多少个batch &nbsp; &nbsp;net.train() &nbsp; &nbsp;correct = 0 #准确率 &nbsp; &nbsp;#一个batch一个batch的训练网络 &nbsp; &nbsp;for batch_ind, (X,y) in enumerate(dataloader): &nbsp; &nbsp; &nbsp; &nbsp;X,y=X.to(device),y.to(device) #gpu上运行X,y &nbsp; &nbsp; &nbsp; &nbsp;pred = net(X) &nbsp; &nbsp; &nbsp; &nbsp;#衡量y与y_hat之前的loss &nbsp; &nbsp; &nbsp; &nbsp;#y:128 ,pred:128*10 CrossEntropyLoss处理的 &nbsp; &nbsp; &nbsp; &nbsp;loss=loss_fn(pred,y) &nbsp; &nbsp;#基于loss信息利用优化器从后向前更新网络的全部参数 &nbsp; &nbsp; &nbsp; &nbsp;optimizer.zero_grad() &nbsp; &nbsp; &nbsp; &nbsp;loss.backward() &nbsp; &nbsp; &nbsp; &nbsp;optimizer.step() &nbsp; &nbsp; &nbsp; &nbsp;# print(f"batch index: {ind},loss:{loss.item()}")#item()方法将单个tensor转化成一个数字 &nbsp; &nbsp; &nbsp; &nbsp;epoch_loss+=loss.item() #每一个batch产生一个loss &nbsp; &nbsp; &nbsp; &nbsp;correct+=(pred.argmax(1)==y).type(torch.float).sum().item() &nbsp; &nbsp; &nbsp; &nbsp;#f-str &nbsp; &nbsp; &nbsp; &nbsp;if batch_ind % 100==0: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;print(f'[{batch_ind+1 :&gt;5d} / {batch_num :&gt;5d}], loss:{loss.item()}') &nbsp; &nbsp;#统计一个周期的平均loss &nbsp; &nbsp;avg_loss=epoch_loss/batch_num &nbsp; &nbsp;avg_accurcy = correct / size &nbsp; &nbsp;return &nbsp;avg_accurcy,avg_loss def test(dataloder,net,loss): &nbsp; &nbsp;size=len(dataloder.dataset) &nbsp; &nbsp;batch_num = len(dataloder) &nbsp; &nbsp;net.eval() &nbsp; &nbsp;losses=0 &nbsp; &nbsp;correct = 0 &nbsp; &nbsp;with torch.no_grad(): &nbsp; &nbsp; &nbsp; &nbsp;for X,y in dataloder: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;X,y = X.to(device),y.to(device) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;pred = net(X) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;loss = loss_fn(pred,y) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;losses+=loss.item() #一个batch产生一个loss &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;correct+=(pred.argmax(1) ==y).type(torch.int).sum().item() &nbsp; &nbsp;accuracy = correct / size &nbsp; &nbsp;avg_loss = losses /batch_num &nbsp; &nbsp;print(f'accuracy is {accuracy*100}%') &nbsp; &nbsp;return accuracy,avg_loss #数据的获取 class getData: &nbsp; &nbsp;def __init__(self): &nbsp; &nbsp; &nbsp; &nbsp;self.train_ds = datasets.MNIST( &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;root='data', &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;download=True, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;train=True, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;transform=ToTensor(), &nbsp;# 将原始的数据格式（PIL）转换为Tensor格式 &nbsp; &nbsp; &nbsp; &nbsp;) &nbsp; &nbsp; &nbsp; &nbsp;# 测试集:评估模型的性能/效果 &nbsp; &nbsp; &nbsp; &nbsp;self.text_ds = datasets.MNIST( &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;root='data', &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;download=True, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;train=False, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;transform=ToTensor(), &nbsp; &nbsp; &nbsp; &nbsp;) &nbsp; &nbsp; &nbsp; &nbsp;# print(train_ds[0]) &nbsp; &nbsp; &nbsp; &nbsp;# print(text_ds[0]) &nbsp; &nbsp; &nbsp; &nbsp;# (3) &nbsp; &nbsp; &nbsp; &nbsp;self.train_loader = DataLoader( &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;dataset=self.train_ds, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;batch_size=128, &nbsp;# 将60000个数据分成每一段的大小是128 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;shuffle=True, &nbsp;# 每一次我去拿那个128的数据都是打乱的，不是有序的。1-60000 打乱数据的次序，一般用于训练集，不需要在测试集 &nbsp; &nbsp; &nbsp; &nbsp;) &nbsp; &nbsp; &nbsp; &nbsp;# (4) &nbsp; &nbsp; &nbsp; &nbsp;# 训练集共有469个batch &nbsp;469*128 &nbsp; &nbsp; &nbsp; &nbsp;# print("训练集的batch{}".format(len(train_loader))) &nbsp; &nbsp; &nbsp; &nbsp;self.test_loader = DataLoader( &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;dataset=self.text_ds, &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;batch_size=128, &nbsp; &nbsp; &nbsp; &nbsp;) if __name__=='__main__': &nbsp; &nbsp;#（0）测试机器是否支持GPU &nbsp; &nbsp;batch_size=128 &nbsp; &nbsp;device='cuda' if torch.cuda.is_available() else 'cpu' &nbsp; &nbsp;# print(device) &nbsp; &nbsp;train_ds = datasets.MNIST( &nbsp; &nbsp; &nbsp; &nbsp;root='data', &nbsp; &nbsp; &nbsp; &nbsp;download=True, &nbsp; &nbsp; &nbsp; &nbsp;train=True, &nbsp; &nbsp; &nbsp; &nbsp;transform=ToTensor(), &nbsp;# 将原始的数据格式（PIL）转换为Tensor格式 &nbsp; &nbsp;) &nbsp; &nbsp;# 测试集:评估模型的性能/效果 &nbsp; &nbsp;text_ds = datasets.MNIST( &nbsp; &nbsp; &nbsp; &nbsp;root='data', &nbsp; &nbsp; &nbsp; &nbsp;download=True, &nbsp; &nbsp; &nbsp; &nbsp;train=False, &nbsp; &nbsp; &nbsp; &nbsp;transform=ToTensor(), &nbsp; &nbsp;) &nbsp; #（1.1）将训练集划分为训练集+验证集 &nbsp; &nbsp;train_ds,val_ds= torch.utils.data.random_split(train_ds,[50000,10000]) &nbsp; &nbsp;# (3) &nbsp; &nbsp;train_loader = DataLoader( &nbsp; &nbsp; &nbsp; &nbsp;dataset=train_ds, &nbsp; &nbsp; &nbsp; &nbsp;batch_size=batch_size, &nbsp;# 将60000个数据分成每一段的大小是128 &nbsp; &nbsp; &nbsp; &nbsp;shuffle=True, &nbsp;# 每一次我去拿那个128的数据都是打乱的，不是有序的。1-60000 打乱数据的次序，一般用于训练集，不需要在测试集 &nbsp; &nbsp;) &nbsp; &nbsp;val_loder = DataLoader( &nbsp; &nbsp; &nbsp; &nbsp;dataset=val_ds, &nbsp; &nbsp; &nbsp; &nbsp;batch_size=batch_size, &nbsp; &nbsp;) &nbsp; &nbsp;# (4) &nbsp; &nbsp;# 训练集共有469个batch &nbsp;469*128 &nbsp; &nbsp;# print("训练集的batch{}".format(len(train_loader))) &nbsp; &nbsp;test_loader = DataLoader( &nbsp; &nbsp; &nbsp; &nbsp;dataset=text_ds, &nbsp; &nbsp; &nbsp; &nbsp;batch_size=batch_size, &nbsp; &nbsp;) &nbsp; &nbsp;#(6)网络的输入、输出以及测试网络的性能（不经过任何训练的网络） &nbsp; &nbsp;net=MyNet().to(device)#to()GPU上运行该网络 &nbsp; &nbsp;#网络训练模型 &nbsp; &nbsp;#X, 真实的标签y, 网络预测的标签y_hat &nbsp; &nbsp;#目标: y_hat越来越接近y &nbsp; &nbsp;#算法：mini-bacth 梯度下降 &nbsp; &nbsp;#优化器 &nbsp; &nbsp;#具体实现梯度下降算法的传播 &nbsp; &nbsp;#SGD随机梯度下降学习度 &nbsp; &nbsp;#y=ax+b &nbsp; &nbsp;optimizer=torch.optim.SGD(net.parameters(),lr=0.15) &nbsp; &nbsp;#损失函数 &nbsp; &nbsp;#衡量yy与y_hat之前的差异 &nbsp; &nbsp;loss_fn=nn.CrossEntropyLoss() &nbsp; &nbsp;#训练一下 &nbsp; &nbsp;# train(train_loader,net,loss_fn,optimizer) &nbsp; &nbsp;#一个周期表示一个完整的训练集 &nbsp; &nbsp;#训练100个周期epoch &nbsp; &nbsp;train_loss=[] #记录所有周期的平均loss &nbsp; &nbsp;train_acc_list,train_loss_list,val_acc_list,val_loss_list=[],[],[],[] &nbsp; &nbsp;#找出周期内最好的模型 &nbsp; &nbsp;#评价标准：验证集的精度 &nbsp; &nbsp;best_acc=0 &nbsp; &nbsp;for epoch in range(50): &nbsp; &nbsp; &nbsp; &nbsp;print('-'*50) &nbsp; &nbsp; &nbsp; &nbsp;print(f'eopch:{epoch+1}') &nbsp; &nbsp; &nbsp; &nbsp;train_accuracy,train_loss=train(train_loader,net,loss_fn,optimizer) &nbsp; &nbsp; &nbsp; &nbsp;val_accuracy,val_loss =test(train_loader,net,loss_fn) &nbsp; &nbsp; &nbsp; &nbsp;print(f'train acc:{train_accuracy},train_val:{train_loss}') &nbsp; &nbsp; &nbsp; &nbsp;train_acc_list.append(train_accuracy) &nbsp; &nbsp; &nbsp; &nbsp;train_loss_list.append(train_loss) &nbsp; &nbsp; &nbsp; &nbsp;val_acc_list.append(val_accuracy) &nbsp; &nbsp; &nbsp; &nbsp;val_loss_list.append(val_loss) &nbsp; &nbsp; &nbsp; &nbsp;if val_accuracy &gt; best_acc: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;best_acc = val_accuracy &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;#保存当前模型 &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;torch.save(net.state_dict(),'model_best.pth') &nbsp; &nbsp;#（7）评估模型` &nbsp; &nbsp;#加载最好的模型 &nbsp; &nbsp;net.load_state_dict(torch.load('model_best.pth')) &nbsp; &nbsp;print('the best val_acc is:') &nbsp; &nbsp;test(test_loader,net,loss_fn) &nbsp; &nbsp;plt.figure() &nbsp; &nbsp;ax1=plt.subplot(121) &nbsp; &nbsp;plt.plot([i for i in range(len(train_acc_list))],train_acc_list,ls='-',c='b') &nbsp; &nbsp;plt.title('accuracy') &nbsp; &nbsp;plt.xlabel('epoch') &nbsp; &nbsp;plt.ylabel('number') &nbsp; &nbsp;ax = plt.subplot(122) &nbsp; &nbsp;plt.plot([i for i in range(len(train_loss_list))], train_loss_list, ls='-', c='b') &nbsp; &nbsp;plt.title('loss') &nbsp; &nbsp;plt.xlabel('epoch') &nbsp; &nbsp;plt.ylabel('number') &nbsp; &nbsp;plt.show()</td> 
 </tr> 
 </tbody> 
</table>

最后无激活函数时结果如图所示：

![bdc16d6aac994d9e1dea463d1e7f3514.png][]

有激活函数时结果如图所示：

![1db0c6cde0262c88c69a7ee701ef3ddd.png][]

**3** **结语**

通过实验发现，在未使用激活函数时，通过不断地训练模型，模型的准确率和损失率都时比较稳定地上升和下降，但是在上升和下降地过程中会出现抖动地情况，但是使用激活函数之后，模型的准确率和损失率就会上升和下降的非常平滑，更有利于实验的进行，以及对模型行为的预测。

[bdc16d6aac994d9e1dea463d1e7f3514.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/22/64b0d66b2fc542eba476b7f540e90ea5.png
[1db0c6cde0262c88c69a7ee701ef3ddd.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/03/22/6924908fe9ea4b568facd5bd559a51d6.png