机器学习-自适应学习率和损失函数的反向传播神经网络BPNN

末蓝、 2023-01-15 14:21 146阅读 0赞

**问题背景：**多分类问题，输入参数为8组特征值，采用Excel表格导入，输出参数为5种不同的分类结果。数据量为1000组左右，训练组与测试组保持为7:3。

BP神经网络模型：隐藏层计算为10层，达到最大准确率，且时间较短。

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTY0NTI4MQ_size_16_color_FFFFFF_t_70][]

学习率更新公式：

![20210422205116792.png][]

损失函数更新公式：

![20210422205135697.png][]

有几个超参数需要提前设置好，根据自己的问题背景环境。

import numpy as np
    import datetime
    import xlrd
    import matplotlib.pyplot as plt
    from sklearn.metrics import roc_curve
    
    def noramlization(data):
        minVals = data.min(0)
        maxVals = data.max(0)
        ranges = maxVals - minVals
        normData = (data - minVals)/ranges
        return normData
    
    # 0.读入数据
    def load_excel(path):
        result_array=[]
        # 读取初始数据
        data = xlrd.open_workbook(path)
        table = data.sheet_by_index(0)
        # table.nrows表示总行数
        for i in range(table.nrows):
            # 读取每行数据，保存在line里面，line是list
            line = table.row_values(i)
            # 将line加入到result_array中，result_array是二维list
            result_array.append(line)
        # 将result_array从二维list变成数组
        result_array = np.array(result_array)
        return result_array
    
    
    # 1.初始化参数
    def initialize_parameters(n_x, n_h, n_y):
        np.random.seed(2)
    
        # 权重和偏置矩阵
        w1 = np.random.rand(n_h, n_x) * 0.1
        #b1 = np.random.rand(n_h, 1)
        b1 = np.zeros(shape=(n_h, 1))
        w2 = np.random.rand(n_y, n_h) * 0.1
        #b2 = np.random.rand(n_y, 1)
        b2 = np.zeros(shape=(n_y, 1))
    
        # 通过字典存储参数
        parameters = {'w1': w1, 'b1': b1, 'w2': w2, 'b2': b2}
        return parameters
    
    #  更新权重和阈值
    def update_temp(parameters):
        w1 = parameters['w1']
        b1 = parameters['b1']
        w2 = parameters['w2']
        b2 = parameters['b2']
    
        temp = {'w1': w1, 'b1': b1, 'w2': w2, 'b2': b2}
    
        return temp
    
    def update_wb(parameters,temp):
        w1 = parameters['w1']
        b1 = parameters['b1']
        w2 = parameters['w2']
        b2 = parameters['b2']
        temp_w1 = temp['w1']
        temp_b1 = temp['b1']
        temp_w2 = temp['w2']
        temp_b2 = temp['b2']
        m1 = w1- temp_w1
        n1 = b1- temp_b1
        m2 = w2 - temp_w2
        n2 = b2 - temp_b2
        update_para = {'m1': m1, 'n1': n1, 'm2': m2, 'n2': n2}
    
        return update_para
    
    # 2.前向传播
    def forward_propagation(X, parameters):
        w1 = parameters['w1']
        b1 = parameters['b1']
        w2 = parameters['w2']
        b2 = parameters['b2']
    
        # 通过前向传播来计算a2
        z1 = np.dot(w1, X) + b1     # 这个地方需注意矩阵加法：虽然(w1*X)和b1的维度不同，但可以相加
        a1 = np.tanh(z1)           # 使用tanh作为第一层的激活函数
        z2 = np.dot(w2, a1) + b2
    
        # exp = np.exp(z2)
        # sum_exp = np.sum(np.exp(z2), axis=0, keepdims=True)
        # a2 = exp / sum_exp
    
        a2 = 1 / (1 + np.exp(-z2))  # 使用sigmoid作为第二层的激活函数
        # 通过字典存储参数
        cache = {'z1': z1, 'a1': a1, 'z2': z2, 'a2': a2}
    
        return a2, cache
    
    
    # 3.计算代价函数
    def compute_cost(a2, Y, parameters):
        m = Y.shape[1]      # Y的列数即为总的样本数
        # 采用交叉熵（cross-entropy）作为代价函数
        logprobs = np.multiply(np.log(a2), Y) + np.multiply((1 - Y), np.log(1 - a2))
        #logprobs = np.multiply(np.log(a2), Y)
        cost = - np.sum(logprobs) / m
        return cost
    
    
    # 4.反向传播（计算代价函数的导数）
    def backward_propagation(parameters, cache, X, Y):
        m = Y.shape[1]
    
        w2 = parameters['w2']
    
        a1 = cache['a1']
        a2 = cache['a2']
    
        # 反向传播，计算dw1、db1、dw2、db2
        dz2 = a2 - Y
        dw2 = (1 / m) * np.dot(dz2, a1.T)
        #print(dw2.shape)
        db2 = (1 / m) * np.sum(dz2, axis=1, keepdims=True)
        dz1 = np.multiply(np.dot(w2.T, dz2), 1 - np.power(a1, 2))
        dw1 = (1 / m) * np.dot(dz1, X.T)
        db1 = (1 / m) * np.sum(dz1, axis=1, keepdims=True)
        grads = {'dw1': dw1, 'db1': db1, 'dw2': dw2, 'db2': db2}
    
        return grads
    
    def update_learn(learning_rate,i,num_iterations):
        learning=learning_rate*(1-i/num_iterations)
        return learning
    
    
    # 5.更新参数
    def update_parameters(parameters, grads,update_para,learning_rate,k):
        w1 = parameters['w1']
        b1 = parameters['b1']
        w2 = parameters['w2']
        b2 = parameters['b2']
    
        dw1 = grads['dw1']
        db1 = grads['db1']
        dw2 = grads['dw2']
        db2 = grads['db2']
    
        m1 = update_para['m1']
        n1 = update_para['n1']
        m2 = update_para['m2']
        n2 = update_para['n2']
    
        # 更新参数
        w1 = w1 - dw1 * learning_rate + m1 * k*0.9
        b1 = b1 - db1 * learning_rate + n1 * k*0.9
        w2 = w2 - dw2 * learning_rate + m2 * k*0.9
        b2 = b2 - db2 * learning_rate + n2 * k*0.9
    
        parameters = {'w1': w1, 'b1': b1, 'w2': w2, 'b2': b2}
    
        return parameters
    
    
    # 建立神经网络
    def nn_model(X, Y, n_h, n_input, n_output, num_iterations=10000, print_cost=False):
        np.random.seed(3)
    
        n_x = n_input           # 输入层节点数
        n_y = n_output          # 输出层节点数
    
        # 1.初始化参数
        learning_rate=0.1
        parameters = initialize_parameters(n_x, n_h, n_y)
        temp = update_temp(parameters)
        cost_list=[]
        grads_list=[]
        j = 1
        # 梯度下降循环
        for i in range(0, num_iterations):
            learning=update_learn(learning_rate,i,num_iterations)
            # 2.前向传播
            a2, cache = forward_propagation(X, parameters)
            # 3.计算代价函数
            cost = compute_cost(a2, Y, parameters)
            cost_list.append(cost)
            # 4.反向传播
            grads = backward_propagation(parameters, cache, X, Y)
            # 5.更新参数
            update_para = update_wb(parameters, temp)
            k=1
            if j==i:
                temp=update_temp(parameters)
                j=j+1
                k=cost/ttt
            ttt = cost
            parameters = update_parameters(parameters, grads, update_para,learning_rate=learning,k=k)
            grads_list.append(k)
            # 每1000次迭代，输出一次代价函数
            if print_cost and i % 1000 == 0:
                print('迭代第%i次，代价函数为：%f' % (i, cost))
        plt.plot(cost_list)
        #plt.plot(grads_list)
        return parameters,cost_list
    
    
    # 对模型进行测试
    def predict(parameters, x_test, y_test):
        w1 = parameters['w1']
        b1 = parameters['b1']
        w2 = parameters['w2']
        b2 = parameters['b2']
    
        z1 = np.dot(w1, x_test) + b1
        a1 = np.tanh(z1)
    
        z2 = np.dot(w2, a1) + b2
        # exp = np.exp(z2)
        # sum_exp = np.sum(np.exp(z2), axis=0, keepdims=True)
        # a2 = exp / sum_exp
        a2 = 1 / (1 + np.exp(-z2))
    
        # 结果的维度
        n_rows = y_test.shape[0]
        n_cols = y_test.shape[1]
    
        # 预测值结果存储
        output = np.empty(shape=(n_rows, n_cols), dtype=int)
    
        # 取出每条测试数据的预测结果
        for i in range(n_cols):
            # 将每条测试数据的预测结果（概率）存为一个行向量
            temp = np.zeros(shape=n_rows)
            for j in range(n_rows):
                temp[j] = a2[j][i]
    
            # 将每条结果（概率）从小到大排序，并获得相应下标
            sorted_dist = np.argsort(temp)
            length = len(sorted_dist)
    
            # 将概率最大的置为1，其它置为0
            for k in range(length):
                if k == sorted_dist[length - 1]:
                    output[k][i] = 1
                else:
                    output[k][i] = 0
    
        print('预测结果：')
        print(output)
        print('真实结果：')
        print(y_test)
    
        count = 0
        for k in range(0, n_cols):
            if output[0][k] == y_test[0][k] and output[1][k] == y_test[1][k] and output[2][k] == y_test[2][k]\
                    and output[3][k] == y_test[3][k] and output[4][k] == y_test[4][k]:
                count = count + 1
    
        acc = count / int(y_test.shape[1]) * 100
        print('准确率：%.2f%%' % acc)
        return output,acc
    
    
    def cross_validation():
        pass
    
    
    if __name__ == "__main__":
        # 读取数据
        data_normal = load_excel('E:/exp/1.xlsx')
        data_sunken = load_excel('E:/exp/2.xlsx')
        data_undercut = load_excel('E:/exp/3.xlsx')
        data_pore = load_excel('E:/exp/4.xlsx')
        data_burnthrough = load_excel('E:/exp/5.xlsx')
        #test_weld = load_excel('E:/exp/test.xlsx')
        # 生成数据集
        y11 = np.array([1,0,0,0,0])
        y1 = np.tile(y11,(data_normal.shape[0],1))
        #y11 = np.zeros((data_normal.shape[0], 1))
        c1 = np.concatenate((data_normal, y1), axis=1) 
    
        y22 = np.array([0,1,0,0,0])
        y2 = np.tile(y22,(data_sunken.shape[0],1))
        #y2 = np.ones((data_sunken.shape[0], 1)) * 1
        c2 = np.concatenate((data_sunken, y2), axis=1)  
    
        y33 = np.array([0,0,1,0,0])
        y3 = np.tile(y33,(data_undercut.shape[0],1))
        #y3 = np.ones((data_undercut.shape[0], 1)) * 2
        c3 = np.concatenate((data_undercut, y3), axis=1) 
        y44 = np.array([0,0,0,1,0])
        y4 = np.tile(y44,(data_pore.shape[0],1))
        #y4 = np.ones((data_pore.shape[0], 1)) * 3
        c4 = np.concatenate((data_pore, y4), axis=1)  
    
        y55 = np.array([0,0,0,0,1])
        y5 = np.tile(y55,(data_burnthrough.shape[0],1))
        #y5 = np.ones((data_burnthrough.shape[0], 1)) * 4
        c5 = np.concatenate((data_burnthrough, y5), axis=1)  
    #######load samples##
        # np.random.shuffle(c1)
        # np.random.shuffle(c2)
        # np.random.shuffle(c3)
        # np.random.shuffle(c4)
        # np.random.shuffle(c5)
        # yuan = np.concatenate((c1[:75,:], c2[:75,:], c3[:75,:], c4[:75,:], c5[:75,:]), axis=0)
    #########
        #all_data=load_excel('E:/temp.xlsx')
        all_data = np.concatenate((c1, c2, c3, c4, c5), axis=0)
        np.random.shuffle(all_data)
        # train_data_x1 = all_data[:1089, :8].T
        # train_data_y1 = all_data[:1089, 8:13].T
        # train_data_x2 = all_data[1218:, :8].T
        # train_data_y2 = all_data[1218:, 8:13].T
        # train_data_x=np.concatenate((train_data_x1,train_data_x2),axis=1)
        # train_data_y = np.concatenate((train_data_y1, train_data_y2), axis=1)
        train_data_x = all_data[:812, :8].T
        train_data_y = all_data[:812, 8:13].T  # 训练集
        # #test_data_x=test_weld[:,:8].T
        # #test_data_y=test_weld[:,8:13].T
        test_data_x = all_data[812:, :8].T
        test_data_y = all_data[812:, 8:13].T  # 测试集
        # test_data_x = all_data[1089:1218, :8].T
        # test_data_y = all_data[1089:1218, 8:13].T
        train_data_y = train_data_y.astype('uint8')
        train_data_x=noramlization(train_data_x)
        test_data_x=noramlization(test_data_x)
    
        # 开始训练
        start_time = datetime.datetime.now()
        # 输入8个节点，隐层10个节点，输出5个节点，迭代10000次
        parameters,result= nn_model(train_data_x, train_data_y, n_h=13, n_input=8, n_output=5, num_iterations=10000, print_cost=True)
        end_time = datetime.datetime.now()
        print("用时：" + str((end_time - start_time).seconds) + 's' + str(round((end_time - start_time).microseconds / 1000)) + 'ms')
    
        # 对模型进行测试
        test_data_y = test_data_y.astype('uint8')
        #o,t=predict(parameters, train_data_x, train_data_y)
        p,acc=predict(parameters, test_data_x,test_data_y)
        #np.savetxt('E:/exp/weld detect/result_test.txt',yuan)
        #np.savetxt('E:/exp/weld detect/result_predict.txt', p.T)

损失函数：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTY0NTI4MQ_size_16_color_FFFFFF_t_70 1][]

准确率：

![20210422205859708.png][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTY0NTI4MQ_size_16_color_FFFFFF_t_70]: /images/20221022/924ea5bbb76b4c77a00677c161970afc.png
[20210422205116792.png]: /images/20221022/f13875094e62416fab61c6b862a48e20.png
[20210422205135697.png]: /images/20221022/b7c059962f2a4ff68936d94bf1612cc5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTY0NTI4MQ_size_16_color_FFFFFF_t_70 1]: /images/20221022/caedaa3c9ada4fa382036fd40cfb87a7.png
[20210422205859708.png]: /images/20221022/7dbe6c14ad154483acc145675b35514e.png