【吴恩达深度学习编程作业】2.1改善深层神经网络——初始化、正则化、梯度校验

怼烎@ 2022-12-17 02:25 206阅读 0赞

参考文章：[改善深层神经网络-初始化、正则化、梯度校验][-]

至今为止，数据集的加载、决策边界的曲线绘制代码不熟悉，向量与字典的相互转化的代码没细看。

代码实现功能如下：

1.  初始化参数：  
    1.1：使用0来初始化参数  
    1.2：使用随机数来初始化参数  
    1.3：使用抑梯度异常初始化参数（参见视频中的梯度消失和梯度爆炸）
2.  正则化模型：  
    2.1：使用二范数对二分类模型正则化，尝试避免过拟合。  
    2.2：使用随机删除节点的方法精简模型，同样是为了尝试避免过拟合。
3.  梯度校验 ：  
    对模型使用梯度校验，检测它是否在梯度下降的过程中出现误差过大的情况。

## 1.初始化 ##

### innitialize.py ###

""" 初始化参数： 1：使用0来初始化参数 2：使用随机数来初始化参数 3：使用抑梯度异常初始化参数（参见视频中的梯度消失和梯度爆炸） """
    import numpy as np
    import matplotlib.pyplot as plt
    import Deep_Learning.test2_1.init_utils
    
    # 绘图初始设置
    plt.rcParams['figure.figsize'] = (7.0, 4.0)     # 设置图片尺寸为600*400像素
    plt.rcParams['image.interpolation'] = 'nearest' # 设置插值方式：最邻近插值
    plt.rcParams['image.cmap'] = 'gray'             # 使用灰度输出而不是彩色输出
    
    # 读取数据集
    train_X, train_Y, test_X, test_Y = Deep_Learning.test2_1.init_utils.load_dataset(is_plot=True)
    plt.show()  # 数据呈现蓝点和红点，我们要做的就是将二者分开
    
    
    # 初始化为0
    def initialize_parameters_zeros(layers_dims):
        """ 将模型的参数全部设置为0 :param layers_dims: -列表，模型的层数和对应每一层的节点的数量 :return: parameters -包含了所有W和b的字典(l属于[0,L]) Wl -权重矩阵，维度为(layers_dims[l], layers_dims[0]) bl -偏置向量，维度为(layers_dims[l], 1) """
        parameters = { }
        L = len(layers_dims)    # 神经网络层数
    
        for l in range(1, L):
            parameters["W" + str(l)] = np.zeros((layers_dims[l], layers_dims[l - 1]))
            parameters["b" + str(l)] = np.zeros((layers_dims[l], 1))
    
            # 使用断言确保格式的正确性
            assert (parameters["W" + str(l)].shape) == ((layers_dims[l], layers_dims[l - 1]))
            assert (parameters["b" + str(l)].shape) == ((layers_dims[l], 1))
    
        return parameters
    
    # 测试initialize_parameters_zeros
    print("====================测试initialize_parameters_zeros====================")
    parameters = initialize_parameters_zeros([3, 2, 1])
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    
    
    # 随机初始化：可以把参数随机赋值，每个神经元可以学习其输入的不同功能
    def initialize_parameters_random(layers_dims):
        np.random.seed(3)   # 指定随机种子
        parameters = { }
        L = len(layers_dims)    # 层数
        for l in range(1, L):
            parameters['W' + str(l)] = np.random.rand(layers_dims[l], layers_dims[l - 1]) * 10  # 使用10倍缩放
            parameters['b' + str(l)] = np.zeros((layers_dims[l], 1))
            assert (parameters["W" + str(l)].shape) == ((layers_dims[l],layers_dims[l - 1]))
            assert (parameters["b" + str(l)].shape) == ((layers_dims[l],1))
        return parameters
    
    # 测试initialize_parameters_random
    print("====================测试initialize_parameters_random====================")
    parameters = initialize_parameters_random([3, 2, 1])
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    
    
    # 抑梯度异常初始化
    def initialize_parameters_he(layers_dims):
        np.random.seed(3)
        parameters = { }
        L = len(layers_dims)
        for l in range(1, L):
            parameters["W" + str(l)] = np.random.randn(layers_dims[l], layers_dims[l - 1]) * np.sqrt(2 / layers_dims[l - 1])
            parameters["b" + str(l)] = np.zeros((layers_dims[l], 1))
            assert (parameters["W" + str(l)].shape) == ((layers_dims[l],layers_dims[l - 1]))
            assert (parameters["b" + str(l)].shape) == ((layers_dims[l],1))
        return parameters
    
    # 测试initialize_parameters_he
    print("====================测试initialize_parameters_he====================")
    parameters = initialize_parameters_random([2, 4, 1])
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    
    
    # 定义模型
    def model01(X, Y, learning_rate=0.01, num_interations=15000, print_cost=True, initialization="he", is_Plot=True):
        """ 实现一个三层的神经网络：LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID :param X: -输入数据，维度为（2，要训练/测试的数量） :param Y: -标签，【0 | 1】，维度为（1，对应的是输入数据的标签） :param learning_rate: -学习速率 :param num_interations: -迭代次数 :param print_cost: -是否打印成本值，每1000次打印一次 :param initialization: -字符串类型，初始化的类型【"zeros" | "random | "he"】 :param is_Plot: -是否绘制梯度下降的曲线图 :return: parameters -学习后的参数 """
        grads = { }
        costs = []
        m = X.shape[1]
        layers_dims = [X.shape[0], 10, 5, 1]
    
        # 选择初始化参数的类型
        if initialization == "zeros":
            parameters = initialize_parameters_zeros(layers_dims)
        elif initialization == "random":
            parameters = initialize_parameters_random(layers_dims)
        elif initialization == "he":
            parameters = initialize_parameters_he(layers_dims)
        else:
            print("错误的初始化参数！程序退出")
            exit
    
        # 开始学习
        for i in range(0, num_interations):
            # 前向传播
            a3, cache = Deep_Learning.test2_1.init_utils.forward_propagation(X, parameters)
    
            # 计算成本
            cost = Deep_Learning.test2_1.init_utils.compute_loss(a3, Y)
    
            # 反向传播
            grads = Deep_Learning.test2_1.init_utils.backward_propagation(X, Y, cache)
    
            # 更新参数
            parameters = Deep_Learning.test2_1.init_utils.update_parameters(parameters, grads, learning_rate)
    
            # 记录成本
            if i % 1000 == 0:
                costs.append(cost)
                # 打印成本
                if print_cost:
                    print("第" + str(i) + "次迭代，成本值为：", np.squeeze(cost))
    
        # 学习完毕，绘制成本曲线
        if is_Plot:
            plt.plot(costs)
            plt.ylabel('cost')
            plt.xlabel('iterations (per hundreds)')
            plt.title("Learning rate = " + str(learning_rate))
            plt.show()
    
        # 返回学习完毕后的参数
        return parameters
    
    
    
    # 使用不同的初始化参数训练模型
    parameters = model01(train_X, train_Y, initialization="zeros")
    # parameters = model01(train_X, train_Y, initialization="random")
    # parameters = model01(train_X, train_Y, initialization="he")
    
    # 查看预测结果
    print("训练集：")
    predictions_train = Deep_Learning.test2_1.init_utils.predict(train_X, train_Y, parameters)
    print("测试集：")
    predictions_test = Deep_Learning.test2_1.init_utils.predict(test_X, test_Y, parameters)
    
    # 查看预测和决策边界的细节
    print("predictions_train = " + str(predictions_train))
    print("predictions_test = " + str(predictions_test))
    
    
    plt.title("Model01 with Zeros initialization")
    axes = plt.gca()    # 获得当前的Axes轴对象
    axes.set_xlim([-1.5, 1.5])  # 设置x轴边界
    axes.set_ylim([-1.5, 1.5])  # 设置y轴边界
    Deep_Learning.test2_1.init_utils.plot_decision_boundary(lambda x: Deep_Learning.test2_1.init_utils.predict_dec(parameters, x.T), train_X, train_Y)
    
    """ 1.零初始化结果显示为一条直线，学习率没有变化，模型没有学习，分类失败，零初始化导致神经网络无法打破对称性 2.随机初始化有助于打破对称，不同隐藏层单元可以学习到不同的参数，但是误差开始很高，这是由于具有较大的随机权重，若初始化参数没有很好的话会导致梯度消失、爆炸，也会减慢优化算法。 3.抑梯度异常初始化的结果很好 """

### 查看数据集 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center]

### 1.1零初始化时的运行结果 ###

====================测试initialize_parameters_zeros====================
    W1 = [[0. 0. 0.]
     [0. 0. 0.]]
    b1 = [[0.]
     [0.]]
    W2 = [[0. 0.]]
    b2 = [[0.]]
    第0次迭代，成本值为： 0.6931471805599453
    第1000次迭代，成本值为： 0.6931471805599453
    第2000次迭代，成本值为： 0.6931471805599453
    第3000次迭代，成本值为： 0.6931471805599453
    第4000次迭代，成本值为： 0.6931471805599453
    第5000次迭代，成本值为： 0.6931471805599453
    第6000次迭代，成本值为： 0.6931471805599453
    第7000次迭代，成本值为： 0.6931471805599453
    第8000次迭代，成本值为： 0.6931471805599453
    第9000次迭代，成本值为： 0.6931471805599453
    第10000次迭代，成本值为： 0.6931471805599455
    第11000次迭代，成本值为： 0.6931471805599453
    第12000次迭代，成本值为： 0.6931471805599453
    第13000次迭代，成本值为： 0.6931471805599453
    第14000次迭代，成本值为： 0.6931471805599453
    训练集：
    Accuracy: 0.5
    测试集：
    Accuracy: 0.5
    predictions_train = [[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0]]
    predictions_test = [[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
      0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 1]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 2]  
零初始化结果显示为一条直线，学习率没有变化，模型没有学习，分类失败，零初始化导致神经网络无法打破对称性。

### 1.2随机初始化时的运行结果 ###

====================测试initialize_parameters_random====================
    W1 = [[5.50797903 7.08147823 2.90904739]
     [5.10827605 8.92946954 8.96293089]]
    b1 = [[0.]
     [0.]]
    W2 = [[1.2558531  2.07242878]]
    b2 = [[0.]]
    第0次迭代，成本值为： inf
    第1000次迭代，成本值为： 0.7621259370184228
    第2000次迭代，成本值为： 0.6690029827916156
    第3000次迭代，成本值为： 0.6513359019864347
    第4000次迭代，成本值为： 0.6470045598995088
    第5000次迭代，成本值为： 0.6447240738113086
    第6000次迭代，成本值为： 0.6409840974187322
    第7000次迭代，成本值为： 0.6278546078886772
    第8000次迭代，成本值为： 0.5964022514198947
    第9000次迭代，成本值为： 0.5493267350966603
    第10000次迭代，成本值为： 0.5069866933465077
    第11000次迭代，成本值为： 0.4899755730226258
    第12000次迭代，成本值为： 0.47892254564188913
    第13000次迭代，成本值为： 0.47441470732170654
    第14000次迭代，成本值为： 0.47138671562861467
    训练集：
    Accuracy: 0.6833333333333333
    测试集：
    Accuracy: 0.67
    predictions_train = [[1 1 1 1 1 1 1 1 1 1 1 0 1 1 1 1 0 1 1 0 1 1 1 0 1 0 1 1 1 1 0 1 1 1 1 1
      1 1 1 1 1 1 1 0 1 1 1 1 0 1 0 1 1 1 1 1 1 1 1 1 1 0 1 1 1 1 0 1 1 1 1 0
      1 1 0 1 1 0 1 0 1 1 1 1 0 1 1 1 1 1 1 0 0 1 1 1 0 1 1 1 1 0 1 1 1 1 1 1
      1 0 1 1 1 0 1 1 0 1 1 1 1 1 1 0 1 0 0 1 1 1 1 1 1 1 1 1 0 1 1 0 0 1 1 1
      1 0 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 1 1 1 1 0 1 1 1 1 1 1 1 1
      1 1 1 0 1 0 1 1 1 1 1 1 1 0 1 1 0 1 1 0 1 0 1 1 1 1 1 1 1 0 1 0 1 1 0 1
      1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 1 0 1 0 1 1 1 1 1 1 1 1 0 1 1 1 0 1 1
      1 1 1 1 1 0 1 1 0 1 0 1 0 0 1 1 0 1 1 1 1 1 1 0 0 0 0 1 1 1 1 1 1 1 1 1
      1 1 1 1 1 0 1 1 1 1 1 0]]
    predictions_test = [[1 1 1 1 1 1 0 1 1 0 1 1 1 0 0 1 1 1 1 1 1 0 1 1 1 0 1 1 1 1 1 0 1 0 1 1
      0 1 1 0 0 1 1 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 1 1 1
      1 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 0 1 0 1 1 1 1 1 0 1]]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 3]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 4]  
随机初始化有助于打破对称，不同隐藏层单元可以学习到不同的参数，但是误差开始很高，这是由于具有较大的随机权重，若初始化参数没有很好的话会导致梯度消失、爆炸，也会减慢优化算法。

### 1.3抑梯度异常初始化运行结果 ###

====================测试initialize_parameters_he====================
    W1 = [[5.50797903 7.08147823]
     [2.90904739 5.10827605]
     [8.92946954 8.96293089]
     [1.2558531  2.07242878]]
    b1 = [[0.]
     [0.]
     [0.]
     [0.]]
    W2 = [[0.51467203 4.40809844 0.29876211 4.56833224]]
    b2 = [[0.]]
    第0次迭代，成本值为： 0.8830537463419761
    第1000次迭代，成本值为： 0.6879825919728063
    第2000次迭代，成本值为： 0.6751286264523371
    第3000次迭代，成本值为： 0.6526117768893807
    第4000次迭代，成本值为： 0.6082958970572938
    第5000次迭代，成本值为： 0.5304944491717495
    第6000次迭代，成本值为： 0.4138645817071794
    第7000次迭代，成本值为： 0.3117803464844441
    第8000次迭代，成本值为： 0.23696215330322562
    第9000次迭代，成本值为： 0.18597287209206836
    第10000次迭代，成本值为： 0.15015556280371817
    第11000次迭代，成本值为： 0.12325079292273552
    第12000次迭代，成本值为： 0.09917746546525932
    第13000次迭代，成本值为： 0.08457055954024274
    第14000次迭代，成本值为： 0.07357895962677362
    训练集：
    Accuracy: 0.9933333333333333
    测试集：
    Accuracy: 0.96
    predictions_train = [[1 0 1 1 0 0 1 0 1 1 1 0 1 0 0 0 0 1 1 0 1 0 1 0 0 0 1 0 1 1 0 1 1 0 0 0
      0 1 0 1 1 1 1 0 0 1 1 1 0 1 0 1 1 1 1 0 0 1 1 1 1 1 0 1 0 1 0 1 0 1 1 0
      0 0 0 0 1 0 0 0 1 0 1 0 0 1 1 1 1 1 1 0 0 0 1 1 0 1 1 0 1 0 0 1 0 1 1 0
      0 0 1 1 0 0 1 0 0 1 0 0 1 1 1 0 0 0 0 1 0 1 1 0 1 1 1 1 0 1 1 0 0 0 0 0
      0 0 1 0 1 0 1 0 1 1 1 0 0 1 1 0 1 0 1 1 0 1 0 1 1 0 1 1 1 1 0 1 0 0 1 0
      1 0 0 0 1 0 1 1 1 0 0 1 1 0 0 1 0 1 0 0 1 0 1 1 0 1 1 1 1 0 1 0 1 0 0 1
      0 1 0 0 0 1 1 1 1 0 1 0 0 1 1 0 0 1 0 1 0 0 1 0 0 1 1 0 0 0 0 1 1 0 1 0
      1 1 0 1 1 0 1 0 0 0 0 0 0 0 0 1 0 0 0 1 1 1 1 0 0 0 0 0 1 1 0 0 1 1 1 1
      1 1 1 0 0 0 0 1 1 0 1 0]]
    predictions_test = [[1 0 1 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 1 1 0 0 1 0 0 0 0 1 1 1 1 0 1 0 0 1
      0 1 1 0 0 1 1 1 1 0 0 0 1 1 0 1 0 1 1 0 1 0 1 0 1 1 1 1 1 1 1 1 1 1 1 0
      1 1 1 1 0 1 1 0 0 1 0 0 0 0 1 0 1 1 0 0 0 1 1 0 0 0 0 0]]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 5]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 6]  
抑梯度异常初始化的结果很好

## 2.正则化 ##

### regularization.py ###

""" 正则化模型： 1：使用二范数对二分类模型正则化，尝试避免过拟合。 2：使用随机删除节点的方法精简模型，同样是为了尝试避免过拟合。 我们要做的就是使用模型画一条线将红色和蓝色分界，分别使用以下三种 1.不使用正则化 2.使用正则化 2.1使用L2正则化：将lambd输入设置为非零值，不使用lambda是因为其是python的关键保留字 2.2使用随机节点删除：将keep_prob设置为小于1的值 """
    
    import numpy as np
    import matplotlib.pyplot as plt
    import Deep_Learning.test2_1.reg_utils
    
    # 绘图初始设置
    plt.rcParams['figure.figsize'] = (7.0, 4.0)     # set default size of plots
    plt.rcParams['image.interpolation'] = 'nearest'
    plt.rcParams['image.cmap'] = 'gray'
    
    
    # 若数据集足够大会导致过拟合问题，为避免过拟合使用正则化模型
    
    # 读取并绘制数据集
    train_X, train_Y, test_X, test_Y = Deep_Learning.test2_1.reg_utils.load_2D_dataset(is_plot=True)
    plt.show()
    
    """ 我们要做的就是使用模型画一条线将红色和蓝色分界，分别使用以下三种 1.不使用正则化 2.使用正则化 2.1使用L2正则化：将lambd输入设置为非零值，不使用lambda是因为其是python的关键保留字 2.2使用随机节点删除：将keep_prob设置为小于1的值 """
    
    
    def model02(X,Y,learning_rate=0.3,num_iterations=30000,print_cost=True,is_Plot=True,lambd=0,keep_prob=1):
        """ 实现一个三层的神经网络：LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID :param X: -输入数据，维度为（2，要训练/测试的数量） :param Y: -标签，【0（蓝色） | 1（红色）】，维度为（1，对应的是输入数据的标签） :param learning_rate: -学习速率 :param num_iterations: -迭代次数 :param print_cost: -是否打印成本值，每迭代10000次打印一次，但是每1000次记录一个成本值 :param is_Plot: -是否绘制梯度下降的曲线图 :param lambd: -正则化的超参数、实数 :param keep_prob: -随机删除节点的概率 :return: parameters -学习后的参数 """
        grads = { }
        costs = []
        m = X.shape[1]
        layers_dims = [X.shape[0],20,3,1]
    
        # 初始化参数
        parameters = Deep_Learning.test2_1.reg_utils.initialize_parameters(layers_dims)
    
        # 开始学习
        for i in range(0,num_iterations):
            # 前向传播
            ## 是否随机删除节点
            if keep_prob == 1:
                ### 不随机删除节点
                a3,cache = Deep_Learning.test2_1.reg_utils.forward_propagation(X,parameters)
            elif keep_prob < 1:
                ### 随机删除节点
                a3,cache = forward_propagation_with_dropout(X,parameters,keep_prob)
            else:
                print("keep_prob参数错误，程序退出")
    
            # 计算成本
            ## 是否使用二范数
            if lambd == 0:
                ### 不使用L2正则化
                cost = Deep_Learning.test2_1.reg_utils.compute_cost(a3,Y)
            else:
                ### 使用L2正则化
                cost = compute_cost_with_regularization(a3,Y,parameters,lambd)
    
            # 反向传播
            ## 可以同时使用L2正则化和随机删除节点，但本次实验不同时使用
            assert (lambd == 0 or keep_prob == 1)
    
            ## 两个参数的使用情况
            if (lambd == 0 and keep_prob == 1):
                ### 不使用L2正则化和不使用随机删除节点
                grads = Deep_Learning.test2_1.reg_utils.backward_propagation(X,Y,cache)
            elif lambd != 0:
                ### 使用L2正则化，不使用随机删除节点
                grads = backward_propagation_with_regularization(X,Y,cache,lambd)
            elif keep_prob < 1:
                ### 使用随机删除节点，不使用L2正则化
                grads = backward_propagation_with_dropout(X,Y,cache,keep_prob)
    
            # 更新参数
            parameters = Deep_Learning.test2_1.reg_utils.update_parameters(parameters,grads,learning_rate)
    
            # 记录并打印成本
            if i % 1000 == 0:
                ## 记录成本
                costs.append(cost)
                if (print_cost and i % 10000 == 0):
                    # 打印成本
                    print("第" + str(i) + "次迭代，成本值为：" + str(cost))
    
        # 是否绘制成本曲线
        if is_Plot:
            plt.plot(costs)
            plt.ylabel('cost')
            plt.xlabel('iterations(x1, 000)')
            plt.title("Learning rate = " + str(learning_rate))
            plt.show()
    
        # 返回学习后的参数
        return parameters
    
    
    # 不使用正则化
    parameters = model02(train_X,train_Y)
    print("训练集：")
    predictions_train = Deep_Learning.test2_1.reg_utils.predict(train_X,train_Y,parameters)
    print("测试集：")
    predictions_test = Deep_Learning.test2_1.reg_utils.predict(test_X,test_Y,parameters)
    
    # 绘制边界曲线
    plt.title("Model without regularization")
    axes = plt.gca()
    axes.set_xlim([-0.75,0.40])
    axes.set_ylim([-0.75,0.65])
    Deep_Learning.test2_1.reg_utils.plot_decision_boundary(lambda x: Deep_Learning.test2_1.reg_utils.predict_dec(parameters,x.T),train_X,train_Y)
    
    
    # 使用L2正则化
    def compute_cost_with_regularization(A3,Y,parameters,lambd):
        """ 实现L2正则化计算成本 :param A3: -正向传播的输出结果，维度为（输出节点数量，训练/测试的数量） :param Y: -标签向量，与数据一一对应，维度为（输出节点数量，训练/测试的数量） :param parameters: -包含模型学习后的参数的字典 :param lambd: -正则化的超参数、实数 :return: cost -正则化损失的值 """
        m = Y.shape[1]
        W1 = parameters["W1"]
        W2 = parameters["W2"]
        W3 = parameters["W3"]
    
        # 交叉熵成本
        cross_entropy_cost = Deep_Learning.test2_1.reg_utils.compute_cost(A3,Y)
    
        # L2正则化项
        L2_regularization_cost = lambd * (np.sum(np.square(W1)) + np.sum(np.square(W2)) + np.sum(np.square(W3))) / (2 * m)
    
        cost = cross_entropy_cost + L2_regularization_cost
    
        return cost
    
    
    # 由于改变了成本函数，也必须改变向后传播函数，所有的梯度都必须根据这个新的成本值来计算
    def backward_propagation_with_regularization(X,Y,cache,lambd):
        """ 实现添加了L2正则化的模型的后向传播 :param X: -输入数据集，维度为（输入节点数量，数据集里面的数量） :param Y: -标签，维度为（输出节点数量，数据集里面的数量） :param cache: -来自forward_propagation()的cache输出 :param lambd: -regularization超参数，实数 :return: gradients -包含每个参数、激活值和预激活值变量的梯度的字典 """
        m = X.shape[1]
    
        (Z1,A1,W1,b1,Z2,A2,W2,b2,Z3,A3,W3,b3) = cache
        dZ3 = A3 - Y
    
        dW3 = (1 / m) * np.dot(dZ3,A2.T) + ((lambd * W3) / m)
        db3 = (1 / m) * np.sum(dZ3,axis=1,keepdims=True)
    
        dA2 = np.dot(W3.T,dZ3)
        dZ2 = np.multiply(dA2,np.int64(A2 > 0))  # 参见笔记3.8：若A2>0结果为1，否则为0
        dW2 = (1 / m) * np.dot(dZ2,A1.T) + ((lambd * W2) / m)
        db2 = (1 / m) * np.sum(dZ2,axis=1,keepdims=True)
    
        dA1 = np.dot(W2.T,dZ2)
        dZ1 = np.multiply(dA1,np.int64(A1 > 0))
        dW1 = (1 / m) * np.dot(dZ1,X.T) + ((lambd * W1) / m)
        db1 = (1 / m) * np.sum(dZ1,axis=1,keepdims=True)
    
        gradients = { "dZ3": dZ3,"dW3": dW3,"db3": db3,"dA2": dA2,
                     "dZ2": dZ2,"dW2": dW2,"db2": db2,"dA1": dA1,
                     "dZ1": dZ1,"dW1": dW1,"db1": db1}
    
        return gradients
    
    
    # 测试L2正则化
    parameters = model02(train_X,train_Y,lambd=0.7)
    print("使用正则化，训练集：")
    predictions_train = Deep_Learning.test2_1.reg_utils.predict(train_X,train_Y,parameters)
    print("使用正则化，测试集：")
    predictions_test = Deep_Learning.test2_1.reg_utils.predict(test_X,test_Y,parameters)
    
    
    # 查看分类结果
    plt.title("Model with L2-regularization")
    axes = plt.gca()
    axes.set_xlim([-0.75,0.40])
    axes.set_ylim([-0.75,0.65])
    Deep_Learning.test2_1.reg_utils.plot_decision_boundary(
        lambda x: Deep_Learning.test2_1.reg_utils.predict_dec(parameters,x.T),train_X,train_Y)
    """ lambd的值是可以使用开发集调整时的超参数，L2正则化会使决策边界更加平滑，若lambd太大，会平滑过渡模型产生高偏差 L2正则化依赖于较小权重的模型比具有较大权重的模型更简单这样的假设，因此通过削弱成本函数中权重的平方值，可以将所有权重值逐渐变小 权重高的话会有更平滑的模型，其中输入变化时输出变化更慢，L2正则化对以下内容有影响 1.成本计算：正则化的计算需要添加到成本函数中 2.反向传播：在权重矩阵中，梯度计算时也要依据正则化作出相应的计算 3.重量变小（重量衰减）：权重被逐渐改变到较小的值 """
    
    
    # 随机删除节点
    def forward_propagation_with_dropout(X,parameters,keep_prob=0.5):
        """ 实现具有随机舍弃节点的前向传播 LINEAR -> RELU + DROPOUT -> LINEAR -> RELU + DROPOUT -> LINEAR -> SIGMOID :param X: -输入数据集，维度为（2，示例数） :param parameters: -包含参数"W1","b1","W2","b2","W3","b3"的字典 :param keep_prob: -随机删除的概率，实数 :return: A3 -最后的激活值，维度为（1,1),正向传播的输出 cache -存储一些用于计算反向传播的数值的元组 """
        np.random.seed(1)
    
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        W3 = parameters["W3"]
        b3 = parameters["b3"]
    
        Z1 = np.dot(W1,X) + b1
        A1 = Deep_Learning.test2_1.reg_utils.relu(Z1)
    
        # DROPOUT
        # 1.初始化矩阵D1 = np.random.rand(...,...)
        D1 = np.random.rand(A1.shape[0],A1.shape[1])
        # 2.将D1的值转换为0或者1，(使用keep_prob作为阈值)
        # 高于keep_prob的节点设置为，低于的设置为1？？？？？？？？？？？
        D1 = D1 < keep_prob
        # 3.舍弃A1的一些节点（将其值设为0或False）
        A1 = A1 * D1
        # 4.缩放未舍弃的节点（不为0）的值
        A1 = A1 / keep_prob
        """ 示例代码帮助理解： import numpy as np np.random.seed(1) # ??????????rand和randn区别 A1 = np.random.randn(1, 3) print("A1:", A1) # A1: [[ 1.62434536 -0.61175641 -0.52817175]] D1 = np.random.rand(A1.shape[0], A1.shape[1]) print("D1:", D1) # D1: [[0.39676747 0.53881673 0.41919451]] keep_prob = 0.5 D1 = D1 < keep_prob print(D1) # [[ True False True]] A1 = 0.01 A1 = A1 * D1 A1 = A1 / keep_prob print(A1) # [[0.02 0. 0.02]] """
        # ??????????? W.T
        Z2 = np.dot(W2,A1) + b2
        A2 = Deep_Learning.test2_1.reg_utils.relu(Z2)
    
        D2 = np.random.rand(A2.shape[0],A2.shape[1])
        D2 = D2 < keep_prob
        A2 = A2 * D2
        A2 = A2 / keep_prob
    
        Z3 = np.dot(W3,A2) + b3
        A3 = Deep_Learning.test2_1.reg_utils.sigmoid(Z3)
    
        cache = (Z1,D1,A1,W1,b1,Z2,D2,A2,W2,b2,Z3,A3,W3,b3)
    
        return A3,cache
    
    
    # 随机删除的模型的后向传播
    def backward_propagation_with_dropout(X,Y,cache,keep_prob):
        """ :param X: -输入数据集，维度为（2，示例数） :param Y: -标签，维度为（输出节点数量，示例数量） :param cache: -来自forward_propagation_with_dropout()的cache输出 :param keep_prob: -随机删除的概率，实数 :return:gradients -一个关于每个参数，激活值和预激活变量的梯度值的字典 """
        m = X.shape[1]
        (Z1,D1,A1,W1,b1,Z2,D2,A2,W2,b2,Z3,A3,W3,b3) = cache
    
        dZ3 = A3 - Y
        # ???????????????.T
        dW3 = (1 / m) * np.dot(dZ3,A2.T)
        db3 = (1. / m) * np.sum(dZ3,axis=1,keepdims=True)
        dA2 = np.dot(W3.T,dZ3)
    
        # 使用正向传播相同的节点，舍弃那些关闭的节点，（因为任何数乘以0或者False都为0或者False）
        dA2 = dA2 * D2
        # 缩放未舍弃的节点（不为0）的值
        dA2 = dA2 / keep_prob
    
        dZ2 = np.multiply(dA2,np.int64(A2 > 0))
        dW2 = 1. / m * np.dot(dZ2,A1.T)
        db2 = 1. / m * np.sum(dZ2,axis=1,keepdims=True)
    
        dA1 = np.dot(W2.T,dZ2)
    
        dA1 = dA1 * D1
        dA1 = dA1 / keep_prob
    
        dZ1 = np.multiply(dA1,np.int64(A1 > 0))
        dW1 = 1. / m * np.dot(dZ1,X.T)
        db1 = 1. / m * np.sum(dZ1,axis=1,keepdims=True)
    
        gradients = { "dZ3": dZ3,"dW3": dW3,"db3": db3,"dA2": dA2,
                     "dZ2": dZ2,"dW2": dW2,"db2": db2,"dA1": dA1,
                     "dZ1": dZ1,"dW1": dW1,"db1": db1}
    
        return gradients
    
    
    # 测试DROPOUT
    parameters = model02(train_X,train_Y,keep_prob=0.86,learning_rate=0.3)
    print("使用随机删除节点，训练集：")
    predictions_train = Deep_Learning.test2_1.reg_utils.predict(train_X,train_Y,parameters)
    print("使用随机删除节点，测试集：")
    predictions_test = Deep_Learning.test2_1.reg_utils.predict(test_X,test_Y,parameters)
    
    # 绘制决策边界
    plt.title("Model with dropout")
    axes = plt.gca()
    axes.set_xlim([-0.75,0.40])
    axes.set_ylim([-0.75,0.65])
    Deep_Learning.test2_1.reg_utils.plot_decision_boundary(
        lambda x: Deep_Learning.test2_1.reg_utils.predict_dec(parameters,x.T),train_X,train_Y)
    """ 正则化会把训练集的准确度降低，但是测试集的准确度提高了 """

### 查看数据集 ###

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 7]

### 2.1无正则化的运行结果 ###

第0次迭代，成本值为：0.6557412523481002
    第10000次迭代，成本值为：0.16329987525724216
    第20000次迭代，成本值为：0.1385164242327309
    训练集：
    Accuracy: 0.9478672985781991
    测试集：
    Accuracy: 0.915

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 8]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 9]  
无正则化时，分割曲线有明显的过拟合特征。

### 2.2使用L2正则化后的运行结果 ###

第0次迭代，成本值为：0.6974484493131264
    第10000次迭代，成本值为：0.2684918873282239
    第20000次迭代，成本值为：0.2680916337127301
    使用正则化，训练集：
    Accuracy: 0.9383886255924171
    使用正则化，测试集：
    Accuracy: 0.93

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 10]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 11]  
lambd的值是可以使用开发集调整时的超参数，L2正则化会使决策边界更加平滑，若lambd太大，会平滑过渡模型产生高偏差  
L2正则化依赖于较小权重的模型比具有较大权重的模型更简单这样的假设，因此通过削弱成本函数中权重的平方值，可以将所有权重值逐渐变小，权重高的话会有更平滑的模型，其中输入变化时输出变化更慢，L2正则化对以下内容有影响：

1.  成本计算：正则化的计算需要添加到成本函数中
2.  反向传播：在权重矩阵中，梯度计算时也要依据正则化作出相应的计算
3.  重量变小（重量衰减）：权重被逐渐改变到较小的值

### 2.3使用Dropout后的运行结果 ###

第0次迭代，成本值为：0.6543912405149825
    第10000次迭代，成本值为：0.0610169865749056
    第20000次迭代，成本值为：0.060582435798513114
    使用随机删除节点，训练集：
    Accuracy: 0.9289099526066351
    使用随机删除节点，测试集：
    Accuracy: 0.95

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 12]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 13]  
正则化会把训练集的准确度降低，但是测试集的准确度提高了

## 3.梯度校验 ##

### gradient\_check.py ###

""" 梯度校验：对模型使用梯度校验，检测它是否在梯度下降的过程中出现误差过大的情况。 """
    import numpy as np
    import Deep_Learning.test2_1.gc_utils
    
    # 梯度检验
    
    # 一维线性模型的梯度检查计算过程
    def forward_propagation(x, theta):
        """ 实现线性前向传播（计算J）（J(theta) = theta * x） :param x: -一个实值输入 :param theta: -参数，也是一个实数 :return: J -函数J的值 """
        J = np.dot(theta, x)
        return J
    
    def backward_propagation(x, theta):
        """ 计算J相对于theta的导数 :param x: -一个实值输入 :param theta: -参数，也是一个实数 :return: dtheta -相对于theta的成本梯度 """
        dtheta = x
        return dtheta
    
    # 当difference<10e-7时结果正确
    def gradient_check(x, theta, epsilon=1e-7):
        """ :param x: -一个实值输入 :param theta: -参数，也是一个实数 :param epsilon: -计算输入的微笑偏移以计算近似梯度 :return: 近似梯度和后向传播梯度之间的差异 """
        thetaplus = theta + epsilon
        thetaminus = theta - epsilon
        J_plus = forward_propagation(x, thetaplus)
        J_minus = forward_propagation(x, thetaminus)
        gradapprox = (J_plus - J_minus) / (2 * epsilon)
    
        # 检查gradapprox是否足够接近backward_propagation()的输出
        grad = backward_propagation(x, theta)
    
        numerator = np.linalg.norm(grad - gradapprox)   # 求范数：linalg=linear（线性）+algebra（代数），norm则表示范数。
        denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox)
        difference = numerator / denominator
    
        if difference < 1e-7:
            print("一维线性梯度检查：梯度正常")
        else:
            print("一维线性梯度检查：梯度超出阈值")
    
        return difference
    
    # 测试gradient_check
    print("=================测试gradient_check====================")
    x, theta = 2, 4
    difference = gradient_check(x, theta)
    print("difference = " + str(difference))
    """ 运行结果： 一维线性梯度检查：梯度正常 difference = 2.919335883291695e-10 """
    
    
    # 高维模型的梯度检查计算过程
    def forward_propagation_n(X, Y, parameters):
        """ :param X: -训练集为m个例子 :param Y: -m个示例的标签 :param parameters: -包含参数"W1","b1","W2","b2","W3","b3"的字典（layers_dims=[4,5,3,1]） :return: cost -成本函数 """
        m = X.shape[1]
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        W3 = parameters["W3"]
        b3 = parameters["b3"]
    
        # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
        Z1 = np.dot(W1, X) + b1
        A1 = Deep_Learning.test2_1.gc_utils.relu(Z1)
    
        Z2 = np.dot(W2, A1) + b2
        A2 = Deep_Learning.test2_1.gc_utils.relu(Z2)
    
        Z3 = np.dot(W3, A2) + b3
        A3 = Deep_Learning.test2_1.gc_utils.relu(Z3)
    
        # 计算成本
        logprobs = np.multiply(-np.log(A3), Y) + np.multiply(-np.log(1 - A3), 1 - Y)
        cost = (1 / m) * np.sum(logprobs)
    
        cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)
    
        return cost, cache
    
    def backward_propagation_n(X, Y, cache):
        """ :param X: -输入数据点（输入节点数，1） :param Y: -标签 :param cache: -来自forward_propagation_n()的cache输出 :return: gradients -字典：包含每个参数、激活和激活前变量相关的成本梯度 """
        m = X.shape[1]
        (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache
    
        dZ3 = A3 - Y
        # 1/和1./的无区别
        dW3 = (1. / m) * np.dot(dZ3, A2.T)
        db3 = (1. / m) * np.sum(dZ3, axis=1, keepdims=True)
    
        dA2 = np.dot(W3.T, dZ3)
        dZ2 = np.multiply(dA2, np.int64(A2 > 0))
        dW2 = 1. / m * np.dot(dZ2, A1.T)
        db2 = 1. / m * np.sum(dZ2, axis=1, keepdims=True)
    
        dA1 = np.dot(W2.T, dZ2)
        dZ1 = np.multiply(dA1, np.int64(A1 > 0))
        dW1 = 1. / m * np.dot(dZ1, X.T)
        db1 = 1. / m * np.sum(dZ1, axis=1, keepdims=True)
    
        gradients = { "dZ3": dZ3,"dW3": dW3,"db3": db3,"dA2": dA2,
                     "dZ2": dZ2,"dW2": dW2,"db2": db2,"dA1": dA1,
                     "dZ1": dZ1,"dW1": dW1,"db1": db1}
    
        return gradients
    
    def gradient_check_n(parameters, gradients, X, Y, epsilon=1e-7):
        """ 检查backward_propagation_n是否正确计算forward_propagation_n输出的成本梯度 :param parameters: -包含参数"W1","b1","W2","b2","W3","b3"的字典 :param gradients: -backward_propagation_n的输出，包含与参数相关的成本梯度 :param X: -输入数据点，维度为（输入节点数量，1） :param Y: -标签 :param epsilon: -计算输入的微小偏移以计算近似梯度 :return: difference -近似梯度和后向传播梯度之间的差异 """
        # 初始化参数
        # 将"parameters"字典转换为一个称为"values"的向量，通过将所有参数(W1,b1,W2,b2,W3,b3)整形为向量并将它们连接起来获得
        parameters_values, keys = Deep_Learning.test2_1.gc_utils.dictionary_to_vector(parameters)   # keys用不到
        grad = Deep_Learning.test2_1.gc_utils.gradients_to_vector(gradients)
        num_parameters = parameters_values.shape[0]
        J_plus = np.zeros((num_parameters, 1))
        J_minus = np.zeros((num_parameters, 1))
        gradapprox = np.zeros((num_parameters, 1))
    
        # 计算gradapprox
        for i in range(num_parameters):
            # 计算J_plus[i] 输入="parameters_values,epsilon" 输出=”J_plus[i]“
            thetaplus = np.copy(parameters_values)
            thetaplus[i][0] = thetaplus[i][0] + epsilon
            J_plus[i], cache = forward_propagation_n(X, Y, Deep_Learning.test2_1.gc_utils.vector_to_dictionary(thetaplus))
    
            # 计算J_minus[i] 输入="parameters_values,epsilon" 输出="J_minus[i]"
            thetaminus = np.copy(parameters_values)
            thetaminus[i][0] = thetaminus[i][0] - epsilon
            J_minus[i], cache = forward_propagation_n(X, Y, Deep_Learning.test2_1.gc_utils.vector_to_dictionary(thetaminus))
    
            # 计算gradapprox[i]
            gradapprox[i] = (J_plus[i] - J_minus[i]) / (2 * epsilon)
    
        # 通过计算差异比较gradapprox和向后传播梯度
        numerator = np.linalg.norm(grad - gradapprox)
        denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox)
        difference = numerator / denominator
    
        if difference < 1e-7:
            print("高维梯度检查：梯度正常")
        else:
            print("高维梯度检查：梯度超出阈值")
    
        return difference
    
    # 测试gradient_check_n，
    print("=======================测试gradient_check_n====================")
    np.random.seed(2)
    X = np.random.randn(4, 3)
    Y = np.array([1, 1, 0])
    W1 = np.random.randn(5, 4)
    b1 = np.zeros((5, 1))
    W2 = np.random.randn(3, 5)
    b2 = np.zeros((3, 1))
    W3 = np.random.randn(1, 3)
    b3 = np.zeros((1, 1))
    parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2, "W3": W3, "b3": b3}
    cost, cache = forward_propagation_n(X, Y, parameters)
    gradients = backward_propagation_n(X, Y, cache)
    difference = gradient_check_n(parameters, gradients, X, Y)
    print("difference = " + str(difference))
    """ 运行结果：（由于没有具体的实验数据结果显示超出阈值，准确率和对X初始化的种子有关） 高维梯度检查：梯度超出阈值 difference = nan """

## 工具包 ##

### init\_utils.py ###

# -*- coding: utf-8 -*-
    
    import numpy as np
    import matplotlib.pyplot as plt
    import sklearn
    import sklearn.datasets
    
    
    def sigmoid(x):
        """ Compute the sigmoid of x Arguments: x -- A scalar or numpy array of any size. Return: s -- sigmoid(x) """
        s = 1/(1+np.exp(-x))
        return s
     
    def relu(x):
        """ Compute the relu of x Arguments: x -- A scalar or numpy array of any size. Return: s -- relu(x) """
        s = np.maximum(0,x)
        
        return s
        
    def compute_loss(a3, Y):
        
        """ Implement the loss function Arguments: a3 -- post-activation, output of forward propagation Y -- "true" labels vector, same shape as a3 Returns: loss - value of the loss function """
        
        m = Y.shape[1]
        np.seterr(divide='ignore', invalid='ignore')  # 忽略除以0的情况，消除警告
        logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y)
        loss = 1./m * np.nansum(logprobs)
        
        return loss
        
    def forward_propagation(X, parameters):
        """ Implements the forward propagation (and computes the loss) presented in Figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) Y -- true "label" vector (containing 0 if cat, 1 if non-cat) parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": W1 -- weight matrix of shape () b1 -- bias vector of shape () W2 -- weight matrix of shape () b2 -- bias vector of shape () W3 -- weight matrix of shape () b3 -- bias vector of shape () Returns: loss -- the loss function (vanilla logistic loss) """
            
        # retrieve parameters
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        W3 = parameters["W3"]
        b3 = parameters["b3"]
        
        # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
        z1 = np.dot(W1, X) + b1
        a1 = relu(z1)
        z2 = np.dot(W2, a1) + b2
        a2 = relu(z2)
        z3 = np.dot(W3, a2) + b3
        a3 = sigmoid(z3)
        
        cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3)
        
        return a3, cache
     
    def backward_propagation(X, Y, cache):
        """ Implement the backward propagation presented in figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) Y -- true "label" vector (containing 0 if cat, 1 if non-cat) cache -- cache output from forward_propagation() Returns: gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables """
        m = X.shape[1]
        (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache
        
        dz3 = 1./m * (a3 - Y)
        dW3 = np.dot(dz3, a2.T)
        db3 = np.sum(dz3, axis=1, keepdims = True)
        
        da2 = np.dot(W3.T, dz3)
        dz2 = np.multiply(da2, np.int64(a2 > 0))
        dW2 = np.dot(dz2, a1.T)
        db2 = np.sum(dz2, axis=1, keepdims = True)
        
        da1 = np.dot(W2.T, dz2)
        dz1 = np.multiply(da1, np.int64(a1 > 0))
        dW1 = np.dot(dz1, X.T)
        db1 = np.sum(dz1, axis=1, keepdims = True)
        
        gradients = { "dz3": dz3, "dW3": dW3, "db3": db3,
                     "da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2,
                     "da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1}
        
        return gradients
     
    def update_parameters(parameters, grads, learning_rate):
        """ Update parameters using gradient descent Arguments: parameters -- python dictionary containing your parameters grads -- python dictionary containing your gradients, output of n_model_backward Returns: parameters -- python dictionary containing your updated parameters parameters['W' + str(i)] = ... parameters['b' + str(i)] = ... """
        
        L = len(parameters) // 2 # number of layers in the neural networks
     
        # Update rule for each parameter
        for k in range(L):
            parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)]
            parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)]
            
        return parameters
        
    def predict(X, y, parameters):
        """ This function is used to predict the results of a n-layer neural network. Arguments: X -- data set of examples you would like to label parameters -- parameters of the trained model Returns: p -- predictions for the given dataset X """
        
        m = X.shape[1]
        p = np.zeros((1,m), dtype = np.int)
        
        # Forward propagation
        a3, caches = forward_propagation(X, parameters)
        
        # convert probas to 0/1 predictions
        for i in range(0, a3.shape[1]):
            if a3[0,i] > 0.5:
                p[0,i] = 1
            else:
                p[0,i] = 0
     
        # print results
        print("Accuracy: "  + str(np.mean((p[0,:] == y[0,:]))))
        
        return p
    
    
    def load_dataset(is_plot=True):
        np.random.seed(1)
        train_X, train_Y = sklearn.datasets.make_circles(n_samples=300, noise=.05)
        np.random.seed(2)
        test_X, test_Y = sklearn.datasets.make_circles(n_samples=100, noise=.05)
        # Visualize the data
        if is_plot:
            plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral);
        train_X = train_X.T
        train_Y = train_Y.reshape((1, train_Y.shape[0]))
        test_X = test_X.T
        test_Y = test_Y.reshape((1, test_Y.shape[0]))
        return train_X, train_Y, test_X, test_Y
    
    
    """ 绘图决策边界的代码还是不能深入理解 """
    def plot_decision_boundary(model, X, y):
        # Set min and max values and give it some padding
        x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1
        y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1
        h = 0.01
        # Generate a grid of points with distance h between them
        xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
        # Predict the function value for the whole grid
        Z = model(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)
        # Plot the contour and training examples
        plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
        plt.ylabel('x2')
        plt.xlabel('x1')
        plt.scatter(X[0, :], X[1, :], c=y, cmap=plt.cm.Spectral)
        plt.show()
     
    def predict_dec(parameters, X):
        """ Used for plotting decision boundary. Arguments: parameters -- python dictionary containing your parameters X -- input data of size (m, K) Returns predictions -- vector of predictions of our model (red: 0 / blue: 1) """
        
        # Predict using forward propagation and a classification threshold of 0.5
        a3, cache = forward_propagation(X, parameters)
        predictions = (a3>0.5)
        return predictions

### reg\_utils.py ###

# -*- coding: utf-8 -*-
    
    import numpy as np
    import matplotlib.pyplot as plt
    import scipy.io as sio
    
    def sigmoid(x):
        """ Compute the sigmoid of x Arguments: x -- A scalar or numpy array of any size. Return: s -- sigmoid(x) """
        s = 1/(1+np.exp(-x))
        return s
     
    def relu(x):
        """ Compute the relu of x Arguments: x -- A scalar or numpy array of any size. Return: s -- relu(x) """
        s = np.maximum(0,x)
        
        return s
    
    
    def initialize_parameters(layer_dims):
        """ Arguments: layer_dims -- python array (list) containing the dimensions of each layer in our network Returns: parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL": W1 -- weight matrix of shape (layer_dims[l], layer_dims[l-1]) b1 -- bias vector of shape (layer_dims[l], 1) Wl -- weight matrix of shape (layer_dims[l-1], layer_dims[l]) bl -- bias vector of shape (1, layer_dims[l]) Tips: - For example: the layer_dims for the "Planar Data classification model" would have been [2,2,1]. This means W1's shape was (2,2), b1 was (1,2), W2 was (2,1) and b2 was (1,1). Now you have to generalize it! - In the for loop, use parameters['W' + str(l)] to access Wl, where l is the iterative integer. """
        
        np.random.seed(3)
        parameters = { }
        L = len(layer_dims) # number of layers in the network
     
        for l in range(1, L):
            parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1]) / np.sqrt(layer_dims[l-1])
            parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))
    
            # yntaxWarning: assertion is always true, perhaps remove parentheses?
            # assert(parameters['W' + str(l)].shape == layer_dims[l], layer_dims[l-1])
            # assert(parameters['W' + str(l)].shape == layer_dims[l], 1)
     
            
        return parameters
    
    def forward_propagation(X, parameters):
        """ Implements the forward propagation (and computes the loss) presented in Figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) Y -- true "label" vector (containing 0 if cat, 1 if non-cat) parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": W1 -- weight matrix of shape () b1 -- bias vector of shape () W2 -- weight matrix of shape () b2 -- bias vector of shape () W3 -- weight matrix of shape () b3 -- bias vector of shape () Returns: loss -- the loss function (vanilla logistic loss) """
            
        # retrieve parameters
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        W3 = parameters["W3"]
        b3 = parameters["b3"]
        
        # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
        z1 = np.dot(W1, X) + b1
        a1 = relu(z1)
        z2 = np.dot(W2, a1) + b2
        a2 = relu(z2)
        z3 = np.dot(W3, a2) + b3
        a3 = sigmoid(z3)
        
        cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3)
        
        return a3, cache
    
    
     
    def compute_cost(a3, Y):
        """ Implement the cost function Arguments: a3 -- post-activation, output of forward propagation Y -- "true" labels vector, same shape as a3 Returns: cost - value of the cost function """
        m = Y.shape[1]
        
        logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y)
        cost = 1./m * np.nansum(logprobs)
        
        return cost
    
    def backward_propagation(X, Y, cache):
        """ Implement the backward propagation presented in figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) Y -- true "label" vector (containing 0 if cat, 1 if non-cat) cache -- cache output from forward_propagation() Returns: gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables """
        m = X.shape[1]
        (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache
        
        dz3 = 1./m * (a3 - Y)
        dW3 = np.dot(dz3, a2.T)
        db3 = np.sum(dz3, axis=1, keepdims = True)
        
        da2 = np.dot(W3.T, dz3)
        dz2 = np.multiply(da2, np.int64(a2 > 0))
        dW2 = np.dot(dz2, a1.T)
        db2 = np.sum(dz2, axis=1, keepdims = True)
        
        da1 = np.dot(W2.T, dz2)
        dz1 = np.multiply(da1, np.int64(a1 > 0))
        dW1 = np.dot(dz1, X.T)
        db1 = np.sum(dz1, axis=1, keepdims = True)
        
        gradients = { "dz3": dz3, "dW3": dW3, "db3": db3,
                     "da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2,
                     "da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1}
        
        return gradients
    
    def update_parameters(parameters, grads, learning_rate):
        """ Update parameters using gradient descent Arguments: parameters -- python dictionary containing your parameters grads -- python dictionary containing your gradients, output of n_model_backward Returns: parameters -- python dictionary containing your updated parameters parameters['W' + str(i)] = ... parameters['b' + str(i)] = ... """
        
        L = len(parameters) // 2 # number of layers in the neural networks
     
        # Update rule for each parameter
        for k in range(L):
            parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)]
            parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)]
            
        return parameters
    
    
    
        
    def load_2D_dataset(is_plot=True):
        data = sio.loadmat('datasets/data.mat')
        train_X = data['X'].T
        train_Y = data['y'].T
        test_X = data['Xval'].T
        test_Y = data['yval'].T
        if is_plot:
            plt.scatter(train_X[0, :], train_X[1, :], c=train_Y, s=40, cmap=plt.cm.Spectral);
        
        return train_X, train_Y, test_X, test_Y
    
    def predict(X, y, parameters):
        """ This function is used to predict the results of a n-layer neural network. Arguments: X -- data set of examples you would like to label parameters -- parameters of the trained model Returns: p -- predictions for the given dataset X """
        
        m = X.shape[1]
        p = np.zeros((1,m), dtype = np.int)
        
        # Forward propagation
        a3, caches = forward_propagation(X, parameters)
        
        # convert probas to 0/1 predictions
        for i in range(0, a3.shape[1]):
            if a3[0,i] > 0.5:
                p[0,i] = 1
            else:
                p[0,i] = 0
     
        # print results
        print("Accuracy: "  + str(np.mean((p[0,:] == y[0,:]))))
        
        return p
    
    def plot_decision_boundary(model, X, y):
        # Set min and max values and give it some padding
        x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1
        y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1
        h = 0.01
        # Generate a grid of points with distance h between them
        xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
        # Predict the function value for the whole grid
        Z = model(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)
        # Plot the contour and training examples
        plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
        plt.ylabel('x2')
        plt.xlabel('x1')
        plt.scatter(X[0, :], X[1, :], c=y, cmap=plt.cm.Spectral)
        plt.show()
     
    def predict_dec(parameters, X):
        """ Used for plotting decision boundary. Arguments: parameters -- python dictionary containing your parameters X -- input data of size (m, K) Returns predictions -- vector of predictions of our model (red: 0 / blue: 1) """
        
        # Predict using forward propagation and a classification threshold of 0.5
        a3, cache = forward_propagation(X, parameters)
        predictions = (a3>0.5)
        return predictions

### gc\_utils.py ###

# -*- coding: utf-8 -*-
    
    import numpy as np
    import matplotlib.pyplot as plt
    
    def sigmoid(x):
        """ Compute the sigmoid of x Arguments: x -- A scalar or numpy array of any size. Return: s -- sigmoid(x) """
        s = 1/(1+np.exp(-x))
        return s
     
    def relu(x):
        """ Compute the relu of x Arguments: x -- A scalar or numpy array of any size. Return: s -- relu(x) """
        s = np.maximum(0,x)
        
        return s
    
    
    
    def dictionary_to_vector(parameters):
        """ Roll all our parameters dictionary into a single vector satisfying our specific required shape. """
        keys = []
        count = 0
        for key in ["W1", "b1", "W2", "b2", "W3", "b3"]:
            
            # flatten parameter
            new_vector = np.reshape(parameters[key], (-1,1))
            keys = keys + [key]*new_vector.shape[0]
            
            if count == 0:
                theta = new_vector
            else:
                theta = np.concatenate((theta, new_vector), axis=0)
            count = count + 1
     
        return theta, keys
     
    def vector_to_dictionary(theta):
        """ Unroll all our parameters dictionary from a single vector satisfying our specific required shape. """
        parameters = { }
        parameters["W1"] = theta[:20].reshape((5,4))
        parameters["b1"] = theta[20:25].reshape((5,1))
        parameters["W2"] = theta[25:40].reshape((3,5))
        parameters["b2"] = theta[40:43].reshape((3,1))
        parameters["W3"] = theta[43:46].reshape((1,3))
        parameters["b3"] = theta[46:47].reshape((1,1))
     
        return parameters
     
    def gradients_to_vector(gradients):
        """ Roll all our gradients dictionary into a single vector satisfying our specific required shape. """
        
        count = 0
        for key in ["dW1", "db1", "dW2", "db2", "dW3", "db3"]:
            # flatten parameter
            new_vector = np.reshape(gradients[key], (-1,1))
            
            if count == 0:
                theta = new_vector
            else:
                theta = np.concatenate((theta, new_vector), axis=0)
            count = count + 1
     
        return theta

[-]: https://blog.csdn.net/u013733326/article/details/79847918
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/b86c5b27f5f5443d8d4d42ad5420b418.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/edbf29feffaf425ebcc0913ef1c14694.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221123/acdacc24c1ef496d816d024a760635b1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221123/7336890da79a42d5af3e54ef1e2a55aa.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 4]: /images/20221123/04768aa7b43d46aeb8b14649fecb4cf4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 5]: /images/20221123/d5bd3a47767543268ec3dba9a6710c14.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 6]: /images/20221123/dab37dd62a0744379ce4d3cdf45a5eb0.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 7]: /images/20221123/4db949fb73684ea780cfa471ef8ccc65.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 8]: /images/20221123/6a085d61fb6b417a93ed9bb73734101a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 9]: /images/20221123/be150df3536240998b8224c743c0c73c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 10]: /images/20221123/2a858a00fdf14e5186ccc1a82bd17e98.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 11]: /images/20221123/469a05f018f34accb9f763abce4d311d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 12]: /images/20221123/d1d5b01096174ffd9de769e1e91b0354.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 13]: /images/20221123/9a50060cf602444e9631019eee445626.png