【吴恩达深度学习编程作业】2.2改善深层神经网络——优化算法实战

傷城~ 2022-11-20 08:22 199阅读 0赞

参考文档：[优化算法实战][Link 1]

代码实现：

1.  分割数据集
2.  优化梯度下降算法：  
    2.1不使用任何优化算法  
    2.2 mini-batch梯度下降法  
    2.3 使用具有动量的梯度下降算法  
    2.4 使用Adam算法

## main.py ##

""" 代码实现： 1. 分割数据集    2. 优化梯度下降算法： 2.1 不使用任何优化算法 2.2 mini-batch梯度下降法 2.3 使用具有动量的梯度下降算法 2.4 使用Adam算法 """
    import numpy as np
    import matplotlib.pyplot as plt
    import scipy.io
    import math
    import sklearn
    import sklearn.datasets
    import Deep_Learning.test2_2.opt_utils
    import Deep_Learning.test2_2.testCase
    
    plt.rcParams['figure.figsize'] = (7.0, 4.0)
    plt.rcParams['image.interpolation'] = 'nearest'
    plt.rcParams['image.cmap'] = 'gray'
    
    
    """ 批量梯度下降 """
    def update_parameters_with_gd(parameters, grads, learning_rate):
        """ 使用梯度下降更新参数 :param parameters: -字典，包含了要更新的参数 :param grads: -字典，包含了每一个梯度值用来更新参数 :param learning_rate: -学习率 :return:parameters -字典，包含了更新后的参数 """
        L = len(parameters) // 2    # 2
    
        # 更新每个参数
        for l in range(L):
            parameters["W" + str(l + 1)] = parameters["W" + str(l + 1)] - learning_rate * grads["dW" + str(l + 1)]
            parameters["b" + str(l + 1)] = parameters["b" + str(l + 1)] - learning_rate * grads["db" + str(l + 1)]
    
        return parameters
    
    # 测试update_parameters_with_gd
    print("=====================测试update_parameters_with_gd======================")
    parameters, grads, learning_rate = Deep_Learning.test2_2.testCase.update_parameters_with_gd_test_case()
    parameters = update_parameters_with_gd(parameters, grads, learning_rate)
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    
    
    """ mini-batch """
    def random_mini_batches(X, Y, mini_batch_size=64, seed=0):
        """ 从（X, Y）中创建一个随机的mini-batch列表 :param X: -输入数据，维度为（输入节点数量，样本的数量） :param Y: -对应的是X的标签，【0 | 1】(蓝 | 红)，维度为（1，样本的数量） :param mini_batch: -每个mini=batch的样本数量 :param seed: -随机种子 :return: mini-batch -一个同步列表，维度为（mini-batch_X, mini_batch_Y） """
        np.random.seed(seed)    # 指定随机种子
        m = X.shape[1]
        mini_batches= []
    
        # 1.打乱顺序
        permutation = list(np.random.permutation(m))    # 返回一个长度为m的随机数组，且里面的数是0到m-1
        shuffled_X = X[:, permutation]  # 将每一列的数据按permutation的顺序来重新排列
        shuffled_Y = Y[:, permutation].reshape((1, m))
    
        # 2.分割
        num_complete_minibatches = math.floor(m / mini_batch_size)     # floor向下取整，分割训练集
        for k in range(0, num_complete_minibatches):
            mini_batch_X = shuffled_X[:, k * mini_batch_size:(k + 1) * mini_batch_size]     # 注意下标索引从0开始，所以是从k*mini_batch_size+1开始
            mini_batch_Y = shuffled_Y[:, k * mini_batch_size:(k + 1) * mini_batch_size]
    
            mini_batch = (mini_batch_X, mini_batch_Y)
            mini_batches.append(mini_batch)
    
        # 如果训练集大小不是mini_batch_size的整数倍，那么最后会剩下一些需要处理
        if m % mini_batch_size != 0:
            # 获取最后剩余的部分
            mini_batch_X = shuffled_X[:, mini_batch_size * num_complete_minibatches:]
            mini_batch_Y = shuffled_Y[:, mini_batch_size * num_complete_minibatches:]
    
            mini_batch = (mini_batch_X, mini_batch_Y)
            mini_batches.append(mini_batch)
    
        return mini_batches
    
    # 测试random_mini_batches
    print("==========================测试random_mini_batches=========================")
    X_assess, Y_assess, mini_batch_size = Deep_Learning.test2_2.testCase.random_mini_batches_test_case()
    mini_batches = random_mini_batches(X_assess, Y_assess, mini_batch_size)
    print("第1个mini_batch_X的维度为：", mini_batches[0][0].shape)
    print("第1个mini_batch_Y的维度为：", mini_batches[0][1].shape)
    print("第2个mini_batch_X的维度为：", mini_batches[1][0].shape)
    print("第2个mini_batch_Y的维度为：", mini_batches[1][1].shape)
    print("第3个mini_batch_X的维度为：", mini_batches[2][0].shape)
    print("第3个mini_batch_Y的维度为：", mini_batches[2][1].shape)
    
    
    
    # 包含动量的梯度下降
    # 初始化
    def initialize_velocity(parameters):
        """ 使用动量可以减少振荡，动量考虑了过去的梯度以平滑更新，把以前梯度的方向存储在变量v中，形式上就是前面梯度的指数加权平均值 初始化速度，velocity是一个字典： - keys:"dW1","db1",...,"dWl","dbL" - values:与相应的梯度/参数维度相同的值为0的矩阵 :param parameters: -字典，包含以下参数： parameters["W" + str(l)] = Wl parameters["b" + str(l)] = bl :return: v -字典，包含以下参数： v["dW" + str(l)] = dWl的速度 v["db" + str(l)] = dbl的速度 """
        L = len(parameters) // 2    # 神经网络层数
        v = { }
    
        for l in range(L):
            v["dW" + str(l + 1)] = np.zeros_like(parameters["W" + str(l + 1)])
            v["db" + str(l + 1)] = np.zeros_like(parameters["b" + str(l + 1)])
    
        return v
    
    # 测试initialize_velocity
    print("====================测试initialize_velocity======================")
    parameters = Deep_Learning.test2_2.testCase.initialize_velocity_test_case()
    v = initialize_velocity(parameters)
    print('v["dW1"] = ' + str(v["dW1"]))
    print('v["db1"] = ' + str(v["db1"]))
    print('v["dW2"] = ' + str(v["dW2"]))
    print('v["db2"] = ' + str(v["db2"]))
    
    def update_parameters_with_momentum(parameters, grads, v, beta, learning_rate):
        """ 使用动量更新参数 :param parameters: -字典 :param grads: -包含梯度值的字典 :param v: -包含当前速度的字典 :param beta: -超参数，动量，实数 :param learning_rate: -学习率，实数 :return: parameters -更新后的字典 v -包含更新后的速度变量 """
        L = len(parameters) // 2
        for l in range(L):
            # 计算速度
            v["dW" + str(l + 1)] = beta * v["dW" + str(l + 1)] + (1 - beta) * grads["dW" + str(l + 1)]
            v["db" + str(l + 1)] = beta * v["db" + str(l + 1)] + (1 - beta) * grads["db" + str(l + 1)]
    
            # 更新参数
            parameters["W" + str(l + 1)] = parameters["W" + str(l + 1)] - learning_rate * v["dW" + str(l + 1)]
            parameters["b" + str(l + 1)] = parameters["b" + str(l + 1)] - learning_rate * v["db" + str(l + 1)]
    
        return parameters, v
    
    # 测试update_parameters_with_momentum
    print("============================测试update_parameters_with_momentum=====================")
    parameters, grads, v = Deep_Learning.test2_2.testCase.update_parameters_with_momentum_test_case()
    update_parameters_with_momentum(parameters, grads, v, beta=0.9, learning_rate=0.01)
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    print('v["dW1"] = ' + str(v["dW1"]))
    print('v["db1"] = ' + str(v["db1"]))
    print('v["dW2"] = ' + str(v["dW2"]))
    print('v["db2"] = ' + str(v["db2"]))
    
    
    
    # Adams算法是RMSprop算法与Momentum算法的结合体
    def initialize_adams(parameters):
        """ 初始化v和s，它们都是字典类型的变量 :param parameters: 字典变量 :return: v -包含梯度的指数加权平均值 s -包含平方梯度的指数加权平均值 """
        L = len(parameters) // 2
        v = { }
        s = { }
    
        for l in range(L):
            v["dW" + str(l + 1)] = np.zeros_like(parameters["W" + str(l + 1)])
            v["db" + str(l + 1)] = np.zeros_like(parameters["b" + str(l + 1)])
    
            s["dW" + str(l + 1)] = np.zeros_like(parameters["W" + str(l + 1)])
            s["db" + str(l + 1)] = np.zeros_like(parameters["b" + str(l + 1)])
    
        return (v, s)
    
    # 测试initialize_adams
    print("======================测试initialize_adams==================")
    parameters = Deep_Learning.test2_2.testCase.initialize_adam_test_case()
    v, s = initialize_adams(parameters)
    print('v["dW1"] = ' + str(v["dW1"]))
    print('v["db1"] = ' + str(v["db1"]))
    print('v["dW2"] = ' + str(v["dW2"]))
    print('v["db2"] = ' + str(v["db2"]))
    print('s["dW1"] = ' + str(s["dW1"]))
    print('s["db1"] = ' + str(s["db1"]))
    print('s["dW2"] = ' + str(s["dW2"]))
    print('s["db2"] = ' + str(s["db2"]))
    
    
    def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate=0.01, beta1=0.9, beta2=0.999, epsilon=1e-8):
        """ 使用Adam更新参数 :param parameters: -字典 :param grads: -包含梯度值的字典 :param v: -Adam的变量，第一个梯度的移动平均值，字典变量 :param s: -Adam的变量，平方梯度的移动平均值，字典变量 :param t: -当前迭代的次数 :param learning_rate: -学习率 :param beta1: -动量，超参数，用于第一阶段，使得曲线的Y不从0开始 :param beta2: -RMSprop的一个参数，超参数 :param epsilon: -防止除0操作 :return:parameters -更新后的参数 v -第一个梯度的移动平均值，字典变量 s -平方梯度的移动平均值，字典变量 """
        L = len(parameters) // 2
        v_corrected = { }    # 偏差修正后的值
        s_corrected = { }    # 偏差修正后的值
    
        for l in range(L):
            # 梯度的移动平均值，输入："v, grads, beta1", 输出："v"
            v["dW" + str(l + 1)] = beta1 * v["dW" + str(l + 1)] + (1 - beta1) * grads["dW" + str(l + 1)]
            v["db" + str(l + 1)] = beta1 * v["db" + str(l + 1)] + (1 - beta1) * grads["db" + str(l + 1)]
    
            # 计算第一阶段的偏差修正后的估计值，输入:"v, beta1, t", 输出:"v_corrected"
            v_corrected["dW" + str(l + 1)] = v["dW" + str(l + 1)] / (1 - np.power(beta1, t))
            v_corrected["db" + str(l + 1)] = v["db" + str(l + 1)] / (1 - np.power(beta1, t))
    
            # 计算平方梯度的移动平均值，输入:"s, grads, beta2" 输出:"s"
            s["dW" + str(l + 1)] = beta2 * s["dW" + str(l + 1)] + (1 - beta2) * np.square(grads["dW" + str(l + 1)])
            s["db" + str(l + 1)] = beta2 * s["db" + str(l + 1)] + (1 - beta2) * np.square(grads["db" + str(l + 1)])
    
            # 计算第二阶段的偏差修正后的估计值，输入:"s, beta2, t", 输出:"s_corrected"
            s_corrected["dW" + str(l + 1)] = s["dW" + str(l + 1)] / (1 - np.power(beta2, t))
            s_corrected["db" + str(l + 1)] = s["db" + str(l + 1)] / (1 - np.power(beta2, t))
    
            # 更新参数，输入:"parameters, learning_rate, v_corrected, s_corrected, epsilon" 输出:"parameters"
            parameters["W" + str(l + 1)] = parameters["W" + str(l + 1)] - learning_rate * (v_corrected["dW" + str(l + 1)] / np.sqrt(s_corrected["dW" + str(l + 1)] + epsilon))
            parameters["b" + str(l + 1)] = parameters["b" + str(l + 1)] - learning_rate * (v_corrected["db" + str(l + 1)] / np.sqrt(s_corrected["db" + str(l + 1)] + epsilon))
    
        return (parameters, v, s)
    
    # 测试update_parameters_with_adam
    print("=============================测试update_parameters_with_adam=========================")
    parameters, grads, v, s, = Deep_Learning.test2_2.testCase.update_parameters_with_adam_test_case()
    update_parameters_with_adam(parameters, grads, v, s, t = 2)
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))
    print('v["dW1"] = ' + str(v["dW1"]))
    print('v["db1"] = ' + str(v["db1"]))
    print('v["dW2"] = ' + str(v["dW2"]))
    print('v["db2"] = ' + str(v["db2"]))
    print('s["dW1"] = ' + str(s["dW1"]))
    print('s["db1"] = ' + str(s["db1"]))
    print('s["dW2"] = ' + str(s["dW2"]))
    print('s["db2"] = ' + str(s["db2"]))
    
    
    
    # 加载数据集
    train_X, train_Y = Deep_Learning.test2_2.opt_utils.load_dataset(is_plot=True)
    plt.show()
    
    
    # 定义模型
    def model(X, Y, layers_dims, optimizer, learning_rate=0.0007,
              mini_batch_size=64, beta=0.9, beta1=0.9, beta2=0.999,
              epsilon=1e-8, num_epochs=10000, print_cost=True, is_plot=True):
        """ 可以运行在不同的优化器下的三层神经网络模型 :param X: -输入数据，维度为（2，输入的数据集里面样本数量） :param Y: -与X对应的标签 :param layers_dims: -包含层数和节点数量的列表 :param optimizer: -字符串类型的参数，用于选择优化类型，【"gd" | "momentum | "adam"】 :param learning_rate: -学习率 :param mini_batch_size: -每个小批量数据集的大小 :param beta: -用于动量优化的一个超参数 :param beta1: -用于计算梯度后的指数衰减的估计的超参数 :param beta2: -用于计算平方梯度后的指数衰减的估计的超参数 :param epsilon: -用于在Adam中避免除零操作的超参数，一般不更改 :param num_epochs: -整个训练集的遍历次数，相当于num_iteration :param print_cost: -是否打印误差值，每1000次数据集打印一次，但是每100次记录一个误差值，又称每1000代打印一次 :param is_plot: -是否绘制曲线 :return: parameters -包含了学习后的参数 """
        L = len(layers_dims)
        costs = []
        t = 0   # 每学习完一个mini-batch就增加1
        seed = 10   # 随机种子
        
        # 初始化参数
        parameters = Deep_Learning.test2_2.opt_utils.initialize_parameters(layers_dims)
        
        # 选择优化器
        if optimizer == "gd":
            pass    # 不使用任何优化器，直接使用梯度下降法
        elif optimizer == "momentum":
            v = initialize_velocity(parameters)     # 使用动量
        elif optimizer == "adam":
            v, s = initialize_adams(parameters)     # 使用Adam优化
        else:
            print("optimizer参数错误，程序退出")
            exit(1)
    
        # 开始学习
        for i in range(num_epochs):
            # 定义随机mini-batches,在每次遍历数据集之后增加种子以重新排列数据集，使每次数据的顺序都不同
            seed = seed + 1
            minibatches = random_mini_batches(X, Y, mini_batch_size, seed)
    
            for minibatch in minibatches:
                # 选择一个minibatch
                (minibatch_X, minibatch_Y) = minibatch
    
                # 前向传播
                A3, cache = Deep_Learning.test2_2.opt_utils.forward_propagation(minibatch_X, parameters)
    
                # 计算误差
                cost = Deep_Learning.test2_2.opt_utils.compute_cost(A3, minibatch_Y)
    
                # 反向传播
                grads = Deep_Learning.test2_2.opt_utils.backward_propagation(minibatch_X, minibatch_Y, cache)
    
                # 更新参数
                if optimizer == "gd":
                    parameters = update_parameters_with_gd(parameters, grads, learning_rate)
                elif optimizer == "momentum":
                    parameters, v = update_parameters_with_momentum(parameters, grads, v, beta, learning_rate)
                elif optimizer == "adam":
                    t = t + 1
                    parameters, v, s, = update_parameters_with_adam(parameters, grads, v, s, t, learning_rate, beta1, beta2, epsilon)
    
            # 记录误差值
            if i % 100 == 0:
                costs.append(cost)
                # 是否打印误差值
                if print_cost and i % 1000 == 0:
                    print("第" + str(i) + "次遍历整个数据集，当前的误差值：" + str(cost))
    
        # 是否绘制曲线
        if is_plot:
            plt.plot(costs)
            plt.ylabel('cost')
            plt.xlabel('epochs (per 100)')
            plt.title("Learning rate = " + str(learning_rate))
            plt.show()
    
        return parameters
    
    
    
    """ 梯度下降测试 """
    layers_dims = [train_X.shape[0], 5, 2, 1]
    
    # 1.使用普通的梯度下降
    parameters = model(train_X, train_Y, layers_dims, optimizer="gd")
    
    # 2.具有动量的梯度下降
    # parameters = model(train_X, train_Y, layers_dims, beta=0.9, optimizer="momentum")
    
    # 3.Adam优化后的梯度下降
    # parameters = model(train_X, train_Y, layers_dims, optimizer="adam")
    
    # 预测
    predictions = Deep_Learning.test2_2.opt_utils.predict(train_X, train_Y, parameters)
    # 绘制决策边界
    plt.title("Model with Gradient Descent optimization")
    axes = plt.gca()
    axes.set_xlim([-1.5, 2.5])
    axes.set_ylim([-1, 1.5])
    Deep_Learning.test2_2.opt_utils.plot_decision_boundary(lambda x:Deep_Learning.test2_2.opt_utils.predict_dec(parameters, x.T), train_X, train_Y)
    
    """ 总结： 具有动量的梯度下降通过有很好的效果，但是由于小的学习速率和简单的数据集所以它的影响是轻微的。 Adam明显优于小批量梯度下降和具有动量的梯度下降，如果在这个简单的模型上运行更多的时间的数据集，这三种方法结果都会很好，但是Adamm明显收敛更快 """

## opt\_utils.py ##

# -*- coding: utf-8 -*-
    
    import numpy as np
    import matplotlib.pyplot as plt
    import sklearn
    import sklearn.datasets
    
    def sigmoid(x):
        """ Compute the sigmoid of x Arguments: x -- A scalar or numpy array of any size. Return: s -- sigmoid(x) """
        s = 1/(1+np.exp(-x))
        return s
     
    def relu(x):
        """ Compute the relu of x Arguments: x -- A scalar or numpy array of any size. Return: s -- relu(x) """
        s = np.maximum(0,x)
        
        return s
    
    
    def load_params_and_grads(seed=1):
        np.random.seed(seed)
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
     
        dW1 = np.random.randn(2,3)
        db1 = np.random.randn(2,1)
        dW2 = np.random.randn(3,3)
        db2 = np.random.randn(3,1)
        
        return W1, b1, W2, b2, dW1, db1, dW2, db2
        
    def initialize_parameters(layer_dims):
        """ Arguments: layer_dims -- python array (list) containing the dimensions of each layer in our network Returns: parameters -- python dictionary containing your parameters "W1", "b1", ..., "WL", "bL": W1 -- weight matrix of shape (layer_dims[l], layer_dims[l-1]) b1 -- bias vector of shape (layer_dims[l], 1) Wl -- weight matrix of shape (layer_dims[l-1], layer_dims[l]) bl -- bias vector of shape (1, layer_dims[l]) Tips: - For example: the layer_dims for the "Planar Data classification model" would have been [2,2,1]. This means W1's shape was (2,2), b1 was (1,2), W2 was (2,1) and b2 was (1,1). Now you have to generalize it! - In the for loop, use parameters['W' + str(l)] to access Wl, where l is the iterative integer. """
        
        np.random.seed(3)
        parameters = { }
        L = len(layer_dims) # number of layers in the network
     
        for l in range(1, L):
            parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1])*  np.sqrt(2 / layer_dims[l-1])
            parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))
            
            # assert(parameters['W' + str(l)].shape == layer_dims[l], layer_dims[l-1])
            # assert(parameters['W' + str(l)].shape == layer_dims[l], 1)
            
        return parameters
        
    def forward_propagation(X, parameters):
        """ Implements the forward propagation (and computes the loss) presented in Figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) parameters -- python dictionary containing your parameters "W1", "b1", "W2", "b2", "W3", "b3": W1 -- weight matrix of shape () b1 -- bias vector of shape () W2 -- weight matrix of shape () b2 -- bias vector of shape () W3 -- weight matrix of shape () b3 -- bias vector of shape () Returns: loss -- the loss function (vanilla logistic loss) """
        
        # retrieve parameters
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]
        W3 = parameters["W3"]
        b3 = parameters["b3"]
        
        # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
        z1 = np.dot(W1, X) + b1
        a1 = relu(z1)
        z2 = np.dot(W2, a1) + b2
        a2 = relu(z2)
        z3 = np.dot(W3, a2) + b3
        a3 = sigmoid(z3)
        
        cache = (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3)
        
        return a3, cache
     
    def backward_propagation(X, Y, cache):
        """ Implement the backward propagation presented in figure 2. Arguments: X -- input dataset, of shape (input size, number of examples) Y -- true "label" vector (containing 0 if cat, 1 if non-cat) cache -- cache output from forward_propagation() Returns: gradients -- A dictionary with the gradients with respect to each parameter, activation and pre-activation variables """
        m = X.shape[1]
        (z1, a1, W1, b1, z2, a2, W2, b2, z3, a3, W3, b3) = cache
        
        dz3 = 1./m * (a3 - Y)
        dW3 = np.dot(dz3, a2.T)
        db3 = np.sum(dz3, axis=1, keepdims = True)
        
        da2 = np.dot(W3.T, dz3)
        dz2 = np.multiply(da2, np.int64(a2 > 0))
        dW2 = np.dot(dz2, a1.T)
        db2 = np.sum(dz2, axis=1, keepdims = True)
        
        da1 = np.dot(W2.T, dz2)
        dz1 = np.multiply(da1, np.int64(a1 > 0))
        dW1 = np.dot(dz1, X.T)
        db1 = np.sum(dz1, axis=1, keepdims = True)
        
        gradients = { "dz3": dz3, "dW3": dW3, "db3": db3,
                     "da2": da2, "dz2": dz2, "dW2": dW2, "db2": db2,
                     "da1": da1, "dz1": dz1, "dW1": dW1, "db1": db1}
        
        return gradients
     
    def compute_cost(a3, Y):
        
        """ Implement the cost function Arguments: a3 -- post-activation, output of forward propagation Y -- "true" labels vector, same shape as a3 Returns: cost - value of the cost function """
        m = Y.shape[1]
        
        logprobs = np.multiply(-np.log(a3),Y) + np.multiply(-np.log(1 - a3), 1 - Y)
        cost = 1./m * np.sum(logprobs)
        
        return cost
     
    def predict(X, y, parameters):
        """ This function is used to predict the results of a n-layer neural network. Arguments: X -- data set of examples you would like to label parameters -- parameters of the trained model Returns: p -- predictions for the given dataset X """
        
        m = X.shape[1]
        p = np.zeros((1,m), dtype = np.int)
        
        # Forward propagation
        a3, caches = forward_propagation(X, parameters)
        
        # convert probas to 0/1 predictions
        for i in range(0, a3.shape[1]):
            if a3[0,i] > 0.5:
                p[0,i] = 1
            else:
                p[0,i] = 0
     
        # print results
     
        #print ("predictions: " + str(p[0,:]))
        #print ("true labels: " + str(y[0,:]))
        print("Accuracy: "  + str(np.mean((p[0,:] == y[0,:]))))
        
        return p
     
    def predict_dec(parameters, X):
        """ Used for plotting decision boundary. Arguments: parameters -- python dictionary containing your parameters X -- input data of size (m, K) Returns predictions -- vector of predictions of our model (red: 0 / blue: 1) """
        
        # Predict using forward propagation and a classification threshold of 0.5
        a3, cache = forward_propagation(X, parameters)
        predictions = (a3 > 0.5)
        return predictions
     
    def plot_decision_boundary(model, X, y):
        # Set min and max values and give it some padding
        x_min, x_max = X[0, :].min() - 1, X[0, :].max() + 1
        y_min, y_max = X[1, :].min() - 1, X[1, :].max() + 1
        h = 0.01
        # Generate a grid of points with distance h between them
        xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
        # Predict the function value for the whole grid
        Z = model(np.c_[xx.ravel(), yy.ravel()])
        Z = Z.reshape(xx.shape)
        # Plot the contour and training examples
        plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
        plt.ylabel('x2')
        plt.xlabel('x1')
        plt.scatter(X[0, :], X[1, :], c=y, cmap=plt.cm.Spectral)
        plt.show()
     
    def load_dataset(is_plot = True):
        np.random.seed(3)
        train_X, train_Y = sklearn.datasets.make_moons(n_samples=300, noise=.2) #300 #0.2 
        # Visualize the data
        if is_plot:
            plt.scatter(train_X[:, 0], train_X[:, 1], c=train_Y, s=40, cmap=plt.cm.Spectral);
        train_X = train_X.T
        train_Y = train_Y.reshape((1, train_Y.shape[0]))
        
        return train_X, train_Y

## testCase.py ##

# -*- coding: utf-8 -*-
    
    import numpy as np
    
    def update_parameters_with_gd_test_case():
        np.random.seed(1)
        learning_rate = 0.01
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
    
        dW1 = np.random.randn(2,3)
        db1 = np.random.randn(2,1)
        dW2 = np.random.randn(3,3)
        db2 = np.random.randn(3,1)
        
        parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2}
        grads = { "dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2}
        
        return parameters, grads, learning_rate
    
    """ def update_parameters_with_sgd_checker(function, inputs, outputs): if function(inputs) == outputs: print("Correct") else: print("Incorrect") """
    
    def random_mini_batches_test_case():
        np.random.seed(1)
        mini_batch_size = 64
        X = np.random.randn(12288, 148)
        Y = np.random.randn(1, 148) < 0.5
        return X, Y, mini_batch_size
    
    def initialize_velocity_test_case():
        np.random.seed(1)
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
        parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2}
        return parameters
    
    def update_parameters_with_momentum_test_case():
        np.random.seed(1)
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
    
        dW1 = np.random.randn(2,3)
        db1 = np.random.randn(2,1)
        dW2 = np.random.randn(3,3)
        db2 = np.random.randn(3,1)
        parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2}
        grads = { "dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2}
        v = { 'dW1': np.array([[ 0.,  0.,  0.],
            [ 0.,  0.,  0.]]), 'dW2': np.array([[ 0.,  0.,  0.],
            [ 0.,  0.,  0.],
            [ 0.,  0.,  0.]]), 'db1': np.array([[ 0.],
            [ 0.]]), 'db2': np.array([[ 0.],
            [ 0.],
            [ 0.]])}
        return parameters, grads, v
        
    def initialize_adam_test_case():
        np.random.seed(1)
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
        parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2}
        return parameters
    
    def update_parameters_with_adam_test_case():
        np.random.seed(1)
        v, s = ({ 'dW1': np.array([[ 0.,  0.,  0.],
             [ 0.,  0.,  0.]]), 'dW2': np.array([[ 0.,  0.,  0.],
             [ 0.,  0.,  0.],
             [ 0.,  0.,  0.]]), 'db1': np.array([[ 0.],
             [ 0.]]), 'db2': np.array([[ 0.],
             [ 0.],
             [ 0.]])}, { 'dW1': np.array([[ 0.,  0.,  0.],
             [ 0.,  0.,  0.]]), 'dW2': np.array([[ 0.,  0.,  0.],
             [ 0.,  0.,  0.],
             [ 0.,  0.,  0.]]), 'db1': np.array([[ 0.],
             [ 0.]]), 'db2': np.array([[ 0.],
             [ 0.],
             [ 0.]])})
        W1 = np.random.randn(2,3)
        b1 = np.random.randn(2,1)
        W2 = np.random.randn(3,3)
        b2 = np.random.randn(3,1)
    
        dW1 = np.random.randn(2,3)
        db1 = np.random.randn(2,1)
        dW2 = np.random.randn(3,3)
        db2 = np.random.randn(3,1)
        
        parameters = { "W1": W1, "b1": b1, "W2": W2, "b2": b2}
        grads = { "dW1": dW1, "db1": db1, "dW2": dW2, "db2": db2}
        
        return parameters, grads, v, s

## 初始化测试结果 ##

原始数据集：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center]

=====================测试update_parameters_with_gd======================
    W1 = [[ 1.63535156 -0.62320365 -0.53718766]
     [-1.07799357  0.85639907 -2.29470142]]
    b1 = [[ 1.74604067]
     [-0.75184921]]
    W2 = [[ 0.32171798 -0.25467393  1.46902454]
     [-2.05617317 -0.31554548 -0.3756023 ]
     [ 1.1404819  -1.09976462 -0.1612551 ]]
    b2 = [[-0.88020257]
     [ 0.02561572]
     [ 0.57539477]]
    ==========================测试random_mini_batches=========================
    第1个mini_batch_X的维度为： (12288, 64)
    第1个mini_batch_Y的维度为： (1, 64)
    第2个mini_batch_X的维度为： (12288, 64)
    第2个mini_batch_Y的维度为： (1, 64)
    第3个mini_batch_X的维度为： (12288, 20)
    第3个mini_batch_Y的维度为： (1, 20)
    ====================测试initialize_velocity======================
    v["dW1"] = [[0. 0. 0.]
     [0. 0. 0.]]
    v["db1"] = [[0.]
     [0.]]
    v["dW2"] = [[0. 0. 0.]
     [0. 0. 0.]
     [0. 0. 0.]]
    v["db2"] = [[0.]
     [0.]
     [0.]]
    ============================测试update_parameters_with_momentum=====================
    W1 = [[ 1.62544598 -0.61290114 -0.52907334]
     [-1.07347112  0.86450677 -2.30085497]]
    b1 = [[ 1.74493465]
     [-0.76027113]]
    W2 = [[ 0.31930698 -0.24990073  1.4627996 ]
     [-2.05974396 -0.32173003 -0.38320915]
     [ 1.13444069 -1.0998786  -0.1713109 ]]
    b2 = [[-0.87809283]
     [ 0.04055394]
     [ 0.58207317]]
    v["dW1"] = [[-0.11006192  0.11447237  0.09015907]
     [ 0.05024943  0.09008559 -0.06837279]]
    v["db1"] = [[-0.01228902]
     [-0.09357694]]
    v["dW2"] = [[-0.02678881  0.05303555 -0.06916608]
     [-0.03967535 -0.06871727 -0.08452056]
     [-0.06712461 -0.00126646 -0.11173103]]
    v["db2"] = [[0.02344157]
     [0.16598022]
     [0.07420442]]
    ======================测试initialize_adams==================
    v["dW1"] = [[0. 0. 0.]
     [0. 0. 0.]]
    v["db1"] = [[0.]
     [0.]]
    v["dW2"] = [[0. 0. 0.]
     [0. 0. 0.]
     [0. 0. 0.]]
    v["db2"] = [[0.]
     [0.]
     [0.]]
    s["dW1"] = [[0. 0. 0.]
     [0. 0. 0.]]
    s["db1"] = [[0.]
     [0.]]
    s["dW2"] = [[0. 0. 0.]
     [0. 0. 0.]
     [0. 0. 0.]]
    s["db2"] = [[0.]
     [0.]
     [0.]]
    =============================测试update_parameters_with_adam=========================
    W1 = [[ 1.63178673 -0.61919778 -0.53561312]
     [-1.08040999  0.85796626 -2.29409733]]
    b1 = [[ 1.75225313]
     [-0.75376553]]
    W2 = [[ 0.32648046 -0.25681174  1.46954931]
     [-2.05269934 -0.31497584 -0.37661299]
     [ 1.14121081 -1.09245036 -0.16498684]]
    b2 = [[-0.88529978]
     [ 0.03477238]
     [ 0.57537385]]
    v["dW1"] = [[-0.11006192  0.11447237  0.09015907]
     [ 0.05024943  0.09008559 -0.06837279]]
    v["db1"] = [[-0.01228902]
     [-0.09357694]]
    v["dW2"] = [[-0.02678881  0.05303555 -0.06916608]
     [-0.03967535 -0.06871727 -0.08452056]
     [-0.06712461 -0.00126646 -0.11173103]]
    v["db2"] = [[0.02344157]
     [0.16598022]
     [0.07420442]]
    s["dW1"] = [[0.00121136 0.00131039 0.00081287]
     [0.0002525  0.00081154 0.00046748]]
    s["db1"] = [[1.51020075e-05]
     [8.75664434e-04]]
    s["dW2"] = [[7.17640232e-05 2.81276921e-04 4.78394595e-04]
     [1.57413361e-04 4.72206320e-04 7.14372576e-04]
     [4.50571368e-04 1.60392066e-07 1.24838242e-03]]
    s["db2"] = [[5.49507194e-05]
     [2.75494327e-03]
     [5.50629536e-04]]

## 2.1不使用任何优化算法的运行结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 1]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 2]

第0次遍历整个数据集，当前的误差值：0.690735512291113
    第1000次遍历整个数据集，当前的误差值：0.6852725328458241
    第2000次遍历整个数据集，当前的误差值：0.6470722240719002
    第3000次遍历整个数据集，当前的误差值：0.6195245549970402
    第4000次遍历整个数据集，当前的误差值：0.5765844355950943
    第5000次遍历整个数据集，当前的误差值：0.6072426395968575
    第6000次遍历整个数据集，当前的误差值：0.5294033317684576
    第7000次遍历整个数据集，当前的误差值：0.46076823985930115
    第8000次遍历整个数据集，当前的误差值：0.465586082399045
    第9000次遍历整个数据集，当前的误差值：0.4645179722167684
    Accuracy: 0.7966666666666666

## 2.3使用具有动量的梯度下降算法的运行结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 3]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 4]

第0次遍历整个数据集，当前的误差值：0.6907412988351506
    第1000次遍历整个数据集，当前的误差值：0.6853405261267578
    第2000次遍历整个数据集，当前的误差值：0.6471448370095255
    第3000次遍历整个数据集，当前的误差值：0.6195943032076022
    第4000次遍历整个数据集，当前的误差值：0.5766650344073023
    第5000次遍历整个数据集，当前的误差值：0.607323821900647
    第6000次遍历整个数据集，当前的误差值：0.5294761758786997
    第7000次遍历整个数据集，当前的误差值：0.46093619004872366
    第8000次遍历整个数据集，当前的误差值：0.465780093701272
    第9000次遍历整个数据集，当前的误差值：0.4647395967922748
    Accuracy: 0.7966666666666666

## 2.4使用Adam算法的运行结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 5]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 6]

第0次遍历整个数据集，当前的误差值：0.6905522446113365
    第1000次遍历整个数据集，当前的误差值：0.18550136438550577
    第2000次遍历整个数据集，当前的误差值：0.1508304657525321
    第3000次遍历整个数据集，当前的误差值：0.0744543857099718
    第4000次遍历整个数据集，当前的误差值：0.1259591565133716
    第5000次遍历整个数据集，当前的误差值：0.10434443534245483
    第6000次遍历整个数据集，当前的误差值：0.10067637504120651
    第7000次遍历整个数据集，当前的误差值：0.031652030135115625
    第8000次遍历整个数据集，当前的误差值：0.11197273131244206
    第9000次遍历整个数据集，当前的误差值：0.1979400715246548
    Accuracy: 0.94

## 总结： ##

*  具有动量的梯度下降通过有很好的效果，但是由于小的学习速率和简单的数据集所以它的影响是轻微的。
 *  Adam明显优于小批量梯度下降和具有动量的梯度下降，如果在这个简单的模型上运行更多的时间的数据集，这三种方法结果都会很好，但是Adamm明显收敛更快

[Link 1]: https://blog.csdn.net/u013733326/article/details/79907419
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center]: /images/20221120/d8b2c4f36d2a4e5bb520b69979dda501.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 1]: https://img-blog.csdnimg.cn/20201027210834474.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 2]: https://img-blog.csdnimg.cn/20201027210842801.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 3]: https://img-blog.csdnimg.cn/20201027211405748.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 4]: https://img-blog.csdnimg.cn/20201027211405768.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 5]: https://img-blog.csdnimg.cn/20201027212431197.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8_size_16_color_FFFFFF_t_70_pic_center 6]: https://img-blog.csdnimg.cn/20201027212439900.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlaXZlbl8=,size_16,color_FFFFFF,t_70#pic_center