《利用Python 进行数据分析》第四章：Numpy基础

小咪咪 2022-12-06 15:12 257阅读 0赞

对《利用Python 进行数据分析》（Wes Mckinney著）一书中的第四章中Numpy基础进行代码实验。原书中采用的是Python2.7，而我采用的Python3.7在Pycharm调试的，因此对书中源代码进行了一定的修改，每步打印结果（除“随机”相关外）与原文校验对照一致（输出结果在注释中，简单的输出就没写结果），全手工敲写，供参考。  
       Pdf文档和数据集参见：[《利用Python 进行数据分析》第二章：引言中的分析代码（含pdf和数据集下载链接）][Python _pdf]

###     Numpy基础： ###

*  1、Numpy的ndarray:一种多维数组对象
 *   *  1.1 ndarray数据类型
     *  1.2 数组和标量之间的运算
     *  1.3 基本的索引和切片
     *  1.4 切片索引
     *  1.5 布尔型索引
     *  1.6 花式索引
     *  1.7 数组转置和轴转换
 *  2、通用函数：快速的元素级数组函数
 *   *  2.1 利用数组进行数据处理
     *  2.2 将条件逻辑表述为数组运算
     *  2.3 数学和统计方法
     *  2.4 用于布尔数组的方法
     *  2.5 排序
     *  2.6 唯一以及其他集合逻辑
 *  3、用于数组的文件输入输出
 *   *  3.1 将数组以二进制格式保存到磁盘
     *  3.2 存取文本文件
 *  4、线性代数
 *  5、随机数生成
 *  6、范例：随机漫步

因为代码过长，放在一个代码段中显得冗长，因此进行了拆分，如下的库引入每个代码段中均可能有必要引入。

# -*- coding:utf-8 -*-
    
    import numpy as np
    from numpy.random import randn

# 1、Numpy的ndarray:一种多维数组对象 #

data = [[0.9526, -0.246, -0.8856],
            [0.5639, 0.22379, 0.9104]]
    
    data = np.array(data)
    print(data)
    print(data * 10)
    print(data + data)
    print(data.shape)  # 数据维度
    print(data.dtype)  # 数据类型
    
    data1 = [6, 7.5, 8, 0, 1]
    arr1 = np.array(data1)
    print(arr1) # [6. 7.5 8. 0. 1. ]
    
    data2 = [[1, 2, 3, 4], [5, 6, 7, 8]]
    arr2 = np.array(data2)
    print(arr2)
    print(arr2.ndim)  # ->2
    print(arr2.shape)  # (2, 4)
    data1 = [6, 7.5, 8, 0, 1]
    arr1 = np.array(data1)
    print(arr1.dtype)  # 数据类型为浮点型float64
    print(arr2.dtype)  # 数据类型为整型int32
    
    print(np.zeros(10))  # [0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
    print(np.zeros((3, 6)))  # 不要写成np.zeros(3,6)
    '''[[0. 0. 0. 0. 0. 0.] [0. 0. 0. 0. 0. 0.] [0. 0. 0. 0. 0. 0.]]'''
    print(np.empty((2, 3, 2)))  # 返回未初始化的垃圾值
    '''[[[6.23042070e-307 3.56043053e-307] [1.60219306e-306 2.44763557e-307] [1.69119330e-306 1.33514617e-307]] [[1.42417221e-306 1.37961641e-306] [1.27945651e-307 8.01097889e-307] [1.20161526e-306 1.42410974e-306]]]'''
    
    print(np.arange(15))  # [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14]

## 1.1 ndarray数据类型 ##

arr1 = np.array([1, 2, 3], dtype=np.float64)
    arr2 = np.array([1, 2, 3], dtype=np.int32)
    print(arr1.dtype)  # float64
    print(arr2.dtype)  # int32
    
    arr = np.array([1, 2, 3, 4, 5])
    print(arr.dtype)  # int32
    float_arr = arr.astype(np.float64)
    print(float_arr.dtype)  # float64
    arr = np.array([3.7, -1.2, -2.6, 0.5, 12.9, 10.1])
    print(arr)
    print(arr.astype(np.int32))  # [ 3 -1 -2 0 12 10]
    
    numeric_strings = np.array(["1.25", "-9.6", "42"], dtype=np.string_)
    print(numeric_strings.astype(float))  # [ 1.25 -9.6 42. ]
    
    empty_unit32 = np.empty(8, dtype='u4')
    print(empty_unit32)  # [1 2 3 4 5 6 7 8] 与原文结果不一致？？原文为随机出来的值

## 1.2 数组和标量之间的运算 ##

arr = np.array([[1., 2., 3.], [4., 5., 6.]])
    print(arr)
    print(arr * arr)
    '''[[ 1. 4. 9.] [16. 25. 36.]]'''
    print(arr - arr)
    print(1 / arr)
    '''[[1. 0.5 0.33333333] [0.25 0.2 0.16666667]]'''
    print(arr ** 0.5)
    '''[[1. 1.41421356 1.73205081] [2. 2.23606798 2.44948974]]'''

## 1.3 基本的索引和切片 ##

arr = np.arange(10)
    print(arr[5])
    print(arr[5:8])
    arr[5:8] = 12  # 更改numpy数组中的元素
    print(arr)  # 由于广播机制 输出->[ 0 1 2 3 4 12 12 12 8 9]
    
    arr_slice = arr[5:8]
    arr_slice[1] = 12345
    print(arr)  # [0 1 2 3 4 12 12345 12 8 9]
    arr_slice[:] = 64
    print(arr)  # [ 0 1 2 3 4 64 64 64 8 9]
    
    arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    print(arr2d[2])  # [7 8 9]
    print(arr2d[0][2])  # 3
    print(arr2d[0, 2])  # 3
    
    arr3d = np.array([[[1, 2, 3], [4, 5, 6]],
                      [[7, 8, 9], [10, 11, 12]]])
    print(arr3d)
    print(arr3d.shape)  # (2, 2, 3)
    print(arr3d[0])
    old_values = arr3d[0].copy()
    arr3d[0] = 42
    print(arr3d)
    '''[[[42 42 42] [42 42 42]] [[ 7 8 9] [10 11 12]]]'''
    
    arr3d[0] = old_values
    print(arr3d)

## 1.4 切片索引 ##

# print(arr) #[ 0 1 2 3 4 64 64 64 8 9] (arr值参见“基本的索引和切片”中）
    # arr2d = np.array([[1,2,3],[4,5,6],[7,8,9]])
    print(arr[1:6])  # ->[ 1 2 3 4 64]
    print(arr2d[:2])  # ->[[1 2 3], [4 5 6]]
    print(arr2d[:2, 1:])  # -> [[2 3], [5 6]]
    print(arr2d[2, :1])  # -> 7
    print(arr2d[1, :2])  # ->[4,5]
    print(arr2d[:, :1])  # 冒号表示整轴选取 ->[[1],[4],[7]]
    arr2d[:2, 1:] = 0
    print(arr2d)  # ->[[1 0 0], [4 0 0], [7 8 9]]

## 1.5 布尔型索引 ##

names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
    data = randn(7, 4)
    print(data)
    '''[[ 0.81586467 0.22772329 -1.0415652 -0.3245785 ] [-1.2945232 0.53349873 -0.75547509 -1.43954103] [-1.26108396 0.67093283 0.31324912 0.30667379] [-0.20715904 0.46115929 -0.99327577 -0.81750551] [ 0.27188598 0.50276487 0.33443927 1.19355676] [ 0.57509673 -0.1303353 -1.26974167 -1.24611761] [-0.61857004 -0.14564834 1.54315148 -0.43394236]]'''
    print(names == 'Bob')  # ->[ True False False True False False False]
    print(data[names == 'Bob'])
    '''[[ 0.81586467 0.22772329 -1.0415652 -0.3245785 ] [-0.20715904 0.46115929 -0.99327577 -0.81750551]]'''
    print(data[names == 'Bob', 2:])
    '''[[-1.0415652 -0.3245785 ] [-0.99327577 -0.81750551]]'''
    
    print(data[names == 'Bob', 3])  # [-0.3245785 -0.81750551]
    print(names != 'Bob')  # [False True True False True True True]
    print(data[~(names == 'Bob')])  # python3中使用“~”而不是“-”，与！=效果一样
    '''[[-1.2945232 0.53349873 -0.75547509 -1.43954103] [-1.26108396 0.67093283 0.31324912 0.30667379] [ 0.27188598 0.50276487 0.33443927 1.19355676] [ 0.57509673 -0.1303353 -1.26974167 -1.24611761] [-0.61857004 -0.14564834 1.54315148 -0.43394236]]'''
    mask = (names == 'Bob') | (names == 'Will')
    print(mask)  # [ True False True True True False False]
    print(data[mask])
    '''[[ 0.81586467, 0.22772329, -1.0415652 , -0.3245785 ], [-1.26108396, 0.67093283, 0.31324912, 0.30667379], [-0.20715904, 0.46115929, -0.99327577, -0.81750551], [ 0.27188598, 0.50276487, 0.33443927, 1.19355676]]'''
    data[data < 0] = 0  # 将所有负值设为0
    print(data)
    ''' [[ 0.81586467 0.22772329 0. 0. ] [0. 0.53349873 0. 0. ] [0. 0.67093283 0.31324912 0.30667379] [0. 0.46115929 0. 0. ] [ 0.27188598 0.50276487 0.33443927 1.19355676] [ 0.57509673 0. 0. 0. ] [0. 0. 1.54315148 0. ]]'''
    
    # 通过一维布尔数组设置整行或整列
    data[names != 'Joe'] = 7  
    print(data)
    ''' [[7. 7. 7. 7. ] [0. 0.53349873 0. 0. ] [7. 7. 7. 7. ] [7. 7. 7. 7. ] [7. 7. 7. 7. ] [ 0.57509673 0. 0. 0. ] [0. 0. 1.54315148 0. ]]'''

## 1.6 花式索引 ##

arr = np.empty((8,4))
    for i in range(8):
        arr[i] = i
    print(arr)
    '''[[0. 0. 0. 0.] [1. 1. 1. 1.] [2. 2. 2. 2.] [3. 3. 3. 3.] [4. 4. 4. 4.] [5. 5. 5. 5.] [6. 6. 6. 6.] [7. 7. 7. 7.]]'''
    
    # 按特定顺序选取子集
    print(arr[[4,3,0,6]]) 
    '''[[4. 4. 4. 4.] [3. 3. 3. 3.] [0. 0. 0. 0.] [6. 6. 6. 6.]]'''
     
    # 加负号后会从后面选取
    print(arr[[-3,-5,-7]]) 
    '''[[5. 5. 5. 5.] [3. 3. 3. 3.] [1. 1. 1. 1.]]'''
    arr = np.arange(32).reshape((8,4))
    print(arr)
    '''[[ 0 1 2 3] [ 4 5 6 7] [ 8 9 10 11] [12 13 14 15] [16 17 18 19] [20 21 22 23] [24 25 26 27] [28 29 30 31]]'''
    print(arr[[1,5,7,2],[0,3,1,2]]) # [ 4 23 29 10]
    print(arr[[1,5,7,2]][:,[0,3,1,2]])
    '''[[ 4 7 5 6] [20 23 21 22] [28 31 29 30] [ 8 11 9 10]]'''
    print(arr[np.ix_([1,5,7,2], [0,3,1,2])])
    '''[[ 4 7 5 6] [20 23 21 22] [28 31 29 30] [ 8 11 9 10]]'''

## 1.7 数组转置和轴转换 ##

arr = np.arange(15).reshape((3,5))
    print(arr)
    '''[[ 0 1 2 3 4] [ 5 6 7 8 9] [10 11 12 13 14]]'''
    print(arr.T) #数组转置
    '''[[ 0 5 10] [ 1 6 11] [ 2 7 12] [ 3 8 13] [ 4 9 14]]'''
    
    arr = np.random.randn(6,3)
    print(np.dot(arr.T, arr))  # 内积计算
    
    arr = np.arange(16).reshape((2,2,4))
    print(arr)
    '''[[[ 0 1 2 3] [ 4 5 6 7]] [[ 8 9 10 11] [12 13 14 15]]]'''
    
    #高维数组需要一个轴编号才能转置(！！没看懂）
    print(arr.transpose((1,0,2))) 
    '''[[[ 0 1 2 3] [ 8 9 10 11]] [[ 4 5 6 7] [12 13 14 15]]]'''
    
    print(arr.swapaxes(1,2))
    '''[[[ 0 4] [ 1 5] [ 2 6] [ 3 7]] [[ 8 12] [ 9 13] [10 14] [11 15]]]'''

# 2、通用函数：快速的元素级数组函数 #

## 2.1 利用数组进行数据处理 ##

arr = np.arange(10)
    print(np.sqrt(arr)) # 求平方根
    '''[0. 1. 1.41421356 1.73205081 2. 2.23606798 2.44948974 2.64575131 2.82842712 3. ]'''
     
    # 求指数e的指数
    print(np.exp(arr)) 
    '''[1.00000000e+00 2.71828183e+00 7.38905610e+00 2.00855369e+01 5.45981500e+01 1.48413159e+02 4.03428793e+02 1.09663316e+03 2.98095799e+03 8.10308393e+03]'''
    x = randn(8)
    y = randn(8)
    print(x)
    '''[ 0.63811097 -0.51027577 -0.17726455 0.97772163 -2.14367248 -0.28716377 -0.20549874 0.46916848]'''
    print(y)
    '''[ 0.63032743 1.21848016 -1.86789595 0.42896225 0.66737185 1.31428271 -0.81983392 -0.33119709]'''
     
    # 求元素级最大值，x和y对应元素比较
    print(np.maximum(x,y)) 
    '''[ 0.63811097 1.21848016 -0.17726455 0.97772163 0.66737185 1.31428271 -0.20549874 0.46916848]'''
    
    arr = randn(7) *5
    print(arr)
    '''[-1.04444876 1.31390346 -2.57308959 9.68938735 -7.54923318 -2.23501749 -4.00171934]'''
     
    # 将数组各元素的小数和整数部分以两个独立数组形式返回(书上没写清楚)
    print(np.modf(arr))
    '''(array([-0.04444876, 0.31390346, -0.57308959, 0.68938735, -0.54923318, -0.23501749, -0.00171934]), array([-1., 1., -2., 9., -7., -2., -4.]))'''
    
    points = np.arange(-5, 5, 0.01)  #1000个间隔相等的点
    xs, ys = np.meshgrid(points, points)
    import matplotlib.pyplot as plt
    z = np.sqrt(xs**2 + ys**2)
    print(z)
    '''[[7.07106781 7.06400028 7.05693985 ... 7.04988652 7.05693985 7.06400028] [7.06400028 7.05692568 7.04985815 ... 7.04279774 7.04985815 7.05692568] [7.05693985 7.04985815 7.04278354 ... 7.03571603 7.04278354 7.04985815] ... [7.04988652 7.04279774 7.03571603 ... 7.0286414 7.03571603 7.04279774] [7.05693985 7.04985815 7.04278354 ... 7.03571603 7.04278354 7.04985815] [7.06400028 7.05692568 7.04985815 ... 7.04279774 7.04985815 7.05692568]]'''
    plt.imshow(z, cmap=plt.cm.gray)
    plt.colorbar()
    plt.title("Image plot of sqart{x^2+y^2} for a gid values")
    plt.show()

## 2.2 将条件逻辑表述为数组运算 ##

xarr = np.array([1.1, 1.2, 1.3, 1.4, 1.5])
    yarr = np.array([2.1 ,2.2, 2.3, 2.4 ,2.5])
    cond = np.array([True, False, True, True, False])
    
    #根据真假值选取新值,为真则从xarr中选取，否则从yarr中选取
    result = [(x if c else y) for x, y, c in zip(xarr, yarr, cond)] 
    print(result) # [1.1, 2.2, 1.3, 1.4, 2.5]
    
    result = np.where(cond, xarr, yarr) # 新的表达方式
    print(result) # [1.1 2.2 1.3 1.4 2.5]
    
    arr = randn(4,4)
    print(arr)
    '''[[ 0.0346191 1.22446873 -1.22899471 -0.3369665 ] [ 1.47604911 -0.99256909 0.00272824 -1.95770979] [-0.74920869 -0.4935277 1.94493423 -0.69479795] [ 0.48006709 1.10094828 0.06475939 -1.31085929]]'''
     
    # 将正值替换为2，负值替换
    print(np.where(arr > 0, 2, -2)) 为-2
    '''[[ 2 2 -2 -2] [ 2 -2 2 -2] [-2 -2 2 -2] [ 2 2 2 -2]]'''
     
    # 将正值替换为2，负值不动
    print(np.where(arr > 0, 2, arr)) 
    '''[[ 2. 2. -1.22899471 -0.3369665 ] [ 2. -0.99256909 2. -1.95770979] [-0.74920869 -0.4935277 2. -0.69479795] [ 2. 2. 2. -1.31085929]]'''
    
    cond1 = np.array([True, False, False, True, True, False])
    cond2 = np.array([True, True, False, False, False, True])
    result = []
    for i in range(6):
        if cond1[i] and cond2[i]:
            result.append(0)
        elif cond1[i]:
            result.append(1)
        elif cond2[i]:
            result.append(2)
        else:
            result.append(3)
    print(result) # [0, 2, 3, 1, 1, 2]
    print(np.where(cond1&cond2,0, np.where(cond1, 1, np.where(cond2,2,3)))) # [0 2 3 1 1 2], 与上述逻辑表达式效果一致

## 2.3 数学和统计方法 ##

arr = np.random.randn(5,4)
    print(arr)
    '''[[ 0.86379303 -0.5274916 0.49188262 0.3381654 ] [-1.25237271 -1.02041665 -0.45664488 2.14356969] [ 0.94932601 -1.49865692 -0.60859843 -0.33866988] [-0.14164747 -0.73383431 -0.6075893 0.44304416] [-0.50505771 -1.49829159 0.09844207 0.67972674]]'''
    print(arr.mean()) # 求均值 -0.15906608693333124
    print(np.mean(arr)) # -0.15906608693333124
    print(arr.sum()) #求和 -3.181321738666625
    print(arr.mean(axis = 1)) # 按列求均值 [ 0.29158736 -0.14646614 -0.37414981 -0.26000673 -0.30629512]
    print(arr.sum(0)) # 按行求和 [-0.08595885 -5.27869107 -1.08250792 3.2658361 ]
    
    arr = np.array([[0,1,2],[3,4,5],[6,7,8]])
    print(arr)
    print(arr.cumsum(0)) # 按行求累计和
    print(arr.cumprod(1)) # 按列求累计积

## 2.4 用于布尔数组的方法 ##

arr = randn(100)
    print((arr > 0).sum())  #求正数的数量
    
    bools = np.array([False, False, True, False])
    print(bools.any()) # True. 检查是否存在一个或者多个True
    print(bools.all()) # Fasle. 检查是否都为True

## 2.5 排序 ##

arr = randn(8)
    print(arr)
    '''[ 0.52054089 0.11640705 1.11630459 -0.488596 1.385098 2.35582882 0.40295761 -0.98801818]'''
      
    # 对数组排序,注意与sorted(arr)区分， 前者会改变原数组，后者不会
    arr.sort() 
    print(arr)
    '''[-0.98801818 -0.488596 0.11640705 0.40295761 0.52054089 1.11630459 1.385098 2.35582882]'''
    
    arr = randn(5,3)
    print(arr)
    '''[[ 0.77209717 1.11046007 0.38905073] [ 0.1855081 0.1714953 0.64185395] [ 1.35953558 -1.32439862 -1.03117232] [-1.65155383 -0.45173971 1.52520206] [ 0.50943937 0.16410033 -0.3458179 ]]'''
    arr.sort(1) #在某个轴上进行排序
    print(arr)
    '''[[ 0.38905073 0.77209717 1.11046007] [ 0.1714953 0.1855081 0.64185395] [-1.32439862 -1.03117232 1.35953558] [-1.65155383 -0.45173971 1.52520206] [-0.3458179 0.16410033 0.50943937]]'''
    
    large_arr = randn(1000)
    large_arr.sort()
    print(large_arr[int(0.05 * len(large_arr))]) # 5%分位数

## 2.6 唯一以及其他集合逻辑 ##

names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
    print(np.unique(names)) # ['Bob' 'Joe' 'Will']
    ints = np.array([3,3,3,2,2,1,1,4,4])
    print(np.unique(ints)) # [1 2 3 4]
    print(sorted(set(names)))  # ['Bob', 'Joe', 'Will']
    vaules = np.array([6,0,0,3,2,5,6])
    print(np.in1d(vaules, [2,3,6])) # [ True False False True True False True]

# 3、用于数组的文件输入输出 #

## 3.1 将数组以二进制格式保存到磁盘 ##

arr = np.arange(10)
    # 保存时自动扩展为.npy格式文件
    np.save("some_array", arr) 
    ret = np.load("some_array.npy")
    print(ret) # [0 1 2 3 4 5 6 7 8 9]
    
    # 将多个数组保存到一个压缩文件中，数组以关键字参数的形式传入
    np.savez("array_achive.npz", a = arr, b = arr) 
    arch = np.load("array_achive.npz")
    print(arch['b']) # [0 1 2 3 4 5 6 7 8 9]

## 3.2 存取文本文件 ##

arr = np.loadtxt("python_data/ch04/array_ex.txt", delimiter=',')
    print(arr)
    '''[[ 0.580052 0.18673 1.040717 1.134411] [ 0.194163 -0.636917 -0.938659 0.124094] [-0.12641 0.268607 -0.695724 0.047428] [-1.484413 0.004176 -0.744203 0.005487] [ 2.302869 0.200131 1.670238 -1.88109 ] [-0.19323 1.047233 0.482803 0.960334]]'''

# 4、线性代数 #

x = np.array([[1.,2.,3.],[4.,5.,6.]])
    y = np.array([[6.,23.],[-1,7],[8,9]])
    print(x)
    '''[[1. 2. 3.] [4. 5. 6.]]'''
    print(y)
    '''[[ 6. 23.] [-1. 7.] [ 8. 9.]]'''
    print(x.dot(y)) # 相当于np.dot(x,y)
    '''[[ 28. 64.] [ 67. 181.]]'''
    
    print(np.dot(x, np.ones(3))) # [ 6. 15.]
    
    from numpy.linalg import inv, qr
    X= randn(5,5)
    mat = X.T.dot(X)
    print(inv(mat))  # 计算矩阵的逆
    t = mat.dot(inv(mat))
    q,r = qr(mat)   # 计算QR分解

# 5、随机数生成 #

samples = np.random.normal(size=(4,4))
    print(samples)
    
    from random import normalvariate
    N =100000
    samples = [normalvariate(0,1) for _ in range(N)] #正态分布; python3.7中没有xrange函数
    ret = np.random.normal(size=N)

# 6、范例：随机漫步 #

import random
    position = 0
    walk =[position]
    steps = 1000
    for i in range(steps):
        step = 1 if random.randint(0,1) else - 1
        position += step
        walk.append(position)
    
    plt.plot(walk)
    plt.title("Random walk with +1/-1 steps")
    plt.show()
    
    nsteps = 1000
    draws = np.random.randint(0,2,size=nsteps)
    steps = np.where(draws > 0, 1, -1) # 感觉这个比较实用
    walk = steps.cumsum()
    print(walk)
    print(walk.min())
    print(walk.max())
    print((np.abs(walk)>10).argmax())
    
    #一次模拟多个随机漫步
    nwalks = 5000
    nsteps = 1000
    draws = np.random.randint(0,2, size=(nwalks, nsteps)) # 0或1
    steps = np.where(draws > 0, 1, -1)
    walks = steps.cumsum(1)
    print(walks)
    print(walk.max())
    print(walk.min())
    # 计算30的最小穿越时间
    hits30 = (np.abs(walks) > 30).any(1)
    print(hits30)
    # 达到30的数量
    print(hits30.sum()) 
    # 选出超过30（绝对值）的随机漫步，并获取在轴以上的穿越时间
    crossing_times = (np.abs(walks[hits30])>=30).argmax(1) 
    print(crossing_times.mean())

[Python _pdf]: https://blog.csdn.net/zhou4411781/article/details/108556600