Pandas基础教程

红太狼 2021-05-12 11:49 591阅读 0赞

# Pandas基础教程 #

#### 目录 ####

基本概念

创建Series

创建DataFrame

DataFrame的属性

数据筛选

数据更改

处理丢失数据

导入导出数据

DataFrame的合并

### 基本概念： ###

Pandas模块的数据结构主要有两种:
    	-1.Series
    Series是一维数组，相当于numpy中的一维数组，本身也是基于numpy中的ndarray结构
    	-2.DataFrame
    DataFrame相当于二维数组（矩阵），与numpy中不同的是，DataFrame的数据结构是类似Excel，DataFrame有行名和列名的二维表结构

### 首先需要导入: ###

import numpy as np
    import pandas as pd

### 创建Series： ###

pd.Series([list],index=[list])
    	参数list,
    	参数index为可选参数，若不填写则默认index从0开始

a = pd.Series([1,2,3,np.nan,4,5])
    print(a)

![在这里插入图片描述][20200203160707189.png]

### 创建DataFrame： ###

pd.DataFrame(data=,index=,columns=,dtype=)
    	参数data:存放在DataFrame中的数据
    	参数index:二维表的行名
    	参数colunms:二维表的列名
    	参数dtype:数据类型
    index和colums可以是单个数，也可以是list类型，如果是list类型需要和DataFrame中的数据大小匹配。

# 创建DataFrame方法1
    default_df = pd.DataFrame(np.random.randn(3,4))  # 不设置行名和列名
    print(default_df)

![在这里插入图片描述][20200203160718987.png]

# 创建DataFrame方法2
    df = pd.DataFrame(np.arange(6).reshape(2,3), index=["one","two"], columns=["a",'b','c'])
    print(df)

![在这里插入图片描述][20200203160735332.png]

# 创建DataFrame方法3,key代表列名，value代表每列下面的值 （下面生成4行5列的表）
    dic_df = pd.DataFrame({ "a":1,
                          "b":pd.Timestamp("20200131"),
                          "c":pd.Series(1,index=list(range(4))),
                          "d":np.array([3]*4,dtype="int32"),                  "e":pd.Categorical(["apple","banana","weatermelon","orange"])})
    print(dic_df)

![在这里插入图片描述][20200203160745184.png]

### DataFrame的属性： ###

print("数据格式类型：",dic_df.dtypes)
    print("查看前几行数据，默认前五行：",dic_df.head())
    print("查看后几行数据，默认后五行：",dic_df.tail())
    print("列名：",dic_df.columns)
    print("行名：",dic_df.index)
    print("表中数据值：",dic_df.values)
    print("表中数据值的最大值，最小值，平均值等，只有是数字格式才有：",dic_df.describe())
    print("表格转置(类似矩阵转置)：",dic_df.T)
    print("axis=0,对行名进行排序:",dic_df.sort_index(axis=0, ascending=False))
    print("axis=1,对列名进行排序:",dic_df.sort_index(axis=1, ascending=False))
    print("对值进行排序：",dic_df.sort_values(by="e"))

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70]

### 数据筛选： ###

# 先创建DataFrame
    dates = pd.date_range("20200131",periods=6)
    df = pd.DataFrame(np.arange(24).reshape([6,4]), index=dates,columns=list("ABCD"))
    print(df)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 1]

# 索引某一列
    print(df["A"])
    # 或者
    print(df.A)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 2]

# 切片
    print(df[0:2])  # 这个只能索引行，0-2行
    print(df["20200201":"20200204"]) # 这个只能索引行，"20200201"行-"20200204"行

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 3]

# 切片
    # select by label : loc 通过行名列名索引
    print(df.loc["20200201",:])   # 行:列, "20200201"行和全部列
    print(df.loc[:,["A","B"]])    # 所有行和"A"和"B"两列
    print(df.loc["20200202",["A"]]) # 保留某行某列

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 4]

# 切片
    # select by position：iloc 通过列名索引
    print(df.iloc[0:2,:])  # 0-2行，整列
    print(df.iloc[0:2,1:3])  # 0-2行，1-3列
    print(df.iloc[[1,3,4],1:3]) # 1,3,4行，1-3列

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 5]

# 切片
    # mixed select : ix (索引配标签)
    print(df.ix[0:2,["A","C"]])

![在这里插入图片描述][2020020316090333.png]

# Boolean indexing
    print(df[df.A>8])
    print(df[df.A>8][df.A<=16])

### 数据更改： ###

dates = pd.date_range("20200131",periods=6)
    df = pd.DataFrame(np.arange(24).reshape([6,4]), index=dates, columns=list("ABCD"))
    print("原来的DataFrame：\n")
    df.loc["20200202","A"] = 2020   # 更改某个值
    df.iloc[2,3] = 4040           # 更改某个值
    df.ix["20200131","A"] = 8.24 # 更改某个值

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 6]

# 对比
    df[df.A>4] = 0   # 对于整个DataFrame而言
    print(df)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 7]

# 对比
    df.A[df.A>4] = 0    # 对于列A而言

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 8]

# 对比
    df.B[df.B>4] = 0  # 对于列B而言

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 9]

# 增加一列空值
    df["E"] = np.nan

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 10]

# 增加一列实值
    df["F"]=pd.Series(np.arange(6),index=pd.date_range("20200131", periods=6))

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 11]

### 处理丢失数据: ###

dates = pd.date_range("20200131", periods=6)
    df = pd.DataFrame(np.arange(24).reshape([6,4]), index=dates, columns=list("ABCD"))
    # df中替换几个空值 
    df.loc["20200131","A"] = np.nan  # 用loc
    df.iloc[1,1] = np.nan  # 用iloc
    df.ix[2,"C"] = np.nan  # 用ix

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 12]

# 处理数据的方法：丢弃
    # axis=0 丢弃行，axis=1 丢弃列
    # how默认等于any, any是只要有一个为NaN就丢弃，all是全部为NaN才丢弃
    print(df.dropna(axis=0, how="any"))
    print(df.dropna(axis=1, how="any"))

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 13]

# 处理数据的方法：用其他数据填充NaN
    print(df.fillna(value=0))  # 用0填充

![在这里插入图片描述][20200203161033281.png]

# 检查DataFrame中是否有缺失数据（为NaN）
    print(df.isnull())  # 返回True或者False

![在这里插入图片描述][20200203161050753.png]

# 如果表格比较大，检查起来不方便 可以用np.any()
    print(np.any(df.isnull()) == True)

![在这里插入图片描述][2020020316111122.png]

### 导入导出数据(比较简单): ###

<table> 
 <thead> 
  <tr> 
   <th>Read数据</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>read_csv</td> 
  </tr> 
  <tr> 
   <td>read_execl</td> 
  </tr> 
  <tr> 
   <td>read_hdf</td> 
  </tr> 
  <tr> 
   <td>read_sql</td> 
  </tr> 
  <tr> 
   <td>read_json</td> 
  </tr> 
  <tr> 
   <td>read_msgpack</td> 
  </tr> 
  <tr> 
   <td>read_html</td> 
  </tr> 
  <tr> 
   <td>read_gbq</td> 
  </tr> 
  <tr> 
   <td>read_stata</td> 
  </tr> 
  <tr> 
   <td>read_sas</td> 
  </tr> 
  <tr> 
   <td>read_clipboard</td> 
  </tr> 
  <tr> 
   <td>read_pickle</td> 
  </tr> 
 </tbody> 
</table>

<table> 
 <thead> 
  <tr> 
   <th>Write数据</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>to_csv</td> 
  </tr> 
  <tr> 
   <td>to_excel</td> 
  </tr> 
  <tr> 
   <td>to_hdf</td> 
  </tr> 
  <tr> 
   <td>to_sql</td> 
  </tr> 
  <tr> 
   <td>to_json</td> 
  </tr> 
  <tr> 
   <td>to_msgpack</td> 
  </tr> 
  <tr> 
   <td>to_html</td> 
  </tr> 
  <tr> 
   <td>to_gbq</td> 
  </tr> 
  <tr> 
   <td>to_stata</td> 
  </tr> 
  <tr> 
   <td>to_clipboard</td> 
  </tr> 
  <tr> 
   <td>to_pickle</td> 
  </tr> 
 </tbody> 
</table>

### DataFrame的合并: ###

# concat
    df0 = pd.DataFrame(np.ones([3,4])*0,columns=["a","b","c","d"])
    df1 = pd.DataFrame(np.ones([3,4])*1,columns=["a","b","c","d"])
    df2 = pd.DataFrame(np.ones([3,4])*2,columns=["a","b","c","d"])
    print(df0)
    
    # axis=0 上下合并 axis=1 左右合并
    result0 = pd.concat([df0,df1,df2],axis=0)  # 上下合并
    result1 = pd.concat([df0,df1,df2],axis=1)  # 左右合并
    print(result0)
    print(result1)
    # 合并后行索引自动对齐 ignore_index=True
    result_ignore = pd.concat([df0,df1,df2],axis=0,ignore_index=True)
    print(result_ignore)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 14]

# join, 属性--"outer"和"inner"
    df0 = pd.DataFrame(np.ones([3,4])*0, columns=list("abcd"), index=[1,2,3])
    df1 = pd.DataFrame(np.ones([3,4])*1, columns=list("bcde"), index=[2,3,4])
    print(df0)
    print(df1)
    # 默认join=outer,保留全部
    result_outer = pd.concat([df0,df1],join="outer",ignore_index=True)
    print(result_outer)
    # join=inner, 裁剪不同，只保留相同
    result_inner = pd.concat([df0,df1], join="inner", ignore_index=True)
    print(result_inner)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 15]

# join_axes 按指定保留的索引进行合并
    df0 = pd.DataFrame(np.ones([3,4])*0, columns=list("abcd"), index=[1,2,3])
    df1 = pd.DataFrame(np.ones([3,4])*1, columns=list("bcde"), index=[2,3,4])
    # 上下合并保留根据某个df来保留
    save_df0_col = pd.concat([df0,df1],axis=0,join_axes=[df0.columns], ignore_index=True)
    print(save_df0_col)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 16]

# append
    df0 = pd.DataFrame(np.ones([3,4])*0, columns=list("abcd"))
    df1 = pd.DataFrame(np.ones([3,4])*1, columns=list("abcd"))
    # append 单个DataFrame
    result = df0.append(df1, ignore_index=True)
    print(result)
    # append 多个
    result = result.append([df1,df0], ignore_index=True)
    print(result)
    # append Series
    one_series = pd.Series(np.arange(4), index=list("abcd"))
    result = result.append(one_series, ignore_index=True)
    print(result)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 17]

[20200203160707189.png]: /images/20210511/65a63883392146f98a9a396a8d9be981.png
[20200203160718987.png]: /images/20210511/e8c45ef1016a41b0a17ea34ebb63906e.png
[20200203160735332.png]: /images/20210511/860d6ac37bdb47e9b3f7a6d2f68a449e.png
[20200203160745184.png]: /images/20210511/b3586d5ffa274c1cb3d55b391516aeda.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70]: /images/20210511/290215bbb865449dab166bb5ae6a4861.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 1]: /images/20210511/9767b71a23d14f6ca9188fc5c5269695.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 2]: /images/20210511/4cb11cc599f74b11863fe9fe73912d06.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 3]: /images/20210511/3f6f6773ebd7455bbd2a53de4a0b2474.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 4]: /images/20210511/ecb7cb8c78254205ba2a9c4d839392a3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 5]: /images/20210511/079a318dea5d4916b0f7758f5f7454dd.png
[2020020316090333.png]: /images/20210511/c6fef322d6c34d52bc7eabc75f689cd6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 6]: /images/20210511/c29610e480d649b9a073bc2482d4988e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 7]: /images/20210511/fac931b1a4824f548fc910d7699a019b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 8]: /images/20210511/d453d595910a4b4ebb1f188ac2c036f5.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 9]: /images/20210511/11f7170d4dac4fd2a9794536f73d2a88.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 10]: /images/20210511/a789ebfd0e144f62ad450007e9287829.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 11]: /images/20210511/acf11f6ebdd5403ba7a83c5182afb38a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 12]: /images/20210511/35e53c73c13a4261932856894324b76a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 13]: /images/20210511/d868059d9f374866be9b4707c07662a2.png
[20200203161033281.png]: /images/20210511/9a9ee4ddefda41228de7c467e3728c1b.png
[20200203161050753.png]: /images/20210511/a1577b94a96a4ae390b6115a0ebc093e.png
[2020020316111122.png]: /images/20210511/97846f3c82e24f83a3cbd3c2cc4f78bd.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 14]: /images/20210511/b86fb4d592e946468238077903bc7a20.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 15]: /images/20210511/c9b7c0d29ad142d2beae4b51b1be8dbd.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 16]: /images/20210511/3734892e722f4113a286261f856b3a11.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkxMjE1OQ_size_16_color_FFFFFF_t_70 17]: /images/20210511/13791e147d0e46a391a53a1adbc24ab9.png