MFS分布式文件系统，分布式存储+高可用（pacemaker+corosync+pcs）+磁盘共享（iscsi）+fence

迷南。 2022-01-30 06:55 478阅读 0赞

环境：RHEL7.3

*  server1：172.25.13.1 mfsmaster
 *  server2：172.25.13.2 chunkserver
 *  server3：172.25.13.3 chunkserver
 *  server4：172.25.13.4 backup-mfsserver
 *  client：172.25.13.250 client

### 一、MFS安装、部署、配置 ###

*  配置mfsmaster端

#### 1.安装部署MFS ####

[root@server1 ~]# cd 3.0.103/
    [root@server1 3.0.103]# ls
    moosefs-cgi-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-cgiserv-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-cli-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-client-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm
    moosefs-metalogger-3.0.103-1.rhsystemd.x86_64.rpm
    ===================================================
    [root@server1 3.0.103]# yum install -y \
    moosefs-cgi-3.0.103-1.rhsystemd.x86_64.rpm \
    moosefs-cgiserv-3.0.103-1.rhsystemd.x86_64.rpm \
    moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm \
    moosefs-cli-3.0.103-1.rhsystemd.x86_64.rpm

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70]  
主配置文件目录

/etc/mfs/

数据目录

/var/lib/mfs/

#### 2.开启master服务，查看端口 ####

[root@server1 3.0.103]# systemctl start moosefs-master
    [root@server1 3.0.103]# netstat -antlp
    Active Internet connections (servers and established)
    Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
    tcp        0      0 0.0.0.0:9419            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:9420            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:9421            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:22              0.0.0.0:*               
    ...

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 1]

#### 3.开启cgiserv服务（web图形处理工具），查看端口 ####

[root@server1 3.0.103]# systemctl start moosefs-cgiserv.service 
    [root@server1 3.0.103]# netstat -antlp
    Active Internet connections (servers and established)
    Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
    tcp        0      0 0.0.0.0:9419            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:9420            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:9421            0.0.0.0:*               LISTEN      11577/mfsmaster     
    tcp        0      0 0.0.0.0:9425            0.0.0.0:*     
    ...

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 2]

#### 4.给server\{1~4\}、client添加解析 ####

[root@server1 3.0.103]# vim /etc/hosts
    =======================================
    172.25.13.1 server1 mfsmaster

5.打开浏览器访问： [http://172.25.13.1:9425][http_172.25.13.1_9425]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 3]

*  配置chunkserver  
    给server2-3发送chunkserver安装

scp moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm server2:
     scp moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm server3:

#### 1.安装chunkserver软件 ####

[root@server2 ~]# ls
    moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm
    [root@server2 ~]# yum install -y moosefs-chunkserver-3.0.103-1.rhsystemd.x86_64.rpm

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 4]

#### 2.创建共享目录 ####

[root@server2 ~]# mkdir /mnt/chunk1

#### 3.修改配置文件 ####

[root@server2 ~]# vim /etc/mfs/mfshdd.cfg
    ===========================================
     35 /mnt/chunk1

![在这里插入图片描述][20190518155104899.png]

#### 4.virt-manager上给server2添加硬盘 ####

[root@server2 ~]# fdisk -l
    ================================
    ...
    Disk /dev/vda: 21.5 GB, 21474836480 bytes, 41943040 sectors
    Units = sectors of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 5]  
![在这里插入图片描述][20190518155207349.png]

#### 5.分区，格式化 ####

[root@server2 ~]# fdisk /dev/vda
    Welcome to fdisk (util-linux 2.23.2).
    
    Changes will remain in memory only, until you decide to write them.
    Be careful before using the write command.
    
    Device does not contain a recognized partition table
    Building a new DOS disklabel with disk identifier 0xb8a1873f.
    
    Command (m for help): p
    
    Disk /dev/vda: 21.5 GB, 21474836480 bytes, 41943040 sectors
    Units = sectors of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    Disk label type: dos
    Disk identifier: 0xb8a1873f
    
       Device Boot      Start         End      Blocks   Id  System
    
    Command (m for help): n
    Partition type:
       p   primary (0 primary, 0 extended, 4 free)
       e   extended
    Select (default p): p
    Partition number (1-4, default 1): 1
    First sector (2048-41943039, default 2048): 
    Using default value 2048
    Last sector, +sectors or +size{ K,M,G} (2048-41943039, default 41943039): 
    Using default value 41943039
    Partition 1 of type Linux and of size 20 GiB is set
    
    Command (m for help): p
    
    Disk /dev/vda: 21.5 GB, 21474836480 bytes, 41943040 sectors
    Units = sectors of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    Disk label type: dos
    Disk identifier: 0xb8a1873f
    
       Device Boot      Start         End      Blocks   Id  System
    /dev/vda1            2048    41943039    20970496   83  Linux
    
    Command (m for help): wq
    The partition table has been altered!
    
    Calling ioctl() to re-read partition table.
    Syncing disks.
    
    [root@server2 ~]# fdisk -l
    Disk /dev/vda: 21.5 GB, 21474836480 bytes, 41943040 sectors
    Units = sectors of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    Disk label type: dos
    Disk identifier: 0xb8a1873f
    
       Device Boot      Start         End      Blocks   Id  System
    /dev/vda1            2048    41943039    20970496   83  Linux
    
    #格式化
    
    [root@server2 ~]# mkfs.xfs /dev/vda1
    meta-data=/dev/vda1              isize=512    agcount=4, agsize=1310656 blks
             =                       sectsz=512   attr=2, projid32bit=1
             =                       crc=1        finobt=0, sparse=0
    data     =                       bsize=4096   blocks=5242624, imaxpct=25
             =                       sunit=0      swidth=0 blks
    naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
    log      =internal log           bsize=4096   blocks=2560, version=2
             =                       sectsz=512   sunit=0 blks, lazy-count=1
    realtime =none                   extsz=4096   blocks=0, rtextents=0

![在这里插入图片描述][20190518155415142.png]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 6]  
![在这里插入图片描述][20190518155330456.png]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 7]

#### 6.挂载，并在目录中添加/etc/\* ####

[root@server2 ~]# mount /dev/vda1 /mnt/chunk1/
    [root@server2 ~]# df
    Filesystem            1K-blocks    Used Available Use% Mounted on
    /dev/mapper/rhel-root  18855936 1094104  17761832   6% /
    devtmpfs                 497292       0    497292   0% /dev
    tmpfs                    508264       0    508264   0% /dev/shm
    tmpfs                    508264   13108    495156   3% /run
    tmpfs                    508264       0    508264   0% /sys/fs/cgroup
    /dev/sda1               1038336  141504    896832  14% /boot
    tmpfs                    101656       0    101656   0% /run/user/0
    /dev/vda1              20960256   32944  20927312   1% /mnt/chunk1
    [root@server2 ~]# cp /etc/* /mnt/chunk1/

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 8]  
![在这里插入图片描述][20190518155530715.png]

#### 7.修改新建目录的所有者和所属组 ####

[root@server2 ~]# cd /mnt/
    [root@server2 mnt]# chown mfs.mfs chunk1/

![在这里插入图片描述][20190518155906705.png]

#### 8.开启moosefs-chunkserver服务 ####

[root@server2 ~]# systemctl start moosefs-chunkserver.service

**server3同server2操作**

#### 9.浏览器访问：[http://172.25.13.1:9425][http_172.25.13.1_9425] ####

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 9]  
从节点添加成功！

*  client部署服务

#### 1.安装moosefs-client ####

[root@foundation13 Desktop]# yum install -y moosefs-client-3.0.103-1.rhsystemd.x86_64.rpm

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 10]

#### 2.修改配置文件 ####

[root@foundation13 Desktop]# vim /etc/mfs/mfsmount.cfg
    ======================================================
    16  /mnt/mfs

![在这里插入图片描述][20190518160055995.png]

#### 3.新建mfs目录 ####

[root@foundation13 Desktop]# cd /mnt/
    [root@foundation13 mnt]# mkdir mfs

![在这里插入图片描述][20190518160127879.png]

#### 4.挂载 ####

[root@foundation13 mnt]# mfsmount 
    mfsmaster accepted connection with parameters: read-write,restricted_ip,admin ; root mapped to root:root

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 11]

#### 5.查看 ####

[root@foundation13 mnt]# df

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 12]

### 二、MFS的存储机制 ###

*  client

#### 1.新建dir1、dir2目录 ####

[root@foundation13 Desktop]# cd /mnt/mfs/
    [root@foundation13 mfs]# mkdir dir1
    [root@foundation13 mfs]# mkdir dir2

![在这里插入图片描述][20190518160243156.png]

#### 2.查看dir1、dir2的文件存储数 ####

[root@foundation13 mfs]# mfsgetgoal dir1/
    dir1/: 2
    [root@foundation13 mfs]# mfsgetgoal dir2/
    dir2/: 2

![在这里插入图片描述][20190518160303158.png]

#### 3.修改dir1的存储数 ####

[root@foundation13 mfs]# mfssetgoal -r 1 dir1/
    dir1/:
     inodes with goal changed:                       1
     inodes with goal not changed:                   0
     inodes with permission denied:                  0
    
    #再次查看
    [root@foundation13 mfs]# mfsgetgoal dir1/
    dir1/: 1
    [root@foundation13 mfs]# mfsgetgoal dir2/
    dir2/: 2

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 13]

#### 4.给dir1中复制/etc/passwd，并查看passwd的信息 ####

[root@foundation13 mfs]# cd dir1/
    [root@foundation13 dir1]# ls
    [root@foundation13 dir1]# cp /etc/passwd .
    [root@foundation13 dir1]# mfsfileinfo passwd 
    passwd:
    	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)

![在这里插入图片描述][20190518160405211.png]

#### 5.给dir2中复制/etc/fstab，并查看fstab的信息 ####

[root@foundation13 dir1]# cd ../dir2/
    [root@foundation13 dir2]# cp /etc/fstab .
    [root@foundation13 dir2]# mfsfileinfo fstab 
    fstab:
    	chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    		copy 2: 172.25.13.3:9422 (status:VALID)

![在这里插入图片描述][20190518160426825.png]

#### 6.给dir1制作大文件bigfile1，并查看备份数 ####

[root@foundation13 dir2]# cd ../dir1/
    [root@foundation13 dir1]# dd if=/dev/zero of=bigfile bs=1M count=200
    200+0 records in
    200+0 records out
    209715200 bytes (210 MB) copied, 1.07711 s, 195 MB/s
    [root@foundation13 dir1]# mfsfileinfo passwd 
    passwd:
    	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)
    [root@foundation13 dir1]# mfsfileinfo bigfile 
    bigfile:
    	chunk 0: 0000000000000003_00000001 / (id:3 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 1: 0000000000000004_00000001 / (id:4 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 2: 0000000000000005_00000001 / (id:5 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 3: 0000000000000006_00000001 / (id:6 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 14]

#### 7.给dir2制作大文件bigfile2，并查看备份数 ####

[root@foundation13 dir1]# cd ../dir2/
    [root@foundation13 dir2]# dd if=/dev/zero of=bigfile2 bs=1M count=200
    200+0 records in
    200+0 records out
    209715200 bytes (210 MB) copied, 9.81156 s, 21.4 MB/s
    [root@foundation13 dir2]# mfsfileinfo bigfile2 
    bigfile2:
    	chunk 0: 0000000000000007_00000001 / (id:7 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    		copy 2: 172.25.13.3:9422 (status:VALID)
    	chunk 1: 0000000000000008_00000001 / (id:8 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    		copy 2: 172.25.13.3:9422 (status:VALID)
    	chunk 2: 0000000000000009_00000001 / (id:9 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    		copy 2: 172.25.13.3:9422 (status:VALID)
    	chunk 3: 000000000000000A_00000001 / (id:10 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    		copy 2: 172.25.13.3:9422 (status:VALID)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 15]

#### 8.测试 ####

#关闭server2的moosefs-chunkserver服务
    [root@server2 ~]# systemctl stop moosefs-chunkserver.service

![在这里插入图片描述][2019051816070586.png]

#查看bigfile2备份详情
    [root@foundation13 dir2]# mfsfileinfo bigfile2 
    bigfile2:
    	chunk 0: 0000000000000007_00000001 / (id:7 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)
    	chunk 1: 0000000000000008_00000001 / (id:8 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)
    	chunk 2: 0000000000000009_00000001 / (id:9 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)
    	chunk 3: 000000000000000A_00000001 / (id:10 ver:1)
    		copy 1: 172.25.13.3:9422 (status:VALID)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 16]

#查看bigfile备份详情
    [root@foundation13 dir2]# cd ../dir1/
    [root@foundation13 dir1]# mfsfileinfo bigfile 
    bigfile:
    	chunk 0: 0000000000000003_00000001 / (id:3 ver:1)
    		no valid copies !!!
    	chunk 1: 0000000000000004_00000001 / (id:4 ver:1)
    		no valid copies !!!
    	chunk 2: 0000000000000005_00000001 / (id:5 ver:1)
    		no valid copies !!!
    	chunk 3: 0000000000000006_00000001 / (id:6 ver:1)
    		no valid copies !!!

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 17]

#查看passwd备份详情
    [root@foundation13 dir1]# mfsfileinfo passwd 
    passwd:
    	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
    		no valid copies !!!
    [root@foundation13 dir1]# ll
    total 204803
    -rw-r--r-- 1 root root 209715200 May 18 11:57 bigfile
    -rw-r--r-- 1 root root      2243 May 18 11:56 passwd

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 18]  
但是passwd确实存在

#开启server2的moosefs-chunkserver服务
    [root@server2 ~]# systemctl start moosefs-chunkserver.service

![在这里插入图片描述][20190518160938622.png]

#查看passwd备份详情
    [root@foundation13 dir1]# mfsfileinfo passwd 
    passwd:
    	chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)

![在这里插入图片描述][20190518161016863.png]

#查看bigfile备份详情
    [root@foundation13 dir1]# mfsfileinfo bigfile 
    bigfile:
    	chunk 0: 0000000000000003_00000001 / (id:3 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 1: 0000000000000004_00000001 / (id:4 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 2: 0000000000000005_00000001 / (id:5 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)
    	chunk 3: 0000000000000006_00000001 / (id:6 ver:1)
    		copy 1: 172.25.13.2:9422 (status:VALID)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 19]

#### 9.杀掉mfsmaster进程 ####

#mfs的特殊文件 metadata.mfs.back[服务运行]<-->metadata.mfs[服务关闭]
    
    [root@server1 mfs]# ps ax
    ...
    11577 ?        S<     1:05 /usr/sbin/mfsmaster start
    
    [root@server1 mfs]# kill -9 11577

![在这里插入图片描述][20190518161154915.png]

#### 10.测试数据文件变化 ####

[root@server1 mfs]# cd /var/lib/mfs/
    [root@server1 mfs]# ll
    total 3616
    -rw-r----- 1 mfs mfs    1062 May 18 12:57 changelog.0.mfs
    -rw-r----- 1 mfs mfs    1693 May 18 11:59 changelog.1.mfs
    -rw-r----- 1 mfs mfs     573 May 18 10:45 changelog.2.mfs
    -rw-r----- 1 mfs mfs     120 May 18 12:00 metadata.crc
    -rw-r----- 1 mfs mfs    3630 May 18 12:00 metadata.mfs.back
    -rw-r----- 1 mfs mfs    3120 May 18 11:00 metadata.mfs.back.1
    -rw-r--r-- 1 mfs mfs       8 Nov 23 20:46 metadata.mfs.empty
    -rw-r----- 1 mfs mfs 3672832 May 18 12:00 stats.mfs

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 20]

#重新开启（肯定会失败，因为上次是非正常关闭，文件metadata.mfs.back为转换为metadata.mfs）
    [root@server1 mfs]# mfsmaster start
    open files limit has been set to: 16384
    working directory: /var/lib/mfs
    lockfile created and locked
    initializing mfsmaster modules ...
    exports file has been loaded
    topology file has been loaded
    loading metadata ...
    can't find metadata.mfs - try using option '-a'
    init: metadata manager failed !!!
    error occurred during initialization - exiting
    ==================================================
    #按照提示修复
    [root@server1 mfs]# mfsmaster -a

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 21]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 22]

#查看文件
    [root@server1 mfs]# ls
    changelog.1.mfs  changelog.3.mfs  metadata.mfs.back    metadata.mfs.empty
    changelog.2.mfs  metadata.crc     metadata.mfs.back.1  stats.mfs

![在这里插入图片描述][20190518161546734.png]

#关闭服务并查看文件
    [root@server1 mfs]# mfsmaster stop
    sending SIGTERM to lock owner (pid:11793)
    waiting for termination terminated
    [root@server1 mfs]# ls
    changelog.2.mfs  changelog.4.mfs  metadata.mfs         metadata.mfs.empty
    changelog.3.mfs  metadata.crc     metadata.mfs.back.1  stats.mfs

![在这里插入图片描述][20190518161626170.png]

#开启服务，查看文件
    [root@server1 mfs]# systemctl start moosefs-master
    [root@server1 mfs]# ls
    changelog.2.mfs  changelog.4.mfs  metadata.mfs.back    metadata.mfs.empty
    changelog.3.mfs  metadata.crc     metadata.mfs.back.1  stats.mfs

![在这里插入图片描述][20190518161709527.png]

### 三、MFS删除文件的暂存机制 ###

#### 1.删除passwd，查看删除文件的暂存时限 ####

[root@foundation13 mfs]# cd /mnt/mfs/dir1/
    [root@foundation13 dir1]# ls
    bigfile  passwd
    [root@foundation13 dir1]# rm -fr passwd 
    [root@foundation13 dir1]# mfsgettrashtime .
    .: 86400

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 23]

#### 2.新建回收站挂载目录，并挂载 ####

[root@foundation13 dir1]# mkdir /mnt/mfsmeta
    [root@foundation13 dir1]# mfsmount -m /mnt/mfsmeta/
    mfsmaster accepted connection with parameters: read-write,restricted_ip

![在这里插入图片描述][20190518161819521.png]

#### 3.查看可挂载设备信息 ####

[root@foundation13 dir1]# mount
    ...
    mfsmaster:9421 on /mnt/mfs type fuse.mfs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
    mfsmaster:9421 on /mnt/mfsmeta type fuse.mfsmeta (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)

![在这里插入图片描述][20190518161857519.png]

#### 4.恢复文件passwd ####

[root@foundation13 dir1]# cd /mnt/mfsmeta/
    [root@foundation13 mfsmeta]# ls
    sustained  trash
    [root@foundation13 mfsmeta]# cd trash/
    [root@foundation13 trash]# ls
    000  112  224  336  448  55A  66C  77E  890  9A2  AB4  BC6  CD8  DEA  EFC
    001  113  225  337  449  55B  66D  77F  891  9A3  AB5  BC7  CD9  DEB  EFD
    002  114  226  338  44A  55C  66E  780  892  9A4  AB6  BC8  CDA  DEC  EFE
    003  115  227  339  44B  55D  66F  781  893  9A5  AB7  BC9  CDB  DED  EFF
    004  116  228  33A  44C  55E  670  782  894  9A6  AB8  BCA  CDC  DEE  F00
    005  117  229  33B  44D  55F  671  783  895  9A7  AB9  BCB  CDD  DEF  F01
    ..........................................................................
    103  215  327  439  54B  65D  76F  881  993  AA5  BB7  CC9  DDB  EED  FFF
    104  216  328  43A  54C  65E  770  882  994  AA6  BB8  CCA  DDC  EEE  undel
    105  217  329  43B  54D  65F  771  883  995  AA7  BB9  CCB  DDD  EEF
    106  218  32A  43C  54E  660  772  884  996  AA8  BBA  CCC  DDE  EF0
    107  219  32B  43D  54F  661  773  885  997  AA9  BBB  CCD  DDF  EF1
    108  21A  32C  43E  550  662  774  886  998  AAA  BBC  CCE  DE0  EF2
    109  21B  32D  43F  551  663  775  887  999  AAB  BBD  CCF  DE1  EF3
    10A  21C  32E  440  552  664  776  888  99A  AAC  BBE  CD0  DE2  EF4
    10B  21D  32F  441  553  665  777  889  99B  AAD  BBF  CD1  DE3  EF5
    10C  21E  330  442  554  666  778  88A  99C  AAE  BC0  CD2  DE4  EF6
    10D  21F  331  443  555  667  779  88B  99D  AAF  BC1  CD3  DE5  EF7
    10E  220  332  444  556  668  77A  88C  99E  AB0  BC2  CD4  DE6  EF8
    10F  221  333  445  557  669  77B  88D  99F  AB1  BC3  CD5  DE7  EF9
    110  222  334  446  558  66A  77C  88E  9A0  AB2  BC4  CD6  DE8  EFA
    111  223  335  447  559  66B  77D  88F  9A1  AB3  BC5  CD7  DE9  EFB
    [root@foundation13 trash]# ls |wc -l
    4097
    [root@foundation13 trash]# find -name *passwd*
    ./004/00000004|dir1|passwd
    [root@foundation13 trash]# cd 004/
    [root@foundation13 004]# ls
    00000004|dir1|passwd  undel
    [root@foundation13 004]# mv 00000004\|dir1\|passwd undel/

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 24]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 25]  
![在这里插入图片描述][20190518162301838.png]

#### 5.client查看是否恢复成功 ####

[root@foundation13 004]# cd /mnt/mfs/dir1/
    [root@foundation13 dir1]# ls
    bigfile  passwd
    [root@foundation13 dir1]# cat passwd 
    root:x:0:0:root:/root:/bin/bash
    bin:x:1:1:bin:/bin:/sbin/nologin
    daemon:x:2:2:daemon:/sbin:/sbin/nologin
    adm:x:3:4:adm:/var/adm:/sbin/nologin
    lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
    sync:x:5:0:sync:/sbin:/bin/sync
    shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
    ............................................
    sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
    tcpdump:x:72:72::/:/sbin/nologin
    kiosk:x:1000:1000:kiosk:/home/kiosk:/bin/bash
    apache:x:48:48:Apache:/usr/share/httpd:/sbin/nologin

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 26]  
恢复成功！

### 四、MFSmaser的高可用 ###

*  配置master，添加高可用yum源

#### 1.更改repo文件 ####

[root@server1 ~]# vim /etc/yum.repos.d/rhel7.3.repo
    ======================================================
    [rhel7.3]
    name=rhel7.3
    baseurl=http://172.25.13.250/rhel7.3/
    gpgcheck=0
    
    [rhel7.3-HighAvailability]
    name=rhel7.3
    baseurl=http://172.25.13.250/rhel7.3/addons/HighAvailability
    gpgcheck=0
    
    [rhel7.3-ResilientStorage]
    name=rhel7.3
    baseurl=http://172.25.13.250/rhel7.3/addons/ResilientStorage
    gpgcheck=0

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 27]

#### 2.查看repo列表 ####

[root@server1 mfs]# yum clean all
    [root@server1 mfs]# yum repolist

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 28]

#### 3.将repo文件发送给server4 ####

[root@server1 mfs]# scp /etc/yum.repos.d/rhel7.3.repo server4:/etc/yum.repos.d/
    root@server4's password: 
    rhel7.3.repo                                  100%  299     0.3KB/s   00:00

![在这里插入图片描述][20190518165424670.png]

#### 4.安装高可用所需软件 ####

[root@server1 mfs]# yum install -y pacemaker corosync pcs

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 29]  
![在这里插入图片描述][20190518165610270.png]

#### 5.制作免密密钥，方便连接 ####

[root@server1 .ssh]# ssh-keygen
    [root@server1 .ssh]# ssh-copy-id server4

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 30]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 31]

#### 6.配置集群 ####

[root@server1 .ssh]# systemctl start pcsd.service 
    [root@server1 .ssh]# systemctl enable pcsd.service 
    Created symlink from /etc/systemd/system/multi-user.target.wants/pcsd.service to /usr/lib/systemd/system/pcsd.service.
    [root@server1 .ssh]# passwd hacluster

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 32]

*  配置backup\_MFSmaster

#### 1.安装moosefs-master ####

[root@server4 ~]# ls
    moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm
    [root@server4 ~]# yum install -y moosefs-master-3.0.103-1.rhsystemd.x86_64.rpm

#### 2.查看yum源列表 ####

[root@server4 ~]# yum clean all
    [root@server4 ~]# yum repolist

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 33]

#### 3.安装高可用所需软件 ####

[root@server4 ~]# yum install -y pacemaker corosync pcs

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 34]  
![在这里插入图片描述][20190518170747948.png]

#### 4.配置集群 ####

[root@server4 ~]# systemctl start pcsd.service 
    [root@server4 ~]# systemctl enable pcsd.service 
    Created symlink from /etc/systemd/system/multi-user.target.wants/pcsd.service to /usr/lib/systemd/system/pcsd.service.
    [root@server4 ~]# passwd hacluster

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 35]

*  创建集群

#创建集群
    [root@server1 3.0.103]# pcs cluster auth server1 server4
    #集群修改名称
    [root@server1 3.0.103]# pcs cluster setup --name mycluster server1 server4

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 36]  
![在这里插入图片描述][20190518171133301.png]

*  查看集群状态

#查看集群状态，有报错，是因为有部分服务没有开启
    [root@server1 3.0.103]# pcs status nodes
    Error: error running crm_mon, is pacemaker running?
    #开启所有服务
    [root@server1 3.0.103]# pcs cluster start --all
    #再次查看节点信息
    [root@server1 3.0.103]# pcs status nodes

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 37]

#验证corosync是否正常
    [root@server1 3.0.103]# corosync-cfgtool -s

![在这里插入图片描述][20190518171518146.png]

#查看corosync状态
    [root@server1 3.0.103]# pcs status corosync

![在这里插入图片描述][20190518171615364.png]

*  创建主备集群

#检查配置,有报错
    [root@server1 3.0.103]# crm_verify -L -V
    #更改属性，禁用STONITH
    [root@server1 3.0.103]# pcs property set stonith-enabled=false
    #再次检查，没有报错
    [root@server1 3.0.103]# crm_verify -L -V
    #查看状态
    [root@server1 3.0.103]# pcs status

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 38]

[root@server1 3.0.103]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=172.25.13.100 cidr_netmask=32 op monitor interval=30s
    #查看，创建成功
    [root@server1 3.0.103]# ip a

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 39]

#查看监控
    [root@server1 3.0.103]# crm_mon

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 40]

#### 5.执行故障转移 ####

#关闭master，查看监控vip会自动漂移
    [root@server1 ~]# pcs cluster start server1
    server1: Starting Cluster...

#在server4查看
    [root@server4 ~]# crm_mon

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 41]

#master重新开启，不会竞争资源
    [root@server1 ~]# pcs cluster start server1
    server1: Starting Cluster...
    
    #关闭backup_MFSmaster，vip又自动漂移到master上
    [root@server4 ~]# pcs cluster stop server4
    server4: Stopping Cluster (pacemaker)...
    server4: Stopping Cluster (corosync)...

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 42]  
以上实验我们实现了MFS的高可用，当master出故障时，backup-master会立刻接替master的工作，保证客户端可以正常得到服务。但master恢复时不会与backup\_MFSmaster竞争资源，保证了用户服务的稳定性。

#获取可用资源标准列表
    [root@server1 ~]# pcs resource standards

![在这里插入图片描述][20190518180047812.png]

#查看资源提供者的列表
    [root@server1 ~]# pcs resource providers

![在这里插入图片描述][20190518180236248.png]

#查看特定的可用资源代理
    [root@server1 ~]# pcs resource agents ocf:heartbeat

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 43]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 44]

### 五、共享存储（使用VIP方式实现共享） ###

#### 1.恢复环境，配置解析 ####

*  client

[root@foundation13 ~]# umount /mnt/mfsmeta/
    [root@foundation13 ~]# umount /mnt/mfs
    [root@foundation13 ~]# vim /etc/hosts
    ==========================================
    172.25.13.100 mfsmaster

*  server1

[root@server1 ~]# systemctl stop moosefs-master.service
    [root@server1 ~]# vim /etc/hosts
    =================================
    172.25.13.100  mfsmaster

*  server2

[root@server2 ~]# systemctl stop moosefs-chunkserver.service
    [root@server2 ~]# vim /etc/hosts
    ==================================
    172.25.13.100  mfsmaster

*  server3

[root@server3 ~]# systemctl stop moosefs-chunkserver.service
    [root@server3 ~]# vim /etc/hosts
    =================================
    172.25.13.100  mfsmaster

#### 2.给chunkserver（server2）添加一块磁盘 ####

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 45]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 46]

#### 3.查看是否成功添加磁盘 ####

[root@server2 ~]# fdisk -l

![在这里插入图片描述][20190518205430297.png]

#### 4.配置共享存储服务 ####

#安装远程块存储设备
    [root@server2 ~]# yum install -y targetcli
    #开启服务
    [root@server2 ~]# systemctl start target.service
    #配置iSCSI服务 
    [root@server2 ~]# targetcli 
    Warning: Could not load preferences file /root/.targetcli/prefs.bin.
    targetcli shell version 2.1.fb41
    Copyright 2011-2013 by Datera, Inc and others.
    For help on commands, type 'help'.
    
    /> cd backstores/block 
    /backstores/block> create my_disk1 /dev/vda
    Created block storage object my_disk1 using /dev/vda.
    /backstores/block> cd ../../
    /> cd iscsi 
    /iscsi> create iqn.2019-05.com.example:server2
    Created target iqn.2019-05.com.example:server2.
    Created TPG 1.
    Global pref auto_add_default_portal=true
    Created default portal listening on all IPs (0.0.0.0), port 3260.
    /iscsi> cd iqn.2019-05.com.example:server2/
    /iscsi/iqn.20...ample:server2> cd tpg1/luns 
    /iscsi/iqn.20...er2/tpg1/luns> create /backstores/block/my_disk1 
    Created LUN 0.
    /iscsi/iqn.20...er2/tpg1/luns> cd ../acls 
    /iscsi/iqn.20...er2/tpg1/acls> create iqn.2019-05.com.example:client
    Created Node ACL for iqn.2019-05.com.example:client
    Created mapped LUN 0.
    /iscsi/iqn.20...er2/tpg1/acls> cd 
    
    /iscsi/iqn.20...er2/tpg1/acls> cd ..
    /iscsi/iqn.20...:server2/tpg1> cd ..
    /iscsi/iqn.20...ample:server2> cd ..
    /iscsi> cd ..
    /> ls
    /> exit 
    Global pref auto_save_on_exit=true
    Last 10 configs saved in /etc/target/backup.
    Configuration saved to /etc/target/saveconfig.json
    [root@server2 ~]#

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 47]

*  配置master，使之可以使用共享磁盘服务

#### 1.在master（server1）上安装iscsi客户端软件 ####

[root@server1 ~]# yum install -y iscsi-*
    [root@server1 ~]# vim /etc/iscsi/initiatorname.iscsi
    ============================================
    InitiatorName=iqn.2019-05.com.example:client
    
    # 发现远程设备
    [root@server1 ~]# iscsiadm -m discovery -t st -p 172.25.13.2:3260
    172.25.13.2:3260,1 iqn.2019-05.com.example:server2
    # 登录
    [root@server1 ~]# iscsiadm -m node -l
    Logging in to [iface: default, target: iqn.2019-05.com.example:server2, portal: 172.25.13.2,3260] (multiple)
    Login to [iface: default, target: iqn.2019-05.com.example:server2, portal: 172.25.13.2,3260] successful.
    [root@server1 ~]#

![在这里插入图片描述][20190518215635160.png]

#### 2.查看远程共享出来的磁盘 ####

[root@server1 ~]# fdisk -l

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 48]

#### 3.使用磁盘，建立分区 ####

[root@server1 ~]# fdisk /dev/sdb

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 49]

#### 4.格式化分区 ####

[root@server1 ~]# mkfs.xfs /dev/sdb1

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 50]

#### 5.挂载 ####

[root@server1 ~]# mount /dev/sdb1 /mnt/
    [root@server1 ~]# df

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 51]

#### 6.将mfs目录下的文件复制到/mnt ####

[root@server1 ~]# cd /var/lib/mfs/
    [root@server1 mfs]# cp -p * /mnt/
    [root@server1 mfs]# cd /mnt/
    [root@server1 mnt]# ll

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 52]

#### 7.修改/mnt目录所有者与所属组，卸载共享设备，并挂载到新的目录 ####

# 当目录属于mfs用户和组时，才能正常使用
    [root@server1 mnt]# chown mfs.mfs /mnt
    [root@server1 mnt]# ll -d /mnt
    drwxr-xr-x 2 mfs mfs 256 May 19 06:04 /mnt
    [root@server1 mnt]# cd 
    [root@server1 ~]# umount /mnt/
    # 使用分区，测试是否可以使用共享磁盘
    [root@server1 ~]# mount /dev/sdb1 /var/lib/mfs/
    [root@server1 ~]# df -h

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 53]

#### 8.开启moosefs-master服务，并查看进程 ####

#服务开启成功，就说明数据文件拷贝成功，共享磁盘可以正常使用
    [root@server1 ~]# systemctl start moosefs-master
    [root@server1 ~]# ps ax

![在这里插入图片描述][20190518221106568.png]  
关闭服务

[root@server1 ~]# systemctl stop moosefs-master.service

*  配置backup-master（server4），使之也可以使用共享磁盘

#### 1.安装共享设备客户端软件，修改解析 ####

#修改解析
    [root@server4 ~]# vim /etc/hosts
    =================================
    172.25.13.100  mfsmaster
    #安装iscsi客户端软件
    [root@server4 ~]# yum install -y iscsi-*
    [root@server4 ~]# vim /etc/iscsi/initiatorname.iscsi
    ==================================================
    InitiatorName=iqn.2019-04.com.example:client
    #发现远程设备
    [root@server4 ~]# iscsiadm -m discovery -t st -p 172.25.13.2
    172.25.13.2:3260,1 iqn.2019-05.com.example:server2
    #登录
    [root@server4 ~]# iscsiadm -m node -l
    Logging in to [iface: default, target: iqn.2019-05.com.example:server2, portal: 172.25.13.2,3260] (multiple)
    Login to [iface: default, target: iqn.2019-05.com.example:server2, portal: 172.25.13.2,3260] successful.

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 54]

#### 2.查看设备 ####

[root@server4 ~]# fdisk -l

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 55]

#### 3.测试磁盘是否可以正常使用 ####

[root@server4 ~]# mount /dev/sdb1 /var/lib/mfs/ # 此处使用的磁盘和master是同一块，因为master已经做过配置了，所以我们只需要使用即可，不用再次配置
    [root@server4 ~]# systemctl start moosefs-master # 测试磁盘是否可以正常使用
    [root@server4 ~]# systemctl stop moosefs-master.service
    [root@server4 ~]# pcs cluster start server4
    server4: Starting Cluster...

![在这里插入图片描述][20190518222803380.png]

*  master（server1）上创建MFS文件系统

[root@server1 ~]# pcs resource create mfsdata ocf:heartbeat:Filesystem device=/dev/sdb1 directory=/var/lib/mfs fstype=xfs op monitor interval=30
    [root@server1 ~]# pcs resource show
    [root@server1 ~]# crm_mon

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 56]  
![在这里插入图片描述][20190518223628735.png]

# 创建mfsd系统
    [root@server1 ~]# pcs resource create mfsd systemd:moosefs-master op monitor interval=1min

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 57]

# 把vip，mfsdata，mfsd 集中在一个组中
    [root@server1 ~]# pcs resource group add mfsgroup vip mfsdata mfsd
    # 当关闭master之后，master上的服务就会迁移到backup-master上
    pcs cluster stop server1

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 58]  
![在这里插入图片描述][20190518224209335.png]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 59]

### 六、fence解决脑裂问题 ###

1.开启master

[root@server1 cluster]# systemctl start moosefs-master

2.客户端挂载共享设备，并上传文件

[root@vits mnt]# mfsmount 
    mfsmaster accepted connection with parameters: read-write,restricted_ip,admin ; root mapped to root:root
    [root@vits mnt]# cd /mnt/mfs/dir1/
    [root@vits dir1]# dd if=/dev/zero of=file2 bs=1M count=500
    500+0 records in
    500+0 records out
    524288000 bytes (524 MB) copied, 2.90685 s, 180 MB/s

![在这里插入图片描述][20190518233618870.png]  
3.关闭提供服务的服务端，查看文件

[root@server4 ~]# pcs cluster stop server4
    [root@vits dir1]# mfsfileinfo file2

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 60]

*  配置fence  
    1.两个master端安装fence服务

[root@server1 3.0.103]# yum install -y fence-virt
    [root@server1 3.0.103]# mkdir /etc/cluster
    ==================================================
    [root@server4 ~]# yum install -y fence-virt
    [root@server4 ~]# mkdir /etc/cluster

2.客户端生成fence密钥文件，传给服务端

[root@vits ~]# yum install -y fence-virtd
    [root@vits ~]# yum install fence-virtd-libvirt.x86_64
    [root@vits ~]# yum install -y fence-virtd-multicast.x86_64
    [root@vits ~]# fence_virtd -c
    ================================
    Interface [virbr0]: br0
    ==================================

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 61]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 62]

3.client端创建存放密钥的目录，并把生成的密钥发送给master端

[root@vits ~]# dd if=/dev/urandom of=/etc/cluster/fence_xvm.key bs=128 count=1
    [root@vits ~]# systemctl start fence_virtd.service
    [root@vits cluster]# scp fence_xvm.key 172.25.13.1:/etc/cluster/
    [root@vits cluster]# scp fence_xvm.key 172.25.13.4:/etc/cluster/
    [root@vits cluster]# netstat -anulp | grep 1229
    udp        0      0 0.0.0.0:1229            0.0.0.0:*                           8248/fence_virtd

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 63]

4.client查看主机域名

[root@vits cluster]# virsh list

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 64]  
5.配置fence服务，并使server4强制重启

[root@server1 cluster]# fence_xvm -H server4_mh

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 65]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70]: /images/20220130/af12f49033644ae5b8d98a4e3aa14a59.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 1]: /images/20220130/79640fafd9e149b1a2c536be7b296199.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 2]: /images/20220130/516a891a750d422d9b83194b5e99a069.png
[http_172.25.13.1_9425]: http://172.25.13.1:9425
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 3]: /images/20220130/3192da1960014f1291f6a4d58f5c7fb6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 4]: /images/20220130/9923dea7506e4698b03dd7af3c3bd692.png
[20190518155104899.png]: /images/20220130/c5dfd3fe3fd748f7bef601c1da5ea913.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 5]: /images/20220130/66b287e3c81447188ba1f338b9fae1e1.png
[20190518155207349.png]: /images/20220130/1daf4dda43d648268ae470a9e1252362.png
[20190518155415142.png]: /images/20220130/029508394f064c86ad94833a99fbf4de.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 6]: /images/20220130/897877c7087b4a0bbcdea3a57642ceb4.png
[20190518155330456.png]: /images/20220130/e6673a449cf8438c9a4cfe4df782dd11.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 7]: /images/20220130/2b7792a0ee2042feb3c7365cc10c84e3.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 8]: /images/20220130/abf078d1e3084f23be6b590e35eef6f0.png
[20190518155530715.png]: /images/20220130/1c17021323d54de4aade2ba860155fe0.png
[20190518155906705.png]: /images/20220130/0af1adc729014c6085b5eae6b5b4f055.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 9]: /images/20220130/6509b8b798644bee9d79c53b6e3500b0.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 10]: /images/20220130/5ff1cb47cc734f03a86eec4515aa18da.png
[20190518160055995.png]: /images/20220130/4e3e3fbee4294af094c9633d336ec77d.png
[20190518160127879.png]: /images/20220130/0153930f3dec4093ab11b16f143781d7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 11]: /images/20220130/32e02c609708420ab17eb81e59539c22.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 12]: /images/20220130/1667c85d5e824e1fa535cfb23db1a037.png
[20190518160243156.png]: /images/20220130/0ed67f3c27f844b693c164639dc021ed.png
[20190518160303158.png]: /images/20220130/44d92034825b4415bafbce061663b6a4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 13]: /images/20220130/2633c170bfcb4d81a8ed62a99e0ac510.png
[20190518160405211.png]: /images/20220130/d89bd79d2ede4d849de1f443ba240d43.png
[20190518160426825.png]: /images/20220130/a16d16c9081144439acd7c02d5da6711.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 14]: /images/20220130/16f857c177af4def8bad8809df93eecb.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 15]: /images/20220130/b1d55f8c23d64197a52ca6f4eaa08338.png
[2019051816070586.png]: /images/20220130/15be7c59bea545b5b24c6c2c52768562.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 16]: /images/20220130/70e18db5345b42ed86e868896a7e3aa8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 17]: /images/20220130/8cb815268cb1474cbc24e40303f21d73.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 18]: /images/20220130/6776c0c045444a508f03a94bb9266bf7.png
[20190518160938622.png]: /images/20220130/ea1c917b185a44fa88e99b9862e05caa.png
[20190518161016863.png]: /images/20220130/e340c68f0eec4f16a677a4727409c7c6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 19]: /images/20220130/5602e32f4ade4d5daadf27835e1622f0.png
[20190518161154915.png]: /images/20220130/ae12193ad5d34b998ad818bf0e39dfdf.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 20]: /images/20220130/b99b3f71abe04219889f567266889525.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 21]: /images/20220130/229d4f77081146a8bbfa67ec97cd7101.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 22]: /images/20220130/328f493490184206b13595bd0198d946.png
[20190518161546734.png]: /images/20220130/e2383621cdef4fa9a89932f8a9030b3e.png
[20190518161626170.png]: /images/20220130/74955f0c47884837b17c15d047dab2ff.png
[20190518161709527.png]: /images/20220130/58a442fa1cdc4f00973ac4b441bb74e8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 23]: /images/20220130/398240d34a624b77a26deefcdadaeea5.png
[20190518161819521.png]: /images/20220130/757bba02d9c947e2a80835b296346039.png
[20190518161857519.png]: /images/20220130/8a3023cf777d46ec8e816156da7b4462.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 24]: /images/20220130/9565e9fb0c4d4356b1d45a50dd5292fd.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 25]: /images/20220130/7c539a0970384ec9a7579a0aa916fe0a.png
[20190518162301838.png]: /images/20220130/9cb9e6307984432f9c709029e47beaec.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 26]: /images/20220130/96e997c93af64888bbc8ef9b335c6cc7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 27]: /images/20220130/6bd7995a3d6d478a9c243bd9ae4df784.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 28]: /images/20220130/2b38cebaee5f4546bc616e98f9c0bb8c.png
[20190518165424670.png]: /images/20220130/9dab46d29f0e4334aaff50a644b32533.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 29]: /images/20220130/71d50694850a4b18a97532965aa3057c.png
[20190518165610270.png]: /images/20220130/46348ff83c7e477780af9e69fc1b34cc.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 30]: /images/20220130/442f93b727b44f6486a485e1f32d1211.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 31]: /images/20220130/eafa1506e20648bda02905dffc00166f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 32]: /images/20220130/87ce538edaf945e3af246d6590c7263c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 33]: /images/20220130/bb814dce974a4cdfaeffa7b5f4ba6ca1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 34]: /images/20220130/f1adc7e9fee84319a7b500c363d1276e.png
[20190518170747948.png]: /images/20220130/74c3ab0b63534a93bdb8f2e225df181b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 35]: /images/20220130/2999625db7dd41b98ac087d79ce8ef51.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 36]: /images/20220130/26ccec578f4448d5b382826357a9091a.png
[20190518171133301.png]: /images/20220130/14d7ca9e9a9c4749a5af672302f06303.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 37]: /images/20220130/fde76dd3ef9d444ba418efbe7b915592.png
[20190518171518146.png]: /images/20220130/ff0ee66033d845d3a949d6c88c52d3fa.png
[20190518171615364.png]: /images/20220130/9f3d21da99af4a5ba45c738a2e374e53.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 38]: /images/20220130/27761b6a55bc47f5b0b1bb5eb99f18ac.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 39]: /images/20220130/710e05d28b884792a00790f40903c8b9.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 40]: /images/20220130/d38683d2726842baad95c1575576ac27.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 41]: /images/20220130/5c550ba4895b4f48b8c351c8bffa78e7.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 42]: /images/20220130/31488a19fe1f46bfa11265b3d2e43f10.png
[20190518180047812.png]: /images/20220130/062c0372043f4a1088fb52f3e0819821.png
[20190518180236248.png]: /images/20220130/391b32e24ae04aa5bf61fa14d2894008.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 43]: /images/20220130/647a1e4e2a294270a9119a6ee2d69118.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 44]: /images/20220130/f496544312a5471cac3c1803681f40da.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 45]: /images/20220130/c6fdc5f77e6c4c53a7696eb7d9bfd742.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 46]: /images/20220130/a0d13b9a9ca743aabba753caf35d0c77.png
[20190518205430297.png]: /images/20220130/f5696d26a46a472c98d3362fdd9a93b4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 47]: /images/20220130/2b15386e50da47a99794054391bb8e5e.png
[20190518215635160.png]: /images/20220130/6f32b9ad1a1441d4b2fb45fa1eca81d4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 48]: /images/20220130/1805d5209d1046b6a19baba78e232c60.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 49]: /images/20220130/4b562ce9a0fb4b13b1f2aa674338cb8e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 50]: /images/20220130/3cc3e4fd23f5476295ca705ba667b394.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 51]: /images/20220130/49c9344625664123941b43037e1ecfb0.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 52]: /images/20220130/228ce560e5cb4384affa2693979dfc12.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 53]: /images/20220130/90f792f1801c48f5b0ebedad51ae8eb7.png
[20190518221106568.png]: /images/20220130/1135ce18f30048e99afdce7d4642abc2.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 54]: /images/20220130/2ff14aaf23bf4f459862ec805d461a79.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 55]: /images/20220130/c40bffc3d5a04b6b8f65b5ad1dec22c4.png
[20190518222803380.png]: /images/20220130/265a95cfe50c478e9964269bc0180348.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 56]: /images/20220130/d8980a0b85a1485cb9a9826f1eb015ef.png
[20190518223628735.png]: /images/20220130/05caef8581a34360b20b6d604ea5a482.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 57]: /images/20220130/97c0608af23f412599c06c7c68b06743.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 58]: /images/20220130/5ad632274e494830ae7e6940989425e5.png
[20190518224209335.png]: /images/20220130/20cd7e2948b849a0bcb33b7a04dabe47.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 59]: /images/20220130/06866b40fe47461fa34640d81436e95f.png
[20190518233618870.png]: /images/20220130/ee5d69c09b34442999f670748c190e67.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 60]: /images/20220130/c2a22b296352444685ca0125d404d3e1.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 61]: /images/20220130/cc67f0fb874f4522819d343f18525bb2.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 62]: /images/20220130/8a88739539e94375bf6bddf47afe0632.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 63]: /images/20220130/11fd31a018974707b838f388d1c424ec.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 64]: /images/20220130/c9becc3b4ae144d1b3b534daa44d0069.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQzNTcwMzY5_size_16_color_FFFFFF_t_70 65]: /images/20220130/52e7b2f0cabc40509fa35dc5edc4c65b.png