hadoop大数据集群完全分布式部署实操篇：HDFS2.9.2、HBASE2.2.6、YARN2.9.2、SPARK2.4.7，ZOOKEEPER3.6.2

女爷i 2023-10-05 15:51 6阅读 0赞

*  推荐以rpm方式安装jdk，有些软件比如cm只认`/usr/java/default`目录
 *  配置`/etc/hosts`文件
 *  配置ssh免密登录
 *  配置时间同步

## HDFS安装 ##

*  单机伪分布式安装
    
     *  ssh远程登录找不到$JAVA\_HOME环境变量，所以要写成绝对路径

[root@cm-m0 home]# ll
    总用量 745248
    drwxr-xr-x 3 root root      4096 12月  2 11:15 cm-install
    -rwxr-xr-x 1 root root 366447449 12月 14 12:44 hadoop-2.9.2.tar.gz
    -rwxr-xr-x 1 root root 220469021 12月 14 11:52 hbase-2.2.6-bin.tar.gz
    -rw-r--r-- 1 root root 176209195 11月 27 09:22 jdk-8u201-linux-x64.rpm
    drwxr-xr-x 2 root root        56 10月 20 10:24 ntp-install
    [root@cm-m0 home]# mkdir /opt/bigdata
    [root@cm-m0 home]# vim /etc/profile
        export JAVA_HOME=/usr/java/default
        export HADOOP_HOME=/opt/bigdata/hadoop-2.9.2
        export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
    [root@cm-m0 home]# source /etc/profile
    [root@cm-m0 home]# cd $HADOOP_HOME/etc/hadoop 
    [root@cm-m0 hadoop]# ll
    总用量 160
    -rw-r--r-- 1 501 dialout  7861 11月 13 2018 capacity-scheduler.xml
    -rw-r--r-- 1 501 dialout  1335 11月 13 2018 configuration.xsl
    -rw-r--r-- 1 501 dialout  1211 11月 13 2018 container-executor.cfg
    -rw-r--r-- 1 501 dialout   774 11月 13 2018 core-site.xml
    -rw-r--r-- 1 501 dialout  4133 11月 13 2018 hadoop-env.cmd
    -rw-r--r-- 1 501 dialout  4969 11月 13 2018 hadoop-env.sh
    -rw-r--r-- 1 501 dialout  2598 11月 13 2018 hadoop-metrics2.properties
    -rw-r--r-- 1 501 dialout  2490 11月 13 2018 hadoop-metrics.properties
    -rw-r--r-- 1 501 dialout 10206 11月 13 2018 hadoop-policy.xml
    -rw-r--r-- 1 501 dialout   775 11月 13 2018 hdfs-site.xml
    -rw-r--r-- 1 501 dialout  2230 11月 13 2018 httpfs-env.sh
    -rw-r--r-- 1 501 dialout  1657 11月 13 2018 httpfs-log4j.properties
    -rw-r--r-- 1 501 dialout    21 11月 13 2018 httpfs-signature.secret
    -rw-r--r-- 1 501 dialout   620 11月 13 2018 httpfs-site.xml
    -rw-r--r-- 1 501 dialout  3518 11月 13 2018 kms-acls.xml
    -rw-r--r-- 1 501 dialout  3139 11月 13 2018 kms-env.sh
    -rw-r--r-- 1 501 dialout  1788 11月 13 2018 kms-log4j.properties
    -rw-r--r-- 1 501 dialout  5939 11月 13 2018 kms-site.xml
    -rw-r--r-- 1 501 dialout 14016 11月 13 2018 log4j.properties
    -rw-r--r-- 1 501 dialout  1076 11月 13 2018 mapred-env.cmd
    -rw-r--r-- 1 501 dialout  1507 11月 13 2018 mapred-env.sh
    -rw-r--r-- 1 501 dialout  4113 11月 13 2018 mapred-queues.xml.template
    -rw-r--r-- 1 501 dialout   758 11月 13 2018 mapred-site.xml.template
    -rw-r--r-- 1 501 dialout    10 11月 13 2018 slaves
    -rw-r--r-- 1 501 dialout  2316 11月 13 2018 ssl-client.xml.example
    -rw-r--r-- 1 501 dialout  2697 11月 13 2018 ssl-server.xml.example
    -rw-r--r-- 1 501 dialout  2250 11月 13 2018 yarn-env.cmd
    -rw-r--r-- 1 501 dialout  4876 11月 13 2018 yarn-env.sh
    -rw-r--r-- 1 501 dialout   690 11月 13 2018 yarn-site.xml
    [root@cm-m0 hadoop]#
    [root@cm-m0 hadoop]# vim hadoop-env.sh 
        export JAVA_HOME=/usr/java/default
    [root@cm-m0 hadoop]# vim core-site.xml
        <configuration>
            <property>
                <name>fs.defaultFS</name>
                <value>hdfs://cm-m0:9000</value>
            </property>
        </configuration>
    [root@cm-m0 hadoop]# vim hdfs-site.xml
        <configuration>
            <property>
                <name>dfs.replication</name>
                <value>1</value>
            </property>
            <property>
    			<name>dfs.namenode.name.dir</name>
    			<value>/var/bigdata/hadoop/local/dfs/name</value>
    		</property>
    		<property>
    			<name>dfs.datanode.data.dir</name>
    			<value>/var/bigdata/hadoop/local/dfs/data</value>
    		</property>
    		<property>
    			<name>dfs.namenode.secondary.http-address</name>
    			<value>cm-m0:50090</value>
    		</property>
    		<property>
    			<name>dfs.namenode.checkpoint.dir</name>
    			<value>/var/bigdata/hadoop/local/dfs/secondary</value>
    		</property>
        </configuration>
    [root@cm-m0 hadoop]# vim slaves
        cm-m0
    [root@cm-m0 hadoop]# cd /var
    [root@cm-m0 var]# ls
    adm    crash  empty  gopher    lib    lock  mail  opt       run    tmp
    cache  db     games  kerberos  local  log   nis   preserve  spool  yp
    [root@cm-m0 var]# hdfs namenode -format
    [root@cm-m0 var]# start-dfs.sh 
    Starting namenodes on [cm-m0]
    cm-m0: starting namenode, logging to /opt/bigdata/hadoop-2.9.2/logs/hadoop-root-namenode-cm-m0.out
    cm-m0: starting datanode, logging to /opt/bigdata/hadoop-2.9.2/logs/hadoop-root-datanode-cm-m0.out
    Starting secondary namenodes [cm-m0]
    cm-m0: starting secondarynamenode, logging to /opt/bigdata/hadoop-2.9.2/logs/hadoop-root-secondarynamenode-cm-m0.out
    [root@cm-m0 var]# jps
    4849 DataNode
    5473 Jps
    4700 NameNode
    5038 SecondaryNameNode
    [root@cm-m0 var]#
    访问：http://192.168.101.119:50070/

*  完全分布式安装

[root@cm-m0 var]# stop-dfs.sh 
    Stopping namenodes on [cm-m0]
    cm-m0: stopping namenode
    cm-m0: stopping datanode
    Stopping secondary namenodes [cm-m0]
    cm-m0: stopping secondarynamenode
    [root@cm-m0 var]# jps
    7018 Jps
    [root@cm-m0 var]#
    [root@cm-m0 var]# cd /opt/bigdata/hadoop-2.9.2/etc/hadoop
    [root@cm-m0 hadoop]# vim hdfs-site.xml 
        <configuration>
            <property>
                <name>dfs.replication</name>
                <value>2</value>
            </property>
            <property>
                    <name>dfs.namenode.name.dir</name>
                    <value>/var/bigdata/hadoop/full/dfs/name</value>
            </property>
            <property>
                    <name>dfs.datanode.data.dir</name>
                    <value>/var/bigdata/hadoop/full/dfs/data</value>
            </property>
            <property>
                    <name>dfs.namenode.secondary.http-address</name>
                    <value>cm-n0:50090</value>
            </property>
            <property>
                    <name>dfs.namenode.checkpoint.dir</name>
                    <value>/var/bigdata/hadoop/full/dfs/secondary</value>
            </property>
        </configuration>
    [root@cm-m0 hadoop]# vim slaves
        cm-n0
        cm-n1
    [root@cm-m0 hadoop]# cd /opt
    [root@cm-m0 opt]# rm -fr $HADOOP_HOME/share/doc/
    [root@cm-m0 opt]# ansible all -m shell -a 'mkdir /opt/bigdata'
    cm-m0 | CHANGED | rc=0 >>
    cm-n1 | CHANGED | rc=0 >>
    cm-n0 | CHANGED | rc=0 >>
    [root@cm-m0 opt]#
    [root@cm-m0 opt]# scp -r bigdata/hadoop-2.9.2 cm-n0:/opt/bigdata
    [root@cm-m0 opt]# scp -r bigdata/hadoop-2.9.2 cm-n1:/opt/bigdata
    [root@cm-m0 opt]# hdfs namenode -format
    ...
    20/12/14 15:38:22 INFO common.Storage: Storage directory /var/bigdata/hadoop/full/dfs/name has been successfully formatted.
    20/12/14 15:38:22 INFO namenode.FSImageFormatProtobuf: Saving image file /var/bigdata/hadoop/full/dfs/name/current/fsimage.ckpt_0000000000000000000 using no compression
    20/12/14 15:38:22 INFO namenode.FSImageFormatProtobuf: Image file /var/bigdata/hadoop/full/dfs/name/current/fsimage.ckpt_0000000000000000000 of size 323 bytes saved