搭建大数据集群环境(记录)

傷城~ 2022-04-24 12:12 240阅读 0赞

全文：

记录一下依赖关系、安装思路。

一、主机配置

# service iptables stop     关闭防火墙
    # vim /etc/sysconfig/network
        HOSTNAME=hadoop01       修改本机名
    # hostname hadoop01         立即生效
    # source /etc/sysconfig/network    永久生效
    # vim /etc/hosts
    
        192.168.220.128 hadoop01
        192.168.220.129 hadoop02
        192.168.220.130 hadoop03

二、免密配置

# ssh-keygen                   生成的文件会自动放在/root/.ssh目录下
         一直回车ENTER生成
    # ssh-copy-id root@hadoop02    把公钥发往另外两台远程机器，输入远程机器主机名及密码(hosts文件)
    # ssh-copy-id root@hadoop03

三、JDK配置

# cd /home/software/
    # tar -xzvf jdk-8u131-linux-x64.tar.gz    生成jdk1.8.0_131文件夹
    # vim /etc/profile
    
    export JAVA_HOME=/home/software/jdk1.8.0_131
    export JRE_HOME=/home/software/jdk1.8.0_131/jre
    export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
    export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
    
    # source /etc/profile                    立即生效
    # java -version                          显示则成功

四、Zookeeper配置

# cd /home/software/
    # tar xzvf zookeeper-3.4.7.tar.gz
    # cd zookeeper-3.4.7/conf/            进入配置文件
    # cp zoo_sample.cfg zoo.cfg           复制样本
    # vim conf/zoo.cfg    
    
        2 tickTime=2000                                   zookeeper中使用的基本时间单位, 毫秒值.
        5 initLimit=10                                    初始化连接时 follower和leader之间的最长心跳时间
        8 syncLimit=5                                     follow和leader之间通信应答时间
        12 dataDir=/home/software/zookeeper-3.4.7/tmp     指定数据目录
        14 clientPort=2181                                监听client连接的端口号：默认2181
    
        29 server.1=192.168.220.129:2888:3888   server.选举id：选举中使用，需比较大小，不能重复
        30 server.2=192.168.220.130:2888:3888   原子广播端口：2888
        31 server.3=192.168.220.131:2888:3888   选举端口：3888
    
    # cd ..
    # mkdir tmp                        创建数据目录
    # vim tmp/myid
    
        1    节点编号，启动优先级，不能相同
    
    # vim /etc/profile
    
        export ZOOKEEPER_HOME=/home/software/zookeeper-3.4.7
        export PATH=$ZOOKEEPER_HOME/bin
    
    # source /etc/profile                    立即生效
    # zkServer.sh start                   启动zookeeper服务器

五、Hadoop配置

# cd /home/software/
    # tar -xzvf hadoop-2.7.1_64bit.tar.gz
    # mv hadoop-2.7.1 hadoop.cluster
    # cd hadoop.cluster/etc/hadoop            Hadoop配置文件目录
    
    # vim hadoop-env.sh
    
        25 export JAVA_HOME=/home/software/jdk1.8.0_131                        绑定jdk路径
        33 export HADOOP_CONF_DIR=/home/software/hadoop-2.7.1/etc/hadoop       指定配置目录
    
    # source hadoop-env.sh                 配置文件生效
    # vim core-site.xml
    
    	
    	<property>
    		<name>fs.defaultFS</name>
    		<value>hdfs://ns</value>
    	</property>
    	
    	<property>
    		<name>hadoop.tmp.dir</name>
    		<value>/home/software/hadoop.cluster/tmp</value>
    	</property>
    	
    	<property>
    		<name>ha.zookeeper.quorum</name>
    		<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
    	</property>
    
    # vim hdfs-site.xml
    
    	
    	<property>
    		<name>dfs.nameservices</name>
    		<value>ns</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.ha.namenodes.ns</name>
    		<value>nn1,nn2</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.rpc-address.ns.nn1</name>
    		<value>hadoop01:9000</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.http-address.ns.nn1</name>
    		<value>hadoop01:50070</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.rpc-address.ns.nn2</name>
    		<value>hadoop02:9000</value>
    	</property>
    	
    	<property>
    		<name>dfs.namenode.http-address.ns.nn2</name>
    		<value>hadoop02:50070</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.namenode.shared.edits.dir</name>
    		<value>qjournal://hadoop01:8485;hadoop02:8485;hadoop03:8485/ns</value>
    	</property>
    	
    	<property>
    		<name>dfs.journalnode.edits.dir</name>
    		<value>/home/software/hadoop.cluster/journal</value>
    	</property>
    	
    	<property>
    		<name>dfs.ha.automatic-failover.enabled</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>dfs.client.failover.proxy.provider.ns</name>
    		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.ha.fencing.methods</name>
    		<value>sshfence</value>
    	</property>
    	
    	<property>
    		<name>dfs.ha.fencing.ssh.private-key-files</name>
    		<value>/root/.ssh/id_rsa</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.namenode.name.dir</name>
    		<value>file:///home/software/hadoop.cluster/tmp/namenode</value>
    	</property>
    	
    	<property>
    		<name>dfs.datanode.data.dir</name>
    		<value>file:///home/software/hadoop.cluster/tmp/datanode</value>
    	</property>
    
    	
    	<property>
    		<name>dfs.replication</name>
    		<value>3</value>
    	</property>
    	
    	<property>
    		<name>dfs.permissions</name>
    		<value>false</value>
    	</property>
    
    # vim mapred-site.xml
    
    	<property>
    		
    		<name>mapreduce.framework.name</name>
    		<value>yarn</value>
    	</property>
    
    # vim yarn-site.xml
    
    	  
    	<property>
    		<name>yarn.resourcemanager.ha.enabled</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.ha.rm-ids</name>
    		<value>rm1,rm2</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.hostname.rm1</name>
    		<value>hadoop01</value>
    	</property>
    	<property>
    		<name>yarn.resourcemanager.hostname.rm2</name>
    		<value>hadoop03</value>
    	</property>
    
    	
    	<property>
    		<name>yarn.resourcemanager.recovery.enabled</name>
    		<value>true</value>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.store.class</name>	            <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
    	</property>
    
    	
    	<property>
    		<name>yarn.resourcemanager.zk-address</name>
    		<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
    		<description>For multiple zk services, separate them with comma</description>
    	</property>
    	
    	<property>
    		<name>yarn.resourcemanager.cluster-id</name>
    		<value>yarn-ha</value>
    	</property>
    	<property>
    		
    		<name>yarn.resourcemanager.hostname</name>
    		<value>hadoop03</value>
    	</property>
    	<property>
    		
    		<name>yarn.nodemanager.aux-services</name>
    		<value>mapreduce_shuffle</value>
    	</property>
    
    # vim slaves          配置从节点
    
        hadoop01
        hadoop02
        hadoop03
    
    # vim /etc/profile    
    
    export HADOOP_HOME=/home/software/hadoop.cluster
    export JAVA_LIBRAY_PATH=/usr/local/hadoop/lib/native
    export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
    
    # source /etc/profile             重新生效
    
    # cd /home/software/hadoop.cluster/          创建相关目录
    # mkdir journal
    # mkdir tmp
    # mkdir namenode
    # mkdir datanode
    # vim /etc/profile
    # scp -r cluster hadoop02:/home/software     发送到其他主机(02、03)上

启动hadoop、格式化

在zookeeper的leader节点上执行
    # hdfs zkfc -formatZK                    格式化zookeeper
    
    在01、02、03节点上执行
    # hadoop-daemon.sh start journalnode     启动journalnode集群
    
    在01节点(192.168.220.128)上执行
    # hadoop namenode -format                格式化01节点的namenode
    # hadoop-daemon.sh start namenode        启动01节点的namenode
    
    在02节点(192.168.220.129)上执行
    # hdfs namenode  -bootstrapStandby       把02节点的 namenode节点变为standby namenode节点
    # hadoop-daemon.sh start namenode        启动02节点的namenode节点
    
    在01,02,03节点上启动datanode节点
    #  hadoop-daemon.sh start datanode
    
    在01,02节点上执行
    # hadoop-daemon.sh start zkfc            启动zkfc（启动FalioverControllerActive)
    
    在01节点上执行
    # start-yarn.sh                          在01节点上启动 主Resourcemanager
    
    在03节点上执行
    # yarn-daemon.sh start resourcemanager   在03节点上启动 副Resoucemanager

结果图：

![20190425120425487.png][]

![201904251204465.png][]

![20190425120507634.png][]

六、Flume配置

# cd /home/software/
    # tar xzvf apache-flume-1.6.0-bin.tar.gz
    # cd /home/software/apache-flume-1.6.0-bin/
    
    # cd conf/
    # vim zebra.conf            配置启动文件
    
    	a1.sources=r1
    	a1.channels=c1
    	a1.sinks=s1
    
    	a1.sources.r1.type=spooldir
    	a1.sources.r1.spoolDir=/home/tmp/zebra        监听linux上文件夹
    	a1.sources.r1.interceptors=i1
    	a1.sources.r1.interceptors.i1.type=timestamp
    
    	a1.channels.c1.type=memory
    
    	a1.sinks.s1.type=hdfs
    	a1.sinks.s1.hdfs.path=hdfs://hadoop01:9000/zebra/reportTime=%Y-%m-%d    输出到HDFS下/zebra文件夹
    	a1.sinks.s1.hdfs.fileType=DataStream
    	a1.sinks.s1.hdfs.rollInterval=30
    	a1.sinks.s1.hdfs.rollSize=0
    	a1.sinks.s1.hdfs.rollCount=0
    
    	a1.sources.r1.channels=c1
    	a1.sinks.s1.channel=c1
    
    # cd ../bin
    # ./flume-ng agent -n a1 -c ../conf -f ../conf/zebra.conf -Dflume.root.logger=INFO,console       启动

六、Mysql配置

# cd /home/softaware/
    # rpm -ivh MySQL-server-5.6.29-1.linux_glibc2.5.x86_64.rpm    安装Mysql服务器
    # rpm -ivh MySQL-client-5.6.29-1.linux_glibc2.5.x86_64.rpm    安装Mysql客户端
    
    # vim /usr/my.cnf
    
    	[client]
        default-character-set=utf8
        [mysql]
        default-character-set=utf8
        [mysqld]
        character_set_server=utf8
    
    # cp /usr/share/mysql/mysql.server /etc/init.d/mysqld        加入系统服务
    # service mysqld start                                       启动mysqI
    # vim /root/.mysql_secret                                    查看密码
    # mysqladmin -u root -p password root
            输入密码
    
    # mysql -uroot -proot                                             登陆
    
    mysql>    create database hive character set latin1;              创建hive数据库
    mysql>    grant all on *.* to 'root'@'%' identified by 'root';    权限开发
    mysql>    flush privileges;                                       执行权限
    
    mysql>    create database hive character set latin1;              创建hive数据库,hive要求存储元数据的字符集必须是iso8859-1

七、Hive配置

# cd /home/software
    # tar xzvf apache-hive-1.2.0-bin.tar.gz
    
    # hadoop fs -rmr /user/hive        删除hdfs中的/user/hive
    # cp mysql-connector-java-5.1.38-bin.jar /home/software/apache-hive-1.2.0-bin/lib/      将mysql驱动包上传到hive安装目录的lib目录下
    
    # cd apache-hive-1.2.0-bin
    # /conf
    # vim hive-site.xml 
    
    	<configuration>
    	<property>
    	<name>javax.jdo.option.ConnectionURL</name>
    	<value>jdbc:mysql://hadoop01:3306/hive?createDatabaseIfNotExist=true</value>
    	</property>
    	<property>
    	<name>javax.jdo.option.ConnectionDriverName</name>
    	<value>com.mysql.jdbc.Driver</value>
    	</property>
    	<property>
    	<name>javax.jdo.option.ConnectionUserName</name> 
    	<value>root</value> 
    	</property>
    	<property>
    	<name>javax.jdo.option.ConnectionPassword</name>
    	<value>root</value>
    	</property>
    	</configuration>
    # cd bin
    # sh hive

八、Sqoop配置

# cd /home/software/
    # tar xzvf sqoop-1.4.4.bin__hadoop-2.0.4-alpha.tar.gz
    
    # cp mysql-connector-java-5.1.38-bin.jar /home/software/sqoop-1.4.4.bin__hadoop-2.0.4-alpha/lib/         将mysql驱动包上传到hive安装目录的lib目录下
    
    
    # vim /etc/profile
    
        export SQOOP_HOME=/home/software/sqoop-1.4.4.bin__hadoop-2.0.4-alpha
        export PATH=$PATH:$SQOOP_HOME/bin
    
    # source /etc/profile

[20190425120425487.png]: /images/20220215/834e85ded055440ba2269c797a0593a3.png
[201904251204465.png]: /images/20220215/c21438d1ea8e43d2a53b5e66c789d7ef.png
[20190425120507634.png]: /images/20220215/b0242dbb4885469a960b89de9b5cf762.png