CentOS下编译hadoop-2.9.2

你的名字 2022-03-01 00:50 221阅读 0赞

# CentOS 下编译 Hadoop #

## 系统环境 ##

系统： CentOS-7.3_x86_64
    protobuf: protoc-2.5.0
    maven： maven-3.6.0
    hadoop: hadoop-2.9.2
    java： jdk1.8.0_131
    cmake: cmake-2.8.12.2
    OpenSSL: OpenSSL 1.0.2k-fips
    findbugs： findbugs-1.3.9

## 准备工作 ##

# 安装编译需要的依赖库
    yum install -y lzo-devel zlib-devel autoconf automake libtool cmake openssl-devel cmake gcc gcc-c++

## 安装maven ##

# 下载maven
    $ wget https://mirrors.shu.edu.cn/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz
    
    # 解压缩
    $ tar -zxvf apache-maven-3.6.0-bin.tar.gz -C /usr/local/
    
    # 配置到系统环境变量
    $ vim /etc/profile
    
    export MAVEN_HOME=/usr/local/apache-maven-3.6.0
    export PATH=$PATH:$MAVEN_HOME/bin
    
    # 刷新配置文件 
    $ source /etc/profile
    
    # 验证maven安装是是否成功
    $ mvn -version
    
    [root@localhost ~]# mvn -version
    Apache Maven 3.6.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2018-10-25T03:39:06+08:00)
    Maven home: /usr/local/apache-maven-3.6.0
    Java version: 1.8.0_131, vendor: Oracle Corporation
    Java home: /usr/local/jdk1.8.0_131/jre
    Default locale: en_US, platform encoding: UTF-8
    OS name: "linux", version: "4.20.13-1.el7.elrepo.x86_64", arch: "amd64", family: "unix"

## 安装protobuf ##

# 安装依赖环境
    $ yum groupinstall Development tools -y
    
    # 下载
    $ https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
    
    # 解压缩
    $ tar -zxvf protobuf-2.5.0.tar.gz
    
    # 进入解压目录 配置安装路径（--prefix=/usr/local/protobuf-2.5.0）
    $ ./configure --prefix=/usr/local/protobuf-2.5.0
    
    # 编译
    $ make
    
    # 验证编译文件
    $ make check
    
    # 安装
    $ make install
    
    # 配置protobuf环境变量
    $ vim /etc/profile
    
    # 刷新配置文件 
    $ source /etc/profile
    
    export PROTOCBUF_HOME=/usr/local/protobuf-2.5.0
    export PATH=$PATH:$PROTOCBUF_HOME/bin
    
    # 验证是否安装成功
    $ protoc --version
    
    [root@localhost ~]# protoc --version
    libprotoc 2.5.0

## 安装findbugs ##

# 下载
    $ https://jaist.dl.sourceforge.net/project/findbugs/findbugs/1.3.9/findbugs-1.3.9.tar.gz
    
    # 解压缩
    $ tar -zxvf findbugs-1.3.9.tar.gz -C /usr/local/findbugs-1.3.9
    
    # 配置系统环境变量
    $ vim /etc/profile
    
    export FINDBUGS_HOME=/usr/local/findbugs-1.3.9
    export PATH=$PATH:$FINDBUGS_HOME/bin
    
    # 刷新配置文件 
    $ source /etc/profile
    
    # 验证是否安装成功
    $ findbugs -version
    
    [root@localhost ~]# findbugs -version
    1.3.9

## 编译hadoop ##

# 下载hadoop的源码
    $ wget https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.9.2/hadoop-2.9.2-src.tar.gz
    
    # 解压缩源码
    $ tar -zxvf hadoop-2.9.2-src.tar.gz
    
    # 进入hadoop的源码目录开始编译
    $ mvn package -Pdist,native -DskipTests -Dtar
    
    # 编译完成
    
    [INFO] Reactor Summary:
    [INFO] 
    [INFO] Apache Hadoop Main ................................. SUCCESS [ 36.760 s]
    [INFO] Apache Hadoop Build Tools .......................... SUCCESS [ 22.223 s]
    [INFO] Apache Hadoop Project POM .......................... SUCCESS [ 10.227 s]
    [INFO] Apache Hadoop Annotations .......................... SUCCESS [ 10.297 s]
    [INFO] Apache Hadoop Assemblies ........................... SUCCESS [  0.291 s]
    [INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [ 17.249 s]
    [INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [ 16.091 s]
    [INFO] Apache Hadoop MiniKDC .............................. SUCCESS [01:55 min]
    [INFO] Apache Hadoop Auth ................................. SUCCESS [01:23 min]
    [INFO] Apache Hadoop Auth Examples ........................ SUCCESS [  5.916 s]
    [INFO] Apache Hadoop Common ............................... SUCCESS [02:48 min]
    [INFO] Apache Hadoop NFS .................................. SUCCESS [  7.323 s]
    [INFO] Apache Hadoop KMS .................................. SUCCESS [ 21.010 s]
    [INFO] Apache Hadoop Common Project ....................... SUCCESS [  0.077 s]
    [INFO] Apache Hadoop HDFS Client .......................... SUCCESS [01:05 min]
    [INFO] Apache Hadoop HDFS ................................. SUCCESS [01:12 min]
    [INFO] Apache Hadoop HDFS Native Client ................... SUCCESS [  2.842 s]
    [INFO] Apache Hadoop HttpFS ............................... SUCCESS [ 24.130 s]
    [INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [ 16.353 s]
    [INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [  5.314 s]
    [INFO] Apache Hadoop HDFS-RBF ............................. SUCCESS [ 29.657 s]
    [INFO] Apache Hadoop HDFS Project ......................... SUCCESS [  0.063 s]
    [INFO] Apache Hadoop YARN ................................. SUCCESS [  0.075 s]
    [INFO] Apache Hadoop YARN API ............................. SUCCESS [ 14.514 s]
    [INFO] Apache Hadoop YARN Common .......................... SUCCESS [01:14 min]
    [INFO] Apache Hadoop YARN Registry ........................ SUCCESS [  6.258 s]
    [INFO] Apache Hadoop YARN Server .......................... SUCCESS [  0.072 s]
    [INFO] Apache Hadoop YARN Server Common ................... SUCCESS [ 36.000 s]
    [INFO] Apache Hadoop YARN NodeManager ..................... SUCCESS [ 14.787 s]
    [INFO] Apache Hadoop YARN Web Proxy ....................... SUCCESS [  4.074 s]
    [INFO] Apache Hadoop YARN ApplicationHistoryService ....... SUCCESS [ 19.797 s]
    [INFO] Apache Hadoop YARN Timeline Service ................ SUCCESS [ 10.855 s]
    [INFO] Apache Hadoop YARN ResourceManager ................. SUCCESS [ 22.955 s]
    [INFO] Apache Hadoop YARN Server Tests .................... SUCCESS [  1.288 s]
    [INFO] Apache Hadoop YARN Client .......................... SUCCESS [  6.973 s]
    [INFO] Apache Hadoop YARN SharedCacheManager .............. SUCCESS [  4.901 s]
    [INFO] Apache Hadoop YARN Timeline Plugin Storage ......... SUCCESS [  4.682 s]
    [INFO] Apache Hadoop YARN Router .......................... SUCCESS [  5.892 s]
    [INFO] Apache Hadoop YARN TimelineService HBase Backend ... SUCCESS [01:06 min]
    [INFO] Apache Hadoop YARN Timeline Service HBase tests .... SUCCESS [ 16.535 s]
    [INFO] Apache Hadoop YARN Applications .................... SUCCESS [  0.052 s]
    [INFO] Apache Hadoop YARN DistributedShell ................ SUCCESS [  3.429 s]
    [INFO] Apache Hadoop YARN Unmanaged Am Launcher ........... SUCCESS [  2.224 s]
    [INFO] Apache Hadoop YARN Site ............................ SUCCESS [  0.042 s]
    [INFO] Apache Hadoop YARN UI .............................. SUCCESS [  0.052 s]
    [INFO] Apache Hadoop YARN Project ......................... SUCCESS [  7.005 s]
    [INFO] Apache Hadoop MapReduce Client ..................... SUCCESS [  0.213 s]
    [INFO] Apache Hadoop MapReduce Core ....................... SUCCESS [ 20.400 s]
    [INFO] Apache Hadoop MapReduce Common ..................... SUCCESS [ 18.649 s]
    [INFO] Apache Hadoop MapReduce Shuffle .................... SUCCESS [  4.872 s]
    [INFO] Apache Hadoop MapReduce App ........................ SUCCESS [  9.835 s]
    [INFO] Apache Hadoop MapReduce HistoryServer .............. SUCCESS [  6.539 s]
    [INFO] Apache Hadoop MapReduce JobClient .................. SUCCESS [  5.527 s]
    [INFO] Apache Hadoop MapReduce HistoryServer Plugins ...... SUCCESS [  2.240 s]
    [INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [  6.176 s]
    [INFO] Apache Hadoop MapReduce ............................ SUCCESS [  2.566 s]
    [INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [  8.532 s]
    [INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [  8.918 s]
    [INFO] Apache Hadoop Archives ............................. SUCCESS [  2.482 s]
    [INFO] Apache Hadoop Archive Logs ......................... SUCCESS [  2.645 s]
    [INFO] Apache Hadoop Rumen ................................ SUCCESS [  6.271 s]
    [INFO] Apache Hadoop Gridmix .............................. SUCCESS [  4.607 s]
    [INFO] Apache Hadoop Data Join ............................ SUCCESS [  3.110 s]
    [INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [  2.631 s]
    [INFO] Apache Hadoop Extras ............................... SUCCESS [  3.344 s]
    [INFO] Apache Hadoop Pipes ................................ SUCCESS [  0.479 s]
    [INFO] Apache Hadoop OpenStack support .................... SUCCESS [  5.677 s]
    [INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [01:57 min]
    [INFO] Apache Hadoop Azure support ........................ SUCCESS [ 25.907 s]
    [INFO] Apache Hadoop Aliyun OSS support ................... SUCCESS [ 30.414 s]
    [INFO] Apache Hadoop Client ............................... SUCCESS [  7.438 s]
    [INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [  1.394 s]
    [INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [  8.115 s]
    [INFO] Apache Hadoop Resource Estimator Service ........... SUCCESS [ 10.685 s]
    [INFO] Apache Hadoop Azure Data Lake support .............. SUCCESS [ 18.561 s]
    [INFO] Apache Hadoop Tools Dist ........................... SUCCESS [ 18.124 s]
    [INFO] Apache Hadoop Tools ................................ SUCCESS [  0.245 s]
    [INFO] Apache Hadoop Distribution ......................... SUCCESS [01:01 min]
    [INFO] Apache Hadoop Cloud Storage ........................ SUCCESS [  1.359 s]
    [INFO] Apache Hadoop Cloud Storage Project ................ SUCCESS [  0.028 s]
    [INFO] ------------------------------------------------------------------------
    [INFO] BUILD SUCCESS
    [INFO] ------------------------------------------------------------------------
    [INFO] Total time: 25:06 min
    [INFO] Finished at: 2019-03-21T11:54:52+08:00
    [INFO] Final Memory: 135M/384M
    [INFO] ------------------------------------------------------------------------
    [root@localhost hadoop-2.9.2-src]# 
    
    # 编译完成的文件在hadoop-dist/target/目录下
    $ ll hadoop-dist/target/
    
    [root@localhost hadoop-2.9.2-src]# ll hadoop-dist/target/
    total 949892
    drwxr-xr-x 2 root root      4096 Mar 21 11:53 antrun
    drwxr-xr-x 3 root root      4096 Mar 21 11:53 classes
    -rw-r--r-- 1 root root      2209 Mar 21 11:53 dist-layout-stitching.sh
    -rw-r--r-- 1 root root       635 Mar 21 11:53 dist-tar-stitching.sh
    drwxr-xr-x 9 root root      4096 Mar 21 11:53 hadoop-2.9.2
    -rw-r--r-- 1 root root 323710978 Mar 21 11:54 hadoop-2.9.2.tar.gz
    -rw-r--r-- 1 root root     31694 Mar 21 11:53 hadoop-dist-2.9.2.jar
    -rw-r--r-- 1 root root 648827621 Mar 21 11:54 hadoop-dist-2.9.2-javadoc.jar
    -rw-r--r-- 1 root root     29003 Mar 21 11:54 hadoop-dist-2.9.2-sources.jar
    -rw-r--r-- 1 root root     29003 Mar 21 11:54 hadoop-dist-2.9.2-test-sources.jar
    drwxr-xr-x 2 root root      4096 Mar 21 11:54 javadoc-bundle-options
    drwxr-xr-x 2 root root      4096 Mar 21 11:53 maven-archiver
    drwxr-xr-x 3 root root      4096 Mar 21 11:53 maven-shared-archive-resources
    drwxr-xr-x 3 root root      4096 Mar 21 11:53 test-classes
    drwxr-xr-x 2 root root      4096 Mar 21 11:53 test-dir

## 总结 ##

到此hadoop的编译已经完成 过程中遇到了很多各种各样的问题上述步骤都是经过验证的如在安装过程中遇到问题可以留言，谢谢！