Hadoop 2.6.4 分布式搭建

悠悠 2022-06-03 07:54 339阅读 0赞

多集群安装(3个节点)  
节点详情：

vm01---192.168.110.101(master)
    vm02---192.168.110.102(SecondNamenode)
    vm03---192.168.110.103
    在slave里配置SecondNamenode

master:
    192.168.110.101
    slaves:
    vm02:192.168.110.102
    vm03:192.168.110.103

core-site.xml

<configuration>
            <property>
                    <name>fs.defaultFS</name>
                    <value>hdfs://192.168.110.101:9000</value>
            </property>
            <property>
                    <name>hadoop.tmp.dir</name>
                    <value>/opt/hadoop-2.6.4/tmp</value>
            </property>
            <property>
                    <name>fs.checkpoint.period</name>
                    <value>3600</value>
            </property>
            <property>
                    <name>fs.checkpoint.size</name>
                    <value>67108864</value>
            </property>
    </configuration>

hdfs-site.xml

<configuration>
            <property>
                    <name>dfs.replication</name>
                    <value>3</value>
            </property>
            <property>
                    <name>dfs.http.address</name>
                    <value>vm01:50070</value>
            </property>
            <property>
                    <name>dfs.namenode.secondary.http-address</name>
                    <value>vm02:50090</value>
            </property>
            <property>
                    <name>dfs.namenode.name.dir</name>
                    <value>/opt/hadoop-2.6.4/name</value>
            </property>
            <property>
                    <name>dfs.datanode.data.dir</name>
                    <value>/opt/hadoop-2.6.4/data</value>
            </property>
    <property>
                    <name>dfs.permissions</name>
                    <value>false</value>
            </property>
    </configuration>

yarn-site.xml

<configuration>
    
    
            <property>
                    <name>yarn.nodemanager.aux-services</name>
                    <value>mapreduce_shuffle</value>
            </property>
            <property>
                    <name>yarn.resourcemanager.hostname</name>
                    <value>vm01</value>
            </property>
            
            <property>
                    <name>yarn.log-aggregation-enable</name>
                    <value>true</value>
            </property>
            
            <property>
                    <name>yarn.log-aggregation.retain-seconds</name>
                    <value>106800</value>
            </property>
            <property>
                    <name>yarn.resourcemanager.address</name>
                    <value>vm01:8032</value>
             </property>
            <property>
                    <name>yarn.resourcemanager.scheduler.address</name>
                    <value>vm01:8030</value>
            </property>
            <property>
                    <name>yarn.resourcemanager.resource-tracker.address</name>
                    <value>vm01:8031</value>
            </property>
            
            <property>
                    <name>yarn.nodemanager.vmem-check-enabled</name>
                    <value>false</value>
                    <description>Whether virtual memory limits will be enforced for containers</description>
            </property>
            
            <property>
                    <name>yarn.nodemanager.vmem-pmem-ratio</name>
                    <value>4</value>
                    <description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
            </property>
    
    </configuration>

mapred-site.xml

<configuration>
        <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
        </property>
        
         <property>
            <name>mapred.tasktracker.map.tasks.maximum</name>
            <value>1</value>
        </property>
         <property>
            <name>mapred.tasktracker.reduce.tasks.maximum</name>
            <value>1</value>
         </property>
    
    </configuration>

遇到一个严重的问题，`虚拟内存溢出`，一开始以为是开的虚拟机内存不足，后来加了内存后，还是存在这个问题  
![这里写图片描述][70]  
后来在mapred-site.xml注释掉与JVM运行内存空间的相关的配置就可以了，具体原理有待研究。

[70]: /images/20220603/a0a9cd225dd84a7cb1ba3a6d3f474180.png