hadoop yarn HA集群搭建

时间：2023-03-09 16:52:36
hadoop yarn HA集群搭建

可先完成hadoop namenode HA的搭建：http://www.cnblogs.com/kisf/p/7458519.html

搭建yarnde HA只需要在namenode HA配置基础上修改yarn-site.xml。

1. 修改yarn-site.xml  (yarn另一台机器上一定要将yarn.resourcemanager.ha.id改成rm1)

<configuration>

    <!-- Site specific YARN configuration properties -->

    <!--rm失联后重新链接的时间-->

    <property>

        <name>yarn.resourcemanager.connect.retry-interval.ms</name>

        <value>2000</value>

    </property>

    <!--开启resourcemanagerHA,默认为false-->

    <property>

        <name>yarn.resourcemanager.ha.enabled</name>

        <value>true</value>

    </property>

    <!--配置resourcemanager-->

    <property>

        <name>yarn.resourcemanager.ha.rm-ids</name>

        <value>rm1,rm2</value>

    </property>

    <property>

        <name>ha.zookeeper.quorum</name>

        <value>zk1:2181,zk2:2181,zk3:2181</value>

    </property>

    <!--开启故障自动切换-->

    <property>

        <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>

        <value>true</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm1</name>

        <value>master1</value>

    </property>

    <property>

        <name>yarn.resourcemanager.hostname.rm2</name>

        <value>master2</value>

    </property>

    <!--

     在master1上配置rm1,在master2上配置rm2,

     注意：一般都喜欢把配置好的文件远程复制到其它机器上，但这个在YARN的另一个机器上一定要修改

     -->

    <property>

        <name>yarn.resourcemanager.ha.id</name>

        <value>rm1</value>

        <description>If we want to launch more than one RM in single node,we need this configuration</description>

    </property>

    <!--开启自动恢复功能-->

    <property>

        <name>yarn.resourcemanager.recovery.enabled</name>

        <value>true</value>

    </property>

    <!--配置与zookeeper的连接地址-->

    <property>

        <name>yarn.resourcemanager.zk-state-store.address</name>

        <value>zk1:2181,zk2:2181,zk3:2181</value>

    </property>

    <property>

        <name>yarn.resourcemanager.store.class</name>

        <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>

    </property>

    <property>

        <name>yarn.resourcemanager.zk-address</name>

        <value>zk1:2181,zk2:2181,zk3:2181</value>

    </property>

    <property>

        <name>yarn.resourcemanager.cluster-id</name>

        <value>appcluster-yarn</value>

    </property>

    <!--schelduler失联等待连接时间-->

    <property>

        <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name>

        <value>5000</value>

    </property>

    <!--配置rm1-->

    <property>

        <name>yarn.resourcemanager.address.rm1</name>

        <value>master1:8032</value>

    </property>

    <property>

        <name>yarn.resourcemanager.scheduler.address.rm1</name>

        <value>master1:8030</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm1</name>

        <value>master1:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.resource-tracker.address.rm1</name>

        <value>master1:8031</value>

    </property>

    <property>

        <name>yarn.resourcemanager.admin.address.rm1</name>

        <value>master1:8033</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.admin.address.rm1</name>

        <value>master1:23142</value>

    </property>

    <!--配置rm2-->

    <property>

        <name>yarn.resourcemanager.address.rm2</name>

        <value>master2:8032</value>

    </property>

    <property>

        <name>yarn.resourcemanager.scheduler.address.rm2</name>

        <value>master2:8030</value>

    </property>

    <property>

        <name>yarn.resourcemanager.webapp.address.rm2</name>

        <value>master2:8088</value>

    </property>

    <property>

        <name>yarn.resourcemanager.resource-tracker.address.rm2</name>

        <value>master2:8031</value>

    </property>

    <property>

        <name>yarn.resourcemanager.admin.address.rm2</name>

        <value>master2:8033</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.admin.address.rm2</name>

        <value>master2:23142</value>

    </property>

    <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

    </property>

    <property>

        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

        <value>org.apache.hadoop.mapred.ShuffleHandler</value>

    </property>

    <property>

        <name>yarn.nodemanager.local-dirs</name>

        <value>/letv/soft/hadoop-2.7.3/yarn/local</value>

    </property>

    <property>

        <name>yarn.nodemanager.log-dirs</name>

        <value>/letv/soft/hadoop-2.7.3/yarn/log</value>

    </property>

    <property>

        <name>mapreduce.shuffle.port</name>

        <value>23080</value>

    </property>

    <!--故障处理类-->

    <property>

        <name>yarn.client.failover-proxy-provider</name>

        <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>

    </property>

    <property>

        <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>

        <value>/yarn-leader-election</value>

        <description>Optionalsetting.Thedefaultvalueis/yarn-leader-election</description>

    </property>

</configuration>

2. 启动

正常格式化，hdfs,yarn正常启动。

./sbin/start-dfs.sh 

./sbin/start-yarn.sh　　

rm2上yarn需要单独启动：

./sbin/yarn-daemon.sh start resourcemanager

over　　

　　

相关文章