Hadoop集群环境搭建

1.

 

2.集群搭建

(1) hadoop-2.2.0/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/usr/java/jdk1.7.0_79

 

(2) hadoop-2.2.0/etc/hadoop/core-site.xml

<configuration>

      <!-- 指定hdfs的nameservice为ns1 -->

        <property>

             <name>fs.defaultFS</name>

             <value>hdfs://ns1</value>

       </property>

      <!-- 指定hadoop临时目录 -->

        <property>

            <name>hadoop.tmp.dir</name>

            <value>/usr/hc/hadoop-2.2.0/tmp</value>

        </property>

      <!-- 指定zookeeper地址,注意多个之间用英文逗号分隔 -->

        <property>

            <name>ha.zookeeper.quorum</name>

            <value>nameNode:2181,dataNode01:2181,dataNode02:2181</value>

        </property>

</configuration>

 (3) hadoop-2.2.0/etc/hadoop/hdfs-site.xml

<configuration>

      <!-- 指定hdfs的nameservice为ns1，需要和core-site.xml里面保持一致 -->

        <property>

            <name>dfs.nameservcies</name>

            <value>ns1</value>

        </property>

        <!-- ns1下面的NameNode，是nn1 -->

        <property>

            <name>dfs.ha.namenodes.ns1</name>

            <value>nn1</value>

        </property>

        <!-- ns1下面的NameNode，是nn1 -->

        <property>

            <name>dfs.ha.namenodes.ns1</name>

            <value>nn1</value>

        </property>

        <!-- nn1的RPC通信地址 -->

        <property>

            <name>dfs.namenode.rpc-address.ns1.nn1</name>

            <value>nameNode:9000</value>

        </property>

        <!-- nn1的http通信地址 -->

        <property>

            <name>dfs.namenode.http-address.ns1.nn1</name>

            <value>nameNode:50070</value>

        </property>

        <!-- 指定NameNode的元数据在JournalNade上的存放 -->

        <property>

            <name>dfs.namenode.shared.edits.dir</name>

            <value>qjournal://nameNode:</value>

        </property>

        <!-- 指定JournalNode在本地磁盘存放数据 -->

        <property>

            <name>dfs.namenode.shared.edits.dir</name>

            <value>/usr/hc/hadoop-2.2.0/jour</value>

        </property>

        <!-- 开启NameNode失败自动切换 -->

        <property>

            <name>dfs.ha.automatic-failover.enabled</name>

            <value>true</value>

        </property>

        <!-- 配置失败自动切换实现方式 -->

        <property>

            <name>dfs.client.failover.proxy.provider.ns1</name>

            <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

        </property>

        <!-- 配置隔离机制方法，多个机制用换行分隔 -->

        <property>

            <name>dfs.ha.fencing.methods</name>

            <value>

                  sshfence

                  shell(/bin/true)

            </value>

        </property>

        <!-- 使用sshfence隔离机制时需要ssh免登陆 -->

        <property>

            <name>dfs.ha.fencing.ssh.private-key-files</name>

            <value>/root/.ssh/id_rsa</value>

        </property>

        <!-- 配置sshfence隔离机制超时时间 -->

        <property>

            <name>dfs.ha.fencing.ssh.connect-timeout</name>

            <value>3000</value>

        </property>

</configuration>

(4) hadoop-2.2.0/etc/hadoop/mapred-site.xml.template 重命名为mapred-site.xml

<configuration>

<!-- 指定mr框架为yarn方式 -->

        <property>

             <name>mapreduce.framework.name</name>

             <value>yarn</value>

        </property>

</configuration>

(5) hadoop-2.2.0/etc/hadoop/yarn-site.xml

<configuration>

     <!-- 指定resourcemaneger地址-->

    <property>

        <name>yarn.resourcemaneger.hostname</name>

        <value>nameNode</value>

   </property>

    <!-- 指定nodemanager启动时加载server -->

    <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

   </property>

</configuration>

(6) hadoop-2.2.0/etc/hadoop/slaves

dataNode01

dataNode02

六个配置文件修改完了，请严格按照顺序，首先启动zookeeper

(7) 在hadoop-2.2.0/sbin下面有hadoop-daemon.sh和hadoop-daemons.sh分别启动单个进程和多个进程。

启动journalnode

./hadoop-daemon.sh start journalnode

(8)格式化hdfs

hdfs namenode -fromat

(9)格式化ZK

hdfs zkfs -formatZK

 

(10)分别启动hdfs和yarn再进行测试