hadoop文件配置

伪分布式配置：　

core-site.xml

<configuration>

        <property>

                <name>fs.defaultFS</name>

                <value>hdfs://localhost:9000</value>

        </property>

</configuration>

hdfs-site.xml

<configuration>

        <property>

                <name>dfs.replication</name>

                <value>3</value>

        </property>

        <property>

                <name>dfs.namenode.name.dir</name>

                <value>file:/usr/local/hadoop/hadoop_data/hdfs/namenode</value>

        </property>

        <property>

                <name>dfs.datanode.data.dir</name>

                <value>file:/usr/local/hadoop/hadoop_data/hdfs/datanode</value>

        </property>

</configuration>

mapred-site.xml (可能需要重命名mapred-site.xml.template)

cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml

<configuration>

        <property>

                <name>mapreduce.framework.name</name>

                <value>yarn</value>

        </property>

</configuration>

yarn-site.xml

<configuration>

        <property>

                <name>yarn.nodemanager.aux-services</name>

                <value>mapreduce_shuffle</value>

        </property>

        <property>

                <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

                <value>org.apache.hadoop.mapred.ShuffleHandler</value>

        </property>

</configuration>

初始化

mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode

mkdir -p /usr/local/hadoop/hadoop_data/hdfs/datanode

chown root:root -R /usr/local/hadoop # 更改目录所有者

hadoop name node -format # 格式化

启动

start-all.sh # 同时启动HDFS，yarn

stop-all.sh # 关闭

报错The authenticity of host '0.0.0.0 (0.0.0.0)' can't be established，

关闭防火墙

systemctl stop firewalld

systemctl disable firewalld

查看

[root@localhost ~]# jps

2458 NodeManager

1524 DataNode

1445 NameNode

2746 Jps

2358 ResourceManager

1890 SecondaryNameNode

启动成功。

HDFS： NameNode,SecondaryNameNode,DataNode

MapReduce2(Yarn): ResourceManager,NodeManager

打开Hadoop Resource-Manager Web界面

　　http://localhost:8088/

分布式集群配置：

master:

$ vim /etc/hostname

master

$ mkdir -p /usr/local/hadoop/hadoop_data/hdfs/namenode

$ vim /etc/hosts

192.168.31.110 master

192.168.31.111 data1

192.168.31.112 data2

192.168.31.113 data3

$ vim /usr/local/hadoop//etc/hadoop/slaves

data1

data2

data3

$ vim /usr/local/hadoop/etc/hadoop/core-site.xml

<configuration>

        <property>

                <name>fs.defaultFS</name>

                <value>hdfs://master:9000</value>

        </property>

</configuration>

$ vim /usr/local/hadoop/etc/hadoop/yarn-site.xml

# ResourceManager主机与NodeManager的连接地址为8025

# ResourceManager 与 ApplicationMaster的连接地址为8030

# ResourceManager 与客户端的连接地址为8050

<configuration>

        <property>

                <name>yarn.nodemanager.aux-services</name>

                <value>mapreduce_shuffle</value>

        </property>

        <property>

                <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

                <value>org.apache.hadoop.mapred.ShuffleHandler</value>

        </property>

        <property>

                <name>yarn.resourcemanager.resource-tracker.address</name>

                <value>master:8025</value>

        </property>

        <property>

                <name>yarn.resourcemanager.scheduler.address</name>

                <value>master:8030</value>

        </property>

        <property>

                <name>yarn.resourcemanager.address</name>

                <value>master:8050</value>

        </property>

</configuration>

$ vim /usr/local/hadoop/etc/hadoop/mapred-site.xml

<configuration>

        <property>

                <name>mapred.job.tracker</name>

                <value>master:54311</value>

        </property>

</configuration>

$ vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml

<configuration>

        <property>

                <name>dfs.replication</name>

                <value>3</value>

        </property>

        <property>

                <name>dfs.namenode.name.dir</name>

                <value>file:/usr/local/hadoop/hadoop_data/hdfs/namenode</value>

        </property>

</configuration>

data1 ~ data3: