Hadoop完全分布式搭建教程（完整版）

这篇具有很好参考价值的文章主要介绍了Hadoop完全分布式搭建教程（完整版）。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

分别创建三个节点 master slave1 slave2

在master节点下安装jdk

# 解压
[root@master /]# tar -zxvf /opt/software/jdk-8u212-linux-x64.tar.gz -C /opt/module/
# 修改安装包名为 java
[root@master /]# mv /opt/module/jdk1.8.0—212/  /opt/module/java

# 配置环境变量并使其生效
[root@master /]# vi /etc/profile
# java 
export JAVA_HOME=/opt/module/java
export PATH=$PATH:$JAVA_HOME/bin

# 验证是否成功
[root@master /]# source /etc/profile
[root@master /]# java -version
[root@master /]# javac

配置免密登录

# 配置映射环境
[root@master /]# vi /etc/host
# 添加三个节点的IP映射
192.168.100.4   master
192.168.100.2   slave1
192.168.100.3   slave2
# 创建密钥 (三个节点都要创建)
[root@master /]# ssh-keygen -t rsa -P ''
# 将公钥传输给master slave1 slave2
[root@master /]# ssh-copy-id root@master
[root@master /]# ssh-copy-id root@slave1
[root@master /]# ssh-copy-id root@slave2
# 将Java安装文件传送给slave1 slave2
[root@master /]# scp -r /opt/module/java/ root@slave1:/opt/module/
[root@master /]# scp -r /opt/module/java/ root@slave2:/opt/module/

Hadoop安装与配置

# 解压Hadoop到指定文件中
[root@master /]# tar -zxvf /opt/software/hadoop-3.1.3.tar.gz -C /opt/module/
# 修改Hadoop名称
[root@master /]# mv /opt/module/hadoop-3.1.3/ /opt/module/hadoop
# 配置环境变量
#hadoop
export HADOOP_HOME=/opt/module/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

切换到Hadoop的etc\hadoop目录下并进行相关配置

1.配置workers文件

 # 配置workers 文件
 [root@master /]# vi workers
 master
 slave1
 slave2

2.配置Hadoop-env.sh文件

# 配置Hadoop-env.sh文件
export JAVA_HOME=/opt/module/java
export HADOOP_PERFIX=/opt/module/hadoop
export HADOOP_OPTS="-Djava.library.path=$HADOOP_PERFIX/lib:$HADOOP_PERFIX/lib/native"
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

3. 配置core-site.xml 文件

<!-- 配置core-site.xml 文件 -->
[root@master hadoop]# vi core-site.xml 
<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/opt/module/hadoop/tmp</value>
        </property>
</configuration>

4.配置mapred-site.xml 文件

<!--配置mapred-site.xml 文件-->
<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>master:10020</value>
        </property>
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>master:19888</value>
        </property>
</configuration>

5. 配置hdfs-site.xml文件

<!--配置hdfs-site.xml文件-->
<configuration>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/opt/module/hadoop/dfs/name</value>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/opt/module/hadoop/dfs/data</value>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>3</value>
        </property>
</configuration>

6. 配置yarn-site.xml 文件

<!--配置yarn-site.xml 文件-->
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
     </property>
     <property>
         <name>yarn.resourcemanager.hostname</name>
         <value>master</value>
     </property>
</configuration>

# 创建
[root@master hadoop]# mkdir -p /opt/module/hadoop/dfs/name
[root@master hadoop]# mkdir -p /opt/module/hadoop/dfs/data
[root@master hadoop]# mkdir -p /opt/module/hadoop/tmp  
#分发
[root@master hadoop]# scp -r /opt/module/hadoop/ root@slave1:/opt/module/
[root@master hadoop]# scp -r /opt/module/hadoop/ root@slave2:/opt/module/
#同步环境
[root@master hadoop]# scp /etc/profile root@slave1:/etc
[root@master hadoop]# scp /etc/profile root@slave2:/etc
# 生效环境（三台都要）
[root@master hadoop]# source /etc/profile

Hadoop集群启动

# 初始化namenode
[root@master hadoop]# hdfs namenode -format 
# 启动hdfs
[root@master /]# start-dfs.sh
# 启动yarn
[root@master /]# start-yarn.sh

验证

# jps查看三个节点进程
# master
[root@master /]# jps
2833 Jps
1509 NameNode
1654 DataNode
2262 ResourceManager
1898 SecondaryNameNode
2429 NodeManager

# slave1
[root@slave1 /]# jps
642 NodeManager
787 Jps
511 DataNode

# slave2
[root@slave1 /]# jps
642 NodeManager
787 Jps
511 DataNode

#端口验证
HDFS ：9870

YARN ：8088

端口测试

master：9870查看

hadoop完全分布式搭建步骤,大数据集群搭建,hadoop,大数据,Powered by 金山文档

ii. master：8088查看

mapreduce测试

[root@master /]# vi a.txt
[root@master /]# hadoop fs -put ~/a.txt /input
# 进入到jar包测试文件目录下
[root@master /]# cd /opt/module/hadoop/share/hadoop/mapreduce/
# 测试mapreduce
[root@master mapreduce]# hadoop jar hadoop-mapreduce-examples-3.1.3.jar wordcount /input/a.txt /output
# 测试结果
    Map-Reduce Framework
        Map input records=3
        Map output records=6
        Map output bytes=59
        Map output materialized bytes=53
        Input split bytes=95
        Combine input records=6
        Combine output records=4
        Reduce input groups=4
        Reduce shuffle bytes=53
        Reduce input records=4
        Reduce output records=4
        Spilled Records=8
        Shuffled Maps =1
        Failed Shuffles=0
        Merged Map outputs=1
        GC time elapsed (ms)=172
        CPU time spent (ms)=1580
        Physical memory (bytes) snapshot=664891392
        Virtual memory (bytes) snapshot=5363101696
        Total committed heap usage (bytes)=745537536
        Peak Map Physical memory (bytes)=382193664
        Peak Map Virtual memory (bytes)=2678300672
        Peak Reduce Physical memory (bytes)=282697728
        Peak Reduce Virtual memory (bytes)=2684801024
    Shuffle Errors
        BAD_ID=0
        CONNECTION=0
        IO_ERROR=0
        WRONG_LENGTH=0
        WRONG_MAP=0
        WRONG_REDUCE=0
    File Input Format Counters 
        Bytes Read=35
    File Output Format Counters 
        Bytes Written=31

[root@master mapreduce]# hadoop fs -ls /output
Found 2 items
-rw-r--r--   3 root supergroup          0 2023-03-19 18:02 /output/_SUCCESS
-rw-r--r--   3 root supergroup         31 2023-03-19 18:02 /output/part-r-00000
[root@master mapreduce]# hadoop fs -cat /output/part-r-00000
2023-03-19 18:03:45,508 INFO sasl.SaslDataTransferClient: SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false
EOPE    1
HADOOP    1
HELLO    3
JAVA    1

注意：如果出现一下问题：

Last 4096 bytes of prelaunch.err :
Last 4096 bytes of stderr :
Error: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster
Please check whether your etc/hadoop/mapred-site.xml contains the below configuration:
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=${full path of your hadoop distribution directory}</value>
</property>

解决方法：

<!-- 输入以下命令回车-->
[root@master mapreduce]# hadoop classpath
//将生成的这堆代码添加至yarn-site.xml中
/opt/module/hadoop/etc/hadoop:/opt/module/hadoop/share/hadoop/common/lib/*:/opt/module/hadoop/share/hadoop/common/*:/opt/module/hadoop/share/hadoop/hdfs:/opt/module/hadoop/share/hadoop/hdfs/lib/*:/opt/module/hadoop/share/hadoop/hdfs/*:/opt/module/hadoop/share/hadoop/mapreduce/lib/*:/opt/module/hadoop/share/hadoop/mapreduce/*:/opt/module/hadoop/share/hadoop/yarn:/opt/module/hadoop/share/hadoop/yarn/lib/*:/opt/module/hadoop/share/hadoop/yarn/*

[root@master hadoop]# vi yarn-site.xml  //添加至最后
 <property>
          <name>yarn.application.classpath</name>
          <value>/opt/module/hadoop/etc/hadoop:/opt/module/hadoop/share/hadoop/common/lib/*:/opt/module/hadoop/share/hadoop/common/*:/opt/module/hadoop/share/hadoop/hdfs:/opt/module/hadoop/share/hadoop/hdfs/lib/*:/opt/module/hadoop/share/hadoop/hdfs/*:/opt/module/hadoop/share/hadoop/mapreduce/lib/*:/opt/module/hadoop/share/hadoop/mapreduce/*:/opt/module/hadoop/share/hadoop/yarn:/opt/module/hadoop/share/hadoop/yarn/lib/*:/opt/module/hadoop/share/hadoop/yarn/*
</value>
 </property>