Debian下Hadoop集群安装
依赖安装
jdk 8
sudo apt-get update && sudo apt-get install -y wget apt-transport-https
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | sudo tee /etc/apt/keyrings/adoptium.asc
echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://mirrors.tuna.tsinghua.edu.cn/Adoptium/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | sudo tee /etc/apt/sources.list.d/adoptium.list
sudo apt-get update
sudo apt-get install -y temurin-8-jdk
hadoop
mkdir -p /root/packages
wget -P /root/packages https://dlcdn.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6-aarch64.tar.gz
tar -zxvf /root/packages/hadoop-3.3.6-aarch64.tar.gz -C /usr/local
配置环境变量
export JAVA_HOME=/usr/lib/jvm/temurin-8-jdk-amd64
export HADOOP_HOME="/usr/local/hadoop-3.3.6"
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
环境配置
设置hosts
127.0.0.1 localhost
192.168.50.201 node1.node1.com node1
192.168.50.202 node2.node2.com node2
192.168.50.203 node3.node3.com node3
# The following lines are desirable for IPv6 capable hosts
::1 localhost ip6-localhost ip6-loopback
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
生成SSH rsa
ssh-keygen -t rsa -C "node1@example.com"
允许root登录
vim /etc/ssh/sshd_config
修改PermitRootLogin yes
复制ssh到其他主机
ssh-copy-id node1
ssh-copy-id node2
ssh-copy-id node3
至少需要完成从node1 -> node1,node2,node3的免密登录
Hadoop配置
编辑Hadoop配置文件
hadoop-env.sh
vim $HADOOP_HOME/etc/hadoop/hadoop-env.sh
在文件末尾加上以下内容
# 配置JAVA_HOME
export JAVA_HOME=$JAVA_HOME
# 设置用户以执行对应角色shell命令
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
core-site.xml
vim $HADOOP_HOME/etc/hadoop/core-site.xml
在configuration标签中添加以下内容
<!-- 默认文件系统的名称。通过URI中schema区分不同文件系统 -->
<!-- file://本地文件系统 hdfs://hadoop分布式文件系统 -->
<!-- gfs://google文件系统 -->
<!-- hdfs文件系统访问地址:http://node1:8020 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://node1:8020</value>
</property>
<!-- 设置Hadoop本地保存数据路径 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/export/data/hadoop</value>
</property>
<!-- 设置HDFS web UI用户身份 -->
<property>
<name>hadoop.http.staticuser.user</name>
<value>root</value>
</property>
mapred-site.xml
vim $HADOOP_HOME/etc/hadoop/mapred-site.xml
在configuration标签中添加以下内容
<!-- 设置MR程序默认运行模式: yarn集群模式 local本地模式 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<!-- MR程序历史服务地址 -->
<property>
<name>mapreduce.jobhistory.address</name>
<value>node1:10020</value>
</property>
<!-- MR程序历史服务器web端地址 -->
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>node1:19888</value>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
</property>
yarn-site.xml
vim $HADOOP_HOME/etc/hadoop/yarn-site.xml
在configuration标签中添加以下内容
<!-- 设置YARN集群主角色运行机器位置 -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>node1</value>
</property>
<!-- ModeManager上运行的附属服务,需配置成mapreduce_shuffle才可运行程序。 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- 每个容器请求的最小内存资源(MB为单位) -->
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>256</value>
</property>
<!-- 每个容器请求的最大内存资源(MB为单位) -->
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>512</value>
</property>
workers
vim $HADOOP_HOME/etc/hadoop/workers
在workers
文件中添加主机名称或IP
node1
node2
node3
Hadoop启动
NameNode format(格式化操作)
首次启动HDFS时,必须对其进行格式化操作
format本质上是初始化工作,进行HDFS清理和准备工作
hdfs namenode -format
当格式化日志中出现以下内容说明格式化成功
STARTUP_MSG: build = https://github.com/apache/hadoop.git -r 1be78238728da9266a4f88195058f08fd012bf9c; compiled by 'ubuntu' on 2023-06-18T23:15Z
STARTUP_MSG: java = 1.8.0_382
************************************************************/
2023-09-12 21:18:41,575 INFO namenode.NameNode: registered UNIX signal handlers for [TERM, HUP, INT]
2023-09-12 21:18:41,758 INFO namenode.NameNode: createNameNode [-format]
2023-09-12 21:18:42,039 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2023-09-12 21:18:43,011 INFO namenode.NameNode: Formatting using clusterid: CID-fcf657e4-d0df-4a9a-8f7d-a2a8f0a910df
2023-09-12 21:18:43,082 INFO namenode.FSEditLog: Edit logging is async:true
2023-09-12 21:18:43,167 INFO namenode.FSNamesystem: KeyProvider: null
2023-09-12 21:18:43,173 INFO namenode.FSNamesystem: fsLock is fair: true
2023-09-12 21:18:43,179 INFO namenode.FSNamesystem: Detailed lock hold time metrics enabled: false
2023-09-12 21:18:43,229 INFO namenode.FSNamesystem: fsOwner = root (auth:SIMPLE)
2023-09-12 21:18:43,233 INFO namenode.FSNamesystem: supergroup = supergroup
2023-09-12 21:18:43,237 INFO namenode.FSNamesystem: isPermissionEnabled = true
2023-09-12 21:18:43,238 INFO namenode.FSNamesystem: isStoragePolicyEnabled = true
2023-09-12 21:18:43,240 INFO namenode.FSNamesystem: HA Enabled: false
2023-09-12 21:18:43,321 INFO common.Util: dfs.datanode.fileio.profiling.sampling.percentage set to 0. Disabling file IO profiling
2023-09-12 21:18:43,593 INFO blockmanagement.DatanodeManager: dfs.block.invalidate.limit : configured=1000, counted=60, effected=1000
2023-09-12 21:18:43,598 INFO blockmanagement.DatanodeManager: dfs.namenode.datanode.registration.ip-hostname-check=true
2023-09-12 21:18:43,613 INFO blockmanagement.BlockManager: dfs.namenode.startup.delay.block.deletion.sec is set to 000:00:00:00.000
2023-09-12 21:18:43,618 INFO blockmanagement.BlockManager: The block deletion will start around 2023 Sep 12 21:18:43
2023-09-12 21:18:43,625 INFO util.GSet: Computing capacity for map BlocksMap
2023-09-12 21:18:43,628 INFO util.GSet: VM type = 64-bit
2023-09-12 21:18:43,637 INFO util.GSet: 2.0% max memory 475.6 MB = 9.5 MB
2023-09-12 21:18:43,638 INFO util.GSet: capacity = 2^20 = 1048576 entries
2023-09-12 21:18:43,665 INFO blockmanagement.BlockManager: Storage policy satisfier is disabled
2023-09-12 21:18:43,668 INFO blockmanagement.BlockManager: dfs.block.access.token.enable = false
2023-09-12 21:18:43,702 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.threshold-pct = 0.999
2023-09-12 21:18:43,709 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.min.datanodes = 0
2023-09-12 21:18:43,711 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.extension = 30000
2023-09-12 21:18:43,717 INFO blockmanagement.BlockManager: defaultReplication = 3
2023-09-12 21:18:43,722 INFO blockmanagement.BlockManager: maxReplication = 512
2023-09-12 21:18:43,723 INFO blockmanagement.BlockManager: minReplication = 1
2023-09-12 21:18:43,724 INFO blockmanagement.BlockManager: maxReplicationStreams = 2
2023-09-12 21:18:43,726 INFO blockmanagement.BlockManager: redundancyRecheckInterval = 3000ms
2023-09-12 21:18:43,727 INFO blockmanagement.BlockManager: encryptDataTransfer = false
2023-09-12 21:18:43,730 INFO blockmanagement.BlockManager: maxNumBlocksToLog = 1000
2023-09-12 21:18:43,923 INFO namenode.FSDirectory: GLOBAL serial map: bits=29 maxEntries=536870911
2023-09-12 21:18:43,927 INFO namenode.FSDirectory: USER serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,928 INFO namenode.FSDirectory: GROUP serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,931 INFO namenode.FSDirectory: XATTR serial map: bits=24 maxEntries=16777215
2023-09-12 21:18:43,964 INFO util.GSet: Computing capacity for map INodeMap
2023-09-12 21:18:43,967 INFO util.GSet: VM type = 64-bit
2023-09-12 21:18:43,973 INFO util.GSet: 1.0% max memory 475.6 MB = 4.8 MB
2023-09-12 21:18:43,974 INFO util.GSet: capacity = 2^19 = 524288 entries
2023-09-12 21:18:43,979 INFO namenode.FSDirectory: ACLs enabled? true
2023-09-12 21:18:43,980 INFO namenode.FSDirectory: POSIX ACL inheritance enabled? true
2023-09-12 21:18:43,980 INFO namenode.FSDirectory: XAttrs enabled? true
2023-09-12 21:18:43,987 INFO namenode.NameNode: Caching file names occurring more than 10 times
2023-09-12 21:18:43,996 INFO snapshot.SnapshotManager: Loaded config captureOpenFiles: false, skipCaptureAccessTimeOnlyChange: false, snapshotDiffAllowSnapRootDescendant: true, maxSnapshotLimit: 65536
2023-09-12 21:18:44,006 INFO snapshot.SnapshotManager: SkipList is disabled
2023-09-12 21:18:44,016 INFO util.GSet: Computing capacity for map cachedBlocks
2023-09-12 21:18:44,018 INFO util.GSet: VM type = 64-bit
2023-09-12 21:18:44,022 INFO util.GSet: 0.25% max memory 475.6 MB = 1.2 MB
2023-09-12 21:18:44,034 INFO util.GSet: capacity = 2^17 = 131072 entries
2023-09-12 21:18:44,070 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.window.num.buckets = 10
2023-09-12 21:18:44,076 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.num.users = 10
2023-09-12 21:18:44,082 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.windows.minutes = 1,5,25
2023-09-12 21:18:44,141 INFO namenode.FSNamesystem: Retry cache on namenode is enabled
2023-09-12 21:18:44,143 INFO namenode.FSNamesystem: Retry cache will use 0.03 of total heap and retry cache entry expiry time is 600000 millis
2023-09-12 21:18:44,156 INFO util.GSet: Computing capacity for map NameNodeRetryCache
2023-09-12 21:18:44,157 INFO util.GSet: VM type = 64-bit
2023-09-12 21:18:44,159 INFO util.GSet: 0.029999999329447746% max memory 475.6 MB = 146.1 KB
2023-09-12 21:18:44,159 INFO util.GSet: capacity = 2^14 = 16384 entries
2023-09-12 21:18:44,234 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1199298961-192.168.50.201-1694524724216
2023-09-12 21:18:44,315 INFO common.Storage: Storage directory /export/data/hadoop/dfs/name has been successfully formatted.
2023-09-12 21:18:44,450 INFO namenode.FSImageFormatProtobuf: Saving image file /export/data/hadoop/dfs/name/current/fsimage.ckpt_0000000000000000000 using no compression
2023-09-12 21:18:44,588 INFO namenode.FSImageFormatProtobuf: Image file /export/data/hadoop/dfs/name/current/fsimage.ckpt_0000000000000000000 of size 396 bytes saved in 0 seconds .
2023-09-12 21:18:44,615 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
2023-09-12 21:18:44,656 INFO namenode.FSNamesystem: Stopping services started for active state
2023-09-12 21:18:44,660 INFO namenode.FSNamesystem: Stopping services started for standby state
2023-09-12 21:18:44,672 INFO namenode.FSImage: FSImageSaver clean checkpoint: txid=0 when meet shutdown.
2023-09-12 21:18:44,677 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at node1.node1.com/192.168.50.201
************************************************************/
Hadoop集群启动关闭-手动逐个进程启停
每台机器上每次手动启动关闭一个角色进程
- HDFS集群
hdfs --daemon start namenode|datanode|secondarynamenode
hdfs --daemon stop namenode|datanode|secondarynamenode
- YARN集群
yarn --daemon start resourcemanager|nodemanager
yarn --daemon stop resourcemanager|nodemanager
在node1中启动
hdfs --daemon start namenode
hdfs --daemon start datanode
yarn --daemon start resourcemanager
yarn --daemon start nodemanager
jps
在node2中启动
hdfs --daemon start datanode
hdfs --daemon start secondarynamenode
yarn --daemon start nodemanager
jps
在node3中启动
hdfs --daemon start datanode
yarn --daemon start nodemanager
jps
Hadoop集群启动关闭-shell脚本一键启停
在node1上,使用自带的shell脚本一键启动
前提:配置好机器之间的SSH免密登录和workers文件。
- HDFS集群
start-dfs.sh
stop-dfs.sh
- YARN集群
start-yarn.sh
stop-yarn.sh
- Hadoop集群
start-all.sh
stop-all.sh
Hadoop Web UI页面-HDFS集群
地址:http://node1:9870文章来源:https://www.toymoban.com/news/detail-708491.html
Hadoop Web UI页面-YARN集群
地址:http://node1:8088文章来源地址https://www.toymoban.com/news/detail-708491.html
到了这里,关于Debian下Hadoop集群安装的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!