1. 准备3台机器
| IP | 机器名 | root密码 | 用户名 | 密码 | 进程 |
|---|---|---|---|---|---|
| 192.168.20.44 | master | hd | hd | hd | namenode,ResourceManager，SecondaryNameNode |
| 192.168.20.45 | slave01 | hd | hd | hd | datanode，NodeManager |
| 192.168.20.46 | slave02 | hd | hd | hd | datanode，NodeManager |
2.用户权限
root 超级管理员（系统文件修改）
hd 普通角色用户 （在/home/hd/* 进行所有增删改查）
密码统一设置成：hd 或者 123456
3. 修改静态IP地址
3.1 第一台机器
设置网卡为静态的地址
切换root用户
[hd@bogon Desktop]$ su root
Password:
修改网卡
[root@bogon Desktop]# vim /etc/sysconfig/network-scripts/ifcfg-eth0 DEVICE="eth0"
BOOTPROTO="static"
ONBOOT="yes"
TYPE="Ethernet"
IPADDR=192.168.245.20
重启网卡
[root@bogon Desktop]# service network restart
查看IP信息
[root@bogon Desktop]# ifconfig
3.2第二台机器
切换root用户
[hd@bogon Desktop]$ su root
Password:
修改网卡
[root@bogon Desktop]# vim /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO="static"
ONBOOT="yes"
TYPE="Ethernet"
IPADDR=192.168.245.30
删除电脑网卡与系统绑定关系
[root@localhost Desktop]# rm -rf /etc/udev/rules.d/70-persistent-net.rules
重启系统
[root@localhost Desktop]# reboot
查看IP信息
[root@bogon Desktop]# ifconfig
3.3 第三台机器
切换root用户
[hd@bogon Desktop]$ su root
Password:
修改网卡
[root@bogon Desktop]# vim /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE="eth0"
BOOTPROTO="static"
ONBOOT="yes"
TYPE="Ethernet"
IPADDR=192.168.245.40
删除电脑网卡与系统绑定关系
[root@localhost Desktop]# rm -rf /etc/udev/rules.d/70-persistent-net.rules
重启系统
[root@localhost Desktop]# reboot
查看IP信息
[root@bogon Desktop]# ifconfig
4. 获取机器的IP地址
[root@localhost ~]# ifconfig
eno16777736: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.20.46 netmask 255.255.255.0 broadcast 192.168.20.46
#如果没有查看IP的命令就安装步骤
[hd@localhost root]$ su root
Password:
[root@localhost ~]# yum install -y net-tools
#也可查看ip addr 命令查看IP地址。
[hd@slave02 ~]$ ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:4f:71:df brd ff:ff:ff:ff:ff:ff
inet 192.168.20.46/24 brd 192.168.20.255 scope global noprefixroute dynamic ens160
valid_lft 32521sec preferred_lft 32521sec
inet6 fe80::f52e:651e:721b:a15a/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::5ebb:a198:b90f:fe68/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::c180:4b3e:a100:224c/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
5. Linux安装Java环境
Linux常用的命令
su 切换用户
pwd 显示当前目录
tar 解压，压宿
cd 进入目录
mv 移动文件，也可以改文件名
source /etc/profile 加载环境变量
vi 文本编辑器，相当于我们的记事本一样，但是使用的都是命令的操作。
5.1 把Linux自带Java环境删除
[hd@localhost ~]$ su root
Password:
[root@localhost hd]# yum remove -y java*
5.2 上传Java包
[root@localhost hd]# su hd
[hd@localhost ~]$ pwd
/home/hd
[hd@localhost ~]$ mkdir apps #上传到此目录
[hd@localhost ~]$ cd apps/
[hd@localhost apps]$
#上传过程
[hd@localhost apps]$ ll
total 178952
-rw-rw-r--. 1 hd hd 183246769 Apr 26 2018 jdk-8u121-linux-x64.tar.gz
5.3 解压java包
#解压
[hd@localhost apps]$ tar -zxvf jdk-8u121-linux-x64.tar.gz
[hd@localhost apps]$ ll
total 178956
drwxr-xr-x. 8 hd hd 4096 Dec 12 2016 jdk1.8.0_121
-rw-rw-r--. 1 hd hd 183246769 Apr 26 2018 jdk-8u121-linux-x64.tar.gz
[hd@localhost apps]$
#目录改名
[hd@localhost apps]$ mv jdk1.8.0_121/ java
[hd@localhost apps]$ ll
total 178956
drwxr-xr-x. 8 hd hd 4096 Dec 12 2016 java
-rw-rw-r--. 1 hd hd 183246769 Apr 26 2018 jdk-8u121-linux-x64.tar.gz
5.4 配置java环境
[hd@localhost apps]$ su root
Password:
[root@localhost apps]# cd java/
[root@localhost java]# pwd
/home/hd/apps/java
[root@localhost java]#
[root@localhost java]# vi /etc/profile
使用vi编辑器，在/etc/profile增加java环境变量
export JAVA_HOME=/home/hd/apps/java
export PATH=$PATH:$JAVA_HOME/bin
重加载一下系统环境
[root@localhost java]# source /etc/profile
[root@localhost java]# java -version
java version "1.8.0_121"
Java(TM) SE Runtime Environment (build 1.8.0_121-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.121-b13, mixed mode)
5.5 配置第二台，第三台机器的Java环境
使用scp远程拷贝命令
scp file2 [[user@]host2:]file2
1.把每一台机的java目录，拷贝到第二台机器
[root@localhost apps]# su hd
[hd@localhost apps]$
[hd@localhost apps]$ scp -r java hd@192.168.126.129:/home/hd/apps/
2.把每一台机的profile文件，拷贝到第二台机器
[hd@localhost apps]$ su root
Password:
[root@localhost apps]# scp /etc/profile root@192.168.126.129:/etc/
The authenticity of host '192.168.126.129 (192.168.126.129)' can't be established.
ECDSA key fingerprint is fb:0a:7a:9f:9a:bc:4f:ff:66:29:1d:1d:b9:a0:35:d1.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.126.129' (ECDSA) to the list of known hosts.
root@192.168.126.129's password:
profile 100% 1820 1.8KB/s 00:00
[root@localhost apps]#
3.第二台机器加载profile
[hd@localhost apps]$ source /etc/profile
[hd@localhost apps]$
[hd@localhost apps]$ java -version
java version "1.8.0_121"
Java(TM) SE Runtime Environment (build 1.8.0_121-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.121-b13, mixed mode)
[hd@localhost apps]$
第三台机，执行以上的步骤
6. 安装hadoop之前准备
6.1 修改主机名
- 第一台机器master
- 第二台机器slave01
- 第三台机器slave02
[hd@localhost ~]$ hostnamectl set-hostname master
==== AUTHENTICATING FOR org.freedesktop.hostname1.set-static-hostname ===
Authentication is required to set the statically configured local host name, as well as the pretty host name.
Authenticating as: root
Password:
==== AUTHENTICATION COMPLETE ===
[hd@localhost ~]$ hostnamectl set-hostname slave01
[hd@localhost ~]$ hostnamectl set-hostname slave02
6.2 修改/etc/hosts 文件
[hd@master ~]$ su root
Password:
[root@master hd]# vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.126.128 master
192.168.126.129 slave01
192.168.126.130 slave02
同步到第二，三台机器
#第二台机器
[root@master hd]# scp /etc/hosts root@slave01:/etc/
#第三台机器
[root@master hd]# scp /etc/hosts root@slave02:/etc/
6.3 关闭防火墙
启动：systemctl start firewalld.service
查看状态： systemctl status firewalld.service
停止： systemctl stop firewalld.service
禁用： systemctl disable firewalld.service
重启: systemctl restart firewalld.service
6.4 免密登录
需要做的免密的机器
| 机器---->机器（免密登录） |
|---|
| master ----> slave01 |
| master ----> slave02 |
| master ----> master |
6.4.1 生成密钥
[hd@master ~]$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/hd/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/hd/.ssh/id_rsa.
Your public key has been saved in /home/hd/.ssh/id_rsa.pub.
The key fingerprint is:
ef:ff:98:6c:a4:66:ca:66:a0:cd:a4:da:75:9c:c0:9f hd@slave02
The key's randomart image is:
+--[ RSA 2048]----+
| |
| |
| |
| . |
| o S |
| o+ + . |
| *..E .o |
| .o.ooo.+..o |
| ... oo+.o=.. |
+-----------------+
6.4.2 拷贝密钥到你需要免密登录的机器
[hd@master ~]$ ssh-copy-id slave02
The authenticity of host 'slave02 (192.168.126.130)' can't be established.
ECDSA key fingerprint is 09:57:a3:56:3b:5f:f0:01:55:0e:42:f3:4c:43:3d:d5.
Are you sure you want to continue connecting (yes/no)? yes
/usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed
/usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys
hd@slave02's password:
Number of key(s) added: 1
Now try logging into the machine, with: "ssh 'slave02'"
and check to make sure that only the key(s) you wanted were added.
6.4.3 测试免密登录
[hd@master ~]$ ssh slave01
Last login: Sun Jul 12 23:54:32 2020 from 192.168.22.130
7. Hadoop安装
7.1 上传hadoop安装包
[hd@master hadoop]$ rz
rz waiting to receive.
zmodem trl+C ȡ
7.2 解压安装包
[hd@master apps]$ su hd
Password:
[hd@master apps]$ pwd
/home/hd/apps
[hd@master apps]$ tar -zxvf hadoop-3.0.0.tar.gz
7.3 改目录名称
hd@master apps]$ mv hadoop-3.0.0 hadoop
[hd@master apps]$ ll
total 324644
drwxr-xr-x. 12 hd hd 192 Jul 11 00:09 hadoop
7.4 修改hadoop配置文件
7.4.1 修改hadoop-env.sh
[hd@master ~]$ cd /home/hd/apps/hadoop/etc/hadoop/
[hd@master hadoop]$ pwd
/home/hd/apps/hadoop/etc/hadoop
[hd@master hadoop]$ vi hadoop-env.sh
#在文件的尾部（按“G”可以跳到文档的尾部），增加
export JAVA_HOME=/home/hd/apps/java
7.4.2 修改core-site.xml
<configuration>
<!-- 指定HADOOP所使用的文件系统schema（URI），HDFS的老大（NameNode）的地址 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
<!-- 指定hadoop运行时产生文件的存储目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hd/apps/hadoop/tmpdata</value>
</property>
</configuration>
7.4.3 修改hdfs-site.xml
<configuration>
<!-- 指定HDFS副本的数量 -->
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<!-- 设置namenode的http通讯地址 -->
<property>
<name>dfs.namenode.http-address</name>
<value>master:50070</value>
</property>
<!-- 设置secondarynamenode的http通讯地址 -->
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>master:50090</value>
</property>
<!-- 设置namenode存放的路径 -->
<property>
<name>dfs.namenode.name.dir</name>
<value>/home/hd/apps/hadoop/namenode</value>
</property>
<!-- 设置datanode存放的路径 -->
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hd/apps/hadoop/datanode</value>
</property>
</configuration>
7.4.4 修改mapred-site.xml
<configuration>
<!-- 指定mr运行在yarn上 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=/home/hd/apps/hadoop</value>
</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=/home/hd/apps/hadoop</value>
</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=/home/hd/apps/hadoop</value>
</property>
</configuration>
7.4.5 修改yarn-site.xml
<configuration>
<!-- 指定YARN的老大（ResourceManager）的地址 -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>master</value>
</property>
<!-- reducer获取数据的方式 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
7.4.6 修改workers
[hd@master hadoop]$ vi workers
slave01
slave02
7.4.7 修改环境变量
[hd@master hadoop]$ su root
Password:
[root@master hadoop]# vi /etc/profile
#增加
export HADOOP_HOME=/home/hd/apps/hadoop
#增加
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
7.5 拷贝到第二，三台机
[root@master hadoop]# su hd
[hd@master hadoop]$ scp -r /home/hd/apps/hadoop hd@slave01:/home/hd/apps/
[hd@master hadoop]$ scp -r /home/hd/apps/hadoop hd@slave02:/home/hd/apps/
[hd@master hadoop]$ su root
Password:
[root@master hadoop]# scp /etc/profile root@slave01:/etc/
root@slave01's password:
profile 100% 1896 1.9KB/s 00:00
[root@master hadoop]# scp /etc/profile root@slave02:/etc/
profile 100% 1896 1.9KB/s 00:00
在第三台机器里加载环境
[root@master hadoop]# source /etc/profile
[hd@master hadoop]$ hadoop version
Hadoop 3.0.0
7.6 格式化
[hd@master hadoop]$ ll /home/hd/apps/hadoop/namenode
ls: cannot access /home/hd/apps/hadoop/namenode: No such file or directory
[hd@master hadoop]$ hadoop namenode -format
7.7 启动hadoop
start-dfs.sh 启动HDFS分布式文件系统，停止stop-dfs.sh
start-yarn.sh 启动Yarn资源管理器，停止stop-yarn.sh
start-all.sh HDFS分布式文件系统与Yarn启动，停止stop-all.sh
7.8 jps查看进程
[hd@master ~]$ jps
23668 SecondaryNameNode
23467 NameNode
23903 ResourceManager
24207 Jps
[hd@slave01 ~]$ jps
22341 DataNode
22649 Jps
22458 NodeManager
[hd@slave02 ~]$ jps
23367 Jps
23176 NodeManager
23051 DataNode
7.9 测试
hdfs 文件系统访问地址：http://192.168.126.128:50070/dfshealth.html#tab-overview文章来源：https://www.toymoban.com/news/detail-440720.html
Yarn资源管理器访问地址：http://192.168.126.128:8088/cluster文章来源地址https://www.toymoban.com/news/detail-440720.html
到了这里，关于【大数据学习篇2】Hadoop集群安装的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
