1、常规初始化操作
三个节点都需要执行
hostnamectl set-hostname cdhmaster
yum -y install vim lrzsz net-tools
cat >>/etc/hosts<<EOF
192.168.180.210 cdhmaster
192.168.180.200 cdhslave01
192.168.180.190 cdhslave02
EOF
systemctl stop firewalld.service && systemctl disable firewalld.service
sed -i ‘/^SELINUX=/s/enforcing/disabled/’ /etc/selinux/config && setenforce 0
2、配置时间同步
cdhmaster:
yum install chrony -y
vim /etc/chrony.conf
server ntp6.aliyun.com iburst
allow 192.168.0.0/16
systemctl enable chronyd.service && systemctl restart chronyd.service
chronyc sources && chronyc -a makestep
cdhslave01/cdhslave02:
yum install chrony -y
vim /etc/chrony.conf
server cdhmaster iburst
systemctl enable chronyd.service && systemctl restart chronyd.service && chronyc sources
3、配置本机SSH免密码登录
ssh-keygen -t rsa
ssh-copy-id cdhmaster
ssh-copy-id cdhslave01
ssh-copy-id cdhslave02
4、安装jdk
三个节点都要执行
tar zxf jdk-8u171-linux-x64.tar.gz -C /opt/
mv /opt/jdk1.8.0_171/ /opt/jdk
cat >>/etc/profile<<EOF
export JAVA_HOME=/opt/jdk
export PATH=
J
A
V
A
H
O
M
E
/
b
i
n
:
JAVA_HOME/bin:
JAVAHOME/bin:PATH
EOF
source /etc/profile && java -version
5、禁用交换分区和透明大页
三个节点都要操作
sysctl -w vm.swappiness=0
echo “vm.swappiness=0” >> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo “echo never > /sys/kernel/mm/transparent_hugepage/defrag” >> /etc/rc.d/rc.local
echo “echo never > /sys/kernel/mm/transparent_hugepage/enabled” >> /etc/rc.d/rc.local
chmod +x /etc/rc.d/rc.local
6、安装数据库
只在主节点cdhmastar上操作
rpm -qa | grep mariadb
rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64
yum install -y libaio
useradd mysql -s /sbin/nologin
tar zvxf mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz
mv mysql-5.7.22-linux-glibc2.12-x86_64 /usr/local/mysql
cd /usr/local/mysql/bin
./mysqld --initialize --user=mysql --datadir=/usr/local/mysql/data
自动生成数据库密码 IrszF>8vd(gy
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
vim /etc/my.cnf
[client]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
[mysqld]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
pid_file = /usr/local/mysql/data/mysql.pid
datadir = /usr/local/mysql/data
default_storage_engine = InnoDB
max_allowed_packet = 512M
max_connections = 2048
open_files_limit = 65535
skip-name-resolve
lower_case_table_names=1
character-set-server = utf8mb4
collation-server = utf8mb4_unicode_ci
init_connect=‘SET NAMES utf8mb4’
innodb_buffer_pool_size = 1024M
innodb_log_file_size = 2048M
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 0
key_buffer_size = 64M
log-error = /usr/local/mysql/data/mysql_error.log
log-bin = /usr/local/mysql/data/mysql-bin
slow_query_log = 1
slow_query_log_file = /usr/local/mysql/data/mysql_slow_query.log
long_query_time = 5
tmp_table_size = 32M
max_heap_table_size = 32M
query_cache_type = 0
query_cache_size = 0
server-id=1
/etc/init.d/mysqld start
ln -s /usr/local/mysql/bin/* /usr/sbin/
mysqladmin -uroot -p password ‘123456’
mysql -u root -p123456
alter user ‘root’@‘localhost’;
alter user ‘root’@‘localhost’ identified by ‘123456’;
create database hive default charset utf8 collate utf8_general_ci;
create database oozie default charset utf8 collate utf8_general_ci;
grant all privileges on . to ‘cdh’@‘localhost’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘cdh’@‘%’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘root’@‘127.0.0.1’ identified by ‘123456’ with grant option;
grant all privileges on . to ‘scm’@‘127.0.0.1’ identified by ‘123456’ with grant option;
flush privileges;
7、安装CDH
所有节点上安装依赖包
yum install -y psmisc libxslt libxslt-python perl
上传cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz、mysql-connector-java-5.1.46.tar.gz到服务器
tar zxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/
mv /opt/cm-5.14.0/ /opt/cm
tar zxf mysql-connector-java-5.1.46.tar.gz
cd mysql-connector-java-5.1.46
chmod +x mysql-connector-java-5.1.46-bin.jar
cp mysql-connector-java-5.1.46-bin.jar /opt/cm/share/cmf/lib/mysql-connector-java.jar
mkdir /usr/share/java
cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
在三台机器上创建cloudera-scm用户
useradd --system --home=/opt/cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment “Cloudera SCM User” cloudera-scm
相关选项说明
–no-create-home 相当于-M,不创建用户的主目录
–shell 用户登录的shell,/bin/false禁止用户登录
–system 创建一个系统用户,在sys_uid_max中取ID
–comment 添加备注
/opt/cm/share/cmf/schema/scm_prepare_database.sh mysql cm -h localhost -uroot -p’123456’ scm ‘123456’
vim /opt/cm/etc/cloudera-scm-agent/config.ini
server_host=192.168.180.210
同步Cloudera Manager 数据到其他节点
scp -r /opt/cm cdhslave01:/opt/
scp -r /opt/cm cdhslave02:/opt/
上传manifest.json、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1文件到cdhmaster服务器上
mv manifest.json CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 /opt/cloudera/parcel-repo/
cd /opt/cloudera/parcel-repo/
mv CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha
vim /opt/cm/etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.useSSL=true
mkdir /var/lib/cloudera-scm-server
/opt/cm/etc/init.d/cloudera-scm-server start
/opt/cm/etc/init.d/cloudera-scm-agent start
netstat -anpt | grep 7180
在slave上只需启动代理服务
/opt/cm/etc/init.d/cloudera-scm-agent start
netstat -anpt | grep -E ‘19001|9000’
8、安装配置CDH 群集
访问CDH控制台
http://192.168.180.210:7180 默认的用户名和密码分别是admin 和admin。
三台机器都执行
上传mysql-connector-java-5.1.46.tar.gz到服务器
cd mysql-connector-java-5.1.46/
cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
cp mysql-connector-java-5.1.46-bin.jar /var/lib/oozie/mysql-connector-java.jar
将上述三个文件复制到两台从节点
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/var/lib/oozie/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar
scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/var/lib/oozie/mysql-connector-java.jar
9、配置并添加Kafka 服务
先激活kafka
上传软件
mv KAFKA-1.2.0.jar /opt/cloudera/csd/
mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel /opt/cloudera/parcel-repo/文章来源:https://www.toymoban.com/news/detail-759724.html
mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha1 /opt/cloudera/parcel-repo/KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha文章来源地址https://www.toymoban.com/news/detail-759724.html
到了这里,关于物联网架构之CDH的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!