Linux centOS 7部署ELK(elasticSearch、logstash、kibana)
目录
1【yum模式安装】
1.1【安装ElasticSearch】
1.2【安装Kibana】
1.3【安装Logstash】
1.4【安装Filebeat】
2【压缩包方式安装elk】
1【yum模式安装】
原文链接:
- How To Install Elasticsearch, Logstash, and Kibana (Elastic Stack) on Ubuntu 16.04 | DigitalOcean
- 十分钟搞定CentOS 7部署ELK_哔哩哔哩_bilibili
1.1【安装ElasticSearch】
全文搜索属于最常见的需求,开源的Elasticsearch(以下简称 es)是目前全文搜索引擎的首选。
它可以快速地储存、搜索和分析海量数据,维基百科、Stack Overflow、Github都采用它。
Elasticsearch简称es,在企业内同样是一款应用非常广泛的搜索引擎服务,很多服务中的搜索功能,都是基于es来实现的。
1【添加yum仓库】
# root执行
# 导入仓库密钥
rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch# 添加yum源
# 编辑文件
vim /etc/yum.repos.d/elasticsearch.repo[elasticsearch-7.x]
name=Elasticsearch repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
# 更新yum缓存
yum makecache
2【安装es 】
yum install -y elasticsearch
3【配置es 】
vim /etc/elasticsearch/elasticsearch.yml
# 17行,设置集群名称
cluster.name: my-cluster# 23行,设置节点名称
node.name: node-1# 56行,允许外网访问
network.host: 0.0.0.0# 74行,配置集群master节点
cluster.initial_master_nodes: ["node-1"]
4【启动es 】
systemctl start | stop | status | enable | disable elasticsearch
5【关闭防火墙 】
systemctl stop firewalld
systemctl disable firewalld
6【测试】
浏览器打开:http://x.x.x.x/9200
1.2【安装Kibana】
yum install -y kibana
vim /etc/kibana/kibana.yml
systemctl start kibana
systemctl status kibana
1.3【安装Logstash】
yum install -y logstash
vim /etc/logstash/conf.d/02-beats-input.conf
监听5044端口
input {
beats {
port => 5044
}
}
vim /etc/logstash/conf.d/10-syslog-filter.conffilter {
if [fileset][module] == "system" {
if [fileset][name] == "auth" {
grok {
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} %{DATA:[system][auth][ssh][method]} for (invalid user )?%{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]} port %{NUMBER:[system][auth][ssh][port]} ssh2(: %{GREEDYDATA:[system][auth][ssh][signature]})?",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: %{DATA:[system][auth][ssh][event]} user %{DATA:[system][auth][user]} from %{IPORHOST:[system][auth][ssh][ip]}",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sshd(?:\[%{POSINT:[system][auth][pid]}\])?: Did not receive identification string from %{IPORHOST:[system][auth][ssh][dropped_ip]}",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} sudo(?:\[%{POSINT:[system][auth][pid]}\])?: \s*%{DATA:[system][auth][user]} :( %{DATA:[system][auth][sudo][error]} ;)? TTY=%{DATA:[system][auth][sudo][tty]} ; PWD=%{DATA:[system][auth][sudo][pwd]} ; USER=%{DATA:[system][auth][sudo][user]} ; COMMAND=%{GREEDYDATA:[system][auth][sudo][command]}",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} groupadd(?:\[%{POSINT:[system][auth][pid]}\])?: new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} useradd(?:\[%{POSINT:[system][auth][pid]}\])?: new user: name=%{DATA:[system][auth][user][add][name]}, UID=%{NUMBER:[system][auth][user][add][uid]}, GID=%{NUMBER:[system][auth][user][add][gid]}, home=%{DATA:[system][auth][user][add][home]}, shell=%{DATA:[system][auth][user][add][shell]}$",
"%{SYSLOGTIMESTAMP:[system][auth][timestamp]} %{SYSLOGHOST:[system][auth][hostname]} %{DATA:[system][auth][program]}(?:\[%{POSINT:[system][auth][pid]}\])?: %{GREEDYMULTILINE:[system][auth][message]}"] }
pattern_definitions => {
"GREEDYMULTILINE"=> "(.|\n)*"
}
remove_field => "message"
}
date {
match => [ "[system][auth][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
geoip {
source => "[system][auth][ssh][ip]"
target => "[system][auth][ssh][geoip]"
}
}
else if [fileset][name] == "syslog" {
grok {
match => { "message" => ["%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}"] }
pattern_definitions => { "GREEDYMULTILINE" => "(.|\n)*" }
remove_field => "message"
}
date {
match => [ "[system][syslog][timestamp]", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
}
vim /etc/logstash/conf.d/30-elasticsearch-output.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t
[root@node1 ~]# sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t
Using bundled JDK: /usr/share/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
Sending Logstash logs to /var/log/logstash which is now configured via log4j2.properties
[2023-03-15T14:57:44,236][INFO ][logstash.runner ] Log4j configuration path used is: /etc/logstash/log4j2.properties
[2023-03-15T14:57:44,252][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.17.9", "jruby.version"=>"jruby 9.2.20.1 (2.5.8) 2021-11-30 2a2962fbd1 OpenJDK 64-Bit Server VM 11.0.18+10 on 11.0.18+10 +indy +jit [linux-x86_64]"}
[2023-03-15T14:57:44,255][INFO ][logstash.runner ] JVM bootstrap flags: [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djdk.io.File.enableADS=true, -Djruby.compile.invokedynamic=true, -Djruby.jit.threshold=0, -Djruby.regexp.interruptible=true, -XX:+HeapDumpOnOutOfMemoryError, -Djava.security.egd=file:/dev/urandom, -Dlog4j2.isThreadContextMapInheritable=true]
[2023-03-15T14:57:47,633][INFO ][org.reflections.Reflections] Reflections took 155 ms to scan 1 urls, producing 119 keys and 419 values
Configuration OK
[2023-03-15T14:57:49,304][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash
[root@node1 ~]#
systemctl status logstash
systemctl start logstash
systemctl enable logstash
1.4【安装Filebeat】
701 yum install -y filebeat
702 vim /etc/filebeat/filebeat.yml
703 filebeat modules enable system
704 filebeat modules list
705 vim /etc/filebeat/modules.d/system.yml
706 sudo filebeat setup --template -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["node1:9200"]'
707 sudo filebeat setup -e -E output.logstash.enabled=false -E output.elasticsearch.hosts=['node1:9200'] -E setup.kibana.host=node1:5601
708 systemctl start filebeat
709 systemctl enable filebeat
710 systemctl status filebeat
711 curl -XGET 'http://node1:9200/filebeat-*/_search?pretty'
712 systemctl enable kibana
713 systemctl status kibana
714 systemctl start kibana
715 systemctl status kibana
716 systemctl status elasticsearch
717 systemctl status logstash.service
718 systemctl start kibana
674 systemctl start elasticsearch
675 java -version
676 yum install -y kibana
677 vim /etc/kibana/kibana.yml
678 systemctl start kibana
679 systemctl status kibana
680 yum install -y logstash
681 vim /etc/kibana/kibana.yml
682 vim /etc/logstash/conf.d/02-beats-input.conf
683 vim /etc/logstash/conf.d/10-syslog-filter.conf
684 vim /etc/logstash/conf.d/30-elasticsearch-output.conf
685 sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t
686 export LANG="en_US";export LANGUAGE="en_US";export LC_ALL="en_US";top
687 vim /etc/kibana/kibana.yml
688 vim /etc/logstash/conf.d/02-beats-input.conf
689 vim /etc/logstash/conf.d/10-syslog-filter.conf
690 vim /etc/logstash/conf.d/30-elasticsearch-output.conf
691 user add logstash
692 useradd logstash
693 sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t
694 systemctl start elasticsearch
695 systemctl status elasticsearch
696 systemctl enable elasticsearch
697 systemctl start logstash
698 systemctl status logstash
699 systemctl enable logstash
700 systemctl status logstash
701 yum install -y filebeat
702 vim /etc/filebeat/filebeat.yml
703 filebeat modules enable system
704 filebeat modules list
705 vim /etc/filebeat/modules.d/system.yml
706 sudo filebeat setup --template -E output.logstash.enabled=false -E 'output.elasticsearch.hosts=["node1:9200"]'
707 sudo filebeat setup -e -E output.logstash.enabled=false -E output.elasticsearch.hosts=['node1:9200'] -E setup.kibana.host=node1:5601
708 systemctl start filebeat
709 systemctl enable filebeat
710 systemctl status filebeat
711 curl -XGET 'http://node1:9200/filebeat-*/_search?pretty'
712 systemctl enable kibana
713 systemctl status kibana
714 systemctl start kibana
715 systemctl status kibana
716 systemctl status elasticsearch
717 systemctl status logstash.service
718 systemctl start kibana
719 history
[root@node1 ~]#
2【压缩包方式安装elk】
- ELK 日志采集分析框架 【elasticsearch、logstash、kibana】_哔哩哔哩_bilibili
安装node.js,版本“node-v14.21.3-linux-x64.tar.xz”。
- 安装教程:linux安装nodejs【详细教程】_菜鸟fox的博客-CSDN博客
- 下载地址:Index of /download/release/v14.21.3/
./elasticsearch
logstash -f /opt/module/logstash-8.5.1/config/test/mysql01.conf
./kibana文章来源:https://www.toymoban.com/news/detail-729065.html
npm run start &文章来源地址https://www.toymoban.com/news/detail-729065.html
1 vim /etc/sysconfig/network-scripts/ifcfg-ens33
2 ifconfig
3 systemctl restart network
4 cat /etc/host
5 cat /etc/hostname
6 cat /etc/hosts
7 vim /etc/hosts
8 ifconfig
9 ll
10 vim /etc/sysconfig/network-scripts/ifcfg-ens33
11 cat /etc/hosts
12 reboot
13 vim /etc/sysconfig/network-scripts/ifcfg-ens33
14 ifconfig
15 systemctl restart network
16 cat /etc/host
17 cat /etc/hostname
18 cat /etc/hosts
19 vim /etc/hosts
20 ifconfig
21 ll
22 vim /etc/sysconfig/network-scripts/ifcfg-ens33
23 cat /etc/hosts
24 reboot
25 hostname
26 vim /etc/hosts
27 ifconfig
28 ping baidu.com
29 ping node1
30 hostname
31 ifconfig
32 vim /etc/hosts
33 ping www.baidu.com
34 yum install -y epel-release
35 systemctl stop firewalld
36 systemctl disable firewalld.service
37 vim /etc/sudoers
38 cd /opt
39 ll
40 exit
41 set +o history;
42 cd /opt
43 ll
44 su - vlu
45 vim /etc/sudoers
46 ll
47 rpm -qa | grep -i java
48 rpm -qa | grep -i java | xargs -n1 rpm -e --nodeps
49 rpm -qa | grep -i java
50 java -version
51 set +o history;
52 cd /opt/software/elk-8.5.1/
53 l;
54 ll
55 set +o history;
56 cd /opt/software
57 ls
58 tar -zxvf jdk-8u212-linux-x64.tar.gz -C /opt/module/
59 tar -zxvf jdk-11.0.18_linux-x64_bin.tar.gz -C /opt/module/
60 sudo vim /etc/profile.d/my_env.sh
61 source /etc/profile
62 java -version
63 java
64 javac
65 java -version
66 tar -zxvf hadoop-3.1.3.tar.gz -C /opt/module/
67 cd /opt/software/
68 tar -zxvf hadoop-3.1.3.tar.gz -C /opt/module/
69 cd /opt/software/elk-8.5.1
70 tar -zxvf elasticsearch-8.5.1-linux-x86_64.tar.gz -C /opt/module/
71 pwd
72 cd /opt/module/elasticsearch-8.5.1
73 ls
74 cd config
75 vim elasticsearch.yml
76 useradd es
77 passwd es
78 chown es:es elasticsearch-8.5.1/
79 chown es:es /opt/module/elasticsearch-8.5.1
80 pwd
81 cd ../
82 ls -l
83 cd ../
84 ls -l
85 chown -R es:es /opt/module/elasticsearch-8.5.1
86 vim /etc/security/limits.conf
87 cd /etc/security/limits.d/
88 ls -l
89 vim /etc/security/limits.d/20-nproc.conf
90 vim /etc/sysctl.conf
91 sysctl -p
92 su es
93 set +o history;
94 history
95 cd /opt/module/logstash-8.5.1/bin
96 pwd
97 ./logstash -f ../config/test/mysql01.conf
98 ./logstash -f /opt/module/logstash-8.5.1/config/test
99 ./logstash -f /opt/module/logstash-8.5.1/config/test/mysql01.conf
100 set +o history;
101 history
102 cd /opt/software/elk-8.5.1
103 ls
104 ll
105 tar -zxvf logstash-8.5.1-linux-x86_64.tar.gz -C /opt/module/
106 tar -zxvf kibana-8.5.1-linux-x86_64.tar.gz -C /opt/module/
107 cd /opt/module/logstash-8.5.1
108 cd config
109 cd ../
110 cd bin
111 cd /opt/module/elasticsearch-8.5.1/bin
112 ./elasticsearch
113 su es
114 set +o history;
115 pwd
116 ./kibana &
117 su es
118 sudo chown -R wudles /opt/module/kibana-8.5.1/
119 sudo chown -R es /opt/module/kibana-8.5.1/
120 su es
121 reboot
122 set +o history;
123 cd /opt/module/kibana-8.5.1/bin
124 ./kibana
125 su es
126 set +o history;
127 cd /opt/module/elasticsearch-8.5.1/bin
128 ./elasticsearch
129 su es
130 cd /opt/software
131 ll
132 tar -zxvf node-v18.15.0.tar.gz -C /opt/module
133 node -v
134 set +o history;
135 cd /opt/software
136 tar -zxvf node-v18.15.0-linux-x64.tar.xz /opt/module
137 tar -xf node-v18.15.0-linux-x64.tar.xz /opt/module
138 tar -xf node-v18.15.0-linux-x64.tar.xz /opt/module/
139 cd /opt/module
140 tar -xvf node-v18.15.0-linux-x64.tar.xz
141 cd node-v18.15.0-linux-x64/
142 cd bin
143 ./node -v
144 pwd
145 ./node -v
146 ./node -npm
147 vi /etc/profile
148 source /etc/profile
149 vi /etc/profile
150 source /etc/profile
151 pwd
152 npm
153 ./node -v
154 vi /etc/profile
155 source /etc/profile
156 sudo ln -s /opt/module/node-v18.15.0-linux-x64/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
157 sudo ln -s /opt/module/node-v18.15.0-linux-x64/bin/node /usr/local/bin/node
158 source /etc/profile
159 node -v
160 ln -s /opt/module/node-v18.15.0-linux-x64/bin/node /usr/local/bin/
161 ln -s /opt/module/node-v18.15.0-linux-x64/bin/npm /usr/local/bin/
162 node -v
163 vi /etc/profile
164 source /etc/profile
165 node -v
166 pwd
167 ./node -v
168 cd /opt/software
169 ll
170 tar -xvf node-v14.21.3-linux-x64.tar.xz /opt/module/
171 tar -xvf node-v14.21.3-linux-x64.tar.xz
172 cd /opt/module
173 tar -xvf node-v14.21.3-linux-x64.tar.xz
174 cd node-v14.21.3-linux-x64/
175 cd bin
176 ./node -v
177 ./npm -v
178 vi /etc/profil
179 source /etc/profile
180 ln -s /opt/module/node-v14.21.3-linux-x64/bin /usr/local/bin
181 ln -s /opt/module/node-v14.21.3-linux-x64/bin/node /usr/local/bin
182 ln -s /opt/module/node-v14.21.3-linux-x64/bin/npm /usr/local/bin/
183 cd /usr/local/bin
184 ll
185 rm -rf node
186 rm -rf npm
187 ll
188 ln -s /opt/module/node-v14.21.3-linux-x64/bin/node /usr/local/bin/
189 ln -s /opt/module/node-v14.21.3-linux-x64/bin/npm /usr/local/bin/
190 node -v
191 npm -v
192 cd /opt/software
193 ll
194 tar -zxvf elasticsearch-head-5.0.0.tar.gz -C /opt/module/
195 cd /opt/module/elasticsearch-head-5.0.0/_site
196 vim app.js
197 cd /opt/module
198 ls
199 ll
200 java
201 javac
202 java -version
203 cd elasticsearch-8.5.1/
204 cd bin
205 ./elasticsearch 2>&1 &
206 jps
207 su es
208 set +o history;
209 cd /opt/software/
210 ll
211 tar -zxvf elasticsearch-head-master.tar.gz -C /opt/module
212 jps
213 cd /opt/module/elasticsearch-head-master
214 npm run start &
215 set +o history;
216 history
[root@hadoop100 ~]# su es
[es@hadoop100 root]$ history
1 cd /opt/module/elasticsearch-8.5.1
2 cd bin/
3 ./elasticsearch
4 pwd
5 ./kibana
6 reboot
7 exit
8 cd /opt/module/elasticsearch-8.5.1/bin
9 ./elasticsearch
10 ./elasticsearch -d
11 cd /opt/module/kibana-8.5.1/bin
12 ./kibana
13 chmod u+w /etc/sudoers
14 su root
15 ./kibana &
16 sudo chown -R wudles /opt/module/es/kibana-7.6.1-linux-x86_64/
17 sudo chown -R wudles /opt/module/kibana-8.5.1/
18 su root
19 ./kibana
20 ./elasticsearch
21 ./elasticsearch 2>&1 1
22 jps
23 ./elasticsearch
24 history
[es@hadoop100 root]$
到了这里,关于Linux centOS 7部署ELK(elasticSearch、logstash、kibana)的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!