使用elasticsearch+filebeat+kibana收集pod指定目录日志,filebeat用于收集日志,es用于存储,kibana用于展示。本例以收集部署于k8s内的nginx日志为例子。
1、部署es+kibana
version: '3.9'
services:
elasticsearch:
image: elasticsearch:7.13.3
container_name: elasticsearch
networks:
- net-es
volumes:
- ./data/elasticsearch/data:/usr/share/elasticsearch/data #这里将elasticsearch的数据文件映射本地,以保证下次如果删除>了容器还有数据
environment:
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
ports:
- "9200:9200"
elastichd:
image: containerize/elastichd:latest
container_name: elasticsearch-hd
networks:
- net-es
ports:
- "9800:9800"
depends_on:
- "elasticsearch"
links:
- "elasticsearch:demo"
kibana:
image: kibana:7.13.3
container_name: kibana
depends_on:
- elasticsearch #kibana在elasticsearch启动之后再启动
environment:
ELASTICSEARCH_HOSTS: http://ip:9200 #设置访问elasticsearch的地址
I18N_LOCALE: zh-CN
ports:
- 5601:5601
#这里要注意,es和eshd要在相同网络才能被links
networks:
net-es:
external: false
2.创建filebeat配置文件(基于elasticsearch存储)
[root@master filebeat]# cat filebeat-es-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeatconf
data:
filebeat.yml: |-
filebeat.inputs:
- input_type: log
paths:
- /data/log/*/*.log # 收集日志的路径
tail_files: true
fields:
pod_name: '${podName}'
pod_ip: '${podIp}'
pod_deploy_name: '${podDeployName}'
pod_namespace: '${podNamespace}'
output.elasticsearch: # 可以使用kafka,redis做缓存,此处直接将数据存储在elasticsearch中
hosts: ["ip:9200"] # elasticsearch 的ip+port
index: "app-%{+yyyy.MM.dd}" # 索引名称定义
setup.template.name: "filebeat-sidecar"
setup.template.pattern: "filebeat-sidecar"
3、创建nginx-filebeat Sidecar(基于elasticsearch的配置)
[root@k8s-master elk]# cat nginx.yml
# lishanbin-nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: lishanbin-nginx
spec:
selector:
matchLabels:
app: lishanbin-nginx
template:
metadata:
labels:
app: lishanbin-nginx
spec:
containers:
- name: filebeat
image: registry.cn-beijing.aliyuncs.com/dotbalo/filebeat:7.10.2
resources:
requests:
memory: "100Mi"
cpu: "10m"
limits:
cpu: "200m"
memory: "300Mi"
imagePullPolicy: IfNotPresent
env:
- name: podIp
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: podName
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: podNamespace
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: podDeployName
value: app
- name: TZ
value: "Asia/Shanghai"
securityContext:
runAsUser: 0
volumeMounts:
- name: data-log
mountPath: /data/log/app/
- name: filebeatconf
mountPath: /usr/share/filebeat/filebeat.yml
subPath: usr/share/filebeat/filebeat.yml
- name: lishanbin-nginx
image: nginx:1.23.1
env:
- name: "username"
value: "lishanbin"
- name: "password"
value: "admin123"
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
ports:
- containerPort: 80
volumeMounts:
- name: data-log
mountPath: /var/log/nginx
volumes:
- name: data-log
hostPath:
path: /tmp/hostpath
- emptyDir: {}
name: logpath
- name: local-time
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: filebeatconf
configMap:
name: filebeatconf # 挂载filebeat配置文件
items:
- key: filebeat.yml
path: usr/share/filebeat/filebeat.yml
---
apiVersion: v1
kind: Service
metadata:
name: lishanbin-nginx
labels:
app: lishanbin-nginx
spec:
ports:
- port: 80
protocol: TCP
name: http
selector:
app: lishanbin-nginx
---
apiVersion: v1
kind: Service
metadata:
name: ishanbin-nginx-nodeport
spec:
selector:
app: lishanbin-nginx
ports:
- protocol: TCP
port: 80
targetPort: 80
nodePort: 30080
type: NodePort
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: lishanbin-nginx
namespace: default
spec:
ingressClassName: lishanbin # 使用 nginx 的 IngressClass(关联的 ingress-nginx 控制器)
rules:
- host: ngdemo.qikqiak.com # 将域名映射到 lishanbin-nginx 服务
http:
paths:
- path: /
pathType: ImplementationSpecific
backend:
service: # 将所有请求发送到 lishanbin-nginx 服务的 80 端口
name: lishanbin-nginx
port:
number: 80
# 不过需要注意大部分Ingress控制器都不是直接转发到Service
# 而是只是通过Service来获取后端的Endpoints列表,直接转发到Pod,这样可以减少网络跳转,提高性能
4.kibana创建索引,查看采集到的日志
文章来源地址https://www.toymoban.com/news/detail-550010.html
文章来源:https://www.toymoban.com/news/detail-550010.html
到了这里,关于k8s-EFK (filebeat)日志收集的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!