Logstash启动时遇到的两个问题
Logstash启动需要对/usr/share/logstash/data目录有读写权限
根据日志,logstash需要对这个文件有读写权限,根据日志logstash还会在这个目录写创建文件
Logstash启动时要求pipelines文件在/usr/share/logstash/config/pipelines.yml
根据日志,Logstash启动过程中,会去默认位置读取pipelines.yml文件,而在其他位置都不行文章来源：https://www.toymoban.com/news/detail-773680.html
启动Logstash容器的命令
docker run --name logstash \ -v /root/elk_workspace/logstash/config:/usr/share/logstash/config \ -v /var/log/nginx:/usr/share/logstash/data \ -v /root/elk_workspace/logstash/pipeline:/usr/share/logstash/pipeline \ -d logstash:7.14.0文章来源地址https://www.toymoban.com/news/detail-773680.html
虚拟机中的文件目录及文件信息
[root@iZbp1eq14sxovxncihmbb1Z logstash]# pwd
/root/elk_workspace/logstash
[root@iZbp1eq14sxovxncihmbb1Z logstash]# ll
total 8
drwxr-xr-x 2 root root 4096 Jan 28 22:23 config
drwxr-xr-x 2 root root 4096 Jan 28 22:12 pipeline
[root@iZbp1eq14sxovxncihmbb1Z logstash]# cd config/
[root@iZbp1eq14sxovxncihmbb1Z config]# ll
total 8
-rw-r--r-- 1 root root 526 Jan 28 21:04 logstash.yml
-rw-r--r-- 1 root root 84 Jan 28 21:21 pipelines.yml
[root@iZbp1eq14sxovxncihmbb1Z config]# cat logstash.yml
node.name: logstash-203
# 日志文件目录配置
path.logs: /usr/share/logstash/logs
# # 验证配置文件及存在性
config.test_and_exit: false
# # 配置文件改变时是否自动加载
config.reload.automatic: false
# # 重新加载配置文件间隔
config.reload.interval: 60s
# # debug模式 开启后会打印解析后的配置文件 包括密码等信息 慎用
# # 需要同时配置日志等级为debug
config.debug: true
log.level: debug
# # The bind address for the metrics REST endpoint.
http.host: 0.0.0.0
[root@iZbp1eq14sxovxncihmbb1Z config]# cat pipelines.yml
- pipeline.id: pipline_1
path.config: /usr/share/logstash/pipeline/pipline_1.conf
[root@iZbp1eq14sxovxncihmbb1Z config]# ll
total 8
-rw-r--r-- 1 root root 526 Jan 28 21:04 logstash.yml
-rw-r--r-- 1 root root 84 Jan 28 21:21 pipelines.yml
[root@iZbp1eq14sxovxncihmbb1Z config]# cd ..
[root@iZbp1eq14sxovxncihmbb1Z logstash]# ll
total 8
drwxr-xr-x 2 root root 4096 Jan 28 22:23 config
drwxr-xr-x 2 root root 4096 Jan 28 22:12 pipeline
[root@iZbp1eq14sxovxncihmbb1Z logstash]# cd pipeline/
[root@iZbp1eq14sxovxncihmbb1Z pipeline]# ll
total 4
-rw-r--r-- 1 root root 168 Jan 28 21:09 pipline_1.conf
[root@iZbp1eq14sxovxncihmbb1Z pipeline]# cat pipline_1.conf
input {
file{
path => "/usr/share/logstash/data/test.log"
codec => json
start_position => "beginning"
}
}
output {
stdout {
codec => rubydebug
}
}
[root@iZbp1eq14sxovxncihmbb1Z pipeline]#
通过docker logs -f logstash查询出来的日志文件
Using bundled JDK: /usr/share/logstash/jdk
warning: no jvm.options file found
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2023-01-28 14:03:51.580 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +jit [linux-x86_64]"}
[DEBUG] 2023-01-28 14:03:51.587 [main] scaffold - Found module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[DEBUG] 2023-01-28 14:03:51.587 [main] registry - Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x2331ac37 @directory="/usr/share/logstash/modules/fb_apache/configuration", @module_name="fb_apache", @kibana_version_parts=["6", "0", "0"]>}
[DEBUG] 2023-01-28 14:03:51.587 [main] scaffold - Found module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[DEBUG] 2023-01-28 14:03:51.588 [main] registry - Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x588e4caa @directory="/usr/share/logstash/modules/netflow/configuration", @module_name="netflow", @kibana_version_parts=["6", "0", "0"]>}
[FATAL] 2023-01-28 14:03:51.591 [main] runner - An unexpected error occurred! {:error=>#<ArgumentError: Path "/usr/share/logstash/data" must be a writable directory. It is not writable.>, :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/settings.rb:530:in `validate'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:290:in `validate_value'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:201:in `block in validate_all'", "org/jruby/RubyHash.java:1415:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/settings.rb:200:in `validate_all'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:326:in `execute'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:67:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/runner.rb:274:in `run'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/clamp-0.6.5/lib/clamp/command.rb:132:in `run'", "/usr/share/logstash/lib/bootstrap/environment.rb:88:in `<main>'"]}
[FATAL] 2023-01-28 14:03:51.594 [main] Logstash - Logstash stopped processing because of an error: (SystemExit) exit
org.jruby.exceptions.SystemExit: (SystemExit) exit
at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:747) ~[jruby-complete-9.2.19.0.jar:?]
at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:710) ~[jruby-complete-9.2.19.0.jar:?]
at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:89) ~[?:?]
Using bundled JDK: /usr/share/logstash/jdk
warning: no jvm.options file found
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2023-01-28 14:07:51.960 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +jit [linux-x86_64]"}
[DEBUG] 2023-01-28 14:07:51.968 [main] scaffold - Found module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[DEBUG] 2023-01-28 14:07:51.968 [main] registry - Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x7148320d @directory="/usr/share/logstash/modules/fb_apache/configuration", @module_name="fb_apache", @kibana_version_parts=["6", "0", "0"]>}
[DEBUG] 2023-01-28 14:07:51.969 [main] scaffold - Found module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[DEBUG] 2023-01-28 14:07:51.969 [main] registry - Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x24d642eb @directory="/usr/share/logstash/modules/netflow/configuration", @module_name="netflow", @kibana_version_parts=["6", "0", "0"]>}
[INFO ] 2023-01-28 14:07:51.975 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[INFO ] 2023-01-28 14:07:51.975 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[DEBUG] 2023-01-28 14:07:52.239 [LogStash::Runner] runner - -------- Logstash Settings (* means modified) ---------
[DEBUG] 2023-01-28 14:07:52.240 [LogStash::Runner] runner - *node.name: "logstash-203" (default: "56e81483a095")
[DEBUG] 2023-01-28 14:07:52.240 [LogStash::Runner] runner - path.data: "/usr/share/logstash/data"
[DEBUG] 2023-01-28 14:07:52.240 [LogStash::Runner] runner - modules.cli: <Java::OrgLogstashUtil::ModulesSettingArray:1 []>
[DEBUG] 2023-01-28 14:07:52.240 [LogStash::Runner] runner - modules: []
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - modules_list: []
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - modules_variable_list: []
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - modules_setup: false
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - config.test_and_exit: false
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - config.reload.automatic: false
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - *config.reload.interval: #<Java::OrgLogstashUtil::TimeValue:0x5195117> (default: #<Java::OrgLogstashUtil::TimeValue:0x7da1bf14>)
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - config.support_escapes: false
[DEBUG] 2023-01-28 14:07:52.243 [LogStash::Runner] runner - config.field_reference.parser: "STRICT"
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - metric.collect: true
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.id: "main"
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.system: false
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.workers: 2
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.batch.size: 125
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.batch.delay: 50
[DEBUG] 2023-01-28 14:07:52.244 [LogStash::Runner] runner - pipeline.unsafe_shutdown: false
[DEBUG] 2023-01-28 14:07:52.248 [LogStash::Runner] runner - pipeline.java_execution: true
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - pipeline.reloadable: true
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - pipeline.plugin_classloaders: false
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - pipeline.separate_logs: false
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - pipeline.ordered: "auto"
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - pipeline.ecs_compatibility: "disabled"
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - path.plugins: []
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - *config.debug: true (default: false)
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - *log.level: "debug" (default: "info")
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - version: false
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - help: false
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - log.format: "plain"
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - http.enabled: true
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - *http.host: "0.0.0.0" (default: "127.0.0.1")
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - http.port: 9600..9700
[DEBUG] 2023-01-28 14:07:52.249 [LogStash::Runner] runner - http.environment: "production"
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.type: "memory"
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.drain: false
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.page_capacity: 67108864
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.max_bytes: 1073741824
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.max_events: 0
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.checkpoint.acks: 1024
[DEBUG] 2023-01-28 14:07:52.250 [LogStash::Runner] runner - queue.checkpoint.writes: 1024
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - queue.checkpoint.interval: 1000
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - queue.checkpoint.retry: false
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - dead_letter_queue.enable: false
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - dead_letter_queue.max_bytes: 1073741824
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - dead_letter_queue.flush_interval: 5000
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - slowlog.threshold.warn: #<Java::OrgLogstashUtil::TimeValue:0x4f98ae97>
[DEBUG] 2023-01-28 14:07:52.256 [LogStash::Runner] runner - slowlog.threshold.info: #<Java::OrgLogstashUtil::TimeValue:0x738f6e44>
[DEBUG] 2023-01-28 14:07:52.268 [LogStash::Runner] runner - slowlog.threshold.debug: #<Java::OrgLogstashUtil::TimeValue:0x4913778f>
[DEBUG] 2023-01-28 14:07:52.269 [LogStash::Runner] runner - slowlog.threshold.trace: #<Java::OrgLogstashUtil::TimeValue:0xa22be4c>
[DEBUG] 2023-01-28 14:07:52.269 [LogStash::Runner] runner - keystore.classname: "org.logstash.secret.store.backend.JavaKeyStore"
[DEBUG] 2023-01-28 14:07:52.269 [LogStash::Runner] runner - keystore.file: "/usr/share/logstash/config/logstash.keystore"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - path.queue: "/usr/share/logstash/data/queue"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - path.dead_letter_queue: "/usr/share/logstash/data/dead_letter_queue"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - path.settings: "/usr/share/logstash/config"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - path.logs: "/usr/share/logstash/logs"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.enabled: false
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.logstash.poll_interval: #<Java::OrgLogstashUtil::TimeValue:0x37ee58cf>
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.pipeline.id: ["main"]
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.elasticsearch.username: "logstash_system"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.elasticsearch.hosts: ["https://localhost:9200"]
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.elasticsearch.ssl.verification_mode: "certificate"
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.management.elasticsearch.sniffing: false
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.monitoring.enabled: false
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.hosts: ["http://localhost:9200"]
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.monitoring.collection.interval: #<Java::OrgLogstashUtil::TimeValue:0x7c5f244c>
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.monitoring.collection.timeout_interval: #<Java::OrgLogstashUtil::TimeValue:0x77cfbbba>
[DEBUG] 2023-01-28 14:07:52.270 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.username: "logstash_system"
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.ssl.verification_mode: "certificate"
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - xpack.monitoring.elasticsearch.sniffing: false
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - xpack.monitoring.collection.pipeline.details.enabled: true
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - xpack.monitoring.collection.config.enabled: true
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - monitoring.enabled: false
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - monitoring.elasticsearch.hosts: ["http://localhost:9200"]
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - monitoring.collection.interval: #<Java::OrgLogstashUtil::TimeValue:0x1bd051c4>
[DEBUG] 2023-01-28 14:07:52.271 [LogStash::Runner] runner - monitoring.collection.timeout_interval: #<Java::OrgLogstashUtil::TimeValue:0x3be76b8d>
[DEBUG] 2023-01-28 14:07:52.278 [LogStash::Runner] runner - monitoring.elasticsearch.username: "logstash_system"
[DEBUG] 2023-01-28 14:07:52.278 [LogStash::Runner] runner - monitoring.elasticsearch.ssl.verification_mode: "certificate"
[DEBUG] 2023-01-28 14:07:52.278 [LogStash::Runner] runner - monitoring.elasticsearch.sniffing: false
[DEBUG] 2023-01-28 14:07:52.278 [LogStash::Runner] runner - monitoring.collection.pipeline.details.enabled: true
[DEBUG] 2023-01-28 14:07:52.279 [LogStash::Runner] runner - monitoring.collection.config.enabled: true
[DEBUG] 2023-01-28 14:07:52.279 [LogStash::Runner] runner - node.uuid: ""
[DEBUG] 2023-01-28 14:07:52.279 [LogStash::Runner] runner - --------------- Logstash Settings -------------------
[DEBUG] 2023-01-28 14:07:52.283 [LogStash::Runner] multilocal - Reading pipeline configurations from YAML {:location=>"/usr/share/logstash/config/pipelines.yml"}
ERROR: Failed to read pipelines yaml file. Location: /usr/share/logstash/config/pipelines.yml
usage:
bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
bin/logstash --help
[FATAL] 2023-01-28 14:07:52.304 [LogStash::Runner] Logstash - Logstash stopped processing because of an error: (SystemExit) exit
org.jruby.exceptions.SystemExit: (SystemExit) exit
at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:747) ~[jruby-complete-9.2.19.0.jar:?]
at org.jruby.RubyKernel.exit(org/jruby/RubyKernel.java:710) ~[jruby-complete-9.2.19.0.jar:?]
at usr.share.logstash.lib.bootstrap.environment.<main>(/usr/share/logstash/lib/bootstrap/environment.rb:89) ~[?:?]
到了这里，关于Logstash启动时遇到的两个问题的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！
