Linux

ノード(Apache+Fluentd)からAggregator(Fluentd+Logstash+Elasticsearch+Kibana)にログ出力でグラフ化

| by 優

 

ノード(Apache+Fluentd)

# vi /etc/security/limits.conf

root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536

 

# vi /etc/sysctl.conf

net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240    65535



※デフォルトで設定されていた


net.core.somaxconn = 1024
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem 　= 4096 12582912 16777216
net.ipv4.tcp_max_syn_backlog = 8096
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240 65535

 

 

# hostnamectl set-hostname Apache-Fluentd-Node
# reboot

 

# ulimit -n
65536

 

# yum install httpd
# systemctl start httpd
# systemctl enable httpd
# chmod o+x /var/log/httpd

 

# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.5.sh | sh
# systemctl restart td-agent

 

動作テスト

# curl -X POST -d 'json={"json":"優テスト"}' http://localhost:8888/debug.test

 

# tail -n 30 /var/log/td-agent/td-agent.log

2018-10-26 15:27:23 +0900 [info]: listening fluent socket on 0.0.0.0:24224
2018-10-26 15:27:23 +0900 [info]: listening dRuby uri="druby://127.0.0.1:24230" object="Engine"
2018-10-26 15:29:50 +0900 debug.test: {"json":"優テスト"}

 

 

# rm -rf /etc/td-agent/td-agent.conf

 

# vi /etc/td-agent/td-agent.conf
 
 
<source>
  type tail
  path /var/log/httpd/access_log
  tag log.yuulinux.tokyo-access
  pos_file /var/log/td-agent/.pos
  format apache2
</source>

<match **>
  type forward
  send_timeout 60s
  <server>
    host 133.130.116.232
    port 24224
  </server>
</match>

 

 

# systemctl restart td-agent
# systemctl enable td-agent

 

 

 

Aggregator(Fluentd+Logstash+Elasticsearch+Kibana)

 

# vi /etc/security/limits.conf

root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536

 

 

# vi /etc/sysctl.conf

net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240    65535



※デフォルトで設定されていた


net.core.somaxconn = 1024
net.core.netdev_max_backlog = 5000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_wmem = 4096 12582912 16777216
net.ipv4.tcp_rmem 　= 4096 12582912 16777216
net.ipv4.tcp_max_syn_backlog = 8096
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240 65535

 

# hostnamectl set-hostname tdSrv
# reboot

 

 

# ulimit -n
65536

 

# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.5.sh | sh
# systemctl start td-agent

 

動作テスト

# curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test

 

 

# tail -n 10 /var/log/td-agent/td-agent.log

2018-10-26 15:27:23 +0900 [info]: listening fluent socket on 0.0.0.0:24224
2018-10-26 15:27:23 +0900 [info]: listening dRuby uri="druby://127.0.0.1:24230" object="Engine"
2018-10-26 15:29:50 +0900 debug.test: {"json":"優テスト"}

 

 

 

# vi /etc/td-agent/td-agent.conf


<source>
    type forward
    port 24224
    bind 0.0.0.0
</source>


<match log.yuulinux.tokyo-access>
  type elasticsearch
  host localhost
  port 9200
  logstash_format true
  logstash_prefix log.yuulinux.tokyo-access
  flush_intarval 30s
</match>

 

# mkdir /var/log/fluentd
# chown td-agent:td-agent /var/log/fluentd

 

# systemctl enable td-agent
# systemctl restart td-agent

 

 

JAVAのインストール

 

# yum install -y java


Installed:
  java-1.8.0-openjdk.x86_64 1:1.8.0.191.b12-0.el7_5

Dependency Installed:
  copy-jdk-configs.noarch 0:3.3-10.el7_5                          fontconfig.x86_64 0:2.10.95-11.el7
  fontpackages-filesystem.noarch 0:1.44-8.el7                     giflib.x86_64 0:4.1.6-9.el7
  java-1.8.0-openjdk-headless.x86_64 1:1.8.0.191.b12-0.el7_5      javapackages-tools.noarch 0:3.4.1-11.el7
  libICE.x86_64 0:1.0.9-9.el7                                     libSM.x86_64 0:1.2.2-2.el7
  libXcomposite.x86_64 0:0.4.4-4.1.el7                            libXext.x86_64 0:1.3.3-3.el7
  libXfont.x86_64 0:1.5.2-1.el7                                   libXi.x86_64 0:1.7.9-1.el7
  libXrender.x86_64 0:0.9.10-1.el7                                libXtst.x86_64 0:1.2.3-1.el7
  libfontenc.x86_64 0:1.1.3-3.el7                                 libxslt.x86_64 0:1.1.28-5.el7
  lksctp-tools.x86_64 0:1.0.17-2.el7                              lyx-fonts.noarch 0:2.2.3-1.el7
  python-javapackages.noarch 0:3.4.1-11.el7                       python-lxml.x86_64 0:3.2.1-4.el7
  ttmkfdir.x86_64 0:3.0.9-42.el7                                  tzdata-java.noarch 0:2018e-3.el7
  xorg-x11-font-utils.x86_64 1:7.5-20.el7                         xorg-x11-fonts-Type1.noarch 0:7.5-9.el7

Dependency Updated:
  nspr.x86_64 0:4.19.0-1.el7_5                       nss.x86_64 0:3.36.0-7.el7_5
  nss-softokn.x86_64 0:3.36.0-5.el7_5                nss-softokn-freebl.x86_64 0:3.36.0-5.el7_5
  nss-sysinit.x86_64 0:3.36.0-7.el7_5                nss-tools.x86_64 0:3.36.0-7.el7_5
  nss-util.x86_64 0:3.36.0-1.el7_5

Complete!

 

 

# rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch

 

# vi /etc/yum.repos.d/elasticsearch.repo

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

 

# yum install -y elasticsearch
# systemctl restart elasticsearch
# systemctl enable elasticsearch

 

# curl -X GET http://localhost:9200/

{
  "name" : "E6N3bBZ",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "-Gy2wQNqTfSb8D_tRk-XHQ",
  "version" : {
    "number" : "6.4.2",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "04711c2",
    "build_date" : "2018-09-26T13:34:09.098244Z",
    "build_snapshot" : false,
    "lucene_version" : "7.4.0",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

 

# vi /etc/elasticsearch/elasticsearch.yml


#network.host: 192.168.0.1

↓変更

#network.host: 192.168.0.1
network.host: 0.0.0.0

 

# systemctl restart elasticsearch

 

# yum groupinstall 'Development tools'


プラグインのインストール
td-agent-gem install fluent-plugin-filter
td-agent-gem install fluent-plugin-forest
td-agent-gem install fluent-plugin-config-expander
td-agent-gem install fluent-plugin-elasticsearch
td-agent-gem install fluent-plugin-typecast
td-agent-gem install fluent-plugin-parser

 

 

# yum install logstash
# systemctl restart logstash
# systemctl enable logstash

 

 

# ls -laht /var/log/fluentd/

total 316K
-rw-r--r--   1 td-agent td-agent 308K Oct 28 22:56 yuulinux.tokyo-access.20181028.b5794a49412530255
drwxr-xr-x   2 td-agent td-agent 4.0K Oct 28 22:54 .
drwxr-xr-x. 12 root     root     4.0K Oct 28 22:22 ..

 

 

 

# vi /etc/yum.repos.d/kibana.repo

[kibana-6.x]
name=Kibana repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

 

 

# yum -y install kibana

 

# vi /etc/kibana/kibana.yml


#server.port: 5601
server.port: 5601

#server.host: "localhost"
server.host: "0.0.0.0"

#elasticsearch.url: "http://localhost:9200"
elasticsearch.url: "http://133.130.116.232:9200"

 

# systemctl restart kibana
# systemctl enable kibana

 

 

# curl 'localhost:9200/_cat/indices?v'
health status index                                uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   .kibana                              8PhOyTySRcWDH68CgqfJow   1   0          2            0     10.2kb         10.2kb
yellow open   log.yuulinux.tokyo-access-2018.10.30 JuxxdnZFSIWtJ21ELtisOA   5   1        398            0    271.5kb        271.5kb

 

http://133.xxx.yyy.232:5601/

 

 

 

 

 

 

関連記事 - More from my site -

Amazonおすすめ

iPad 9世代 2021年最新作

iPad 9世代出たから買い替え。安いぞ！🐱 初めてならiPad。Kindleを外で見るならiPad mini。ほとんどの人には通常のiPadをおすすめします＞＜

コメントを残す

メールアドレスが公開されることはありません。 * が付いている欄は必須項目です

日本語が含まれない投稿は無視されますのでご注意ください。（スパム対策）