{"id":164,"date":"2020-03-14T19:38:01","date_gmt":"2020-03-14T10:38:01","guid":{"rendered":"http:\/\/www.skyer9.pe.kr\/wordpress\/?p=164"},"modified":"2020-03-28T22:05:34","modified_gmt":"2020-03-28T13:05:34","slug":"filebeat-kafka-logstash","status":"publish","type":"post","link":"https:\/\/www.skyer9.pe.kr\/wordpress\/?p=164","title":{"rendered":"Filebeat + Kafka + Logstash"},"content":{"rendered":"

Filebeat + Kafka + Logstash<\/h1>\n

\uc5b4\ub5a0\ud55c \uc774\uc720\ub85c ELK \uc2a4\ud0dd\uc774 \ub2e4\uc6b4\ub418\uc5c8\uc744 \ub54c, \uc800\uc7a5\ud558\uc9c0 \ubabb\ud55c \ub85c\uadf8\ub97c \ubcf4\uad00\ud558\uae30 \uc704\ud574 Apache Kafka<\/code> \ub97c \uc774\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n

JAVA 8 \uc774\uc0c1\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\uc73c\uba74 \uc5ec\uae30<\/a> \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4.<\/p>\n

ELK \uc2a4\ud0dd\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\ub2e4\uba74 \uc5ec\uae30<\/a> \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4.<\/p>\n

Filebeat \uac00 \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\ub2e4\uba74 \uc5ec\uae30<\/a> \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4.<\/p>\n

Kafka \uac00 \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\uc73c\uba74 \uc5ec\uae30<\/a> \ub97c \ucc38\uc870\ud558\uc5ec \uc124\uce58\ud569\ub2c8\ub2e4.<\/p>\n

Kafka \ub85c \ub85c\uadf8 \uc804\uc1a1\ud558\uae30<\/h2>\n

\uae30\uc874\uc5d0 filebeat \uc5d0\uc11c logstash \ub85c \uc9c1\uc811 \ub85c\uadf8\ub97c \uc804\uc1a1\ud558\ub358 \ubc29\uc2dd\uc5d0\uc11c, Kafka \ub97c \uac70\uccd0 \uc804\uc1a1\ub418\ub3c4\ub85d \uc218\uc815\ud569\ub2c8\ub2e4.<\/p>\n

\uc6b0\uc120 filebeat \uac00 Kafka \ub85c \ub85c\uadf8\ub97c \uc804\uc1a1\ud558\ub3c4\ub85d \uc218\uc815\ud569\ub2c8\ub2e4.<\/p>\n

sudo vi \/etc\/filebeat\/filebeat.yml<\/code><\/pre>\n
filebeat.inputs:\n- type: log\n  paths:\n    - \/var\/log\/nginx\/*.log\n  exclude_files: ['\\.gz$']\n  # scan_frequency: 10s\n\noutput.kafka:\n  hosts: ["<\uce74\ud504\uce74\uc11c\ubc84\uc544\uc774\ud53c>:9092"]\n  codec.format:\n    string: '%{[message]}'\n  topic: 'skyer9.pe.kr'\n  partition.round_robin:\n    reachable_only: false\n\n# output.logstash:\n#   hosts: ["172.31.31.196:5400"]\n#   # ssl.certificate_authorities: ["\/etc\/elk-certs\/elk-ssl.crt"]\n#   # ssl.certificate: "\/etc\/elk-certs\/elk-ssl.crt"\n#   # ssl.key: "\/etc\/elk-certs\/elk-ssl.key"<\/code><\/pre>\n
\uc11c\ube44\uc2a4\ub97c \uc7ac\uc2dc\uc791 \ud569\ub2c8\ub2e4.<\/p>\n
sudo systemctl restart filebeat<\/code><\/pre>\n
Logstash \uc5d0\uc11c \ub85c\uadf8 \uc218\uc2e0\ud558\uae30<\/h2>\n
sudo vi \/etc\/logstash\/conf.d\/logstash-nginx-es.conf<\/code><\/pre>\n
input {\n    kafka {\n        bootstrap_servers => "<\uce74\ud504\uce74\uc11c\ubc84\uc544\uc774\ud53c>:9092"\n            topics => ["skyer9.pe.kr"]\n            consumer_threads => 2\n            # group_id => "Your group"\n    }\n}\n\nfilter {\n    grok {\n        match => [ "message" , "%{COMBINEDAPACHELOG}+%{GREEDYDATA:extra_fields}"]\n        overwrite => [ "message" ]\n    }\n    mutate {\n        convert => ["response", "integer"]\n        convert => ["bytes", "integer"]\n        convert => ["responsetime", "float"]\n    }\n    # geoip {\n    #     source => "clientip"\n    #     target => "geoip"\n    #     add_tag => [ "nginx-geoip" ]\n    # }\n    date {\n        match => [ "timestamp" , "dd\/MMM\/YYYY:HH:mm:ss Z" ]\n        remove_field => [ "timestamp" ]\n    }\n    useragent {\n        source => "agent"\n    }\n}\n\noutput {\n    elasticsearch {\n        hosts => ["localhost:9200"]\n        index => "weblogs-%{+YYYY.MM.dd}"\n        document_type => "nginx_logs"\n    }\n    stdout { codec => rubydebug }\n}<\/code><\/pre>\n
\uc11c\ube44\uc2a4\ub97c \uc7ac\uc2dc\uc791\ud569\ub2c8\ub2e4.<\/p>\n
sudo systemctl restart logstash<\/code><\/pre>\n
\uc778\ub371\uc2a4\uc5d0 \uc785\ub825\uc774 \uc798 \uc548\ub420 \ub54c\ub294 \uc778\ub371\uc2a4\ub97c \uc0ad\uc81c\ud558\uace0 \ub2e4\uc2dc \ud14c\uc2a4\ud2b8 \ud569\ub2c8\ub2e4.<\/p>\n","protected":false},"excerpt":{"rendered":"
Filebeat + Kafka + Logstash \uc5b4\ub5a0\ud55c \uc774\uc720\ub85c ELK \uc2a4\ud0dd\uc774 \ub2e4\uc6b4\ub418\uc5c8\uc744 \ub54c, \uc800\uc7a5\ud558\uc9c0 \ubabb\ud55c \ub85c\uadf8\ub97c \ubcf4\uad00\ud558\uae30 \uc704\ud574 Apache Kafka \ub97c \uc774\uc6a9\ud569\ub2c8\ub2e4. JAVA 8 \uc774\uc0c1\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\uc73c\uba74 \uc5ec\uae30 \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4. ELK \uc2a4\ud0dd\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\ub2e4\uba74 \uc5ec\uae30 \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4. Filebeat \uac00 \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\ub2e4\uba74 \uc5ec\uae30 \ub97c \ucc38\uc870\ud574\uc11c \uc124\uce58\ud569\ub2c8\ub2e4. Kafka \uac00 \uc124\uce58\ub418\uc5b4 \uc788\uc9c0 \uc54a\uc73c\uba74\u2026 Read More »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[6],"tags":[],"class_list":["post-164","post","type-post","status-publish","format-standard","hentry","category-elasticsearch"],"_links":{"self":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/164","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=164"}],"version-history":[{"count":9,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/164\/revisions"}],"predecessor-version":[{"id":347,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=\/wp\/v2\/posts\/164\/revisions\/347"}],"wp:attachment":[{"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=164"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=164"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.skyer9.pe.kr\/wordpress\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=164"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}