https://www.jianshu.com/p/0edcc3addf3f
https://blog.csdn.net/u010622525/article/details/100066743
kafka->logstash->elasticsearch->kibana
一.镜像拉取
docker pull wurstmeister/zookeeper
docker pull wurstmeister/kafka
二.定义docker-compose.yml
version: '3' services: zookeeper: image: wurstmeister/zookeeper ports: - "2181:2181" kafka: image: wurstmeister/kafka depends_on: [ zookeeper ] ports: - "9092:9092" environment: KAFKA_ADVERTISED_HOST_NAME: 192.168.220.150 KAFKA_CREATE_TOPICS: "test:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - /data/product/zj_bigdata/data/kafka/docker.sock:/var/run/docker.sock
在docker-compose.yml文件目录进行服务打包
[root@VM_0_16_centos kafka] # docker-compose build
zookeeper uses an image, skipping
kafka uses an image, skipping
三.启动服务
[root@VM_0_16_centos kafka]# docker-compose up -d
Starting kafka_kafka_1 ... done
Starting kafka_zookeeper_1 ... done
四.启动测试
记住启动的启动名称,kafka为 kafka_kafka_1 ,zookeeper 为 kafka_zookeeper_1 .
如果docker-compose正常启动,此时docker ps会看到以上两个容器。进入kafka容器
docker exec -it kafka_kafka_1 bash
创建一个topic
$KAFKA_HOME/bin/kafka-topics.sh --create --topic topic --partitions 4 --zookeeper kafka_zookeeper_1:2181 --replication-factor 1
注意–zookeeper后面的参数为,容器的name
查看刚刚创建的topic
$KAFKA_HOME/bin/kafka-topics.sh --zookeeper kafka_zookeeper_1:2181 --describe --topic test
发布信息
bash-4.4# $KAFKA_HOME/bin/kafka-console-producer.sh --topic=test --broker-list kafka_kafka_1:9092
>ni
>haha
同样注意--broker-list后面的参数
接收消息
bash-4.4# $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server kafka_kafka_1:9092 --from-beginning --topic test
ni
haha
五.kafka集群管理界面
docker run -itd --name=kafka-manager -p 9000:9000 -e ZK_HOSTS="192.168.220.150:2181" sheepkiller/kafka-manager
访问
http://192.168.220.150:9000/
bash-4.4# $KAFKA_HOME/bin/kafka-topics.sh --create --topic tttt --partitions 4 --zookeeper sde_zookeeper_1:2181 --replication-factor 1 Created topic tttt. bash-4.4# bin/kafka-topics.sh --list^C bash-4.4# $KAFKA_HOME/bin/kafka-console-producer.sh --topic=tttt --broker-list sde_kafka_1:9092 >^Cbash-4.$KAFKA_HOME/bin/kafka-topics.sh --create --topic topic --partitions 4 --zookeeper sde_zookeeper_1:2181 --replication-factor 1 ^C bash-4.4# $KAFKA_HOME/bin/kafka-console-consumer.sh --bootstrap-server sde_kafka_1:9092 --from-beginning --topic=tttt
sde@sde:~$ pwd /home/sde sde@sde:~$ cat config/ elasticsearch.yml kibana.yml sde@sde:~$ cat config/elasticsearch.yml cluster.name: "docker-cluster" network.host: 0.0.0.0 xpack: ml.enabled: false monitoring.enabled: false security.enabled: false watcher.enabled: false sde@sde:~$ cat config/kibana.yml server.name: kibana server.host: "0" elasticsearch.url: "http://10.18.1.2:9200" xpack.monitoring.ui.container.elasticsearch.enabled: true i18n.locale: "zh-CN" sde@sde:~$
docker run -itd --name=kafka-manager -p 9000:9000 -e ZK_HOSTS="10.18.1.2:2181" sheepkiller/kafka-manager
sudo sysctl -w vm.max_map_count=262144
docker run --rm -d -it -p 9200:9200 -p 9300:9300 -v "$PWD"/config/elasticsearch.yml:/home/jeon/elasticsearch/config/elasticsearch.yml --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:6.8.2
docker run --rm -d -it -p 5601:5601 -v "$PWD"/config/kibana.yml:/usr/share/kibana/config/kibana.yml --name kibnaa docker.elastic.co/kibana/kibana:6.5.4
docker run --rm -d -it -v ~/logstash/pipeline:/usr/share/logstash/pipeline -v ~/logstash/conf/logstash.yml:/usr/share/logstash/config/logstash.yml -v /home/sde/logstash/plugin:/plugin -p 5044:5044 -p 9600:9600
--privileged=true --name logstash docker.elastic.co/logstash/logstash:7.10.0 -f /usr/share/logstash/pipeline/logstash.conf
rlogger-->kafka-->elk
http://10.18.1.2:9000/clusters/cluster1/topics
http://10.18.1.2:9200/
http://10.18.1.2:5601/
sde@sde:~/kafka$ tree . ├── docker-compose.yml ├── elasticsearch │ └── elasticsearch.yml ├── kibana │ └── kibana.yml └── logstash ├── conf │ └── logstash.yml ├── pipeline │ ├── logstash.conf │ └── logstash.conf.bak └── plugin 6 directories, 6 files
sde@sde:~/kafka$ cat docker-compose.yml version: '3' services: zookeeper: image: wurstmeister/zookeeper ports: - "2181:2181" restart: always kafka: image: wurstmeister/kafka depends_on: [ zookeeper ] ports: - "9092:9092" restart: always environment: KAFKA_ADVERTISED_HOST_NAME: 10.18.1.2 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 kafka_manager: image: sheepkiller/kafka-manager depends_on: [ zookeeper ] ports: - "9000:9000" restart: always environment: ZK_HOSTS: zookeeper:2181 logstash: image: docker.elastic.co/logstash/logstash:7.10.0 depends_on: - elasticsearch - kafka ports: - "5044:5044" - "9600:9600" volumes: - ./logstash/pipeline:/usr/share/logstash/pipeline - ./logstash/conf/logstash.yml:/usr/share/logstash/config/logstash.yml - ./logstash/plugin:/plugin privileged: true restart: always # command: logstash -f /usr/share/logstash/pipeline/logstash.conf elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.8.2 ports: - "9200:9200" - "9300:9300" restart: always volumes: - ./elasticsearch/elasticsearch.yml:/home/jeon/elasticsearch/config/elasticsearch.yml kibana: image: docker.elastic.co/kibana/kibana:6.5.4 ports: - "5601:5601" restart: always volumes: - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
sde@sde:~/kafka$ ls docker-compose.yml elasticsearch kibana logstash sde@sde:~/kafka$ cat elasticsearch/elasticsearch.yml cluster.name: "docker-cluster" network.host: 0.0.0.0 xpack: ml.enabled: false monitoring.enabled: false security.enabled: false watcher.enabled: false sde@sde:~/kafka$ cat kibana/kibana.yml server.name: kibana server.host: "0" elasticsearch.url: "http://10.18.1.2:9200" xpack.monitoring.ui.container.elasticsearch.enabled: true i18n.locale: "zh-CN" sde@sde:~/kafka$ cat logstash/conf/logstash.yml http.host: "0.0.0.0" #path.config: /home/sde/logstash/pipeline xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: http://10.18.1.2:9200 sde@sde:~/kafka$ cat logstash/pipeline/logstash.conf input { kafka { bootstrap_servers => '10.18.1.2:9092' #topics => ["topic_amf", "topic_smf"] topics_pattern => "topic_.*" codec => json {} } } output { elasticsearch { hosts => ["http://10.18.1.2:9200"] index => "topic_2" sniffing => false } stdout { codec => rubydebug } }