kafka启动:
sudo sh kafka-server-start.sh ../conf/server.properties
kafka关闭:
sudo sh kafka-server-stop.sh
kafka查看topic:
sudo sh kafka-topics.sh --zookeeper localhost:2181 --list(相关命令: --describe --create --alter or --delete)
修改分区数量:
sudo sh kafka-topics.sh --zookeeper localhost:2181 --alter --topic ono --partitions 3
查看指定topic信息:
sudo sh kafka-topics.sh --zookeeper localhost:2181 --describe --topic hello
java版的kafka生产者之基本使用
package com.yangpeng.producer; import java.nio.channels.NonReadableChannelException; import java.util.Map; import java.util.Properties; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; public class KafkaProducerDemo { Properties properties = null; KafkaProducer<String ,String> producer = null; Logger logger = LoggerFactory.getLogger(KafkaProducerDemo.class); public KafkaProducerDemo() { super(); Properties kafkaProps = new Properties(); kafkaProps.put("bootstrap.servers", "***.**.**.30:9092"); kafkaProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); kafkaProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); producer = new KafkaProducer<String, String>(kafkaProps); } public Properties getProperties() { return properties; } public void setProperties(Properties properties) { this.properties = properties; } public void sendMessage() { Thread.currentThread().setContextClassLoader(null); ProducerRecord<String, String> record = new ProducerRecord<String, String>("test", "key", "this is value from topic 'test' ,is ok!"); try { producer.send(record); } catch (Exception e) { // TODO: handle exception } } public static void main(String[] args) { new KafkaProducerDemo().sendMessage(); } }
其中有以下几个注意点:如果出现logger没有加载器的情况,可在maven下加
<dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>1.8.0-beta0</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-simple</artifactId> <version>1.8.0-beta0</version> </dependency>
kafka消费者基本使用:
package com.yangpeng.consumer; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.springframework.stereotype.Component; import com.sun.org.apache.xml.internal.security.Init; public class KafkaConsumerDemo { Properties props = new Properties(); public void ininProps() { props.put("bootstrap.servers","119.23.226.30:9092"); props.put("group.id","test"); props.put("enable.auto.commit", "false"); //不自动提交offset props.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer"); props.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer"); } public void ReceiMsg() { KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props); consumer.subscribe(Arrays.asList("test")); final int minBatchSize = 200; List<ConsumerRecord<String, String>> buffer =new ArrayList<ConsumerRecord<String,String>>(); while(true) { ConsumerRecords<String, String> records = consumer.poll(Duration.ofMinutes(1)); for(ConsumerRecord<String, String> record:records) { buffer.add(record); System.out.println(record.toString()); } consumer.commitAsync(); buffer.clear(); } } public static void main(String[] args) { KafkaConsumerDemo demo = new KafkaConsumerDemo(); demo.ininProps(); demo.ReceiMsg(); } }
spring kafka傻瓜式配置:
spring.kafka.bootstrap-servers=192.168.1.180:9092 #设置一个默认组 spring.kafka.consumer.group-id=0 #key-value序列化反序列化 spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer #每次批量发送消息的数量 spring.kafka.producer.batch-size=65536 spring.kafka.producer.buffer-memory=524288
注意:
ConsumerOffsetChecker以及被取消了.....
应该使如下的命令:
bin/kafka-consumer-groups.sh --bootstrap-server 192.168.47.133:9092 --describe --group test-consumer-group,得到如下的数据
查看配置信息:
sudo sh kafka-configs.sh