Java curator操作zookeeper获取kafka
Curator是Netflix公司开源的一个Zookeeper客户端，与Zookeeper提供的原生客户端相比，Curator的抽象层次更高，简化了Zookeeper客户端的开发量。
原文地址：http://blogxinxiucan.sh1.newtouch.com/2017/08/01/Java-curator操作zookeeper获取kafka/
Curator的Maven依赖
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.3.0</version>
</dependency>
本地启动kafka
启动zookeeper
./bin/zookeeper-server-start.sh config/zookeeper.properties
启动kafka
./bin/kafka-server-start.sh config/server.properties
启动kafkaManager
sudo ./bin/kafka-manager
代码
项目结构
先看测试
package com.xinxiucan.test;
import java.util.List;
import com.xinxiucan.api.BrokerApi;
import com.xinxiucan.api.TopicApi;
import com.xinxiucan.zookeeper.ZKBean;
public class AppTest {
public static void main(String[] args) {
ZKBean zk = new ZKBean();
zk.setZkURL("127.0.0.1:2181");
List<String> ids = BrokerApi.findBrokerIds(zk);
for (String id : ids) {
String idInfo = BrokerApi.findBrokerById(zk, id);
System.out.println(idInfo);
}
List<String> topicNames = TopicApi.findAllTopicName(zk);
for (String topicName : topicNames) {
System.out.println("topicName:" + topicName);
}
List<String> topics = TopicApi.findAllTopics(zk);
for (String topic : topics) {
System.out.println("topic:" + topic);
}
}
}
运行结果
{"jmx_port":-1,"timestamp":"1501636761404","endpoints":["PLAINTEXT://can:9092"],"host":"can","version":3,"port":9092}
topicName:test_xin_1
topicName:xin
topic:{"version":1,"partitions":{"0":[0]}}
topic:{"version":1,"partitions":{"0":[0]}}
剩余代码
ZKBean
package com.xinxiucan.zookeeper;
import java.io.Serializable;
public class ZKBean implements Serializable {
private static final long serialVersionUID = -6057956208558425192L;
private int id = -1;
private String name;
private String zkURL;
private String version = "0.8.2.2";
private boolean jmxEnable;
private String jmxAuthUsername;
private String jmxAuthPassword;
private boolean jmxWithSsl;
private int zkConnectionTimeout = 30;
private int maxRetryCount = 3;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getZkURL() {
return zkURL;
}
public void setZkURL(String zkURL) {
this.zkURL = zkURL;
}
public boolean isJmxEnable() {
return jmxEnable;
}
public void setJmxEnable(boolean jmxEnable) {
this.jmxEnable = jmxEnable;
}
public String getJmxAuthUsername() {
return jmxAuthUsername;
}
public void setJmxAuthUsername(String jmxAuthUsername) {
this.jmxAuthUsername = jmxAuthUsername;
}
public String getJmxAuthPassword() {
return jmxAuthPassword;
}
public void setJmxAuthPassword(String jmxAuthPassword) {
this.jmxAuthPassword = jmxAuthPassword;
}
public boolean isJmxWithSsl() {
return jmxWithSsl;
}
public void setJmxWithSsl(boolean jmxWithSsl) {
this.jmxWithSsl = jmxWithSsl;
}
public int getZkConnectionTimeout() {
return zkConnectionTimeout;
}
public void setZkConnectionTimeout(int zkConnectionTimeout) {
this.zkConnectionTimeout = zkConnectionTimeout;
}
public int getMaxRetryCount() {
return maxRetryCount;
}
public void setMaxRetryCount(int maxRetryCount) {
this.maxRetryCount = maxRetryCount;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
@Override
public int hashCode() {
return this.id;
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (this == obj) {
return true;
}
if (obj instanceof ZKBean) {
ZKBean anotherZKCluster = (ZKBean) obj;
return this.id == anotherZKCluster.getId();
} else {
return false;
}
}
}
ZKConnectionFactory
package com.xinxiucan.zookeeper;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
public class ZKConnectionFactory {
public static CuratorFramework buildCuratorFramework(ZKBean zk) {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(zk.getZkConnectionTimeout() * 1000, zk.getMaxRetryCount());
CuratorFramework client = CuratorFrameworkFactory.newClient(zk.getZkURL(), retryPolicy);
client.start();
return client;
}
}
BrokerApi
package com.xinxiucan.api;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import com.xinxiucan.KafkaZKPath;
import com.xinxiucan.zookeeper.ZKBean;
import com.xinxiucan.zookeeper.ZKConnectionFactory;
/**
* 获取broker信息
* @author xinxiucan
*/
public class BrokerApi {
/**
* 根据zk获取Broker ID列表
* @param zkBean
* @return
*/
public static List<String> findBrokerIds(ZKBean zkBean) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
List<String> brokerIdStrs = client.getChildren().forPath(KafkaZKPath.BROKER_IDS_PATH);
return brokerIdStrs;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
/**
* 根据zk&brokerId获取Broker信息
* @param zkBean
* @return
*/
public static String findBrokerById(ZKBean zkBean, String id) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
String brokerIdPath = KafkaZKPath.getBrokerSummeryPath(id);
String brokerInfo = new String(client.getData().forPath(brokerIdPath), "UTF-8");
return brokerInfo;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
public static Map<String, String> findAllBrokerSummarys(ZKBean zkBean) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
List<String> brokerIds = client.getChildren().forPath(KafkaZKPath.BROKER_IDS_PATH);
if (brokerIds == null || brokerIds.size() == 0) {
return null;
}
Map<String, String> brokerMap = new HashMap<String, String>();
for (String brokerId : brokerIds) {
String brokerIdPath = KafkaZKPath.getBrokerSummeryPath(brokerId);
String brokerInfo = new String(client.getData().forPath(brokerIdPath), "UTF-8");
brokerMap.put(brokerId, brokerInfo);
}
return brokerMap;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
}
TopicApi
package com.xinxiucan.api;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import org.apache.zookeeper.data.Stat;
import com.xinxiucan.KafkaZKPath;
import com.xinxiucan.ReplicasAssignmentBuilder;
import com.xinxiucan.util.Object2Array;
import com.xinxiucan.zookeeper.ZKBean;
import com.xinxiucan.zookeeper.ZKConnectionFactory;
/**
* 获取Topic信息
* @author can
*/
public class TopicApi {
/**
* 创建Topic
* @param zkBean
* @param topic
* @param partitionsCount
* @param replicationFactorCount
* @param topicConfig
*/
public static void createTopic(ZKBean zkBean, String topic, int partitionsCount, int replicationFactorCount,
Map<String, String> topicConfig) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
List<String> brokerIds = client.getChildren().forPath(KafkaZKPath.BROKER_IDS_PATH);
List<Integer> ids = new ArrayList<>();
for (String str : brokerIds) {
int i = Integer.parseInt(str);
ids.add(i);
}
Map<String, List<Integer>> replicasAssignment = ReplicasAssignmentBuilder.assignReplicasToBrokers(ids, partitionsCount,
replicationFactorCount);
Map<String, Object> topicSummaryMap = new HashMap<String, Object>();
topicSummaryMap.put("version", 1);
topicSummaryMap.put("partitions", replicasAssignment);
byte[] topicSummaryData = Object2Array.objectToByteArray(topicSummaryMap);
String topicSummaryPath = KafkaZKPath.getTopicSummaryPath(topic);
Stat stat = client.checkExists().forPath(topicSummaryPath);
if (stat == null) {
// create
client.create().creatingParentsIfNeeded().forPath(topicSummaryPath, topicSummaryData);
} else {
// update
client.setData().forPath(topicSummaryPath, topicSummaryData);
}
Map<String, Object> topicConfigMap = new HashMap<String, Object>();
topicConfigMap.put("version", 1);
topicConfigMap.put("config", topicConfig);
byte[] topicConfigData = Object2Array.objectToByteArray(topicConfigMap);
String topicConfigPath = KafkaZKPath.getTopicConfigPath(topic);
Stat stat2 = client.checkExists().forPath(topicConfigPath);
if (stat2 == null) {
// create
client.create().creatingParentsIfNeeded().forPath(topicConfigPath, topicConfigData);
} else {
// update
client.setData().forPath(topicConfigPath, topicConfigData);
}
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
/**
* 删除Topic
* @param zkBean
* @param topicName
*/
public static void deleteTopic(ZKBean zkBean, String topicName) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
String adminDeleteTopicPath = KafkaZKPath.getDeleteTopicPath(topicName);
client.create().creatingParentsIfNeeded().forPath(adminDeleteTopicPath, null);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
/**
* 查询topic
* @param zkBean
* @return
*/
public static List<String> findAllTopics(ZKBean zkBean) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
List<String> topicNames = client.getChildren().forPath(KafkaZKPath.BROKER_TOPICS_PATH);
List<String> topicSummarys = new ArrayList<String>();
for (String topicName : topicNames) {
byte[] topicData = client.getData().forPath(KafkaZKPath.getTopicSummaryPath(topicName));
topicSummarys.add(new String(topicData, "UTF-8"));
}
// add delete flag
List<String> deleteTopics = client.getChildren().forPath(KafkaZKPath.ADMIN_DELETE_TOPICS_PATH);
return topicSummarys;
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
public static List<String> findAllTopicName(ZKBean zkBean) {
CuratorFramework client = null;
try {
client = ZKConnectionFactory.buildCuratorFramework(zkBean);
return client.getChildren().forPath(KafkaZKPath.BROKER_TOPICS_PATH);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (client != null) {
client.close();
}
}
}
}
KafkaZKPath
package com.xinxiucan;
public class KafkaZKPath {
public static final String COMSUMERS_PATH = "/consumers";
public static final String BROKER_IDS_PATH = "/brokers/ids";
public static final String BROKER_TOPICS_PATH = "/brokers/topics";
public static final String CONFIG_TOPICS_PATH = "/config/topics";
public static final String CONFIG_CHANGES_PATH = "/config/changes";
public static final String CONTROLLER_PATH = "/controller";
public static final String CONTROLLER_EPOCH_PATH = "/controller_epoch";
public static final String ADMIN_PATH = "/admin";
public static final String ADMIN_REASSGIN_PARTITIONS_PATH = "/admin/reassign_partitions";
public static final String ADMIN_DELETE_TOPICS_PATH = "/admin/delete_topics";
public static final String ADMIN_PREFERED_REPLICA_ELECTION_PATH = "/admin/preferred_replica_election";
public static String getTopicSummaryPath(String topic) {
return BROKER_TOPICS_PATH + "/" + topic;
}
public static String getTopicPartitionsStatePath(String topic,String partitionId) {
return getTopicSummaryPath(topic) + "/partitions/" + partitionId+ "/state";
}
public static String getTopicConfigPath(String topic) {
return CONFIG_TOPICS_PATH + "/" + topic;
}
public static String getDeleteTopicPath(String topic) {
return ADMIN_DELETE_TOPICS_PATH + "/" + topic;
}
public static String getBrokerSummeryPath(String id) {
return BROKER_IDS_PATH + "/" + id;
}
public static String getConsumerOffsetPath(String groupId) {
return COMSUMERS_PATH + "/" + groupId +"/offsets";
}
public static String getTopicConsumerOffsetPath(String groupId,String topic){
return COMSUMERS_PATH + "/" + groupId +"/offsets/"+topic;
}
public static String getTopicPartitionConsumerOffsetPath(String groupId, String topic,String partitionId){
return COMSUMERS_PATH + "/" + groupId +"/offsets/"+topic + "/" + partitionId;
}
public static String getTopicPartitionConsumerOwnerPath(String groupId, String topic, String partitionId) {
return COMSUMERS_PATH + "/" + groupId +"/owners/"+topic + "/" + partitionId;
}
}
ReplicasAssignmentBuilder
package com.xinxiucan;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class ReplicasAssignmentBuilder {
public static Map<String, List<Integer>> assignReplicasToBrokers(List<Integer> brokerListSet, int nPartitions, int nReplicationFactor) {
return assignReplicasToBrokers(brokerListSet, nPartitions, nReplicationFactor, -1, -1);
}
/**
* There are 2 goals of replica assignment:
* 1. Spread the replicas evenly among brokers.
* 2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
*
* To achieve this goal, we:
* 1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
* 2. Assign the remaining replicas of each partition with an increasing shift.
*
* Here is an example of assigning
* broker-0 broker-1 broker-2 broker-3 broker-4
* p0 p1 p2 p3 p4 (1st replica)
* p5 p6 p7 p8 p9 (1st replica)
* p4 p0 p1 p2 p3 (2nd replica)
* p8 p9 p5 p6 p7 (2nd replica)
* p3 p4 p0 p1 p2 (3nd replica)
* p7 p8 p9 p5 p6 (3nd replica)
*/
public static Map<String, List<Integer>> assignReplicasToBrokers(List<Integer> brokerListSet, int nPartitions, int nReplicationFactor,
int fixedStartIndex, int startPartitionId) {
// sort
brokerListSet.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1 - o2;
}
});
if (nPartitions <= 0) {
throw new RuntimeException("num of partition cannot less than 1");
}
if (nReplicationFactor <= 0) {
throw new RuntimeException("num of replication factor cannot less than 1");
}
if (nReplicationFactor > brokerListSet.size()) {
throw new RuntimeException("broker num is less than replication factor num");
}
Random random = new Random();
Map<String, List<Integer>> result = new HashMap<String, List<Integer>>();
int startIndex = fixedStartIndex >= 0 ? fixedStartIndex : random.nextInt(brokerListSet.size());
int currentPartitionId = startPartitionId >= 0 ? startPartitionId : 0;
int nextReplicaShift = fixedStartIndex >= 0 ? fixedStartIndex : random.nextInt(brokerListSet.size());
for (int i = 0; i < nPartitions; i++) {
if (currentPartitionId > 0 && ((currentPartitionId % brokerListSet.size()) == 0)) {
nextReplicaShift++;
}
int firstReplicaIndex = (currentPartitionId + startIndex) % brokerListSet.size();
List<Integer> replicaList = new ArrayList<Integer>();
replicaList.add(brokerListSet.get(firstReplicaIndex));
for (int j = 0; j < (nReplicationFactor - 1); j++) {
replicaList.add(brokerListSet.get(getReplicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerListSet.size())));
}
result.put("" + currentPartitionId, getReverseList(replicaList));
currentPartitionId++;
}
return result;
}
private static List<Integer> getReverseList(List<Integer> list) {
List<Integer> result = new ArrayList<Integer>();
for (int i = list.size() - 1; i >= 0; i--) {
result.add(list.get(i));
}
return result;
}
private static int getReplicaIndex(int firstReplicaIndex, int nextReplicaShift, int replicaIndex, int nBrokers) {
int shift = 1 + (nextReplicaShift + replicaIndex) % (nBrokers - 1);
int result = (firstReplicaIndex + shift) % nBrokers;
// System.out.println(firstReplicaIndex+"|"+nextReplicaShift+"|"+replicaIndex+"="+result);
return result;
}
private static void printAssignReplicas(Map<String, List<Integer>> map, List<Integer> brokerListSet) {
Map<Integer, List<String>> brokerMap = new HashMap<Integer, List<String>>();
for (int i = 0; i < brokerListSet.size(); i++) {
List<String> partitions = new ArrayList<String>();
for (int j = 0; j < map.size(); j++) {
List<Integer> brokerIds = map.get(j + "");
if (brokerIds.contains(i)) {
partitions.add("" + j);
}
}
brokerMap.put(i, partitions);
}
}
}