kafka三种分区策略及代码示例

分区partition

1、HashPartitioner

1.1代码

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
 
import java.util.Map;
 
/**
*要想在生产数据的时候指定partitioner
*
*自定义的Partitioner
*/
public class HashPartitioner implements Partitioner{
 
@Override
public void close(){
}
 
@Override
public void configure(Map<String,?>configs){
}
/**
*hash分区key.hash()%partitionNum
*@param topic这条消息要被发送给哪个topic
*@param key这条消息中的key
*@param keyBytes这条消息的key的字节序列对象
*@param value这条消息中的value
*@param valueBytes这条消息的value的字节序列对象
*@param clusterkafkacluster的上下文对象，通过该对象获取集群的相关信息
*@return 经过分区算法之后返回当前消息应该被投递的分区
*/
@Override
public int partition(String topic,Object key,byte[] keyBytes,Object value,byte[] valueBytes,Cluster cluster){
    if(key==null){
        return0;
    }
int hash=Math.abs(key.hashCode());      //规避hash值为负数的情况
System.out.println("key:"+key+",hash:"+hash);
int partitionNum=cluster.partitionsForTopic(topic).size();
return hash%partitionNum;
}
}

1.2配置文件

在producer中添加：
#nameofthepartitionerclassforpartitioningevents;defaultpartitionspreadsdatarandomly
partitioner.class=com.moshang.com.HashPartitioner

2、轮询

2.1代码

/** * 轮询 * 1-->0 * 2-->1 * 3-->2 * 4-->0 * 5-->1 * 6-->2 */
public class RoundRobinPartitioner implements Partitioner{    
@Override    
public void close() {     
}    
@Override    
public void configure(Map<String, ?> configs) {     
} //    private int count = 0;    
private AtomicInteger count = new AtomicInteger();    
@Override    
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {        
int partition = count.getAndIncrement() % cluster.partitionsForTopic(topic).size();        System.out.println("key: " + key + ", partition: " + partition);        
return partition;    
}
}

2.2配置文件

在producer中添加：
#name of the partitionerclassforpartitioningevents;default partitionspreadsdatarandomly
partitioner.class=com.moshang.com.RoundRobinPartitioner

3、随机

1.1代码

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster; 
import java.util.Map;
import java.util.Random; 

public class RandomPartitioner implements Partitioner{ 
@Override
public void close(){} 
@Override
public void configure(Map<String,?>configs){} //privateintcount=0;
private Random random=new Random();
@Override
public int partition(Stringtopic,Objectkey,byte[]keyBytes,Objectvalue,byte[]valueBytes,Clustercluster){
intpartition=random.nextInt(cluster.partitionsForTopic(topic).size());
System.out.println("key:"+key+",partition:"+partition);
return partition;
}
}

1.2配置文件

在producer中添加：
#nameofthepartitionerclassforpartitioningevents;defaultpartitionspreadsdatarandomly
partitioner.class=com.moshang.com.RandomPartitioner