并行消费kafka存放本地文件

package com.tvm.writefile;

/**
 * Created by zhangx on 2017/1/25.
 */

import com.alibaba.fastjson.JSONObject;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
import kafka.serializer.StringDecoder;
import kafka.utils.VerifiableProperties;
import lombok.extern.log4j.Log4j2;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;


@Log4j2
public class KafkaConsumer implements Runnable{

    static boolean stop = false;

    public static boolean isStop() {
        return stop;
    }
    public static void setStop(boolean stop) {
        KafkaConsumer.stop = stop;
    }

    private static Thread mainThread = Thread.currentThread();
    public static Properties properties() throws IOException {
        Properties properties_kafkainfo=new Properties();

        InputStream in = new FileInputStream("/opt/logs/java_jar/kafkainfo.properties");
        properties_kafkainfo.load(in);
        return properties_kafkainfo;
    }
    public static Properties properties_topic() throws IOException {
        Properties properties_kafkatopic=new Properties();
        InputStream in = new FileInputStream("/opt/logs/java_jar/topic.properties");
        properties_kafkatopic.load(in);
        return properties_kafkatopic;
    }

    public final ConsumerConnector consumer;
    public KafkaConsumer() throws IOException {
        Properties props = new Properties();
        //props.put("zookeeper.connect", "10.20.30.91:2181,10.20.30.92:2181,10.20.30.93:2181");
        props.put("zookeeper.connect", properties().getProperty("zookeeper_connect"));
        props.put("group.id", properties().getProperty("group"));
        props.put("zookeeper.session.timeout.ms", properties().getProperty("session_timeout_ms"));
        props.put("zookeeper.sync.time.ms", properties().getProperty("zookeeper_sync_time_ms"));
        props.put("auto.commit.interval.ms", properties().getProperty("auto_commit_interval_ms"));
        props.put("auto.commit.enable",properties().getProperty("auto_commit_enable"));
        props.put("auto.offset.reset", properties().getProperty("auto_offset_reset")); //largest smallest
        props.put("serializer.class", properties().getProperty("serializer_class"));
        ConsumerConfig config = new ConsumerConfig(props);
        consumer =   kafka.consumer.Consumer.createJavaConsumerConnector(config);
    }
    public void run() {
        final int numThreads = 6;
        final Iterator topic;
        try {
            topic = properties_topic().stringPropertyNames().iterator();
            Map topicCountMap = new HashMap();
            List topicList = new ArrayList();
            while(topic.hasNext()){
                final String key_topic = topic.next();
                topicList.add(key_topic);
                topicCountMap.put(key_topic, new Integer(1));
            }
            StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
            StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
            final Map>> consumerMap =
                    consumer.createMessageStreams(topicCountMap,keyDecoder,valueDecoder);
            for(int i=0;i> stream = consumerMap.get(key_topic1);
                                ExecutorService executor = Executors.newFixedThreadPool(numThreads);
                                for (KafkaStream streams : stream) {
                                    executor.submit(new KafkaConsumerThread(streams, key_topic1));
                                }
                            }
                        }).start();
                }
            }


        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    public static void main(String[] args) throws IOException {
        Thread t = new Thread(new KafkaConsumer());
        t.start();
        Runtime.getRuntime().addShutdownHook(new Thread(){
            @Override
            public void run() {
                KafkaConsumer.setStop(true);
                try {
                    sleep(10000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

package com.tvm.writefile;

import com.alibaba.fastjson.JSONObject;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.message.MessageAndMetadata;
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Created by zhangx on 2017/3/10.
 */
public class KafkaConsumerThread implements Runnable{

    private KafkaStream stream;
    private String topicNum;
    public KafkaConsumerThread(KafkaStream stream, String topicNum) {
        this.stream = stream;
        this.topicNum = topicNum;

    }
    public void run() {
        ConsumerIterator it = stream.iterator();
        while (it.hasNext()){
            if(KafkaConsumer.isStop()){
                flushData();
                break;
            }
            MessageAndMetadata m = it.next();
            JSONObject jsStr = JSONObject.parseObject(m.message());
            String dateTime = new SimpleDateFormat("yyyyMMddHH").format(jsStr.get("createTime"));
            String name = dateTime + ".log";
            String path_send = "/opt/logs/kafkafile/";
            String path_send_topic = path_send + topicNum;
            String file_send = path_send_topic + name;//发放福利路径
            FileWrapper writer = getFileByName(file_send);
            writer.write(jsStr.toString());
        }
    }

    static Map fileMap=new ConcurrentHashMap<>();
    FileWrapper getFileByName(String name){
        FileWrapper bufferedWriter = fileMap.get(name);
        if(bufferedWriter!=null){
            return bufferedWriter;
        }
        synchronized (fileMap){
            bufferedWriter=fileMap.get(name);
            if(bufferedWriter!=null){
                return bufferedWriter;
            }
            File file = new File(name);
            System.out.println("file.name==========="+file.getName());
            FileWrapper result = new FileWrapper(name);
            if(result==null){
                new Throwable("result为空！");
            }
            fileMap.put(name,result);
            return result;
        }

    }
    private static class FileWrapper {
        private final String name;
        private AtomicInteger calc = new AtomicInteger();
        private final BufferedWriter br;
        private long starttime=new Date().getTime();
        private FileOutputStream fos;
        private OutputStreamWriter osw;

        public FileWrapper(String name) {
            this.name=name;
            try {
                fos=new FileOutputStream(name,true);
                osw=new OutputStreamWriter(fos,"utf-8");
                this.br = new BufferedWriter(osw);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
                throw new RuntimeException("文件不可用"+name,e);
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
                throw new RuntimeException("文件不可用"+name,e);
            }
        }

        public void write (String txt){
            try {
                //int i = calc.incrementAndGet();
                String txt2=txt+"\n";
                br.write(txt2);
                //if(i%10000==0){
                    br.flush();
                    //calc.set(0);
                //}
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        public void close(){
            try {
                br.close();
            } catch (IOException e) {
            }
            try {
                osw.close();
            } catch (IOException e) {
            }
            try {
                fos.close();
                fos=null;
            } catch (IOException e) {
            }
        }
    }


    static void flushData(){
        long nowtime = new Date().getTime();
        synchronized (fileMap){
            for (FileWrapper fileWrapper : fileMap.values()) {
                //fileWrapper.br.flush();
                if(nowtime-fileWrapper.starttime>2*60*60*1000){
                    fileMap.remove(fileWrapper.name);
                    fileWrapper.close();
                }
            }
        }
    }
    static {
        Timer timer = new Timer("flush-task", false);
        timer.schedule(new TimerTask() {
            @Override
            public void run() {
                flushData();
            }
        },60000,60000);
    }

}

package com.tvm.util;

import java.text.SimpleDateFormat;

/**
 * Created by zhangx on 2017/3/10.
 */
public class DateUtil {

    //时间转换为yyyyMMddHH格式
    public static String getTFormatString(Object date1) {
        SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHH");
        String str = format.format(date1);
        return str;
    }

}

group 'com.kafka'
version '1.0-SNAPSHOT'

apply plugin: 'java'
apply plugin: 'idea'

[compileJava, compileTestJava, javadoc]*.options*.encoding = 'UTF-8'
sourceCompatibility = 1.7
targetCompatibility=1.7

repositories {
    mavenLocal()
    mavenCentral()
    jcenter()
    maven {
        url = 'http://115.159.154.56:8081/nexus/content/groups/public'
    }
}

dependencies {
    testCompile group: 'junit', name: 'junit', version: '4.11'
    compile "org.apache.kafka:kafka_2.10:0.8.2.2"
    compile group: 'com.alibaba', name: 'fastjson', version: '1.2.21'
    compile "org.projectlombok:lombok:1.16.6"
    compile "org.apache.logging.log4j:log4j-api:2.3"
    compile "org.apache.logging.log4j:log4j-core:2.3"
    fileTree(dir: 'lib', include: '*.jar')
}
jar {
    String someString = ''
    configurations.runtime.each {someString = someString + " lib//"+it.name}//遍历项目的所有依赖的jar包赋值给变量someString
    manifest {
        attributes 'Main-Class': 'com.tvm.writefile.KafkaConsumer'
        attributes 'Class-Path': someString
    }
}

//清除上次的编译过的文件
task clearPj(type:Delete){
    delete 'build','target'
}
task copyJar(type:Copy){
    from configurations.runtime
    into ('build/libs/lib')
}
//把JAR复制到目标目录
task release(type: Copy,dependsOn: [build,copyJar]) {
// from 'conf'
// into ('build/libs/eachend/conf') // 目标位置
}

通过

gradle release

命令来进行打包

在linux系统中

nohup java -jar KafkaConsumer_log-1.0-SNAPSHOT.jar > /dev/null 2>&1 &

后台运行