Linux搭建Kafka+Spark实时处理系统

服务器要求：jdk-8u121-linux-x64.tar.gz、kafka_2.12-0.10.2.1.tgz、spark-1.3.1-bin-hadoop2-without-hive.tgz

1、生产者—SpringMVC+Kafka

1.1、准备工作

所需资源：kafka_2.10-0.8.2.2.jar、kafka-clients-0.10.0.0.jar，把这两个jar包导入到项目中

1.2、配置

关于生产者Kafka服务器的配置，如下：

bootstrap.servers=172.17.0.2:9092
acks=all
retries=3
batch.size=16384
linger.ms=1
buffer.memory=33554432
key.serializer=org.apache.kafka.common.serialization.StringSerializer
value.serializer=org.apache.kafka.common.serialization.StringSerializer

1.3、编写代码

1.3.1、线程池工厂

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;

public class ExecutorServiceFactory {
	
	private ExecutorService executors;
	
	private ExecutorServiceFactory() {
		
	}
	
	public static final ExecutorServiceFactory getInstance() {
        return KafkaProducerPoolHolder.instance;
    }
	
	private static class KafkaProducerPoolHolder {
		private static final ExecutorServiceFactory instance = new ExecutorServiceFactory();
	}
	
	public ExecutorService createScheduledThreadPool() {
		// CPU个数
        int availableProcessors = Runtime.getRuntime().availableProcessors();
        // 创建
        executors = Executors.newScheduledThreadPool(availableProcessors * 10, new KafkaProducterThreadFactory());
        return executors;
	}
	
	public ExecutorService createSingleThreadExecutor() {
        // 创建
        executors = Executors.newSingleThreadExecutor(new KafkaProducterThreadFactory());
        return executors;
    }
	
	public ExecutorService createCachedThreadPool() {
        // 创建
        executors = Executors.newCachedThreadPool(new KafkaProducterThreadFactory());
        return executors;
    }
	
	public ExecutorService createFixedThreadPool(int count) {
        // 创建
        executors = Executors.newFixedThreadPool(count, new KafkaProducterThreadFactory());
        return executors;
    }
	
	private class KafkaProducterThreadFactory implements ThreadFactory {
		@Override
		public Thread newThread(Runnable runnable) {
			SecurityManager s = System.getSecurityManager();
			ThreadGroup group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
	        Thread t = new Thread(group, runnable);
	        return t;
		}
	}
}

1.3.2 、生产者线程池

import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

public class KafkaProducerPool {
	
	private ExecutorService executor;
	
	private KafkaProducerPool() {
		executor = ExecutorServiceFactory.getInstance().createFixedThreadPool(2);
	}
	
	public static final KafkaProducerPool getInstance() {
		return KafkaProducerPoolHolder.instance;
	}
	
	private static class KafkaProducerPoolHolder {
		private static final KafkaProducerPool instance = new KafkaProducerPool();
	}
	
	/**
     * 关闭线程池，这里要说明的是：调用关闭线程池方法后，线程池会执行完队列中的所有任务才退出
     * 
     * @author SHANHY
     * @date   2015年12月4日
     */
    public void shutdown(){
        executor.shutdown();
    }

    /**
     * 提交任务到线程池，可以接收线程返回值
     * 
     * @param task
     * @return
     * @author SHANHY
     * @date   2015年12月4日
     */
    public Future submit(Runnable task) {
        return executor.submit(task);
    }

    /**
     * 提交任务到线程池，可以接收线程返回值
     * 
     * @param task
     * @return
     * @author SHANHY
     * @date   2015年12月4日
     */
    public Future submit(Callable task) {
        return executor.submit(task);
    }

    /**
     * 直接提交任务到线程池，无返回值
     * 
     * @param task
     * @author SHANHY
     * @date   2015年12月4日
     */
    public void execute(Runnable task){
        executor.execute(task);
    }
}

1.3.3、生产者

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import com.cda91.common.HttpJResopnse;

public class KafkaProducerT implements Runnable {
	
	private static Producer producer;  
	private String topic = "kafka_producer_title";
	private static Map> messages;
	private static List keys;
	private String key = null;
	private Object value = null;
	private static Properties props;
	private static final String productPro = "/kafka.properties";
	private KafkaProducterListener listener;
	
	private KafkaProducerT() {
		System.out.println("KafkaProducerT构造函数被调用");
		props = new Properties();
		messages = new HashMap>();
		keys = new ArrayList();
		try {
			InputStream is = KafkaProducerT.class.getResourceAsStream(productPro);
			props.load(is);
			is.close();
			is = null;
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		producer = new KafkaProducer(props);
	}
	
	public static final KafkaProducerT getInstance() {
		return KafkaProducerTHolder.instance;
	}
	
	private static class KafkaProducerTHolder{
        private static final KafkaProducerT instance = new KafkaProducerT();
    }
	
	public void setTopic(String topic) {
		this.topic = topic;
	}

	public void push(String key,Object value){
		if(!keys.contains(key)) {
			keys.add(key);
		}
		if(!messages.containsKey(key)) {
			messages.put(key, new ArrayList