# | hostname | ip | software | notes |
---|---|---|---|---|
1 | apollo.dt.com | 192.168.56.181 | zookeeper | Kafka: broker.id=181 |
2 | artemis.dt.com | 192.168.56.182 | zookeeper | kafka: borker.id=182 |
3 | uranus.dt.com | 192.168.56.183 | zookeeper | kafka: broker.id=183 |
4 | pandora.dt.com | 192.168.56.184 | zookeeper | kafka: broker.id=184 |
有关zookeeper详细集群搭建请参考:CentOS安装配置Zookeeper集群
# | hostname | ip | software | notes |
---|---|---|---|---|
1 | apollo.dt.com | 192.168.56.181 | kafka | Kafka: broker.id=181 |
2 | artemis.dt.com | 192.168.56.182 | kafka | kafka: borker.id=182 |
3 | uranus.dt.com | 192.168.56.183 | kafka | kafka: broker.id=183 |
4 | pandora.dt.com | 192.168.56.184 | kafka | kafka: broker.id=184 |
有关Kafka详细集群搭建请参考:CentOS7.0安装配置Kafka集群](http://blog.csdn.net/jssg_tzw/article/details/73106299)
[root@apollo ~]# zkServer.sh start
[root@artemis ~]# zkServer.sh start
[root@uranus ~]# zkServer.sh start
[root@pandora ~]# zkServer.sh start
[root@apollo ~]# kafka-server-start.sh /opt/kafka/config/server.properties
[root@artemis ~]# kafka-server-start.sh /opt/kafka/config/server.properties
[root@uranus ~]# kafka-server-start.sh /opt/kafka/config/server.properties
[root@pandora ~]# kafka-server-start.sh /opt/kafka/config/server.properties
# 定义agent
a1.sources=r1
a1.channels=c1
a1.sinks=k1
# 定义source
a1.sources.r1.type=avro
a1.sources.r1.bind=localhost
a1.sources.r1.port=44446
# 定义channel
a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=1000
a1.channels.c1.keep-alive=30
# 定义sink (Kafka)
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = dt-receipts
a1.sinks.k1.kafka.bootstrap.servers = 192.168.56.181:9092,192.168.56.182:9092,192.168.56.183:9092,192.168.56.184:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 1
a1.sinks.k1.kafka.producer.compression.type = snappy
# 绑定source, sink到channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
[itlocals-MacBook-Pro:flume david.tian]$ bin/flume-ng agent -n a1 -c conf/ --conf-file conf/flume2kafka.conf -Dflume.root.logger=DEBUG,console
源码请从我的git上下载:https://github.com/david-louis-tian/dBD
"http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
com.dvtn.www
dBD
1.0-SNAPSHOT
jar
dBD
http://maven.apache.org
UTF-8
1.7.25
1.2.17
junit
junit
3.8.1
test
org.slf4j
slf4j-api
${slf4j.version}
org.slf4j
slf4j-log4j12
${slf4j.version}
log4j
log4j
${log4j.version}
org.json
json
20170516
org.apache.avro
avro
1.8.2
org.apache.flume
flume-ng-core
1.7.0
org.apache.flume.flume-ng-clients
flume-ng-log4jappender
1.7.0
org.apache.avro
avro-ipc
1.8.2
################### set log levels ###############
log4j.rootLogger = INFO,stdout,file,flume
################### flume ########################
log4j.appender.flume = org.apache.flume.clients.log4jappender.Log4jAppender
log4j.appender.flume.layout = org.apache.log4j.PatternLayout
log4j.appender.flume.Hostname = localhost
log4j.appender.flume.Port = 44446
################## stdout #######################
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Threshold = INFO
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n
################## file ##########################
log4j.appender.file = org.apache.log4j.DailyRollingFileAppender
log4j.appender.file.Threshold = INFO
log4j.appender.file.File = /Users/david.tian/logs/tracker/tracker.log
log4j.appender.file.Append = true
log4j.appender.file.DatePattern = '.'yyyy-MM-dd
log4j.appender.file.layout = org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n
package com.dvtn.www.log4j.jsonlog;
import com.dvtn.www.log4j.logfile.LogProducer;
import com.dvtn.www.model.Area;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.*;
/**
* Created by david.tian on 08/09/2017.
*/
public class SendReceipts {
private static Logger LOG = LoggerFactory.getLogger(LogProducer.class);
private static String path = SendReceipts.class.getResource("/").getPath();
private static String areaJsonString;
private static String city;
private static String cityKey;
private static String province;
private static String provinceKey;
private static int separator;
private static String phonePrefix;
//private static final Random rnd = new Random();
private static String[] payers = {"Merchants", "Individuals"};
private static String[] managers = {"david", "josen", "fab", "simon", "banana", "tom", "scott", "ekrn", "sunshine", "lily", "kudu", "scala", "spark", "flume", "storm", "kafka", "avro", "linux"};
private static String[] terminalTypes = {"RNM", "CNM", "RNM", "GNM", "CNJ", "GNJ", "RNJ", "GNM", "CNM"};
private static String[] stores = {"连锁店", "分营店", "工厂店", "会员店", "直销店"};
private static String[] items = {"面包","酒","油","牛奶","蔬菜","猪肉","牛肉","羊肉","曲奇","手机","耳机","面粉","大米","糖","苹果","茶叶","书","植物","玩具","床","锅","牙膏","洗衣粉","酱油","金鱼","干货"};
private static String[] itemsType ={"食物","酒水","饮料","日用品","电子","数码","娱乐","家俱"};
public static void main(String[] args) {
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
Random rnd = new Random();
ProduceReceipts pr = new ProduceReceipts();
areaJsonString = pr.readJSON(path, "area.json");
String transactionID = System.currentTimeMillis() + ""+Math.round(Math.random() * 9000 + 1000);
String transactionDate = System.currentTimeMillis() + "";
String taxNumber = Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000);
String invoiceId = System.currentTimeMillis() + "";
String invoiceNumber = Math.round(Math.random() * 900000000 + 100000000) + "";
String invoiceDate = System.currentTimeMillis() + "";
List listArea = pr.listArea(areaJsonString);
int idx = rnd.nextInt(listArea.size());
String provinceID = listArea.get(idx).getProvinceID();
String provinceName = listArea.get(idx).getProvinceName();
String cityID = listArea.get(idx).getCityID();
String cityName = listArea.get(idx).getCityName();
String telephone = provinceID + "-" + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000);
int managerSize = managers.length;
String manger = managers[rnd.nextInt(managerSize)];
int payerSize = payers.length;
String payer = payers[rnd.nextInt(payerSize)];
String operator = "OP" + Math.round(Math.random() * 90000 + 10000);
int terminalTypeSize = terminalTypes.length;
String terminalNumber = terminalTypes[rnd.nextInt(terminalTypeSize)] + Math.round(Math.random() * 90000 + 10000);
String account = pr.StringReplaceWithStar(Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000));
String tcNumber = Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + " " + Math.round(Math.random() * 9000 + 1000) + "";
File file = new File(path + "receipts.avsc");
String line = null;
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(file));
while ((line = reader.readLine()) != null) {
// System.out.println("========>" + line);
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
}
}
}
try {
//获得整个Schema
Schema schema = new Schema.Parser().parse(new File(path + "receipts.avsc"));
GenericRecord record = new GenericData.Record(schema);
//获取schema中的字段
int storesSize = stores.length;
//获取店面的Schema
Schema.Field storeField = schema.getField("store");
Schema storeSchema = storeField.schema();
GenericRecord storeRecord = new GenericData.Record(storeSchema);
String storeNumber = Math.round(Math.random() * 9000 + 1000) + "";
String address = provinceName + cityName;
String storeName = provinceName + cityName + stores[rnd.nextInt(storesSize)];
storeRecord.put("store_number",storeNumber);
storeRecord.put("store_name",storeName);
storeRecord.put("address",address);
int itemsSize = items.length;
int itemsTypeSize = itemsType.length;
List productRecordList = new ArrayList();
//获取product的schema
Schema.Field productField = schema.getField("products");
Schema productSchema = productField.schema();
for (int i=0; i< 10; i++){
String itemName = items[rnd.nextInt(1000)%itemsSize];
String itemType = itemsType[rnd.nextInt(1000)%itemsTypeSize];
String quantity = String.valueOf(rnd.nextInt(100));
String price = String.valueOf(rnd.nextFloat()*100);
String discount = String.valueOf(rnd.nextFloat());
GenericRecord productRecord = new GenericData.Record(productSchema);
productRecord.put("item",itemName);
productRecord.put("item_type",itemType);
productRecord.put("quantity",quantity);
productRecord.put("price",price);
productRecord.put("discount",discount);
productRecordList.add(productRecord);
}
record.put("transaction_id",transactionID);
record.put("transaction_date",transactionDate);
record.put("invoice_id",invoiceId);
record.put("invoice_number",invoiceNumber);
record.put("telephone",telephone);
record.put("payer",payer);
record.put("store",storeRecord);
record.put("operator",operator);
record.put("terminal_number",terminalNumber);
record.put("products",productRecordList);
record.put("account",account);
record.put("tc_number",terminalNumber);
LOG.info(record.toString());
} catch (IOException e) {
e.printStackTrace();
}
}
}, 0, 1000);
}
}
在kafka机器上执行命令kafka-console-consumer.sh读取topic “dt-receipts”中看是否日志已被kafka收集:
[root@apollo ~]# kafka-console-consumer.sh --bootstrap-server 192.168.56.181:9092,192.168.56.182:9092,192.168.56.183:9092,192.168.56.184:9092 --from-beginning --topic dt-receipts
我们可以看到,数据已经收集到kafka里的dt-receipts的topic里面: