mysql数据同步至elasticsearch简单练习(binlog+canal+kafka+docker+elasticsearch)
mysql同步es学习记录。操作系统是window。kafka+zookeeper、elasticsearch+kibana都是直接docker拉取使用
1. 配置mysql
windows是my.ini，linux是my.cnf。
[mysqld]
log-bin=mysql-bin # 开启 binlog
binlog-format=ROW # 选择 ROW 模式
server_id=1 # 配置 MySQL replaction 需要定义，不要和 canal 的 slaveId 重复
canal是伪装成slave进行binlog读取同步的，所以还需要配置slave账户
CREATE USER canal IDENTIFIED BY 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
-- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ;
FLUSH PRIVILEGES;
2. 下载配置canal
canal的github地址为：https://github.com/alibaba/canal，这里用的1.1.4
版本：https://github.com/alibaba/canal/releases/download/canal-1.1.4/canal.deployer-1.1.4.tar.gz。
解压后修改 canal.deployer-1.1.4\conf\example\instance.properties
#数据库地址
canal.instance.master.address=127.0.0.1:3306
#数据库账户
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
#监听数据配置，规则如下
canal.instance.filter.regex=test.test
canal.instance.filter.regex的配置方式：
-. 所有表：.* or .\…
-. canal schema下所有表： canal\…*
-. canal下的以canal打头的表：canal\.canal.*
-. canal schema下的一张表：canal\.test1
-. 多个规则组合使用：canal\…*,mysql.test1,mysql.test2 (逗号分隔)
修改canal.deployer-1.1.4\conf\canal.properties
#修改MQ为kafka
canal.serverMode = kafka
#配置kafka地址
canal.mq.servers = 127.0.0.1:9092
启动 canal.deployer-1.1.4\bin\startup.bat
3. docker运行kafka和es
这里用的镜像是spotify/kafka和nshou/elasticsearch-kibana
#启动nshou/elasticsearch-kibana
docker run -d -p 9200:9200 -p 9300:9300 -p 5601:5601 --name eskibana nshou/elasticsearch-kibana
#启动spotify/kafka
docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=127.0.0.1 --env ADVERTISED_PORT=9092 spotify/kafka
4. 测试
com.alibaba.otter
canal.client
1.1.4
org.apache.kafka
kafka-clients
0.10.1.0
org.elasticsearch.client
elasticsearch-rest-high-level-client
7.3.1
import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;
import com.alibaba.otter.canal.protocol.CanalEntry.*;
import com.thinkgem.jeesite.API.weixin.util.JsonUtil;
import org.apache.http.HttpHost;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.protocol.Message;
/**
* Kafka client example
*
* @author machengyuan @ 2018-6-12
* @version 1.0.0
*/
public class test {
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClient.class);
private KafkaCanalConnector connector;
static volatile boolean running = false;
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
public test(String zkServers, String servers, String topic, Integer partition, String groupId) {
connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
}
public static RestHighLevelClient client;
public static void setClient(RestHighLevelClient client) {
CanalKafkaClient.client = client;
}
public static void main(String[] args) {
client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("localhost", 9200, "http")));
try {
final CanalKafkaClient kafkaCanalClientExample = new CanalKafkaClient("192.168.1.3:2181",
"192.168.1.3:9092",
"example",
null,
"3");
logger.info("## start the kafka consumer: {}-{}", "example", "1");
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
}
});
while (running)
;
} catch (Throwable e) {
logger.error("## Something goes wrong when starting up the kafka consumer:", e);
System.exit(0);
}
}
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
}
public void stop() {
if (!running) {
return;
}
running = false;
if (thread != null) {
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
}
}
private void process() {
while (!running) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
while (running) {
try {
connector.connect();
connector.subscribe();
while (running) {
try {
List messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS); // 获取message
if (messages == null) {
continue;
}
for (Message message : messages) {
long batchId = message.getId();
int size = message.getEntries().size();
if (batchId == -1 || size == 0) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
} else {
//取到数据开始处理
sendToElastic(message.getEntries());
logger.info(message.toString());
}
}
connector.ack(); // 提交确认
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
connector.unsubscribe();
connector.disconnect();
}
private static void sendToElastic(List entrys) throws IOException {
for (Entry entry : entrys) {
if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
continue;
}
//判断表
if (entry.getHeader().getTableName().equals("test")) {
RowChange rowChage = null;
try {
rowChage = RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
EventType eventType = rowChage.getEventType();
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(),
entry.getHeader().getSchemaName(), entry.getHeader().getTableName(),
eventType));
for (RowData rowData : rowChage.getRowDatasList()) {
//删除操作
if (eventType == EventType.DELETE) {
final String[] id = {"0"};
rowData.getBeforeColumnsList().forEach((Column o) -> {
if (o.getName().equals("id")) {
id[0] = o.getValue();
}
});
DeleteRequest deleteRequest =
new DeleteRequest("test", id[0]);
client.delete(deleteRequest, RequestOptions.DEFAULT);
//插入操作
} else if (eventType == EventType.INSERT) {
EsIndex esIndex = rowDataToEsIndex(rowData);
IndexRequest indexRequest = new IndexRequest("test")
.id(esIndex.getMyId())
.source(JsonUtil.toJSONString(esIndex), XContentType.JSON)
.opType(DocWriteRequest.OpType.CREATE);
client.index(indexRequest, RequestOptions.DEFAULT);
//更新操作
} else {
//rowData转换成对象
EsIndex esIndex = rowDataToEsIndex(rowData);
UpdateRequest request = new UpdateRequest(
"test",
esIndex.getMyId()).doc(JsonUtil.toJSONString(esIndex), XContentType.JSON);
client.update(request, RequestOptions.DEFAULT);
}
}
}
}
}
private static EsIndex rowDataToEsIndex(RowData rowData) throws IOException {
//es索引的文档对象，只需要myId和name属性即可。
EsIndex esIndex = new EsIndex();
for (Column column : rowData.getAfterColumnsList()) {
if (column.getName().equals("id")) {
esIndex.setMyId(column.getValue());
}
if (column.getName().equals("name")) {
esIndex.setName(column.getValue());
}
}
return esIndex;
}
}
可以用kibana或者google的elasticsearch Head插件查看数据变化。
博客地址:那个人好像一条狗啊