flink read 带Kerberos的hbase
package com.hx.test;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.util.Iterator;
public class FlinkReadHbase {
public static void main(String[] args) throws Exception{
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(3,1000));
System.setProperty("java.security.auth.login.config","D:\\workspace\\flink2doris\\src\\main\\resources\\kafka_client_jaas.conf");
System.setProperty("java.security.krb5.conf","D:\\workspace\\flink2doris\\src\\main\\resources\\krb5.ini");
System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
DataStream
result.print();
env.execute();
}
public static class HBaseReader extends RichSourceFunction
private Connection conn = null;
private Table table = null;
private Scan scan = null;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "cdp-kfk01.hx.tc,cdp-kfk02.hx.tc,cdp-kfk03.hx.tc");
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, "2181"); //2181
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase"); //hbase
// 认证需要的参数
// hbase authentication is kerberos ,rpc is privacy
//
conf.set("hadoop.security.authentication", "kerberos");
conf.set("hbase.security.authentication","kerberos");
conf.set("hbase.rpc.protection", "privacy");
// 指定kerberos配置参数
conf.set("keytab.file", "D:/soft/kerbros/hbase.keytab");
conf.set("hbase.master.kerberos.principal", "hbase/[email protected]");
conf.set("hbase.regionserver.kerberos.principal", "hbase/[email protected]");
conf.set("kerberos.principal", "hbase/[email protected]");
// 设置配置文件信息
UserGroupInformation.setConfiguration(conf);
// 通过keytab登录用户
System.out.println("--------------> 开始进行keytab认证 <--------------");
UserGroupInformation.loginUserFromKeytab("hbase/[email protected]", "D:/soft/kerbros/hbase.keytab");
System.out.println("--------------> keytab认证结束 <--------------");
conn = ConnectionFactory.createConnection(conf);
table = conn.getTable(TableName.valueOf("ods_sr_prd:tbl_store"));
scan = new Scan();
scan.addFamily(Bytes.toBytes("cf"));
scan.withStartRow(Bytes.toBytes("1"));
scan.withStopRow(Bytes.toBytes("1"),true);
}
@Override
public void run(SourceContext
ResultScanner rs = table.getScanner(scan);
Iterator
while (iterator.hasNext()) {
Result result = iterator.next();
String rowkey = Bytes.toString(result.getRow());
StringBuffer sb = new StringBuffer();
for (Cell cell : result.listCells()) {
String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
sb.append(value).append(",");
}
String valueString = sb.replace(sb.length() - 1, sb.length(), "").toString();
Tuple2
tuple2.setFields(rowkey, valueString);
ctx.collect(tuple2);
}
}
@Override
public void cancel() {
try {
if (table != null) {
table.close();
}
if (conn != null) {
conn.close();
}
} catch (IOException e) {
System.out.println("Close HBase Exception666:"+ e.toString());
}
}
}
}
-------------------------------------------------pom 文件---------------------------------------------
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">