Hbase命令行+JAVA API+hivehbase整合
笔记
1.Utils
public class HbaseUtils {
public static String value = "qianfeng01,qianfeng02,qianfeng03";
public static String key = "hbase.zookeeper.quorum";
public static Configuration configuration = new Configuration();
static {
configuration.set(key,value);
}
public static Admin getAdmin(){
try {
Admin admin = ConnectionFactory.createConnection(configuration).getAdmin();
return admin;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public static void close(Admin admin){
if (admin != null){
try {
admin.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
2.namespace
1.hbase shell【命令行】
【增加】
create_namespace 'ns1'
create_namespace 'ns2' , {'name' => 'LL'}
【删除】
drop_namespace 'ns1'
【修改】
alter_namespace 'ns1', {METHOD => 'set', 'sex' => 'woman'}
alter_namespace 'yuyao', {METHOD => 'unset', NAME => 'sex'}
【查询】
descirbe_namespace 'ns1'
//查询hbase中的所有的namespace，支持使用正则表达式
list_namespace 'ns1'
list_namespace 'ns.*'
//查询指定的namespace有多少表
list_namespace_tables 'ns1'
2.API
public static Admin admin;
@Before
public void getadmin(){
admin = HbaseUtils.getAdmin();
}
/**
* 新增加一个namespace
*/
@Test
public void create_namespace() throws IOException {
NamespaceDescriptor descriptor = NamespaceDescriptor.create("yy").build();
admin.createNamespace(descriptor);
}
/**
*删除namespace
* @throws IOException
*/
@Test
public void drop_namespace() throws IOException {
admin.deleteNamespace("yy");
}
/**
* 修改namespace属性
*/
public void alter_namespace() throws IOException {
//获取到指定表的namespaceDescriptor
NamespaceDescriptor namespaceDescriptor = admin.getNamespaceDescriptor("yuyao");
//查看name属性的值
namespaceDescriptor.getConfigurationValue("name");
//查看namespace里面的全部属性
namespaceDescriptor.getConfiguration();
//修改属性name的值为yy
namespaceDescriptor.setConfiguration("name","yy");
//删除namespace的sex属性
namespaceDescriptor.removeConfiguration("sex");
//需要NamespaceDescriptor
admin.modifyNamespace(namespaceDescriptor);
}
/**
* 查看namespace的详情
*/
public void describe_namespace() throws IOException {
System.out.println(admin.getNamespaceDescriptor("ns1"));
}
/**
* list_namespace/describe_namespace
* 查看所有的namespace
* @throws IOException
*/
@Test
public void list_namespace() throws IOException {
NamespaceDescriptor[] namespaceDescriptors = admin.listNamespaceDescriptors();
for (NamespaceDescriptor namespaceDescriptor:namespaceDescriptors)
{
//所有namespace的名字和对相应的属性
System.out.println(namespaceDescriptor);
//查看所有namespace的某一个属性值
System.out.println(namespaceDescriptor.getConfigurationValue("name"));
//所有namespace的属性
System.out.println(namespaceDescriptor.getConfiguration());
System.out.print(namespaceDescriptor.getName() + "\t");
//Map configuration1 = namespaceDescriptor.getConfiguration();
Map configuration = namespaceDescriptor.getConfiguration();
// 获取到所有的属性的kv
for (Map.Entry entry : configuration.entrySet()) {
System.out.print(entry.getKey() + "-->" + entry.getValue() + "\t");
}
System.out.println();
}
}
/**
* 获取指定namespace里面的表
* @throws IOException
*/
@Test
public void list_namespace_tables() throws IOException {
TableName[] tables = admin.listTableNamesByNamespace("ns1");
for (TableName table : tables){
System.out.println(table);
}
}
@After
public void close() throws IOException {
admin.close();
}
3.DDL[表操作]
1.Hbase shell
【增加】
create 'ns1:t1' , {NAME => 'f1',VERSIONS => 5}
create 'ns1:t2', {NAME => 'f1'}, {NAME => 'f2'}, {NAME => 'f3'}
【删除】
drop 'ns1:t1'
disable 'ns1:t1'
enable 'ns1:t1'
exists 'ns1:t1'
【修改】
//添加列簇/修改列簇
alter 't1', NAME => 'f1', VERSIONS => 5
//删除列簇
alter 'ns1:t1', NAME => 'f1', METHOD => 'delete'
alter 'ns1:t1', 'delete' => 'f1'
【查询】
describe 'ns2:t1'
list
2.API
/**
* 创建表
* @throws IOException
*/
@Test
public void create_Table() throws IOException {
//表名
TableName tableName = TableName.valueOf("yuyao:t1");
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableName);
//列簇
HColumnDescriptor hColumnDescriptor = new HColumnDescriptor("f1");
hColumnDescriptor.setVersions(1,6);
hTableDescriptor.addFamily(hColumnDescriptor);
admin.createTable(hTableDescriptor);
}
/**
*删除表
*/
public void drop_rable() throws IOException {
TableName tableName = TableName.valueOf("ns1:t2");
boolean exists = admin.tableExists(tableName);
if(exists)
{
boolean tableDisabled = admin.isTableDisabled(tableName);
if (!tableDisabled){
admin.disableTable(tableName);
}
admin.deleteTable(tableName);
}else {
System.out.println(tableName + "表不存在");
}
}
/**
* alter修改列簇的属性/修改列簇
*/
@Test
public void alter_table() throws IOException {
TableName tableName = TableName.valueOf("ns1:t2");
//得到表描述信息
HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName);
//删除列簇
tableDescriptor.removeFamily(Bytes.toBytes("f1"));
//添加列簇
HColumnDescriptor hColumnDescriptor = new HColumnDescriptor("f1");
tableDescriptor.addFamily(hColumnDescriptor);
//修改列簇
HColumnDescriptor columnDescriptor = tableDescriptor.getFamily(Bytes.toBytes("f1"));
columnDescriptor.setVersions(5,6);
columnDescriptor.setConfiguration("name","yy");
admin.modifyTable(tableName,tableDescriptor);
}
/**
* 查询表信息 list
*/
@Test
public void list_tableNames() throws IOException {
TableName[] tableNames = admin.listTableNames();
for (TableName tableName:tableNames)
{
System.out.println(tableName);
}
}
/**
* 查询表的详细信息
*/
@Test
public void desc_tables() throws IOException {
HTableDescriptor[] hTableDescriptors = admin.listTables("ns1:t2");
for (HTableDescriptor hTableDescriptor:hTableDescriptors)
{
System.out.println(hTableDescriptor);
}
}
4.DML
1.Hbase shell
【增加】 put 'ns1:t1','r1','f1:sex','man' , [ts1] append 't1','r1','f2:sex','woman' 【删除】 delete 'ns1:t1', 'r1', 'f1:sex', [ts1] 【修改】 【查询】 get 'ns1:t1', 'r1' scan 'ns1:t1' 【自增】 incr 'ns1：t1','r1','f1:age'
2.API
package com.qf.Hbaseutils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;
import java.io.IOException;
public class TableUtils {
private final static String ZK_CLUSTER_Value = "qphone01,qphone02,qphone03";
private final static String ZK_CLUSTER_KEY = "hbase.zookeeper.quorum";
private final static String DEFAULT_TABLE = "t1";
private static Configuration configuration = new Configuration();
static {
configuration.set(ZK_CLUSTER_KEY,ZK_CLUSTER_Value);
}
public static Admin getadmin() throws IOException {
Admin admin = ConnectionFactory.createConnection(configuration).getAdmin();
return admin;
}
public static Table getTable() throws IOException {
return getTable(DEFAULT_TABLE);
}
public static Table getTable(String tablename) throws IOException {
Table table = ConnectionFactory.createConnection(configuration).getTable(TableName.valueOf(tablename));
return table;
}
public static void println(Result result) {
if (result == null) {
return;
}
while (result.advance()) { // 判断如果有cell可以推进就返回true，否则为false
Cell current = result.current(); // 获取到当前的cell
System.out.print(new String(CellUtil.cloneRow(current)) + "\t");
System.out.print(new String(CellUtil.cloneFamily(current)) + "\t");
System.out.print(new String(CellUtil.cloneQualifier(current)) + "\t");
System.out.print(new String(CellUtil.cloneValue(current)) + "\t");
System.out.println();
}
}
/**
* 打印scanner中的结果
*/
public static void println(ResultScanner scanner) {
//3. 查看
Iterator iterator = scanner.iterator();
while (iterator.hasNext()) {
Result result = iterator.next();
println(result);
}
}
public static void close(Admin admin) throws IOException {
admin.close();
}
public static void close(Table table) throws IOException {
table.close();
}
public static void close(Admin admin,Table table) throws IOException {
close(admin);
close(table);
}
}
/**
* 添加
* @throws IOException
*/
@Test
public void put_append() throws IOException {
//Put put = new Put(Bytes.toBytes("r1"),1594621012395L);
Put put = new Put(Bytes.toBytes("r1"));
put.addColumn(Bytes.toBytes("f1"),Bytes.toBytes("name"),Bytes.toBytes("aa"));
table.put(put);
Append append = new Append(Bytes.toBytes("f1"));
append.add(Bytes.toBytes("f1"),Bytes.toBytes("name"),Bytes.toBytes("aa"));
table.append(append);
}
/**
*
*/
@Test
public void get() throws IOException {
//行键
Get get = new Get(Bytes.toBytes("r1"));
Result result = table.get(get);
Filterlist.println(result);
}
@Test
public void scan() throws IOException {
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
Filterlist.println(scanner);
}
5.过滤器
###
package com.qf;
import com.qf.Hbaseutils.HbaseUtils;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.Table;
import org.junit.After;
import org.junit.Test;
import java.io.IOException;
public class Filter {
public static Table table;
@Test
public void singleColumnValueFilter() throws IOException {
table = HbaseUtils.getTable("ns1:t2");
SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(
Bytes.toBytes("f1"),
Bytes.toBytes("age"),
CompareFilter.CompareOp.GREATER,
Bytes.toBytes("10")
);
Scan scan = new Scan();
scan.setFilter(singleColumnValueFilter);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void filterList() throws IOException {
table = HbaseUtils.getTable("t2");
SingleColumnValueFilter singleColumnValueFilter1 = new SingleColumnValueFilter(
Bytes.toBytes(""),
Bytes.toBytes(""),
CompareFilter.CompareOp.GREATER,
Bytes.toBytes("")
);
SingleColumnValueFilter singleColumnValueFilter2 = new SingleColumnValueFilter(
Bytes.toBytes(""),
Bytes.toBytes(""),
CompareFilter.CompareOp.GREATER,
Bytes.toBytes("")
);
singleColumnValueFilter1.setFilterIfMissing(true);
singleColumnValueFilter2.setFilterIfMissing(true);
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL,singleColumnValueFilter1,singleColumnValueFilter2);
Scan scan = new Scan();
scan.setFilter(filterList);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
//Hbase 2.0版本之后
public void columnValueFilter() throws IOException {
table = HbaseUtils.getTable("t2");
//new ColumnValueFilter();
}
@Test
public void regexStringComparator() throws IOException {
table = HbaseUtils.getTable("ns1:t2");
RegexStringComparator regexStringComparators = new RegexStringComparator(".*m.*");
SingleColumnValueFilter singleColumnValueFilters = new SingleColumnValueFilter(
Bytes.toBytes("f1"),
Bytes.toBytes("sex"),
CompareFilter.CompareOp.EQUAL,
regexStringComparators
);
Scan scan = new Scan();
scan.setFilter(singleColumnValueFilters);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void substringComparator() throws IOException {
table = HbaseUtils.getTable("ns1:t2");
SubstringComparator substring = new SubstringComparator("m");
SingleColumnValueFilter singleColumnValueFilters = new SingleColumnValueFilter(
Bytes.toBytes("f1"),
Bytes.toBytes("sex"),
CompareFilter.CompareOp.EQUAL,
substring
);
Scan scan = new Scan();
scan.setFilter(singleColumnValueFilters);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void binaryComparator() throws IOException {
table = HbaseUtils.getTable();
BinaryComparator binary = new BinaryComparator(Bytes.toBytes("11"));
SingleColumnValueFilter singleColumnValueFilters = new SingleColumnValueFilter(
Bytes.toBytes("f1"),
Bytes.toBytes("age"),
CompareFilter.CompareOp.GREATER,
binary
);
Scan scan = new Scan();
scan.setFilter(singleColumnValueFilters);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void binaryPrefixComparator() throws IOException {
table = HbaseUtils.getTable();
BinaryPrefixComparator binaryPrefix= new BinaryPrefixComparator(Bytes.toBytes("m"));
SingleColumnValueFilter singleColumnValueFilters = new SingleColumnValueFilter(
Bytes.toBytes("f1"),
Bytes.toBytes("sex"),
CompareFilter.CompareOp.EQUAL,
binaryPrefix
);
Scan scan = new Scan();
scan.setFilter(singleColumnValueFilters);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void binaryComponentComparator() throws IOException {
table = HbaseUtils.getTable();
//BinaryComponentComparator binaryComponent = new BinaryComponentComparator();
/**
* byte[] partialValue = Bytes.toBytes("partial_value");
* int partialValueOffset =
* Filter partialValueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER,
* new BinaryComponentComparator(partialValue,partialValueOffset));
*/
}
@Test
public void familyFilter() throws IOException {
table = HbaseUtils.getTable();
BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator(Bytes.toBytes("f1"));
FamilyFilter family = new FamilyFilter(CompareFilter.CompareOp.EQUAL,binaryPrefixComparator);
Scan scan = new Scan();
scan.setFilter(family);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void qualifierFilter() throws IOException {
table = HbaseUtils.getTable();
BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator(Bytes.toBytes("ag"));
QualifierFilter qualifier = new QualifierFilter(CompareFilter.CompareOp.EQUAL,binaryPrefixComparator);
Scan scan = new Scan();
scan.setFilter(qualifier);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
/**
* 列名前缀
* @throws IOException
*/
@Test
public void columnPrefixFilter() throws IOException {
table = HbaseUtils.getTable();
ColumnPrefixFilter columnPrefix = new ColumnPrefixFilter(Bytes.toBytes("ag"));
Scan scan = new Scan();
scan.setFilter(columnPrefix);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void multipleColumnPrefixFilter() throws IOException {
table = HbaseUtils.getTable();
byte[][] bytes = {Bytes.toBytes("age"),Bytes.toBytes("name")};
MultipleColumnPrefixFilter multipleColumnPrefix = new MultipleColumnPrefixFilter(bytes);
Scan scan = new Scan();
scan.setFilter(multipleColumnPrefix);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void columnRangeFilter() throws IOException {
table = HbaseUtils.getTable();
ColumnRangeFilter rangeFilter = new ColumnRangeFilter(Bytes.toBytes("age"), false, Bytes.toBytes("pp"), true);
Scan scan = new Scan();
scan.setFilter(rangeFilter);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void rowFilter() throws IOException {
table = HbaseUtils.getTable();
BinaryComparator binaryComparator = new BinaryComparator(Bytes.toBytes("r1"));
RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, binaryComparator);
Scan scan = new Scan();
scan.setFilter(rowFilter);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@Test
public void firstKeyOnlyFilter() throws IOException {
table = HbaseUtils.getTable("ns1:t2");
FirstKeyOnlyFilter firstKeyOnlyFilter = new FirstKeyOnlyFilter();
Scan scan = new Scan();
scan.setFilter(firstKeyOnlyFilter);
ResultScanner scanner = table.getScanner(scan);
HbaseUtils.println(scanner);
}
@After
public void close(){
HbaseUtils.close(table);
}
}
hive整合hbase
创建hive的同时创建hbase create database if not exists hive2hbase; use hive2hbase; create table if not exists hive2hbase ( uid int, uname string, age int, sex string ) stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' with serdeproperties( "hbase.columns.mapping"=":key,base_info:name,base_info:age,base_info:sex" ) tblproperties( "hbase.table.name"="hive2hbase1" );
hbase整合hive
先创建hbase表格 在创建hive表格 如下： create external table if not exists hbase2hive ( uid int, age int, name string ) stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' with serdeproperties( "hbase.columns.mapping"=":key,ff:age,ff:name" ) tblproperties( "hbase.table.name"="ns1:hbase" );