[HBase基础]-- java 调用hbase api创建hbase分区表

java 调用hbase api创建hbase分区表

运行环境：idea16+jdk-1.7+hbase-cdh-1.2.0+cdh-5.8.0

 

一、新建maven项目，编写pom.xml文件

    4.0.0

    com.enn
    testDB
    1.0-SNAPSHOT
    
    
        UTF-8
        2.6.0-cdh5.8.0
        1.2.0-cdh5.8.0
    

    
    
        
            cloudera
            https://repository.cloudera.com/artifactory/cloudera-repos/
        
    


    
        
        
            org.apache.hadoop
            hadoop-common
            ${hadoop.version}
            
                
                    org.slf4j
                    slf4j-log4j12
                
            
        
        
            org.apache.hadoop
            hadoop-client
            ${hadoop.version}
        

        
            org.apache.hadoop
            hadoop-hdfs
            ${hadoop.version}
            
                
                    tomcat
                    jasper-runtime
                
                
                    tomcat
                    jasper-compiler
                
            
        

        
        
            org.apache.hbase
            hbase-client
            ${hbase.version}
            
                
                    org.slf4j
                    slf4j-log4j12
                
            
        
        
            org.apache.hbase
            hbase-common
            ${hbase.version}
            
                
                    org.slf4j
                    slf4j-log4j12
                
            
        
        
            org.apache.hbase
            hbase-server
            ${hbase.version}
            
                
                    org.slf4j
                    slf4j-log4j12
                
            
        

        
        
            ch.qos.logback
            logback-core
            1.1.3
        
        
            ch.qos.logback
            logback-classic
            1.1.3
        
        
            ch.qos.logback
            logback-access
            1.1.3
        

    
    
    
        
            
            
                org.apache.maven.plugins
                maven-compiler-plugin
                3.1
                
                    1.7
                    1.7
                    ${project.build.sourceEncoding}
                
            
        
    

 

二、编写配置文件：CreateTable.xml(注意：该文件需要放到resources目录下)

 

    
        1
        0
        65536
        true
        false
        2147483647
        ROW
        LZ4
        false
        false
        false
        true
        1
        false
        PREFIX_TREE
    
    fgreatgas:enn_test222232
    
        slave-29.dev.cluster:2181,slave-30.dev.cluster:2181,slave-31.dev.cluster:2181
        /hbase
    

 

三、编写工具类：HBaseConf.java

 

 

 

package createTable;

import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;

import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.URL;
import java.util.NoSuchElementException;

/**
 * 获取创建表配置文件的内容
 */
public class HBaseConf {
    Logger logger = LoggerFactory.getLogger(HBaseConf.class);
    //列属性配置
    int MaxVersions;//最大版本数
    int MinVersions;//最小版本数
    int Blocksize;//块大小
    boolean BlockCacheEnabled;//块是否可缓存
    boolean InMemory;//
    int TimeToLive;//存储时长
    BloomType BloomFilterType;//存储HFile过滤器
    Algorithm CompressionType;//压缩类型
    boolean CacheBloomsOnWrite;//
    boolean CacheDataOnWrite;//
    boolean CacheIndexesOnWrite;//
    String CompressTags;//
    int Scope;//
    boolean KeepDeletedCells;//是否保存删除的数据
    DataBlockEncoding DataBlockEncoding;//
    //表属性配置
    String FamilyName;//列族名
    String TableName;//表名
    String RegionNum;//分区数
    //hbase连接配置
    String Quorum;//zookeeper主机
    String Parent;//zookeeper存储目录

    public HBaseConf readXml() {
        try {
            XMLConfiguration xmlConfiguration = getXml();
            xmlConfiguration.setThrowExceptionOnMissing(true);//获取不到指定属性则报错
            this.MaxVersions = xmlConfiguration.getInt("column.MaxVersions");
            this.MinVersions = xmlConfiguration.getInt("column.MinVersions");
            this.Blocksize = xmlConfiguration.getInt("column.BlockSize");
            this.BlockCacheEnabled = xmlConfiguration.getBoolean("column.BlockCacheEnabled");
            this.InMemory = xmlConfiguration.getBoolean("column.InMemory");
            this.TimeToLive = xmlConfiguration.getInt("column.TimeToLive");
            this.BloomFilterType = BloomType.valueOf(xmlConfiguration.getString("column.BloomFilterType").toUpperCase());
            this.CompressionType = Algorithm.valueOf(xmlConfiguration.getString("column.CompressionType").toUpperCase());
            this.CacheBloomsOnWrite = xmlConfiguration.getBoolean("column.CacheBloomsOnWrite");
            this.CacheDataOnWrite = xmlConfiguration.getBoolean("column.CacheDataOnWrite");
            this.CacheIndexesOnWrite = xmlConfiguration.getBoolean("column.CacheIndexesOnWrite");
            this.CompressTags = xmlConfiguration.getString("column.CompressTags");
            this.Scope = xmlConfiguration.getInt("column.Scope");
            this.KeepDeletedCells = xmlConfiguration.getBoolean("column.KeepDeletedCells");
            this.DataBlockEncoding = DataBlockEncoding.valueOf(xmlConfiguration.getString("column.DataBlockEncoding").toUpperCase());

            this.FamilyName = xmlConfiguration.getString("table.FamilyName");
            this.TableName = xmlConfiguration.getString("table.TableName");
            this.RegionNum = xmlConfiguration.getString("table.RegionNum");

            this.Quorum = xmlConfiguration.getString("zookeeper.Quorum");
            this.Parent = xmlConfiguration.getString("zookeeper.Parent");
            return this;
        } catch (ConfigurationException ex) {
            logger.error("创建xml解读类出现错误");
            ex.printStackTrace();
            return null;
        } catch (NoSuchElementException ex) {
            logger.error("读取'建表的参数配置文件'出现错误,请确保已配置如下参数:{}.", ex.getMessage());
            ex.printStackTrace();
            return null;
        }

    }

    /**
     * 读取默认的建表配置文件
     *
     * @return 配置文件
     * @throws ConfigurationException
     */
    private XMLConfiguration getXml() throws ConfigurationException {
        URL url = HBaseConf.class.getClassLoader().getResource("CreateTable.xml");
        XMLConfiguration xmlConfiguration=new XMLConfiguration();
        xmlConfiguration.setDelimiterParsingDisabled(true);
        xmlConfiguration.setAttributeSplittingDisabled(true);
        xmlConfiguration.load(url);
        return xmlConfiguration;
    }

    /**
     * 读取指定文件
     *
     * @param file 指定的建表配置文件名
     * @return 配置文件
     * @throws ConfigurationException
     */
    private XMLConfiguration getXml(String file) throws ConfigurationException {
        XMLConfiguration xmlConfiguration=new XMLConfiguration();
        xmlConfiguration.setDelimiterParsingDisabled(true);
        xmlConfiguration.setAttributeSplittingDisabled(true);
        xmlConfiguration.load(file);
        return xmlConfiguration;
    }
}

 

 

编写主类：CreateTable.java(注意：如果有kerberos认证，则需要2个文件---》krb5.conf和e_lvbin.keytab)

 

package createTable;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;

import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

/**
 * 创建Hbase分区表的实现
 * 测试通过
 */
public class CreateTable {

    static Logger logger = LoggerFactory.getLogger(CreateTable.class);//创建日志对象

    public static void main(String[] args) {
        HBaseConf conf = null;//创建配置文件类
        conf = new HBaseConf().readXml();//获取配置文件的配置参数类
        try {
            for (String parent : conf.Parent.split(",")) {
                for (String table : conf.TableName.split(",")) {
                    create(conf, parent, table);
                }
            }

        } catch (Exception e) {
            logger.error("建表失败!");
            e.printStackTrace();
        }
    }
    /**
     * 建表
     *
     * @param conf   配置类
     * @param parent 库名
     * @param table  表名
     *       作用：    创建分区表
     * @throws IOException
     */
    public static void create(HBaseConf conf, String parent, String table) throws IOException {

        System.setProperty("java.security.krb5.conf","F:/krb5.conf");  //windows下 ,本地执行需要
        Configuration configuration = HBaseConfiguration.create();//创建Hbase连接配置
        configuration.set("hbase.zookeeper.quorum", "slave-31.dev.cluster.em.cn:2181");
        configuration.set("hbase.rootdir", "hdfs://mycluster/hbase");
        configuration.set("hadoop.security.authentication", "kerberos");
        configuration.set("hbase.security.authentication", "kerberos");
        configuration.set("hbase.security.authorization", "true");
        configuration.set("hbase.master.kerberos.principal", "hbase/[email protected]");
        configuration.set("hbase.thrift.kerberos.principal", "hbase/[email protected]");
        configuration.set("hbase.regionserver.kerberos.principal", "hbase/[email protected]");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        configuration.set(TableInputFormat.INPUT_TABLE, table);
        configuration.set("hbase.rpc.timeout", "10000");
        configuration.set("hbase.client.retries.number", "5");
        configuration.set("hbase.client.pause", "5000");
        configuration.set("hbase.client.scanner.timeout.period", "50000");
        String user = "[email protected]";
        String keyPath = "F:/e_lvbin.keytab";                   //本地认证需要的文件
        UserGroupInformation.setConfiguration(configuration);
        UserGroupInformation.loginUserFromKeytab(user, keyPath);
        HBaseAdmin hBaseAdmin = null;//创建表管理
        try {
            hBaseAdmin = new HBaseAdmin(configuration);
            logger.info("hbase库是:{}............", parent);
            if (hBaseAdmin.tableExists(table)) {
                logger.error("{}表已经存在，无法创建！", table);
            } else {
                logger.info("正在创建表:{}..............", table);
                HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(table));//创建表描述器
                //hTableDescriptor.setMaxFileSize();//集群配置中应该有，这些不设置应该也行
                for (String familyName : conf.FamilyName.split(",")) {//针对不同的列族建立列描述器
                    HColumnDescriptor hd = new HColumnDescriptor(familyName);//创建列描述器
                    hd.setBlockCacheEnabled(conf.BlockCacheEnabled);
                    hd.setBlocksize(conf.Blocksize);
                    hd.setBloomFilterType(conf.BloomFilterType);
                    hd.setCacheBloomsOnWrite(conf.CacheBloomsOnWrite);
                    hd.setCacheDataOnWrite(conf.CacheDataOnWrite);
                    hd.setCacheIndexesOnWrite(conf.CacheIndexesOnWrite);
                    hd.setCompressionType(conf.CompressionType);
                    hd.setDataBlockEncoding(conf.DataBlockEncoding);
                    hd.setInMemory(conf.InMemory);
                    hd.setMaxVersions(conf.MaxVersions);
                    hd.setMinVersions(conf.MinVersions);
                    hd.setTimeToLive(conf.TimeToLive);
                    hd.setKeepDeletedCells(conf.KeepDeletedCells);
                    hd.setScope(conf.Scope);
                    hTableDescriptor.addFamily(hd);//表描述器加载该列描述器
                }
                //注意建立预分区的startKey与endKey类型要与你插入数据时的hash值对应起来，此处是short数，那个你插入时rowkey的前缀应该是short的byte数组而不是字符串的byte数组
                hBaseAdmin.createTable(hTableDescriptor,
                        Bytes.toBytes((short)(0)), Bytes.toBytes((short)(0x7FFF)), Integer.parseInt(conf.RegionNum));//创建表-参数分别是：表描述、起始key、结束key、分区数
                logger.info("表创建完成:{}!", table);
            }

        } catch (IOException ex) {
            ex.printStackTrace();
        } finally {
            if (null != hBaseAdmin) {
                try {
                    hBaseAdmin.close();
                } catch (IOException e) {
                    logger.info("表连接关闭失败:{}!", table);
                    e.printStackTrace();
                }
            }
        }
    }
}

 

四、本地执行主类：CreateTable.java

 

五、xshell上查看是否创建分区表成功

 

注意：如果出现空指针异常，请将resources目录下的CreateTable.xml另起一个名称CreateTable222.xml，

     即可解决!!