spring data hadoop操作hbase

一、使用的jar版本

1、hbase-client:0.96.1.1-cdh5.0.0

2、spring-data-hadoop:2.2.1.RELEASE

3、Spring:4.2.1.RELEASE

二、hbase-site.xml配置



 

  hbase.zookeeper.quorum

  chhliu1,chhliu2,chhliu3

  

  

  hbase.zookeeper.property.clientPort

  2181

  

注:由于 hbase 依赖了 zookeeper ,所以需要指定 zookeeper 集群的地址以及端口号,此处需在 hosts 文件中加入 chhliu1 chhliu2 chhliu3 对应的 IP 地址。

三、applicationContext配置文件

  

      

     

    

    

     

      

      

      

      

      

 

注:配置文件中需要加入spring-data-hadoop以及P标签的命名空间,此处,我们使用的是在applicationContext配置文件中,直接引入hbase-site.xml的方式进行集成,当然也可以直接在applicationContext配置文件中,直接配置,如下:

说明:使用spring-data-hadoop操作hbase的核心在于HbaseTemplate类,该类的作用类似于HibernateTemplate以及JpaTemplateRedisTemplate

四、HbaseTemplate接口

HbaseTemplate继承了HbaseAccessor并实现了HbaseOperations,其中HbaseAccessor实现了Hbase的配置以及初始化工作,HbaseOperations则定义了HbaseTemplate的操作,源码如下:

public interface HbaseOperations {

/**
 * 回调方法
 */
 T execute(String tableName, TableCallback action);

/**
 * 根据表名,查找Hbase表中的列簇信息
 */
 T find(String tableName, String family, final ResultsExtractor action);

/**
 * 浏览表的各列信息
 */
 T find(String tableName, String family, String qualifier, final ResultsExtractor action);

 /**
 * 根据Scan查找表信息,并将信息封装到ResultsExtractor中
 */
 T find(String tableName, final Scan scan, final ResultsExtractor action);

 /**
 * 查找表中的列簇信息
 */
 List find(String tableName, String family, final RowMapper action);

/**
 * 查找表的各列信息
 */
 List find(String tableName, String family, String qualifier, final RowMapper action);

/**
 * 将查找到的信息以回调的方式显示
 */
 List find(String tableName, final Scan scan, final RowMapper action);

/**
 * 获取表中指定行的信息
 */
 T get(String tableName, String rowName, final RowMapper mapper);

/**
 * 获取行对应的列簇信息
 */
 T get(String tableName, String rowName, String familyName, final RowMapper mapper);

/**
 * 获取指定行,指定列簇的指定列信息
 */
 T get(String tableName, final String rowName, final String familyName, final String qualifier, final RowMapper mapper);

/**
 * 将数据插入表中指定行,指定列簇,指定列中
 */
void put(String tableName, final String rowName, final String familyName, final String qualifier, final byte[] data);

/**
 * 删除指定行的列簇信息
 */
void delete(String tableName, final String rowName, final String familyName);

/**
 * 删除指定行,指定列簇,指定列的信息
 */
void delete(String tableName, final String rowName, final String familyName, final String qualifier);
}
五、服务封装实现

@Service("hbaseService")
public class HbaseServiceImpl implements HbaseServiceI {

	@Resource(name = "htemplate")
	private HbaseTemplate htemplate;
    // 获取表中指定行,列簇,列的值
	public String get(String tableName, String rowName, String familyName,
			String qualifier) {
		return htemplate.get(tableName, rowName, familyName, qualifier,
				new RowMapper() {
					public String mapRow(Result result, int rowNum)
							throws Exception {
						List ceList = result.listCells();
						String res = "";
						if (ceList != null && ceList.size() > 0) {
							for (Cell cell : ceList) {
								res = Bytes.toString(cell.getValueArray(),
										cell.getValueOffset(),
										cell.getValueLength());
							}
						}
						return res;
					}
				});
	}
    // 浏览表中某些行的信息,例子中指定的起始行和结束行
	public List find(String tableName, String startRow,
			String stopRow) {
        // 使用Scan可以实现过滤的功能
		Scan scan = new Scan();
		if (startRow == null) {
			startRow = "";
		}
		if (stopRow == null) {
			stopRow = "";
		}
		scan.setStartRow(Bytes.toBytes(startRow));
		scan.setStopRow(Bytes.toBytes(stopRow));
		return htemplate.find(tableName, scan, new RowMapper() {
			
			public SocTagInfo mapRow(Result result, int rowNum)
					throws Exception {
				final SocTagInfo info = new SocTagInfo();
				List ceList = result.listCells();
				if (ceList != null && ceList.size() > 0) {
					Segment segment = new Segment();
					Operator operator = new Operator();
					for (Cell cell : ceList) {
						if (null == info.getRowKey()
								|| info.getRowKey().isEmpty()) {
							String row = Bytes.toString(cell.getRowArray(),
									cell.getRowOffset(), cell.getRowLength());
							info.setRowKey(row);
						}
						// 获取值
						String value = Bytes.toString(cell.getValueArray(),
								cell.getValueOffset(), cell.getValueLength());
						// 获取列簇
						String family = Bytes.toString(cell.getFamilyArray(),
								cell.getFamilyOffset(), cell.getFamilyLength());
						// 获取列
						String quali = Bytes.toString(cell.getQualifierArray(),
								cell.getQualifierOffset(),
								cell.getQualifierLength());

                      // 以下为业务封装,可以忽略
						if (family.equals("operator")) {
							PropertiesCopyUtil.copy(operator, family, quali,
									value);
						} else {
							PropertiesCopyUtil.copy(segment, family, quali,
									value);
						}
					}
					info.setOperator(operator);
					info.setSegment(segment);
				}
				return info;
			}
		});
	}
    // 获取一行的数据,并封装成vo
	public SocTagInfo get(String tableName, String rowName) {
		return htemplate.get(tableName, rowName,
				new RowMapper() {
					public SocTagInfo mapRow(Result result, int rowNum)
							throws Exception {
						final SocTagInfo info = new SocTagInfo();
						List ceList = result.listCells();
						if (ceList != null && ceList.size() > 0) {
							Segment segment = new Segment();
							Operator operator = new Operator();
							for (Cell cell : ceList) {
								if (null == info.getRowKey()
										|| info.getRowKey().isEmpty()) {
									String row = Bytes.toString(cell.getRowArray(),
											cell.getRowOffset(), cell.getRowLength());
									info.setRowKey(row);
								}
								// 获取值
								String value = Bytes.toString(cell.getValueArray(),
										cell.getValueOffset(), cell.getValueLength());
								// 获取列簇
								String family = Bytes.toString(cell.getFamilyArray(),
										cell.getFamilyOffset(), cell.getFamilyLength());
								// 获取列
								String quali = Bytes.toString(cell.getQualifierArray(),
										cell.getQualifierOffset(),
										cell.getQualifierLength());
								if (family.equals("operator")) {
									PropertiesCopyUtil.copy(operator, family, quali,
											value);
								} else {
									PropertiesCopyUtil.copy(segment, family, quali,
											value);
								}
							}
							info.setOperator(operator);
							info.setSegment(segment);
						}
						return info;
					}
				});
	}
    // 将值插入表中指定的行,列簇,列中
	public void put(final String tableName, final String rowKey,
			final String familyName, final String column, final String value) {
		htemplate.put(tableName, rowKey, familyName, column, value.getBytes());
	}
}
六、服务测试

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = { "classpath:applicationContext-hbase.xml" })
public class HbaseServiceTest {
	
	@Resource(name="hbaseService")
	private HbaseServiceI hService;
	
	@Test
	public void test(){
		String result = hService.get("HBASE_TEST", "1102", "operator", "pid");
		System.out.println("result:"+result);
	}
	
	@Test
	public void testGet(){
		SocTagInfo info = hService.get("HBASE_TEST", "1102");
		System.out.println(info.getRowKey());
		System.out.println(info.getOperator().getPid());
		System.out.println(info.getOperator().getAgent());
		System.out.println(info.getSegment().getPnrNo());
	}
	
	@Test
	public void testPut(){
		hService.put("HBASE_TEST", "1102", "operator", "agent", "1000");
	}
	
	@Test
	public void testFind(){
		List list = hService.find("HBASE_TEST", "1102", "1112");
		for(SocTagInfo info:list){
			String rowKey = info.getRowKey();
			Segment s = info.getSegment();
			Operator o = info.getOperator();
			
			System.out.println(rowKey);
			System.out.println(o.getAgent());
			System.err.println(s.getPnrNo());
		}
	}
}

最终显示结果如下:

spring data hadoop操作hbase_第1张图片

七、关键类解读

a) HtableInterface

该接口用来和Hbase的单个表打交道,例如我们要想往表中插入一条数据,我们可以自己实现,不使用HbaseTemplate提供的接口方法,代码如下:

public boolean put(final String tableName, final String rowKey,
			final String familyName, final String column, final String value) {
		Assert.hasLength(tableName, "必须指定表名!");
		Assert.hasLength(rowKey, "必须指定行!");
		Assert.hasLength(familyName, "必须指定列簇!");
		Assert.hasLength(column, "必须指定列名!");
		Assert.notNull(value, "要插入表中的值不能为空!");
		htemplate.execute(tableName, new TableCallback() {
			boolean isinsert = false;
			@Override
			public Boolean doInTable(HTableInterface table) {
				try {
					Put put = new Put(rowKey.getBytes());
					put.add(Bytes.toBytes(familyName), Bytes.toBytes(column),
							Bytes.toBytes(value));
					table.put(put);
					isinsert = true;
				} catch (IOException e) {
					e.printStackTrace();
				}
				return isinsert;
			}
		});
	}
该类还提供了很多和表操作相关的方法,这里就不一一介绍了!

你可能感兴趣的:(spring,hbase,大数据)