综合分析平台
package com.sdg.consumer.myhbase
import java.text.SimpleDateFormat
import java.util
import com.sdg.consumer.myutils.{ConnectionInstance, HBaseUtil, PropertiesUtil}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.hadoop.hbase.client.{HTable, Put}
import org.apache.hadoop.hbase.util.Bytes
object HbaseDao {
private val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
private val sdf2 = new SimpleDateFormat("yyyyMMddHHmmss")
private val cacheList = new util.ArrayList[Put]
var conf: Configuration = HBaseConfiguration.create
private var regions = Integer.valueOf(PropertiesUtil.getProperty("hbase.calllog.regions"))
private var namespace: String = PropertiesUtil.getProperty("hbase.calllog.namespace")
private var tableName: String = PropertiesUtil.getProperty("hbase.calllog.tablename")
var table: HTable = null
//首先创建命名空间 然后再创建表
if (!HBaseUtil.isExistTable(conf, tableName)) {
//这里做一个判断就好
HBaseUtil.initNamespace(conf, namespace)
HBaseUtil.createTable(conf, tableName, regions, "f1", "f2")
}
/**
* 把数据写入到hbase
* ori数据样式： 18576581848,17269452013,2017-08-14 13:38:31,1761
* rowkey样式：01_18576581848_20170814133831_17269452013_1_1761
* HBase表的列：call1 call2 build_time build_time_ts flag duration
*
* @param ori
*/
def put(ori: String): Unit = {
//插入方法
//put方式
//hive+hbase方式
//phoenix(sql)==>squueirl（松鼠）
//buldload==>效率最高 hfile:效率最高
if (cacheList.size == 0) {
val connection = ConnectionInstance.getConnection(conf)
table = connection.getTable(TableName.valueOf(tableName)).asInstanceOf[HTable]
table.setAutoFlushTo(false)
//
table.setWriteBufferSize(2 * 1024 * 1024)
}
//对传输过来的字符串用逗号进行分割 18468618874,魏明艳,13980337439,卫艺,2017-11-25 13:05:27,1088,1
val splitOri: Array[String] = ori.split(",")
val caller: String = splitOri(0)
val callee: String = splitOri(1)
val buildTime: String = splitOri(4)
val duration: String = splitOri(5)
//获取region编码
val regionCode: String = HBaseUtil.genRegionCode(caller, buildTime, regions)
//建立通话时间
val buildTimeReplace: String = sdf2.format(sdf1.parse(buildTime))
val buildTimeTs: String = String.valueOf(sdf1.parse(buildTime).getTime)
//生成rowkey
val rowkey: String = HBaseUtil.genRowKey(regionCode, caller, buildTimeReplace, callee, "1", duration)
//向表中插入该条数据
val put: Put = new Put(Bytes.toBytes(rowkey))
//主叫号码
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("call1"), Bytes.toBytes(caller))
//被叫号码
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("call2"), Bytes.toBytes(callee))
//通话日期
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("build_time"), Bytes.toBytes(buildTime))
//通话时间
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("build_time_ts"), Bytes.toBytes(buildTimeTs))
//通过标识
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("flag"), Bytes.toBytes("1"))
//通话时长
put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("duration"), Bytes.toBytes(duration))
cacheList.add(put)
if (cacheList.size >= 0) {
// val value: List[Put] = util.List[Put]
//这个类型必须是java的List,scala 的类型是不支持的
table.put(cacheList)
//把数据提交到hbase的表中
table.flushCommits()
println("插入数据成功")
//清除缓冲的内容
cacheList.clear()
}
//1.定时接受数据
//2.for循环之外在提交一次
}
}
package com.sdg.consumer.mykafka
//命名包名的时候不要冲突
import java.util
import com.sdg.consumer.myhbase.HbaseDao
import com.sdg.consumer.myutils.PropertiesUtil
import org.apache.kafka.clients.consumer.{ConsumerRecords, KafkaConsumer}
import scala.collection.JavaConversions._
/**
- KafkaToHbase
- 把数据写入到hbase 中
*/
object HbaseConsumer {
def main(args: Array[String]): Unit = {
// HbaseDao.put(“18468618874,魏明艳,13980337439,卫艺,2017-11-25 13:05:27,1088,1”)
//testHbase()
//创建kafka消费者的对象
val kafkaConsumer = new KafkaConsumerString, String
//订阅指定的topic 用于数据的消费
kafkaConsumer.subscribe(util.Arrays.asList(PropertiesUtil.getProperty(“kafka.topics”)))
println(“等待消费数据--------------”)
while (true) {
//每0.1S 从指定topic中消费数据
val records: ConsumerRecords[String, String] = kafkaConsumer.poll(100)
//这个是scala和java集合类型之间的转换
for (cr <- records) {
//得到每条数据的value
val str: String = cr.value()
println(str)
//把数据写入到hbase中
HbaseDao.put(str)
}
}
}
def testHbase(): Unit = {
val str = “18468618874,魏明艳,13980337439,卫艺,2017-11-25 13:05:27,1088,1”
HbaseDao.put(str)
}
}
package com.sdg.consumer.myutils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client.{Connection, ConnectionFactory}
//获取连接的
object ConnectionInstance {
private var conn: Connection = null
def getConnection(conf: Configuration): Connection = {
if (conn == null || conn.isClosed) {
conn = ConnectionFactory.createConnection(conf);
}
conn
}
}
package com.sdg.consumer.myutils
import java.text.DecimalFormat
import java.util
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client.{Admin, Connection, ConnectionFactory}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HColumnDescriptor, HTableDescriptor, NamespaceDescriptor, TableName}
object HBaseUtil {
/**
* regionCode_call1_buildTime_call2_flag_duration
*
* @param regionCode 区号
* @param caller 主叫号码
* @param buildTime 建立时间
* @param callee 被叫号码
* @param flag 主动被动标识
* @param duration 通话时长
* @return
*/
def genRowKey(regionCode: String, caller: String, buildTime: String, callee: String, flag: String, duration: String): String = {
val sb = new StringBuilder
sb.append(regionCode + "_")
.append(caller + "_")
.append(buildTime + "_")
.append(caller + "_")
.append(flag + "_")
.append(duration)
sb.toString()
}
/**
* 获取区号
*
* @param call1
* @param buildTime
* @param regions
* @return
*/
def genRegionCode(call1: String, buildTime: String, regions: Integer): String = {
//电话号码的长度
val len: Int = call1.length
//取出后4位号码
val lastPhone: String = call1.substring(len - 4)
//取出建立通过时间的年月2018-02-02
val ym: String = buildTime.replaceAll("-", "")
.replaceAll(":", "")
.replaceAll(" ", "")
.substring(0, 6)
//离散操作1 ^ 这个符号是异或运算 转成二进制 对应位置相同为 0 不同就为1
val x: Integer = Integer.valueOf(lastPhone) ^ Integer.valueOf(ym)
//离散操作2
val y: Int = x.hashCode
//生成分区号
val regionCode: Int = y % regions
//格式化分区号
val df = new DecimalFormat("00")
df.format(regionCode)
}
/**
* 预分区键
* 创建区号
* @param regions
* @return
*/
def genSplitKeys(regions: Integer): Array[Array[Byte]] = {
//定义一个存放分区键的数组
val keys: Array[String] = new Array[String](regions)
//目前推算，region个数不会超过2位数，所以region分区键格式化为两位数字所代表的字符串
val df: DecimalFormat = new DecimalFormat("00")
//对region个数遍历
for (i <- 0 until regions) {
//使用 | 拼接一下
keys(i) = df.format(i) + "|"
}
//定义一个二维数组
val splitKeys = new Array[Array[Byte]](regions)
//比较器 BYTES_COMPARATOR :升序排序
val treeSet: util.TreeSet[Array[Byte]] = new util.TreeSet[Array[Byte]](Bytes.BYTES_COMPARATOR)
for (i <- 0 until regions) {
//把我们生成keys 放进去
treeSet.add(Bytes.toBytes(keys(i)));
}
val splitKeysIterator: util.Iterator[Array[Byte]] = treeSet.iterator
var index = 0
while (splitKeysIterator.hasNext) {
val b: Array[Byte] = splitKeysIterator.next
println(b)
splitKeys(index) = b
index = index + 1
}
splitKeys
}
/**
*
* @param conf
* @param tableName 表名
* @param regions 分区个数
* @param columnFamily 列簇（连个列簇）
*/
def createTable(conf: Configuration, tableName: String, regions: Integer, columnFamily: String*) = {
val connection: Connection = ConnectionFactory.createConnection(conf)
val admin: Admin = connection.getAdmin
//if (isExistTable(conf, tableName)) return
val htd = new HTableDescriptor(TableName.valueOf(tableName))
for (cf <- columnFamily) {
htd.addFamily(new HColumnDescriptor(cf))
}
//创建表的时候指定支持协处理器
//htd.addCoprocessor("hbase.CalleeWriteObserver")
//指定与分区指定分区的个数
admin.createTable(htd, genSplitKeys(regions))
//此下方法创建的表只能有一个region分区
// admin.createTable(htd)
admin.close()
connection.close()
}
def main(args: Array[String]): Unit = {
/* val conf: Configuration = HbaseDao.conf
val connection: Connection = ConnectionFactory.createConnection(conf)
val admin: Admin = connection.getAdmin
//if (isExistTable(conf, tableName)) return
val htd = new HTableDescriptor(TableName.valueOf(" "))
//增加协处理器
//htd.addCoprocessor("hbase.CalleeWriteObserver")
//指定与分区指定分区的个数
admin.createTable(htd, genSplitKeys(6))
admin.close()
connection.close()*/
/*val array = genSplitKeys(6)
println(Bytes.toString(array(0)))
println(Bytes.toString(array(1)))
println(Bytes.toString(array(2)))
println(Bytes.toString(array(3)))
println(Bytes.toString(array(4)))
println(Bytes.toString(array(5)))
println(Bytes.toString(array(6)))*/
val str: String = genRegionCode("13526949099", "2018-02-02", 6)
println(str)
}
/**
* 初始化命名空间
*
* @param conf
* @param namespace
*/
def initNamespace(conf: Configuration, namespace: String) = {
//获取hbase链接
val connection: Connection = ConnectionFactory.createConnection(conf)
//获取admin对象
val admin: Admin = connection.getAdmin
//创建命令空间
val nd: NamespaceDescriptor = NamespaceDescriptor.create(namespace)
.addConfiguration("CREATE_TIME", String.valueOf(System.currentTimeMillis)).addConfiguration("AUTHOR", "liuhe").build
admin.createNamespace(nd)
admin.close()
connection.close()
}
/**
* 判断表是否存在
*
* @param conf
* @param tableName
*/
def isExistTable(conf: Configuration, tableName: String): Boolean = {
val connection: Connection = ConnectionFactory.createConnection(conf)
val admin: Admin = connection.getAdmin
//判断表是否存在
val result: Boolean = admin.tableExists(TableName.valueOf(tableName))
admin.close()
connection.close()
result
}
}
package com.sdg.consumer.myutils
import java.io.InputStream
import java.util.Properties
//读取配置文件信息
object PropertiesUtil {
val is: InputStream = ClassLoader.getSystemResourceAsStream(“hbase_consumer.properties”)
var properties = new Properties
properties.load(is)
//根据key 取出来对应的值
def getProperty(key: String): String = {
val str: String = properties.getProperty(key)
str
}
}