ogg for bigdata hbase安装步骤

一、源端的准备工作(oracle库）
1. 设置参数
alter system set enable_goldengate_replication=true;
alter system set open_cursors=500;

2. 创建GolenGate表空间
create tablespace ts_ogg datafile size 500m;
create temporary tablespace ts_ogg_temp tempfile size 500m;

3. 创建GoldenGate用户并授权
create user ogg identified by oracle default tablespace ts_ogg temporary tablespace ts_ogg_temp;
grant dba to ogg;

4. 开启归档
shutdown immediate;
startup mount
alter database archivelog;
alter database open;

5. 检查是否开始附加日志和强制日志
SELECT force_logging, supplemental_log_data_min FROM v$database;        -- 再检查一下
alter database force logging;
alter database add supplemental log data;
alter system switch logfile;
alter system switch logfile;
alter system switch logfile;
SELECT force_logging, supplemental_log_data_min FROM v$database;        -- 再检查一下
ALTER SYSTEM SET ENABLE_GOLDENGATE_REPLICATION = TRUE SCOPE=BOTH;

二、源端安装OGG

1. 配置环境变量
export OGG_HOME=/oracle/gg/
export LD_LIBRARY_PATH=/oracle/product/db12c/lib:/lib:/usr/lib:/oracle/product/db12c/rdbms/lib
export CLASSPATH=/oracle/product/db12c/JRE:/oracle/product/db12c/jlib:/oracle/product/db12c/rdbms/jlib:/oracle/product/db12c/network/jlib

2. 安装GoldenGate
unzip fbo_ggs_Linux_x64_shiphome.zip
cd fbo_ggs_Linux_x64_shiphome/Disk1
export DISPLAY=172.26.0.85:0.0
./runInstaller

3.配置mgr进程
edit params mgr

port 7809
AUTOSTART EXTRACT *
AUTORESTART EXTRACT *, RETRIES 5, WAITMINUTES 1, RESETMINUTES 60
PURGEOLDEXTRACTS /u01/app/oracle/ogg/dirdat/*, usecheckpoints, minkeepdays 1

start mgr

4.添加表的trandata
dblogin userid ogg,password ogg

add trandata tiger.b_terminal

5.查询源端数据库SCN（动态抓取，当期scn号码，当前状态）
select current_scn from v$database;

11344815

5. 添加抽取配置文件
--动态抓取
--源端抽取进程
dblogin userid ogg password ogg
ADD EXTRACT exbb TRANLOG SCN 11344815
ADD EXTTRAIL /oracle/gg/dirdat/bb, EXTRACT exbb,megabytes 200
add trandata tiger.b_terminal

edit param exbb

extract exbb
SETENV (ORACLE_SID=SORDB)
userid ogg password ogg
exttrail /oracle/gg/dirdat/bb
logallsupcols
NOCOMPRESSDELETES
GETUPDATEBEFORES
updaterecordformat compact
discardfile /oracle/gg/dirrpt/bb.dsc, append
reportcount every 5 minutes, rate
FETCHOPTIONS, NOUSESNAPSHOT, NOUSELATESTVERSION, MISSINGROW REPORT
STATOPTIONS REPORTFETCH
table tiger.b_terminal; 

--源端投递进程
add extract pubb, SCN 11344815, exttrailsource /oracle/gg/dirdat/bb 
add rmttrail /home/oracle/ogg12/dirdat/bb, extract pubb, megabytes 200

extract pubb
userid ogg password ogg
discardfile /oracle/gg/dirrpt/bb.dsc, append
rmthost 172.20.25.250, mgrport 7809
rmttrail /home/oracle/ogg12/dirdat/bb

table tiger.b_terminal;

三、目标端的准备工作
1. 安装Oracle Database

2. 安装GoldenGate（目标端oracle）
unzip fbo_ggs_Linux_x64_shiphome.zip 
cd fbo_ggs_Linux_x64_shiphome/Disk1/
./runInstaller

3. 安装123010_ggs_Adapters_Linux_x64.zip（目标端bigdata）
unzip 123010_ggs_Adapters_Linux_x64.zip
mv ggs_Adapters_Linux_x64.tar /home/oracle/ogg12
cd /home/oracle/ogg12

4. 安装jdk-8u131-linux-x64.rpm

5. 配置profile环境变量
vi /root/.bash_profile

export JAVA_HOME=/usr/java/jdk1.8.0_77
export PATH=$JAVA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$LD_LIBRARY_PATH

（/jre/lib/amd64/server根据自己实现安装好的路径添加，需要自己验证）
6. 创建目标表（目录）
这里主要是当目标端为HDFS目录或者Hive表或者MySQL数据库时需要手动先在目标端创建好目录或者表，创建方法都类似，这里我们模拟实时传入到HDFS目录，故手动创建一个接收目录即可
hadoop –fs mkdir /ogg/replication/hive/

HBase可以自动创建表

7.配置mgr进程
edit params mgr

port 7809
AUTOSTART EXTRACT *
AUTORESTART EXTRACT *, RETRIES 5, WAITMINUTES 1, RESETMINUTES 60
PURGEOLDEXTRACTS /u01/app/oracle/ogg/dirdat/*, usecheckpoints, minkeepdays 1

8.配置目标端复制进程
add replicat rebb,nodbcheckpoint,exttrail ./dirdat/bb

edit param rebb

replicat rebb
TARGETDB LIBFILE libggjava.so SET property=dirprm/hbase.props
REPORTCOUNT EVERY 1 MINUTES, RATE
GROUPTRANSOPS 1000
MAP tiger.b_terminal, TARGET tiger.b_terminal;

--配置hbase.props
vi /home/oracle/ogg12/dirprm/hbase.props
gg.handlerlist=hbase

gg.handler.hbase.type=hbase
gg.handler.hbase.hBaseColumnFamilyName=cf
gg.handler.hbase.keyValueDelimiter=CDATA[=]
gg.handler.hbase.keyValuePairDelimiter=CDATA[,]
gg.handler.hbase.encoding=UTF-8
gg.handler.hbase.pkUpdateHandling=abend
gg.handler.hbase.nullValueRepresentation=CDATA[NULL]
gg.handler.hbase.authType=none
gg.handler.hbase.includeTokens=false

gg.handler.hbase.mode=tx

goldengate.userexit.timestamp=utc
goldengate.userexit.writers=javawriter
javawriter.stats.display=TRUE
javawriter.stats.full=TRUE

gg.log=log4j
gg.log.level=INFO

gg.report.time=30sec

#Sample gg.classpath for Apache HBase
gg.classpath=/opt/cslc/hbase-1.2.1/lib/*:/opt/cslc/hbase-1.2.1/conf/
#Sample gg.classpath for CDH
#gg.classpath=/opt/cloudera/parcels/CDH/lib/hbase/lib/*:/etc/hbase/conf
#Sample gg.classpath for HDP
#gg.classpath=/usr/hdp/current/hbase-client/lib/*:/etc/hbase/conf

javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=ggjava/ggjava.jar

--初始化数据
dblogin userid ogg password ogg
add trandata tiger.b_terminal

ADD EXTRACT excb, sourceistable scn 11344815

extract excb
userid ogg,password ogg
rmthost 172.20.25.250, mgrport 7809
RMTFILE ./dirdat/cb
table tiger.b_terminal;

start excb

add replicat recb,specialrun

specialrun
end runtime
extfile ./dirdat/cb
TARGETDB LIBFILE libggjava.so SET property=dirprm/hbase.props
REPORTCOUNT EVERY 1 MINUTES, RATE
GROUPTRANSOPS 1000
MAP tiger.b_terminal, TARGET tiger.b_terminal;

./replicat paramfile dirprm/recb.prm reportfile dirrpt/recb.rpt -p initialdataload

解决方法：
添加变量
export HADOOP_USER_NAME=hdfs
参考链接
https://github.com/sequenceiq/docker-spark/issues/30