HBase的子节点的RegionServer无法启动
一：异常
[root@slave1 ~]# jps
8364 Jps4555 NodeManager
7978 QuorumPeerMain
4450 DataNode
[root@slave1 ~]# tail -100 /usr/home/hbase-0.98.19-hadoop2/l
lib/ logs/
[root@slave1 ~]# tail -100 /usr/home/hbase-0.98.19-hadoop2/logs/
hbase-root-regionserver-slave1.log hbase-root-regionserver-slave1.out.2 hbase-root-regionserver-slave1.out.5
hbase-root-regionserver-slave1.out hbase-root-regionserver-slave1.out.3 SecurityAuth.audit
hbase-root-regionserver-slave1.out.1 hbase-root-regionserver-slave1.out.4
[root@slave1 ~]# tail -100 /usr/home/hbase-0.98.19-hadoop2/logs/hbase-root-regionserver-slave1.log
at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2197)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:305)
at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2195)
at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:897)
at java.lang.Thread.run(Thread.java:722)
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.ClockOutOfSyncException): org.apache.hadoop.hbase.ClockOutOfSyncException: Server slave1,60020,1469550349546 has been rejected; Reported time is too far out of sync with master. Time difference of 28941955ms > max allowed of 30000ms
at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:358)
at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:251)
at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1410)
at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2197)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1491)
at org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1693)
at org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1760)
at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.regionServerStartup(RegionServerStatusProtos.java:8277)
at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2193)
... 2 more
2016-07-27 00:25:50,680 FATAL [regionserver60020] regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: []
2016-07-27 00:25:50,680 INFO [regionserver60020] regionserver.HRegionServer: STOPPED: Unhandled: org.apache.hadoop.hbase.ClockOutOfSyncException: Server slave1,60020,1469550349546 has been rejected; Reported time is too far out of sync with master. Time difference of 28941955ms > max allowed of 30000ms
at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:358)
at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:251)
at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1410)
at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910)
at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2197)
at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
2016-07-27 00:25:50,680 INFO [regionserver60020] ipc.RpcServer: Stopping server on 60020
2016-07-27 00:25:50,680 INFO [regionserver60020] regionserver.HRegionServer: Stopping infoServer
2016-07-27 00:25:50,682 INFO [regionserver60020] mortbay.log: Stopped [email protected]:60030
2016-07-27 00:25:50,782 INFO [regionserver60020] snapshot.RegionServerSnapshotManager: Stopping RegionServerSnapshotManager abruptly.
2016-07-27 00:25:50,783 INFO [regionserver60020] regionserver.HRegionServer: aborting server null
2016-07-27 00:25:50,783 DEBUG [regionserver60020] catalog.CatalogTracker: Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@242eb7e
2016-07-27 00:25:50,783 INFO [regionserver60020] client.HConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x156262ed7ef000c
2016-07-27 00:25:50,785 INFO [regionserver60020] zookeeper.ZooKeeper: Session: 0x156262ed7ef000c closed
2016-07-27 00:25:50,786 INFO [regionserver60020-EventThread] zookeeper.ClientCnxn: EventThread shut down
2016-07-27 00:25:50,786 INFO [regionserver60020] regionserver.HRegionServer: stopping server null; all regions closed.
2016-07-27 00:25:50,887 INFO [regionserver60020] regionserver.Leases: regionserver60020 closing leases
2016-07-27 00:25:50,887 INFO [regionserver60020] regionserver.Leases: regionserver60020 closed leases
2016-07-27 00:25:50,887 INFO [regionserver60020] regionserver.CompactSplitThread: Waiting for Split Thread to finish...
2016-07-27 00:25:50,888 INFO [regionserver60020] regionserver.CompactSplitThread: Waiting for Merge Thread to finish...
2016-07-27 00:25:50,888 INFO [regionserver60020] regionserver.CompactSplitThread: Waiting for Large Compaction Thread to finish...
2016-07-27 00:25:50,888 INFO [regionserver60020] regionserver.CompactSplitThread: Waiting for Small Compaction Thread to finish...
2016-07-27 00:25:50,896 DEBUG [regionserver60020] zookeeper.RecoverableZooKeeper: Node /hbase/rs/slave1,60020,1469550349546 already deleted, retry=false
2016-07-27 00:25:50,896 WARN [regionserver60020] regionserver.HRegionServer: Failed deleting my ephemeral node
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase/rs/slave1,60020,1469550349546
at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.delete(RecoverableZooKeeper.java:178)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1239)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1228)
at org.apache.hadoop.hbase.regionserver.HRegionServer.deleteMyEphemeralNode(HRegionServer.java:1396)
at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1069)
at java.lang.Thread.run(Thread.java:722)
2016-07-27 00:25:50,900 INFO [regionserver60020] zookeeper.ZooKeeper: Session: 0x156262ed7ef000b closed
2016-07-27 00:25:50,900 INFO [regionserver60020] regionserver.HRegionServer: stopping server null; zookeeper connection closed.
2016-07-27 00:25:50,900 INFO [regionserver60020] regionserver.HRegionServer: regionserver60020 exiting
2016-07-27 00:25:50,900 INFO [regionserver60020-EventThread] zookeeper.ClientCnxn: EventThread shut down
2016-07-27 00:25:50,901 ERROR [main] regionserver.HRegionServerCommandLine: Region server exiting
java.lang.RuntimeException: HRegionServer Aborted
at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66)
at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:85)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2581)
2016-07-27 00:25:50,903 INFO [Thread-9] regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@608916f9
2016-07-27 00:25:50,903 INFO [Thread-9] regionserver.ShutdownHook: Starting fs shutdown hook thread.
2016-07-27 00:25:50,905 INFO [Thread-9] regionserver.ShutdownHook: Shutdown hook finished.
二、问题分析：时间同步上有问题，查了一下RegionServer和Master两台机的时间，果然不一致。由于之前把主节点的时间同步而未同步子节点的时间。
三、解决方案：
同步时间
$hwclock -r
$hwclock -w
$/usr/sbin/ntpdate us.pool.ntp.org
同步操作步骤详见：http://blog.csdn.net/xiaoshunzi111/article/details/52033957