搭建企业级Hbase

搭建Hbase步骤

1.查看linux系统最大进程数和最大文件打开数

ulimit-a

查看open files（最大文件打开数）、max user processes（最大进程数），默认为1024。

2.设置linux系统最大进程数和最大文件打开数

su

vim /etc/security/limits.conf

例如对root和hadoop用户进行配置：

root    soft  nproc  50000

root    hard  nproc  50000

root    soft  nofile  25535

root    hard  nofile  25535

hadoop    soft  nproc  50000

hadoop    hard  nproc  50000

hadoop    soft  nofile  25535

hadoop    hard  nofile  25535

3.调整linux内核参数

vim /etc/sysctl.conf

添加下述配置：

net.ipv4.ip_forward= 0

net.ipv4.conf.default.rp_filter= 1

net.ipv4.conf.default.accept_source_route= 0

kernel.core_users_pid= 1

net.ipv4.tcp_syncookies= 1

net.bridge.bridge-nf-call-ip6tables= 0

net.bridge.bridge-nf-call-iptables= 0

net.bridge.bridge-nf-call-arptables= 0

kernel.mggmnb= 65536

kernel.mggmax= 65536

kernel.shmmax= 68719476736

kernel.shmall= 268435456

net.ipv4.tcp_max_syn_backlog= 65000

net.core.netdev_max_backlog= 32768

net.core.somaxconn= 32768

fs.file-max= 65000

net.core.wmem_default= 8388608

net.core.rmem_default= 8388608

net.core.rmem_max= 16777216

net.core.wmem_max= 16777216

net.ipv4.tcp_timestamps= 1

net.ipv4.tcp_synack_retries= 2

net.ipv4.tcp_syn_retries= 2

net.ipv4.tcp_mem= 94500000 915000000 927000000

net.ipv4.tcp_max_orphans= 3276800

net.ipv4.tcp_tw_reuse= 1

net.ipv4.tcp_tw_recycle= 1

net.ipv4.tcp_keepalive_time= 1200

net.ipv4.tcp_syncookies= 1

net.ipv4.tcp_fin_timeout= 10

net.ipv4.tcp_keepalive_intvl= 15

net.ipv4.tcp_keepalive_probes= 3

net.ipv4.ip_local_port_range= 1024 65535

net.ipv4.conf.eml.send_redirects= 0

net.ipv4.conf.lo.send_redirects= 0

net.ipv4.conf.default.send_redirects= 0

net.ipv4.conf.all.send_redirects= 0

net.ipv4.icmp_echo_ignore_broadcasts= 1

net.ipv4.conf.eml.accept_source_route= 0

net.ipv4.conf.lo.accept_source_route= 0

net.ipv4.conf.default.accept_source_route= 0

net.ipv4.conf.all.accept_source_route= 0

net.ipv4.icmp_ignore_bogus_error_responses= 1

kernel.core_pattern= /tmp/core

vm.overcommit_memory= 1

执行生效：
sysctl -p

4.配置环境

（1）修改hbase-env.sh

cd  ./hbase-1.2.0-cdh5.7.1/conf/

vim hbase-env.sh

具体设置如下：

# 配置Java home
export JAVA_HOME=/home/hadoop/app/jdk1.7.0_79

# 设置HBase内存堆的大小
export HBASE_HEAPSIZE=1024

# 设置HMaster最大可用内存
export HBASE_MASTER_OPTS="-Xmx512m"

# 设置HRegionServer最大可用内存
export HBASE_REGIONSERVER_OPTS="-Xmx512m"

# 设置HBase的日志目录
export HBASE_LOG_DIR=${HBASE_HOME}/logs

# 设置HBase的pid目录
export HBASE_PID_DIR=${HBASE_HOME}/pids

# 使用独立的ZooKeeper集群
export HBASE_MANAGES_ZK=false

（2）配置hbase-site.xml

vim hbase-site.xml

配置如下：

  
  
    hbase.master.distributed.log.splitting
    false
  

  
  
    hbase.rootdir
    hdfs://educluster/hbase
  

    
  
    hbase.cluster.distributed
    true
  

  
  
    hbase.master.port
    60000
  

    
  
    hbase.zookeeper.quorum
    namenode1,namenode2,datanode
  

  
  
    hbase.zookeeper.property.dataDir
    /opt/hadoopdata/zookeeperdata
  

  
  
    hbase.client.scanner.caching
    2000
  

 
  
    hbase.hregion.max.filesize
    10737418240
  
  
  
  
    hbase.master.info.port
    16010
  
  
  
  
    hbase.tmp.dir
    /home/hadoop/app/cdh/hbase-1.2.0-cdh5.7.1/tmp
  
 
  
  
  
    hbase.zookeeper.property.clientPort
    2181
  
 
 
  
  
    hbase.regionserver.reginoSplitLimit
    2000
  
  
  
  
    hbase.hstore.compactionThreshold
    6
  
  
  
  
    hbase.hstore.blockingStoreFiles
    14
  
  
  
  
    hbase.hregion.memstore.block.multiplier
    20
  
  
  
  
    hbase.server.thread.wakefrequency
    500
  
  
  
  
    hbase.zookeeper.property.maxClientCnxns
    2000
  
  
  
  
    hbase.regionserver.global.memstore.lowerLimit
    0.3
  
  
    hbase.regionserver.global.memstore.upperLimit
    0.39
  
  
    hbase.block.cache.size
    0.4
  
  
  
  
    hbase.reginoserver.handler.count
    300
  
  
  
  
    hbase.client.retries.number
    5
  
  
  
  
    hbase.client.pause
    100
  

待续...