Hadoop 安装配置

1.创建用户组和用户

groupadd hadoops  
useradd hadoop -g hadoops
passwd hadoop

2.解压jdk和hadoop

tar -zxvf jdk1.7.0_80.tar.gz -C /usr/local
mv /usr/local/jdk1.7.0_80 /usr/local/java

tar -zxvf hadoop-2.7.3.tar.gz -C /u01
mv /u01/hadoop-2.7.3  /u01/hadoop

chown -R hadoop:hadoop /u01/hadoop/etc/hadoop/**
chown -R hadoop:hadoop /u01/hadoop/bin/**
chown -R hadoop:hadoop /u01/hadoop/sbin/**

3.配置环境变量

vi /etc/profile

export HADOOP_HOME=/u01/hadoop
export PATH=$HADOOP_HOME/bin:$PATH
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

4.配置hosts

vi /etc/hosts

192.168.0.9 hadmaster
192.168.0.27 hadslave1
192.168.0.28 hadslave2

5.创建目录

mkdri /u01/hadoopData
chown hadoop:hadoop /u01/hadoopData

vi /u01/hadoop/etc/hadoop/slaves

hadslave1
hadslave2

vi /u01/hadoop/etc/hadoop/masters
hadmaster

6.配置hadoop

• 6.1.编辑hadoop-env.sh

  vi /u01/hadoop/etc/hadoop/hadoop-env.sh
  
  export JAVA_HOME=/usr/local/java
  export HADOOP_OPTS=-Djava.net.preferIPv4Stack=true
  

• 6.2.编辑core-site.xml

  vi /u01/hadoop/etc/hadoop/core-site.xml
  
  
      hadoop.tmp.dir
      /u01/hadoopData/tmp
  
  
          fs.default.name
          hdfs://hadmaster:9000
  
  
      fs.defaultFS
      hdfs://hadmaster:9000
  
    
      hadoop.proxyuser.hadoop.hosts
      *  
    
    
      hadoop.proxyuser.hadoop.groups  
      *  
  
  

• 6.3.编辑hdfs-site.xml

  vi /u01/hadoop/etc/hadoop/hdfs-site.xml
  
  
      dfs.replication
      2
  
  
      dfs.namenode.data.dir
      file:/u01/hadoopData/data,file:/u01/hadoopData/data2
  
  
      dfs.namenode.name.dir
      file:/u01/hadoopData/name,file:/u01/hadoopData/name2
  
  

• 6.4.编辑mapred-site.xml

  cp /u01/hadoop/etc/hadoop/mapred-site.xml.template  /u01/hadoop/etc/hadoop/mapred-site.xml
  vi /u01/hadoop/etc/hadoop/mapred-site.xml
  
  
      mapred.job.tracker
      hadmaster:9001
  
  
      mapred.local.dir
      /u01/hadoopData/tmp
  
  

• 6.5.编辑yarn-site.xml

  vi /u01/hadoop/etc/hadoop/mapred-site.xml
  
  
      yarn.resourcemanager.resource-tracker.address
      hadmaster:8025
  
  
          yarn.resourcemanager.scheduler.address
          hadmaster:8035
  
  
          yarn.resourcemanager.address
          hadmaster:8050
  
  
          yarn.resourcemanager.webapp.address
          hadmaster:8058
  
  

7.ssh免密码登陆

su - hadoop
cd /home/hadoop
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cp id_rsa.pub authorized_keys
cd /home/hadoop/.ssh
scp authorized_keys hadslave1:/home/hadoop/.ssh
scp authorized_keys hadslave2:/home/hadoop/.ssh
ssh hadmaster 
exit
ssh hadslave1
exit
ssh hadslave2
exit
rsync -avxP /u01/hadoop/ hadoop@hadslave1:/u01/hadoop/
rsync -avxP /u01/hadoop/ hadoop@hadslave2:/u01/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave1:/u01/hadoop/etc/hadoop/
rsync -avxP /u01/hadoop/etc/hadoop/ hadoop@hadslave2:/u01/hadoop/etc/hadoop/

8.hive元数据

create database hive_meta; 
create user 'hive' identified by 'hive'; 
grant all privileges on hive_meta.* to
'hive'@'5'
identified by 'hive' WITH GRANT OPTION; 
flush privileges; 

9.hive环境变量

tar -zxvf apache-hive-2.1.0-bin.tar.gz -C /u01
mv /u01/apache-hive-2.1.0-bin /u01/hive
chown hadoop:hadoop  /u01/hive
vi /etc/profile
export HIVE_HOME=/u01/hive
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile

10.hive配置

• 10.1.copy配置文件

  cd /u01/hive/conf
  cp hive-env.sh.template hive-env.sh
  cp hive-default.xml.template hive-site.xml  
  cp hive-log4j2.properties.template hive-log4j2.properties
  cp hive-exec-log4j2.properties.template hive-exec-log4j2.properties
  

• 10.2.配置hive-env.sh

  vi hive-env.sh
  export JAVA_HOME=/usr/local/java
  export HADOOP_HOME=/u01/hadoop
  export HIVE_HOME=/u01/hive
  export HIVE_CONF_DIR=/u01/hive/conf
  

• 10.3.Mysql驱动

  cp  mysql-connector-java-5.1.39.jar $HIVE_HOME/lib
  chown hadoop:hadoop $HIVE_HOME/lib/**
  

• 10.4.配置hive-site.xml

  ##替换hive-site.xml文件中的 ${system:java.io.tmpdir} 和 ${system:user.name}
  
      hive.exec.scratchdir
      /tmp/hive-${user.name}
      HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. For each
          connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/ is created, with
          ${hive.scratch.dir.permission}.
      
  
  
      hive.exec.local.scratchdir
      /tmp/${user.name}
      Local scratch space for Hive jobs
  
  
      hive.downloaded.resources.dir
      /tmp/hive/resources
      Temporary local directory for added resources in the remote file system.
  
  
      hive.querylog.location
      /tmp/${user.name}
      Location of Hive run time structured log file
  
  
      hive.server2.logging.operation.log.location
      /tmp/${user.name}/operation_logs
      Top level directory where operation logs are stored if logging functionality is enabled
      
  
  

  ##配置Hive Metastore
  
      javax.jdo.option.ConnectionURL
      jdbc:mysql://192.168.0.7:3306/hive_meta?createDatabaseIfNotExist=true&characterEncoding=UTF-8&useSSL=false
  
  
      javax.jdo.option.ConnectionDriverName
      com.mysql.jdbc.Driver
  
  
      javax.jdo.option.ConnectionUserName
      hive
  
  
      javax.jdo.option.ConnectionPassword
      hive
  
  

11.hdfs目录

hdfs dfs -mkdir /tmp
hdfs dfs -mkdir -p /user/hive/warehouse
hdfs dfs -chmod g+w /tmp
hdfs dfs -chmod g+w /user/hive/warehouse

12.初始化hive元数据

cd /u01/hive/bin
./schematool -dbType mysql -initSchema

13.测试hive

hive
show tables

14.连接hive

hive --service metastore >/dev/null 2>/dev/null &
hive --service hiveserver2 10000 >/dev/null 2>/dev/null &

$HIVE_HOME/bin/hive --service hiveserver2 10000 >/dev/null 2>/dev/null &
连接Hive JDBC URL：jdbc:hive2://192.168.6.116:10000/default     （Hive默认端口：10000  默认数据库名：default）

beeline
!connect jdbc:hive2://localhost:10000 hadoop hadoop

15.hiveServer2WebUI

    hive.server2.webui.host
    192.168.0.9



    hive.server2.webui.port
    10002

git仓库地址