hadoop3.1.3-spark-2.4.6-bin-hadoop2.7-hive-3.1.2单机版本安装

环境：centos7.4

vim /etc/profile

export JAVA_HOME=/usr/java/java8
export JRE_HOME=/usr/java/java8/jre
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HIVE_HOME=/usr/java/hive
export HBASE_HOME=/usr/java/hbase
export SPARK_HOME=/usr/java/spark
export SCALA_HOME=/usr/java/scala
export HADOOP_HOME=/usr/java/hadoop
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$HADOOP_HOME/bin:$SCALA_HOME/bin:$HADOOP_HOME/sbin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin

一、安装hadoop

修改配置文件：core-site.xml

       
		fs.defaultFS
		hdfs://bigdata:8020
	
	
	
		hadoop.tmp.dir
		/usr/java/hadoop/datas/tmp
	
    
	
		io.file.buffer.size
		8192
	
    
	
		fs.trash.interval
		10080
	
	
		hadoop.proxyuser.hadoop.hosts
		*
	
	
		hadoop.proxyuser.hadoop.groups
		root
	

修改hdfs-site.xml

	
		dfs.namenode.name.dir
		file:///usr/java/hadoop/datas/datas/namenode/namenodedatas
	
	
		dfs.blocksize
		134217728
	
	
		dfs.namenode.handler.count
		10
	
	
		dfs.datanode.data.dir
		file:///usr/java/hadoop/datas/datas/datanode/datanodeDatas
	
	
		dfs.namenode.http-address
		bigdata:50070
	
	
		dfs.replication
		1
	
	
		dfs.permissions.enabled
		false
	
	
		dfs.namenode.checkpoint.edits.dir
		file:///usr/java/hadoop/datas/dfs/nn/snn/edits
	
	
		dfs.namenode.secondary.http-address
		bigdata:50090
	
	
		dfs.namenode.edits.dir
		file:///usr/java/hadoop/datas/datas/dfs/nn/edits
	
	
		dfs.namenode.checkpoint.dir
		file:///usr/java/hadoop/datas/datas/dfs/snn/name
	

修改mapred-site.xml

	
		mapreduce.framework.name
		yarn
	
	
		mapreduce.map.memory.mb
		1024
	
	
		mapreduce.map.java.opts
		-Xmx512M
	
	
		mapreduce.reduce.memory.mb
		1024
	
	
		mapreduce.reduce.java.opts
		-Xmx512M
	
	
		mapreduce.task.io.sort.mb
		256
	
	
		mapreduce.task.io.sort.factor
		100
	
	
		mapreduce.reduce.shuffle.parallelcopies
		25
	
	
		mapreduce.jobhistory.address
		bigdata:10020
	
	
		mapreduce.jobhistory.webapp.address
		bigdata:19888
	
	
		mapreduce.jobhistory.intermediate-done-dir
		/usr/java/hadoop/datas/jobhsitory/intermediateDoneDatas
	
	
		mapreduce.jobhistory.done-dir
		/usr/java/hadoop/datas/jobhsitory/DoneDatas
	
	
	  yarn.app.mapreduce.am.env
	  HADOOP_MAPRED_HOME=/usr/java/hadoop
	
	
	  mapreduce.map.env
	  HADOOP_MAPRED_HOME=/usr/java/hadoop/
	
	
	  mapreduce.reduce.env
	  HADOOP_MAPRED_HOME=/usr/java/hadoop
	

修改yarn-site.xml

	
		dfs.namenode.handler.count
		100
	
	
		yarn.log-aggregation-enable
		true
	
	
		yarn.resourcemanager.address
		bigdata:8032
	
	
		yarn.resourcemanager.scheduler.address
		bigdata:8030
	
	
		yarn.resourcemanager.resource-tracker.address
		bigdata:8031
	
	
		yarn.resourcemanager.admin.address
		bigdata:8033
	
	
		yarn.resourcemanager.webapp.address
		bigdata:8088
	
	
		yarn.resourcemanager.hostname
		bigdata
	
	
		yarn.scheduler.minimum-allocation-mb
		1024
	
	
		yarn.scheduler.maximum-allocation-mb
		2048
	
	
		yarn.nodemanager.vmem-pmem-ratio
		2.1
	
	
	
		yarn.nodemanager.vmem-check-enabled
		false
	
	
		yarn.nodemanager.resource.memory-mb
		1024
	
	
		yarn.nodemanager.resource.detect-hardware-capabilities
		true
	
	
		yarn.nodemanager.local-dirs
		file:///usr/java/hadoop/datas/nodemanager/nodemanagerDatas
	
	
		yarn.nodemanager.log-dirs
		file:///usr/java/hadoop/datas/nodemanager/nodemanagerLogs
	
	
		yarn.nodemanager.log.retain-seconds
		10800
	
	
		yarn.nodemanager.remote-app-log-dir
		/usr/java/hadoop/datas/remoteAppLog/remoteAppLogs
	
	
		yarn.nodemanager.remote-app-log-dir-suffix
		logs
	
	
		yarn.nodemanager.aux-services
		mapreduce_shuffle
	
	
		yarn.log-aggregation.retain-seconds
		18144000
	
	
		yarn.log-aggregation.retain-check-interval-seconds
		86400
	
	
	
        yarn.app.mapreduce.am.resource.mb
        1024
	

二、安装Hive

(1) 复制mysql驱动程序到hive的lib目录下。

(2) 配置hive-site.xml

vim hive-site.xml

	javax.jdo.option.ConnectionPassword
	root
	password to use against metastore database


	javax.jdo.option.ConnectionUserName
	root
	Username to use against metastore database


	javax.jdo.option.ConnectionURL
	jdbc:mysql://127.0.0.1:3306/hive


	javax.jdo.option.ConnectionDriverName
	com.mysql.jdbc.Driver
	Driver class name for a JDBC metastore

(3) 在msyql中创建存放hive信息的数据库
create database if not exists hive;
insert into mysql.user(Host,User,Password)values('%','hadoop',password('hadoop'));
grant all privileges on hivedb.* to 'hadoop'@'%' identified by 'hadoop';
(4) 初始化hive的元数据(表结构)到mysql中
cd /soft/hive/bin
schematool -dbType mysql -initSchema

不同版本的hadoop或hive里面的guava包版本都不同，重点是hadoop里的包比hive里的版本高。
解决问题：
删除hive中低版本的guava包，把hadoop里的复制到hive的lib目录下即可。

三、安装spark

vim spark-env.sh
export SCALA_HOME=/usr/java/scala
export JAVA_HOME=/usr/java/java8
export HADOOP_HOME=/usr/java/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_LOG_DIR=/usr/java/spark/logs
export SPARK_PID_DIR=/usr/java/spark
export SPARK_DRIVER_MEMORY=512M
export SPARK_MASTER_HOST=bigdata

vim spark-defaults.conf
spark.yarn.jars=hdfs://ns/spark-lib/*.jar
spark.yarn.am.memory=512M
spark.driver.memory=512M
spark.executor.memory=512M
spark.driver.extraClassPath=/usr/java/spark/lib/*