openEuler 部署 hive3

openEuler 部署 hive3

启动Hadoop

# 启动hadoop
start-all.sh
# 检查hadoop进程
jps
# 检查各端口
netstat -aplnt | grep java

检查MySQL是否启动成功

ps -aux | grep mysql
netstat -aplnt | grep 3306

安装hive

# 将软件上传到 /opt/soft 目录
# 解压hive
tar -zxvf apache-hive-3.1.3-bin.tar.gz 
# 目录改名
mv apache-hive-3.1.3-bin hive3
# 进入配置文件目录
cd /opt/soft/hive3/conf
# 复制配置文件
cp hive-env.sh.template  hive-env.sh
cp hive-default.xml.template  hive-site.xml
# 编辑环境配置文件
vim hive-env.sh
# 编辑配置文件
vim hive-site.xml

hive-env.sh

# hadoop 安装路径
export HADOOP_HOME=/opt/soft/hadoop3/
# hive 配置文件路径
export HIVE_CONF_DIR=/opt/soft/hive3/conf/

hive-site.xml

需要修改的位置提炼如下：

<configuration>
    
    <property>
        <name>javax.jdo.option.ConnectionURLname>
        <value>jdbc:mysql://spark03:3306/hive?useUnicode=true&createDatabaseIfNotExist=true&characterEncoding=UTF8&useSSL=false&serverTimeZone=Asia/Shanghaivalue>
    property>

    
    <property>
        <name>javax.jdo.option.ConnectionDriverNamename>
        <value>com.mysql.cj.jdbc.Drivervalue>
    property>

    
    <property>
        <name>javax.jdo.option.ConnectionUserNamename>
        <value>rootvalue>
    property>
    <property>
        <name>javax.jdo.option.ConnectionPasswordname>
        <value>Lihaozhe!!@@1122value>
    property>

    <property>
        <name>hive.metastore.warehouse.dirname>
        <value>/user/hive/warehousevalue>
    property>

    <property>
        <name>hive.exec.scratchdirname>
        <value>/user/hive/tmpvalue>
    property>
    
    property>

    <property>
        <name>hive.exec.local.scratchdirname>
        <value>/user/hive/localvalue>
        <description>Local scratch space for Hive jobsdescription>
    property>

    <property>
        <name>hive.downloaded.resources.dirname>
        <value>/user/hive/resourcesvalue>
        <description>Temporary local directory for added resources in the remote file system.description>
    property>

    
    <property>
        <name>hive.querylog.locationname>
        <value>/user/hive/logvalue>
    property>

    
    <property>
        <name>hive.metastore.urisname>
        <value>thrift://spark01:9083value>
    property>

    
    <property> 
        <name>hive.server2.thrift.portname> 
        <value>10000value>
    property>
    <property> 
        <name>hive.server2.thrift.bind.hostname> 
        <value>0.0.0.0value>
    property>
    <property>
        <name>hive.server2.webui.hostname>
        <value>0.0.0.0value>
    property>

    
    <property>
        <name>hive.server2.webui.portname>
        <value>10002value>
    property>

    <property> 
        <name>hive.server2.long.polling.timeoutname> 
        <value>5000value>                               
    property>

    <property>
        <name>hive.server2.enable.doAsname>
        <value>truevalue>
    property>
    
    <property>
        <name>hive.execution.enginename>
        <value>mrvalue>
    property>
    <property>
        <name>hive.metastore.schema.verificationname>
        <value>falsevalue>
        <description>
          Enforce metastore schema version consistency.
          True: Verify that version information stored in is compatible with one from Hive jars.  Also disable automatic
                schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
                proper metastore schema migration. (Default)
          False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
        description>
      property>
configuration>

注意：上面配置文件中的路径在 vi 编辑器下 全局替换

:%s@\${system:java.io.tmpdir}@/tmp/hive-logp@g

不要使用图形化 不然每次保存后3215行都会有个  特殊字符 如果产生删除即可 具体报错信息 后面有单独的描述

上传 MySQL 连接驱动 jar 包到 hive 安装目录的lib目录下:

/opt/soft/hive3/lib

jar 包有两个 分别为：

• mysql-connector-java-8.0.32.jar
• protobuf-java-3.22.2.jar

删除原有的 protobuf-java-2.5.0.jar 文件

guava版本冲突

删除 hive/lib目录中的 guava-19.0.jar

拷贝hadoop/share/hadoop/common/lib目录中的 guava-27.0-jre.jar 到 hive/lib 目录

rm -f /opt/soft/hive3/lib/guava-19.0.jar
cp -v /opt/soft/hadoop3/share/hadoop/common/lib/guava-27.0-jre.jar /opt/soft/hive3/lib

配置环境变量

vim /etc/profile

export HIVE_HOME=/opt/soft/hive3
export PATH=$PATH:$HIVE_HOME/bin

source /etc/profile

初始化hive的元数据库

注意：初始初始元素中库之前 保证 hadoop 和 mysql 正常启动

schematool -initSchema -dbType  mysql

报错解解决：

Exception in thread "main" java.lang.RuntimeException: com.ctc.wstx.exc.WstxParsingException: Illegal character entity: expansion character (code 0x8
at [row,col,system-id]: [3215,96,"file:/usr/local/hive/conf/hive-site.xml"]
at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:3051)
...
at org.apache.hadoop.util.RunJar.main(RunJar.java:236)
Caused by: com.ctc.wstx.exc.WstxParsingException: Illegal character entity: expansion character (code 0x8
at [row,col,system-id]: [3215,96,"file:/usr/local/hive/conf/hive-site.xml"]
at com.ctc.wstx.sr.StreamScanner.constructWfcException(StreamScanner.java:621)
...
at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:3034)
... 17 more

报错原因：

hive-site.xml配置文件中，3215行（见报错记录第二行）有特殊字符

解决办法：

进入hive-site.xml文件，跳转到对应行，删除里面的 &#8 特殊字符即可

Caused by: java.net.URISyntaxException: Relative path in absolute URI: ${system:java.io.tmpdir%7D/$%7Bsystem:user.name%7D
	at java.net.URI.checkPath(URI.java:1822)
	at java.net.URI.<init>(URI.java:745)
	at org.apache.hadoop.fs.Path.initialize(Path.java:260)

解决方案：将hive-site.xml配置文件的
hive.querylog.location
hive.exec.local.scratchdir
hive.downloaded.resources.dir
三个值（原始为$标识的相对路径）写成绝对值

# 全局替换
:%s@\${system:java.io.tmpdir}@/tmp/hive-logp@g

远程模式

# 启动服务端
hive --service metastore &
hive --service hiveserver2 &


hiveserver2 start
nohup hiveserver2 >/dev/null 2>&1 &

# 客户端连接
hive
beeline -u jdbc:hive2://spark01:10000 -n root
beeline jdbc:hive2://spark01:10000> show databases;

体验

use default;
create table person (
	id int,
    phonenum bigint,
    salary double,
    name string
);
create table ps (
	id int,
    phonenum bigint,
    salary double,
    name string
);
show tables;
insert into person values (1001,13966668888,9999.99,"张三");

songsong,bingbing_lili,xiao song:18_xiaoxiao song:19
longlong,pingping_liuliu,xiao long:8_xiaoxiao long:9

drop table person;
create table person (
    name string,
    friends array,
    childrens map
)
 row format delimited fields terminated by ',' 
 collection items terminated by '_' 
 map keys terminated by ':' 
 lines terminated by '\n';

load data local inpath  '/root/person.txt' into table person; 

drop table data;
create table data (
    name string, 
   	amount int
)
 row format delimited fields terminated by ',' 
 lines terminated by '\n';
 load data local inpath  '/root/data.txt' into table data; 

select count(*) from data;
select count(*) from data group by name;
select name,max(t) from data group by name;
select name,max(t) from data group by name order by max(t) ;