在我们日常的工作中,肯定不会一条一条的使用SQL去取出你想要的数据,这个时候我们就可以用Linux中的定时器Shell脚本。我们可以编写一个定时任务,一小时或者一天将SQL写入脚本,让SQL自动执行然后取出数据即可,这样我们就可以实现了数据的定时调度。
create external table students_pt1
(
id bigint
,name string
,age int
,gender string
,clazz string
)
PARTITIONED BY(pt string)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
LOCATION '/student/input1';
alter table students_pt1 add partition(pt='20220220');
alter table students_pt1 add partition(pt='20220219');
alter table students_pt1 add partition(pt='20220218');
alter table students_pt1 add partition(pt='20220221');
alter table students_pt1 add partition(pt='20220222');
alter table students_pt1 add partition(pt='20220223');
alter table students_pt1 add partition(pt='20220224');
show partitions students_pt1;
insert into table students_pt1 partition(pt='20220220') select * from student1;
load data local inpath '/usr/local/soft/data/students.txt' into table students_pt1 partition(pt='20200221');
hive -e "select * from test1.students limit 10"
在/usr/local/soft/目录下创建scripts目录,并创建stu.sql文件
cd /usr/local/soft/
mkdir scripts
vim stu.sql
在stu.sql里编辑一条SQL(这里写最简单的),注意这里不带分号;
select * from test1.students limit 10
[root@master scripts]# hive -f stu.sql
vim hql.sh
#!/bin/sh
#date="2022-02-22"
date=$(date "+%Y%m%d")
#sql1="select * from test1.students_pt1 where pt='${date}'"
#hive -e "${sql1}"
sed -i "s/!everydate!/${date}/" /usr/local/soft/scripts/stu_pt.sql
cat /usr/local/soft/scripts/stu_pt.sql
hive -f /usr/local/soft/scripts/stu_pt.sql
sed -i "s/${date}/!everydate!/" /usr/local/soft/scripts/stu_pt.sql
cat /usr/local/soft/scripts/stu_pt.sql
chmod a+x hql.sh
vim stu_pt.sql
select * from test1.students_pt where pt='!everydate!';
[root@master scripts]# mkdir logs
crontab -e
*/1 * * * * /usr/local/soft/scripts/hql.sh >> /usr/local/soft/scripts/logs/1.log
sh hql.sh
或
./hql.sh
cd /usr/local/soft/scripts/logs/
cat 1.log
到底啦!关注靓仔学习更多的大数据技术!( •̀ ω •́ )✧