join大表和小表对查询性能的影响和索引类型的选择

join大表和小表对查询性能的影响和索引类型的选择

示例数据库采用PostgreSQL

1.创建数据表和相关函数

drop function if exists gen_random_gps(bigint);
drop function if exists gps_save(bigint,bigint,timestamptz,geometry);
drop function if exists gen_random_plate(int,int);
drop function if exists gen_random_az(int);
drop function if exists gen_random_num(int);
drop table if exists  vehgps;
drop table if exists  vehicles;

/****************************************************************************************
    车辆基本信息表
drop table if exists  vehicles;
****************************************************************************************/
create table vehicles(
    objectid int not null,                       --唯一编号
    numplate text not null,                      --车牌号
    describe text,                               --备注
  generate timestamptz default now() not null, --注册时间
    constraint pk_vehicles_objectid primary key (objectid)
);
create index idx_vehicles_numplate on vehicles(numplate);
/****************************************************************************************
    车辆GPS轨迹表
drop table if exists  vehiclegps;
****************************************************************************************/
create table if not exists vehiclegps(
    objectid bigint not null,                           --唯一编号
    vehid int not null,                                 --车辆编号,外键
    generate timestamptz default now() not null,        --GPS时间,实际应用应按此字段分表
    constraint pk_vehiclegps_objectid primary key (objectid),
    constraint fk_vehiclegps_vehid foreign key(vehid)  references vehicles(objectid) on delete cascade
);
create index idx_vehiclegps_vehid on vehiclegps(vehid); --车辆编号
select AddGeometryColumn ('vehiclegps','geom',4326,'POINT',2); -- GPS位置,GPS采用4326 ,类型为点,二维坐标
create index idx_vehiclegps_generate on vehiclegps using brin(generate)  with (pages_per_range='1');
/****************************************************************************************
    车辆基本信息函数
drop function if exists gen_random_plate(int,int);
drop function if exists gen_random_az(int);
drop function if exists gen_random_num(int);
****************************************************************************************/
--生成指定位数的数字
create or replace function gen_random_num(int)
    returns text
as $$
	select string_agg(chr((random()*(57-48)+48 )::integer) , '')  from generate_series(1,$1);
$$ language sql;
--生成指定位数的大写A-Z
create or replace function gen_random_az(int)
    returns text
as $$
	select string_agg(chr((random()*(90-65)+65 )::integer) , '')  from generate_series(1,$1);
$$ language sql;
--随机生成车牌号
create or replace function gen_random_plate(int default 1,int default 5)
    returns text
as $$
	with cte as(
		select array['浙','粤','京','津','冀','晋','蒙','辽','黑','沪','吉','苏','皖','赣','鲁','豫','鄂','湘','桂','琼','渝','川','贵','云','藏','陕','甘','青','宁'] as provinces
	),provinces as(
		select array_to_string(array[provinces[(random()*(29-1)+1)::integer],gen_random_az($1)],'') as pro,
				gen_random_num($2) as num
			from cte
	) select array_to_string(array[pro,num],'-') from provinces
$$ language sql;
/****************************************************************************************
    随机插入车辆GPS轨迹函数,调用一次插件1千条记录,gps时间控制在2018-01-01至2018-04-16之间
drop function if exists gen_random_gps(bigint);
drop function if exists gps_save(bigint,bigint,timestamptz,geometry);
****************************************************************************************/
create or replace function gen_random_gps(bigint)
    returns table(objectid bigint,vehid bigint,generate timestamptz,geom geometry)
as $$
	with cte as(
		select start,(start + 999) as end,
			extract(epoch from '2018-01-01'::timestamptz) as sdate,
			extract(epoch from '2018-04-16'::timestamptz) as edata,
			(select array[min(objectid),max(objectid)] from vehicles) as vehid			
		from (select $1 as start) as tmp
	),points as(
		select id,
			((random()*(vehid[2]-vehid[1])+vehid[1])::bigint) as vehid, --车辆id是连续的才这么干
			to_timestamp((random()*(edata-sdate)+sdate)) as gpstime,
			ST_SetSRID(ST_Point(
				round((random()*(135.085831-73.406586)+73.406586)::numeric,6),
				round((random()*(53.880950-3.408477)+3.408477)::numeric,6)
			),4326) as geom
		from cte,generate_series(cte.start,cte.end) as id
	)select * from points
$$ language sql;
--保存GPS轨迹
create or replace function gps_save(bigint,bigint,timestamptz,geometry)
    returns bigint
as $$
	insert into vehiclegps(objectid,vehid,generate,geom) values($1,$2,$3,$4) returning objectid;
$$ language sql;

2.插入测试数据

/****************************************************************************************
    随机插入车辆基本信息测试数据(50W)
****************************************************************************************/
insert into vehicles  select id, gen_random_plate() from generate_series(1,500000) as id;
/****************************************************************************************
    随机插入车辆GPS轨迹测试数据(10000000W)
****************************************************************************************/
do $$
	declare
		v_id bigint;
	begin
		for i in 1..10000 loop
			select max(id) into v_id from (select gps_save(objectid,vehid,generate,geom) as id from gen_random_gps((i-1)*1000 + 1)) as tmp;
			raise notice  '%', v_id;
		end loop;
	end;
$$;

为了不影响执行计划数据插入完成后运行vacuum

vacuum full freeze VERBOSE vehicles;
vacuum full freeze VERBOSE vehiclegps;

3.在本案例中b树索引和brin范围索引比较

3.1 测试brin类型索引

在本案例中,因为单位时间内的gps数据非常密集,brin类型索引只有在一个非常小的范围内(大约3小时以内的数据,返回的数据约为14000行,执行时间约为360ms,以实测为准,误差基本不会太大)才有效(注意pages_per_range已经设置为1).

drop index if exists  idx_vehiclegps_generate;
create index idx_vehiclegps_generate on vehiclegps using brin(generate) with (pages_per_range='1');
--测试获取3小时25分内的数据
explain (analyze,verbose,costs,buffers,timing)
select * from vehiclegps
where generate>=('2018-01-01 12:00:00'::timestamptz) and generate<('2018-01-01 15:25:00'::timestamptz)
--测试获取一天内的数据
explain (analyze,verbose,costs,buffers,timing)
select * from vehiclegps
where generate>=('2018-01-01 12:00:00'::timestamptz) and generate<('2018-01-01 15:25:00'::timestamptz)

3.2 测试B树索引

在本案例中,B对索引表现优于brin.

drop index if exists  idx_vehiclegps_generate;
create index idx_vehiclegps_generate on vehiclegps(generate);
--测试获取3小时25分内的数据
explain (analyze,verbose,costs,buffers,timing)
select * from vehiclegps
where generate>=('2018-01-01 12:00:00'::timestamptz) and generate<('2018-01-01 15:25:00'::timestamptz)
--测试获取一天内的数据
explain (analyze,verbose,costs,buffers,timing)
select * from vehiclegps
where generate>=('2018-01-01 12:00:00'::timestamptz) and generate<('2018-01-01 15:25:00'::timestamptz)

本章小结

• 任何事物都有适用范围,实际如何使用要根据业务需求测试后选择.
  
• brin类型的索引适用于数据比较稀疏,跨度比较大的应用
  
• B树索引适用于大多数应用
  

4.join查询

根据第3章的结论,后面的示例采用B树索引.

4.1 大表join小表

/****************************************************************************************
    大表join小表
****************************************************************************************/
explain (analyze,verbose,costs,buffers,timing)
select gps.* from vehiclegps as gps
inner join vehicles as veh on veh.objectid=gps.vehid
where gps.generate>=('2018-01-01'::timestamptz) and gps.generate<('2018-01-02'::timestamptz)

4.2 小表join大表

explain (analyze,verbose,costs,buffers,timing)
select gps.* from vehicles as veh
inner join vehiclegps as gps on gps.vehid=veh.objectid
where gps.generate>=('2018-01-01'::timestamptz) and gps.generate<('2018-01-02'::timestamptz)

本章小结

• 基本没有区别.