Oracle-大表改造分区表实施步骤

前言：

对于业务交易繁忙的数据库，在运行了一定时间后往往会产生一些数据量较大的表，特别是对于每天新增数据较多的日志表或者流水表，单表数据量通常会达到几十G，甚至是上百G。大表对于日常的运维非常的不方便，特别是在表数据的清理、迁移，查询性能会随着数据量的增大而受到影响，因此，通常对于大表我们需要进行优化拆分，在Oracle数据库，比较常见的大表优化拆分是将大表改造为分区表，这种方式最主要的好处就是对于应用透明，应用层面的改动很少。

本文接下来的内容将主要讲述Oracle大表改造分区表的步骤，主要有通过expdp/impdp进行非在线的改造方式以及通过dbms_redefinition进行在线的改造方式

 

expdp/impdp方式

首先获取表的表结构，索引DDL信息

---获取源表的表结构
set longc 9999
set long 99999
set linesize 400
set pagesize 400
select dbms_metadata.get_ddl('TABLE','TAB','TEST') from dual;
------
  CREATE TABLE "TEST"."TAB"
   (  "BEGNDT" DATE NOT NULL ENABLE,
  "OVERDT" DATE NOT NULL ENABLE,
  "CORPCODE" VARCHAR2(32) NOT NULL ENABLE,
  "PRODCD" VARCHAR2(32) NOT NULL ENABLE,
  "SRCSYS" VARCHAR2(64) NOT NULL ENABLE,
  "ACCT_SERNO" VARCHAR2(64) NOT NULL ENABLE,
  "ACCT_NUM" VARCHAR2(64),
  "TACCTNO" VARCHAR2(64),
  "CUSTNO" VARCHAR2(64)
   ) SEGMENT CREATION IMMEDIATE
  PCTFREE 10 PCTUSED 40 INITRANS 1 MAXTRANS 255
 NOCOMPRESS LOGGING
  STORAGE(INITIAL 41943040 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
  PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
  BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
  TABLESPACE "TEST"
---获取源表索引结构
select owner,index_name,index_type
from dba_indexes
where table_owner='TEST' and table_name='TAB';
OWNER             INDEX_NAME          INDEX_TYPE
------------------------------ ------------------------------ ---------------------------
TEST             IDX_TAB01        NORMAL
​
set longc 9999
set long 99999
set linesize 400
set pagesize 400
select dbms_metadata.get_ddl('INDEX','IDX_TAB01','TEST') from dual;
​
  CREATE INDEX "TEST"."IDX_TAB01" ON "TEST"."TAB" ("ACCT_NUM")
  PCTFREE 10 INITRANS 2 MAXTRANS 255 COMPUTE STATISTICS
  STORAGE(INITIAL 65536 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
  PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
  BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
  TABLESPACE "TEST"
;

获取表的注释、授权，结构、约束、依赖对象，这些需要保证改造前后一致

---查看表信息
set linesize 400
select owner,table_name,NUM_ROWS,tablespace_name,degree,partitioned,temporary,row_movement,iot_type
from dba_tableswhere owner='TEST' and table_name='TAB'
​
OWNER             TABLE_NAME      NUM_ROWS TABLESPACE_NAME    DEGREE           PAR T ROW_MOVE IOT_TYPE
------------------------------ ------------------------------ ---------- ------------------------------ ---------------------------------------- --- - -------- ------------
TEST             TAB      45583680 TEST          1         NO  N DISABLED
​
---获取表,列注释
select 'comment on table '||owner||'.'||table_name||' is '''||comments||''';'
from dba_tab_comments
where owner='TEST' and table_name='TAB';
​
select 'comment on column '||owner||'.'table_name||'.'||column_name||' is '''||comments||''';'
from dba_col_comments
where owner='TEST' and table_name='TAB';
​
---获取表的授权
---查看表的授权
select grantor,grantee,PRIVILEGE,owner,table_name
from dba_tab_privs
where table_name ='TAB' and owner='TEST';
​
select 'grant '||privilege||' on '||owner||'.'||table_name||' to '||grantee||';'  
from dba_tab_privs
where table_name ='TAB' and owner='TEST';
----查询当前对象状态数量(改造完之后状态要一致)
select owner,status,count(*)
from dba_objects
WHERE OWNER='TEST'
group by owner,status
order by 1,2;
​
----查询表被依赖的对象(改造完之后需要重新编译)
select a.owner,a.name,a.type,a.REFERENCED_OWNER,a.REFERENCED_NAME,b.status  from dba_DEPENDENCIES a,dba_objects b 
where a.REFERENCED_OWNER='TEST' and a.REFERENCED_NAME='TAB' and a.NAME=b.object_name;
----查询表的约束(确保改造数量，状态前后一致)
select owner,table_name,constraint_name,constraint_Type,status from dba_constraints  where owner='TEST' and table_name='TAB'

确认至少有一倍剩余空间可以存放

----确认表，索引所在的表空间有一倍空间进行存放
with temp_seg as (
select owner,segment_name
from dba_lobs
where owner='TEST' and table_name='TAB'
union
select owner,segment_name
from dba_segments
where owner='TEST' and  segment_name='TAB'
union
select owner,index_name
from dba_indexes
where table_owner='TEST' and table_name='TAB'
)
select a.tablespace_name,sum(a.bytes)/1024/1024
from dba_segments a,temp_seg b
where a.owner =b.owner and a.segment_name=b.segment_name
group by a.tablespace_name;

查看源表的最大，最小日期数据

alter session set nls_date_format='yyyy-mm-dd hh24:mi:ss';
select min(BEGNDT),max(BEGNDT) from TEST.TAB;
​
MIN(BEGNDT)      MAX(BEGNDT)
------------------- -------------------
2021-05-28 00:00:00 2021-09-06 00:00:00

创建新的分区

----创建新的分区表tab_partion
----注意初始化分区要尽可能的小于当前最早的数据日期(因为如果后面插入的数据小于初始化分区，不会新建对应区间的新分区，会直接存放在初始化的分区里面，自动新建分区只作用于大于初始化分区的数据)
----采用自动分区，按天，对于新的数据，如果所在的区间分区(当前为天)不存在,则会自动新建分区(
----如插入2021-01-02 07:00:00则会新建分区less than 2021-01-03,如果后面继续插入2020-12-24 07:00:00则会新建分区less than 2021-12-25，按天去匹配创建，不管数据插入的日期前后，只要大于初始化分区即可自动创建)
----注意，约束，索引名字要修改，避免重复，后面再rename就行
create table TEST.TAB_PART
   ("BEGNDT" DATE NOT NULL ENABLE,
  "OVERDT" DATE NOT NULL ENABLE,
  "CORPCODE" VARCHAR2(32) NOT NULL ENABLE,
  "PRODCD" VARCHAR2(32) NOT NULL ENABLE,
  "SRCSYS" VARCHAR2(64) NOT NULL ENABLE,
  "ACCT_SERNO" VARCHAR2(64) NOT NULL ENABLE,
  "ACCT_NUM" VARCHAR2(64),
  "TACCTNO" VARCHAR2(64),
  "CUSTNO" VARCHAR2(64)
   ) SEGMENT CREATION IMMEDIATE
  PCTFREE 10 PCTUSED 40 INITRANS 1 MAXTRANS 255
 NOCOMPRESS LOGGING
  STORAGE(INITIAL 41943040 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
  PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
  BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
  TABLESPACE "TEST" 
partition by range(BEGNDT)
interval (numtodsinterval(1,'day')) store in (TEST)
(partition p_2000 values less than (to_date('2021-01-01','yyyy-mm-dd')) tablespace TEST);

确认应用没有访问，设置表为只读模式。

alter table TEST.TAB read only;

expdp导出表数据

create directory part_expdp as '/home/oracle/part_backup';
expdp \" /  as sysdba \" directory=part_expdp dumpfile=TEST_TAB_20210907_%u.dmp tables=TEST.TAB logfile=TEST_TAB_20210907.log CONTENT=all exclude=statistics cluster=n parallel=8

impdp导入数据

impdp \" /  as sysdba \" directory=part_expdp dumpfile=TEST_TAB_20210907_%u.dmp remap_table=TAB:TAB_PART logfile=imp_TEST_TAB_20210907.log CONTENT=data_only  cluster=n parallel=8

对新分区表进行统计信息收集

---统计信息收集
begin
  DBMS_STATS.GATHER_TABLE_STATS(ownname=>'TEST',tabname=>'TAB_PART',ESTIMATE_PERCENT=>20,method_opt=>'for all columns size 1',cascade=>true,force=>true,degree=>8,no_invalidate=>FALSE);
end;
/

数据抽查检验

---查看分区数量,以及数据分布
set linesize 300
set pagesize 300
col table_owner for a10
col table_name for a30
WITH PART_TEMP AS 
(SELECT TABLE_OWNER,TABLE_NAME,PARTITION_POSITION
FROM DBA_TAB_PARTITIONS
where table_owner='TEST' and table_name='TAB_PART')
select d.table_owner,d.table_name,d.partition_name,d.high_value,d.NUM_ROWS
FROM DBA_TAB_PARTITIONS d,part_temp t
where d.table_owner=t.table_owner and d.table_name=t.table_name and d.PARTITION_POSITION=t.PARTITION_POSITION 
order by 1,2,3;
----验证对比数据
select count(*) from TEST.TAB;
select count(*) from TEST.TAB_PART;
----分区数据抽验
alter session set nls_date_format='yyyy-mm-dd hh24:mi:ss';
select min(BEGNDT),max(BEGNDT) from TEST.TAB_PART partition(SYS_P18782);
----hash全数据校验(对于关键表可以进行,消耗资源较多)
----批量生成column name的拼接字句
----注意语句的列输出格式要一样，最好在一个窗口执行
SET LINESIZE 400
SET PAGESIZE 400
SET NUMWIDTH 25
with temp1 as 
(select owner,table_name,listagg(column_name||'|'||'|'''||'|'''||'|'||'|') within group( order by COLUMN_ID) concat
from dba_tab_columns
where owner='TEST' and table_name in ('TAB_PART')
group by owner,table_name)
select owner,table_name,substr(concat,0,length(concat)-7) from temp1;
---将表名，以及column name的拼接字句带入
select /*+ parallel(a,4)*/ 'TEST'||'.'||'TAB_PART' TABLE_NAME, 
     count(1) COUNT,
    nvl(sum(dbms_utility.get_hash_value(BEGNDT||'|'||OVERDT||'|'||CORPCODE||'|'||PRODCD||'|'||SRCSYS||'|'||ACCT_SERNO||'|'||ACCT_NUM||'|'||TACCTNO||'|'||CUSTNO,0,1073741824)),0) HASH_VAL 
from TEST.TAB_PART a;
TABLE_NAME             COUNT      HASH_VAL
------------------------------ ------------------------- -------------------------
TEST.TAB_PART       45605625   24485050418392042
​
select /*+ parallel(a,4)*/ 'TEST'||'.'||'TAB' TABLE_NAME, 
     count(1) COUNT,
    nvl(sum(dbms_utility.get_hash_value(BEGNDT||'|'||OVERDT||'|'||CORPCODE||'|'||PRODCD||'|'||SRCSYS||'|'||ACCT_SERNO||'|'||ACCT_NUM||'|'||TACCTNO||'|'||CUSTNO,0,1073741824)),0) HASH_VAL 
from TEST.TAB a;
​
TABLE_NAME             COUNT      HASH_VAL
------------------------------ ------------------------- -------------------------
TEST.TAB        45605625   24485050418392042

对表进行rename替换,将表正式替换为分区表

alter table TEST.TAB rename to TAB_OLD20210907;
alter table TEST.TAB_PART rename to TAB;

对索引，约束进行重建

----获取旧表索引结构
select owner,index_name
from dba_indexes
where table_owner='TEST' and table_name='TAB_OLD20210907';
set longc 9999
set long 99999
set linesize 400
set pagesize 400
select dbms_metadata.get_ddl('INDEX','IDX_TAB01','TEST') from dual;
---
  CREATE INDEX "TEST"."IDX_TAB01" ON "TEST"."TAB_OLD20210907" ("ACCT_NUM")
  PCTFREE 10 INITRANS 2 MAXTRANS 255 COMPUTE STATISTICS
  STORAGE(INITIAL 65536 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
  PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
  BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
  TABLESPACE "TEST"
----rename索引
select 'alter index '||owner||'.'||index_name||' rename to '||index_name||'_old20210907;'
from dba_indexes
where owner='TEST' and table_name='TAB_OLD20210907';
----rename约束名称
----排除了系统自建的约束修改SYS_C
select ' alter table '||owner||'.'||table_name||' rename constraint '||constraint_name||' to '||constraint_name||'_old20210907'
from dba_constraints
where owner='TEST' and table_name='TAB_OLD20210907' and constraint_name not like 'SYS_C%';
----新建新表索引
----对于包含分区建的索引,创建loal本地分区，对于不包含分区键的索引，创建全局分区
create index TEST.index_name on TEST.tab_partion(column_name) tablespace users parallel 4;
or
create index TEST.index_name on TEST.tab_partion(column_name) local tablespace users parallel 4;
​
  CREATE INDEX "TEST"."IDX_TAB01" ON "TEST"."TAB" ("ACCT_NUM")
  PCTFREE 10 INITRANS 2 MAXTRANS 255 COMPUTE STATISTICS
  STORAGE(INITIAL 65536 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
  PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
  BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
  TABLESPACE "TEST" PARALLEL 8;
----关闭索引并行
select 'alter index '||owner||'.'||index_name||' noparallel;'
from dba_indexes
where degree not in (1,0) and owner='TEST';

查看依赖对象有效性，状态是否一致

----查看依赖对象有效性
select a.owner,a.name,a.type,a.REFERENCED_OWNER,a.REFERENCED_NAME,b.status  from dba_DEPENDENCIES a,dba_objects b 
where a.REFERENCED_OWNER='TEST' and a.REFERENCED_NAME='TAB' and a.NAME=b.object_name;
----手动进行编译
alter procedure  . compile;
alter package . compile;
alter package . compile body;
alter view . compile;
alter trigger  compile;
----查询当前对象状态数量(改造完之后状态要一致)
select owner,status,count(*)
from dba_objects
WHERE OWNER='TEST'
group by owner,status
order by 1,2;

检查完成之后，大表改造分区完成

 

dbms_redefinition在线重定义方式

创建中间表

CREATE TABLE par_table (
id NUMBER(10),
create_date DATE,
name VARCHAR2(100)
)
PARTITION BY RANGE (create_date)
(PARTITION unpar_table_2005 VALUES LESS THAN (TO_DATE('01/01/2005', 'DD/MM/YYYY')),
PARTITION unpar_table_2006 VALUES LESS THAN (TO_DATE('01/01/2006', 'DD/MM/YYYY')),
PARTITION unpar_table_2007 VALUES LESS THAN (MAXVALUE));

检查源表(非分区unpar_tab)是否具备迁移条件

EXEC Dbms_Redefinition.can_redef_table(USER, 'unpar_table');
---检查源表是否具备迁移条件
---注意表需要有主键，否则会报以下错误
ORA-12089: cannot online redefine table "TEST"."UNPAR_TABLE" with no primary key

如果检查没有报错，则可以进行以下数据迁移

---此操作期间对于要迁移的数据，如果行上锁还未释放，操作会产生TX 6锁
BEGIN
DBMS_REDEFINITION.start_redef_table(
uname => USER, 
orig_table => 'unpar_table',
int_table => 'par_table');
END;
/

手动同步数据到新表，在创建索引之前，避免数据差异过大，即使finish_redef_table也会再一次同步数据

BEGIN
dbms_redefinition.sync_interim_table(
uname => USER, 
orig_table => 'unpar_table',
int_table => 'par_table');
END;
/
注意:在操作期间，目标表无法进行，也不该进行人为的dml操作，会报以下错误
update par_table set id='x';
update par_table set id='x'
                        *
ERROR at line 1:
ORA-01732: data manipulation operation not legal on this view

可以通过以下视图查看在线重定义表的状态

CDB_REDEFINITION_ERRORS
CDB_REDEFINITION_OBJECTS
CDB_REDEFINITION_STATUS
DBA_REDEFINITION_ERRORS
DBA_REDEFINITION_OBJECTS
DBA_REDEFINITION_STATUS
​

主要关注当前的操作，以及操作状态，如果当前没有正在进行的redefiniton操作，则视图都为空

在进行切换之前，对中间表创建索引以及约束​

ALTER TABLE par_table ADD (
CONSTRAINT unpar_table_pk2 PRIMARY KEY (id
)
)
;
CREATE INDEX create_date_ind2 ON par_table(create_date);

在进行切换之前，进行表统计信息收集

EXEC DBMS_STATS.gather_table_stats(USER, 'par_table', cascade => TRUE);

完成重定义操作

---此操作期间需要短暂获取TM 6锁，以完成表的切换
SQL> BEGIN
dbms_redefinition.finish_redef_table(
uname => USER,
orig_table => 'unpar_table',
int_table => 'par_table');
END;
/
At this point the interim table (along with its index) has become the "real" table and their names have been switched in the name dictionary. 
SQL> select table_name from user_tab_partitions where table_name in ('UNPAR_TABLE','PAR_TABLE')
;
TABLE_NAME
------------------------------
UNPAR_TABLE
UNPAR_TABLE
UNPAR_TABLE
SQL> select table_name,index_name from user_indexes where table_name in ('UNPAR_TABLE','PAR_TABLE');
TABLE_NAME INDEX_NAME
------------------------------ ------------------------------
PAR_TABLE CREATE_DATE_IND
PAR_TABLE UNPAR_TABLE_PK
UNPAR_TABLE UNPAR_TABLE_PK2
UNPAR_TABLE CREATE_DATE_IND2

检查完成之后，大表改造分区完成

​