postgresql 服务器日志

开启审计日志

• 编辑 $PGDATA/postgresql.conf 文件

vi $PGDATA/postgresql.conf
# 做以下修改，下面2部分未提及的全部备注掉
# - Where to Log -
log_destination = 'csvlog' # 日志输出格式
logging_collector = on # 日志收集器，打开后某些不会出现在审计日志中的日志会被重定向到审计日志
log_directory = 'pg_log' # 相对于 $PGDATA 的相对路径，全路径即为 $PGDATA/pg_log
# 保留近7天的审计日志，轮询替换
log_filename = 'postgresql.%a' #日志名称
log_file_mode = 0600 # 只有postgres有读写权限
log_truncate_on_rotation = on # 覆盖同名日志
log_rotation_size = 0 # 不限制日志大小

log_min_messages = warning # 控制哪些消息级别被写入到审计日志
log_min_error_statement = error # 控制哪些导致一个错误情况的 SQL 语句被记录在服务器日志中
log_min_duration_statement = 0 # 记录所有sql运行时长，可以查慢sql

# - What to Log -
log_duration = on # 导致每一个完成的语句的持续时间被记录
log_lock_waits = on # 等锁超时记录日志，超时时间参数 deadlock_timeout
log_statement = 'mod' # mod记录所有ddl语句，外加数据修改语句例如INSERT, UPDATE、DELETE、TRUNCATE, 和COPY FROM
log_replication_commands = off # 不记录流复制命令
log_timezone = 'Asia/Shanghai' # 时区，查看操作系统时区 timedatectl | grep "Time zone"

• 重启数据库

pg_ctl restart -mf

sql直接读取日志

• 安装 file_fdw 插件

create extension file_fdw;

• 创建外部表

drop foreign table if exists pg_log_mon;
create foreign table pg_log_mon(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Mon.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_mon is '每周一当天审计日志';
comment on column pg_log_mon.log_time is '日志时间';
comment on column pg_log_mon.user_name is '用户名';
comment on column pg_log_mon.database_name is '数据库名';
comment on column pg_log_mon.process_id is '进程id';
comment on column pg_log_mon.connection_from is '客户端ip:端口';
comment on column pg_log_mon.session_id is '会话id';
comment on column pg_log_mon.session_line_num is '当前会话的第几次查询';
comment on column pg_log_mon.command_tag is '命令类型';
comment on column pg_log_mon.session_start_time is '会话开始时间';
comment on column pg_log_mon.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_mon.transaction_id is '事务id';
comment on column pg_log_mon.error_severity is '错误级别';
comment on column pg_log_mon.sql_state_code is 'sql状态代码';
comment on column pg_log_mon.message is '信息';
comment on column pg_log_mon.detail is '详细信息';
comment on column pg_log_mon.hint is '提示信息';
comment on column pg_log_mon.query is '查询的sql';
comment on column pg_log_mon.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_tue;
create foreign table pg_log_tue(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Tue.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_tue is '每周二当天审计日志';
comment on column pg_log_tue.log_time is '日志时间';
comment on column pg_log_tue.user_name is '用户名';
comment on column pg_log_tue.database_name is '数据库名';
comment on column pg_log_tue.process_id is '进程id';
comment on column pg_log_tue.connection_from is '客户端ip:端口';
comment on column pg_log_tue.session_id is '会话id';
comment on column pg_log_tue.session_line_num is '当前会话的第几次查询';
comment on column pg_log_tue.command_tag is '命令类型';
comment on column pg_log_tue.session_start_time is '会话开始时间';
comment on column pg_log_tue.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_tue.transaction_id is '事务id';
comment on column pg_log_tue.error_severity is '错误级别';
comment on column pg_log_tue.sql_state_code is 'sql状态代码';
comment on column pg_log_tue.message is '信息';
comment on column pg_log_tue.detail is '详细信息';
comment on column pg_log_tue.hint is '提示信息';
comment on column pg_log_tue.query is '查询的sql';
comment on column pg_log_tue.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_wed;
create foreign table pg_log_wed(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Wed.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_wed is '每周三当天审计日志';
comment on column pg_log_wed.log_time is '日志时间';
comment on column pg_log_wed.user_name is '用户名';
comment on column pg_log_wed.database_name is '数据库名';
comment on column pg_log_wed.process_id is '进程id';
comment on column pg_log_wed.connection_from is '客户端ip:端口';
comment on column pg_log_wed.session_id is '会话id';
comment on column pg_log_wed.session_line_num is '当前会话的第几次查询';
comment on column pg_log_wed.command_tag is '命令类型';
comment on column pg_log_wed.session_start_time is '会话开始时间';
comment on column pg_log_wed.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_wed.transaction_id is '事务id';
comment on column pg_log_wed.error_severity is '错误级别';
comment on column pg_log_wed.sql_state_code is 'sql状态代码';
comment on column pg_log_wed.message is '信息';
comment on column pg_log_wed.detail is '详细信息';
comment on column pg_log_wed.hint is '提示信息';
comment on column pg_log_wed.query is '查询的sql';
comment on column pg_log_wed.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_thu;
create foreign table pg_log_thu(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Thu.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_thu is '每周四当天审计日志';
comment on column pg_log_thu.log_time is '日志时间';
comment on column pg_log_thu.user_name is '用户名';
comment on column pg_log_thu.database_name is '数据库名';
comment on column pg_log_thu.process_id is '进程id';
comment on column pg_log_thu.connection_from is '客户端ip:端口';
comment on column pg_log_thu.session_id is '会话id';
comment on column pg_log_thu.session_line_num is '当前会话的第几次查询';
comment on column pg_log_thu.command_tag is '命令类型';
comment on column pg_log_thu.session_start_time is '会话开始时间';
comment on column pg_log_thu.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_thu.transaction_id is '事务id';
comment on column pg_log_thu.error_severity is '错误级别';
comment on column pg_log_thu.sql_state_code is 'sql状态代码';
comment on column pg_log_thu.message is '信息';
comment on column pg_log_thu.detail is '详细信息';
comment on column pg_log_thu.hint is '提示信息';
comment on column pg_log_thu.query is '查询的sql';
comment on column pg_log_thu.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_fri;
create foreign table pg_log_fri(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Fri.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_fri is '每周五当天审计日志';
comment on column pg_log_fri.log_time is '日志时间';
comment on column pg_log_fri.user_name is '用户名';
comment on column pg_log_fri.database_name is '数据库名';
comment on column pg_log_fri.process_id is '进程id';
comment on column pg_log_fri.connection_from is '客户端ip:端口';
comment on column pg_log_fri.session_id is '会话id';
comment on column pg_log_fri.session_line_num is '当前会话的第几次查询';
comment on column pg_log_fri.command_tag is '命令类型';
comment on column pg_log_fri.session_start_time is '会话开始时间';
comment on column pg_log_fri.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_fri.transaction_id is '事务id';
comment on column pg_log_fri.error_severity is '错误级别';
comment on column pg_log_fri.sql_state_code is 'sql状态代码';
comment on column pg_log_fri.message is '信息';
comment on column pg_log_fri.detail is '详细信息';
comment on column pg_log_fri.hint is '提示信息';
comment on column pg_log_fri.query is '查询的sql';
comment on column pg_log_fri.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_sat;
create foreign table pg_log_sat(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Sat.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_sat is '每周六当天审计日志';
comment on column pg_log_sat.log_time is '日志时间';
comment on column pg_log_sat.user_name is '用户名';
comment on column pg_log_sat.database_name is '数据库名';
comment on column pg_log_sat.process_id is '进程id';
comment on column pg_log_sat.connection_from is '客户端ip:端口';
comment on column pg_log_sat.session_id is '会话id';
comment on column pg_log_sat.session_line_num is '当前会话的第几次查询';
comment on column pg_log_sat.command_tag is '命令类型';
comment on column pg_log_sat.session_start_time is '会话开始时间';
comment on column pg_log_sat.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_sat.transaction_id is '事务id';
comment on column pg_log_sat.error_severity is '错误级别';
comment on column pg_log_sat.sql_state_code is 'sql状态代码';
comment on column pg_log_sat.message is '信息';
comment on column pg_log_sat.detail is '详细信息';
comment on column pg_log_sat.hint is '提示信息';
comment on column pg_log_sat.query is '查询的sql';
comment on column pg_log_sat.application_name is '应用名（客户端名）';

drop foreign table if exists pg_log_sun;
create foreign table pg_log_sun(
     log_time timestamp
    ,user_name text
    ,database_name text
    ,process_id integer
    ,connection_from text
    ,session_id text
    ,session_line_num bigint
    ,command_tag text
    ,session_start_time timestamp
    ,virtual_transaction_id text
    ,transaction_id bigint
    ,error_severity text
    ,sql_state_code text
    ,message text
    ,detail text
    ,hint text
    ,internal_query text
    ,internal_query_pos integer
    ,context text
    ,query text
    ,query_pos integer
    ,location text
    ,application_name text 
) server pg_file_server options(
     filename '/data/pgdata/pg_log/postgresql.Sun.csv'
    ,format 'csv'
    ,header 'false'
    ,delimiter ','
    ,quote '"'
    ,escape '"'
);
comment on foreign table pg_log_sun is '每周日当天审计日志';
comment on column pg_log_sun.log_time is '日志时间';
comment on column pg_log_sun.user_name is '用户名';
comment on column pg_log_sun.database_name is '数据库名';
comment on column pg_log_sun.process_id is '进程id';
comment on column pg_log_sun.connection_from is '客户端ip:端口';
comment on column pg_log_sun.session_id is '会话id';
comment on column pg_log_sun.session_line_num is '当前会话的第几次查询';
comment on column pg_log_sun.command_tag is '命令类型';
comment on column pg_log_sun.session_start_time is '会话开始时间';
comment on column pg_log_sun.virtual_transaction_id is '虚拟事务id';
comment on column pg_log_sun.transaction_id is '事务id';
comment on column pg_log_sun.error_severity is '错误级别';
comment on column pg_log_sun.sql_state_code is 'sql状态代码';
comment on column pg_log_sun.message is '信息';
comment on column pg_log_sun.detail is '详细信息';
comment on column pg_log_sun.hint is '提示信息';
comment on column pg_log_sun.query is '查询的sql';
comment on column pg_log_sun.application_name is '应用名（客户端名）';

create or replace view pg_log as 
select * from pg_log_mon
union all
select * from pg_log_tue
union all
select * from pg_log_wed
union all
select * from pg_log_thu
union all
select * from pg_log_fri
union all
select * from pg_log_sat
union all
select * from pg_log_sun;

备份策略

• 每天00:30备份前一天的审计日志

备份实现

变量替换

postgresql 实现变量替换框架

get_dow_date

postgresql获取指定日期所在星期的星期几所在的日期

增加备份相关时间变量

insert into vardef (
     varname  --变量名
    ,varvalue  --变量值
    ,vardesc  --变量说明
) values (
     'bak_log_span'
    ,$$to_char(get_dow_date(get_dow_date(current_date,1)-1,5),'yyyymmdd')||'_'||to_char(get_dow_date(current_date,4),'yyyymmdd')$$
    ,'周备份范围'
);

shell脚本实现备份

#! /bin/bash

#备份日期默认昨天，也是备份日期上限
BAKUP_DATE_UPPER=`date '+%Y%m%d' -d '-1 day'`
BAKUP_DATE_FLOOR=`date '+%Y%m%d' -d '-6 day'`
BAKUP_DATE=${BAKUP_DATE_UPPER}

showuseage() {
    echo "程序功能:在数据库中备份昨天的pg审计日志
  Useage: [backup_pglog -h 20220830]
       -h [:可选，指定备份日期，可选范围${BAKUP_DATE_FLOOR}-${BAKUP_DATE_UPPER}，其他默认昨天]"
}

# /home/postgres/backup_pglog.sh
# 每天备份昨天的数据


while getopts h: OPTS; do
    case "$OPTS" in
        h)  
            if [ $OPTARG -ge ${BAKUP_DATE_FLOOR} -a $OPTARG -le ${BAKUP_DATE_UPPER} ]; then 
                BAKUP_DATE=$OPTARG
            fi
            ;;
        :)  
            echo "$0 必须为 -$OPTARG 添加一个参数!"
            exit 1
            ;;
        ?)
            showuseage
            exit 1
           ;;
    esac
done            

BAKUP_SQL="
    create table if not exists pg_log_:bak_log_span as 
    select 
         :today::varchar(8) as bak_date
        ,*
    from pg_log
    where 1 = 2
    ;
    delete from pg_log_:bak_log_span where to_char(log_time,'yyyymmdd')::numeric = ${BAKUP_DATE}
    ;
    insert into pg_log_:bak_log_span
    select :today::varchar as bak_date ,* 
    from pg_log
    where to_char(log_time,'yyyymmdd')::numeric = ${BAKUP_DATE}
    ;
    "

echo `date '+%Y-%m-%d %H:%m:%S'`"|INFO|备份数据日期：${BAKUP_DATE}"
echo `date '+%Y-%m-%d %H:%m:%S'`"|INFO|审计日志备份开始"
BAKUP_SQL_EXEC_DATE=`date '+%Y%m%d%H%m%S'`
psql <<EOF > /tmp/bakup_sql_exec_${BAKUP_SQL_EXEC_DATE}.log 2>&1 
\i ~/var/${BAKUP_DATE}.sql
${BAKUP_SQL}
\q
EOF
EXECLOG=`cat /tmp/bakup_sql_exec_${BAKUP_SQL_EXEC_DATE}.log`
rm -f /tmp/bakup_sql_exec_${BAKUP_SQL_EXEC_DATE}.log

echo `date '+%Y-%m-%d %H:%m:%S'`"|INFO|审计日志备份结束"

# 错误
ERRORNUM=`echo "${EXECLOG}" | grep -i 'ERROR' | wc -l`

if [ ${ERRORNUM} -eq 0 ]
then 
    # 发短信通知备份成功
    echo `date '+%Y-%m-%d %H:%m:%S'`"|INFO|审计日志备份成功"
    exit 0
else
    # 发短信通知备份失败
    echo "请及时重新执行备份脚本进行备份" | mail -s "审计日志备份失败" 1445471277@qq.com
    echo `date '+%Y-%m-%d %H:%m:%S'`"|ERROR|审计日志备份失败"
    exit 1
fi

chmod +x /home/postgres/backup_pglog.sh
crontab -e 
30 0 * * * . /home/postgres/backup_pglog.sh >> /home/postgres/backup_pglog.log  2>&1

说明

• 格式符

格式符	说明
%a	星期的英文单词的缩写：如星期一， 则返回 Mon
%A	星期的英文单词的全拼：如星期一，返回 Monday
%b	月份的英文单词的缩写：如一月， 则返回 Jan
%B	月份的引文单词的缩写：如一月， 则返回 January
%c	返回datetime的字符串表示，如03/08/15 23:01:26
%d	返回的是当前时间是当前月的第几天
%f	微秒的表示： 范围: [0,999999]
%H	以24小时制表示当前小时
%I	以12小时制表示当前小时
%j	返回 当天是当年的第几天 范围[001,366]
%m	返回月份 范围[0,12]
%M	返回分钟数 范围 [0,59]
%P	返回是上午还是下午–AM or PM
%S	返回秒数 范围 [0,61]。。。手册说明的
%U	返回当周是当年的第几周 以周日为第一天
%W	返回当周是当年的第几周 以周一为第一天
%w	当天在当周的天数，范围为[0, 6]，6表示星期天
%x	日期的字符串表示 ：03/08/15
%X	时间的字符串表示 ：23:22:08
%y	两个数字表示的年份 15
%Y	四个数字表示的年份 2015
%z	与utc时间的间隔 （如果是本地时间，返回空字符串）
%Z	时区名称（如果是本地时间，返回空字符串）

• 消息严重级别

严重性	用法	syslog	eventlog
DEBUG1…DEBUG5	为开发者提供连续的更详细的信息。	DEBUG	INFORMATION
INFO	提供用户隐式要求的信息，例如来自VACUUM VERBOSE的输出。	INFO	INFORMATION
NOTICE	提供可能对用户有用的信息，例如长标识符截断提示。	NOTICE	INFORMATION
WARNING	提供可能出现的问题的警告，例如在一个事务块外COMMIT。	NOTICE	WARNING
ERROR	报告一个导致当前命令中断的错误。	WARNING	ERROR
LOG	报告管理员可能感兴趣的信息，例如检查点活动。	INFO	INFORMATION
FATAL	报告一个导致当前会话中断的错误。	ERR	ERROR
PANIC	报告一个导致所有数据库会话中断的错误。	CRIT	ERROR