CDH日志清理

备注:
CDH 版本: 6.3.1

背景

CDH集群运行了2个多月了，根目录磁盘使用率接近80%了，需要清理CDH日志。

一.查看问题

CDH的日志文件一般在如下几个目录，可以通过如下代码进行查看。

代码:

cd /var/log/
du -s ./* | sort -nr

cd /var/lib/cloudera-service-monitor
du -s ./* | sort -nr

cd /var/lib/cloudera-host-monitor
du -s ./* | sort -nr

测试记录:

[root@hp1 ~]# cd /var/log/
[root@hp1 log]# du -s ./* | sort -nr
712308  ./hadoop-hdfs
259048  ./cloudera-scm-server
244780  ./hadoop-yarn
191752  ./hive
67408   ./oozie
54472   ./zookeeper
48012   ./cloudera-scm-firehose
40388   ./hue
11224   ./cloudera-scm-agent
7832    ./cloudera-scm-alertpublisher
7280    ./cloudera-scm-eventserver
6712    ./hadoop-mapreduce
5504    ./audit
2328    ./anaconda
1380    ./statestore
684     ./catalogd
560     ./messages-20201227
136     ./httpd
116     ./spark
112     ./messages
108     ./wtmp
108     ./dmesg.old
108     ./dmesg
104     ./boot.log-20201114
100     ./messages-20201206
96      ./boot.log-20201013
88      ./messages-20201220
76      ./messages-20201213
48      ./cron-20201227
48      ./cron-20201213
44      ./cron-20201220
44      ./cron-20201206
36      ./lastlog
36      ./boot.log-20201225
36      ./boot.log-20201126
32      ./mysqld.log
32      ./cron
32      ./boot.log-20201120
32      ./boot.log-20201117
28      ./vmware-vmsvc.log
28      ./mongodb
24      ./tuned
16      ./vmware-vgauthsvc.log.0
16      ./hue-httpd
12      ./yum.log
12      ./tallylog
8       ./secure-20201227
4       ./vmware-network.log
4       ./vmware-network.1.log
4       ./secure-20201220
4       ./secure-20201213
4       ./secure-20201206
4       ./secure
4       ./grubby_prune_debug
4       ./btmp-20201201
0       ./spooler-20201227
0       ./spooler-20201220
0       ./spooler-20201213
0       ./spooler-20201206
0       ./spooler
0       ./rhsm
0       ./ntpstats
0       ./maillog-20201227
0       ./maillog-20201220
0       ./maillog-20201213
0       ./maillog-20201206
0       ./maillog
0       ./impala-minidumps
0       ./firewalld
0       ./chrony
0       ./btmp
0       ./boot.log
[root@hp1 log]# cd /var/lib/cloudera-service-monitor
[root@hp1 cloudera-service-monitor]# du -s ./* | sort -nr
7822564 ./ts
892788  ./subject_record
40852   ./yarn
148     ./impala
68      ./reports
[root@hp1 cloudera-service-monitor]# cd /var/lib/cloudera-host-monitor
[root@hp1 cloudera-host-monitor]# du -s ./* | sort -nr
4720888 ./ts
43356   ./subject_record
[root@hp1 cloudera-host-monitor]# 
[root@hp1 cloudera-host-monitor]# 
[root@hp1 cloudera-host-monitor]# df -h
文件系统                                容量  已用  可用 已用% 挂载点
devtmpfs                                7.8G     0  7.8G    0% /dev
tmpfs                                   7.8G     0  7.8G    0% /dev/shm
tmpfs                                   7.8G   17M  7.8G    1% /run
tmpfs                                   7.8G     0  7.8G    0% /sys/fs/cgroup
/dev/mapper/centos_10--31--1--123-root   50G   36G   15G   72% /
/dev/sda1                              1014M  150M  865M   15% /boot
/dev/mapper/centos_10--31--1--123-home  441G  116G  326G   27% /home
cm_processes                            7.8G  176M  7.6G    3% /run/cloudera-scm-agent/process
tmpfs                                   1.6G     0  1.6G    0% /run/user/0
[root@hp1 cloudera-host-monitor]# 

二.清理空间

首先是 /var/log的清理,因为组件较多，需要进入到各个组件的目录下,将out文件和log文件进行删除

cd /var/log/hadoop-mapreduce
cd /var/log/hadoop-hdfs
cd /var/log/zookeeper
cd /var/log/hive
cd /var/log/cloudera-scm-agent
cd /var/log/audit
cd /var/log/hadoop-yarn
cd /var/log/hue-httpd

rm -rf *.out.* 
rm -rf *.log.*

其实是监控数据的删除，两个监控累积占了15G左右的空间。
删除监控服务的日志。

cd /var/lib/cloudera-host-monitor/ts/type/partitions
rm -rf type*
cd /var/lib/cloudera-host-monitor/ts/stream/partitions
rm -rf stream*
cd /var/lib/cloudera-host-monitor/ts/ts_stream_rollup_PT600S/partitions/
rm -rf ts_stream*
cd /var/lib/cloudera-host-monitor/ts/ts_type_rollup_PT600S/partitions/
rm -rf ts_type*

cd /var/lib/cloudera-service-monitor/ts/stream/partitions/
rm -rf stream*
cd /var/lib/cloudera-service-monitor/ts/type/partitions/
rm -rf type*
cd /var/lib/cloudera-service-monitor/ts/ts_stream_rollup_PT600S/partitions/
rm -rf ts_stream*
cd /var/lib/cloudera-service-monitor/ts/ts_type_rollup_PT600S/partitions/
rm -rf ts_type*

三.配置shell脚本自动清理

#!/bin/bash

# 清理日志
cd /var/log/hadoop-mapreduce
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/hadoop-hdfs
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/zookeeper
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/hive
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/cloudera-scm-agent
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/audit
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/hadoop-yarn
rm -rf *.out.* 
rm -rf *.log.*

cd /var/log/hue-httpd
rm -rf *.out.* 
rm -rf *.log.*

# 清理监控日志
cd /var/lib/cloudera-host-monitor/ts/type/partitions
rm -rf type*
cd /var/lib/cloudera-host-monitor/ts/stream/partitions
rm -rf stream*
cd /var/lib/cloudera-host-monitor/ts/ts_stream_rollup_PT600S/partitions/
rm -rf ts_stream*
cd /var/lib/cloudera-host-monitor/ts/ts_type_rollup_PT600S/partitions/
rm -rf ts_type*

cd /var/lib/cloudera-service-monitor/ts/stream/partitions/
rm -rf stream*
cd /var/lib/cloudera-service-monitor/ts/type/partitions/
rm -rf type*
cd /var/lib/cloudera-service-monitor/ts/ts_stream_rollup_PT600S/partitions/
rm -rf ts_stream*
cd /var/lib/cloudera-service-monitor/ts/ts_type_rollup_PT600S/partitions/
rm -rf ts_type*

参考:

1.https://www.cnblogs.com/qinglanmei/p/11455010.html