SRE运维工程师笔记-文本处理三剑客之 grep

内容概述

• 文本处理三剑客之grep

文本处理三剑客

grep 命令主要对文本的（正则表达式）行基于模式进行过滤
sed：stream editor，文本编辑工具
awk：Linux上的实现gawk，文本报告生成器

1. 文本处理三剑客之 grep

grep: Global search REgular expression and Print out the line
作用：文本搜索工具，根据用户指定的“模式”对目标文本逐行进行匹配检查；打印匹配到的行
模式：由正则表达式字符及文本字符所编写的过滤条件
格式：

grep [OPTIONS] PATTERN [FILE...]

常见选项：

--color=auto 对匹配到的文本着色显示
-m # 匹配#次后停止
-v 显示不被pattern匹配到的行
-i 忽略字符大小写
-n 显示匹配的行号
-c 统计匹配的行数
-o 仅显示匹配到的字符串
-q 静默模式，不输出任何信息
-A # after, 后#行
-B # before, 前#行

-C # context, 前后各#行
-e 实现多个选项间的逻辑or关系,如：grep –e ‘cat ’ -e ‘dog’ file
-w 匹配整个单词
-E 使用ERE，相当于egrep
-F 不支持正则表达式，相当于fgrep
-f file 根据模式文件处理
-r 递归目录，但不处理软链接
-R 递归目录，但处理软链接

[root@CentOS-8 ~]#grep -v '^#' /etc/ssh/sshd_config |grep -v '^$'
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
PermitRootLogin yes
AuthorizedKeysFile	.ssh/authorized_keys
PasswordAuthentication yes
ChallengeResponseAuthentication no
GSSAPIAuthentication yes
GSSAPICleanupCredentials no
UsePAM yes
X11Forwarding yes
PrintMotd no
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem	sftp	/usr/libexec/openssh/sftp-server
[root@CentOS-8 ~]#grep -v '^[[:space:]]*#\|^$' /etc/ssh/sshd_config
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
PermitRootLogin yes
AuthorizedKeysFile	.ssh/authorized_keys
PasswordAuthentication yes
ChallengeResponseAuthentication no
GSSAPIAuthentication yes
GSSAPICleanupCredentials no
UsePAM yes
X11Forwarding yes
PrintMotd no
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem	sftp	/usr/libexec/openssh/sftp-server
[root@CentOS-8 ~]#grep -Ev '^[[:space:]]*#|^$' /etc/ssh/sshd_config
HostKey /etc/ssh/ssh_host_rsa_key
HostKey /etc/ssh/ssh_host_ecdsa_key
HostKey /etc/ssh/ssh_host_ed25519_key
SyslogFacility AUTHPRIV
PermitRootLogin yes
AuthorizedKeysFile	.ssh/authorized_keys
PasswordAuthentication yes
ChallengeResponseAuthentication no
GSSAPIAuthentication yes
GSSAPICleanupCredentials no
UsePAM yes
X11Forwarding yes
PrintMotd no
AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
AcceptEnv XMODIFIERS
Subsystem	sftp	/usr/libexec/openssh/sftp-server

[root@CentOS-8 ~]#grep -in ROOT /etc/shadow
1:root:$6$w094OhpSTIlvjEmg$IZd0Lf9REhlU5Ns.6dIRTrH32IuQsWQ8y9fVSaSeljDmae14bJe9Iv.Z0fxB/zxO77kVoER7LrsG.GhKbUrcO1::0:99999:7:::
[root@CentOS-8 ~]#grep -ic ROOT /etc/shadow
1
[root@CentOS-8 ~]#grep -o root /etc/shadow
root
[root@CentOS-8 ~]#grep -q root /etc/shadow
[root@CentOS-8 ~]#echo $?
0
[root@CentOS-8 ~]#grep -nA3 root /etc/passwd
1:root:x:0:0:root:/root:/bin/bash
2-bin:x:1:1:bin:/bin:/sbin/nologin
3-daemon:x:2:2:daemon:/sbin:/sbin/nologin
4-adm:x:3:4:adm:/var/adm:/sbin/nologin
--
10:operator:x:11:0:operator:/root:/sbin/nologin
11-games:x:12:100:games:/usr/games:/sbin/nologin
12-ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
13-nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
[root@CentOS-8 ~]#grep -nB3 root /etc/passwd
1:root:x:0:0:root:/root:/bin/bash
--
7-shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
8-halt:x:7:0:halt:/sbin:/sbin/halt
9-mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
10:operator:x:11:0:operator:/root:/sbin/nologin
[root@CentOS-8 ~]#grep -nC3 root /etc/passwd
1:root:x:0:0:root:/root:/bin/bash
2-bin:x:1:1:bin:/bin:/sbin/nologin
3-daemon:x:2:2:daemon:/sbin:/sbin/nologin
4-adm:x:3:4:adm:/var/adm:/sbin/nologin
--
7-shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
8-halt:x:7:0:halt:/sbin:/sbin/halt
9-mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
10:operator:x:11:0:operator:/root:/sbin/nologin
11-games:x:12:100:games:/usr/games:/sbin/nologin
12-ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
13-nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
[root@CentOS-8 ~]#grep -e root -e bash /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
du:x:1000:1000:duwenshuo,it,100001,111111:/home/du:/bin/bash
wang:x:1001:1001::/home/wang:/bin/bash
mage:x:1002:1002::/home/mage:/bin/bash
[root@CentOS-8 ~]#grep -E "root|bash" /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
du:x:1000:1000:duwenshuo,it,100001,111111:/home/du:/bin/bash
wang:x:1001:1001::/home/wang:/bin/bash
mage:x:1002:1002::/home/mage:/bin/bash
[root@CentOS-8 ~]#grep -w root /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
[root@CentOS-8 ~]#egrep  "root|bash" /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
du:x:1000:1000:duwenshuo,it,100001,111111:/home/du:/bin/bash
wang:x:1001:1001::/home/wang:/bin/bash
mage:x:1002:1002::/home/mage:/bin/bash
[root@CentOS-8 ~]#grep -F 'r..t' /etc/passwd
[root@CentOS-8 ~]#fgrep  'r..t' /etc/passwd
[root@CentOS-8 ~]#grep  'r..t' /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
clevis:x:977:976:Clevis Decryption Framework unprivileged user:/var/cache/clevis:/sbin/nologin
[root@CentOS-8 ~]#vi re.txt
[root@CentOS-8 ~]#cat re.txt 
r..t
bash
[root@CentOS-8 ~]#grep -f re.txt /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
clevis:x:977:976:Clevis Decryption Framework unprivileged user:/var/cache/clevis:/sbin/nologin
du:x:1000:1000:duwenshuo,it,100001,111111:/home/du:/bin/bash
wang:x:1001:1001::/home/wang:/bin/bash
mage:x:1002:1002::/home/mage:/bin/bash
[root@CentOS-8 ~]#cat f1.txt 
a
b
c
[root@CentOS-8 ~]#cat f2.txt 
1
a
2
c
[root@CentOS-8 ~]#cat f1.txt f2.txt 
a
b
c
1
a
2
c
[root@CentOS-8 ~]#cat f1.txt f2.txt |sort|uniq -c
      1 1
      1 2
      2 a
      1 b
      2 c
[root@CentOS-8 ~]#cat f1.txt f2.txt |sort|uniq -d
a
c
[root@CentOS-8 ~]#grep -f f1.txt f2.txt 
a
c

范例：

grep root /etc/passwd
grep "USER" /etc/passwd
grep 'USER' /etc/passwd
grep whoami /etc/passwd

范例：取两个文件的相同行

[root@centos8 ~]#cat /data/f1.txt
a 
b 
1 
c
[root@centos8 ~]#cat /data/f2.txt
b 
e 
f 
c 
1 
2
[root@centos8 ~]#grep -f /data/f1.txt /data/f2.txt
b 
c
1

范例:

df | grep '^/dev/sd' |tr -s ' ' %|cut -d% -f5|sort -n|tail -1

范例:

[root@centos8 ~]#ss -nt | grep "^ESTAB" |tr -s ' ' : |cut -d: -f6|sort |uniq -c|sort -nr|head -n3
3 10.0.0.1
1 172.16.4.100
1 172.16.31.188
[root@CentOS-8 ~]#grep -v '^state' ss.log |grep -Eo '^[[:alpha:]-]+\>' |sort|uniq -c
    108 ESTAB
      1 FIN-WAIT
      3 LAST-ACK
      1 State
[root@CentOS-8 ~]#grep -v '^state' ss.log |cut -d " " -f1 |sort|uniq -c
    108 ESTAB
      1 FIN-WAIT-1
      3 LAST-ACK
      1 State
[root@CentOS-8 ~]#ss -nt|grep -v '^state'|grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}:[0-9]+ *$'|grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}'
10.0.0.1
[root@CentOS-8 ~]#ss -nt|grep -v '^state'|grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}:[0-9]+ *$'|grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}'|sort|uniq -c
      1 10.0.0.1

范例:

[root@centos8 ~]#grep -v "^#" /etc/profile | grep -v '^$'
[root@centos8 ~]#grep -v "^#\|^$" /etc/profile
[root@centos8 ~]#grep -v "^\(#\|$\)" /etc/profile
[root@centos8 ~]#grep -Ev "^(#|$)" /etc/profile
[root@centos8 ~]#egrep -v "^(#|$)" /etc/profile
[root@centos6 ~]#egrep -v '^(#|$)' /etc/httpd/conf/httpd.conf

范例:

[root@centos8 ~]#grep -o 'r..t' /etc/passwd
root
root
root
root
r/ft
rypt

范例:

[root@centos8 ~]#ifconfig | grep -E '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]
{1,3}'
inet 10.0.0.8 netmask 255.255.255.0 broadcast 10.0.0.255
inet 172.16.0.123 netmask 255.255.0.0 broadcast 172.16.255.255
inet6 fe80::c11e:4792:7e77:12a4 prefixlen 64 scopeid 0x20
inet 127.0.0.1 netmask 255.0.0.0
[root@centos8 ~]#ifconfig | grep -E '([0-9]{1,3}.){3}[0-9]{1,3}'
inet 10.0.0.8 netmask 255.255.255.0 broadcast 10.0.0.255
inet 172.16.0.123 netmask 255.255.0.0 broadcast 172.16.255.255
inet6 fe80::c11e:4792:7e77:12a4 prefixlen 64 scopeid 0x20
inet 127.0.0.1 netmask 255.0.0.0
[root@centos8 ~]#ifconfig eth0 | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}'|head -1
10.0.0.8
[root@centos8 ~]#cat regex.txt
([0-9]{1,3}\.){3}[0-9]{1,3}
[root@centos8 ~]#ifconfig | grep -oEf regex.txt
10.0.0.8
255.255.255.0
10.0.0.255
127.0.0.1
255.0.0.0

范例:

[root@centos8 ~]#grep -E 'root|bash' /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
wang:x:1000:1000:wang:/home/wang:/bin/bash
mage:x:1001:1001::/home/mage:/bin/bash
xiaoming:x:1002:1002::/home/xiaoming:/bin/bash
roob:x:1003:1003::/home/roob:/bin/bash
[root@centos8 ~]#grep -e 'root' -e 'bash' /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
wang:x:1000:1000:wang:/home/wang:/bin/bash
mage:x:1001:1001::/home/mage:/bin/bash
xiaoming:x:1002:1002::/home/xiaoming:/b

范例:

[root@centos8 ~]#grep -w root /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin
[root@centos8 ~]#grep '\' /etc/passwd
root:x:0:0:root:/root:/bin/bash
operator:x:11:0:operator:/root:/sbin/nologin

范例：

[root@centos8 ~]#grep "^\(.*\)\>.*\<\1$" /etc/passwd
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
bash:x:1008:1008::/home/bash:/bin/bash
nologin:x:1011:1011::/home/nologin:/sbin/nologin

[root@centos8 ~]#grep -E "^(.*)\>.*\<\1$" /etc/passwd
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
bash:x:1008:1008::/home/bash:/bin/bash
nologin:x:1011:1011::/home/nologin:/sbin/nologin

[root@centos8 ~]#egrep "^(.*)\>.*\<\1$" /etc/passwd
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
bash:x:1008:1008::/home/bash:/bin/bash
nologin:x:1011:1011::/home/nologin:/sbin/nologin

范例：面试题，算出所有人的年龄总和

[root@centos8 ~]#cat /data/nianling.txt
xiaoming=20
xiaohong=18
xiaoqiang=22

[root@centos8 ~]#cut -d"=" -f2 /data/nianling.txt|tr '\n' + | grep -Eo ".*[0-
9]"|bc
60
[root@centos8 ~]#grep -Eo "[0-9]+" /data/nianling.txt | tr '\n' + | grep -Eo ".*
[0-9]"|bc
60

[root@CentOS-8 ~]#vim user.txt
[root@CentOS-8 ~]#cat user.txt 
xiaoming=20
xiaohong=18
xiaoqiang=22
wangcai=21
[root@CentOS-8 ~]#cut -d"=" -f2 user.txt|tr '\n' +
20+18+22+21+[root@CentOS-8 ~]#cut -d"=" -f2 user.txt
20
18
22
21
[root@CentOS-8 ~]#cut -d"=" -f2 user.txt|paste -s
20	18	22	21
[root@CentOS-8 ~]#cut -d"=" -f2 user.txt|paste -d+ -s
20+18+22+21
[root@CentOS-8 ~]#cut -d"=" -f2 user.txt|paste -d+ -s|bc
81
[root@CentOS-8 ~]#grep -Eo '[0-9]+' user.txt
20
18
22
21
[root@CentOS-8 ~]#grep -Eo '[0-9]+' user.txt|paste -d+ -s|bc
81