awk学习
pipe 管道用法:
if you open a pipe with this:
print $1 | "sort -r > names.sorted"
then you must close it with this:
close("sort -r > names.sorted")
实时查看网卡流量 查看1分钟
seq 2 | awk -vRS="" 'function ip(s){"ifconfig eth1"|getline;match($0,s" bytes:([0-9]+)",t);close("ifconfig eth1");return t[1]}BEGIN{oldR=ip("RX");oldT=ip("TX")}{while(j<60){system("sleep 1");newR=ip("RX");newT=ip("TX");system("clear");printf "SpeedRX:%dKbytes/s;\tSpeedTX:%dKbytes/s\n",(newR-oldR)/1024,(newT-oldT)/1024;oldR=newR;oldT=newT;j++}}'
apache IP流量
[root@cAdmin tmp]# awk '{a[$1]+=$10}END{for (i in a){printf ("%-20s %10d\n",i,a[i])}}' access.log | sort -k 2nr | head
192.168.1.118 3795645246
192.168.1.115 138155222
192.168.1.112 101585240
192.168.1.110 26680852
192.168.1.121 25349146
192.168.1.123 12843290
192.168.1.124 12778395
192.168.1.127 2152647
192.168.1.117 1129855
192.168.1.101 920865
统计apache访问量前10IP和访问量前10URL
[root@test /tmp]$ awk 'function g(m,n){for(j in m){if(m[j]==n){delete m[j];return j}}}{a[$1]++;b[$7]++}END{y=asort(a,e);s=asort(b,c);for(i=1;i<=10;i++){print i"\t"e[y]"\t"g(a,e[y])"\t"c[s]"\t"g(b,c[s]);y--;s--}}' access.log
1 458 192.168.11.155 40 /svn/TEST/!svn/vcc/default
2 28 192.168.1.158 32 /svn/TEST
3 28 192.168.11.153 24 /svn/TEST/Code/HW/digit_a
4 28 192.168.1.153 16 /svn/TEST/Code/HW
5 22 192.168.11.159 12 /svn/TEST/!svn/bln/1822
6 14 192.168.11.53 10 /svn/TEST/Code/HW/Digital_d/abc
7 7 192.168.11.55 10 /svn/TEST/Code/HW/digit_a/ram
8 7 192.168.1.103 8 /svn/TEST/
9 6 192.168.11.57 4 /svn/TEST/Code/HW/digit_a/ram/
10 1 127.0.0.1 4 /svn/TEST/Code/HW/work
删除重复文件,保留最近的一个. 按md5检查
ls -lS --time-style=+"%s" | awk '{"md5sum "$7|getline m;close("md5sum "$7);split(m,t)}a[t[1]]{if(b[t[1]]>$6){system("rm -f "$7)}else{system("rm -f "a[t[1]])}next}{a[t[1]]=$7;b[t[1]]=$6;}'
asort排序
[root@cAdmin tmp]# more re
12
1
13
123
23
2
21
10
9
90
98
18
[root@cAdmin tmp]# awk '{a[NR]=$1}END{num=asort(a,b); for (i=1;i<=num;i++){print b[i]}}' re
1
2
9
10
12
13
18
21
23
90
98
123
更改文本方向
[root@cAdmin 123]# more test
1 2 3 4 5 6
2 3 4 5 6 1
3 4 5 6 1 2
4 5 6 1 2 3
[root@cAdmin 123]# awk '{for (i=1;i<=NF;i++){a[NR,i]=$i}}END{for (k=1;k<=6;k++){for (j=4;j>0;j--) {printf ("%s ", a[j,k])};print ""}}' test
4 3 2 1
5 4 3 2
6 5 4 3
1 6 5 4
2 1 6 5
3 2 1 6
按变量删除任一列 变量>2 变量<NF
$ more 2
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| id | select_type | table | type | possible_keys| key | key_len| ref | rows | Extra |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| 1 | SIMPLE | db_aa | ALL | NULL | NULL | NULL | NULL| 248 | Using where|
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
$ awk -vj=2 'BEGIN{FS="|"}{if(NR%2){print}else{for(i=2;i<=(NF-1);i++){if(i!=j){printf FS$i}};print "|"}}' 2
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| select_type | table | type | possible_keys| key | key_len| ref | rows | Extra |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| SIMPLE | db_aa | ALL | NULL | NULL | NULL | NULL| 248 | Using where|
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
$ awk -vj=3 'BEGIN{FS="|"}{if(NR%2){print}else{for(i=2;i<=(NF-1);i++){if(i!=j){printf FS$i}};print "|"}}' 2
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| id | table | type | possible_keys| key | key_len| ref | rows | Extra |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| 1 | db_aa | ALL | NULL | NULL | NULL | NULL| 248 | Using where|
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
$ awk -vj=9 'BEGIN{FS="|"}{if(NR%2){print}else{for(i=2;i<=(NF-1);i++){if(i!=j){printf FS$i}};print "|"}}' 2
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| id | select_type | table | type | possible_keys| key | key_len| rows | Extra |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| 1 | SIMPLE | db_aa | ALL | NULL | NULL | NULL | 248 | Using where|
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
$ awk -vj=11 'BEGIN{FS="|"}{if(NR%2){print}else{for(i=2;i<=(NF-1);i++){if(i!=j){printf FS$i}};print "|"}}' 2
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| id | select_type | table | type | possible_keys| key | key_len| ref | rows |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
| 1 | SIMPLE | db_aa | ALL | NULL | NULL | NULL | NULL| 248 |
+----+-------------+-----------+------+---------------+------+---------+------+------+-------------+
在多个文件中如果列一包含"35.0"的数据就把第一行和第二行加上35.0这一行后面的数据再加上1
$ more 2
101.860703 25.689400
99.698402 27.823200
41.0 3.698
42.0 3.715
43.0 3.732
44.0 3.747
45.0 3.762
$ more 3
102.143997 24.700199
103.717903 27.322100
33.0 3.575
34.0 3.595
35.0 3.617
36.0 3.639
$ more 1
101.535599 25.029301
100.735100 23.501900
35.0 3.591
36.0 3.618
37.0 3.647
38.0 3.681
39.0 3.719
40.0 3.752
$ awk 'FNR<=2{a[FNR]=$2" "$1};$1=="35.0"{print a[1],a[2],$2,"1"}' 1 2 3
25.029301 101.535599 23.501900 100.735100 3.591 1
24.700199 102.143997 27.322100 103.717903 3.617 1
最后一有逗号的另一行，前面包含前三列
[root@cAdmin 123]# more 10
a b c 1.2.3.4,5.6.7.8
b b c 2.3.4.5
c b c 2.3.4.5,6.7.8.9
d b c 3.4.5.6
[root@cAdmin 123]# awk '{sub(",","\n"$1" "$2" "$3" ",$4);print}' 10
a b c 1.2.3.4
a b c 5.6.7.8
b b c 2.3.4.5
c b c 2.3.4.5
c b c 6.7.8.9
d b c 3.4.5.6
去掉 [ 右边的, 删除中文或者字母前面的两位数字(如果有), 小数点那一列右对齐
$ more 2.txt
11111111 22222222 333333 100.00 11 AB失败 [L8412][01]
03015810 00015802 226592 10.00 重复 [L101836][1.90]
11111111 222 222 333333 2340.00 14 余额 [L8412][01]
456 923 00015802 226592 230.00 13 失败，A重复 [L101836] [1.90]
$ awk -F "[" '{if(substr($1,41,3)+1>1){print substr($1,1,40),substr($1,45)}else{print $1}}' 2.txt
11111111 22222222 333333 100.00 AB失败
03015810 00015802 226592 10.00 重复
11111111 222 222 333333 2340.00 余额
456 923 00015802 226592 230.00 失败，A重复
把source name="bbb"下面的max="10000"换成max="12345"
$ more 2.txt
<source name="aaa">
<parameter free="0" max="10000" min="0.0001" name="Prefactor" scale="1e-12" value="3.616513122"/>
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="2.55996"/>
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="648.478027"/>
<source name="bbb">
<parameter free="0" max="10000" min="0.0001" name="Prefactor" scale="1e-12" value="2.327612278" />
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="2.4095" />
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="915.795898" />
<source name="ccc">
<parameter free="0" max="10000" min="0.0001" name="Prefactor" scale="1e-14" value="1.119141435" />
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="1.99974" />
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="5288.33252" />
$ awk '/bbb/{print;getline;gsub(/max=\"10000\"/,"max=\"12345\"")}1' 2.txt
<source name="aaa">
<parameter free="0" max="10000" min="0.0001" name="Prefactor" scale="1e-12" value="3.616513122"/>
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="2.55996"/>
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="648.478027"/>
<source name="bbb">
<parameter free="0" max="12345" min="0.0001" name="Prefactor" scale="1e-12" value="2.327612278" />
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="2.4095" />
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="915.795898" />
<source name="ccc">
<parameter free="0" max="10000" min="0.0001" name="Prefactor" scale="1e-14" value="1.119141435" />
<parameter free="0" max="10" min="0" name="Index" scale="-1" value="1.99974" />
<parameter free="0" max="500000" min="30" name="Scale" scale="1" value="5288.33252" />
第四列 的时间 减去第三列的时间 时间差变换成天数 输出到第五列中
$ more 2.txt
13400041414|18702652237|20150302131949|20150302132312
13400041414|15052124614|20150107111304|20150302131709
13400041414|18279684920|20150302133159|20150313092141
13400041414|14779568524|20150316062001|20150316063117
13400041414|15103434017|20150119152514|20150313092146
13400041414|18479343074|20150316061904|20150316062042
13400041414|13405761414|20150316063544|20150316063639
13400041414|18870680194|20150316060912|20150316061428
13400041414|14762010034|20150105110754|20150115104932
13400041414|15152494334|20141230100609|20141230101153
13400643973|13400644513|20150515154622|20150515155231
13400644913|13489937792|20150519111937|20150609144721
$ awk -F "|" '{m=$0;for(i=3;i<5;i++){gsub(/[0-9][0-9]/,"& ",$i);t[i]=mktime(gensub(" ","",1,$i))}printf("%s|%0s\n",m,int((t[4]-t[3])/86400))}' 2.txt
13400041414|18702652237|20150302131949|20150302132312|0
13400041414|15052124614|20150107111304|20150302131709|54
13400041414|18279684920|20150302133159|20150313092141|10
13400041414|14779568524|20150316062001|20150316063117|0
13400041414|15103434017|20150119152514|20150313092146|52
13400041414|18479343074|20150316061904|20150316062042|0
13400041414|13405761414|20150316063544|20150316063639|0
13400041414|18870680194|20150316060912|20150316061428|0
13400041414|14762010034|20150105110754|20150115104932|9
13400041414|15152494334|20141230100609|20141230101153|0
13400643973|13400644513|20150515154622|20150515155231|0
13400644913|13489937792|20150519111937|20150609144721|21
AWK猜数字
[root@cAdmin tmp]# awk 'BEGIN{a="Input Num:";print a;while(1){getline <"/dev/tty";if($0>9){print "Too Big\n"a}else if($0<9){print "Too Small\n"a}else{print "Right!";break}}}' file
Input Num:
3
Too Small
Input Num:
10
Too Big
Input Num:
8
Too Small
Input Num:
9
Right!
把 "第2列是中文" "第5列是邮箱" "第6列是时间" "第7列是ip" 的就整行输出
$ more 2.txt 12354 s品种ID 45986737595 dfg./,4543rrgd;435t,de4rewfsdf3 [email protected] 11:12:5 2 114.12.54.78 878797 897gjg575uu76gf 7567567567fghrdyr754tyhfgy54yfdhh [email protected] 114.1 2.54.78 11:12:52 $ awk '$5~/\w+@\w+/&&$6~/[0-1][0-9]:[0-5][0-9]:[0-5][0-9]/&&$7~/[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/{for(i=1;i<=length($2);i++){if(substr($2,i,1)>"\177"){print;break}}}' 2.txt 12354 s品种ID 45986737595 dfg./,4543rrgd;435t,de4rewfsdf3 [email protected] 11:12:52 114.12.54.78
得到Channel后面的数字、ESSID后面引号中的字符串、Address后面的MAC地址、Authentication Suites后面的字符串以及Pairwise Ciphers后面的字符串
Cell 01 - Address: 56:E6:FC:77:F7:E2
Protocol:802.11b/g/n
ESSID:"marugao-u"
Mode:Managed
Frequency:2.412 GHz (Channel 1)
Quality=100/100 Signal level=-21 dBm Noise level=-92 dBm
Encryption key:on
Bit Rates:54 Mb/s
IE: IEEE 802.11i/WPA2 Version 1
Group Cipher : CCMP
Pairwise Ciphers (1) : CCMP
Authentication Suites (1) : PSK
Cell 02 - Address: 8C:21:0A:64:47:66
Protocol:802.11b/g/n
ESSID:"TP-LINK_644766"
Mode:Managed
Frequency:2.412 GHz (Channel 6)
Quality=57/100 Signal level=-67 dBm Noise level=-92 dBm
Encryption key:on
Bit Rates:54 Mb/s
IE: WPA Version 1
Group Cipher : CCMP
Pairwise Ciphers (1) : CCMP
Authentication Suites (1) : PSK
IE: IEEE 802.11i/WPA2 Version 1
Group Cipher : CCMP
Pairwise Ciphers (1) : CCMP
Authentication Suites (1) : PSK
Cell 03 - Address: 42:16:9F:A6:B3:C4
Protocol:802.11b/g/n
ESSID:"kaminari"
Mode:Managed
Frequency:2.412 GHz (Channel 11)
Quality=100/100 Signal level=-35 dBm Noise level=-92 dBm
Encryption key:on
Bit Rates:54 Mb/s
IE: IEEE 802.11i/WPA2 Version 1
Group Cipher : CCMP
Pairwise Ciphers (1) : CCMP
Authentication Suites (1) : PSK
Cell 04 - Address: 3C:46:D8:0B:27:1C
Protocol:802.11b/g/n
ESSID:"TP-LINK_HyFi_9C"
Mode:Managed
Frequency:2.412 GHz (Channel 1)
Quality=0/100 Signal level=-91 dBm Noise level=-91 dBm
Encryption key:off
Bit Rates:54 Mb/s
#gensub () 的分组可以在后面用 \\1 \\2 取得
$ awk -vRS="Cell" 'function t(str){return gensub(".*"str"[: ]+([^\n]*).*","\\1",1,$0)}{gsub(/"|)/,"");print t("Channel"),t("ESSID"),t("Address"), /Auth/ ? t("Authentication Suites ...")"/"t("Pairwise Ciphers ...") : ""}' 2.txt | column -t
1 marugao-u 56:E6:FC:77:F7:E2 PSK/CCMP
6 TP-LINK_644766 8C:21:0A:64:47:66 PSK/CCMP
11 kaminari 42:16:9F:A6:B3:C4 PSK/CCMP
1 TP-LINK_HyFi_9C 3C:46:D8:0B:27:1C
1.遇到cco1{},将cco1{} 替换为cco1(),并将{}内的逗号,全替换为| (cco1内的字段数量是不固定的)
2.遇到cbax[]，将[]内的{}全替换为(), 并将[]内的逗号,全替换为| (cbax内的字段数量是不固定的)
(1,1,{1,1,cco1{-1,0, -1,1438081612000,70, 0, 0},cbax[{"OnPeakAccountID", 1800, 997699, 1970-01-01 07:00:00, 0}, {"FreeUnits", 1800, 997699, 1970-01-01 07:00:00, 0}]})
#patsplit中被分隔的数组m, m[0]为首字符串; m[i]在t[i]和t[i+1]之间
$ awk '{patsplit($0,t,"(cco1{[^}]*}|cbax\\[[^\\]]*\\])",m);for(i=1;i<length(m);i++){gsub(",","|",t[i]);gsub("{","(",t[i]);gsub("}",")",t[i]);s=s""t[i]""m[i]}print m[0]""s""m[length(m)]}' FILE
(1,1,{1,1,cco1(-1|0| -1|1438081612000|70| 0| 0),cbax[("OnPeakAccountID"| 1800| 997699| 1970-01-01 07:00:00| 0)| ("FreeUnits"| 1800| 997699| 1970-01-01 07:00:00| 0)]})
把逗号改为换行,双引号里面的逗号不变.
[root@test /tmp]$ echo '"abc,dd",a,aa,"aa,a","asdf,adsf,sdf",adsf,",asdf,sdf"' | awk -F\" -vOFS= '{for(i=1;i<=NF;i+=2){gsub(",","\n",$i)}}1'
abc,dd
a
aa
aa,a
asdf,adsf,sdf
adsf
,asdf,sdf
ip2int
[root@test /tmp]$ echo 192.168.1.11 | awk -F. '{print $1*2^24+$2*2^16+$3*2^8+$4}'
3232235787
[root@test /tmp]$ echo 192.168.1.11 | awk -F. '{for(i=NF;i>0;i--){k+=$i*256^n++}print k}'
3232235787
int2ip
[root@test /tmp]$ echo 3232235787 | awk '{$0=int($1/2^24)"."int($1%2^24/2^16)"."int($1%2^16/2^8)"."int($1%2^8)}1'
192.168.1.11
[root@test /tmp]$ echo 3232235787 | awk '{for(i=0;i<4;i++){ip=ip j strtonum("0x"substr(sprintf("%08x",$0),1+i*2,2));j="."};print ip}'
192.168.1.11
显示任意一行 server ip
[root@test /tmp]$ more b
hello world_1
<connection name="A" timeout="10" >
<server ip="10.1.2.11" type="0"/>
<server ip="10.1.2.12" type="0"/>
<server ip="10.1.2.13" type="0"/>
<server ip="10.1.2.14" type="0"/>
<pconnection>
<connection name="B" timeout="10" >
<server ip="10.1.3.111" type="0"/>
<server ip="10.1.3.112" type="0"/>
<server ip="10.1.3.113" type="0"/>
<server ip="10.1.3.114" type="0"/>
<pconnection>
hello world_2
[root@test /tmp]$ awk '!/server/{print;c=0;next}++c==2{print}' b #第2个 server
hello world_1
<connection name="A" timeout="10" >
<server ip="10.1.2.12" type="0"/>
<pconnection>
<connection name="B" timeout="10" >
<server ip="10.1.3.112" type="0"/>
<pconnection>
hello world_2
删除engine{...foobar...}
more a
mainfunc {
var = "baz"
engine "map" {
func {
var0 = "foo"
border = { 1, 1, 1, 1 }
var1 = "bar"
}
}
}
mainfunc {
var = "baz"
engine "map" {
func {
var0 = "foo"
border = { 1, 1, 1, 1 }
var1 = "foobar"
}
}
}
$ awk '/engine/{c=1}c{b=b?b"\n"$0:$0;if(/{/)a++;if(/}/)a--;if(!a){if(b!~/foobar/)print b;c=0;b="";next}}!c' a #/{/a++;/}/a-- 计算{}数量，看是否在一个区间
mainfunc {
var = "baz"
engine "map" {
func {
var0 = "foo"
border = { 1, 1, 1, 1 }
var1 = "bar"
}
}
}
mainfunc {
var = "baz"
}
把"#"改为index效果
#) Title #) Title #) Title #.#) Subtitle #.#.#) Section #.#) Subtitle #) Title #) Title #.#) Subtitle #.#.#) Section #.#) Subtitle #.#.#) Section #.#.#.#) Subsection #) Title #) Title #.#) Subtitle #.#.#) Section #.#.#.#) Subsection #.#.#.#) Subsection
awk 'function w(){
k=m>s?m:s
for(i=1;i<=k;i++){
if(i>m){
a[i]=0
}
else{
a[i]=(i==m)?++a[i]:a[i]
sub("#",a[i]=a[i]?a[i]:1)
}
}
s=m
}
{m=split($1,t,"#")-1;w()}1' file
1) Title
2) Title
3) Title
3.1) Subtitle
3.1.1) Section
3.2) Subtitle
4) Title
5) Title
5.1) Subtitle
5.1.1) Section
5.2) Subtitle
5.2.1) Section
5.2.1.1) Subsection
6) Title
7) Title
7.1) Subtitle
7.1.1) Section
7.1.1.1) Subsection
7.1.1.2) Subsection