如何从apache/nginx日志分析ip的来源并写入到数据库

如题，分析首先最简单的取出ip

cd /usr/local/nginx/logs
awk '{print $1}'  access.log | sort  |  uniq -c  | sort -rn | head -n 50 | awk '{print $2}' > /root/list
#在nginx/apache 下执行上述代码。把ip写到list列表

然后思路就是 把每个ip的来源地分析出来，我用的淘宝的ip数据库，还是不错的

#!/bin/bash

ipInfo() {
  for i in `cat list`
  do
    TransCoding="/usr/bin/python TransCoding.py"
    JsonDate="curl -s http://ip.taobao.com/service/getIpInfo.php?ip=$i"
    country=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==2{print $3}'`
    area=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==4{print $2}'`
    region=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==6{print $2}'`
    city=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==8{print $2}'`
    county=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==10{print $2}'`
    isp=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==12{print $2}'`
    printf "%-18s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\n" $i $country $isp $area $region $city $county
  done
}

printf "%-18s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\n" IP地址 国家 运营商 区域 省份 城市 县/区
echo -e "\e[1;33m======================================================================\e[0m"

ipInfo

转码代码

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
import sys

def main():
    for line in sys.stdin:
        sys.stdout.write(re.sub(r'\\u\w{4}',
            lambda e: unichr(int(e.group(0)[2:], 16)).encode('utf-8'),
            line))

if __name__ == '__main__':
    main()

说明一下，shell用可以从淘宝的api把每个ip地址来源获取出来，但是是json的格式，需要用python把他转化为utf8的格式。

此时解决ip来源查询的，下一步就是写到数据库，很简单，把shell修改下就行了。当然了前提是去数据库手工创建字段咯，数据类型自己定义就ok。

+-----------------+---------+--------------+--------+--------------------------+-----------------+
52 rows in set (0.00 sec)

mysql> quit
Bye
You have new mail in /var/spool/mail/root
[root@10-6-12-175 ~]# vim ip1.sh 
#!/bin/bash

ipInfo() {
  for i in `cat list`
  do
    TransCoding="/usr/bin/python TransCoding.py"
    JsonDate="curl -s http://ip.taobao.com/service/getIpInfo.php?ip=$i"
    country=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==2{print $3}'`
    area=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==4{print $2}'`
    region=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==6{print $2}'`
    city=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==8{print $2}'`
    county=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==10{print $2}'`
    isp=`$JsonDate | sed 's/,/\n/g' | $TransCoding | tr -d "{}\"" | awk -F ":" 'NR==12{print $2}'`
    printf "%-18s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\t%-8s\n" $i $country $isp $area $region $city $county
    mysql -uroot -pxxxxxx  ip_address << EOF
    use ip_address;
    insert into ip_address values ("$i","$country","$isp","$area","$region","$city");
EOF


  done
}

ipInfo

来个mysql的查询界面：

mysql> SELECT * FROM ip_address;
+-----------------+---------+--------------+--------+--------------------------+-----------------+
| ip              | country | isp          | area   | region                   | city            |
+-----------------+---------+--------------+--------+--------------------------+---------
| 101.199.112.45  | 中国    |              | 华北   | 北京市                   | 北京市          |
| 101.226.125.116 | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 101.226.33.199  | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 101.226.33.222  | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 101.226.61.142  | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 101.81.49.203   | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 106.120.160.109 | 中国    | 电信         | 华北   | 北京市                   | 北京市          |
| 111.73.45.242   | 中国    | 电信         | 华东   | 江西省                   | 上饶市          |
| 111.73.46.22    | 中国    | 电信         | 华东   | 江西省                   | 上饶市          |
| 112.65.191.116  | 中国    | 联通         | 华东   | 上海市                   | 上海市          |
| 115.231.218.126 | 中国    | 电信         | 华东   | 浙江省                   | 杭州市          |
| 116.231.78.64   | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 117.131.99.203  | 中国    | 移动         | 华东   | 上海市                   | 上海市          |
| 117.21.226.160  | 中国    | 电信         | 华东   | 江西省                   | 赣州市          |
| 119.161.140.147 | 中国    |              | 华北   | 北京市                   | 北京市          |
| 12.197.154.136  | 美国    |              |        |                          |                 |
| 120.132.72.219  | 中国    | 天地祥云     | 华北   | 北京市                   | 北京市          |
| 120.204.200.42  | 中国    | 移动         | 华东   | 上海市                   | 上海市          |
| 121.32.128.186  | 中国    | 电信         | 华南   | 广东省                   | 广州市          |
| 121.40.83.80    | 中国    | 阿里云       | 华东   | 浙江省                   | 杭州市          |
| 125.211.216.98  | 中国    | 联通         | 东北   | 黑龙江省                 | 哈尔滨市        |
| 140.207.30.50   | 中国    | 联通         | 华东   | 上海市                   | 上海市          |
| 180.153.163.205 | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 180.153.213.141 | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 180.154.173.64  | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 180.173.226.55  | 中国    | 电信         | 华东   | 上海市                   | 上海市          |
| 183.233.169.183 | 中国    | 移动         | 华南   | 广东省                   | 韶关市          |
| 183.61.171.132  | 中国    | 电信         | 华南   | 广东省                   | 东莞市          |
| 192.3.6.2       | 美国    |              |        |                          |                 |
| 198.13.108.236  | 美国    |              |        |                          |                 |
| 203.156.217.33  | 中国    |              | 华东   | 上海市                   | 上海市          |
| 203.171.238.69  | 中国    |              | 华中   | 河南省                   | 郑州市          |
| 216.99.156.172  | 美国    |              |        |                          |                 |
| 218.84.251.209  | 中国    | 电信         | 西北   | 新疆维吾尔自治区         | 乌鲁木齐市      |
| 220.181.132.193 | 中国    | 电信         | 华北   | 北京市                   | 北京市          |
| 220.181.132.215 | 中国    | 电信         | 华北   | 北京市                   | 北京市          |
| 221.2.205.150   | 中国    | 联通         | 华东   | 山东省                   | 聊城市          |
| 221.203.98.43   | 中国    | 联通         | 东北   | 辽宁省                   | 本溪市          |
| 222.174.5.25    | 中国    | 电信         | 华东   | 山东省                   | 泰安市          |
| 222.186.30.115  | 中国    | 电信         | 华东   | 江苏省                   | 镇江市          |
| 222.186.30.82   | 中国    | 电信         | 华东   | 江苏省                   | 镇江市          |
| 222.186.31.130  | 中国    | 电信         | 华东   | 江苏省                   | 镇江市          |
| 222.186.31.149  | 中国    | 电信         | 华东   | 江苏省                   | 镇江市          |
| 222.215.230.233 | 中国    | 电信         | 西南   | 四川省                   | 眉山市          |
| 223.104.3.186   | 中国    | 移动         | 华北   | 北京市                   | 北京市          |
| 42.62.56.151    | 中国    | 森华易腾     | 华北   | 北京市                   | 北京市          |
| 50.7.37.98      | 美国    |              |        |                          |                 |
| 58.217.255.4    | 中国    | 电信         | 华东   | 江苏省                   | 南京市          |
| 59.108.122.184  | 中国    | 方正网络     | 华北   | 北京市                   | 北京市          |
| 61.148.243.106  | 中国    | 联通         | 华北   | 北京市                   | 北京市          |
| 61.164.211.98   | 中国    | 电信         | 华东   | 浙江省                   | 嘉兴市