使用python开发敏感词检测过滤系统

这里使用python 的 bottle框架来做简易的敏感词过滤系统，算法采用成熟的DFA关键词匹配算法，本系统只提供一套基于http的 api，可以提供给各个应用使用。这里我只把最核心的业务实现，其他的再完善。

检索算法是网上找的DFA算法的python实现版本

smallgfw.py

#encoding=utf-8
#DFA based text filter
#version=0.3
class GFW(object):
    def __init__(self):
        self.d = {}
    
    #give a list of "ming gan ci"
    def set(self,keywords):
        p = self.d
        q = {}
        k = ''
        for word in keywords:
            word += chr(11)
            p = self.d
            for char in word:
                char = char.lower()
                if p=='':
                    q[k] = {}
                    p = q[k]
                if not (char in p):
                    p[char] = ''
                    q = p
                    k = char
                p = p[char]
        
        pass
    
    def replace(self,text,mask):
        """
        >>> gfw = GFW()
        >>> gfw.set(["sexy","girl","love","shit"])
        >>> s = gfw.replace("Shit!,Cherry is a sexy girl. She loves python.","*")
        >>> print s
        *!,Cherry is a * *. She *s python.
        """
        p = self.d
        i = 0 
        j = 0
        z = 0
        result = []
        ln = len(text)
        while i+j<ln:
            #print i,j
            t = text[i+j].lower()
            #print hex(ord(t))
            if not (t in p):
                j = 0
                i += 1
                p = self.d
                continue
            p = p[t]
            j+=1
            if chr(11) in p:
                p = self.d
                result.append(text[z:i])
                result.append(mask)
                i = i+j
                z = i
                j = 0
        result.append(text[z:i+j])
        return "".join(result)
        
    def check(self,text):
        """
        >>> gfw = GFW()
        >>> gfw.set(["abd","defz","bcz"])
        >>> print gfw.check("xabdabczabdxaadefz")
        [(1, 3, 'abd'), (5, 3, 'bcz'), (8, 3, 'abd'), (14, 4, 'defz')]
        """
        p = self.d
        i = 0 
        j = 0
        result = []
        ln = len(text)
        while i+j<ln:
            t = text[i+j].lower()
            #print i,j,hex(ord(t))
            if not (t in p):
                j = 0
                i += 1
                p = self.d
                continue
            p = p[t]
            j+=1
            #print p,i,j
            if chr(11) in p:
                p = self.d
                result.append((i,j,text[i:i+j]))
                i = i+j
                j = 0
        return result
        
if __name__=="__main__":
    import doctest,sys
    doctest.testmod(sys.modules[__name__])
    

    

然后编写bottle框架的api文件代码，localbottle.py

#-*- coding:utf-8 -*-
#localhost testing
#caroltc 2014/10/7
from bottle import route, run, request
from smallgfw import *
import json
import sys

def initWords():
    path = 'words.txt'  
    fp = open(path,'r')  
    word_list = []  
    for line in fp:  
        line = line[0:-1]  
        word_list.append(line)  
    fp.close()
    return word_list

@route('/replace', method="POST")
def replace():
    reload(sys)
    sys.setdefaultencoding('utf8')
    getwords = request.params.words or ""
    gfw = GFW()
    words = initWords()
    gfw.set(words)#设置敏感词列表
    res = gfw.check(getwords.encode('utf8'))
#    for obj in res:
#        print json.dumps(obj),obj[2]
    s = gfw.replace(getwords.encode('utf8'),"**")
    return s

@route('/check',method="POST")
def check():
    reload(sys)
    sys.setdefaultencoding('utf8')
    getwords = request.params.words or ""
    gfw = GFW()
    words = initWords()
    gfw.set(words)#设置敏感词列表
    res = gfw.check(getwords.encode('utf8'))
    resp = {}
    resp['count'] = len(res)
    resp['datas']= res
    return json.dumps(resp)

@route('/test')
def test():
    reload(sys)
    sys.setdefaultencoding('utf8')
    webdata = '<h1>check</h1><form action="/replace" method="post"><input type="text" name="words" /><input type="submit"></from>'
    return webdata


run(host='localhost', port=80, debug=True)

敏感词文件为words.txt,每行一个词就可以了，该文件采用gb2312编码，程序均为utf8编码

测试一下api，均为POST请求

过滤敏感词API，直接返回过滤后的数据

检测敏感词API，返回json格式数据

用bottle来开发这样的小工具相当快，而且敏感词检测系统在很多应用场景都需要，独立出来写成接口可以提高效率，并且易于维护，国内目前第三方敏感词检测服务还不多，天朝的需求又很旺盛，可以试试搞个在线敏感词检测服务平台。