训练中文分词HMM模型，得到A（状态转移矩阵）、B（混淆矩阵）、Pi（初始状态概率）

#!F://python
# page coding=utf-8
#状态转移矩阵：状态之间的转移概率      B（混淆矩阵）：状态为y的情况下，输出为x的概率  Pi：初始状态为s的状态
# 本代码用来训练中文分词HMM模型，得到A矩阵（状态转移矩阵）、B矩阵（混淆矩阵）、Pi向量（初始概率向量）并且用pickle 将他们的utf-8码写到了文件当中去
import pickle
import codecs
A_dic = {}    # 状态转移矩阵
B_dic = {}    # 混淆矩阵
Pi_dic = {}   # 状态初始概率向量
state_list = ['B', 'M', 'E', 'S']
state_M = 4
word_N = 0
Pi_dic_size = 0.0
A_row_count_dic = {}  # 记录A[key]所在行的所有列的总和，也就是在求解A_dic每行的每一个值候(A[key1][key2])，分母就是Count_dic[key]
B_dic_element_size = {}  # 这个词典中记录的是，混淆矩阵中的对应着同一个输出字符的隐藏状态的数量
PROB_SATRT = "prob_start.py"   # 初始概率向量写在这里
INPUT_DATA = "RenMinData.txt"  # 用于训练的语料
PROB_EMIT = "prob_emit.py"     # 混淆矩阵
PROB_TRANS = "prob_trans.py"   # 状态转移矩阵


def init():
    global Pi_dic
    global B_dic
    global A_dic
    global A_row_count_dic
    for state in state_list:
        A_dic[state] = {}
        for state1 in state_list:
            A_dic[state][state1] = 0.0
    for state in state_list:
        Pi_dic[state] = 0.0  # 初始化状态初始概率向量中所有的元素值为0.0
        B_dic[state] = {}    # 初始化混淆矩阵中所有的元素对应为一个词典
        A_row_count_dic[state] = 0
    # print B_dic,"over"


def getList(input_str):     # 生成”山棱“=》“BE” ”君“=》“S” "才敢与君绝"=》“BMMME”
    output_str = []
    input_str_length = len(input_str)
    if input_str_length == 1:
        output_str.append('S')
    else:
        Middle_Num = input_str_length-2
        output_str.append('B')
        output_str.extend(['M']*Middle_Num)  # list.extend(list)直接将一个list合并到另一个list中
        output_str.append('E')
    return output_str


def main(train_file_path):
    init()
    global Pi_dic_size
    global word_N
    global B_dic_element_size
    global A_row_count_dic
    global B_dic
    train_file = codecs.open("copy.txt", "rb", "utf8")
    train_file.read(1)       # utf-8文件读取时候第一个字符的位置是不可见字符，所以要踢掉
    for line in train_file:  # line=山无棱  天地合
        if not line:
            continue
        word_list = line.split(" ")  # word_list:['山无棱'，‘天地合’]
        line_state=[]
        for word in word_list:
            line_state.append(getList(word))  # line_state:[[BME],[BME]]
        print line_state
        if len(line_state)!=len(word_list):
            print "different length for a word and the corresponding state"
            return
        for i in range(len(line_state)):  # 这里的i是第几个[B,M,E]
            Pi_dic[line_state[i][0]] += 1     # 构建初始向量
            Pi_dic_size += len(line_state)
            for j in range(len(word_list[i])):  # word_list[i]:山无棱   #line_state[i]:['B','M','E']
                if word_list[i][j] not in B_dic[line_state[i][j]]:
                    utfWordList=word_list[i][j].encode('utf-8')
                    B_dic[line_state[i][j]][utfWordList] = 1.0  # 构建混淆矩阵:line_state:word_list
                else:
                    B_dic[line_state[i][j]][word_list[i][j].encode('utf-8')] += 1
                if word_list[i][j] not in B_dic_element_size:
                    B_dic_element_size[word_list[i][j].encode('utf-8')] = 1
                else:
                    B_dic_element_size[word_list[i][j].encode('utf-8')] += 1
                if j < len(line_state[i]) - 1:
                    A_dic[line_state[i][j]][line_state[i][j + 1]] += 1  # 状态转移向量
                    A_row_count_dic[line_state[i][j]] += 1
    print B_dic_element_size
    train_file.close()
    probs()


def probs():
    PROB_SATRT = "prob_start.py"  # 初始概率向量写在这里
    INPUT_DATA = "RenMinData.txt"  # 用于训练的语料
    PROB_EMIT = "prob_emit.py"  # 混淆矩阵
    PROB_TRANS = "prob_trans.py"  # 状态转移矩阵
    global Pi_dic
    global Pi_dic_size
    global B_dic
    global A_dic
    global B_dic_element_size
    global A_row_count_dic
    start_fp = open(PROB_SATRT, 'w')
    emit_fp = open(PROB_EMIT, 'w')
    trans_fp = open(PROB_TRANS, 'w')
    print "-------------------以下Pi向量------------------------"
    for key in Pi_dic:
        Pi_dic[key] = Pi_dic[key] / Pi_dic_size
    print Pi_dic
    print "-------------------以下是状态转移矩阵------------------------"
    for key in A_dic:
        for key2 in A_dic[key]:
            if A_row_count_dic[key] != 0:
                A_dic[key][key2] = A_dic[key][key2]/A_row_count_dic[key]
    print A_dic
    print "------------------以下是混淆矩阵-----------------------"
    for key in B_dic:
        for key1 in B_dic[key]:
            B_dic[key][key1]=B_dic[key][key1]/B_dic_element_size[key1]
    for item in B_dic:
        for key in B_dic[item] :
            print item, '-->', key,B_dic[item][key],'   ',
    # for key in B_dic:

    pickle.dump(A_dic, start_fp)
    pickle.dump(B_dic, emit_fp)
    pickle.dump(Pi_dic, trans_fp)

    # for key in Pi_dic:
    #     start_fp.write(key+" : "+str(Pi_dic[key]).encode('utf-8'))
    #     start_fp.write("\n")
    # for key in B_dic:
    #     # emit_fp.write()
    #     emit_fp.write(key+" : ")
    #     for key2 in B_dic[key]:
    #         emit_fp.write(key2.encode('utf-8'))
    #     emit_fp.write("\n")
    # for key in A_dic:
    #     trans_fp.write(key+" : "+str(A_dic[key]).encode('utf-8'))
    #     trans_fp.write("\n")
    start_fp.close()
    emit_fp.close()
    trans_fp.close()
main("RenMinData.txt")