bert细节适配：添加词表之外的词和标点符号的更好处理

bert细节适配：添加词表之外的单词和标点符号的处理细节

由于bert中主要为中文，所以词表中英文单词比较少，但是一般英文单词如果简单的直接使用tokenize函数，往往在一些序列预测问题上存在一些对齐问题，或者很多不存在的单词或符号没办法处理而直接使用　unk　替换了，某些英文单词或符号失去了单词的预训练效果，所以采用以下一种更缓和的方式，来进行ＢＥＲＴ的适配，可以提高模型在中英文文本下，预训练模型的效果

通过重写Tokenize类
①处理vocab中不存在的标点符号，使用替代方式
②不存在的单词，正向匹配　##后缀　的词，一定程度上有接近语义或词性

不过segment_id需要自己计算，一般单句就是全部为0的列表了，即　segment_ids = [0] * len(token_ids)

# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors.


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import re
import unicodedata
import six
import tensorflow as tf


def load_vocab(vocab_file):
  """Loads a vocabulary file into a dictionary."""
  vocab = collections.OrderedDict()
  index = 0
  with tf.gfile.GFile(vocab_file, "r") as reader:
    for token in reader.readlines():
      if not token:
        break
      token = token.strip()
      vocab[token] = index
      index += 1
  return vocab


def convert_by_vocab(vocab, items):
  """Converts a sequence of [tokens|ids] using the vocab."""
  output = []
  for item in items:
    output.append(vocab[item])
  return output


def convert_tokens_to_ids(vocab, tokens):
  return convert_by_vocab(vocab, tokens)


def convert_ids_to_tokens(inv_vocab, ids):
  return convert_by_vocab(inv_vocab, ids)


def whitespace_tokenize(text):
  """Runs basic whitespace cleaning and splitting on a piece of text."""
  text = text.strip()
  if not text:
    return []
  tokens = text.split()
  return tokens


class BasicTokenizer(object):
  """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""

  def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200, do_lower_case=True):
    """Constructs a BasicTokenizer.
    Args:
      do_lower_case: Whether to lower case the input.
    """
    self.do_lower_case = do_lower_case
    self.vocab = vocab
    self.unk_token = unk_token
    self.max_input_chars_per_word = max_input_chars_per_word

    
  def tokenize(self, text):
    """Tokenizes a piece of text."""
    text = self._clean_text(text)

    # This was added on November 1st, 2018 for the multilingual and Chinese
    # models. This is also applied to the English models now, but it doesn't
    # matter since the English models were not trained on any Chinese data
    # and generally don't have any Chinese data in them (there are Chinese
    # characters in the vocabulary because Wikipedia does have some Chinese
    # words in the English Wikipedia.).
    text = self._tokenize_chinese_chars(text)

    orig_tokens = whitespace_tokenize(text)
    split_tokens = []
    for token in orig_tokens:
      if self.do_lower_case:
        token = token.lower()
        token = self._run_strip_accents(token)
      split_tokens.extend(self._run_split_on_punc(token))

    output_tokens = whitespace_tokenize(" ".join(split_tokens))
    return output_tokens

  # 处理不在词表中的词
  def wordpiecetoken(self, tokens):
    output_tokens = []
    for token in tokens:
      if token in self.vocab:
          output_tokens.append(token)
          continue

      chars = list(token)
      # 如果超出长度，则用ｕｎｋ
      if len(chars) > self.max_input_chars_per_word:
        output_tokens.append(self.unk_token)
        continue

      is_bad = False
      start = 0
      sub_tokens = []
      end = len(chars)
      while end > 1:
        cur_substr = None
        while start < end:
          substr = "".join(chars[start:end])
          if start > 0:
            substr = "##" + substr
          if substr in self.vocab:
            cur_substr = substr
            break
        # end -= 1
          start += 1
        if cur_substr is not None:
          sub_tokens.append(cur_substr)
          break
        end -= 1

      if cur_substr is None:
       is_bad = True

      if is_bad:
        output_tokens.append(self.unk_token)
      else:
        output_tokens.extend(sub_tokens)
    return output_tokens
　# 处理vocab中不存在的标点符号，使用替代方式
  def deal_punctuation(self, c):
    r = None
    if c == '“':
      r = '"'
    elif c == '”':
      r = '"'
    elif c == '“':
      r = '"'
    elif c == '“':
      r = '"'
    elif c == '—':
      r = '-'
    elif c == '…':
      r = '...'
    elif c == '……':
      r = '...'
    else:
      r = c
    return r

  def deal_punctuations(self, tokens):
    R = []
    for token in tokens:
      token = self.deal_punctuation(token)
      R.append(token)
    return R


  def encode(self, text, add_cls_sep=True):
    tokens = self.tokenize(text)
    # print(len(tokens))
    # print(tokens)
    tokens = self.deal_punctuations(tokens)
    wordpiecetokens = self.wordpiecetoken(tokens)
    print(wordpiecetokens)
    token_ids = convert_tokens_to_ids(self.vocab, wordpiecetokens)
    if add_cls_sep:
        token_ids.insert(0,self.vocab['[CLS]'])
        token_ids.append(self.vocab['[SEP]'])
    return token_ids


  def _run_strip_accents(self, text):
    """Strips accents from a piece of text."""
    text = unicodedata.normalize("NFD", text)
    output = []
    for char in text:
      cat = unicodedata.category(char)
      if cat == "Mn":
        continue
      output.append(char)
    return "".join(output)

  def _run_split_on_punc(self, text):
    """Splits punctuation on a piece of text."""
    chars = list(text)
    i = 0
    start_new_word = True
    output = []
    while i < len(chars):
      char = chars[i]
      if _is_punctuation(char):
        output.append([char])
        start_new_word = True
      else:
        if start_new_word:
          output.append([])
        start_new_word = False
        output[-1].append(char)
      i += 1

    return ["".join(x) for x in output]

  def _tokenize_chinese_chars(self, text):
    """Adds whitespace around any CJK character."""
    output = []
    for char in text:
      cp = ord(char)
      if self._is_chinese_char(cp):
        output.append(" ")
        output.append(char)
        output.append(" ")
      else:
        output.append(char)
    return "".join(output)

  def _is_chinese_char(self, cp):
    """Checks whether CP is the codepoint of a CJK character."""
    # This defines a "chinese character" as anything in the CJK Unicode block:
    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
    #
    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
    # despite its name. The modern Korean Hangul alphabet is a different block,
    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
    # space-separated words, so they are not treated specially and handled
    # like the all of the other languages.
    if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
        (cp >= 0x3400 and cp <= 0x4DBF) or  #
        (cp >= 0x20000 and cp <= 0x2A6DF) or  #
        (cp >= 0x2A700 and cp <= 0x2B73F) or  #
        (cp >= 0x2B740 and cp <= 0x2B81F) or  #
        (cp >= 0x2B820 and cp <= 0x2CEAF) or
        (cp >= 0xF900 and cp <= 0xFAFF) or  #
        (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
      return True

    return False

  def _clean_text(self, text):
    """Performs invalid character removal and whitespace cleanup on text."""
    output = []
    for char in text:
      cp = ord(char)
      if cp == 0 or cp == 0xfffd or _is_control(char):
        continue
      if _is_whitespace(char):
        output.append(" ")
      else:
        output.append(char)
    return "".join(output)


def _is_whitespace(char):
  """Checks whether `chars` is a whitespace character."""
  # \t, \n, and \r are technically contorl characters but we treat them
  # as whitespace since they are generally considered as such.
  if char == " " or char == "\t" or char == "\n" or char == "\r":
    return True
  cat = unicodedata.category(char)
  if cat == "Zs":
    return True
  return False


def _is_control(char):
  """Checks whether `chars` is a control character."""
  # These are technically control characters but we count them as whitespace
  # characters.
  if char == "\t" or char == "\n" or char == "\r":
    return False
  cat = unicodedata.category(char)
  if cat in ("Cc", "Cf"):
    return True
  return False


def _is_punctuation(char):
  """Checks whether `chars` is a punctuation character."""
  cp = ord(char)
  # We treat all non-letter/number ASCII as punctuation.
  # Characters such as "^", "$", and "`" are not in the Unicode
  # Punctuation class but we treat them as punctuation anyways, for
  # consistency.
  if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or
      (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
    return True
  cat = unicodedata.category(char)
  if cat.startswith("P"):
    return True
  return False


def main():
    import os

    here = os.path.dirname(os.path.abspath(__file__))
    vocab = load_vocab(os.path.join(here, 'chinese_L-12_H-768_A-12','vocab.txt'))
    bsc = BasicTokenizer(vocab=vocab)
    s = bsc.encode('今天和ｂｏyfriend出去散步,!１我们心情很那NIce人也很BEAtiFUlly…对吗')
    # print('lens', len(s))
    print(s)
    # ['今', '天', '和', '##end', '出', '去', '散', '步', ',', '!', '１', '我', '们', '心', '情', '很', '那', 'nice', '人', '也', '很', '##lly', '...', '对', '吗']
 #[101, 791, 1921, 1469, 11652, 1139, 1343, 3141, 3635, 117, 106, 8029, 2769, 812, 2552, 2658, 2523, 6929, 10192, 782, 738, 2523, 9456, 8106, 2190, 1408, 102]

if __name__ == "__main__":
    main()