序列标注任务

基本概念

序列标注

序列标注可以认为是token级别的文本分类。

常见序列标注任务

NER

NER(Named-entity recognition) ：分辨出文本中的名词和实体是 person人名, organization组织机构名, location地点名还是其他。

POS

POS (Part-of-speech tagging)： 对token进行词性标注

Chunk

将同一个短语的tokens组块放在一起

一个NER任务

需要安装的库

• datasets transformers seqeval

数据加载

from datasets import load_dataset, load_metric
datasets = load_dataset("conll2003")
label_list = datasets["train"].features[f"{task}_tags"].feature.names # 查看训练集分类标签

标签：

• ‘O’ for no special entity
• ‘PER’ for person
• ‘ORG’ for organization
• ‘LOC’ for location
• ‘MISC’ for miscellaneous
  每一种实体类别又分别有B-（实体开始的token）前缀和I-（实体中间的token）前缀

数据预处理

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
import transformers
assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)
#tokenize
def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
    labels = []
    for i, label in enumerate(examples[f"{task}_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:
            # Special tokens have a word id that is None. We set the label to -100 so they are automatically
            # ignored in the loss function.
            if word_idx is None:
                label_ids.append(-100) #特殊字符的label设置为-100，在模型中会被忽略掉不计算loss
            # We set the label for the first token of each word.
            elif word_idx != previous_word_idx:
                label_ids.append(label[word_idx])
            # For the other tokens in a word, we set the label to either the current label or -100, depending on
            # the label_all_tokens flag.
            else:
                label_ids.append(label[word_idx] if label_all_tokens else -100)
            previous_word_idx = word_idx

        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs

tokenize_and_align_labels(datasets['train'][:5])
tokenized_datasets = datasets.map(tokenize_and_align_labels, batched=True)

微调预训练模型

from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

task = "ner" 
model_checkpoint = "distilbert-base-uncased"
batch_size = 16

model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))
args = TrainingArguments(
    f"test-{task}",
    evaluation_strategy = "epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=3,
    weight_decay=0.01,
)
#加载数据收集器
from transformers import DataCollatorForTokenClassification
data_collator = DataCollatorForTokenClassification(tokenizer)
#评价指标
import numpy as np
metric = load_metric("seqeval")
def compute_metrics(p):
    predictions, labels = p
    predictions = np.argmax(predictions, axis=2)

    # Remove ignored index (special tokens)
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    true_labels = [
        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]

    results = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": results["overall_precision"],
        "recall": results["overall_recall"],
        "f1": results["overall_f1"],
        "accuracy": results["overall_accuracy"],
    }
# trainer
trainer = Trainer(
    model,
    args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)
# 模型训练
trainer.train()
# 模型评价
trainer.evaluate()

参考

datawhale文档