NLP:定名实体识别及案例(Bert微调) - IT评测·应用市场-qidao123.com技术社区

{
"chunk_tags": [11, 12, 12, 21, 13, 11, 11, 21, 13, 11, 12, 13, 11, 21, 22, 11, 12, 17, 11, 21, 17, 11, 12, 12, 21, 22, 22, 13, 11, 0],
"id": "0",
"ner_tags": [0, 3, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"pos_tags": [12, 22, 22, 38, 15, 22, 28, 38, 15, 16, 21, 35, 24, 35, 37, 16, 21, 15, 24, 41, 15, 16, 21, 21, 20, 37, 40, 35, 21, 7],
"tokens": ["The", "European", "Commission", "said", "on", "Thursday", "it", "disagreed", "with", "German", "advice", "to", "consumers", "to", "shun", "British", "lamb", "until", "scientists", "determine", "whether", "mad", "cow", "disease", "can", "be", "transmitted", "to", "sheep", "."]
}

复制代码

{'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6, 'B-MISC': 7, 'I-MISC': 8}

复制代码

tokens = ['Ap', '##ple', 'is', 'a', 'technology', 'company']
labels = ['B-ORG', 'I-ORG', 'O', 'O', 'O', 'O']

复制代码

from datasets import load_dataset
from transformers import BertTokenizerFast, BertForTokenClassification
from transformers import TrainingArguments, Trainer
# 加载 CoNLL-2003 数据集
dataset = load_dataset("conll2003")
train_dataset = dataset["train"]
eval_dataset = dataset["validation"]
test_dataset = dataset["test"]
# 加载 BERT tokenizer 和模型
tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
label_list = dataset["train"].features["ner_tags"].feature.names
print(train_dataset[0])
def tokenize_and_align_labels(examples):
tokenized_inputs = tokenizer(examples["tokens"], truncation=True, padding="max_length",
is_split_into_words=True)
labels= []
for i, label in enumerate(examples["ner_tags"]):
word_ids = tokenized_inputs.word_ids(batch_index=i)
previous_word_idx = None
label_ids = []
for word_idx in word_ids:
if word_idx is None:
label_ids.append(-100)
elif word_idx != previous_word_idx:
label_ids.append(label[word_idx])
else:
label_ids.append(label[word_idx])
previous_word_idx = word_idx
labels.append(label_ids)
tokenized_inputs["labels"] = labels
print(labels[0])
print(len(tokenized_inputs.input_ids[0]),len(labels[0]))
return tokenized_inputs
train_dataset = train_dataset.map(tokenize_and_align_labels, batched=True)
eval_dataset = eval_dataset.map(tokenize_and_align_labels, batched=True)
test_dataset = test_dataset.map(tokenize_and_align_labels, batched=True)
model= BertForTokenClassification.from_pretrained("bert-base-cased",
num_labels=len(label_list))
training_args = TrainingArguments(
output_dir='./results', # 输出目录
eval_strategy="epoch", # 每个 epoch 进行评估
learning_rate=2e-5, # 学习率
per_device_train_batch_size=16, # 训练 batch size
per_device_eval_batch_size=16, # 评估 batch size
num_train_epochs=3, # 训练 epoch 数
weight_decay=0.01, # 权重衰减
)
# 使用 Trainer API
trainer = Trainer(
model=model, # 待训练的模型
args=training_args, # 训练参数
train_dataset=train_dataset, # 训练数据集
eval_dataset=eval_dataset, # 验证数据集
tokenizer=tokenizer # 使用的 tokenizer
)
trainer.train()
eval_results = trainer.evaluate()
print(eval_results)
pred_results = trainer.predict(test_dataset)
print(pred_results)

复制代码