fastNLP和spaCy的使用

import glob
from fastNLP import Instance

def transfer_sentence(df_input):
    """
    将标注好的句子转为实例

    return: 将单个数据集的特征打包为一个数据示例(Instance)
    """
    words = []
    targets = []
    raw_chars = ''.join(df_input.内容)
    idx = 0
    # 遍历原始数据集,只保留符合标注规范的数据
    while idx < len(df_input.index):
        if df_input.loc[idx,'标记'] == 'O':
            words.append(df.loc[idx,'内容'])
            idx += 1
        elif df_input.loc[idx,'标记'].startswith('B'):
            targets.append(df_input.loc[idx,'标记'][2:])
            tmp = ''
            tmp += df_input.loc[idx,'内容']
            idx += 1
            while idx < len(df_input.index) and df_input.loc[idx,'标记'].startswith('I'):
                tmp += df_input.loc[idx,'内容']
                idx += 1
            words.append(tmp)
        else:
            idx += 1
            return
    # 如果目标列表为空,说明数据集中没有特征数据,丢弃
    if not targets:
        return
    return Instance(
        raw_chars = raw_chars,
        chars = list(df_input.内容),
        target = list(df_input.标记),
        seq_len = len(df_input.标记),
        words = words,
        whole_targets = targets
    )

dataset = DataSet()
for file_path in file_path_list:
    df = pd.read_excel(file_path,index_col=0)
    ins = transfer_sentence(df)
    if ins:
        dataset.append(ins)

+-------------------+------------------+------------------+---------+------------------+-------------------+
| raw_chars         | chars            | target           | seq_len | words            | whole_targets     |
+-------------------+------------------+------------------+---------+------------------+-------------------+
| 代诉：右肺癌,... | ['代', '诉', ... | ['O', 'O', 'O... | 77      | ['代', '诉', ... | ['ZZ', 'ZZ', '... |
| 代诉：服上药后... | ['代', '诉', ... | ['O', 'O', 'O... | 114     | ['代', '诉', ... | ['BW', 'BW', '... |
| 右下肺占位,右...  | ['右', '下', ... | ['O', 'O', 'O... | 94      | ['右', '下', ... | ['ZZ', 'ZZ', '... |
| 右上肺占位,经...  | ['右', '上', ... | ['O', 'O', 'O... | 133     | ['右', '上', ... | ['ZZ', 'ZZ', '... |
| 今年6月左右开...  | ['今', '年', ... | ['O', 'O', 'O... | 100     | ['今', '年', ... | ['ZZ', 'ZZ', '... |
| 自觉体力有所改... | ['自', '觉', ... | ['O', 'O', 'O... | 56      | ['自', '觉', ... | ['ZZ', 'ZZ', '... |
| 疲劳乏力,右胸...  | ['疲', '劳', ... | ['B-ZZ', 'I-Z... | 55      | ['疲劳乏力', ... | ['ZZ', 'BW', '... |
| 2000年右肺细胞... | ['2', '0', '0... | ['O', 'O', 'O... | 86      | ['2', '0', '0... | ['CD', 'ZZ', '... |
| 午后身热,作业...  | ['午', '后', ... | ['O', 'O', 'O... | 52      | ['午', '后', ... | ['ZZ', 'ZZ', '... |
| 右肺94年癌手术... | ['右', '肺', ... | ['O', 'O', 'O... | 72      | ['右', '肺', ... | ['ZZ', 'CD', '... |
+-------------------+------------------+------------------+---------+------------------+-------------------+

vocab = Vocabulary()
#  从该dataset中的chars列建立词表
vocab.from_dataset(dataset, field_name='chars')
#  使用vocabulary将chars列转换为index
vocab.index_dataset(dataset, field_name='chars')

target_vocab = Vocabulary(unknown=None, padding=None)
#  从该dataset中的chars列建立词表
target_vocab.from_dataset(dataset, field_name='target')
#  使用vocabulary将chars列转换为index
target_vocab.index_dataset(dataset, field_name='target')

# 重命名列名, fastnlp这里似乎是写死的, 只有words能被识别读取
dataset.rename_field('chars', 'words')

# *设置输入列和目标列
dataset.set_input('words','target','seq_len')
dataset.set_target('target','seq_len')

# 设置训练集和开发集
train  = dataset[:1750]
dev = dataset[1750:1900]

embed = StaticEmbedding(vocab=vocab, model_dir_or_name='cn-char-fastnlp-100d')

model = BiLSTMCRF(embed=embed, num_classes=len(target_vocab), num_layers=1, hidden_size=200, dropout=0.5, target_vocab=target_vocab)
# 使用其他的模型结构
# model = SeqLabeling(embed=embed, num_classes=len(target_vocab), hidden_size=200)
# model = AdvSeqLabel(embed=embed, num_classes=len(target_vocab), hidden_size=200, dropout=0.5)

metric = SpanFPreRecMetric(tag_vocab=target_vocab)
optimizer = Adam(model.parameters(), lr=1e-2)
loss = LossInForward()

device= 0 if torch.cuda.is_available() else 'cpu'
trainer = Trainer(train, model, loss=loss, optimizer=optimizer,
                    dev_data=dev, metrics=metric, device=device,n_epochs=100)
trainer.train()

import spacy

nlp = spacy.load("./output/model-best")
doc = nlp('巅顶疼痛基本稳定,但转侧不舒,昏重,视物稍糊,不咳,无痰,食纳少味,舌面下唇粘膜辣痛,口不干,尿少,大便正常')

for ent in doc.ents:
    print(ent.text, ent.start_char, ent.end_char, ent.label_)

# This is an auto-generated partial config. To use it with 'spacy train'
# you can run spacy init fill-config to auto-fill all default settings:
# python -m spacy init fill-config ./base_config.cfg ./config.cfg
[paths]
train = null
dev = null

[system]
gpu_allocator = "pytorch"

[nlp]
lang = "zh"
pipeline = ["transformer","ner"]
batch_size = 128

[components]

[components.transformer]
factory = "transformer"

[components.transformer.model]
@architectures = "spacy-transformers.TransformerModel.v1"
name = "bert-base-chinese"
tokenizer_config = {"use_fast": true}

[components.transformer.model.get_spans]
@span_getters = "spacy-transformers.strided_spans.v1"
window = 128
stride = 96

[components.ner]
factory = "ner"

[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = false
nO = null

[components.ner.model.tok2vec]
@architectures = "spacy-transformers.TransformerListener.v1"
grad_factor = 1.0

[components.ner.model.tok2vec.pooling]
@layers = "reduce_mean.v1"

[corpora]

[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 500

[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0

[training]
accumulate_gradient = 3
dev_corpus = "corpora.dev"
train_corpus = "corpora.train"

[training.optimizer]
@optimizers = "Adam.v1"

[training.optimizer.learn_rate]
@schedules = "warmup_linear.v1"
warmup_steps = 250
total_steps = 20000
initial_rate = 5e-5

[training.batcher]
@batchers = "spacy.batch_by_padded.v1"
discard_oversize = true
size = 2000
buffer = 256

[initialize]
vectors = null