<jupyter_start><jupyter_text>Инициализация<jupyter_code>import os
import numpy as np
import pymorphy2
from tqdm import tqdm_notebook
token_filenames = [f for f in os.listdir('./devset/') if '.tokens' in f]<jupyter_output><empty_output><jupyter_text>Функции для работы с файлами<jupyter_code>class Token:
    def __init__(self, position, length, text):
        self._position = position
        self._length = length
        self._text = text
        self._pos = None
        self._tag = None
class Span:
    def __init__(self, token_id):
        self._token_id = token_id
def load_tokens(token_filename, dev=True):
    _set = './devset/' if dev else './testset/'
    tokens = dict()
    with open(_set + token_filename, encoding='utf8') as f:
        for line in f:
            split = line.split()
            if len(split) > 0:
                t = Token(split[1], split[2], split[3])
                tokens[split[0]] = t
    return tokens
def load_spans(token_filename, dev=True):
    _set = './devset/' if dev else './testset/'
    spans = dict()
    with open(_set + token_filename.split('.')[0] + '.spans', encoding='utf8') as f:
        for line in f:
            split = line.split()
            s = Span(split[4])
            spans[split[0]] = s
    return spans
def transform_base_tag(base_tag):
    if base_tag == 'Person':
        return 'PER'
    if base_tag == 'Location' or base_tag == 'LocOrg':
        return 'LOC'
    if base_tag == 'Org':
        return 'ORG'
    else:
        return 'MISC'
def load_objects(token_filename, tokens, spans, dev=True):
    _set = './devset/' if dev else './testset/'
    with open(_set + token_filename.split('.')[0] + '.objects', encoding='utf8') as f:
        for line in f:
            line = line.split(' # ')[0]
            split = line.split()
            base_tag = transform_base_tag(split[1])
            span_ids = split[2:]
            if len(span_ids) == 1:
                tokens[spans[span_ids[0]]._token_id]._tag = 'U-' + base_tag
            else:
                for i, span_id in enumerate(span_ids):
                    if i == 0:
                        tokens[spans[span_ids[i]]._token_id]._tag = 'B-' + base_tag
                    if i == len(span_ids) - 1:
                        tokens[spans[span_ids[i]]._token_id]._tag = 'L-' + base_tag
                    else:
                        tokens[spans[span_ids[i]]._token_id]._tag = 'I-' + base_tag
    return tokens
morph = pymorphy2.MorphAnalyzer()
def fill_pos(tokens):
    for id, token in tokens.items():
        pos = morph.parse(token._text)[0].tag.POS
        if pos is None:
            pos = 'None'
        token._pos = pos
        if token._tag is None:
            token._tag = 'O'
    return tokens
def word2features(sent, i):
    word = sent[i]._text
    postag = sent[i]._pos

    features = {
        'bias': 1.0,
        'word.lower()': word.lower(),
        'word[-3:]': word[-3:],
        'word[-2:]': word[-2:],
        'word.isupper()': word.isupper(),
        'word.istitle()': word.istitle(),
        'word.isdigit()': word.isdigit(),
        'postag': postag,
    }
    if i > 0:
        word1 = sent[i-1]._text
        postag1 = sent[i-1]._pos
        features.update({
            '-1:word.lower()': word1.lower(),
            '-1:word.istitle()': word1.istitle(),
            '-1:word.isupper()': word1.isupper(),
            '-1:postag': postag1,
        })
    else:
        features['BOS'] = True

    if i < len(sent)-1:
        word1 = sent[i+1]._text
        postag1 = sent[i+1]._pos
        features.update({
            '+1:word.lower()': word1.lower(),
            '+1:word.istitle()': word1.istitle(),
            '+1:word.isupper()': word1.isupper(),
            '+1:postag': postag1,
        })
    else:
        features['EOS'] = True

    return features


def sent2features(sent):
    return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):
    return [token._tag for token in sent]

def sent2tokens(sent):
    return [token for token in sent]
def split_tokens_by_sents(tokens):
    sents = []
    sent = []
    for id, token in tokens.items():
        if token._text != '.':
            sent.append(token)
        else:
            sents.append(sent)
            sent = []
    return sents<jupyter_output><empty_output><jupyter_text>Готовим тренировочную выборку<jupyter_code>def generate_sents(token_filename):
    tokens = load_tokens(token_filename)
    spans = load_spans(token_filename)
    tokens = load_objects(token_filename, tokens, spans)
    tokens = fill_pos(tokens)
    sents = split_tokens_by_sents(tokens)
    return sents
sents = []
for token_filename in tqdm_notebook(token_filenames):
    sents += generate_sents(token_filename)
len(sents)
from sklearn.model_selection import train_test_split
import numpy as np

train_ids, test_ids = train_test_split(np.arange(len(sents)))
train_sents = np.array(sents)[train_ids]
test_sents = np.array(sents)[test_ids]
X_train = [sent2features(s) for s in train_sents]
y_train = [sent2labels(s) for s in train_sents]

X_test = [sent2features(s) for s in test_sents]
y_test = [sent2labels(s) for s in test_sents]<jupyter_output><empty_output><jupyter_text>Тренируем модель<jupyter_code>from sklearn_crfsuite import CRF

crf = CRF(algorithm='lbfgs', c1=0.1, c2=0.1, max_iterations=100, all_possible_transitions=True)
crf.fit(X_train, y_train)
from sklearn_crfsuite.metrics import flat_classification_report

print(flat_classification_report(y_test, crf.predict(X_test)))<jupyter_output>precision    recall  f1-score   support

      I-LOC       0.42      0.24      0.30        21
      I-ORG       0.39      0.17      0.24        80
      I-PER       0.89      0.96      0.92       112
      L-LOC       0.27      0.17      0.21        18
      L-ORG       0.41      0.17      0.24        64
      L-PER       0.88      0.99      0.93       106
          O       0.98      0.99      0.99      6126
      U-LOC       0.75      0.88      0.81       179
      U-ORG       0.80      0.66      0.72       101
      U-PER       0.79      0.45      0.57        76

avg / total       0.95      0.96      0.95      6883<jupyter_text>Применяем модель<jupyter_code>def get_entities(tokens):
    rows = []

    buffer = []
    for id, token in tokens.items():
        tag = token._tag
        if tag.startswith('U'):
            rows.append('%s %d %d\n' % (tag.split('-')[1], int(token._position), int(token._length)))
        elif tag.startswith('B') or tag.startswith('I'):
            buffer.append(token)
        elif tag.startswith('L'):
            buffer.append(token)
            start = int(buffer[0]._position)
            length = int(buffer[-1]._position) + int(buffer[-1]._length) - int(start)
            rows.append('%s %d %d\n' % (tag.split('-')[1], start, length))
            buffer = []
    return rows
test_token_filenames = [filename for filename in os.listdir('./testset') if '.tokens' in filename]
for token_filename in tqdm_notebook(test_token_filenames):
    tokens = load_tokens(token_filename, dev=False)
    tokens = fill_pos(tokens)
    sents = split_tokens_by_sents(tokens)
    X = [sent2features(s) for s in sents]
    y_pred = crf.predict(X)
    for i in range(len(y_pred)):
        for j in range(len(y_pred[i])):
            sents[i][j]._tag = y_pred[i][j]
    rows = get_entities(tokens)
    with open('./results_crf/' + token_filename.split('.')[0] + '.task1', 'w') as f:
        f.writelines(rows)<jupyter_output><empty_output><jupyter_text>Проверяем результаты<jupyter_code>!python scripts\t1_eval.py -s .\testset -t .\results_crf -o .\output\<jupyter_output>Type    P        R        F1       TP1      TP2      In Std.  In Test.
per        0.7470   0.6666   0.7045   897.88   897.88     1347     1202
loc        0.3239   0.7976   0.4607   477.78   477.78      599     1475
org        0.7129   0.3661   0.4837   583.87   583.87     1595      819
locorg     1.0000   0.0000   0.0000     0.00     0.00      633        0
overall    0.5605   0.4695   0.5110  1959.54  1959.54     4174     3496
{'fraction_non_alphanumeric': 0.13123294467873978, 'fraction_numerical': 0.046018357727611016, 'mean_word_length': 2.115533230293663, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '507971', 'license': 'MIT', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2830, 'n_words': 620, 'path': 'NER CRF.ipynb', 'repo_name': 'aminought/factRuEval-2016-practice', 'size': '8062'}