<commit_before>#!/usr/bin/env python
#
#   Copyright 2014 <NAME>
#
#   gnTEAM, School of Computer Science, University of Manchester.
#   All rights reserved. This program and the accompanying materials
#   are made available under the terms of the GNU General Public License.
#
#   author: <NAME>
#   email:  <EMAIL>
#
#   For details, see www.cs.man.ac.uk/~filannim/

'''It uses the external CRFs algorithms to predict the labels.'''

from __future__ import division
from abc import ABCMeta, abstractmethod
import subprocess
import re
import codecs
import cPickle
import logging
import os
import shutil
import multiprocessing
from tempfile import NamedTemporaryFile

from crf_utilities import get_scale_factors
from crf_utilities import probabilistic_correction
from crf_utilities import label_switcher
from model import Event
from model import TemporalExpression
from model import SequenceLabel
from settings import PATH_MODEL_FOLDER
from settings import PATH_CRF_PP_ENGINE_TEST
from settings import PATH_CRF_PP_ENGINE_TRAIN
from settings import EVENT_ATTRIBUTES
from settings import NO_ATTRIBUTE
from utilities import Mute_stderr
from utilities import extractors_stamp


def identification_attribute_matrix(documents, dest, subject, training=True):
    """It writes in dest the entire training matrix for the identification.
    """
    assert type(documents) == list, 'Wrong type for documents.'
    assert len(documents) > 0, 'Empty documents list.'
    assert subject in ('EVENT', 'TIMEX'), 'Wrong identification subject.'

    # It stores the attribute matrix
    with codecs.open(dest, 'w', encoding='utf8') as matrix:
        for document in documents:
            for sentence in document.sentences:
                for word in sentence.words:
                    row = [v for _, v in sorted(word.attributes.items())]
                    matrix.write('\t'.join(row))
                    gold_label = word.gold_label.copy()

                    if not gold_label.is_out():
                        # The class of the instances different from the subject
                        # are changed in 'O'
                        if subject == 'EVENT':
                            if gold_label.is_timex():
                                gold_label.set_out()
                            else:
                                # For events, we use the attribute CLASS.
                                try:
                                    gold_label.tag = word.tag_attributes[
                                        'class']
                                except KeyError:
                                    # the clinical event type in i2b2 are
                                    # annotated with the attribute 'TYPE'.
                                    gold_label.tag = word.tag_attributes[
                                        'type']
                        else:
                            if gold_label.is_event():
                                gold_label.set_out()
                    if training:
                        matrix.write('\t' + str(gold_label))
                    matrix.write('\n')
                matrix.write('\n')
        matrix.close()


def normalisation_attribute_matrix(documents, dest, subject, training=True):
    """It writes in dest the entire training matrix for the attribute.
    """
    assert type(documents) == list, 'Wrong type for documents.'
    assert len(documents) > 0, 'Empty documents list.'
    assert subject in EVENT_ATTRIBUTES

    if training:
        get_label = lambda word: word.gold_label
    else:
        get_label = lambda word: word.predicted_label

    # It stores the attribute matrix
    with codecs.open(dest, 'w', encoding='utf8') as matrix:
        for ndoc, document in enumerate(documents):
            for nsen, sentence in enumerate(document.sentences):
                for nwor, word in enumerate(sentence.words):
                    # attributes:
                    row = [v for _, v
                           in sorted(word.attributes.items())]

                    # identification label:
                    # I keep only the EVENT label (not TIMEX)
                    ident_label = get_label(word)
                    if ident_label.is_timex():
                        ident_label.set_out()
                    row.append(str(ident_label))

                    # normalisation label (CLASS) for training
                    if training:
                        label = word.tag_attributes.get(subject, NO_ATTRIBUTE)

                        # Different null representation are collapsed.
                        label_to_be_fixed = any((not label, label == 'None',
                                                 ident_label.is_out()))
                        if label_to_be_fixed:
                            label = NO_ATTRIBUTE

                        label = label.replace(' ', '_').upper()
                        row.append(label)

                    # token coordinates for test purpose
                    else:
                        # I sneak in the coordinates of each token
                        row.append('{}_{}_{}'.format(ndoc, nsen, nwor))
                    matrix.write('\t'.join(row))

                    # I am using CRFs with singleton sequences
                    matrix.write('\n\n')
        matrix.close()


class Classifier(object):
    """This class is an abstract classifier for ManTIME."""
    __metaclass__ = ABCMeta

    def __init__(self):
        self.num_cores = multiprocessing.cpu_count()

    @abstractmethod
    def train(self, documents, model_name):
        """ It returns a ClassificationModel object.
        """
        assert len(set([d.annotation_format for d in documents])) == 1

    @abstractmethod
    def test(self, documents, model, post_processing_pipeline=False):
        """

            return List of <Document> (with .annotations filled in.)
        """
        pass


class IdentificationClassifier(Classifier):<commit_msg>positive event attributes training only<commit_after>#!/usr/bin/env python
#
#   Copyright 2014 <NAME>
#
#   gnTEAM, School of Computer Science, University of Manchester.
#   All rights reserved. This program and the accompanying materials
#   are made available under the terms of the GNU General Public License.
#
#   author: <NAME>
#   email:  <EMAIL>
#
#   For details, see www.cs.man.ac.uk/~filannim/

'''It uses the external CRFs algorithms to predict the labels.'''

from __future__ import division
from abc import ABCMeta, abstractmethod
import subprocess
import re
import codecs
import cPickle
import logging
import os
import shutil
import multiprocessing
from tempfile import NamedTemporaryFile

from crf_utilities import get_scale_factors
from crf_utilities import probabilistic_correction
from crf_utilities import label_switcher
from model import Event
from model import TemporalExpression
from model import SequenceLabel
from settings import PATH_MODEL_FOLDER
from settings import PATH_CRF_PP_ENGINE_TEST
from settings import PATH_CRF_PP_ENGINE_TRAIN
from settings import EVENT_ATTRIBUTES
from settings import NO_ATTRIBUTE
from utilities import Mute_stderr
from utilities import extractors_stamp


def identification_attribute_matrix(documents, dest, subject, training=True):
    """It writes in dest the entire training matrix for the identification.
    """
    assert type(documents) == list, 'Wrong type for documents.'
    assert len(documents) > 0, 'Empty documents list.'
    assert subject in ('EVENT', 'TIMEX'), 'Wrong identification subject.'

    # It stores the attribute matrix
    with codecs.open(dest, 'w', encoding='utf8') as matrix:
        for document in documents:
            for sentence in document.sentences:
                for word in sentence.words:
                    row = [v for _, v in sorted(word.attributes.items())]
                    matrix.write('\t'.join(row))
                    gold_label = word.gold_label.copy()

                    if not gold_label.is_out():
                        # The class of the instances different from the subject
                        # are changed in 'O'
                        if subject == 'EVENT':
                            if gold_label.is_timex():
                                gold_label.set_out()
                            else:
                                # For events, we use the attribute CLASS.
                                try:
                                    gold_label.tag = word.tag_attributes[
                                        'class']
                                except KeyError:
                                    # the clinical event type in i2b2 are
                                    # annotated with the attribute 'TYPE'.
                                    gold_label.tag = word.tag_attributes[
                                        'type']
                        else:
                            if gold_label.is_event():
                                gold_label.set_out()
                    if training:
                        matrix.write('\t' + str(gold_label))
                    matrix.write('\n')
                matrix.write('\n')
        matrix.close()


def normalisation_attribute_matrix(documents, dest, subject, training=True):
    """It writes in dest the entire training matrix for the attribute.
    """
    assert type(documents) == list, 'Wrong type for documents.'
    assert len(documents) > 0, 'Empty documents list.'
    assert subject in EVENT_ATTRIBUTES

    if training:
        get_label = lambda word: word.gold_label
    else:
        get_label = lambda word: word.predicted_label

    prev_label = get_label(documents[0].sentences[0].words[0])

    # It stores the attribute matrix
    with codecs.open(dest, 'w', encoding='utf8') as matrix:
        for ndoc, document in enumerate(documents):
            for nsen, sentence in enumerate(document.sentences):
                for nwor, word in enumerate(sentence.words):
                    # attributes:
                    row = [v for _, v
                           in sorted(word.attributes.items())]

                    # identification label:
                    # I keep only the EVENT label (not TIMEX)
                    ident_label = get_label(word)
                    if ident_label.is_timex():
                        ident_label.set_out()
                    row.append(str(ident_label))

                    # discard words that are not positively labelled
                    if ident_label.is_out():
                        prev_label = ident_label
                        continue

                    # group sequences by identification label
                    if str(prev_label) != str(ident_label):
                        matrix.write('\n')

                    # normalisation label (CLASS) for training
                    if training:
                        label = word.tag_attributes.get(subject, NO_ATTRIBUTE)

                        # Different null representation are collapsed.
                        label_to_be_fixed = any((not label, label == 'None',
                                                 ident_label.is_out()))
                        if label_to_be_fixed:
                            label = NO_ATTRIBUTE

                        label = label.replace(' ', '_').upper()
                        row.append(label)

                    # token coordinates for test purpose
                    else:
                        # I sneak in the coordinates of each token
                        row.append('{}_{}_{}'.format(ndoc, nsen, nwor))
                    matrix.write('\t'.join(row))

                    # I am using CRFs with singleton sequences
                    matrix.write('\n')

                    prev_label = ident_label
        matrix.close()


class Classifier(object):
    """This class is an abstract classifier for ManTIME."""
    __metaclass__ = ABCMeta

    def __init__(self):
        self.num_cores = multiprocessing.cpu_count()

    @abstractmethod
    def train(self, documents, model_name):
        """ It returns a ClassificationModel object.
        """
        assert len(set([d.annotation_format for d in documents])) == 1

    @abstractmethod
    def test(self, documents, model, post_processing_pipeline=False):
        """

            return List of <Document> (with .annotations filled in.)
        """
        pass


class IdentificationClassifier(Classifier):
{'fraction_non_alphanumeric': 0.0743801652892562, 'fraction_numerical': 0.002005937575222659, 'mean_word_length': 1.6655260906757914, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '730702', 'n_tokens_mistral': 3097, 'n_tokens_neox': 2886, 'n_words': 954}