<commit_before>#!/usr/bin/env python

"""
An example for chunking.
Copyright 2010,2011 <NAME>.
"""

import crfutils
import template

templates = (
    (('w', -2), ),
    (('w', -1), ),
    (('w',  0), ),
    (('w',  1), ),
    (('w',  2), ),
    (('w', -1), ('w',  0)),
    (('w',  0), ('w',  1)),
    (('pos', -2), ),
    (('pos', -1), ),
    (('pos',  0), ),
    (('pos',  1), ),
    (('pos',  2), ),
    (('pos', -2), ('pos', -1)),
    (('pos', -1), ('pos',  0)),
    (('pos',  0), ('pos',  1)),
    (('pos',  1), ('pos',  2)),
    (('pos', -2), ('pos', -1), ('pos',  0)),
    (('pos', -1), ('pos',  0), ('pos',  1)),
    (('pos',  0), ('pos',  1), ('pos',  2)),
    )

def feature_extractor(X):
    template.apply(X, templates)
    if X:
        X[0]['F'].append('__BOS__')
        X[-1]['F'].append('__EOS__')

if __name__ == '__main__':
    crfutils.main(feature_extractor, fields='w pos y', sep=' ')
<commit_msg>Revise the chunking example for readability.<commit_after>#!/usr/bin/env python

"""
An example for chunking.
Copyright 2010,2011 <NAME>.
"""

import crfutils
import template

# Feature template. This template is identical to the one bundled in CRF++
# distribution, but written in a Python object.
templates = (
    (('w', -2), ),
    (('w', -1), ),
    (('w',  0), ),
    (('w',  1), ),
    (('w',  2), ),
    (('w', -1), ('w',  0)),
    (('w',  0), ('w',  1)),
    (('pos', -2), ),
    (('pos', -1), ),
    (('pos',  0), ),
    (('pos',  1), ),
    (('pos',  2), ),
    (('pos', -2), ('pos', -1)),
    (('pos', -1), ('pos',  0)),
    (('pos',  0), ('pos',  1)),
    (('pos',  1), ('pos',  2)),
    (('pos', -2), ('pos', -1), ('pos',  0)),
    (('pos', -1), ('pos',  0), ('pos',  1)),
    (('pos',  0), ('pos',  1), ('pos',  2)),
    )

def feature_extractor(X):
    template.apply(X, templates)
    if X:
        X[0]['F'].append('__BOS__')     # BOS feature
        X[-1]['F'].append('__EOS__')    # EOS feature

if __name__ == '__main__':
    fi = sys.stdin
    fo = sys.stdout
    
    # Each line of the input text has token surface, part-of-speech, and
    # chunk label separated by SPACE (' ') characters in this order.
    # An empty line represents an end of a sentence.
    fields = ('w', 'pos', 'y')
    
    if len(sys.argv) < 3:
        # The command line does not have "-m MODEL" specified.
        # This is an example of converting an input tet into the
        # label-attribute format compatible with the CRFsuite utility

        # For each sequence from STDIN.
        for X in crfutil.readiter(fi, fields, ' '):
            # print X   # uncomment this to see how a sequence is stored in X
            feature_extractor(X)
            # print X   # uncomment this to see how a sequence is stored in X
            crfutil.output_features(fo, X, 'y')
        
    elif sys.argv[1] == '-m':
        # The command line has "-m MODEL" specified.
        # This is an example of using the python module for tagging
        
        import crfsuite
        tagger = crfsuite.Tagger()
        tagger.open(sys.argv[2])

        # For each sequence from STDIN.
        for X in crfutil.readiter(fi, fields, ' '):
            # Obtain features.
            feature_extractor(X)
            xseq = crfutil.to_crfsuite(X)
            yseq = tagger.tag(xseq)
            for t in range(len(X)):
                v = X[t]
                # Thru the input text
                fo.write(' '.join([v[f] for f in F]))
                # Append the predicted label
                fo.write(' %s\n' % yseq[t])
            fo.write('\n')
{'fraction_non_alphanumeric': 0.21822272215973004, 'fraction_numerical': 0.023903262092238472, 'mean_word_length': 2.0743301642178045, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4582899', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1277, 'n_words': 381}