<commit_before>#!/usr/bin/env python
'''
kba.pipeline Transform for converting v0_1_0 StreamItems to v0_2_0

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''

from streamcorpus import make_stream_item, ContentItem

def upgrade_streamcorpus(config):
    '''
    returns a kba.pipeline "transform" function that populates takes
    in v0_1_0 StreamItems and emits v0_2_0 StreamItems
    '''
    ## make a closure around config
    def _upgrade_streamcorpus(s1):
        s2 = make_stream_item(s1.stream_time.zulu_timestamp,
                              s1.abs_url)
        s2.schost = s1.schost
        s2.source = s1.source
        s2.source_metadata['kba-2012'] = s1.source_metadata
        s2.body = ContentItem(
            raw = s1.body.raw,
            encoding = s1.body.encoding,
            ## default, might get overwritten below
            media_type = 'text/html',
            )
        if s1.source == 'social':
            s2.body.media_type = 'text/plain'
            ## the separation of content items in the social stream
            ## was artificial and annoying, so smoosh them together
            s2.body.clean_visible = '\n\n'.join([<commit_msg>adding in the old NER data as an extra tagging, instead of discarding it.<commit_after>#!/usr/bin/env python
'''
kba.pipeline Transform for converting v0_1_0 StreamItems to v0_2_0

This software is released under an MIT/X11 open source license.

Copyright 2012 Diffeo, Inc.
'''

from streamcorpus import make_stream_item, make_stream_time, ContentItem, Tagging

def upgrade_streamcorpus(config):
    '''
    returns a kba.pipeline "transform" function that populates takes
    in v0_1_0 StreamItems and emits v0_2_0 StreamItems
    '''
    ## make a closure around config
    def _upgrade_streamcorpus(s1):
        s2 = make_stream_item(s1.stream_time.zulu_timestamp,
                              s1.abs_url)
        s2.schost = s1.schost
        s2.source = s1.source
        s2.source_metadata['kba-2012'] = s1.source_metadata
        s2.body = ContentItem(
            raw = s1.body.raw,
            encoding = s1.body.encoding,
            ## default, might get overwritten below
            media_type = 'text/html',
            taggings = {'stanford': Tagging(
                    tagger_id = 'stanford',
                    raw_tagging = s1.body.ner,
                    generation_time = make_stream_time('2012-06-01T00:00:00.0Z'),
                    tagger_config = 'annotators: {tokenize, cleanxml, ssplit, pos, lemma, ner}, properties: pos.maxlen=100',
                    tagger_version = 'Stanford CoreNLP ver 1.2.0',
                    )}
            )
        if 'keep_old_cleansed_as_clean_visible' in config and \
                config['keep_old_cleansed_as_clean_visible']:
            s2.body.clean_visible = s1.body.cleansed

        if s1.source == 'social':
            s2.body.media_type = 'text/plain'
            ## the separation of content items in the social stream
            ## was artificial and annoying, so smoosh them together
            s2.body.clean_visible = '\n\n'.join([
{'fraction_non_alphanumeric': 0.08817891373801917, 'fraction_numerical': 0.03194888178913738, 'mean_word_length': 2.227835051546392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1548524', 'n_tokens_mistral': 987, 'n_tokens_neox': 941, 'n_words': 244}