<commit_before># -*- coding: utf-8 -*-
#
# Author: <NAME> <<EMAIL>>
# Copyright: © 2011-2013 SARL Romain Dorgueil Conseil
#
from werkzeug.utils import cached_property
from rdc.etl.transform import Transform
from rdc.etl.util import create_http_reader, create_file_reader


class FileExtract(Transform):
    uri = None
    output_field = None

    def __init__(self, uri=None, output_field=None):
        super(FileExtract, self).__init__()

        self.uri = uri or self.uri
        self.output_field = output_field or self.output_field

    @cached_property
    def reader(self):
        if self.uri is None:
            raise RuntimeError('No URI configured in ' + self.__class__.__name__ + ' transformation.')

        if self.uri.find('http://') == 0 or self.uri.find('https://') == 0:
            return create_http_reader(self.uri)
        else:
            return create_file_reader(self.uri)

    @cached_property
    def content(self):
        return self.reader()

    def transform(self, hash):
        hash.set(self.output_field, self.content)
        yield hash

<commit_msg>deprecated transform.extract.file (no reason to statically configure an url when it can be read from the stream).<commit_after># -*- coding: utf-8 -*-
#
# Author: <NAME> <<EMAIL>>
# Copyright: © 2011-2013 SARL Romain Dorgueil Conseil
#
from werkzeug.utils import cached_property
from rdc.etl.transform import Transform
from rdc.etl.util import create_http_reader, create_file_reader


class FileExtract(Transform):
    """ Deprecated in favor of FileJoin """

    uri = None
    output_field = None

    def __init__(self, uri=None, output_field=None):
        super(FileExtract, self).__init__()

        self.uri = uri or self.uri
        self.output_field = output_field or self.output_field

    @cached_property
    def reader(self):
        if self.uri is None:
            raise RuntimeError('No URI configured in ' + self.__class__.__name__ + ' transformation.')

        if self.uri.find('http://') == 0 or self.uri.find('https://') == 0:
            return create_http_reader(self.uri)
        else:
            return create_file_reader(self.uri)

    @cached_property
    def content(self):
        return self.reader()

    def transform(self, hash):
        hash.set(self.output_field, self.content)
        yield hash


{'fraction_non_alphanumeric': 0.10804993542832544, 'fraction_numerical': 0.009470512268618166, 'mean_word_length': 2.8993288590604025, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1771384', 'n_tokens_mistral': 722, 'n_tokens_neox': 690, 'n_words': 187}