<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import csv

from contextlib2 import ExitStack
from contextlib import contextmanager

import attr
from attr.validators import instance_of as is_a

from tasks.utils.files import atomic_output


def unique_using(getter, transformer):
    seen = set()

    def fn(entry):
        value = getter(entry)
        if value not in seen:
            yield transformer(entry)
            seen.add(value)
    return fn



@attr.s(frozen=True)  # pylint: disable=W0232
class CsvOutput(object):
    filename = attr.ib(validator=is_a(basestring))
    headers = attr.ib(validator=is_a(list))
    transformer = attr.ib()
    csv_options = attr.ib(validator=is_a(dict))

    @csv_options.default
    def default_csv_options(self):
        """
        Generate the CSV options to use for writing
        """
        return {
            'delimiter': ',',
            'quotechar': '"',
            'quoting': csv.QUOTE_ALL,
            'lineterminator': '\n',
        }

    def exists(self):
        return os.path.exists(self.filename)
<commit_msg>Provide a default for transformer<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import csv

from contextlib2 import ExitStack
from contextlib import contextmanager

import attr
from attr.validators import instance_of as is_a

from tasks.utils.files import atomic_output


def unique_using(getter, transformer):
    seen = set()

    def fn(entry):
        value = getter(entry)
        if value not in seen:
            yield transformer(entry)
            seen.add(value)
    return fn



@attr.s(frozen=True)  # pylint: disable=W0232
class CsvOutput(object):
    filename = attr.ib(validator=is_a(basestring))
    headers = attr.ib(validator=is_a(list))
    transformer = attr.ib()
    csv_options = attr.ib(validator=is_a(dict))

    @transformer.default
    def default_transformer(self):
        def fn(entry):
            yield entry
        return fn

    @csv_options.default
    def default_csv_options(self):
        """
        Generate the CSV options to use for writing
        """
        return {
            'delimiter': ',',
            'quotechar': '"',
            'quoting': csv.QUOTE_ALL,
            'lineterminator': '\n',
        }

    def exists(self):
        return os.path.exists(self.filename)

{'fraction_non_alphanumeric': 0.08902929350947732, 'fraction_numerical': 0.010338885697874785, 'mean_word_length': 3.0406032482598606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5872839', 'n_tokens_mistral': 1028, 'n_tokens_neox': 963, 'n_words': 341}