<commit_before>import json

from exporters.writers.base_writer import BaseWriter, ItemsLimitReached


class ConsoleWriter(BaseWriter):
    """
    It is just a writer with testing purposes. It prints every item in console.
    """

    def __init__(self, options):
        super(ConsoleWriter, self).__init__(options)
        self.logger.info('ConsoleWriter has been initiated')
        self.pretty_print = self.options.get('pretty_print', False)

    def write_batch(self, batch):
        for item in batch:
            formatted_item = item.formatted

            if self.pretty_print:
                formatted_item = self._format(formatted_item)

            print formatted_item
            self._increment_written_items()
            if self.items_limit and self.items_limit == self.stats['items_count']:
                raise ItemsLimitReached('Finishing job after items_limit reached: {} items written.'.format(self.stats['items_count']))
        self.logger.debug('Wrote items')

    def _format(self, item):
        try:
            return json.dumps(json.loads(item), indent=2)
        except:
            return item<commit_msg>Use sort_keys=True for the ConsoleWritter pretty printing<commit_after>import json

from exporters.writers.base_writer import BaseWriter, ItemsLimitReached


class ConsoleWriter(BaseWriter):
    """
    It is just a writer with testing purposes. It prints every item in console.
    """

    def __init__(self, options):
        super(ConsoleWriter, self).__init__(options)
        self.logger.info('ConsoleWriter has been initiated')
        self.pretty_print = self.options.get('pretty_print', False)

    def write_batch(self, batch):
        for item in batch:
            formatted_item = item.formatted

            if self.pretty_print:
                formatted_item = self._format(formatted_item)

            print formatted_item
            self._increment_written_items()
            if self.items_limit and self.items_limit == self.stats['items_count']:
                raise ItemsLimitReached('Finishing job after items_limit reached: {} items written.'.format(self.stats['items_count']))
        self.logger.debug('Wrote items')

    def _format(self, item):
        try:
            return json.dumps(json.loads(item), indent=2, sort_keys=True)
        except:
            return item
{'fraction_non_alphanumeric': 0.09017094017094017, 'fraction_numerical': 0.0008547008547008547, 'mean_word_length': 2.727707006369427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '652039', 'n_tokens_mistral': 649, 'n_tokens_neox': 618, 'n_words': 158}