<commit_before>"""
peek.reporter
~~~~~~~~~~~~~

:copyright: 2012 DISQUS.
:license: Apache License 2.0, see LICENSE for more details.
"""

__all__ = ('Reporter',)

import json
import os.path


class Reporter(object):
    def __init__(self, origin, collector, output=None):
        self.origin = origin
        self.collector = collector

        if output is None:
            output = 'peekhtml'

        self.output = os.path.normpath(output)

    def report(self):
        raise NotImplementedError


class HTMLReporter(Reporter):
    """
    An HTML reporter.

    This actually just dumps some JSON to a file, which a static HTML page will
    pull in and render.
    """
    def _get_origin(self, data, origin):
        if origin in data:
            return {origin: data[origin]}

        for key, value in data.iteritems():
            if key.startswith(origin):
                return {key: value}

        for key, value in data.iteritems():
            return self._get_origin(value['c'], origin)

    def get_files(self, calls, files=None):
        if files is None:
            files = set()
        for key, value in calls.iteritems():
            if 'f' in value:
                files.add(value['f'])
            if 'c' in value:
                self.get_files(value['c'], files)
        return files

    def report(self):
        calls = self._get_origin(self.collector.get_calls(), self.origin)

        files = self.get_files(calls)

        source = dict((f, [f.strip() for f in open(f, 'r')]) for f in files)

        data = {
            'calls': calls,
            'source': source,
        }<commit_msg>Dont call .strip() as it kills whitespace<commit_after>"""
peek.reporter
~~~~~~~~~~~~~

:copyright: 2012 DISQUS.
:license: Apache License 2.0, see LICENSE for more details.
"""

__all__ = ('Reporter',)

import json
import os.path


class Reporter(object):
    def __init__(self, origin, collector, output=None):
        self.origin = origin
        self.collector = collector

        if output is None:
            output = 'peekhtml'

        self.output = os.path.normpath(output)

    def report(self):
        raise NotImplementedError


class HTMLReporter(Reporter):
    """
    An HTML reporter.

    This actually just dumps some JSON to a file, which a static HTML page will
    pull in and render.
    """
    def _get_origin(self, data, origin):
        if origin in data:
            return {origin: data[origin]}

        for key, value in data.iteritems():
            if key.startswith(origin):
                return {key: value}

        for key, value in data.iteritems():
            return self._get_origin(value['c'], origin)

    def get_files(self, calls, files=None):
        if files is None:
            files = set()
        for key, value in calls.iteritems():
            if 'f' in value:
                files.add(value['f'])
            if 'c' in value:
                self.get_files(value['c'], files)
        return files

    def report(self):
        calls = self._get_origin(self.collector.get_calls(), self.origin)

        files = self.get_files(calls)

        source = dict((f, [f[:-1] for f in open(f, 'r')]) for f in files)

        data = {
            'calls': calls,
            'source': source,
        }
{'fraction_non_alphanumeric': 0.11893870082342177, 'fraction_numerical': 0.003964623360780726, 'mean_word_length': 2.0711610486891385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3660730', 'n_tokens_mistral': 992, 'n_tokens_neox': 950, 'n_words': 302}