<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-


from collections import OrderedDict
import string
import re


def I(s):
    # Return s unchanged
    return s


def camelize(s):
    # Change "Foo bar baz's" to "FooBarBazs"
    return re.sub(r'\W', '', string.capwords(s))


def underscore(s):
    # Change "Foo bar baz's" to "foo_bar_bazs"
    underscored = re.sub(r'\s+', '_', s.strip().downcase())
    return re.sub(r'\W', '', underscored)


class Renderer(object):
    def __init__(self, key_formatter=I):
        self.key_formatter = key_formatter
        super(Renderer, self).__init__()

    def stringify_dict(self, d):
        out = OrderedDict(
            [
                (self.key_formatter(self._render_block(k)), self._valuify(v))
                for k, v in d.items()
            ])
        return out

    def _valuify(self, cm_vals):
        if hasattr(cm_vals, 'items'):
            return self.stringify_dict(cm_vals)
        if len(cm_vals) == 0:
            return ''
        first = cm_vals[0]
        if first.t == 'List':
            return self._render_List(first)
        return "\n\n".join([self._render_block(v) for v in cm_vals])

    def _render_block(self, block):
        method_name = "_render_{0}".format(block.t)
        method = self._render_generic_block
        if hasattr(self, method_name):<commit_msg>Fix bug in underscore method.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-


from collections import OrderedDict
import string
import re


def I(s):
    # Return s unchanged
    return s


def camelize(s):
    # Change "Foo bar baz's" to "FooBarBazs"
    return re.sub(r'\W', '', string.capwords(s))


def underscore(s):
    # Change "Foo bar baz's" to "foo_bar_bazs"
    underscored = re.sub(r'\s+', '_', s.strip().lower())
    return re.sub(r'\W', '', underscored)


class Renderer(object):
    def __init__(self, key_formatter=I):
        self.key_formatter = key_formatter
        super(Renderer, self).__init__()

    def stringify_dict(self, d):
        out = OrderedDict(
            [
                (self.key_formatter(self._render_block(k)), self._valuify(v))
                for k, v in d.items()
            ])
        return out

    def _valuify(self, cm_vals):
        if hasattr(cm_vals, 'items'):
            return self.stringify_dict(cm_vals)
        if len(cm_vals) == 0:
            return ''
        first = cm_vals[0]
        if first.t == 'List':
            return self._render_List(first)
        return "\n\n".join([self._render_block(v) for v in cm_vals])

    def _render_block(self, block):
        method_name = "_render_{0}".format(block.t)
        method = self._render_generic_block
        if hasattr(self, method_name):
{'fraction_non_alphanumeric': 0.1353135313531353, 'fraction_numerical': 0.0029336266960029336, 'mean_word_length': 2.3930348258706466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6117068', 'n_tokens_mistral': 945, 'n_tokens_neox': 896, 'n_words': 207}