<commit_before>from StringIO import StringIO
from mock import patch
from mr.awsome.config import Config
from unittest2 import TestCase
import os
import pytest
import shutil
import tempfile


class ConfigTests(TestCase):
    def testEmpty(self):
        contents = StringIO("")
        config = Config(contents).parse()
        assert config == {}

    def testPlainSection(self):
        contents = StringIO("[foo]")
        config = Config(contents).parse()
        assert config == {'global': {'foo': {}}}

    def testGroupSection(self):
        contents = StringIO("[bar:foo]")
        config = Config(contents).parse()
        config == {'bar': {'foo': {}}}

    def testMixedSections(self):
        contents = StringIO("[bar:foo]\n[baz]")
        config = Config(contents).parse()
        assert config == {
            'bar': {'foo': {}},
            'global': {'baz': {}}}

    def testMacroExpansion(self):
        from mr.awsome.config import ConfigValue
        contents = StringIO("\n".join([
            "[macro]",
            "macrovalue=1",
            "[baz]",
            "<=macro",
            "bazvalue=2"]))
        config = Config(contents).parse()
        assert config == {
            'global': {
                'macro': {'macrovalue': '1'},
                'baz': {'macrovalue': '1', 'bazvalue': '2'}}}
        assert isinstance(config['global']['baz']._dict['macrovalue'], ConfigValue)
        assert isinstance(config['global']['baz']._dict['bazvalue'], ConfigValue)

    def testGroupMacroExpansion(self):
        contents = StringIO("\n".join([
            "[group:macro]",
            "macrovalue=1",
            "[baz]",
            "<=group:macro",
            "bazvalue=2"]))
        config = Config(contents).parse()
        assert config == {
            'global': {
                'baz': {'macrovalue': '1', 'bazvalue': '2'}},
            'group': {
                'macro': {'macrovalue': '1'}}}

    def testCircularMacroExpansion(self):
        contents = StringIO("\n".join([
            "[macro]",
            "<=macro",
            "macrovalue=1"]))
        with self.assertRaises(ValueError):
            Config(contents).parse()

    def testOverrides(self):
        contents = StringIO("\n".join([
            "[section]",
            "value=1"]))
        config = Config(contents).parse()
        assert config == {'global': {'section': {'value': '1'}}}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides=None)
        assert result == {
            'value': '1'}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides={'value': '2'})
        assert result == {
            'value': '2'}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides={'value2': '2'})
        assert result == {
            'value': '1',
            'value2': '2'}
        # make sure nothing is changed afterwards
        assert config == {'global': {'section': {'value': '1'}}}

    def testSpecialKeys(self):
        contents = StringIO("\n".join([
            "[section]",
            "value=1"]))
        config = Config(contents).parse()
        assert config['global']['section']['__name__'] == 'section'
        assert config['global']['section']['__groupname__'] == 'global'


class DummyPlugin(object):
    def __init__(self):
        self.massagers = []

    def get_massagers(self):
        return self.massagers


@pytest.mark.parametrize("value, expected", [
    (True, True),
    (False, False),
    ("true", True),
    ("yes", True),
    ("on", True),
    ("false", False),
    ("no", False),
    ("off", False),
    ("foo", None)])
def test_value_asbool(value, expected):
    from mr.awsome.config import value_asbool
    assert value_asbool(value) == expected


class MassagerTests(TestCase):
    def setUp(self):
        TestCase.setUp(self)
        self.dummyplugin = DummyPlugin()
        self.plugins = dict(
            dummy=dict(
                get_massagers=self.dummyplugin.get_massagers))

    def tearDown(self):
        TestCase.tearDown(self)
        del self.plugins<commit_msg>Test and fix macro cleaners.<commit_after>from StringIO import StringIO
from mock import patch
from mr.awsome.config import Config
from unittest2 import TestCase
import os
import pytest
import shutil
import tempfile


class ConfigTests(TestCase):
    def testEmpty(self):
        contents = StringIO("")
        config = Config(contents).parse()
        assert config == {}

    def testPlainSection(self):
        contents = StringIO("[foo]")
        config = Config(contents).parse()
        assert config == {'global': {'foo': {}}}

    def testGroupSection(self):
        contents = StringIO("[bar:foo]")
        config = Config(contents).parse()
        config == {'bar': {'foo': {}}}

    def testMixedSections(self):
        contents = StringIO("[bar:foo]\n[baz]")
        config = Config(contents).parse()
        assert config == {
            'bar': {'foo': {}},
            'global': {'baz': {}}}

    def testMacroExpansion(self):
        from mr.awsome.config import ConfigValue
        contents = StringIO("\n".join([
            "[macro]",
            "macrovalue=1",
            "[baz]",
            "<=macro",
            "bazvalue=2"]))
        config = Config(contents).parse()
        assert config == {
            'global': {
                'macro': {'macrovalue': '1'},
                'baz': {'macrovalue': '1', 'bazvalue': '2'}}}
        assert isinstance(config['global']['baz']._dict['macrovalue'], ConfigValue)
        assert isinstance(config['global']['baz']._dict['bazvalue'], ConfigValue)

    def testGroupMacroExpansion(self):
        contents = StringIO("\n".join([
            "[group:macro]",
            "macrovalue=1",
            "[baz]",
            "<=group:macro",
            "bazvalue=2"]))
        config = Config(contents).parse()
        assert config == {
            'global': {
                'baz': {'macrovalue': '1', 'bazvalue': '2'}},
            'group': {
                'macro': {'macrovalue': '1'}}}

    def testCircularMacroExpansion(self):
        contents = StringIO("\n".join([
            "[macro]",
            "<=macro",
            "macrovalue=1"]))
        with self.assertRaises(ValueError):
            Config(contents).parse()

    def testMacroCleaners(self):
        dummyplugin = DummyPlugin()
        plugins = dict(
            dummy=dict(
                get_macro_cleaners=dummyplugin.get_macro_cleaners))

        def cleaner(macro):
            if 'cleanvalue' in macro:
                del macro['cleanvalue']

        dummyplugin.macro_cleaners = {'global': cleaner}
        contents = StringIO("\n".join([
            "[group:macro]",
            "macrovalue=1",
            "cleanvalue=3",
            "[baz]",
            "<=group:macro",
            "bazvalue=2"]))
        config = Config(contents, plugins=plugins).parse()
        assert config == {
            'global': {
                'baz': {'macrovalue': '1', 'bazvalue': '2'}},
            'group': {
                'macro': {'macrovalue': '1', 'cleanvalue': '3'}}}

    def testOverrides(self):
        contents = StringIO("\n".join([
            "[section]",
            "value=1"]))
        config = Config(contents).parse()
        assert config == {'global': {'section': {'value': '1'}}}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides=None)
        assert result == {
            'value': '1'}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides={'value': '2'})
        assert result == {
            'value': '2'}
        result = config.get_section_with_overrides(
            'global',
            'section',
            overrides={'value2': '2'})
        assert result == {
            'value': '1',
            'value2': '2'}
        # make sure nothing is changed afterwards
        assert config == {'global': {'section': {'value': '1'}}}

    def testSpecialKeys(self):
        contents = StringIO("\n".join([
            "[section]",
            "value=1"]))
        config = Config(contents).parse()
        assert config['global']['section']['__name__'] == 'section'
        assert config['global']['section']['__groupname__'] == 'global'


class DummyPlugin(object):
    def __init__(self):
        self.macro_cleaners = {}
        self.massagers = []

    def get_macro_cleaners(self, main_config):
        return self.macro_cleaners

    def get_massagers(self):
        return self.massagers


@pytest.mark.parametrize("value, expected", [
    (True, True),
    (False, False),
    ("true", True),
    ("yes", True),
    ("on", True),
    ("false", False),
    ("no", False),
    ("off", False),
    ("foo", None)])
def test_value_asbool(value, expected):
    from mr.awsome.config import value_asbool
    assert value_asbool(value) == expected


class MassagerTests(TestCase):
    def setUp(self):
        TestCase.setUp(self)
        self.dummyplugin = DummyPlugin()
        self.plugins = dict(
            dummy=dict(
                get_massagers=self.dummyplugin.get_massagers))

    def tearDown(self):
        TestCase.tearDown(self)
        del self.plugins
{'fraction_non_alphanumeric': 0.16021688284073995, 'fraction_numerical': 0.005847331490537955, 'mean_word_length': 2.0923734385272845, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053660', 'n_tokens_mistral': 2721, 'n_tokens_neox': 2620, 'n_words': 565}