<commit_before>#include "CppUnitTest.h"
#include <string>
#include <vector>

using namespace Microsoft::VisualStudio::CppUnitTestFramework;
using namespace std;

namespace algo
{
    template<class Traits>
    class tokens
    {
        vector<typename Traits::type> tokens_;
    public:
        bool empty() const { return tokens_.size() == 0; }
        void from_string(string s)
        {
            tokens_.push_back(Traits::tokenize(s));
        }
    };
}

namespace algo { namespace spec
{
    struct test_token_traits
    {
        struct s {};
        typedef s type;
        static s tokenize(const string&) { return s(); }
    };

    typedef tokens<test_token_traits> test_tokens;

    TEST_CLASS(can_recognize_tokens)
	{
	public:

        TEST_METHOD(should_not_have_any_tokens_when_default_constructed)
		{
            test_tokens testTokens;
            Assert::IsTrue(testTokens.empty());
		}

        TEST_METHOD(should_recognize_a_token)
        {
            test_tokens testTokens;
            testTokens.from_string("s");
            Assert::IsFalse(testTokens.empty());
        }

	};
}}<commit_msg>test rename<commit_after>#include "CppUnitTest.h"
#include <string>
#include <vector>

using namespace Microsoft::VisualStudio::CppUnitTestFramework;
using namespace std;

namespace algo
{
    template<class Traits>
    class tokens
    {
        vector<typename Traits::type> tokens_;
    public:
        bool empty() const { return tokens_.size() == 0; }
        void from_string(string s)
        {
            tokens_.push_back(Traits::tokenize(s));
        }
    };
}

namespace algo { namespace spec
{
    struct test_token_traits
    {
        struct s {};
        typedef s type;
        static s tokenize(const string&) { return s(); }
    };

    typedef tokens<test_token_traits> test_tokens;

    TEST_CLASS(can_recognize_tokens)
	{
	public:

        TEST_METHOD(should_not_have_any_tokens_when_default_constructed)
		{
            test_tokens testTokens;
            Assert::IsTrue(testTokens.empty());
		}

        TEST_METHOD(should_recognize_a_token_in_a_string)
        {
            test_tokens testTokens;
            testTokens.from_string("s");
            Assert::IsFalse(testTokens.empty());
        }

	};
}}
{'fraction_non_alphanumeric': 0.10115350488021295, 'fraction_numerical': 0.0008873114463176575, 'mean_word_length': 2.5793650793650795, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1757485', 'n_tokens_mistral': 689, 'n_tokens_neox': 679, 'n_words': 127}