<commit_before>#include <cctype>
#include <string>
#include <vector>
#include <sstream>
#include <cctype>
#include "scope.h"

using std::string;
using std::vector;
using std::ostringstream;

namespace shl {

template<typename T>
vector<string> strtok(const string& str, T predicator) {
    const char* s = str.c_str();
    auto length = str.size();
    vector<string> tokens;

    bool in_separator = true;
    string::size_type start, index;
    for(index = 0; index < length; index++) {
        if (in_separator) {
            if (!predicator(s[index])) {
                in_separator = false;
                start = index;
            }
        } else {
            if (predicator(s[index])) {
                in_separator = true;
                tokens.push_back(str.substr(start, index - start));
            }
        }
    }
    if (!in_separator) {
        tokens.push_back(str.substr(start));
    }

    return tokens;
}

string trim(const string& str) {
    const char* s = str.c_str();
    string::size_type start, end;
    start = 0;
    end = str.size();

    while(start < end && isblank(s[start])) start++;
    while(end > start && isblank(s[end-1])) end--;

    return str.substr(start, end - start);
}

ScopeName::ScopeName(const string& scope_name) {
    _name = scope_name;
    _component = strtok(scope_name, [](char c) { return c == '.';});
}

Scope::Scope(const string& scope) {
    auto scope_str = strtok(scope, isblank);

    for(auto it = scope_str.begin(); it != scope_str.end(); it++) {
        _scope.emplace_back(*it);
    }
}


Scope::Scope(const vector<string>& scope) {
    for (auto& name : scope) {
        if(!trim(name).empty()) _scope.push_back(name);<commit_msg>use emplace<commit_after>#include <cctype>
#include <string>
#include <vector>
#include <sstream>
#include <cctype>
#include "scope.h"

using std::string;
using std::vector;
using std::ostringstream;

namespace shl {

template<typename T>
vector<string> strtok(const string& str, T predicator) {
    const char* s = str.c_str();
    auto length = str.size();
    vector<string> tokens;

    bool in_separator = true;
    string::size_type start, index;
    for(index = 0; index < length; index++) {
        if (in_separator) {
            if (!predicator(s[index])) {
                in_separator = false;
                start = index;
            }
        } else {
            if (predicator(s[index])) {
                in_separator = true;
                tokens.push_back(str.substr(start, index - start));
            }
        }
    }
    if (!in_separator) {
        tokens.push_back(str.substr(start));
    }

    return tokens;
}

string trim(const string& str) {
    const char* s = str.c_str();
    string::size_type start, end;
    start = 0;
    end = str.size();

    while(start < end && isblank(s[start])) start++;
    while(end > start && isblank(s[end-1])) end--;

    return str.substr(start, end - start);
}

ScopeName::ScopeName(const string& scope_name) {
    _name = scope_name;
    _component = strtok(scope_name, [](char c) { return c == '.';});
}

Scope::Scope(const string& scope) {
    auto scope_str = strtok(scope, isblank);

    for(auto it = scope_str.begin(); it != scope_str.end(); it++) {
        _scope.emplace_back(*it);
    }
}


Scope::Scope(const vector<string>& scope) {
    for (auto& name : scope) {
        if(!trim(name).empty()) _scope.emplace_back(name);
{'fraction_non_alphanumeric': 0.15330396475770924, 'fraction_numerical': 0.001762114537444934, 'mean_word_length': 2.646680942184154, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364152', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1107, 'n_words': 260}