<commit_before>// This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

/* feature_set.cc
   <NAME>, 10 June 2003
   Copyright (c) 2003 <NAME>.  All rights reserved.
   $Source$

   Implementation of the feature set.
*/


#include "feature_set.h"
#include "training_data.h"
#include "mldb/base/parse_context.h"
#include "mldb/arch/demangle.h"
#include "mldb/ml/jml/registry.h"
#include <typeinfo>
#include <string>


using namespace std;
using namespace ML;
using namespace ML::DB;
using namespace MLDB;



namespace ML {


/*****************************************************************************/
/* FEATURE_SET                                                               */
/*****************************************************************************/

std::ostream & operator << (std::ostream & stream, const Feature & feature)
{
    return stream << feature.print();
}


/*****************************************************************************/
/* MUTABLE_FEATURE_SET                                                       */
/*****************************************************************************/

/** Structure used to implement the sort() routine. */
struct Mutable_Feature_Set::compare_feature {
    bool operator () (const std::pair<Feature, float> & p1,
                      const std::pair<Feature, float> & p2) const
    {
        safe_less<float> value_cmp;
        if (p1.first < p2.first) return true;
        else if (p2.first < p1.first) return false;
        return value_cmp(p1.second, p2.second);
    }
};

void Mutable_Feature_Set::sort()
{
    Mutable_Feature_Set::do_sort();
}

void Mutable_Feature_Set::do_sort() const
{
    if (is_sorted) return;
    if (locked) throw Exception("mutating locked feature set");
    std::sort(features.begin(), features.end(), compare_feature());
    is_sorted = true;
}

void Mutable_Feature_Set::add(const Feature & feat, float val)
{
    if (locked) throw Exception("mutating locked feature set");
    features.push_back(std::make_pair(feat, val));
    if (features.size() == 1) is_sorted = true;
    else if (is_sorted && compare_feature()(features[features.size() - 2],
                                            features.back()))
        is_sorted = true;
    else is_sorted = false;
}

void Mutable_Feature_Set::
replace(const Feature & feat, float val)
{
    if (locked) throw Exception("mutating locked feature set");
    if (is_sorted) {
        /* Can do this more sensibly, and keep it sorted... */
        features_type::iterator it
            = std::lower_bound(features.begin(), features.end(),
                               make_pair(feat, -INFINITY));
        
        if (it == end() || it->first != feat)
            add(feat, val);
        else {
            it->second = val;
            ++it;
            int start = it - features.begin();
            int pos = 0;

            /* Shift those to delete back to maintain sorted property. */
            int num_to_delete = 0;
            while (pos < features.size() && features[pos].first == feat) {
                ++num_to_delete;
                ++pos;<commit_msg>Fix small logic issue with feature_set<commit_after>// This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

/* feature_set.cc
   <NAME>, 10 June 2003
   Copyright (c) 2003 <NAME>.  All rights reserved.
   $Source$

   Implementation of the feature set.
*/


#include "feature_set.h"
#include "training_data.h"
#include "mldb/base/parse_context.h"
#include "mldb/arch/demangle.h"
#include "mldb/ml/jml/registry.h"
#include <typeinfo>
#include <string>


using namespace std;
using namespace ML;
using namespace ML::DB;
using namespace MLDB;



namespace ML {


/*****************************************************************************/
/* FEATURE_SET                                                               */
/*****************************************************************************/

std::ostream & operator << (std::ostream & stream, const Feature & feature)
{
    return stream << feature.print();
}


/*****************************************************************************/
/* MUTABLE_FEATURE_SET                                                       */
/*****************************************************************************/

/** Structure used to implement the sort() routine. */
struct Mutable_Feature_Set::compare_feature {
    bool operator () (const std::pair<Feature, float> & p1,
                      const std::pair<Feature, float> & p2) const
    {
        safe_less<float> value_cmp;
        if (p1.first < p2.first) return true;
        else if (p2.first < p1.first) return false;
        return value_cmp(p1.second, p2.second);
    }
};

void Mutable_Feature_Set::sort()
{
    Mutable_Feature_Set::do_sort();
}

void Mutable_Feature_Set::do_sort() const
{
    if (is_sorted) return;
    if (locked) throw Exception("mutating locked feature set");
    std::sort(features.begin(), features.end(), compare_feature());
    is_sorted = true;
}

void Mutable_Feature_Set::add(const Feature & feat, float val)
{
    if (locked) throw Exception("mutating locked feature set");
    features.push_back(std::make_pair(feat, val));
    if (features.size() == 1) is_sorted = true;
    else if (is_sorted && compare_feature()(features[features.size() - 2],
                                            features.back()))
        is_sorted = true;
    else is_sorted = false;
}

void Mutable_Feature_Set::
replace(const Feature & feat, float val)
{
    if (locked) throw Exception("mutating locked feature set");
    if (is_sorted) {
        /* Can do this more sensibly, and keep it sorted... */
        features_type::iterator it
            = std::lower_bound(features.begin(), features.end(),
                               make_pair(feat, -INFINITY));
        
        if (it == end() || it->first != feat)
            add(feat, val);
        else {
            it->second = val;
            ++it;
            int start = it - features.begin();
            int pos = start;

            /* Shift those to delete back to maintain sorted property. */
            int num_to_delete = 0;
            while (pos < features.size() && features[pos].first == feat) {
                ++num_to_delete;
                ++pos;
{'fraction_non_alphanumeric': 0.21072615092548647, 'fraction_numerical': 0.008068343616516375, 'mean_word_length': 2.5318435754189945, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315000', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1676, 'n_words': 460}