<commit_before>// Copyright 2010-2014 RethinkDB, all rights reserved.
#include "rdb_protocol/datum.hpp"

#include <float.h>
#include <math.h>
#include <stdint.h>
#include <stdlib.h>

#include <algorithm>

#include "errors.hpp"
#include <boost/detail/endian.hpp>

#include "containers/archive/stl_types.hpp"
#include "containers/scoped.hpp"
#include "rdb_protocol/env.hpp"
#include "rdb_protocol/error.hpp"
#include "rdb_protocol/pseudo_binary.hpp"
#include "rdb_protocol/pseudo_literal.hpp"
#include "rdb_protocol/pseudo_time.hpp"
#include "rdb_protocol/shards.hpp"
#include "stl_utils.hpp"

namespace ql {

const size_t tag_size = 8;

const std::set<std::string> datum_t::_allowed_pts = std::set<std::string>();

const char *const datum_t::reql_type_string = "$reql_type$";

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_null_t) :
    type(R_NULL), r_str(NULL) { }

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_boolean_t, bool _bool) :
    type(R_BOOL), r_bool(_bool) { }

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_binary_t,
                                        scoped_ptr_t<wire_string_t> data) :
    type(R_BINARY), r_str(data.release()) { }

datum_t::data_wrapper_t::data_wrapper_t(double num) :
    type(R_NUM), r_num(num) { }

datum_t::data_wrapper_t::data_wrapper_t(std::string &&str) :
    type(R_STR),
    r_str(wire_string_t::create_and_init(str.size(), str.data()).release()) { }

datum_t::data_wrapper_t::data_wrapper_t(scoped_ptr_t<wire_string_t> str) :
    type(R_STR), r_str(str.release()) { }

datum_t::data_wrapper_t::data_wrapper_t(const char *cstr) :
    type(R_STR),
    r_str(wire_string_t::create_and_init(::strlen(cstr), cstr).release()) { }

datum_t::data_wrapper_t::data_wrapper_t(std::vector<counted_t<const datum_t> > &&array) :
    type(R_ARRAY),
    r_array(new std::vector<counted_t<const datum_t> >(std::move(array))) { }

datum_t::data_wrapper_t::data_wrapper_t(std::map<std::string,
                                                 counted_t<const datum_t> > &&object) :
    type(R_OBJECT),
    r_object(new std::map<std::string, counted_t<const datum_t> >(std::move(object))) { }

datum_t::data_wrapper_t::~data_wrapper_t() {
    switch (type) {
    case R_NULL: // fallthru
    case R_BOOL: // fallthru
    case R_NUM: break;
    case R_BINARY: // fallthru
    case R_STR: {
        delete r_str;
    } break;
    case R_ARRAY: {
        delete r_array;
    } break;
    case R_OBJECT: {
        delete r_object;
    } break;
    default: unreachable();
    }
}

datum_t::datum_t(datum_t::construct_null_t dummy) : data(dummy) { }

datum_t::datum_t(construct_boolean_t dummy, bool _bool) : data(dummy, _bool) { }

datum_t::datum_t(construct_binary_t dummy, scoped_ptr_t<wire_string_t> _data)
    : data(dummy, std::move(_data)) { }

datum_t::datum_t(double _num) : data(_num) {
    // isfinite is a macro on OS X in math.h, so we can't just say std::isfinite.
    using namespace std;
    rcheck(isfinite(data.r_num), base_exc_t::GENERIC,
           strprintf("Non-finite number: %" PR_RECONSTRUCTABLE_DOUBLE, data.r_num));
}

datum_t::datum_t(std::string &&_str) : data(std::move(_str)) {
    check_str_validity(data.r_str);
}

datum_t::datum_t(scoped_ptr_t<wire_string_t> str) : data(std::move(str)) {
    check_str_validity(data.r_str);
}

datum_t::datum_t(const char *cstr) : data(cstr) { }

datum_t::datum_t(std::vector<counted_t<const datum_t> > &&_array,
                 const configured_limits_t &limits)
    : data(std::move(_array)) {
    rcheck_array_size(*data.r_array, limits, base_exc_t::GENERIC);
}

datum_t::datum_t(std::vector<counted_t<const datum_t> > &&_array,
                 no_array_size_limit_check_t) : data(std::move(_array)) { }

datum_t::datum_t(std::map<std::string, counted_t<const datum_t> > &&_object,
                 const std::set<std::string> &allowed_pts)
    : data(std::move(_object)) {
    maybe_sanitize_ptype(allowed_pts);
}

datum_t::datum_t(std::map<std::string, counted_t<const datum_t> > &&_object,
                 no_sanitize_ptype_t)
    : data(std::move(_object)) { }

counted_t<const datum_t> to_datum(grouped_data_t &&gd,
                                  const configured_limits_t &limits) {
    std::map<std::string, counted_t<const datum_t> > map;
    map[datum_t::reql_type_string] = make_counted<const datum_t>("GROUPED_DATA");

    {
        datum_array_builder_t arr(limits);
        arr.reserve(gd.size());
        for (auto kv = gd.begin(); kv != gd.end(); ++kv) {
            arr.add(make_counted<const datum_t>(
                            std::vector<counted_t<const datum_t> >{
                                std::move(kv->first), std::move(kv->second) },
                            limits));
        }
        map["data"] = std::move(arr).to_counted();
    }

    // We don't sanitize the ptype because this is a fake ptype that should only
    // be used for serialization.
    // TODO(2014-08): This is a bad thing.
    return make_counted<datum_t>(std::move(map), datum_t::no_sanitize_ptype_t());
}

datum_t::~datum_t() {
}

counted_t<const datum_t> datum_t::empty_array() {
    return make_counted<datum_t>(std::vector<counted_t<const datum_t> >(),
                                 no_array_size_limit_check_t());
}

counted_t<const datum_t> datum_t::empty_object() {
    return make_counted<datum_t>(std::map<std::string, counted_t<const datum_t> >());
}

counted_t<const datum_t> datum_t::null() {
    return make_counted<datum_t>(construct_null_t());
}

counted_t<const datum_t> datum_t::boolean(bool value) {
    return make_counted<datum_t>(construct_boolean_t(), value);
}

counted_t<const datum_t> datum_t::binary(scoped_ptr_t<wire_string_t> data) {
    return make_counted<datum_t>(construct_binary_t(), std::move(data));
}<commit_msg>fixing build<commit_after>// Copyright 2010-2014 RethinkDB, all rights reserved.
#include "rdb_protocol/datum.hpp"

#include <float.h>
#include <math.h>
#include <stdint.h>
#include <stdlib.h>

#include <algorithm>

#include "errors.hpp"
#include <boost/detail/endian.hpp>

#include "containers/archive/stl_types.hpp"
#include "containers/scoped.hpp"
#include "rdb_protocol/env.hpp"
#include "rdb_protocol/error.hpp"
#include "rdb_protocol/pseudo_binary.hpp"
#include "rdb_protocol/pseudo_literal.hpp"
#include "rdb_protocol/pseudo_time.hpp"
#include "rdb_protocol/shards.hpp"
#include "stl_utils.hpp"

namespace ql {

const size_t tag_size = 8;

const std::set<std::string> datum_t::_allowed_pts = std::set<std::string>();

const char *const datum_t::reql_type_string = "$reql_type$";

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_null_t) :
    type(R_NULL), r_str(NULL) { }

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_boolean_t, bool _bool) :
    type(R_BOOL), r_bool(_bool) { }

datum_t::data_wrapper_t::data_wrapper_t(datum_t::construct_binary_t,
                                        scoped_ptr_t<wire_string_t> _data) :
    type(R_BINARY), r_str(_data.release()) { }

datum_t::data_wrapper_t::data_wrapper_t(double num) :
    type(R_NUM), r_num(num) { }

datum_t::data_wrapper_t::data_wrapper_t(std::string &&str) :
    type(R_STR),
    r_str(wire_string_t::create_and_init(str.size(), str.data()).release()) { }

datum_t::data_wrapper_t::data_wrapper_t(scoped_ptr_t<wire_string_t> str) :
    type(R_STR), r_str(str.release()) { }

datum_t::data_wrapper_t::data_wrapper_t(const char *cstr) :
    type(R_STR),
    r_str(wire_string_t::create_and_init(::strlen(cstr), cstr).release()) { }

datum_t::data_wrapper_t::data_wrapper_t(std::vector<counted_t<const datum_t> > &&array) :
    type(R_ARRAY),
    r_array(new std::vector<counted_t<const datum_t> >(std::move(array))) { }

datum_t::data_wrapper_t::data_wrapper_t(std::map<std::string,
                                                 counted_t<const datum_t> > &&object) :
    type(R_OBJECT),
    r_object(new std::map<std::string, counted_t<const datum_t> >(std::move(object))) { }

datum_t::data_wrapper_t::~data_wrapper_t() {
    switch (type) {
    case R_NULL: // fallthru
    case R_BOOL: // fallthru
    case R_NUM: break;
    case R_BINARY: // fallthru
    case R_STR: {
        delete r_str;
    } break;
    case R_ARRAY: {
        delete r_array;
    } break;
    case R_OBJECT: {
        delete r_object;
    } break;
    default: unreachable();
    }
}

datum_t::datum_t(datum_t::construct_null_t dummy) : data(dummy) { }

datum_t::datum_t(construct_boolean_t dummy, bool _bool) : data(dummy, _bool) { }

datum_t::datum_t(construct_binary_t dummy, scoped_ptr_t<wire_string_t> _data)
    : data(dummy, std::move(_data)) { }

datum_t::datum_t(double _num) : data(_num) {
    // isfinite is a macro on OS X in math.h, so we can't just say std::isfinite.
    using namespace std;
    rcheck(isfinite(data.r_num), base_exc_t::GENERIC,
           strprintf("Non-finite number: %" PR_RECONSTRUCTABLE_DOUBLE, data.r_num));
}

datum_t::datum_t(std::string &&_str) : data(std::move(_str)) {
    check_str_validity(data.r_str);
}

datum_t::datum_t(scoped_ptr_t<wire_string_t> str) : data(std::move(str)) {
    check_str_validity(data.r_str);
}

datum_t::datum_t(const char *cstr) : data(cstr) { }

datum_t::datum_t(std::vector<counted_t<const datum_t> > &&_array,
                 const configured_limits_t &limits)
    : data(std::move(_array)) {
    rcheck_array_size(*data.r_array, limits, base_exc_t::GENERIC);
}

datum_t::datum_t(std::vector<counted_t<const datum_t> > &&_array,
                 no_array_size_limit_check_t) : data(std::move(_array)) { }

datum_t::datum_t(std::map<std::string, counted_t<const datum_t> > &&_object,
                 const std::set<std::string> &allowed_pts)
    : data(std::move(_object)) {
    maybe_sanitize_ptype(allowed_pts);
}

datum_t::datum_t(std::map<std::string, counted_t<const datum_t> > &&_object,
                 no_sanitize_ptype_t)
    : data(std::move(_object)) { }

counted_t<const datum_t> to_datum(grouped_data_t &&gd,
                                  const configured_limits_t &limits) {
    std::map<std::string, counted_t<const datum_t> > map;
    map[datum_t::reql_type_string] = make_counted<const datum_t>("GROUPED_DATA");

    {
        datum_array_builder_t arr(limits);
        arr.reserve(gd.size());
        for (auto kv = gd.begin(); kv != gd.end(); ++kv) {
            arr.add(make_counted<const datum_t>(
                            std::vector<counted_t<const datum_t> >{
                                std::move(kv->first), std::move(kv->second) },
                            limits));
        }
        map["data"] = std::move(arr).to_counted();
    }

    // We don't sanitize the ptype because this is a fake ptype that should only
    // be used for serialization.
    // TODO(2014-08): This is a bad thing.
    return make_counted<datum_t>(std::move(map), datum_t::no_sanitize_ptype_t());
}

datum_t::~datum_t() {
}

counted_t<const datum_t> datum_t::empty_array() {
    return make_counted<datum_t>(std::vector<counted_t<const datum_t> >(),
                                 no_array_size_limit_check_t());
}

counted_t<const datum_t> datum_t::empty_object() {
    return make_counted<datum_t>(std::map<std::string, counted_t<const datum_t> >());
}

counted_t<const datum_t> datum_t::null() {
    return make_counted<datum_t>(construct_null_t());
}

counted_t<const datum_t> datum_t::boolean(bool value) {
    return make_counted<datum_t>(construct_boolean_t(), value);
}

counted_t<const datum_t> datum_t::binary(scoped_ptr_t<wire_string_t> _data) {
    return make_counted<datum_t>(construct_binary_t(), std::move(_data));
}
{'fraction_non_alphanumeric': 0.14826308076889924, 'fraction_numerical': 0.002585983966899405, 'mean_word_length': 4.093064091308165, 'pattern_counts': {'":': 0, '<': 107, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4480309', 'n_tokens_mistral': 4499, 'n_tokens_neox': 4475, 'n_words': 633}