<commit_before>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "rdb_protocol/op.hpp"
#include "rdb_protocol/term.hpp"
#include "rdb_protocol/terms/terms.hpp"

namespace ql {
class json_term_t : public op_term_t {
public:
    json_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }

    counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        std::string data = arg(env, 0)->as_str();
        scoped_cJSON_t cjson(cJSON_Parse(data.c_str()));
        rcheck(cjson.get() != NULL, base_exc_t::GENERIC,
               strprintf("Failed to parse \"%s\" as JSON.",
                 (data.size() > 40 ? (data.substr(0, 37) + "...").c_str() : data.c_str())));
        return new_val(make_counted<const datum_t>(cjson.get()));
    }

    bool is_deterministic() const {
        return true;
    }
    virtual const char *name() const { return "json"; }
};

counted_t<term_t> make_json_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<json_term_t>(env, term);
}
} // namespace ql

<commit_msg>fixed json_term_t is_deterministic<commit_after>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "rdb_protocol/op.hpp"
#include "rdb_protocol/term.hpp"
#include "rdb_protocol/terms/terms.hpp"

namespace ql {
class json_term_t : public op_term_t {
public:
    json_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }

    counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        std::string data = arg(env, 0)->as_str();
        scoped_cJSON_t cjson(cJSON_Parse(data.c_str()));
        rcheck(cjson.get() != NULL, base_exc_t::GENERIC,
               strprintf("Failed to parse \"%s\" as JSON.",
                 (data.size() > 40 ? (data.substr(0, 37) + "...").c_str() : data.c_str())));
        return new_val(make_counted<const datum_t>(cjson.get()));
    }

    virtual const char *name() const { return "json"; }
};

counted_t<term_t> make_json_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<json_term_t>(env, term);
}
} // namespace ql


{'fraction_non_alphanumeric': 0.14799635701275046, 'fraction_numerical': 0.01366120218579235, 'mean_word_length': 3.6546610169491527, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1531377', 'n_tokens_mistral': 846, 'n_tokens_neox': 810, 'n_words': 163}