<commit_before>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <inttypes.h>
#include <stdarg.h>
#include <string.h>
#include <unistd.h>

#include "b64_ntop.h"

#include "bson.h"
#include "bson-md5.h"
#include "bson-memory.h"
#include "bson-string.h"
#include "bson-thread.h"
#include "bson-utf8.h"


/*
 * TODO:
 *
 *   - Put some consideration into if we want to handle OOM. It is a really
 *     difficult thing to do correctly. Almost nobody gets it right. D-BUS
 *     on GNU/Linux might be one of the few things that does.
 */


#define BSON_FLAG_NO_FREE (1 << 0)
#define BSON_FLAG_NO_GROW (1 << 1)
#define BSON_FLAG_CHILD   (1 << 2)


static const bson_uint8_t gZero;


static bson_bool_t bson_as_json_visit_array    (const bson_iter_t *iter,
                                                const char        *key,
                                                const bson_t      *v_array,
                                                void              *data);
static bson_bool_t bson_as_json_visit_document (const bson_iter_t *iter,
                                                const char        *key,
                                                const bson_t      *v_document,
                                                void              *data);


typedef struct
{
   bson_uint32_t  count;
   bson_bool_t    keys;
   bson_string_t *str;
} bson_json_state_t;


/**
 * bson_get_data_fast:
 * @b: A bson_t.
 *
 * Fetches the beginning of the buffer for the BSON document taking into
 * account the posibility that it is a child BSON. A child BSON is a BSON
 * document currently being built within another BSON document.
 *
 * If you append data to @b after calling this function, the address may
 * have changed!
 *
 * Returns: A bson_uint8_t* buffer of at least b->len bytes in size.
 */
static BSON_INLINE bson_uint8_t *
bson_get_data_fast (const bson_t *b)
{
   if (!(b->flags & BSON_FLAG_CHILD))
      return b->top.data;
   return (*b->child.data) + b->child.offset;
}


const bson_uint8_t *
bson_get_data (const bson_t *bson)
{
   bson_return_val_if_fail(bson, NULL);
   return bson_get_data_fast(bson);
}


/**
 * bson_encode_length:
 * @bson: A bson_t.
 *
 * Encodes the length of the bson into the first four bytes of @bson.
 * This should be called any time you add a field. This used to be done
 * in a bson_finish() style call, but instead we just do it every time we
 * add a field internally.
 */
static BSON_INLINE void
bson_encode_length (bson_t *bson)
{
   bson_uint32_t len_le = BSON_UINT32_TO_LE(bson->len);
   bson_uint8_t *data = bson_get_data_fast(bson);
   memcpy(data, &len_le, 4);
}


/**
 * bson_grow_if_needed:
 * @bson: A bson_t to grow.
 * @additional_bytes: Number of additional bytes needed.
 *
 * Will check to see if there are enough bytes allocated for @additional_bytes
 * to be used. If not, it will grow the size of the bson by a power of two of
 * the current allocation size.
 *
 * Returns: @bson or a new memory location if the buffer was grown.
 */
static void
bson_grow_if_needed (bson_t *bson,
                     size_t  additional_bytes)
{
   size_t amin;
   size_t asize;

   bson_return_if_fail(bson);
   bson_return_if_fail(additional_bytes < INT_MAX);

   if ((bson->flags & BSON_FLAG_CHILD)) {
      bson_grow_if_needed(bson->child.toplevel, additional_bytes);
      return;
   }

   amin = bson->len + additional_bytes;

   if (amin <= sizeof bson->top.inlbuf) {
      return;
   }

   asize = 32;

   while (asize < amin) {
      asize <<= 1;
   }

   if (BSON_UNLIKELY(asize >= INT_MAX)) {<commit_msg>bson: start with 64 byte allocation once growing out of inline buffer.<commit_after>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <inttypes.h>
#include <stdarg.h>
#include <string.h>
#include <unistd.h>

#include "b64_ntop.h"

#include "bson.h"
#include "bson-md5.h"
#include "bson-memory.h"
#include "bson-string.h"
#include "bson-thread.h"
#include "bson-utf8.h"


/*
 * TODO:
 *
 *   - Put some consideration into if we want to handle OOM. It is a really
 *     difficult thing to do correctly. Almost nobody gets it right. D-BUS
 *     on GNU/Linux might be one of the few things that does.
 */


#define BSON_FLAG_NO_FREE (1 << 0)
#define BSON_FLAG_NO_GROW (1 << 1)
#define BSON_FLAG_CHILD   (1 << 2)


static const bson_uint8_t gZero;


static bson_bool_t bson_as_json_visit_array    (const bson_iter_t *iter,
                                                const char        *key,
                                                const bson_t      *v_array,
                                                void              *data);
static bson_bool_t bson_as_json_visit_document (const bson_iter_t *iter,
                                                const char        *key,
                                                const bson_t      *v_document,
                                                void              *data);


typedef struct
{
   bson_uint32_t  count;
   bson_bool_t    keys;
   bson_string_t *str;
} bson_json_state_t;


/**
 * bson_get_data_fast:
 * @b: A bson_t.
 *
 * Fetches the beginning of the buffer for the BSON document taking into
 * account the posibility that it is a child BSON. A child BSON is a BSON
 * document currently being built within another BSON document.
 *
 * If you append data to @b after calling this function, the address may
 * have changed!
 *
 * Returns: A bson_uint8_t* buffer of at least b->len bytes in size.
 */
static BSON_INLINE bson_uint8_t *
bson_get_data_fast (const bson_t *b)
{
   if (!(b->flags & BSON_FLAG_CHILD))
      return b->top.data;
   return (*b->child.data) + b->child.offset;
}


const bson_uint8_t *
bson_get_data (const bson_t *bson)
{
   bson_return_val_if_fail(bson, NULL);
   return bson_get_data_fast(bson);
}


/**
 * bson_encode_length:
 * @bson: A bson_t.
 *
 * Encodes the length of the bson into the first four bytes of @bson.
 * This should be called any time you add a field. This used to be done
 * in a bson_finish() style call, but instead we just do it every time we
 * add a field internally.
 */
static BSON_INLINE void
bson_encode_length (bson_t *bson)
{
   bson_uint32_t len_le = BSON_UINT32_TO_LE(bson->len);
   bson_uint8_t *data = bson_get_data_fast(bson);
   memcpy(data, &len_le, 4);
}


/**
 * bson_grow_if_needed:
 * @bson: A bson_t to grow.
 * @additional_bytes: Number of additional bytes needed.
 *
 * Will check to see if there are enough bytes allocated for @additional_bytes
 * to be used. If not, it will grow the size of the bson by a power of two of
 * the current allocation size.
 *
 * Returns: @bson or a new memory location if the buffer was grown.
 */
static void
bson_grow_if_needed (bson_t *bson,
                     size_t  additional_bytes)
{
   size_t amin;
   size_t asize;

   bson_return_if_fail(bson);
   bson_return_if_fail(additional_bytes < INT_MAX);

   if ((bson->flags & BSON_FLAG_CHILD)) {
      bson_grow_if_needed(bson->child.toplevel, additional_bytes);
      return;
   }

   amin = bson->len + additional_bytes;

   if (amin <= sizeof bson->top.inlbuf) {
      return;
   }

   asize = 64;

   while (asize < amin) {
      asize <<= 1;
   }

   if (BSON_UNLIKELY(asize >= INT_MAX)) {
{'fraction_non_alphanumeric': 0.08751051809111672, 'fraction_numerical': 0.00865488640461594, 'mean_word_length': 2.541932737335036, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1005782', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2703, 'n_words': 905}