<commit_before>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include "bson-iter.h"


bson_bool_t
bson_iter_init (bson_iter_t  *iter,
                const bson_t *bson)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);

   if (BSON_UNLIKELY(bson->len < 5)) {
      return FALSE;
   }

   memset(iter, 0, sizeof *iter);

   iter->bson = bson;
   iter->offset = 0;
   iter->next_offset = 4;

   return TRUE;
}


bson_bool_t
bson_iter_recurse (const bson_iter_t *iter,
                   bson_iter_t       *child)
{
   const bson_uint8_t *data;
   bson_uint32_t len;
   bson_t b;

   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(child, FALSE);

   memset(child, 0, sizeof *child);

   if (*iter->type == BSON_TYPE_DOCUMENT) {
      bson_iter_document(iter, &len, &data);
   } else if (*iter->type == BSON_TYPE_ARRAY) {
      bson_iter_array(iter, &len, &data);
   } else {
      return FALSE;
   }

   /*
    * Load the BSON into an inline document inside the iter. Sadly, we have
    * to do this twice for the time being. We can get rid of that by not
    * using memset() in bson_iter_init().
    */
   if (!bson_init_static(&b, data, len) ||
       !bson_iter_init(child, &b) ||
       !bson_init_static(&child->inl_bson, data, len)) {
      return FALSE;
   }

   child->bson = &child->inl_bson;

   return TRUE;
}


bson_bool_t
bson_iter_init_find (bson_iter_t  *iter,
                     const bson_t *bson,
                     const char   *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return (bson_iter_init(iter, bson) && bson_iter_find(iter, key));
}


bson_bool_t
bson_iter_init_find_case (bson_iter_t  *iter,
                          const bson_t *bson,
                          const char   *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return (bson_iter_init(iter, bson) && bson_iter_find_case(iter, key));
}


static bson_bool_t
bson_iter_find_with_len (bson_iter_t *iter,
                         const char  *key,
                         int          keylen)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   if (keylen < 0) {
      keylen = strlen(key);
   }

   while (bson_iter_next(iter)) {
      if (!strncmp(key, bson_iter_key(iter), keylen)) {
         return TRUE;
      }
   }

   return FALSE;
}


bson_bool_t
bson_iter_find (bson_iter_t *iter,
                const char  *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return bson_iter_find_with_len(iter, key, -1);
}


bson_bool_t
bson_iter_find_case (bson_iter_t *iter,
                     const char  *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   while (bson_iter_next(iter)) {
      if (!strcasecmp(key, bson_iter_key(iter))) {
         return TRUE;
      }
   }

   return FALSE;
}


bson_bool_t
bson_iter_find_descendant (bson_iter_t *iter,
                           const char  *dotkey,
                           bson_iter_t *descendant)
{
   bson_iter_t tmp;
   const char *dot;
   size_t sublen;

   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(dotkey, FALSE);
   bson_return_val_if_fail(descendant, FALSE);

   if ((dot = strchr(dotkey, '.'))) {
      sublen = dot - dotkey;
   } else {
      sublen = strlen(dotkey);
   }

   if (bson_iter_find_with_len(iter, dotkey, sublen)) {
      if (!dot) {
         *descendant = *iter;
         return TRUE;
      }
      if (BSON_ITER_HOLDS_DOCUMENT(iter) || BSON_ITER_HOLDS_ARRAY(iter)) {
         if (bson_iter_recurse(iter, &tmp)) {
            return bson_iter_find_descendant(&tmp, dot + 1, descendant);
         }
      }
   }

   return FALSE;
}


const char *
bson_iter_key (const bson_iter_t *iter)
{
   bson_return_val_if_fail(iter, NULL);
   return bson_iter_key_unsafe(iter);
}


bson_type_t
bson_iter_type (const bson_iter_t *iter)
{
   bson_return_val_if_fail(iter, 0);
   bson_return_val_if_fail(iter->type, 0);
   return bson_iter_type_unsafe(iter);
}


bson_bool_t
bson_iter_next (bson_iter_t *iter)
{
   const bson_uint8_t *data;
   bson_uint32_t o;
   const bson_t *b;

   bson_return_val_if_fail(iter, FALSE);

   if (!iter->bson) {
      return FALSE;
   }

   b = iter->bson;
   data = bson_get_data(b);

   iter->offset = iter->next_offset;
   iter->type = &data[iter->offset];
   iter->key = &data[iter->offset + 1];
   iter->data1 = NULL;
   iter->data2 = NULL;
   iter->data3 = NULL;
   iter->data4 = NULL;

   for (o = iter->offset + 1; o < b->len; o++) {
      if (!data[o]) {
         iter->data1 = &data[++o];
         goto fill_data_fields;
      }
   }

   goto mark_invalid;

fill_data_fields:

   switch (*iter->type) {
   case BSON_TYPE_DATE_TIME:
   case BSON_TYPE_DOUBLE:
   case BSON_TYPE_INT64:
   case BSON_TYPE_TIMESTAMP:
      iter->next_offset = o + 8;
      break;
   case BSON_TYPE_CODE:
   case BSON_TYPE_SYMBOL:
   case BSON_TYPE_UTF8:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + 4 + l;

         /*
          * Make sure the string length includes the NUL byte.
          */
         if (BSON_UNLIKELY((l < 1) || (iter->next_offset >= iter->bson->len))) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         /*
          * Make sure the last byte is a NUL byte.
          */
         if (BSON_UNLIKELY(iter->data2[l - 1] != '\0')) {
            iter->err_offset = o + 4 + l - 1;
            goto mark_invalid;
         }
      }
      break;
   case BSON_TYPE_BINARY:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         iter->data3 = &data[o + 5];

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + 5 + l;
      }
      break;
   case BSON_TYPE_ARRAY:
   case BSON_TYPE_DOCUMENT:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + l;
      }
      break;
   case BSON_TYPE_OID:
      iter->next_offset = o + 12;
      break;
   case BSON_TYPE_BOOL:
      iter->next_offset = o + 1;
      break;
   case BSON_TYPE_REGEX:
      {
         bson_bool_t eor = FALSE;
         bson_bool_t eoo = FALSE;

         for (; o < b->len; o++) {
            if (!data[o]) {
               iter->data2 = &data[++o];
               eor = TRUE;
               break;
            }
         }

         if (!eor) {
            iter->err_offset = iter->next_offset;
            goto mark_invalid;
         }

         for (; o < b->len; o++) {
            if (!data[o]) {
               eoo = TRUE;
               break;
            }
         }

         if (!eoo) {
            iter->err_offset = iter->next_offset;
            goto mark_invalid;
         }

         iter->next_offset = o + 1;
      }
      break;
   case BSON_TYPE_DBPOINTER:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);

         iter->data3 = &data[o + 4 + l];
         iter->next_offset = o + 4 + l + 12;
      }
      break;
   case BSON_TYPE_CODEWSCOPE:
      {
         bson_uint32_t l;
         bson_uint32_t doclen;

         if ((o + 8) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         iter->data3 = &data[o + 8];

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         if (l < 14) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->next_offset = o + l;
         if (iter->next_offset >= iter->bson->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         memcpy(&l, iter->data2, 4);
         l = BSON_UINT32_FROM_LE(l);
         if (BSON_UNLIKELY((o + 4 + 4 + l + 4) >= iter->next_offset)) {
            iter->err_offset = o + 4;
            goto mark_invalid;
         }

         iter->data4 = &data[o + 4 + 4 + l];
         memcpy(&doclen, iter->data4, 4);
         doclen = BSON_UINT32_FROM_LE(doclen);
         if ((o + 4 + 4 + l + doclen) != iter->next_offset) {
            iter->err_offset = o + 4 + 4 + l;
            goto mark_invalid;
         }
      }
      break;
   case BSON_TYPE_INT32:
      iter->next_offset = o + 4;
      break;
   case BSON_TYPE_MAXKEY:
   case BSON_TYPE_MINKEY:
   case BSON_TYPE_NULL:
   case BSON_TYPE_UNDEFINED:
      iter->data1 = NULL;
      iter->next_offset = o;
      break;
   case BSON_TYPE_EOD:
   default:
      iter->err_offset = o;
      goto mark_invalid;
   }

   /*
    * Check to see if any of the field locations would overflow the
    * current BSON buffer. If so, set the error location to the offset
    * of where the field starts.
    */
   if (!(iter->next_offset < b->len)) {
      iter->err_offset = o;
      goto mark_invalid;
   }

   iter->err_offset = 0;
   return TRUE;

mark_invalid:
   iter->bson = NULL;
   return FALSE;
}


void
bson_iter_binary (const bson_iter_t   *iter,
                  bson_subtype_t      *subtype,
                  bson_uint32_t       *binary_len,
                  const bson_uint8_t **binary)
{
   bson_return_if_fail(iter);
   bson_return_if_fail(!binary || binary_len);

   if (*iter->type == BSON_TYPE_BINARY) {
      if (subtype) {
         *subtype = (bson_subtype_t)*iter->data2;
      }
      if (binary) {
         memcpy(binary_len, iter->data1, 4);
         *binary_len = BSON_UINT32_FROM_LE(*binary_len);
         *binary = iter->data3;
      }
   }
}<commit_msg>iter: initialize out parameters in failure case.<commit_after>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include "bson-iter.h"


bson_bool_t
bson_iter_init (bson_iter_t  *iter,
                const bson_t *bson)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);

   if (BSON_UNLIKELY(bson->len < 5)) {
      return FALSE;
   }

   memset(iter, 0, sizeof *iter);

   iter->bson = bson;
   iter->offset = 0;
   iter->next_offset = 4;

   return TRUE;
}


bson_bool_t
bson_iter_recurse (const bson_iter_t *iter,
                   bson_iter_t       *child)
{
   const bson_uint8_t *data;
   bson_uint32_t len;
   bson_t b;

   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(child, FALSE);

   memset(child, 0, sizeof *child);

   if (*iter->type == BSON_TYPE_DOCUMENT) {
      bson_iter_document(iter, &len, &data);
   } else if (*iter->type == BSON_TYPE_ARRAY) {
      bson_iter_array(iter, &len, &data);
   } else {
      return FALSE;
   }

   /*
    * Load the BSON into an inline document inside the iter. Sadly, we have
    * to do this twice for the time being. We can get rid of that by not
    * using memset() in bson_iter_init().
    */
   if (!bson_init_static(&b, data, len) ||
       !bson_iter_init(child, &b) ||
       !bson_init_static(&child->inl_bson, data, len)) {
      return FALSE;
   }

   child->bson = &child->inl_bson;

   return TRUE;
}


bson_bool_t
bson_iter_init_find (bson_iter_t  *iter,
                     const bson_t *bson,
                     const char   *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return (bson_iter_init(iter, bson) && bson_iter_find(iter, key));
}


bson_bool_t
bson_iter_init_find_case (bson_iter_t  *iter,
                          const bson_t *bson,
                          const char   *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(bson, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return (bson_iter_init(iter, bson) && bson_iter_find_case(iter, key));
}


static bson_bool_t
bson_iter_find_with_len (bson_iter_t *iter,
                         const char  *key,
                         int          keylen)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   if (keylen < 0) {
      keylen = strlen(key);
   }

   while (bson_iter_next(iter)) {
      if (!strncmp(key, bson_iter_key(iter), keylen)) {
         return TRUE;
      }
   }

   return FALSE;
}


bson_bool_t
bson_iter_find (bson_iter_t *iter,
                const char  *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   return bson_iter_find_with_len(iter, key, -1);
}


bson_bool_t
bson_iter_find_case (bson_iter_t *iter,
                     const char  *key)
{
   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(key, FALSE);

   while (bson_iter_next(iter)) {
      if (!strcasecmp(key, bson_iter_key(iter))) {
         return TRUE;
      }
   }

   return FALSE;
}


bson_bool_t
bson_iter_find_descendant (bson_iter_t *iter,
                           const char  *dotkey,
                           bson_iter_t *descendant)
{
   bson_iter_t tmp;
   const char *dot;
   size_t sublen;

   bson_return_val_if_fail(iter, FALSE);
   bson_return_val_if_fail(dotkey, FALSE);
   bson_return_val_if_fail(descendant, FALSE);

   if ((dot = strchr(dotkey, '.'))) {
      sublen = dot - dotkey;
   } else {
      sublen = strlen(dotkey);
   }

   if (bson_iter_find_with_len(iter, dotkey, sublen)) {
      if (!dot) {
         *descendant = *iter;
         return TRUE;
      }
      if (BSON_ITER_HOLDS_DOCUMENT(iter) || BSON_ITER_HOLDS_ARRAY(iter)) {
         if (bson_iter_recurse(iter, &tmp)) {
            return bson_iter_find_descendant(&tmp, dot + 1, descendant);
         }
      }
   }

   return FALSE;
}


const char *
bson_iter_key (const bson_iter_t *iter)
{
   bson_return_val_if_fail(iter, NULL);
   return bson_iter_key_unsafe(iter);
}


bson_type_t
bson_iter_type (const bson_iter_t *iter)
{
   bson_return_val_if_fail(iter, 0);
   bson_return_val_if_fail(iter->type, 0);
   return bson_iter_type_unsafe(iter);
}


bson_bool_t
bson_iter_next (bson_iter_t *iter)
{
   const bson_uint8_t *data;
   bson_uint32_t o;
   const bson_t *b;

   bson_return_val_if_fail(iter, FALSE);

   if (!iter->bson) {
      return FALSE;
   }

   b = iter->bson;
   data = bson_get_data(b);

   iter->offset = iter->next_offset;
   iter->type = &data[iter->offset];
   iter->key = &data[iter->offset + 1];
   iter->data1 = NULL;
   iter->data2 = NULL;
   iter->data3 = NULL;
   iter->data4 = NULL;

   for (o = iter->offset + 1; o < b->len; o++) {
      if (!data[o]) {
         iter->data1 = &data[++o];
         goto fill_data_fields;
      }
   }

   goto mark_invalid;

fill_data_fields:

   switch (*iter->type) {
   case BSON_TYPE_DATE_TIME:
   case BSON_TYPE_DOUBLE:
   case BSON_TYPE_INT64:
   case BSON_TYPE_TIMESTAMP:
      iter->next_offset = o + 8;
      break;
   case BSON_TYPE_CODE:
   case BSON_TYPE_SYMBOL:
   case BSON_TYPE_UTF8:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + 4 + l;

         /*
          * Make sure the string length includes the NUL byte.
          */
         if (BSON_UNLIKELY((l < 1) || (iter->next_offset >= iter->bson->len))) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         /*
          * Make sure the last byte is a NUL byte.
          */
         if (BSON_UNLIKELY(iter->data2[l - 1] != '\0')) {
            iter->err_offset = o + 4 + l - 1;
            goto mark_invalid;
         }
      }
      break;
   case BSON_TYPE_BINARY:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         iter->data3 = &data[o + 5];

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + 5 + l;
      }
      break;
   case BSON_TYPE_ARRAY:
   case BSON_TYPE_DOCUMENT:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         iter->next_offset = o + l;
      }
      break;
   case BSON_TYPE_OID:
      iter->next_offset = o + 12;
      break;
   case BSON_TYPE_BOOL:
      iter->next_offset = o + 1;
      break;
   case BSON_TYPE_REGEX:
      {
         bson_bool_t eor = FALSE;
         bson_bool_t eoo = FALSE;

         for (; o < b->len; o++) {
            if (!data[o]) {
               iter->data2 = &data[++o];
               eor = TRUE;
               break;
            }
         }

         if (!eor) {
            iter->err_offset = iter->next_offset;
            goto mark_invalid;
         }

         for (; o < b->len; o++) {
            if (!data[o]) {
               eoo = TRUE;
               break;
            }
         }

         if (!eoo) {
            iter->err_offset = iter->next_offset;
            goto mark_invalid;
         }

         iter->next_offset = o + 1;
      }
      break;
   case BSON_TYPE_DBPOINTER:
      {
         bson_uint32_t l;

         if ((o + 4) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);

         iter->data3 = &data[o + 4 + l];
         iter->next_offset = o + 4 + l + 12;
      }
      break;
   case BSON_TYPE_CODEWSCOPE:
      {
         bson_uint32_t l;
         bson_uint32_t doclen;

         if ((o + 8) >= b->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->data2 = &data[o + 4];
         iter->data3 = &data[o + 8];

         memcpy(&l, iter->data1, 4);
         l = BSON_UINT32_FROM_LE(l);
         if (l < 14) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         iter->next_offset = o + l;
         if (iter->next_offset >= iter->bson->len) {
            iter->err_offset = o;
            goto mark_invalid;
         }

         memcpy(&l, iter->data2, 4);
         l = BSON_UINT32_FROM_LE(l);
         if (BSON_UNLIKELY((o + 4 + 4 + l + 4) >= iter->next_offset)) {
            iter->err_offset = o + 4;
            goto mark_invalid;
         }

         iter->data4 = &data[o + 4 + 4 + l];
         memcpy(&doclen, iter->data4, 4);
         doclen = BSON_UINT32_FROM_LE(doclen);
         if ((o + 4 + 4 + l + doclen) != iter->next_offset) {
            iter->err_offset = o + 4 + 4 + l;
            goto mark_invalid;
         }
      }
      break;
   case BSON_TYPE_INT32:
      iter->next_offset = o + 4;
      break;
   case BSON_TYPE_MAXKEY:
   case BSON_TYPE_MINKEY:
   case BSON_TYPE_NULL:
   case BSON_TYPE_UNDEFINED:
      iter->data1 = NULL;
      iter->next_offset = o;
      break;
   case BSON_TYPE_EOD:
   default:
      iter->err_offset = o;
      goto mark_invalid;
   }

   /*
    * Check to see if any of the field locations would overflow the
    * current BSON buffer. If so, set the error location to the offset
    * of where the field starts.
    */
   if (!(iter->next_offset < b->len)) {
      iter->err_offset = o;
      goto mark_invalid;
   }

   iter->err_offset = 0;
   return TRUE;

mark_invalid:
   iter->bson = NULL;
   return FALSE;
}


void
bson_iter_binary (const bson_iter_t   *iter,
                  bson_subtype_t      *subtype,
                  bson_uint32_t       *binary_len,
                  const bson_uint8_t **binary)
{
   bson_return_if_fail(iter);
   bson_return_if_fail(!binary || binary_len);

   if (*iter->type == BSON_TYPE_BINARY) {
      if (subtype) {
         *subtype = (bson_subtype_t)*iter->data2;
      }
      if (binary) {
         memcpy(binary_len, iter->data1, 4);
         *binary_len = BSON_UINT32_FROM_LE(*binary_len);
         *binary = iter->data3;
      }
   }

   if (binary) {
      *binary = NULL;
   }

   if (binary_len) {
      *binary_len = 0;
   }

   if (subtype) {
      *subtype = 0;
   }
}
{'fraction_non_alphanumeric': 0.11675057208237986, 'fraction_numerical': 0.012814645308924484, 'mean_word_length': 1.94726193687618, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 209, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7343188', 'n_tokens_mistral': 8208, 'n_tokens_neox': 7855, 'n_words': 1692}