<commit_before>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <Python.h>
#include <datetime.h>

#include <bson/bson.h>

#include "cbson-dbref.h"
#include "cbson-oid.h"
#include "cbson-util.h"


static bson_context_t *gContext;
static PyObject       *gStr_id;


static BSON_INLINE void
cbson_loads_set_item (PyObject   *obj,
                      const char *key,
                      PyObject   *value)
{
   PyObject *keyobj;

   if (PyDict_Check(obj)) {
      if (*key == '_' && !strcmp(key, "_id")) {
         keyobj = gStr_id;
         Py_INCREF(keyobj);
      } else if (!(keyobj = PyUnicode_DecodeUTF8(key, strlen(key), "strict"))) {
         keyobj = PyString_FromString(key);
      }
      PyDict_SetItem(obj, keyobj, value);
      Py_DECREF(keyobj);
   } else {
      PyList_Append(obj, value);
   }
}


static PyObject *
cbson_regex_new (const char *regex,
                 const char *options)
{
   bson_uint32_t flags = 0;
   bson_uint32_t i;
   PyObject *module;
   PyObject *compile;
   PyObject *strobj;
   PyObject *value;

   /*
    * NOTE: Typically, you would want to cache the module from
    *       PyImport_ImportModule(). However, it is really not typical to
    *       have regexes stored so loading it each time is not a common
    *       case.
    */

   if (!(module = PyImport_ImportModule("re"))) {
      return NULL;
   }

   if (!(compile = PyObject_GetAttrString(module, "compile"))) {
      Py_DECREF(module);
      return NULL;
   }

   for (i = 0; options[i]; i++) {
      switch (options[i]) {
      case 'i':
         flags |= 2;
         break;
      case 'l':
         flags |= 4;
         break;
      case 'm':
         flags |= 8;
         break;
      case 's':
         flags |= 16;
         break;
      case 'u':
         flags |= 32;
         break;
      case 'x':
         flags |= 64;
         break;
      default:
         break;
      }
   }

   strobj = PyUnicode_FromString(regex);
   value = PyObject_CallFunction(compile, (char *)"Oi", strobj, flags);
   Py_DECREF(compile);
   Py_DECREF(strobj);
   Py_DECREF(module);

   return value;
}


static bson_bool_t
cbson_loads_visit_utf8 (const bson_iter_t *iter,
                        const char        *key,
                        size_t             v_utf8_len,
                        const char        *v_utf8,
                        void              *data)
{
   PyObject **ret = data;
   PyObject *value;

   if (!(value = PyUnicode_DecodeUTF8(v_utf8, v_utf8_len, "strict"))) {
      Py_DECREF(*ret);
      *ret = NULL;
      return TRUE;
   }<commit_msg>cbson: check return value of PyUnicode_FromString().<commit_after>/*
 * Copyright 2013 10gen Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <Python.h>
#include <datetime.h>

#include <bson/bson.h>

#include "cbson-dbref.h"
#include "cbson-oid.h"
#include "cbson-util.h"


static bson_context_t *gContext;
static PyObject       *gStr_id;


static BSON_INLINE void
cbson_loads_set_item (PyObject   *obj,
                      const char *key,
                      PyObject   *value)
{
   PyObject *keyobj;

   if (PyDict_Check(obj)) {
      if (*key == '_' && !strcmp(key, "_id")) {
         keyobj = gStr_id;
         Py_INCREF(keyobj);
      } else if (!(keyobj = PyUnicode_DecodeUTF8(key, strlen(key), "strict"))) {
         keyobj = PyString_FromString(key);
      }
      PyDict_SetItem(obj, keyobj, value);
      Py_DECREF(keyobj);
   } else {
      PyList_Append(obj, value);
   }
}


static PyObject *
cbson_regex_new (const char *regex,
                 const char *options)
{
   bson_uint32_t flags = 0;
   bson_uint32_t i;
   PyObject *module;
   PyObject *compile;
   PyObject *strobj;
   PyObject *value = NULL;

   /*
    * NOTE: Typically, you would want to cache the module from
    *       PyImport_ImportModule(). However, it is really not typical to
    *       have regexes stored so loading it each time is not a common
    *       case.
    */

   if (!(module = PyImport_ImportModule("re"))) {
      return NULL;
   }

   if (!(compile = PyObject_GetAttrString(module, "compile"))) {
      Py_DECREF(module);
      return NULL;
   }

   for (i = 0; options[i]; i++) {
      switch (options[i]) {
      case 'i':
         flags |= 2;
         break;
      case 'l':
         flags |= 4;
         break;
      case 'm':
         flags |= 8;
         break;
      case 's':
         flags |= 16;
         break;
      case 'u':
         flags |= 32;
         break;
      case 'x':
         flags |= 64;
         break;
      default:
         break;
      }
   }

   if ((strobj = PyUnicode_FromString(regex))) {
      value = PyObject_CallFunction(compile, (char *)"Oi", strobj, flags);
      Py_DECREF(strobj);
   }

   Py_DECREF(compile);
   Py_DECREF(module);

   return value;
}


static bson_bool_t
cbson_loads_visit_utf8 (const bson_iter_t *iter,
                        const char        *key,
                        size_t             v_utf8_len,
                        const char        *v_utf8,
                        void              *data)
{
   PyObject **ret = data;
   PyObject *value;

   if (!(value = PyUnicode_DecodeUTF8(v_utf8, v_utf8_len, "strict"))) {
      Py_DECREF(*ret);
      *ret = NULL;
      return TRUE;
   }
{'fraction_non_alphanumeric': 0.10921177587844255, 'fraction_numerical': 0.010129787907565685, 'mean_word_length': 2.094515181194907, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6114384', 'n_tokens_mistral': 2044, 'n_tokens_neox': 2003, 'n_words': 579}