<commit_before>/*
** -*- coding: utf-8 -*-
**
** File: document_object.cpp
** by <NAME>
** <EMAIL>
**
*/

#include <Python.h>
#include <structmember.h>
#include <rapidxml.hpp>
#include <vector>
#include <fstream>
#include <streambuf>
#include <cstring>
#include <cerrno>

#include <common.h>

static void rapidxml_DocumentObject_dealloc(rapidxml_DocumentObject* self) {
  delete self->base.base.document;
  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}

static int _parse(rapidxml_DocumentObject* self,
                  Py_buffer* text_buff,
                  bool from_file, bool parse_cdata=false) {
  const char* text;
  std::vector<char> text_vector;

  text = static_cast<const char*>(text_buff->buf);
  if (from_file) {
    std::ifstream f(text, std::ios::binary);
    if (f.fail()) {
      PyErr_SetString(rapidxml_RapidXmlError, strerror(errno));
      return 0;
    }
    text_vector = std::vector<char>((std::istreambuf_iterator<char>(f)),
                                    std::istreambuf_iterator<char>());
    text_vector.push_back(0);
    text = &text_vector[0];
  }
  try {
    self->base.base.document->clear();
	if (parse_cdata){
		(self->base.base.document
			->parse<rapidxml::parse_no_utf8 | rapidxml::parse_no_data_nodes>)
				(self->base.base.document->allocate_string(text));
		
	}else{
		(self->base.base.document
			->parse<rapidxml::parse_declaration_node>)
				(self->base.base.document->allocate_string(text));
	}
    
  } catch (rapidxml::parse_error &e) {
    PyErr_SetString(rapidxml_RapidXmlError, e.what());
    return 0;
  }
  return 1;
}

static PyObject* rapidxml_DocumentObject_parse(rapidxml_DocumentObject* self,
                                               PyObject* args,
                                               PyObject* kwds) {
  Py_buffer text_buff;
  PyObject* from_file_obj = NULL;
  bool read_cdata = false
  char kw_text[] = "text";
  char kw_from_file[] = "from_file";
  char kw_parse_cdata[] = "parse_cdata";

  static char* kwlist[] = {kw_text, kw_from_file, kw_parse_cdata, NULL};
  if (!PyArg_ParseTupleAndKeywords(args, kwds, "s*|Op", kwlist,
                                   &text_buff, &from_file_obj, &read_cdata)) {
    return NULL;
  }

  if (!_parse(self, &text_buff,
              (from_file_obj != NULL) && PyObject_IsTrue(from_file_obj), read_cdata)) {
    return NULL;
  }
  Py_INCREF(Py_None);
  return Py_None;
}

static int rapidxml_DocumentObject_init(rapidxml_DocumentObject* self,
                                        PyObject* args,
                                        PyObject* kwds) {
  Py_buffer text_buff = {0};
  PyObject* from_file_obj = NULL;
  bool read_cdata = false
  char kw_text[] = "text";
  char kw_from_file[] = "from_file";
  char kw_parse_cdata[] = "parse_cdata";

  if (rapidxml_NodeType.tp_init(reinterpret_cast<PyObject*>(self), args, kwds) < 0) {
    return -1;
  }
  static char* kwlist[] = {kw_text, kw_from_file, kw_parse_cdata, NULL};
  if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s*Op", kwlist,
                                   &text_buff, &from_file_obj, &read_cdata)) {
    return -1;
  }
  self->base.base.underlying_obj = new rapidxml::xml_document<>();
  self->base.base.document = static_cast<rapidxml::xml_document<>*>(self->base.base.underlying_obj);
  if (text_buff.buf) {
    return _parse(self, &text_buff, (from_file_obj != NULL) && PyObject_IsTrue(from_file_obj), read_cdata) - 1;
  }
  return 0;
}

static PyObject* rapidxml_DocumentObject_allocate_node(rapidxml_DocumentObject* self,
                                                       PyObject* args,
                                                       PyObject* kwds) {
  const char* name;
  Py_buffer name_buff = {0};
  const char* value;
  Py_buffer value_buff = {0};<commit_msg>cdata decodoing<commit_after>/*
** -*- coding: utf-8 -*-
**
** File: document_object.cpp
** by <NAME>
** <EMAIL>
**
*/

#include <Python.h>
#include <structmember.h>
#include <rapidxml.hpp>
#include <vector>
#include <fstream>
#include <streambuf>
#include <cstring>
#include <cerrno>

#include <common.h>

static void rapidxml_DocumentObject_dealloc(rapidxml_DocumentObject* self) {
  delete self->base.base.document;
  Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}

static int _parse(rapidxml_DocumentObject* self,
                  Py_buffer* text_buff,
                  bool from_file, bool parse_cdata=false) {
  const char* text;
  std::vector<char> text_vector;

  text = static_cast<const char*>(text_buff->buf);
  if (from_file) {
    std::ifstream f(text, std::ios::binary);
    if (f.fail()) {
      PyErr_SetString(rapidxml_RapidXmlError, strerror(errno));
      return 0;
    }
    text_vector = std::vector<char>((std::istreambuf_iterator<char>(f)),
                                    std::istreambuf_iterator<char>());
    text_vector.push_back(0);
    text = &text_vector[0];
  }
  try {
    self->base.base.document->clear();
	if (parse_cdata){
		(self->base.base.document
			->parse<rapidxml::parse_no_utf8 | rapidxml::parse_no_data_nodes>)
				(self->base.base.document->allocate_string(text));
		
	}else{
		(self->base.base.document
			->parse<rapidxml::parse_declaration_node>)
				(self->base.base.document->allocate_string(text));
	}
    
  } catch (rapidxml::parse_error &e) {
    PyErr_SetString(rapidxml_RapidXmlError, e.what());
    return 0;
  }
  return 1;
}

static PyObject* rapidxml_DocumentObject_parse(rapidxml_DocumentObject* self,
                                               PyObject* args,
                                               PyObject* kwds) {
  Py_buffer text_buff;
  PyObject* from_file_obj = NULL;
  bool read_cdata = false;
  char kw_text[] = "text";
  char kw_from_file[] = "from_file";
  char kw_parse_cdata[] = "parse_cdata";

  static char* kwlist[] = {kw_text, kw_from_file, kw_parse_cdata, NULL};
  if (!PyArg_ParseTupleAndKeywords(args, kwds, "s*|Op", kwlist,
                                   &text_buff, &from_file_obj, &read_cdata)) {
    return NULL;
  }

  if (!_parse(self, &text_buff,
              (from_file_obj != NULL) && PyObject_IsTrue(from_file_obj), read_cdata)) {
    return NULL;
  }
  Py_INCREF(Py_None);
  return Py_None;
}

static int rapidxml_DocumentObject_init(rapidxml_DocumentObject* self,
                                        PyObject* args,
                                        PyObject* kwds) {
  Py_buffer text_buff = {0};
  PyObject* from_file_obj = NULL;
  bool read_cdata = false;
  char kw_text[] = "text";
  char kw_from_file[] = "from_file";
  char kw_parse_cdata[] = "parse_cdata";

  if (rapidxml_NodeType.tp_init(reinterpret_cast<PyObject*>(self), args, kwds) < 0) {
    return -1;
  }
  static char* kwlist[] = {kw_text, kw_from_file, kw_parse_cdata, NULL};
  if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s*Op", kwlist,
                                   &text_buff, &from_file_obj, &read_cdata)) {
    return -1;
  }
  self->base.base.underlying_obj = new rapidxml::xml_document<>();
  self->base.base.document = static_cast<rapidxml::xml_document<>*>(self->base.base.underlying_obj);
  if (text_buff.buf) {
    return _parse(self, &text_buff, (from_file_obj != NULL) && PyObject_IsTrue(from_file_obj), read_cdata) - 1;
  }
  return 0;
}

static PyObject* rapidxml_DocumentObject_allocate_node(rapidxml_DocumentObject* self,
                                                       PyObject* args,
                                                       PyObject* kwds) {
  const char* name;
  Py_buffer name_buff = {0};
  const char* value;
  Py_buffer value_buff = {0};
{'fraction_non_alphanumeric': 0.13205060622034792, 'fraction_numerical': 0.003953610964681075, 'mean_word_length': 2.8561991869918697, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 42}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4137554', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2560, 'n_words': 460}