<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <memory.h>
#include <string>
#include <iostream>
#include <vector>

#include <libxml/xmlmemory.h>

#include <zorba/store_consts.h>

#include "store/naive/store_defs.h"
#include "store/naive/string_pool.h"
#include "store/naive/qname_pool.h"
#include "store/naive/simple_store.h"
#include "store/naive/atomic_items.h"
#include "store/naive/node_items.h"
#include "store/naive/dataguide.h"
#include "store/naive/nsbindings.h"
#include "store/naive/loader.h"
#include "store/naive/simple_item_factory.h"
#include "store/naive/node_factory.h"

#include "zorbatypes/datetime.h"
#include "zorbatypes/URI.h"

#include "zorbaerrors/error_manager.h"
#include "zorbaerrors/Assert.h"


namespace zorba { namespace simplestore {

#ifndef NDEBUG

#define LOADER_TRACE(level, msg)             \
{                                            \
  if (level <= loader.theTraceLevel)         \
    std::cout << msg << std::endl;           \
}

#define LOADER_TRACE1(msg) LOADER_TRACE(1, msg);
#define LOADER_TRACE2(msg) LOADER_TRACE(2, msg);
#define LOADER_TRACE3(msg) LOADER_TRACE(3, msg);

#else
#define LOADER_TRACE(msg)
#define LOADER_TRACE1(msg)
#define LOADER_TRACE2(msg)
#define LOADER_TRACE3(msg)
#endif

#define ZORBA_LOADER_CHECK_ERROR(loader) \
  do { \
    if (loader.theErrorManager->hasErrors()) \
      return; \
  } while (0);

/*******************************************************************************

********************************************************************************/
DtdXmlLoader::DtdXmlLoader(
    BasicItemFactory* factory,
    error::ErrorManager* errorManager,
    bool dataguide)
  :
  XmlLoader(factory, errorManager, dataguide),
  theTree(NULL),
  theRootNode(NULL),
  theNodeStack(2048)
{
  theOrdPath.init();

  theTraceLevel = GET_STORE().getTraceLevel();

  memset(&theSaxHandler, 0, sizeof(theSaxHandler) );
  theSaxHandler.initialized = XML_SAX2_MAGIC;
  //theSaxHandler.startDocument = &DtdXmlLoader::startDocument;
  //theSaxHandler.endDocument = &DtdXmlLoader::endDocument;
  //theSaxHandler.startElementNs = &DtdXmlLoader::startElement;
  //theSaxHandler.endElementNs = &DtdXmlLoader::endElement;
  //theSaxHandler.characters = &DtdXmlLoader::characters;
  //theSaxHandler.cdataBlock = &DtdXmlLoader::cdataBlock;
  //theSaxHandler.comment = &DtdXmlLoader::comment;
  //theSaxHandler.processingInstruction = &DtdXmlLoader::processingInstruction;
  theSaxHandler.warning = &DtdXmlLoader::warning;
  theSaxHandler.error = &DtdXmlLoader::error;

  theSaxHandler.getEntity = &DtdXmlLoader::getEntity;
  theSaxHandler.getParameterEntity = &DtdXmlLoader::getParameterEntity;
  theSaxHandler.entityDecl = &DtdXmlLoader::entityDecl;
}


/*******************************************************************************

********************************************************************************/
DtdXmlLoader::~DtdXmlLoader()
{
}


/*******************************************************************************

********************************************************************************/
void DtdXmlLoader::setRoot(XmlNode* root)
{
  theRootNode = root;
  theTree->setRoot(root);
}


/*******************************************************************************
  Method called to do cleanup in case of errors.
********************************************************************************/
void DtdXmlLoader::abortload()
{
  theBaseUri.~zstring();
  theDocUri.~zstring();

  if (theTree != NULL)
  {
    delete theTree;
    theTree = NULL;
  }

  theOrdPath.init();
  theRootNode = NULL;

  while(!theNodeStack.empty())
  {
    XmlNode* node = theNodeStack.top();
    theNodeStack.pop();
    if (node != NULL)
      node->destroy();
  }

  thePathStack.clear();

#ifdef DATAGUIDE
  if(!theGuideStack.empty())
  {
    GuideNode* node = theGuideStack[0];
    if (node != NULL)
      node->deleteTree();
  }

  theGuideStack.clear();
#endif

  while(!theBindingsStack.empty())
  {
    //NsBindingsContext* ctx = theBindingsStack.top();
    theBindingsStack.pop();
    //delete ctx;
  }

  if(ctxt)
  {
    xmlCtxtReset(ctxt);
    xmlFreeParserCtxt(ctxt);
  }

  ctxt = NULL;
}


/*******************************************************************************
  Method used to reset the loader so it can be used to load another document.
********************************************************************************/
void DtdXmlLoader::reset()
{
  theBaseUri.~zstring();
  theDocUri.~zstring();

  theTree = NULL;
  theOrdPath.init();
  theRootNode = NULL;

  if ( !theNodeStack.empty() )
    theNodeStack.pop();

  ZORBA_ASSERT(theNodeStack.empty());
#ifdef DATAGUIDE
  ZORBA_ASSERT(theGuideStack.empty());
#endif
  ZORBA_ASSERT(theBindingsStack.empty());
}


/*******************************************************************************
  Read up to "size" bytes from the given input stream into the given buffer.
  Return the number of bytes actually read, throw an exception if any I/O
  error occured.
********************************************************************************/
std::streamsize DtdXmlLoader::readPacket(std::istream& stream, char* buf, std::streamoff size)
{
  try
  {
    stream.read(buf, size);

    if (stream.bad())
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "Input stream in bad state");
    }

    return stream.gcount();
  }
  catch (std::iostream::failure e)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                              STR0020_LOADER_IO_ERROR, e.what());
  }
  catch (...)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                              STR0020_LOADER_IO_ERROR,
                              "Unknown exception");
  }

  return -1;
}


/*******************************************************************************

********************************************************************************/
store::Item_t DtdXmlLoader::loadXml(
    const zstring& baseUri,
    const zstring& docUri,
    std::istream& stream)
{
  //  xmlParserCtxtPtr ctxt = NULL;
  theTree = GET_STORE().getNodeFactory().createXmlTree();

  xmlSubstituteEntitiesDefault(1);

  if (docUri.empty())
  {
    std::ostringstream uristream;
    uristream << "zorba://internalDocumentURI-" << theTree->getId();

    theDocUri = uristream.str();
  }
  else
  {
    theDocUri = docUri;
  }

  thePathStack.push(PathStepInfo(NULL, baseUri));

  theTree->setDocUri(theDocUri);
  theTree->setBaseUri(baseUri);

  char *theBuffer = NULL;
  try
  {
    stream.seekg(0, std::ios::end);
    std::streamoff fileSize = stream.tellg();
    stream.seekg(0, std::ios::beg);

    theBuffer = new char[static_cast<unsigned int>(fileSize+1)];
    theBuffer[fileSize] = 0;

    std::streamsize numChars = readPacket(stream, theBuffer, fileSize);

    if (numChars < 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "Unknown I/O error");
      abortload();
      delete[] theBuffer;
      return NULL;
    }
    else if (numChars == 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "No input data.");
      delete[] theBuffer;
      abortload();
      return NULL;
    }

    ctxt = xmlCreatePushParserCtxt( NULL, //&theSaxHandler,
                                   this,
                                   theBuffer,
                                   static_cast<int>(numChars),
                                   docUri.c_str());

    if (ctxt == NULL)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                                "Failed to initialize parser");
      delete[] theBuffer;
      abortload();
			return NULL;
    }

    /****** new *******/
    int options = XML_PARSE_COMPACT;
    options |= XML_PARSE_DTDVALID;
    options |= XML_PARSE_DTDLOAD;
    //options |= XML_PARSE_SAX1;
    //xmlSAXDefaultVersion(1);
    xmlLoadExtDtdDefaultValue |= XML_DETECT_IDS;
    xmlLoadExtDtdDefaultValue |= XML_COMPLETE_ATTRS;

    xmlCtxtUseOptions(ctxt, options);

    if ( xmlParseDocument(ctxt)==-1 )
    {
      std::cout << "  xmlParseDocument: Error: Unable to create tree: " <<
          ctxt->lastError.message << std::endl; std::cout.flush();
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                                "Unable to create tree");
      delete[] theBuffer;
      abortload();
      return NULL;
    }
    /****************/
    while ((numChars = readPacket(stream, theBuffer, INPUT_CHUNK_SIZE)) > 0)
    {
      xmlParseChunk(ctxt, theBuffer, static_cast<int>(numChars), 0);

      if (theErrorManager->hasErrors())
      {
        delete[] theBuffer;
        abortload();
        return NULL;
      }
    }

    if (numChars < 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0020_LOADER_IO_ERROR,
                                "Unknown I/O error");
      delete[] theBuffer;
      abortload();
      return NULL;
    }

    xmlParseChunk(ctxt, theBuffer, 0, 1);
    delete[] theBuffer;
  }
  catch(...)
  {
    delete[] theBuffer;
    abortload();
    thePathStack.clear();
    return NULL;
  }

  bool ok = ctxt->wellFormed != 0;

  // The doc may be well formed, but it may have other kinds of errors,
  // e.g., unresolved ns prefixes.
  if (theErrorManager->hasErrors())
  {
    abortload();
    return NULL;
  }
  else if (!ok )
  {
    if (!theDocUri.empty())
    {
      ZORBA_ERROR_PARAM_CONTINUE_OSS(theErrorManager,
                                     STR0021_LOADER_PARSING_ERROR,
                                     "The document with URI " << theDocUri
                                     <<" is not well formed", "");
    }
    else
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0021_LOADER_PARSING_ERROR,
                                "Not well formed XML");
    }

    abortload();
    return NULL;
  }
  else if ( ctxt->lastError.code != XML_ERR_OK )
  {
    std::cout << "-- LastErrorCode: " << ctxt->lastError.code << " " <<
        ctxt->lastError.message << " wellformed:" << ok << std::endl; std::cout.flush();

    if ( ctxt->lastError.code == XML_NS_ERR_UNDEFINED_NAMESPACE ||
        ctxt->lastError.code != XML_ERR_NO_DTD )
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0021_LOADER_PARSING_ERROR,
                                "Not well formed XML");

      abortload();
      return NULL;

    }
  }


  xmlDoc *doc = ctxt->myDoc;
  if (doc == NULL)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                              "Unable to create doc tree");
    abortload();
    return NULL;
  }

  loadDocument(doc);
  /* no need to free the document since is in ctxt */
  //xmlFreeDoc(doc);

  xmlCtxtReset(ctxt);
  xmlFreeParserCtxt(ctxt);
  ctxt = NULL;


  thePathStack.pop();
  assert(thePathStack.empty());

  XmlNode* resultNode;
  resultNode = theRootNode;
  reset();
  return resultNode;
}

void DtdXmlLoader::loadDocument(xmlDoc *doc)
{
  startDocument(this);

  /*Get the root element node */
  //xmlNode *root_element = xmlDocGetRootElement(doc);
  //print_element_names(root_element);
  loadNode(doc->children);

  endDocument(this);
}

void DtdXmlLoader::loadNode(xmlNode *a_node)
{
  xmlNode *cur_node = NULL;

  for (cur_node = a_node; cur_node; cur_node = cur_node->next)
  {
    switch( cur_node->type )
    {
      case XML_ELEMENT_NODE:
      {
        startElement(this, cur_node);
        loadNode(cur_node->children);
        endElement(this, cur_node);

        break;
      }
      case XML_TEXT_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        characters(this, content, xmlStrlen(content));
        xmlFree(content);
        break;
      }
      case XML_CDATA_SECTION_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        cdataBlock(this, content, xmlStrlen(content));
        xmlFree(content);
        break;
      }
      case XML_PI_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        processingInstruction(this, cur_node->name, content);
        xmlFree(content);
        break;
      }
      case XML_COMMENT_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        comment(this, content);
        xmlFree(content);
        break;
      }

      case XML_ATTRIBUTE_NODE:
      case XML_ENTITY_REF_NODE:
      case XML_ENTITY_NODE:
      case XML_DOCUMENT_NODE:
      case XML_DOCUMENT_TYPE_NODE:
      case XML_DOCUMENT_FRAG_NODE:
      case XML_NOTATION_NODE:
      case XML_HTML_DOCUMENT_NODE:
      case XML_DTD_NODE:
      case XML_ELEMENT_DECL:
      case XML_ATTRIBUTE_DECL:
      case XML_ENTITY_DECL:
      case XML_NAMESPACE_DECL:
      case XML_XINCLUDE_START:
      case XML_XINCLUDE_END:

#ifdef LIBXML_DOCB_ENABLED
      case XML_DOCB_DOCUMENT_NODE:
#endif //LIBXML_DOCB_ENABLED
        std::cout << "DtdXmlLoader::loadNode nodeType: " << (int)(cur_node->type)
          << std::endl; std::cout.flush();
        break;
      default:
        std::cout << "AssertError: Unknown libxml2 node type: " << (int)(cur_node->type)
          << std::endl; std::cout.flush();
        ZORBA_ASSERT(false);
    }
  }
}
/*******************************************************************************
  SAX2 callback when the start of document has been detected by the parser.

  ctx: the user data (XML parser context)
********************************************************************************/
void DtdXmlLoader::startDocument(void * ctx)
{
  DtdXmlLoader& loader = *(static_cast<DtdXmlLoader *>(ctx));
  ZORBA_LOADER_CHECK_ERROR(loader);

  try
  {
    XmlNode* docNode = GET_STORE().getNodeFactory().createDocumentNode();

    loader.setRoot(docNode);
    loader.theNodeStack.push(docNode);<commit_msg>Remove debug output.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <memory.h>
#include <string>
#include <iostream>
#include <vector>

#include <libxml/xmlmemory.h>

#include <zorba/store_consts.h>

#include "store/naive/store_defs.h"
#include "store/naive/string_pool.h"
#include "store/naive/qname_pool.h"
#include "store/naive/simple_store.h"
#include "store/naive/atomic_items.h"
#include "store/naive/node_items.h"
#include "store/naive/dataguide.h"
#include "store/naive/nsbindings.h"
#include "store/naive/loader.h"
#include "store/naive/simple_item_factory.h"
#include "store/naive/node_factory.h"

#include "zorbatypes/datetime.h"
#include "zorbatypes/URI.h"

#include "zorbaerrors/error_manager.h"
#include "zorbaerrors/Assert.h"


namespace zorba { namespace simplestore {

#ifndef NDEBUG

#define LOADER_TRACE(level, msg)             \
{                                            \
  if (level <= loader.theTraceLevel)         \
    std::cout << msg << std::endl;           \
}

#define LOADER_TRACE1(msg) LOADER_TRACE(1, msg);
#define LOADER_TRACE2(msg) LOADER_TRACE(2, msg);
#define LOADER_TRACE3(msg) LOADER_TRACE(3, msg);

#else
#define LOADER_TRACE(msg)
#define LOADER_TRACE1(msg)
#define LOADER_TRACE2(msg)
#define LOADER_TRACE3(msg)
#endif

#define ZORBA_LOADER_CHECK_ERROR(loader) \
  do { \
    if (loader.theErrorManager->hasErrors()) \
      return; \
  } while (0);

/*******************************************************************************

********************************************************************************/
DtdXmlLoader::DtdXmlLoader(
    BasicItemFactory* factory,
    error::ErrorManager* errorManager,
    bool dataguide)
  :
  XmlLoader(factory, errorManager, dataguide),
  theTree(NULL),
  theRootNode(NULL),
  theNodeStack(2048)
{
  theOrdPath.init();

  theTraceLevel = GET_STORE().getTraceLevel();

  memset(&theSaxHandler, 0, sizeof(theSaxHandler) );
  theSaxHandler.initialized = XML_SAX2_MAGIC;
  //theSaxHandler.startDocument = &DtdXmlLoader::startDocument;
  //theSaxHandler.endDocument = &DtdXmlLoader::endDocument;
  //theSaxHandler.startElementNs = &DtdXmlLoader::startElement;
  //theSaxHandler.endElementNs = &DtdXmlLoader::endElement;
  //theSaxHandler.characters = &DtdXmlLoader::characters;
  //theSaxHandler.cdataBlock = &DtdXmlLoader::cdataBlock;
  //theSaxHandler.comment = &DtdXmlLoader::comment;
  //theSaxHandler.processingInstruction = &DtdXmlLoader::processingInstruction;
  theSaxHandler.warning = &DtdXmlLoader::warning;
  theSaxHandler.error = &DtdXmlLoader::error;

  theSaxHandler.getEntity = &DtdXmlLoader::getEntity;
  theSaxHandler.getParameterEntity = &DtdXmlLoader::getParameterEntity;
  theSaxHandler.entityDecl = &DtdXmlLoader::entityDecl;
}


/*******************************************************************************

********************************************************************************/
DtdXmlLoader::~DtdXmlLoader()
{
}


/*******************************************************************************

********************************************************************************/
void DtdXmlLoader::setRoot(XmlNode* root)
{
  theRootNode = root;
  theTree->setRoot(root);
}


/*******************************************************************************
  Method called to do cleanup in case of errors.
********************************************************************************/
void DtdXmlLoader::abortload()
{
  theBaseUri.~zstring();
  theDocUri.~zstring();

  if (theTree != NULL)
  {
    delete theTree;
    theTree = NULL;
  }

  theOrdPath.init();
  theRootNode = NULL;

  while(!theNodeStack.empty())
  {
    XmlNode* node = theNodeStack.top();
    theNodeStack.pop();
    if (node != NULL)
      node->destroy();
  }

  thePathStack.clear();

#ifdef DATAGUIDE
  if(!theGuideStack.empty())
  {
    GuideNode* node = theGuideStack[0];
    if (node != NULL)
      node->deleteTree();
  }

  theGuideStack.clear();
#endif

  while(!theBindingsStack.empty())
  {
    //NsBindingsContext* ctx = theBindingsStack.top();
    theBindingsStack.pop();
    //delete ctx;
  }

  if(ctxt)
  {
    xmlCtxtReset(ctxt);
    xmlFreeParserCtxt(ctxt);
  }

  ctxt = NULL;
}


/*******************************************************************************
  Method used to reset the loader so it can be used to load another document.
********************************************************************************/
void DtdXmlLoader::reset()
{
  theBaseUri.~zstring();
  theDocUri.~zstring();

  theTree = NULL;
  theOrdPath.init();
  theRootNode = NULL;

  if ( !theNodeStack.empty() )
    theNodeStack.pop();

  ZORBA_ASSERT(theNodeStack.empty());
#ifdef DATAGUIDE
  ZORBA_ASSERT(theGuideStack.empty());
#endif
  ZORBA_ASSERT(theBindingsStack.empty());
}


/*******************************************************************************
  Read up to "size" bytes from the given input stream into the given buffer.
  Return the number of bytes actually read, throw an exception if any I/O
  error occured.
********************************************************************************/
std::streamsize DtdXmlLoader::readPacket(std::istream& stream, char* buf, std::streamoff size)
{
  try
  {
    stream.read(buf, size);

    if (stream.bad())
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "Input stream in bad state");
    }

    return stream.gcount();
  }
  catch (std::iostream::failure e)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                              STR0020_LOADER_IO_ERROR, e.what());
  }
  catch (...)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                              STR0020_LOADER_IO_ERROR,
                              "Unknown exception");
  }

  return -1;
}


/*******************************************************************************

********************************************************************************/
store::Item_t DtdXmlLoader::loadXml(
    const zstring& baseUri,
    const zstring& docUri,
    std::istream& stream)
{
  //  xmlParserCtxtPtr ctxt = NULL;
  theTree = GET_STORE().getNodeFactory().createXmlTree();

  xmlSubstituteEntitiesDefault(1);

  if (docUri.empty())
  {
    std::ostringstream uristream;
    uristream << "zorba://internalDocumentURI-" << theTree->getId();

    theDocUri = uristream.str();
  }
  else
  {
    theDocUri = docUri;
  }

  thePathStack.push(PathStepInfo(NULL, baseUri));

  theTree->setDocUri(theDocUri);
  theTree->setBaseUri(baseUri);

  char *theBuffer = NULL;
  try
  {
    stream.seekg(0, std::ios::end);
    std::streamoff fileSize = stream.tellg();
    stream.seekg(0, std::ios::beg);

    theBuffer = new char[static_cast<unsigned int>(fileSize+1)];
    theBuffer[fileSize] = 0;

    std::streamsize numChars = readPacket(stream, theBuffer, fileSize);

    if (numChars < 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "Unknown I/O error");
      abortload();
      delete[] theBuffer;
      return NULL;
    }
    else if (numChars == 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0020_LOADER_IO_ERROR,
                                "No input data.");
      delete[] theBuffer;
      abortload();
      return NULL;
    }

    ctxt = xmlCreatePushParserCtxt( NULL, //&theSaxHandler,
                                   this,
                                   theBuffer,
                                   static_cast<int>(numChars),
                                   docUri.c_str());

    if (ctxt == NULL)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                                "Failed to initialize parser");
      delete[] theBuffer;
      abortload();
			return NULL;
    }

    /****** new *******/
    int options = XML_PARSE_COMPACT;
    options |= XML_PARSE_DTDVALID;
    options |= XML_PARSE_DTDLOAD;
    //options |= XML_PARSE_SAX1;
    //xmlSAXDefaultVersion(1);
    xmlLoadExtDtdDefaultValue |= XML_DETECT_IDS;
    xmlLoadExtDtdDefaultValue |= XML_COMPLETE_ATTRS;

    xmlCtxtUseOptions(ctxt, options);

    if ( xmlParseDocument(ctxt)==-1 )
    {
      std::cout << "  xmlParseDocument: Error: Unable to create tree: " <<
          ctxt->lastError.message << std::endl; std::cout.flush();
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                                "Unable to create tree");
      delete[] theBuffer;
      abortload();
      return NULL;
    }
    /****************/
    while ((numChars = readPacket(stream, theBuffer, INPUT_CHUNK_SIZE)) > 0)
    {
      xmlParseChunk(ctxt, theBuffer, static_cast<int>(numChars), 0);

      if (theErrorManager->hasErrors())
      {
        delete[] theBuffer;
        abortload();
        return NULL;
      }
    }

    if (numChars < 0)
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0020_LOADER_IO_ERROR,
                                "Unknown I/O error");
      delete[] theBuffer;
      abortload();
      return NULL;
    }

    xmlParseChunk(ctxt, theBuffer, 0, 1);
    delete[] theBuffer;
  }
  catch(...)
  {
    delete[] theBuffer;
    abortload();
    thePathStack.clear();
    return NULL;
  }

  bool ok = ctxt->wellFormed != 0;

  // The doc may be well formed, but it may have other kinds of errors,
  // e.g., unresolved ns prefixes.
  if (theErrorManager->hasErrors())
  {
    abortload();
    return NULL;
  }
  else if (!ok )
  {
    if (!theDocUri.empty())
    {
      ZORBA_ERROR_PARAM_CONTINUE_OSS(theErrorManager,
                                     STR0021_LOADER_PARSING_ERROR,
                                     "The document with URI " << theDocUri
                                     <<" is not well formed", "");
    }
    else
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0021_LOADER_PARSING_ERROR,
                                "Not well formed XML");
    }

    abortload();
    return NULL;
  }
  else if ( ctxt->lastError.code != XML_ERR_OK )
  {
    std::cout << "-- LastErrorCode: " << ctxt->lastError.code << " " <<
        ctxt->lastError.message << " wellformed:" << ok << std::endl; std::cout.flush();

    if ( ctxt->lastError.code == XML_NS_ERR_UNDEFINED_NAMESPACE ||
        ctxt->lastError.code != XML_ERR_NO_DTD )
    {
      ZORBA_ERROR_DESC_CONTINUE(theErrorManager,
                                STR0021_LOADER_PARSING_ERROR,
                                "Not well formed XML");

      abortload();
      return NULL;

    }
  }


  xmlDoc *doc = ctxt->myDoc;
  if (doc == NULL)
  {
    ZORBA_ERROR_DESC_CONTINUE(theErrorManager, STR0021_LOADER_PARSING_ERROR,
                              "Unable to create doc tree");
    abortload();
    return NULL;
  }

  loadDocument(doc);
  /* no need to free the document since is in ctxt */
  //xmlFreeDoc(doc);

  xmlCtxtReset(ctxt);
  xmlFreeParserCtxt(ctxt);
  ctxt = NULL;


  thePathStack.pop();
  assert(thePathStack.empty());

  XmlNode* resultNode;
  resultNode = theRootNode;
  reset();
  return resultNode;
}

void DtdXmlLoader::loadDocument(xmlDoc *doc)
{
  startDocument(this);

  /*Get the root element node */
  //xmlNode *root_element = xmlDocGetRootElement(doc);
  //print_element_names(root_element);
  loadNode(doc->children);

  endDocument(this);
}

void DtdXmlLoader::loadNode(xmlNode *a_node)
{
  xmlNode *cur_node = NULL;

  for (cur_node = a_node; cur_node; cur_node = cur_node->next)
  {
    switch( cur_node->type )
    {
      case XML_ELEMENT_NODE:
      {
        startElement(this, cur_node);
        loadNode(cur_node->children);
        endElement(this, cur_node);

        break;
      }
      case XML_TEXT_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        characters(this, content, xmlStrlen(content));
        xmlFree(content);
        break;
      }
      case XML_CDATA_SECTION_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        cdataBlock(this, content, xmlStrlen(content));
        xmlFree(content);
        break;
      }
      case XML_PI_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        processingInstruction(this, cur_node->name, content);
        xmlFree(content);
        break;
      }
      case XML_COMMENT_NODE:
      {
        xmlChar * content = xmlNodeGetContent(cur_node);
        comment(this, content);
        xmlFree(content);
        break;
      }

      case XML_ATTRIBUTE_NODE:
      case XML_ENTITY_REF_NODE:
      case XML_ENTITY_NODE:
      case XML_DOCUMENT_NODE:
      case XML_DOCUMENT_TYPE_NODE:
      case XML_DOCUMENT_FRAG_NODE:
      case XML_NOTATION_NODE:
      case XML_HTML_DOCUMENT_NODE:
      case XML_DTD_NODE:
      case XML_ELEMENT_DECL:
      case XML_ATTRIBUTE_DECL:
      case XML_ENTITY_DECL:
      case XML_NAMESPACE_DECL:
      case XML_XINCLUDE_START:
      case XML_XINCLUDE_END:

#ifdef LIBXML_DOCB_ENABLED
      case XML_DOCB_DOCUMENT_NODE:
#endif //LIBXML_DOCB_ENABLED
        break;
      default:
        std::cout << "AssertError: Unknown libxml2 node type: " << (int)(cur_node->type)
          << std::endl; std::cout.flush();
        ZORBA_ASSERT(false);
    }
  }
}
/*******************************************************************************
  SAX2 callback when the start of document has been detected by the parser.

  ctx: the user data (XML parser context)
********************************************************************************/
void DtdXmlLoader::startDocument(void * ctx)
{
  DtdXmlLoader& loader = *(static_cast<DtdXmlLoader *>(ctx));
  ZORBA_LOADER_CHECK_ERROR(loader);

  try
  {
    XmlNode* docNode = GET_STORE().getNodeFactory().createDocumentNode();

    loader.setRoot(docNode);
    loader.theNodeStack.push(docNode);
{'fraction_non_alphanumeric': 0.19382851650991967, 'fraction_numerical': 0.006590238209651953, 'mean_word_length': 3.0197295805739515, 'pattern_counts': {'":': 0, '<': 111, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 78}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3373106', 'n_tokens_mistral': 9182, 'n_tokens_neox': 8831, 'n_words': 1691}