<commit_before>/*
 * Copyright (C) 2003 Apple Computer, Inc.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

/*
 * Copyright (C) 2012 Kolibre
 *
 * This file is part of kolibre-xmlreader.
 *
 * Kolibre-xmlreader is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * Kolibre-xmlreader is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with kolibre-xmlreader. If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * \class XmlReader
 *
 * \brief This is the main interface for the kolibre xmlreader. XmlReader lets you parse HMTL and XML resource (offline and online) and can even cached data for quicker access.
 */

#include "XmlReader.h"

#include "XmlAttributes.h"
#include "XmlError.h"
#include "XmlDefaultHandler.h"

#include "DataSource.h"

#include <malloc.h>
#include <libxml/encoding.h>

#include <cstring>
#include <sstream>

#include <log4cxx/logger.h>

// create logger which will become a child to logger kolibre.xmlreader
log4cxx::LoggerPtr xmlXmlReaderLog(
        log4cxx::Logger::getLogger("kolibre.xmlreader.xmlreader"));

static void startDocumentHandler(void *userData)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    LOG4CXX_DEBUG(xmlXmlReaderLog, "startDocumentHandler()");
    reader->contentHandler()->startDocument();
}

static void endDocumentHandler(void *userData)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    LOG4CXX_DEBUG(xmlXmlReaderLog, "endDocumentHandler()");
    reader->endDocumentHandlerCalled();
    reader->contentHandler()->endDocument();
}

static void startElementHandler(void *userData, const xmlChar *name,
        const xmlChar **libxmlAttributes)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);

    LOG4CXX_DEBUG(xmlXmlReaderLog, "startElementHandler() for " << name);

    if (reader->parserStopped())
    {
        return;
    }

    XmlAttributes attributes(libxmlAttributes);

    XmlNamespace* ns = reader->pushNamespaces(attributes);
    attributes.split(ns);

    const xmlChar *qName = name;
    xmlChar *localName = NULL;
    const xmlChar *uri = NULL;
    const xmlChar *colonPtr = xmlStrstr(qName, (xmlChar *) ":");

    if (colonPtr != NULL)
    {
        localName = xmlStrdup(qName + (colonPtr - qName) + 1);
    }

    uri = reader->xmlNamespace()->uriForPrefix(qName);

    // We pass in the namespace of the element, and then the name both with and without
    // the namespace prefix.
    if (localName != NULL)
    {
        reader->contentHandler()->startElement(uri, localName, qName,
                attributes);
        free(localName);
    }
    else
    {
        reader->contentHandler()->startElement(uri, qName, qName, attributes);
    }
}

static void endElementHandler(void *userData, const xmlChar *name)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    if (reader->parserStopped())
    {
        return;
    }

    LOG4CXX_DEBUG(xmlXmlReaderLog, "endElementHandler() for " << name);

    const xmlChar *qName = name;
    xmlChar *localName = NULL;
    const xmlChar *uri = NULL;
    const xmlChar *colonPtr = xmlStrstr(qName, (xmlChar *) ":");

    if (colonPtr != NULL)
    {
        localName = xmlStrdup(qName + (colonPtr - qName) + 1);
    }

    uri = reader->xmlNamespace()->uriForPrefix(qName);<commit_msg>Change loglevel on elementhandler<commit_after>/*
 * Copyright (C) 2003 Apple Computer, Inc.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

/*
 * Copyright (C) 2012 Kolibre
 *
 * This file is part of kolibre-xmlreader.
 *
 * Kolibre-xmlreader is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 2.1 of the License, or
 * (at your option) any later version.
 *
 * Kolibre-xmlreader is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with kolibre-xmlreader. If not, see <http://www.gnu.org/licenses/>.
 */

/**
 * \class XmlReader
 *
 * \brief This is the main interface for the kolibre xmlreader. XmlReader lets you parse HMTL and XML resource (offline and online) and can even cached data for quicker access.
 */

#include "XmlReader.h"

#include "XmlAttributes.h"
#include "XmlError.h"
#include "XmlDefaultHandler.h"

#include "DataSource.h"

#include <malloc.h>
#include <libxml/encoding.h>

#include <cstring>
#include <sstream>

#include <log4cxx/logger.h>

// create logger which will become a child to logger kolibre.xmlreader
log4cxx::LoggerPtr xmlXmlReaderLog(
        log4cxx::Logger::getLogger("kolibre.xmlreader.xmlreader"));

static void startDocumentHandler(void *userData)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    LOG4CXX_DEBUG(xmlXmlReaderLog, "startDocumentHandler()");
    reader->contentHandler()->startDocument();
}

static void endDocumentHandler(void *userData)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    LOG4CXX_DEBUG(xmlXmlReaderLog, "endDocumentHandler()");
    reader->endDocumentHandlerCalled();
    reader->contentHandler()->endDocument();
}

static void startElementHandler(void *userData, const xmlChar *name,
        const xmlChar **libxmlAttributes)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);

    LOG4CXX_TRACE(xmlXmlReaderLog, "startElementHandler() for " << name);

    if (reader->parserStopped())
    {
        return;
    }

    XmlAttributes attributes(libxmlAttributes);

    XmlNamespace* ns = reader->pushNamespaces(attributes);
    attributes.split(ns);

    const xmlChar *qName = name;
    xmlChar *localName = NULL;
    const xmlChar *uri = NULL;
    const xmlChar *colonPtr = xmlStrstr(qName, (xmlChar *) ":");

    if (colonPtr != NULL)
    {
        localName = xmlStrdup(qName + (colonPtr - qName) + 1);
    }

    uri = reader->xmlNamespace()->uriForPrefix(qName);

    // We pass in the namespace of the element, and then the name both with and without
    // the namespace prefix.
    if (localName != NULL)
    {
        reader->contentHandler()->startElement(uri, localName, qName,
                attributes);
        free(localName);
    }
    else
    {
        reader->contentHandler()->startElement(uri, qName, qName, attributes);
    }
}

static void endElementHandler(void *userData, const xmlChar *name)
{
    XmlReader *reader = static_cast<XmlReader *>(userData);
    if (reader->parserStopped())
    {
        return;
    }

    LOG4CXX_TRACE(xmlXmlReaderLog, "endElementHandler() for " << name);

    const xmlChar *qName = name;
    xmlChar *localName = NULL;
    const xmlChar *uri = NULL;
    const xmlChar *colonPtr = xmlStrstr(qName, (xmlChar *) ":");

    if (colonPtr != NULL)
    {
        localName = xmlStrdup(qName + (colonPtr - qName) + 1);
    }

    uri = reader->xmlNamespace()->uriForPrefix(qName);
{'fraction_non_alphanumeric': 0.0959505286255735, 'fraction_numerical': 0.004189108318372232, 'mean_word_length': 3.7976076555023925, 'pattern_counts': {'":': 4, '<': 31, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 70}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1953674', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2771, 'n_words': 1038}