<commit_before>/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2009 <NAME>
 *
 * This library is open source and may be redistributed and/or modified under
 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
 * (at your option) any later version.  The full license is in LICENSE file
 * included with this distribution, and on the openscenegraph.org website.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * OpenSceneGraph Public License for more details.
*/

#include <osgDB/XmlParser>
#include <osgDB/FileUtils>

#include <osg/Notify>

using namespace osgDB;

XmlNode* osgDB::readXmlFile(const std::string& filename,const Options* options)
{
    std::string foundFile = osgDB::findDataFile(filename, options);
    if (!foundFile.empty())
    {
        XmlNode::Input input;
        input.open(foundFile);
        if (!input)
        {
            osg::notify(osg::NOTICE)<<"Could not open XML file: "<<filename<<std::endl;
            return 0;
        }

        input.readAllDataIntoBuffer();

        osg::ref_ptr<XmlNode> root = new XmlNode;
        root->read(input);

        return root.release();
    }
    else
    {
        osg::notify(osg::NOTICE)<<"Could not find XML file: "<<filename<<std::endl;
        return 0;
    }
}

XmlNode* osgDB::readXmlStream(std::istream& fin)
{
    XmlNode::Input input;
    input.attach(fin);
    if (!input)
    {
        osg::notify(osg::NOTICE)<<"Could not attach to XML stream."<<std::endl;
        return 0;
    }

    input.readAllDataIntoBuffer();

    osg::ref_ptr<XmlNode> root = new XmlNode;
    root->read(input);

    return root.release();
}


XmlNode::Input::Input():
    _currentPos(0)
{
    setUpControlMappings();
}

XmlNode::Input::Input(const Input&):
    _currentPos(0)
{
    setUpControlMappings();
}

XmlNode::Input::~Input()
{
}

void XmlNode::Input::setUpControlMappings()
{
    addControlToCharacter("&amp;",'&');
    addControlToCharacter("&lt;",'<');
    addControlToCharacter("&gt;",'>');
    addControlToCharacter("&quot;",'"');
    addControlToCharacter("&apos;",'\'');
}

void XmlNode::Input::addControlToCharacter(const std::string& control, int c)
{
    _controlToCharacterMap[control] = c;
    _characterToControlMap[c] = control;
}

void XmlNode::Input::open(const std::string& filename)
{
    _fin.open(filename.c_str());
}

void XmlNode::Input::attach(std::istream& fin)
{
    std::ios &fios = _fin;
    fios.rdbuf(fin.rdbuf());
}

void XmlNode::Input::readAllDataIntoBuffer()
{
    while(_fin)
    {
        int c = _fin.get();
        if (c>=0 && c<=255)
        {
            _buffer.push_back(c);
        }
    }
}

void XmlNode::Input::skipWhiteSpace()
{
    while(_currentPos<_buffer.size() && _buffer[_currentPos]==' ')
    {
        // osg::notify(osg::NOTICE)<<"_currentPos="<<_currentPos<<"_buffer.size()="<<_buffer.size()<<" v="<<_buffer[_currentPos]<<std::endl;
        ++_currentPos;
    }
    //osg::notify(osg::NOTICE)<<"done"<<std::endl;
}

XmlNode::XmlNode()
{
    type = UNASSIGNED;
}

bool XmlNode::read(Input& input)
{
    if (type == UNASSIGNED) type = ROOT;

    while(input)
    {<commit_msg>Added tabs to treatment as white space to skyWhiteSpace()<commit_after>/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2009 <NAME>
 *
 * This library is open source and may be redistributed and/or modified under
 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
 * (at your option) any later version.  The full license is in LICENSE file
 * included with this distribution, and on the openscenegraph.org website.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * OpenSceneGraph Public License for more details.
*/

#include <osgDB/XmlParser>
#include <osgDB/FileUtils>

#include <osg/Notify>

using namespace osgDB;

XmlNode* osgDB::readXmlFile(const std::string& filename,const Options* options)
{
    std::string foundFile = osgDB::findDataFile(filename, options);
    if (!foundFile.empty())
    {
        XmlNode::Input input;
        input.open(foundFile);
        if (!input)
        {
            osg::notify(osg::NOTICE)<<"Could not open XML file: "<<filename<<std::endl;
            return 0;
        }

        input.readAllDataIntoBuffer();

        osg::ref_ptr<XmlNode> root = new XmlNode;
        root->read(input);

        return root.release();
    }
    else
    {
        osg::notify(osg::NOTICE)<<"Could not find XML file: "<<filename<<std::endl;
        return 0;
    }
}

XmlNode* osgDB::readXmlStream(std::istream& fin)
{
    XmlNode::Input input;
    input.attach(fin);
    if (!input)
    {
        osg::notify(osg::NOTICE)<<"Could not attach to XML stream."<<std::endl;
        return 0;
    }

    input.readAllDataIntoBuffer();

    osg::ref_ptr<XmlNode> root = new XmlNode;
    root->read(input);

    return root.release();
}


XmlNode::Input::Input():
    _currentPos(0)
{
    setUpControlMappings();
}

XmlNode::Input::Input(const Input&):
    _currentPos(0)
{
    setUpControlMappings();
}

XmlNode::Input::~Input()
{
}

void XmlNode::Input::setUpControlMappings()
{
    addControlToCharacter("&amp;",'&');
    addControlToCharacter("&lt;",'<');
    addControlToCharacter("&gt;",'>');
    addControlToCharacter("&quot;",'"');
    addControlToCharacter("&apos;",'\'');
}

void XmlNode::Input::addControlToCharacter(const std::string& control, int c)
{
    _controlToCharacterMap[control] = c;
    _characterToControlMap[c] = control;
}

void XmlNode::Input::open(const std::string& filename)
{
    _fin.open(filename.c_str());
}

void XmlNode::Input::attach(std::istream& fin)
{
    std::ios &fios = _fin;
    fios.rdbuf(fin.rdbuf());
}

void XmlNode::Input::readAllDataIntoBuffer()
{
    while(_fin)
    {
        int c = _fin.get();
        if (c>=0 && c<=255)
        {
            _buffer.push_back(c);
        }
    }
}

void XmlNode::Input::skipWhiteSpace()
{
    while(_currentPos<_buffer.size() && _buffer[_currentPos]==' ' || _buffer[_currentPos]=='\t' )
    {
        // osg::notify(osg::NOTICE)<<"_currentPos="<<_currentPos<<"_buffer.size()="<<_buffer.size()<<" v="<<_buffer[_currentPos]<<std::endl;
        ++_currentPos;
    }
    //osg::notify(osg::NOTICE)<<"done"<<std::endl;
}

XmlNode::XmlNode()
{
    type = UNASSIGNED;
}

bool XmlNode::read(Input& input)
{
    if (type == UNASSIGNED) type = ROOT;

    while(input)
    {
{'fraction_non_alphanumeric': 0.15833962264150944, 'fraction_numerical': 0.005735849056603773, 'mean_word_length': 3.522866894197952, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '203408', 'n_tokens_mistral': 2277, 'n_tokens_neox': 2207, 'n_words': 471}