<commit_before>/* parse_context.h                                                  -*- C++ -*-
   <NAME>, 27 January 2005
   Copyright (c) 2005 <NAME>.  All rights reserved.
      
   This file is part of "Jeremy's Machine Learning Library", copyright (c)
   1999-2005 <NAME>.
   
   This program is available under the GNU General Public License, the terms
   of which are given by the file "license.txt" in the top level directory of
   the source code distribution.  If this file is missing, you have no right
   to use the program; please contact the author.

   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   for more details.

   ---

   A class to ease the recursive-descent parsing of strings.
*/

#ifndef __utils__parse_context_h__
#define __utils__parse_context_h__

#include "jml/utils/unnamed_bool.h"
#include "jml/arch/exception.h"
#include "jml/compiler/compiler.h"
#include <cmath>
#include <string>
#include <iostream>
#include <list>
#include <limits.h>
#include <string.h>
#include <stdint.h>
#include <boost/utility.hpp>
#include <boost/shared_ptr.hpp>


namespace ML {


class File_Read_Buffer;


/*****************************************************************************/
/* PARSE_CONTEXT                                                             */
/*****************************************************************************/

struct Parse_Context {

    /** Create but don't initialize. */
    Parse_Context();

    /** Initialize from a filename, loading the file and uncompressing if
        necessary. */
    explicit Parse_Context(const std::string & filename);
    
    /** Initialize from a memory region. */
    Parse_Context(const std::string & filename, const char * start,
                  const char * finish, unsigned line = 1, unsigned col = 1);

    Parse_Context(const std::string & filename, const char * start,
                  size_t length, unsigned line = 1, unsigned col = 1);

    /** Initialize from a File_Read_Buffer. */
    explicit Parse_Context(const File_Read_Buffer & buf);

    /** Default chunk size. */
    enum { DEFAULT_CHUNK_SIZE = 65500 };

    /** Initialize from an istream. */
    Parse_Context(const std::string & filename, std::istream & stream,
                  unsigned line = 1, unsigned col = 1,
                  size_t chunk_size = DEFAULT_CHUNK_SIZE);

    ~Parse_Context();

    /** Initialize from a filename, loading the file and uncompressing if
        necessary. */
    void init(const std::string & filename);

    /** Initialize from a memory region. */
    void init(const std::string & filename, const char * start,
              const char * finish, unsigned line = 1, unsigned col = 1);

    /** Initialize from a File_Read_Buffer. */
    void init(const File_Read_Buffer & buf);

    /** Initialize from an istream. */
    void init(const std::string & filename, std::istream & stream,
              unsigned line = 1, unsigned col = 1);

    /** Set the chunk size for the buffers.  Mostly used for testing
        purposes.  Note that this is only useful when initialized from
        a stream. */
    void set_chunk_size(size_t size);

    /** Get the chunk size. */
    size_t get_chunk_size() const { return chunk_size_; }

    /** How many characters are available to read ahead from? */
    size_t readahead_available() const;

    /** How many characters are buffered in total, both before and after
        the current character? */
    size_t total_buffered() const;

    /** Increment.  Note that it always sets up the buffer such that more
        characters are available. */
    JML_ALWAYS_INLINE Parse_Context & operator ++ ()
    {
        if (eof()) exception("unexpected EOF");

        if (*cur_ == '\n') { ++line_;  col_ = 0; }
        ofs_ += 1;  col_ += 1;

        ++cur_;
        if (JML_UNLIKELY(cur_ == ebuf_))
            next_buffer();

        return *this;
    }

    /** Little helper class that allows character at current position to
        be returned.  This allows us to do "*context++", without making
        a (heavyweight!) copy at each stage (as context++ requires a copy
        of the object to be returned).
    */
    struct Last_Char {
        Last_Char(char c) : c(c) {}
        char operator * () const { return c; }
        char c;
    };
    
    Last_Char operator ++ (int)
    {
        char result = operator * ();
        operator ++ ();
        return Last_Char(result);
    }

    char operator * () const
    {
        if (eof()) exception("unexpected EOF");
        return *cur_;
    }
    
    /** Match a literal character.  Return true if matched or false if not.
        Never throws.
    */
    bool match_literal(char c)
    {
        if (eof()) return false;
        if (*cur_ == c) { operator ++();  return true; }
        return false;
    }
    
    /** Expect a literal character.  Throws if the character is not matched. */
    void expect_literal(char c, const char * error = "expected '%c'")
    {
        if (!match_literal(c)) exception_fmt(error, c);
    }
    
    /** Match a literal string.  Returns true if it was matched and false if
        not.  Never throws.
    */
    bool match_literal(const std::string & str)
    {
        return match_literal_str(str.data(), str.length());
    }
    
    /** Expect a literal string.  Throws an exception if the given string<commit_msg>better unexpected character message in Parse_Context<commit_after>/* parse_context.h                                                  -*- C++ -*-
   <NAME>, 27 January 2005
   Copyright (c) 2005 <NAME>.  All rights reserved.
      
   This file is part of "Jeremy's Machine Learning Library", copyright (c)
   1999-2005 <NAME>.
   
   This program is available under the GNU General Public License, the terms
   of which are given by the file "license.txt" in the top level directory of
   the source code distribution.  If this file is missing, you have no right
   to use the program; please contact the author.

   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
   or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   for more details.

   ---

   A class to ease the recursive-descent parsing of strings.
*/

#ifndef __utils__parse_context_h__
#define __utils__parse_context_h__

#include "jml/utils/unnamed_bool.h"
#include "jml/arch/exception.h"
#include "jml/compiler/compiler.h"
#include <cmath>
#include <string>
#include <iostream>
#include <list>
#include <limits.h>
#include <string.h>
#include <stdint.h>
#include <boost/utility.hpp>
#include <boost/shared_ptr.hpp>


namespace ML {


class File_Read_Buffer;


/*****************************************************************************/
/* PARSE_CONTEXT                                                             */
/*****************************************************************************/

struct Parse_Context {

    /** Create but don't initialize. */
    Parse_Context();

    /** Initialize from a filename, loading the file and uncompressing if
        necessary. */
    explicit Parse_Context(const std::string & filename);
    
    /** Initialize from a memory region. */
    Parse_Context(const std::string & filename, const char * start,
                  const char * finish, unsigned line = 1, unsigned col = 1);

    Parse_Context(const std::string & filename, const char * start,
                  size_t length, unsigned line = 1, unsigned col = 1);

    /** Initialize from a File_Read_Buffer. */
    explicit Parse_Context(const File_Read_Buffer & buf);

    /** Default chunk size. */
    enum { DEFAULT_CHUNK_SIZE = 65500 };

    /** Initialize from an istream. */
    Parse_Context(const std::string & filename, std::istream & stream,
                  unsigned line = 1, unsigned col = 1,
                  size_t chunk_size = DEFAULT_CHUNK_SIZE);

    ~Parse_Context();

    /** Initialize from a filename, loading the file and uncompressing if
        necessary. */
    void init(const std::string & filename);

    /** Initialize from a memory region. */
    void init(const std::string & filename, const char * start,
              const char * finish, unsigned line = 1, unsigned col = 1);

    /** Initialize from a File_Read_Buffer. */
    void init(const File_Read_Buffer & buf);

    /** Initialize from an istream. */
    void init(const std::string & filename, std::istream & stream,
              unsigned line = 1, unsigned col = 1);

    /** Set the chunk size for the buffers.  Mostly used for testing
        purposes.  Note that this is only useful when initialized from
        a stream. */
    void set_chunk_size(size_t size);

    /** Get the chunk size. */
    size_t get_chunk_size() const { return chunk_size_; }

    /** How many characters are available to read ahead from? */
    size_t readahead_available() const;

    /** How many characters are buffered in total, both before and after
        the current character? */
    size_t total_buffered() const;

    /** Increment.  Note that it always sets up the buffer such that more
        characters are available. */
    JML_ALWAYS_INLINE Parse_Context & operator ++ ()
    {
        if (eof()) exception("unexpected EOF");

        if (*cur_ == '\n') { ++line_;  col_ = 0; }
        ofs_ += 1;  col_ += 1;

        ++cur_;
        if (JML_UNLIKELY(cur_ == ebuf_))
            next_buffer();

        return *this;
    }

    /** Little helper class that allows character at current position to
        be returned.  This allows us to do "*context++", without making
        a (heavyweight!) copy at each stage (as context++ requires a copy
        of the object to be returned).
    */
    struct Last_Char {
        Last_Char(char c) : c(c) {}
        char operator * () const { return c; }
        char c;
    };
    
    Last_Char operator ++ (int)
    {
        char result = operator * ();
        operator ++ ();
        return Last_Char(result);
    }

    char operator * () const
    {
        if (eof()) exception("unexpected EOF");
        return *cur_;
    }
    
    /** Match a literal character.  Return true if matched or false if not.
        Never throws.
    */
    bool match_literal(char c)
    {
        if (eof()) return false;
        if (*cur_ == c) { operator ++();  return true; }
        return false;
    }
    
    /** Expect a literal character.  Throws if the character is not matched. */
    void expect_literal(char c, const char * error = "expected '%c', got '%c'")
    {
        if (!match_literal(c)) exception_fmt(error, c, (eof() ? '\0' : *cur_));
    }
    
    /** Match a literal string.  Returns true if it was matched and false if
        not.  Never throws.
    */
    bool match_literal(const std::string & str)
    {
        return match_literal_str(str.data(), str.length());
    }
    
    /** Expect a literal string.  Throws an exception if the given string
{'fraction_non_alphanumeric': 0.12611076205008528, 'fraction_numerical': 0.006552374113634324, 'mean_word_length': 2.415695892090742, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2720463', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2948, 'n_words': 1127}