<commit_before>
module Ragel

using Compat
using Switch
import Base.FS
import Base: push!, pop!, endof, append!, empty!, isempty, length, getindex,
             setindex!, (==), takebuf_string, read!


# A simple buffer type, similar to IOBuffer but faster and with fewer features.
# It's essentially just a vector that can grow but is never allowed to
# reallocate to smaller size.

const INITIAL_BUF_SIZE = 1000000

type Buffer{T}
    data::Vector{T}
    pos::Int

    function Buffer(initial_size=10000)
        new(Array(T, initial_size), 1)
    end
end


function isempty(buf::Buffer)
    return buf.pos == 1
end


function length(buf::Buffer)
    return buf.pos - 1
end


function empty!(buf::Buffer)
    buf.pos = 1
end


function getindex(buf::Buffer, i::Integer)
    if !(1 <= i < buf.pos)
        throw(BoundsError())
    end
    @inbounds x = buf.data[i]
    return x
end


function setindex!{T}(buf::Buffer{T}, value::T, i::Integer)
    if !(1 <= i < buf.pos)
        throw(BoundsError())
    end
    @inbounds buf.data[i] = value
end


function ensureroom!(buf::Buffer, n::Int)
    if buf.pos + n > length(buf.data)
        resize!(buf.data, 2 * length(buf.data))
    end
end


function push!{T}(buf::Buffer{T}, c::T)
    ensureroom!(buf, 1)
    @inbounds buf.data[buf.pos] = c
    buf.pos += 1
end


function pop!(buf::Buffer)
    if buf.pos == 1
        throw(BoundsError())
    end
    @inbounds c = buf.data[buf.pos - 1]
    buf.pos -= 1
    return c
end


function endof(buf::Buffer)
    return buf.pos - 1
end


function append!{T}(buf::Buffer{T}, source::Vector{T}, start::Int, stop::Int)
    n = stop - start + 1
    ensureroom!(buf, n)<commit_msg>Fix potential buffer overflow in parsers.<commit_after>
module Ragel

using Compat
using Switch
import Base.FS
import Base: push!, pop!, endof, append!, empty!, isempty, length, getindex,
             setindex!, (==), takebuf_string, read!


# A simple buffer type, similar to IOBuffer but faster and with fewer features.
# It's essentially just a vector that can grow but is never allowed to
# reallocate to smaller size.

const INITIAL_BUF_SIZE = 1000000

type Buffer{T}
    data::Vector{T}
    pos::Int

    function Buffer(initial_size=10000)
        new(Array(T, initial_size), 1)
    end
end


function isempty(buf::Buffer)
    return buf.pos == 1
end


function length(buf::Buffer)
    return buf.pos - 1
end


function empty!(buf::Buffer)
    buf.pos = 1
end


function getindex(buf::Buffer, i::Integer)
    if !(1 <= i < buf.pos)
        throw(BoundsError())
    end
    @inbounds x = buf.data[i]
    return x
end


function setindex!{T}(buf::Buffer{T}, value::T, i::Integer)
    if !(1 <= i < buf.pos)
        throw(BoundsError())
    end
    @inbounds buf.data[i] = value
end


function ensureroom!(buf::Buffer, n::Int)
    if buf.pos + n > length(buf.data)
        newsize = max(2 * length(buf.data), buf.pos + n)
        resize!(buf.data, newsize)
    end
end


function push!{T}(buf::Buffer{T}, c::T)
    ensureroom!(buf, 1)
    @inbounds buf.data[buf.pos] = c
    buf.pos += 1
end


function pop!(buf::Buffer)
    if buf.pos == 1
        throw(BoundsError())
    end
    @inbounds c = buf.data[buf.pos - 1]
    buf.pos -= 1
    return c
end


function endof(buf::Buffer)
    return buf.pos - 1
end


function append!{T}(buf::Buffer{T}, source::Vector{T}, start::Int, stop::Int)
    n = stop - start + 1
    ensureroom!(buf, n)
{'fraction_non_alphanumeric': 0.13577712609970674, 'fraction_numerical': 0.015249266862170088, 'mean_word_length': 3.0704057279236276, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6383951', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1155, 'n_words': 344}