<commit_before>
import Base:
    start,
    next,
    done


immutable EachWindowIterator{T}
    vec::Vector{T}
    K::Int
    step::Int
    npos::Int  # Number of windows, length(vec) - K + step

    EachWindowIterator(vec, K, step=1) = new(vec, K, step, length(vec) - K + 1)
end

immutable EachWindowIteratorState{T}
    i::Int  # start position of current window
end

function eachwindow{T}(K::Integer, vec::Vector{T}, step::Integer=1)
    @assert K >= 0 "K (window size) must be ≥ 0 in EachWindow"
    @assert step >= 1 "step must be ≥ 1"

    return EachWindowIterator{T}(vec, K, step)
end


@inline function start(it::EachWindowIterator)
    return 1
end


@inline function next{T}(it::EachWindowIterator{T}, state::Integer)
    i = state
    window = sub(it.vec, i:i+it.K - 1)
    return window, i + it.step<commit_msg>Remove import Base in windows.jl, it's in Util.jl<commit_after># Iterate through every window of size K in vec

immutable EachWindowIterator{T}
    vec::Vector{T}
    K::Int
    step::Int
    npos::Int  # Number of windows, length(vec) - K + step

    EachWindowIterator(vec, K, step=1) = new(vec, K, step, length(vec) - K + 1)
end

immutable EachWindowIteratorState{T}
    i::Int  # start position of current window
end

function eachwindow{T}(K::Integer, vec::Vector{T}, step::Integer=1)
    @assert K >= 0 "K (window size) must be ≥ 0 in EachWindow"
    @assert step >= 1 "step must be ≥ 1"

    return EachWindowIterator{T}(vec, K, step)
end


@inline function start(it::EachWindowIterator)
    return 1
end


@inline function next{T}(it::EachWindowIterator{T}, state::Integer)
    i = state
    window = sub(it.vec, i:i+it.K - 1)
    return window, i + it.step
{'fraction_non_alphanumeric': 0.13010053222945003, 'fraction_numerical': 0.010644589000591367, 'mean_word_length': 3.6869806094182827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157593', 'n_tokens_mistral': 602, 'n_tokens_neox': 575, 'n_words': 182}