<commit_before>public extension IReadableStream
{
	public func chunks(of: Int) -> ReadableStream<Array<Self.ChunkType>> {
		let chunks = self.map { (elements: [ChunkType]) in
			return elements.count == of ? (elements, []) : (nil, elements)
		}
		return chunks
	}
}
<commit_msg>Add overlappingChunks.<commit_after>public extension IReadableStream
{
	public func chunks(of: Int) -> ReadableStream<Array<Self.ChunkType>> {
		let chunks = self.map { (elements: [ChunkType]) in
			return elements.count == of ? (elements, []) : (nil, elements)
		}
		return chunks
	}

	public func overlappingChunks(of: Int, advancingBy: Int) -> ReadableStream<Array<Self.ChunkType>> {
		let overlappingChunkStream = OverlappingChunkStream<ChunkType>(of, advancingBy)
		_ = self.chunks(of: advancingBy).subscribe {
			overlappingChunkStream.accumulate($0)
		}
		return ReadableStream(overlappingChunkStream)
	}
}

internal class OverlappingChunkStream<T> : Stream<[T]>
{
	internal init(_ chunkSize: Int, _ advanceBy: Int) {
		_chunkSize = chunkSize
		_advanceBy = advanceBy
	}

	func accumulate(_ data: [T]) {
		let toFill = min(data.count, _chunkSize - _accumulator.count)
		let leftovers = toFill < data.count ? data[toFill...data.count - 1] : []
		_accumulator.append(contentsOf: data[0...toFill - 1])
		if (_accumulator.count == _chunkSize) {
			publish(_accumulator)
			_accumulator = Array(_accumulator[_advanceBy..._chunkSize - 1])
		}

		if (leftovers.count != 0) {
			accumulate(Array(leftovers))
		}
	}

	let _chunkSize: Int
	let _advanceBy: Int
	var _accumulator: [T] = []
}

{'fraction_non_alphanumeric': 0.131629392971246, 'fraction_numerical': 0.0038338658146964857, 'mean_word_length': 4.53356890459364, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6845743', 'n_tokens_mistral': 583, 'n_tokens_neox': 520, 'n_words': 113}