<commit_before># This file is part of NIT (http://www.nitlanguage.org).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Add reading and writing binary services
#
# ~~~
# var w = new FileWriter.open("/tmp/data.bin")
# w.write "hello"
# w.write_int64 123456789
# w.write_byte 3u8
# w.write_float 1.25
# w.write_double 1.234567
# w.write_bits(true, false, true)
# assert w.last_error == null
# w.close
#
# var r = new FileReader.open("/tmp/data.bin")
# assert r.read(5) == "hello"
# assert r.read_int64 == 123456789
# assert r.read_byte == 3u8
# assert r.read_float == 1.25
# assert r.read_double == 1.234567
#
# var bits = r.read_bits
# assert bits[0] and not bits[1] and bits[2]
#
# assert r.last_error == null
# r.close
# ~~~
module binary

in "C" `{
	#include <inttypes.h>
	#include <endian.h>

	// Android compatibility
	#ifndef be32toh
		#define be32toh(val) betoh32(val)
		#define le32toh(val) letoh32(val)
	#endif

	#ifndef be64toh
		#define be64toh(val) betoh64(val)
		#define le64toh(val) letoh64(val)
	#endif
`}

# A stream of binary data
abstract class BinaryStream
	super Stream

	# Use the big-endian convention? otherwise use little-endian.
	#
	# By default, `true` to use big-endian.
	var big_endian = true is writable
end

redef abstract class Writer
	super BinaryStream

	# Write a boolean `value` on a byte, using 0 for `false` and 1 for `true`
	fun write_bool(value: Bool) do write_byte if value then 1u8 else 0u8

	# Write up to 8 `Bool` in a byte
	#
	# To be used with `BinaryReader::read_bits`.
	#
	# Ensure: `bits.length <= 8`
	fun write_bits(bits: Bool...)
	do
		assert bits.length <= 8

		var int = 0u8
		for b in bits.length.times do
			if bits[b] then int |= 1u8 << (7 - b)
		end

		write_byte int
	end

	# Write `text` as a null terminated string
	#
	# To be used with `Reader::read_string`.
	#
	# Require: `text` has no null bytes.
	fun write_string(text: Text)
	do
		write text
		write_byte 0x00u8
	end

	# Write the length as a 64 bits integer, then the content of `text`
	#
	# To be used with `Reader::read_block`.
	#
	# Compared to `write_string`, this method supports null bytes in `text`.
	fun write_block(text: Text)
	do
		write_int64 text.length
		write text
	end

	# Write a floating point `value` on 32 bits
	#
	# Using this format may result in a loss of precision as it uses less bits
	# than Nit `Float`.
	fun write_float(value: Float)
	do
		for i in [0..4[ do write_byte value.float_byte_at(i, big_endian)
	end

	# Write a floating point `value` on 64 bits
	fun write_double(value: Float)
	do
		for i in [0..8[ do write_byte value.double_byte_at(i, big_endian)
	end

	# Write `value` as a signed integer on 64 bits
	#
	# Using this format may result in a loss of precision as the length of a
	# Nit `Int` may be more than 64 bits on some platforms.
	fun write_int64(value: Int)
	do
		for i in [0..8[ do write_byte value.int64_byte_at(i, big_endian)
	end

	# TODO:
	#
	# fun write_int8
	# fun write_uint8 # No need for this one, it is the current `read_char`
	# fun write_int16
	# fun write_uint16
	# fun write_int32
	# fun write_uint32
	# fun write_uint64
	# fun write_long_double?
end

redef abstract class Reader
	super BinaryStream

	# Read a single byte and return `true` if its value is different than 0
	#
	# Returns `false` when an error is pending (`last_error != null`).
	fun read_bool: Bool do return read_byte != 0u8

	# Get an `Array` of 8 `Bool` by reading a single byte
	#
	# To be used with `BinaryWriter::write_bits`.
	#
	# Returns an array of `false` when an error is pending (`last_error != null`).
	fun read_bits: Array[Bool]
	do
		var int = read_byte
		if int == null then return new Array[Bool]
		var arr = new Array[Bool]
		for i in [7 .. 0].step(-1) do
			arr.push(((int >> i) & 1u8) != 0u8)
		end
		return arr
	end

	# Read a null terminated string
	#
	# To be used with `Writer::write_string`.
	#
	# Returns a truncated string when an error is pending (`last_error != null`).
	fun read_string: String
	do
		var buf = new FlatBuffer
		loop
			var byte = read_byte
			if byte == null or byte == 0x00u8 then return buf.to_s
			buf.bytes.add byte
		end<commit_msg>lib/binary: Fix binary lib to properly use Bytes instead of FlatBuffer<commit_after># This file is part of NIT (http://www.nitlanguage.org).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Add reading and writing binary services
#
# ~~~
# var w = new FileWriter.open("/tmp/data.bin")
# w.write "hello"
# w.write_int64 123456789
# w.write_byte 3u8
# w.write_float 1.25
# w.write_double 1.234567
# w.write_bits(true, false, true)
# assert w.last_error == null
# w.close
#
# var r = new FileReader.open("/tmp/data.bin")
# assert r.read(5) == "hello"
# assert r.read_int64 == 123456789
# assert r.read_byte == 3u8
# assert r.read_float == 1.25
# assert r.read_double == 1.234567
#
# var bits = r.read_bits
# assert bits[0] and not bits[1] and bits[2]
#
# assert r.last_error == null
# r.close
# ~~~
module binary

in "C" `{
	#include <inttypes.h>
	#include <endian.h>

	// Android compatibility
	#ifndef be32toh
		#define be32toh(val) betoh32(val)
		#define le32toh(val) letoh32(val)
	#endif

	#ifndef be64toh
		#define be64toh(val) betoh64(val)
		#define le64toh(val) letoh64(val)
	#endif
`}

# A stream of binary data
abstract class BinaryStream
	super Stream

	# Use the big-endian convention? otherwise use little-endian.
	#
	# By default, `true` to use big-endian.
	var big_endian = true is writable
end

redef abstract class Writer
	super BinaryStream

	# Write a boolean `value` on a byte, using 0 for `false` and 1 for `true`
	fun write_bool(value: Bool) do write_byte if value then 1u8 else 0u8

	# Write up to 8 `Bool` in a byte
	#
	# To be used with `BinaryReader::read_bits`.
	#
	# Ensure: `bits.length <= 8`
	fun write_bits(bits: Bool...)
	do
		assert bits.length <= 8

		var int = 0u8
		for b in bits.length.times do
			if bits[b] then int |= 1u8 << (7 - b)
		end

		write_byte int
	end

	# Write `text` as a null terminated string
	#
	# To be used with `Reader::read_string`.
	#
	# Require: `text` has no null bytes.
	fun write_string(text: Text)
	do
		write text
		write_byte 0x00u8
	end

	# Write the length as a 64 bits integer, then the content of `text`
	#
	# To be used with `Reader::read_block`.
	#
	# Compared to `write_string`, this method supports null bytes in `text`.
	fun write_block(text: Text)
	do
		write_int64 text.length
		write text
	end

	# Write a floating point `value` on 32 bits
	#
	# Using this format may result in a loss of precision as it uses less bits
	# than Nit `Float`.
	fun write_float(value: Float)
	do
		for i in [0..4[ do write_byte value.float_byte_at(i, big_endian)
	end

	# Write a floating point `value` on 64 bits
	fun write_double(value: Float)
	do
		for i in [0..8[ do write_byte value.double_byte_at(i, big_endian)
	end

	# Write `value` as a signed integer on 64 bits
	#
	# Using this format may result in a loss of precision as the length of a
	# Nit `Int` may be more than 64 bits on some platforms.
	fun write_int64(value: Int)
	do
		for i in [0..8[ do write_byte value.int64_byte_at(i, big_endian)
	end

	# TODO:
	#
	# fun write_int8
	# fun write_uint8 # No need for this one, it is the current `read_char`
	# fun write_int16
	# fun write_uint16
	# fun write_int32
	# fun write_uint32
	# fun write_uint64
	# fun write_long_double?
end

redef abstract class Reader
	super BinaryStream

	# Read a single byte and return `true` if its value is different than 0
	#
	# Returns `false` when an error is pending (`last_error != null`).
	fun read_bool: Bool do return read_byte != 0u8

	# Get an `Array` of 8 `Bool` by reading a single byte
	#
	# To be used with `BinaryWriter::write_bits`.
	#
	# Returns an array of `false` when an error is pending (`last_error != null`).
	fun read_bits: Array[Bool]
	do
		var int = read_byte
		if int == null then return new Array[Bool]
		var arr = new Array[Bool]
		for i in [7 .. 0].step(-1) do
			arr.push(((int >> i) & 1u8) != 0u8)
		end
		return arr
	end

	# Read a null terminated string
	#
	# To be used with `Writer::write_string`.
	#
	# Returns a truncated string when an error is pending (`last_error != null`).
	fun read_string: String
	do
		var buf = new Bytes.empty
		loop
			var byte = read_byte
			if byte == null or byte == 0u8 then
				return buf.to_s
			end
			buf.add byte
		end
{'fraction_non_alphanumeric': 0.09963495812754992, 'fraction_numerical': 0.03006227184882972, 'mean_word_length': 3.54833984375, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7019214', 'n_tokens_mistral': 3557, 'n_tokens_neox': 3264, 'n_words': 1246}