<commit_before>-module(abx_serializer).
-export([serialize_to_file/2, serialize_to_binary/1]).

-include("abx.hrl").

serialize_to_file(FileName, Chunks) ->
	file:write_file(FileName, serialize_to_binary(Chunks)).

serialize_to_binary(Chunks) ->
	StringPool = proplists:get_value(string_pool, Chunks),
	put(string_pool, dict:from_list(lists:zip(StringPool,
		lists:seq(0, length(StringPool) - 1)))),
	Chunk = serialize_chunks(Chunks),
	ChunkSize = byte_size(Chunk) + 8,
	<<?RES_XML_TYPE:16/little, 8:16/little, ChunkSize:32/little, Chunk/binary>>.

serialize_chunks(Chunks) -> serialize_chunks(Chunks, <<>>).
serialize_chunks([], Acc) -> Acc;
serialize_chunks([Chunk | Chunks], Acc) ->
	{Type, HeaderSize, Serialized} = serialize_chunk(Chunk),
	FullSize = byte_size(Serialized) + 8,
	PadBytes = case FullSize rem 4 of 0 -> 0; R -> 4 - R end,
	Padding = binary:copy(<<0>>, PadBytes),
	serialize_chunks(Chunks, <<Acc/binary, Type:16/little, HeaderSize:16/little,
		(FullSize + PadBytes):32/little, Serialized/binary, Padding/binary>>).

serialize_chunk({string_pool, Pool}) ->
	{?RES_STRING_POOL_TYPE, 28, serialize_string_pool(Pool)};
serialize_chunk({element, LineNum, Comment, Namespace, Name, Attributes}) ->
	AttrStart = 20,
	AttrSize = 20,
	AttrCount = length(Attributes),
	NsIndex = query_string_pool(Namespace),
	NameIndex = query_string_pool(Name),
	AttrBytes = << <<(serialize_attribute(A))/binary>> || A <- Attributes >>,
	{?RES_XML_START_ELEMENT_TYPE, 16, <<LineNum:32/little, Comment:32/little,
		NsIndex:32/little, NameIndex:32/little, AttrStart:16/little,
		AttrSize:16/little, AttrCount:16/little, 0:48, AttrBytes/binary>>};
serialize_chunk({unknown, Type, HeaderSize, Payload}) -> {Type, HeaderSize, Payload}.

query_string_pool(null) -> 16#FFFFFFFF;
query_string_pool(String) -> dict:fetch(String, get(string_pool)).

serialize_attribute({Namespace, Name, Value}) -><commit_msg>added resource map serialization<commit_after>-module(abx_serializer).
-export([serialize_to_file/2, serialize_to_binary/1]).

-include("abx.hrl").

serialize_to_file(FileName, Chunks) ->
	file:write_file(FileName, serialize_to_binary(Chunks)).

serialize_to_binary(Chunks) ->
	StringPool = proplists:get_value(string_pool, Chunks),
	put(string_pool, dict:from_list(lists:zip(StringPool,
		lists:seq(0, length(StringPool) - 1)))),
	Chunk = serialize_chunks(Chunks),
	ChunkSize = byte_size(Chunk) + 8,
	<<?RES_XML_TYPE:16/little, 8:16/little, ChunkSize:32/little, Chunk/binary>>.

serialize_chunks(Chunks) -> serialize_chunks(Chunks, <<>>).
serialize_chunks([], Acc) -> Acc;
serialize_chunks([Chunk | Chunks], Acc) ->
	{Type, HeaderSize, Serialized} = serialize_chunk(Chunk),
	FullSize = byte_size(Serialized) + 8,
	PadBytes = case FullSize rem 4 of 0 -> 0; R -> 4 - R end,
	Padding = binary:copy(<<0>>, PadBytes),
	serialize_chunks(Chunks, <<Acc/binary, Type:16/little, HeaderSize:16/little,
		(FullSize + PadBytes):32/little, Serialized/binary, Padding/binary>>).

serialize_chunk({string_pool, Pool}) ->
	{?RES_STRING_POOL_TYPE, 28, serialize_string_pool(Pool)};
serialize_chunk({element, LineNum, Comment, Namespace, Name, Attributes}) ->
	AttrStart = 20,
	AttrSize = 20,
	AttrCount = length(Attributes),
	NsIndex = query_string_pool(Namespace),
	NameIndex = query_string_pool(Name),
	AttrBytes = << <<(serialize_attribute(A))/binary>> || A <- Attributes >>,
	{?RES_XML_START_ELEMENT_TYPE, 16, <<LineNum:32/little, Comment:32/little,
		NsIndex:32/little, NameIndex:32/little, AttrStart:16/little,
		AttrSize:16/little, AttrCount:16/little, 0:48, AttrBytes/binary>>};
serialize_chunk({res_map, Map}) ->
	{?RES_XML_RESOURCE_MAP_TYPE, 8, << <<R:32/little>> || R <- Map>>};
serialize_chunk({unknown, Type, HeaderSize, Payload}) -> {Type, HeaderSize, Payload}.

query_string_pool(null) -> 16#FFFFFFFF;
query_string_pool(String) -> dict:fetch(String, get(string_pool)).

serialize_attribute({Namespace, Name, Value}) ->
{'fraction_non_alphanumeric': 0.16342710997442456, 'fraction_numerical': 0.026854219948849106, 'mean_word_length': 6.997955010224949, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4181417', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1395, 'n_words': 240}