<commit_before>-module(event_stream).

-export([new/0, feed/2]).

-define(MESSAGES_MAX, 100).
-define(OOB_KEY_MAX, 256).
-define(EVENT_OPEN,  "**<", EventType:3/binary, ">").
-define(EVENT_CLOSE, "<>**").
-define(TIMESTAMP,
        Year:2/binary, "/", Month:2/binary, "/",  Day:2/binary, " ",
        Hour:2/binary, ":", Minute:2/binary, ":", Second:2/binary).

new() ->
        {next_stream, {outside_event, {}}}.

feed(Data, {next_stream, State}) -> handle_state(State, Data).

handle_state({outside_event, _StateData}, Data) ->
        case Data of
                {eol, <<?EVENT_OPEN, " ", ?TIMESTAMP, TagBin/binary>>} ->
                        Tags = string:tokens(binary_to_list(TagBin), " "),
                        {next_stream, {inside_event,
                                       {EventType,
                                        {Year, Month, Day, Hour, Minute, Second},
                                        Tags,
                                        message_buffer:new(?MESSAGES_MAX)}}};
                {eol, <<?EVENT_OPEN, " ", Message/binary>>} ->
                        {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_local_time(now()),
                        Messages = message_buffer:new(?MESSAGES_MAX),
                        {next_stream, {outside_event,
                                       event({EventType,
                                              {Year, Month, Day, Hour, Minute, Second},
                                              [],
                                             message_buffer:append(binary_to_list(Message), Messages)})}};
                {_IsEOL, <<BadMessage/binary>>} ->
                        {next_stream, {outside_event,
                                       {errline, binary_to_list(BadMessage) ++
                                        case _IsEOL of
                                                true -> "";
                                                false -> "..."
                                        end}}}
        end;

handle_state({inside_event, {EventType, Time, Tags, Messages} = _StateData}, Data) ->
        case Data of
                {eol, <<?EVENT_CLOSE>>} ->
                        {next_stream, {outside_event,
                                       event({EventType, Time, Tags, Messages})}};
                {_IsEOL, <<Message/binary>>} ->
                        {next_stream, {inside_event,
                                      {EventType, Time, Tags,<commit_msg>Changed _IsEol -> IsEol since the variable is actually used<commit_after>-module(event_stream).

-export([new/0, feed/2]).

-define(MESSAGES_MAX, 100).
-define(OOB_KEY_MAX, 256).
-define(EVENT_OPEN,  "**<", EventType:3/binary, ">").
-define(EVENT_CLOSE, "<>**").
-define(TIMESTAMP,
        Year:2/binary, "/", Month:2/binary, "/",  Day:2/binary, " ",
        Hour:2/binary, ":", Minute:2/binary, ":", Second:2/binary).

new() ->
        {next_stream, {outside_event, {}}}.

feed(Data, {next_stream, State}) -> handle_state(State, Data).

handle_state({outside_event, _StateData}, Data) ->
        case Data of
                {eol, <<?EVENT_OPEN, " ", ?TIMESTAMP, TagBin/binary>>} ->
                        Tags = string:tokens(binary_to_list(TagBin), " "),
                        {next_stream, {inside_event,
                                       {EventType,
                                        {Year, Month, Day, Hour, Minute, Second},
                                        Tags,
                                        message_buffer:new(?MESSAGES_MAX)}}};
                {eol, <<?EVENT_OPEN, " ", Message/binary>>} ->
                        {{Year, Month, Day}, {Hour, Minute, Second}} = calendar:now_to_local_time(now()),
                        Messages = message_buffer:new(?MESSAGES_MAX),
                        {next_stream, {outside_event,
                                       event({EventType,
                                              {Year, Month, Day, Hour, Minute, Second},
                                              [],
                                             message_buffer:append(binary_to_list(Message), Messages)})}};
                {IsEOL, <<BadMessage/binary>>} ->
                        {next_stream, {outside_event,
                                       {errline, binary_to_list(BadMessage) ++
                                        case IsEOL of
                                                true -> "";
                                                false -> "..."
                                        end}}}
        end;

handle_state({inside_event, {EventType, Time, Tags, Messages} = _StateData}, Data) ->
        case Data of
                {eol, <<?EVENT_CLOSE>>} ->
                        {next_stream, {outside_event,
                                       event({EventType, Time, Tags, Messages})}};
                {_IsEOL, <<Message/binary>>} ->
                        {next_stream, {inside_event,
                                      {EventType, Time, Tags,
{'fraction_non_alphanumeric': 0.13495222929936307, 'fraction_numerical': 0.005971337579617835, 'mean_word_length': 1.1876360470178493, 'pattern_counts': {'":': 4, '<': 27, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7528626', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1250, 'n_words': 225}