<commit_before>-module(parsing).
-export([parse_http/1]).

read_line(Data) ->
    read_line1(Data, <<>>).

read_line1(<<>>, Tail) -> Tail;
read_line1(<<Chunk:1/binary, Rest/binary>>, Tail) ->
    case Chunk of
        <<"\r">> ->
            Tail;
        <<"\n">> ->
            Tail;
        _ ->
            read_line1(Rest, <<Chunk:1/binary,Tail/binary>>)
    end.

parse_http(Data) ->
    read_line(Data).<commit_msg>Okay this one works as expected.<commit_after>-module(parsing).
-export([parse_http/1]).

read_line(Data) ->
    read_line1(Data, <<>>).

read_line1(<<>>, Tail) -> Tail;
read_line1(<<Chunk:1/binary, Rest/binary>>, Tail) ->
    case Chunk of
        <<"\r">> ->
            Tail;
        <<"\n">> ->
            Tail;
        _ ->
            read_line1(Rest, <<Chunk:1/binary,Tail/binary>>)
    end;
read_line1(_, Tail) -> Tail.

parse_http(Data) ->
    Line = read_line(Data),
    io:format("[-] Data: ~p~n", [Line]),
    Line.
{'fraction_non_alphanumeric': 0.21473684210526317, 'fraction_numerical': 0.015789473684210527, 'mean_word_length': 2.3020833333333335, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167615', 'n_tokens_mistral': 370, 'n_tokens_neox': 367, 'n_words': 57}